From 5bbd8a1d9f13efc4869d067e3eecdbbb7e6cda97 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sat, 22 Aug 2026 00:34:50 +0200 Subject: [PATCH] Split GEN x2 and G23 paths to eliminate AES hot-path overhead --- cmdline/speed.c | 32 +- raid/avx2.c | 1692 ++++++++++++++++++++++++------- raid/int.c | 262 ++++- raid/internal.h | 75 +- raid/neon.c | 819 +++++++++++++-- raid/neon32.c | 727 ++++++++++++-- raid/sse2.c | 558 ++++++++--- raid/ssse3.c | 2226 +++++++++++++++++++++++++++++++++++++---- raid/test.c | 1 - raid/test/speedtest.c | 31 +- 10 files changed, 5406 insertions(+), 1017 deletions(-) diff --git a/cmdline/speed.c b/cmdline/speed.c index 54b99846..fbefabde 100644 --- a/cmdline/speed.c +++ b/cmdline/speed.c @@ -330,12 +330,7 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8s", raid_gen_tag(RAID_ALGO_CAUCHY_PAR2)); fflush(stdout); - SPEED_START { - raid_gen2_int8(nd, size, v); - } SPEED_STOP - - printf("%8" PRIu64, ds / dt); - fflush(stdout); + printf("%8s", ""); if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { @@ -844,15 +839,17 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #endif } if (raid_cpu_has_ssse3()) { - if (mode == RAID_MODE_CAUCHY_RAID) { + if (mode == RAID_MODE_CAUCHY_AES) { + SPEED_START { + raid_gen5_ssse3_aes(nd, size, v); + } SPEED_STOP + } else { SPEED_START { raid_gen5_ssse3_raid(nd, size, v); } SPEED_STOP - - printf("%8" PRIu64, ds / dt); - } else { - printf("%8s", ""); } + + printf("%8" PRIu64, ds / dt); fflush(stdout); #ifdef CONFIG_X86_64 @@ -984,17 +981,18 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #endif } if (raid_cpu_has_ssse3()) { - if (mode == RAID_MODE_CAUCHY_RAID) { + if (mode == RAID_MODE_CAUCHY_AES) { + SPEED_START { + raid_gen6_ssse3_aes(nd, size, v); + } SPEED_STOP + } else { SPEED_START { raid_gen6_ssse3_raid(nd, size, v); } SPEED_STOP - - printf("%8" PRIu64, ds / dt); - } else { - printf("%8s", ""); } - fflush(stdout); + printf("%8" PRIu64, ds / dt); + fflush(stdout); #ifdef CONFIG_X86_64 if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { diff --git a/raid/avx2.c b/raid/avx2.c index 3be6066a..60440d5b 100644 --- a/raid/avx2.c +++ b/raid/avx2.c @@ -39,9 +39,9 @@ void raid_gen1_avx2(int nd, size_t size, void **vv) } /* - * GEN2 Cauchy AVX2 implementation using the active Q transition + * GEN2 Cauchy AVX2 implementation using only Q *= 2. */ -static __always_inline void raid_gen2_avx2_gen(int nd, size_t size, void **vv, int g23) +static __always_inline void raid_gen2_avx2_x2(int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -55,50 +55,25 @@ static __always_inline void raid_gen2_avx2_gen(int nd, size_t size, void **vv, i raid_avx_begin(); - asm volatile ("vpbroadcastb %0, %%ymm7" : : "m" (gfconst16.poly[0])); + asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.poly[0])); asm volatile ("vpxor %ymm6,%ymm6,%ymm6"); - int g23_start = raid_g23_count(l - 1); for (i = 0; i < size; i += 64) { - int g23_count = g23_start; - int g23_x3; - asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[l][i])); asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[l][i + 32])); asm volatile ("vmovdqa %ymm0,%ymm2"); asm volatile ("vmovdqa %ymm1,%ymm3"); for (d = l - 1; d >= 0; --d) { - g23_x3 = 0; - if (g23) { - g23_x3 = --g23_count == 0; - if (g23_x3) - g23_count = 51; - } - - if (g23 && g23_x3) { - asm volatile ("vmovdqa %ymm2,%ymm4"); - asm volatile ("vmovdqa %ymm3,%ymm5"); - asm volatile ("vpaddb %ymm2,%ymm2,%ymm2"); - asm volatile ("vpaddb %ymm3,%ymm3,%ymm3"); - asm volatile ("vpxor %ymm4,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm5,%ymm3,%ymm3"); - asm volatile ("vpcmpgtb %ymm4,%ymm6,%ymm4"); - asm volatile ("vpcmpgtb %ymm5,%ymm6,%ymm5"); - asm volatile ("vpand %ymm7,%ymm4,%ymm4"); - asm volatile ("vpand %ymm7,%ymm5,%ymm5"); - asm volatile ("vpxor %ymm4,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm5,%ymm3,%ymm3"); - } else { - asm volatile ("vpcmpgtb %ymm2,%ymm6,%ymm4"); - asm volatile ("vpcmpgtb %ymm3,%ymm6,%ymm5"); - asm volatile ("vpaddb %ymm2,%ymm2,%ymm2"); - asm volatile ("vpaddb %ymm3,%ymm3,%ymm3"); - asm volatile ("vpand %ymm7,%ymm4,%ymm4"); - asm volatile ("vpand %ymm7,%ymm5,%ymm5"); - asm volatile ("vpxor %ymm4,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm5,%ymm3,%ymm3"); - } + /* scale Q before adding the current disk */ + asm volatile ("vpcmpgtb %ymm2,%ymm6,%ymm4"); + asm volatile ("vpcmpgtb %ymm3,%ymm6,%ymm5"); + asm volatile ("vpaddb %ymm2,%ymm2,%ymm2"); + asm volatile ("vpaddb %ymm3,%ymm3,%ymm3"); + asm volatile ("vpand %ymm7,%ymm4,%ymm4"); + asm volatile ("vpand %ymm7,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm4,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm5,%ymm3,%ymm3"); asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i])); asm volatile ("vmovdqa %0,%%ymm5" : : "m" (v[d][i + 32])); @@ -117,27 +92,137 @@ static __always_inline void raid_gen2_avx2_gen(int nd, size_t size, void **vv, i raid_avx_end(); } +/* + * GEN2 Cauchy AVX2 implementation using the AES G23 Q transition. + */ +static __always_inline void raid_gen2_avx2_g23(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + + int g23_boundary = raid_g23_boundary(l - 1); + + raid_avx_begin(); + + asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.poly[0])); + asm volatile ("vpxor %ymm6,%ymm6,%ymm6"); + + for (i = 0; i < size; i += 64) { + int boundary = g23_boundary; + + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[l][i])); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[l][i + 32])); + asm volatile ("vmovdqa %ymm0,%ymm2"); + asm volatile ("vmovdqa %ymm1,%ymm3"); + + d = l - 1; + + for (;;) { + /* + * All transitions above 'boundary' are x2. + */ + for (; d > boundary; --d) { + /* scale Q before adding the current disk */ + asm volatile ("vpcmpgtb %ymm2,%ymm6,%ymm4"); + asm volatile ("vpcmpgtb %ymm3,%ymm6,%ymm5"); + asm volatile ("vpaddb %ymm2,%ymm2,%ymm2"); + asm volatile ("vpaddb %ymm3,%ymm3,%ymm3"); + asm volatile ("vpand %ymm7,%ymm4,%ymm4"); + asm volatile ("vpand %ymm7,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm4,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm5,%ymm3,%ymm3"); + + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i])); + asm volatile ("vmovdqa %0,%%ymm5" : : "m" (v[d][i + 32])); + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm4,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm5,%ymm3,%ymm3"); + } + + /* + * boundary == -1 identifies the final x2-only segment. + * At this point all remaining disks have been consumed. + */ + if (boundary < 0) + break; + + /* + * d == boundary. + * + * Perform the fixed G23 x3 transition and then add + * the current disk to P and Q. + */ + asm volatile ("vmovdqa %ymm2,%ymm4"); + asm volatile ("vmovdqa %ymm3,%ymm5"); + asm volatile ("vpaddb %ymm2,%ymm2,%ymm2"); + asm volatile ("vpaddb %ymm3,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm4,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm5,%ymm3,%ymm3"); + asm volatile ("vpcmpgtb %ymm4,%ymm6,%ymm4"); + asm volatile ("vpcmpgtb %ymm5,%ymm6,%ymm5"); + asm volatile ("vpand %ymm7,%ymm4,%ymm4"); + asm volatile ("vpand %ymm7,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm4,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm5,%ymm3,%ymm3"); + + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i])); + asm volatile ("vmovdqa %0,%%ymm5" : : "m" (v[d][i + 32])); + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm4,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm5,%ymm3,%ymm3"); + + --d; + + /* + * G23 boundaries are exactly 51 transitions apart. + * After d = 50 there are no more x3 transitions. + */ + boundary -= 51; + if (boundary < 50) + boundary = -1; + } + + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); + asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); + asm volatile ("vmovntdq %%ymm2,%0" : "=m" (q[i])); + asm volatile ("vmovntdq %%ymm3,%0" : "=m" (q[i + 32])); + } + + raid_avx_end(); +} + void raid_gen2_avx2_raid(int nd, size_t size, void **vv) { - raid_gen2_avx2_gen(nd, size, vv, 0); + raid_gen2_avx2_x2(nd, size, vv); } void raid_gen2_avx2_aes(int nd, size_t size, void **vv) { - raid_gen2_avx2_gen(nd, size, vv, 1); + if (raid_g23_x2_only(nd)) + raid_gen2_avx2_x2(nd, size, vv); + else + raid_gen2_avx2_g23(nd, size, vv); } #ifdef CONFIG_X86_64 /* - * GEN2 Cauchy AVX2 implementation using the extended register set. + * GEN2 Cauchy AVX2 implementation using only Q *= 2. * * Process two data disks at a time and process the two 32-byte lanes * sequentially. * * Note that it uses 16 registers, meaning that x64 is required. */ -static __always_inline void raid_gen2_avx2ext_gen(int nd, size_t size, - void **vv, int g23) +static __always_inline void raid_gen2_avx2ext_x2(int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -153,13 +238,8 @@ static __always_inline void raid_gen2_avx2ext_gen(int nd, size_t size, asm volatile ("vpxor %ymm14,%ymm14,%ymm14"); asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.poly[0])); - int g23_start = raid_g23_count(l - 1); for (i = 0; i < size; i += 64) { - int g23_count = g23_start; - int g23_x3_d; - int g23_x3_d1; - asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[l][i])); asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[l][i + 32])); asm volatile ("vmovdqa %ymm0,%ymm2"); @@ -167,26 +247,12 @@ static __always_inline void raid_gen2_avx2ext_gen(int nd, size_t size, /* process two disks per iteration */ for (d = l - 1; d >= 1; d -= 2) { - g23_x3_d = 0; - g23_x3_d1 = 0; - if (g23) { - g23_x3_d = --g23_count == 0; - if (g23_x3_d) - g23_count = 51; - g23_x3_d1 = --g23_count == 0; - if (g23_x3_d1) - g23_count = 51; - } - asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i])); asm volatile ("vmovdqa %0,%%ymm5" : : "m" (v[d][i + 32])); asm volatile ("vmovdqa %0,%%ymm6" : : "m" (v[d - 1][i])); asm volatile ("vmovdqa %0,%%ymm7" : : "m" (v[d - 1][i + 32])); - if (g23 && g23_x3_d) { - asm volatile ("vmovdqa %ymm2,%ymm12"); - asm volatile ("vmovdqa %ymm3,%ymm13"); - } + /* Q *= 2, then add disk d */ asm volatile ("vpcmpgtb %ymm2,%ymm14,%ymm10"); asm volatile ("vpcmpgtb %ymm3,%ymm14,%ymm11"); asm volatile ("vpaddb %ymm2,%ymm2,%ymm2"); @@ -195,17 +261,10 @@ static __always_inline void raid_gen2_avx2ext_gen(int nd, size_t size, asm volatile ("vpand %ymm15,%ymm11,%ymm11"); asm volatile ("vpxor %ymm10,%ymm2,%ymm2"); asm volatile ("vpxor %ymm11,%ymm3,%ymm3"); - if (g23 && g23_x3_d) - asm volatile ("vpxor %ymm12,%ymm2,%ymm2"); - if (g23 && g23_x3_d) - asm volatile ("vpxor %ymm13,%ymm3,%ymm3"); asm volatile ("vpxor %ymm4,%ymm2,%ymm2"); asm volatile ("vpxor %ymm5,%ymm3,%ymm3"); - if (g23 && g23_x3_d1) { - asm volatile ("vmovdqa %ymm2,%ymm12"); - asm volatile ("vmovdqa %ymm3,%ymm13"); - } + /* Q *= 2, then add disk d - 1 */ asm volatile ("vpcmpgtb %ymm2,%ymm14,%ymm10"); asm volatile ("vpcmpgtb %ymm3,%ymm14,%ymm11"); asm volatile ("vpaddb %ymm2,%ymm2,%ymm2"); @@ -214,10 +273,6 @@ static __always_inline void raid_gen2_avx2ext_gen(int nd, size_t size, asm volatile ("vpand %ymm15,%ymm11,%ymm11"); asm volatile ("vpxor %ymm10,%ymm2,%ymm2"); asm volatile ("vpxor %ymm11,%ymm3,%ymm3"); - if (g23 && g23_x3_d1) - asm volatile ("vpxor %ymm12,%ymm2,%ymm2"); - if (g23 && g23_x3_d1) - asm volatile ("vpxor %ymm13,%ymm3,%ymm3"); asm volatile ("vpxor %ymm6,%ymm2,%ymm2"); asm volatile ("vpxor %ymm7,%ymm3,%ymm3"); @@ -229,41 +284,22 @@ static __always_inline void raid_gen2_avx2ext_gen(int nd, size_t size, /* single remaining disk */ if (d == 0) { - int g23_x3; - - g23_x3 = 0; - if (g23) { - g23_x3 = --g23_count == 0; - if (g23_x3) - g23_count = 51; - } - asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[0][i])); asm volatile ("vmovdqa %0,%%ymm5" : : "m" (v[0][i + 32])); asm volatile ("vpcmpgtb %ymm2,%ymm14,%ymm10"); - asm volatile ("vpaddb %ymm2,%ymm2,%ymm8"); - asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); - if (g23 && g23_x3) - asm volatile ("vpxor %ymm4,%ymm2,%ymm12"); - asm volatile ("vpand %ymm15,%ymm10,%ymm10"); - asm volatile ("vpxor %ymm10,%ymm8,%ymm8"); - if (g23 && g23_x3) - asm volatile ("vpxor %ymm12,%ymm8,%ymm2"); - else - asm volatile ("vpxor %ymm4,%ymm8,%ymm2"); - asm volatile ("vpcmpgtb %ymm3,%ymm14,%ymm11"); - asm volatile ("vpaddb %ymm3,%ymm3,%ymm9"); - asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); - if (g23 && g23_x3) - asm volatile ("vpxor %ymm5,%ymm3,%ymm13"); + asm volatile ("vpaddb %ymm2,%ymm2,%ymm2"); + asm volatile ("vpaddb %ymm3,%ymm3,%ymm3"); + asm volatile ("vpand %ymm15,%ymm10,%ymm10"); asm volatile ("vpand %ymm15,%ymm11,%ymm11"); - asm volatile ("vpxor %ymm11,%ymm9,%ymm9"); - if (g23 && g23_x3) - asm volatile ("vpxor %ymm13,%ymm9,%ymm3"); - else - asm volatile ("vpxor %ymm5,%ymm9,%ymm3"); + asm volatile ("vpxor %ymm10,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm11,%ymm3,%ymm3"); + + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm4,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm5,%ymm3,%ymm3"); } asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); @@ -275,14 +311,176 @@ static __always_inline void raid_gen2_avx2ext_gen(int nd, size_t size, raid_avx_end(); } + +/* + * GEN2 Cauchy AVX2 implementation using the AES G23 Q transition. + * + * Process two data disks at a time and process the two 32-byte lanes + * sequentially. + * + * Note that it uses 16 registers, meaning that x64 is required. + */ +static __always_inline void raid_gen2_avx2ext_g23(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + + int g23_boundary = raid_g23_boundary(l - 1); + + raid_avx_begin(); + + asm volatile ("vpxor %ymm14,%ymm14,%ymm14"); + asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.poly[0])); + + for (i = 0; i < size; i += 64) { + int boundary = g23_boundary; + + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[l][i])); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[l][i + 32])); + asm volatile ("vmovdqa %ymm0,%ymm2"); + asm volatile ("vmovdqa %ymm1,%ymm3"); + + d = l - 1; + + for (;;) { + /* + * All transitions above 'boundary' are x2. + * Do not let a two-disk iteration cross the boundary. + */ + for (; d >= boundary + 2; d -= 2) { + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i])); + asm volatile ("vmovdqa %0,%%ymm5" : : "m" (v[d][i + 32])); + asm volatile ("vmovdqa %0,%%ymm6" : : "m" (v[d - 1][i])); + asm volatile ("vmovdqa %0,%%ymm7" : : "m" (v[d - 1][i + 32])); + + /* Q *= 2, then add disk d */ + asm volatile ("vpcmpgtb %ymm2,%ymm14,%ymm10"); + asm volatile ("vpcmpgtb %ymm3,%ymm14,%ymm11"); + asm volatile ("vpaddb %ymm2,%ymm2,%ymm2"); + asm volatile ("vpaddb %ymm3,%ymm3,%ymm3"); + asm volatile ("vpand %ymm15,%ymm10,%ymm10"); + asm volatile ("vpand %ymm15,%ymm11,%ymm11"); + asm volatile ("vpxor %ymm10,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm11,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm4,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm5,%ymm3,%ymm3"); + + /* Q *= 2, then add disk d - 1 */ + asm volatile ("vpcmpgtb %ymm2,%ymm14,%ymm10"); + asm volatile ("vpcmpgtb %ymm3,%ymm14,%ymm11"); + asm volatile ("vpaddb %ymm2,%ymm2,%ymm2"); + asm volatile ("vpaddb %ymm3,%ymm3,%ymm3"); + asm volatile ("vpand %ymm15,%ymm10,%ymm10"); + asm volatile ("vpand %ymm15,%ymm11,%ymm11"); + asm volatile ("vpxor %ymm10,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm11,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm6,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm7,%ymm3,%ymm3"); + + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm6,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm7,%ymm1,%ymm1"); + } + + /* + * The first x2 segment can have odd length. + * Consume its last x2 transition separately. + */ + if (d > boundary) { + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i])); + asm volatile ("vmovdqa %0,%%ymm5" : : "m" (v[d][i + 32])); + + asm volatile ("vpcmpgtb %ymm2,%ymm14,%ymm10"); + asm volatile ("vpcmpgtb %ymm3,%ymm14,%ymm11"); + asm volatile ("vpaddb %ymm2,%ymm2,%ymm2"); + asm volatile ("vpaddb %ymm3,%ymm3,%ymm3"); + asm volatile ("vpand %ymm15,%ymm10,%ymm10"); + asm volatile ("vpand %ymm15,%ymm11,%ymm11"); + asm volatile ("vpxor %ymm10,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm11,%ymm3,%ymm3"); + + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm4,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm5,%ymm3,%ymm3"); + + --d; + } + + /* + * boundary == -1 identifies the final x2-only segment. + * At this point all remaining disks have been consumed. + */ + if (boundary < 0) + break; + + /* + * d == boundary. + * + * Perform the fixed G23 x3 transition and then add + * the current disk to P and Q. + */ + asm volatile ("vmovdqa %ymm2,%ymm12"); + asm volatile ("vmovdqa %ymm3,%ymm13"); + + asm volatile ("vpcmpgtb %ymm2,%ymm14,%ymm10"); + asm volatile ("vpcmpgtb %ymm3,%ymm14,%ymm11"); + asm volatile ("vpaddb %ymm2,%ymm2,%ymm2"); + asm volatile ("vpaddb %ymm3,%ymm3,%ymm3"); + asm volatile ("vpand %ymm15,%ymm10,%ymm10"); + asm volatile ("vpand %ymm15,%ymm11,%ymm11"); + asm volatile ("vpxor %ymm10,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm11,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm12,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm13,%ymm3,%ymm3"); + + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i])); + asm volatile ("vmovdqa %0,%%ymm5" : : "m" (v[d][i + 32])); + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm4,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm5,%ymm3,%ymm3"); + + --d; + + /* + * G23 boundaries are exactly 51 transitions apart. + * After d = 50 there are no more x3 transitions. + */ + boundary -= 51; + if (boundary < 50) + boundary = -1; + } + + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); + asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); + asm volatile ("vmovntdq %%ymm2,%0" : "=m" (q[i])); + asm volatile ("vmovntdq %%ymm3,%0" : "=m" (q[i + 32])); + } + + raid_avx_end(); +} + + void raid_gen2_avx2ext_raid(int nd, size_t size, void **vv) { - raid_gen2_avx2ext_gen(nd, size, vv, 0); + raid_gen2_avx2ext_x2(nd, size, vv); } void raid_gen2_avx2ext_aes(int nd, size_t size, void **vv) { - raid_gen2_avx2ext_gen(nd, size, vv, 1); + if (raid_g23_x2_only(nd)) + raid_gen2_avx2ext_x2(nd, size, vv); + else + raid_gen2_avx2ext_g23(nd, size, vv); } /* @@ -364,10 +562,11 @@ void raid_genz_avx2ext_raid(int nd, size_t size, void **vv) #ifdef CONFIG_X86_64 /* * GEN3 (triple parity with Cauchy matrix) AVX2 implementation + * using only Q *= 2. * * Note that it uses 16 registers, meaning that x64 is required. */ -static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv, int g23) +static __always_inline void raid_gen3_avx2ext_x2(int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -387,19 +586,13 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv memcpy(v[1 + i], v[0], size); return; } - int g23_start = raid_g23_count(l - 1); raid_avx_begin(); - /* generic case with at least two data disks */ - asm volatile ("vpbroadcastb %0, %%ymm3" : : "m" (gfconst16.poly[0])); - asm volatile ("vpbroadcastb %0, %%ymm11" : : "m" (gfconst16.low4[0])); + asm volatile ("vpbroadcastb %0,%%ymm3" : : "m" (gfconst16.poly[0])); + asm volatile ("vpbroadcastb %0,%%ymm11" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 64) { - int g23_count = g23_start; - int g23_x3_d; - int g23_x3_d1; - /* last disk without the generator multiplication */ asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[l][i])); asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[l][i + 32])); @@ -409,130 +602,103 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv asm volatile ("vmovdqa %ymm12,%ymm8"); asm volatile ("vmovdqa %ymm12,%ymm9"); - asm volatile ("vpsrlw $4,%ymm4,%ymm5"); - asm volatile ("vpsrlw $4,%ymm12,%ymm13"); - asm volatile ("vpand %ymm11,%ymm4,%ymm4"); - asm volatile ("vpand %ymm11,%ymm12,%ymm12"); - asm volatile ("vpand %ymm11,%ymm5,%ymm5"); - asm volatile ("vpand %ymm11,%ymm13,%ymm13"); + asm volatile ("vpsrlw $4,%ymm4,%ymm5"); + asm volatile ("vpsrlw $4,%ymm12,%ymm13"); + asm volatile ("vpand %ymm11,%ymm4,%ymm4"); + asm volatile ("vpand %ymm11,%ymm12,%ymm12"); + asm volatile ("vpand %ymm11,%ymm5,%ymm5"); + asm volatile ("vpand %ymm11,%ymm13,%ymm13"); - asm volatile ("vbroadcasti128 %0,%%ymm10" : : "m" (raid_gfcauchypshufb[l][1][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[l][1][0][0])); asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[l][1][1][0])); - asm volatile ("vpshufb %ymm4,%ymm10,%ymm2"); - asm volatile ("vpshufb %ymm12,%ymm10,%ymm10"); + + asm volatile ("vpshufb %ymm4,%ymm14,%ymm2"); asm volatile ("vpshufb %ymm5,%ymm15,%ymm7"); - asm volatile ("vpshufb %ymm13,%ymm15,%ymm15"); - asm volatile ("vpxor %ymm7,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm15,%ymm10,%ymm10"); + asm volatile ("vpxor %ymm7,%ymm2,%ymm2"); + + asm volatile ("vpshufb %ymm12,%ymm14,%ymm10"); + asm volatile ("vpshufb %ymm13,%ymm15,%ymm7"); + asm volatile ("vpxor %ymm7,%ymm10,%ymm10"); /* process two intermediate disks per iteration */ for (d = l - 1; d > 1; d -= 2) { - g23_x3_d = 0; - g23_x3_d1 = 0; - if (g23) { - g23_x3_d = --g23_count == 0; - if (g23_x3_d) - g23_count = 51; - g23_x3_d1 = --g23_count == 0; - if (g23_x3_d1) - g23_count = 51; - } - + /* disk d */ asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i])); asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i + 32])); - asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[d][1][1][0])); - - if (g23 && g23_x3_d) - asm volatile ("vmovdqa %ymm1,%ymm6"); + /* Q *= 2 */ asm volatile ("vpxor %ymm5,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm13,%ymm13,%ymm13"); asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5"); + asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13"); asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); + asm volatile ("vpaddb %ymm9,%ymm9,%ymm9"); asm volatile ("vpand %ymm3,%ymm5,%ymm5"); + asm volatile ("vpand %ymm3,%ymm13,%ymm13"); asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); - if (g23 && g23_x3_d) - asm volatile ("vpxor %ymm6,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm13,%ymm9,%ymm9"); asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); + asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); asm volatile ("vpsrlw $4,%ymm4,%ymm5"); + asm volatile ("vpsrlw $4,%ymm12,%ymm13"); asm volatile ("vpand %ymm11,%ymm4,%ymm4"); + asm volatile ("vpand %ymm11,%ymm12,%ymm12"); asm volatile ("vpand %ymm11,%ymm5,%ymm5"); + asm volatile ("vpand %ymm11,%ymm13,%ymm13"); + + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[d][1][1][0])); asm volatile ("vpshufb %ymm4,%ymm14,%ymm6"); asm volatile ("vpshufb %ymm5,%ymm15,%ymm7"); asm volatile ("vpxor %ymm7,%ymm6,%ymm6"); asm volatile ("vpxor %ymm6,%ymm2,%ymm2"); - if (g23 && g23_x3_d) - asm volatile ("vmovdqa %ymm9,%ymm6"); - asm volatile ("vpxor %ymm13,%ymm13,%ymm13"); - asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13"); - asm volatile ("vpaddb %ymm9,%ymm9,%ymm9"); - asm volatile ("vpand %ymm3,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm13,%ymm9,%ymm9"); - if (g23 && g23_x3_d) - asm volatile ("vpxor %ymm6,%ymm9,%ymm9"); - - asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); - asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); - - asm volatile ("vpsrlw $4,%ymm12,%ymm13"); - asm volatile ("vpand %ymm11,%ymm12,%ymm12"); - asm volatile ("vpand %ymm11,%ymm13,%ymm13"); - asm volatile ("vpshufb %ymm12,%ymm14,%ymm6"); asm volatile ("vpshufb %ymm13,%ymm15,%ymm7"); asm volatile ("vpxor %ymm7,%ymm6,%ymm6"); asm volatile ("vpxor %ymm6,%ymm10,%ymm10"); + /* disk d - 1 */ asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d - 1][i])); asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d - 1][i + 32])); - asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d - 1][1][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[d - 1][1][1][0])); - - if (g23 && g23_x3_d1) - asm volatile ("vmovdqa %ymm1,%ymm6"); + /* Q *= 2 */ asm volatile ("vpxor %ymm5,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm13,%ymm13,%ymm13"); asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5"); + asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13"); asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); + asm volatile ("vpaddb %ymm9,%ymm9,%ymm9"); asm volatile ("vpand %ymm3,%ymm5,%ymm5"); + asm volatile ("vpand %ymm3,%ymm13,%ymm13"); asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); - if (g23 && g23_x3_d1) - asm volatile ("vpxor %ymm6,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm13,%ymm9,%ymm9"); asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); + asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); asm volatile ("vpsrlw $4,%ymm4,%ymm5"); + asm volatile ("vpsrlw $4,%ymm12,%ymm13"); asm volatile ("vpand %ymm11,%ymm4,%ymm4"); + asm volatile ("vpand %ymm11,%ymm12,%ymm12"); asm volatile ("vpand %ymm11,%ymm5,%ymm5"); + asm volatile ("vpand %ymm11,%ymm13,%ymm13"); + + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d - 1][1][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[d - 1][1][1][0])); asm volatile ("vpshufb %ymm4,%ymm14,%ymm6"); asm volatile ("vpshufb %ymm5,%ymm15,%ymm7"); asm volatile ("vpxor %ymm7,%ymm6,%ymm6"); asm volatile ("vpxor %ymm6,%ymm2,%ymm2"); - if (g23 && g23_x3_d1) - asm volatile ("vmovdqa %ymm9,%ymm6"); - asm volatile ("vpxor %ymm13,%ymm13,%ymm13"); - asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13"); - asm volatile ("vpaddb %ymm9,%ymm9,%ymm9"); - asm volatile ("vpand %ymm3,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm13,%ymm9,%ymm9"); - if (g23 && g23_x3_d1) - asm volatile ("vpxor %ymm6,%ymm9,%ymm9"); - - asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); - asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); - - asm volatile ("vpsrlw $4,%ymm12,%ymm13"); - asm volatile ("vpand %ymm11,%ymm12,%ymm12"); - asm volatile ("vpand %ymm11,%ymm13,%ymm13"); - asm volatile ("vpshufb %ymm12,%ymm14,%ymm6"); asm volatile ("vpshufb %ymm13,%ymm15,%ymm7"); asm volatile ("vpxor %ymm7,%ymm6,%ymm6"); @@ -541,32 +707,31 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv /* single remaining intermediate disk */ if (d == 1) { - g23_x3_d = 0; - if (g23) { - g23_x3_d = --g23_count == 0; - if (g23_x3_d) - g23_count = 51; - } - asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[1][i])); asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[1][i + 32])); - if (g23 && g23_x3_d) - asm volatile ("vmovdqa %ymm1,%ymm6"); asm volatile ("vpxor %ymm5,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm13,%ymm13,%ymm13"); asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5"); + asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13"); asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); + asm volatile ("vpaddb %ymm9,%ymm9,%ymm9"); asm volatile ("vpand %ymm3,%ymm5,%ymm5"); + asm volatile ("vpand %ymm3,%ymm13,%ymm13"); asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); - if (g23 && g23_x3_d) - asm volatile ("vpxor %ymm6,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm13,%ymm9,%ymm9"); asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); + asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); asm volatile ("vpsrlw $4,%ymm4,%ymm5"); + asm volatile ("vpsrlw $4,%ymm12,%ymm13"); asm volatile ("vpand %ymm11,%ymm4,%ymm4"); + asm volatile ("vpand %ymm11,%ymm12,%ymm12"); asm volatile ("vpand %ymm11,%ymm5,%ymm5"); + asm volatile ("vpand %ymm11,%ymm13,%ymm13"); asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[1][1][0][0])); asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[1][1][1][0])); @@ -576,23 +741,6 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv asm volatile ("vpxor %ymm7,%ymm6,%ymm6"); asm volatile ("vpxor %ymm6,%ymm2,%ymm2"); - if (g23 && g23_x3_d) - asm volatile ("vmovdqa %ymm9,%ymm6"); - asm volatile ("vpxor %ymm13,%ymm13,%ymm13"); - asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13"); - asm volatile ("vpaddb %ymm9,%ymm9,%ymm9"); - asm volatile ("vpand %ymm3,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm13,%ymm9,%ymm9"); - if (g23 && g23_x3_d) - asm volatile ("vpxor %ymm6,%ymm9,%ymm9"); - - asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); - asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); - - asm volatile ("vpsrlw $4,%ymm12,%ymm13"); - asm volatile ("vpand %ymm11,%ymm12,%ymm12"); - asm volatile ("vpand %ymm11,%ymm13,%ymm13"); - asm volatile ("vpshufb %ymm12,%ymm14,%ymm6"); asm volatile ("vpshufb %ymm13,%ymm15,%ymm7"); asm volatile ("vpxor %ymm7,%ymm6,%ymm6"); @@ -603,17 +751,6 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[0][i])); asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[0][i + 32])); - g23_x3_d = 0; - if (g23) { - g23_x3_d = --g23_count == 0; - if (g23_x3_d) - g23_count = 51; - } - - if (g23 && g23_x3_d) { - asm volatile ("vmovdqa %ymm1,%ymm6"); - asm volatile ("vmovdqa %ymm9,%ymm14"); - } asm volatile ("vpxor %ymm5,%ymm5,%ymm5"); asm volatile ("vpxor %ymm13,%ymm13,%ymm13"); asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5"); @@ -624,11 +761,298 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv asm volatile ("vpand %ymm3,%ymm13,%ymm13"); asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); asm volatile ("vpxor %ymm13,%ymm9,%ymm9"); - if (g23 && g23_x3_d) { + + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm4,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); + asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); + asm volatile ("vpxor %ymm12,%ymm10,%ymm10"); + + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); + asm volatile ("vmovntdq %%ymm8,%0" : "=m" (p[i + 32])); + asm volatile ("vmovntdq %%ymm1,%0" : "=m" (q[i])); + asm volatile ("vmovntdq %%ymm9,%0" : "=m" (q[i + 32])); + asm volatile ("vmovntdq %%ymm2,%0" : "=m" (r[i])); + asm volatile ("vmovntdq %%ymm10,%0" : "=m" (r[i + 32])); + } + + raid_avx_end(); +} + +/* + * GEN3 (triple parity with Cauchy matrix) AVX2 implementation + * using the AES G23 Q transition. + * + * Note that it uses 16 registers, meaning that x64 is required. + */ +static __always_inline void raid_gen3_avx2ext_g23(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + + /* special case with only one data disk */ + if (l == 0) { + for (i = 0; i < 3; ++i) + memcpy(v[1 + i], v[0], size); + return; + } + + int g23_boundary = raid_g23_boundary(l - 1); + + raid_avx_begin(); + + asm volatile ("vpbroadcastb %0,%%ymm3" : : "m" (gfconst16.poly[0])); + asm volatile ("vpbroadcastb %0,%%ymm11" : : "m" (gfconst16.low4[0])); + + for (i = 0; i < size; i += 64) { + int boundary = g23_boundary; + + /* last disk without the generator multiplication */ + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[l][i])); + asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[l][i + 32])); + + asm volatile ("vmovdqa %ymm4,%ymm0"); + asm volatile ("vmovdqa %ymm4,%ymm1"); + asm volatile ("vmovdqa %ymm12,%ymm8"); + asm volatile ("vmovdqa %ymm12,%ymm9"); + + asm volatile ("vpsrlw $4,%ymm4,%ymm5"); + asm volatile ("vpsrlw $4,%ymm12,%ymm13"); + asm volatile ("vpand %ymm11,%ymm4,%ymm4"); + asm volatile ("vpand %ymm11,%ymm12,%ymm12"); + asm volatile ("vpand %ymm11,%ymm5,%ymm5"); + asm volatile ("vpand %ymm11,%ymm13,%ymm13"); + + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[l][1][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[l][1][1][0])); + + asm volatile ("vpshufb %ymm4,%ymm14,%ymm2"); + asm volatile ("vpshufb %ymm5,%ymm15,%ymm7"); + asm volatile ("vpxor %ymm7,%ymm2,%ymm2"); + + asm volatile ("vpshufb %ymm12,%ymm14,%ymm10"); + asm volatile ("vpshufb %ymm13,%ymm15,%ymm7"); + asm volatile ("vpxor %ymm7,%ymm10,%ymm10"); + + d = l - 1; + + for (;;) { + /* + * Process pairs entirely above the next G23 boundary. + * Disk 0 remains outside this loop. + */ + for (; d >= 2 && d >= boundary + 2; d -= 2) { + /* disk d */ + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i])); + asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i + 32])); + + asm volatile ("vpxor %ymm5,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm13,%ymm13,%ymm13"); + asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5"); + asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13"); + asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); + asm volatile ("vpaddb %ymm9,%ymm9,%ymm9"); + asm volatile ("vpand %ymm3,%ymm5,%ymm5"); + asm volatile ("vpand %ymm3,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm13,%ymm9,%ymm9"); + + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); + asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); + + asm volatile ("vpsrlw $4,%ymm4,%ymm5"); + asm volatile ("vpsrlw $4,%ymm12,%ymm13"); + asm volatile ("vpand %ymm11,%ymm4,%ymm4"); + asm volatile ("vpand %ymm11,%ymm12,%ymm12"); + asm volatile ("vpand %ymm11,%ymm5,%ymm5"); + asm volatile ("vpand %ymm11,%ymm13,%ymm13"); + + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + + asm volatile ("vpshufb %ymm4,%ymm14,%ymm6"); + asm volatile ("vpshufb %ymm5,%ymm15,%ymm7"); + asm volatile ("vpxor %ymm7,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm6,%ymm2,%ymm2"); + + asm volatile ("vpshufb %ymm12,%ymm14,%ymm6"); + asm volatile ("vpshufb %ymm13,%ymm15,%ymm7"); + asm volatile ("vpxor %ymm7,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm6,%ymm10,%ymm10"); + + /* disk d - 1 */ + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d - 1][i])); + asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d - 1][i + 32])); + + asm volatile ("vpxor %ymm5,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm13,%ymm13,%ymm13"); + asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5"); + asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13"); + asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); + asm volatile ("vpaddb %ymm9,%ymm9,%ymm9"); + asm volatile ("vpand %ymm3,%ymm5,%ymm5"); + asm volatile ("vpand %ymm3,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm13,%ymm9,%ymm9"); + + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); + asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); + + asm volatile ("vpsrlw $4,%ymm4,%ymm5"); + asm volatile ("vpsrlw $4,%ymm12,%ymm13"); + asm volatile ("vpand %ymm11,%ymm4,%ymm4"); + asm volatile ("vpand %ymm11,%ymm12,%ymm12"); + asm volatile ("vpand %ymm11,%ymm5,%ymm5"); + asm volatile ("vpand %ymm11,%ymm13,%ymm13"); + + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d - 1][1][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[d - 1][1][1][0])); + + asm volatile ("vpshufb %ymm4,%ymm14,%ymm6"); + asm volatile ("vpshufb %ymm5,%ymm15,%ymm7"); + asm volatile ("vpxor %ymm7,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm6,%ymm2,%ymm2"); + + asm volatile ("vpshufb %ymm12,%ymm14,%ymm6"); + asm volatile ("vpshufb %ymm13,%ymm15,%ymm7"); + asm volatile ("vpxor %ymm7,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm6,%ymm10,%ymm10"); + } + + /* + * The x2 segment may contain one remaining intermediate disk. + */ + if (d > boundary && d > 0) { + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i])); + asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i + 32])); + + asm volatile ("vpxor %ymm5,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm13,%ymm13,%ymm13"); + asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5"); + asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13"); + asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); + asm volatile ("vpaddb %ymm9,%ymm9,%ymm9"); + asm volatile ("vpand %ymm3,%ymm5,%ymm5"); + asm volatile ("vpand %ymm3,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm13,%ymm9,%ymm9"); + + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); + asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); + + asm volatile ("vpsrlw $4,%ymm4,%ymm5"); + asm volatile ("vpsrlw $4,%ymm12,%ymm13"); + asm volatile ("vpand %ymm11,%ymm4,%ymm4"); + asm volatile ("vpand %ymm11,%ymm12,%ymm12"); + asm volatile ("vpand %ymm11,%ymm5,%ymm5"); + asm volatile ("vpand %ymm11,%ymm13,%ymm13"); + + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + + asm volatile ("vpshufb %ymm4,%ymm14,%ymm6"); + asm volatile ("vpshufb %ymm5,%ymm15,%ymm7"); + asm volatile ("vpxor %ymm7,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm6,%ymm2,%ymm2"); + + asm volatile ("vpshufb %ymm12,%ymm14,%ymm6"); + asm volatile ("vpshufb %ymm13,%ymm15,%ymm7"); + asm volatile ("vpxor %ymm7,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm6,%ymm10,%ymm10"); + + --d; + } + + if (boundary < 0) + break; + + /* + * d == boundary: Q *= 3. + */ + asm volatile ("vmovdqa %ymm1,%ymm6"); + asm volatile ("vmovdqa %ymm9,%ymm7"); + + asm volatile ("vpxor %ymm5,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm13,%ymm13,%ymm13"); + asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5"); + asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13"); + asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); + asm volatile ("vpaddb %ymm9,%ymm9,%ymm9"); + asm volatile ("vpand %ymm3,%ymm5,%ymm5"); + asm volatile ("vpand %ymm3,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm13,%ymm9,%ymm9"); asm volatile ("vpxor %ymm6,%ymm1,%ymm1"); - asm volatile ("vpxor %ymm14,%ymm9,%ymm9"); + asm volatile ("vpxor %ymm7,%ymm9,%ymm9"); + + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i])); + asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i + 32])); + + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); + asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); + + asm volatile ("vpsrlw $4,%ymm4,%ymm5"); + asm volatile ("vpsrlw $4,%ymm12,%ymm13"); + asm volatile ("vpand %ymm11,%ymm4,%ymm4"); + asm volatile ("vpand %ymm11,%ymm12,%ymm12"); + asm volatile ("vpand %ymm11,%ymm5,%ymm5"); + asm volatile ("vpand %ymm11,%ymm13,%ymm13"); + + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + + asm volatile ("vpshufb %ymm4,%ymm14,%ymm6"); + asm volatile ("vpshufb %ymm5,%ymm15,%ymm7"); + asm volatile ("vpxor %ymm7,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm6,%ymm2,%ymm2"); + + asm volatile ("vpshufb %ymm12,%ymm14,%ymm6"); + asm volatile ("vpshufb %ymm13,%ymm15,%ymm7"); + asm volatile ("vpxor %ymm7,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm6,%ymm10,%ymm10"); + + --d; + + boundary -= 51; + if (boundary < 50) + boundary = -1; } + /* + * D0 always follows an x2 transition. + */ + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[0][i])); + asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[0][i + 32])); + + asm volatile ("vpxor %ymm5,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm13,%ymm13,%ymm13"); + asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5"); + asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13"); + asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); + asm volatile ("vpaddb %ymm9,%ymm9,%ymm9"); + asm volatile ("vpand %ymm3,%ymm5,%ymm5"); + asm volatile ("vpand %ymm3,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm13,%ymm9,%ymm9"); + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); asm volatile ("vpxor %ymm4,%ymm2,%ymm2"); @@ -649,22 +1073,26 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv void raid_gen3_avx2ext_raid(int nd, size_t size, void **vv) { - raid_gen3_avx2ext_gen(nd, size, vv, 0); + raid_gen3_avx2ext_x2(nd, size, vv); } void raid_gen3_avx2ext_aes(int nd, size_t size, void **vv) { - raid_gen3_avx2ext_gen(nd, size, vv, 1); + if (raid_g23_x2_only(nd)) + raid_gen3_avx2ext_x2(nd, size, vv); + else + raid_gen3_avx2ext_g23(nd, size, vv); } #endif #ifdef CONFIG_X86_64 /* * GEN4 (quad parity with Cauchy matrix) AVX2 implementation + * using only Q *= 2. * * Note that it uses 16 registers, meaning that x64 is required. */ -static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv, int g23) +static __always_inline void raid_gen4_avx2ext_x2(int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -686,7 +1114,6 @@ static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv memcpy(v[1 + i], v[0], size); return; } - int g23_start = raid_g23_count(l - 1); raid_avx_begin(); @@ -694,11 +1121,7 @@ static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv asm volatile ("vpbroadcastb %0,%%ymm6" : : "m" (gfconst16.poly[0])); asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.low4[0])); - /* generic case with at least two data disks */ for (i = 0; i < size; i += 64) { - int g23_count = g23_start; - int g23_x3; - /* last disk without the generator multiplication */ asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[l][i])); asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[l][i + 32])); @@ -715,6 +1138,7 @@ static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv asm volatile ("vpand %ymm7,%ymm12,%ymm12"); asm volatile ("vpand %ymm7,%ymm13,%ymm13"); + /* R */ asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[l][1][0][0])); asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[l][1][1][0])); asm volatile ("vpshufb %ymm4,%ymm14,%ymm2"); @@ -724,6 +1148,7 @@ static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv asm volatile ("vpshufb %ymm13,%ymm15,%ymm11"); asm volatile ("vpxor %ymm11,%ymm10,%ymm10"); + /* S */ asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[l][2][0][0])); asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[l][2][1][0])); asm volatile ("vpshufb %ymm4,%ymm14,%ymm3"); @@ -735,40 +1160,316 @@ static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv /* intermediate disks */ for (d = l - 1; d > 0; --d) { - g23_x3 = 0; - if (g23) { - g23_x3 = --g23_count == 0; - if (g23_x3) - g23_count = 51; - } - - if (g23 && g23_x3) - asm volatile ("vmovdqa %ymm1,%ymm4"); + /* Q *= 2, first lane */ asm volatile ("vpxor %ymm5,%ymm5,%ymm5"); asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5"); asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); asm volatile ("vpand %ymm6,%ymm5,%ymm5"); asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); - if (g23 && g23_x3) - asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i])); asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); asm volatile ("vpsrlw $4,%ymm4,%ymm5"); asm volatile ("vpand %ymm7,%ymm4,%ymm4"); asm volatile ("vpand %ymm7,%ymm5,%ymm5"); - if (g23 && g23_x3) - asm volatile ("vmovdqa %ymm9,%ymm12"); + + /* Q *= 2, second lane */ asm volatile ("vpxor %ymm13,%ymm13,%ymm13"); asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13"); asm volatile ("vpaddb %ymm9,%ymm9,%ymm9"); asm volatile ("vpand %ymm6,%ymm13,%ymm13"); asm volatile ("vpxor %ymm13,%ymm9,%ymm9"); - if (g23 && g23_x3) - asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i + 32])); + asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); + asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); + asm volatile ("vpsrlw $4,%ymm12,%ymm13"); + asm volatile ("vpand %ymm7,%ymm12,%ymm12"); + asm volatile ("vpand %ymm7,%ymm13,%ymm13"); + /* R */ + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("vpshufb %ymm4,%ymm14,%ymm15"); + asm volatile ("vpshufb %ymm12,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm15,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm14,%ymm10,%ymm10"); + + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("vpshufb %ymm5,%ymm14,%ymm15"); + asm volatile ("vpshufb %ymm13,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm15,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm14,%ymm10,%ymm10"); + + /* S */ + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][2][0][0])); + asm volatile ("vpshufb %ymm4,%ymm14,%ymm15"); + asm volatile ("vpshufb %ymm12,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm15,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm14,%ymm11,%ymm11"); + + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("vpshufb %ymm5,%ymm14,%ymm15"); + asm volatile ("vpshufb %ymm13,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm15,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm14,%ymm11,%ymm11"); + } + + /* first disk with all coefficients at 1 */ + asm volatile ("vpxor %ymm5,%ymm5,%ymm5"); + asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5"); + asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); + asm volatile ("vpand %ymm6,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); + + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[0][i])); + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm4,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm4,%ymm3,%ymm3"); + + asm volatile ("vpxor %ymm13,%ymm13,%ymm13"); + asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13"); + asm volatile ("vpaddb %ymm9,%ymm9,%ymm9"); + asm volatile ("vpand %ymm6,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm13,%ymm9,%ymm9"); + + asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[0][i + 32])); + asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); + asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); + asm volatile ("vpxor %ymm12,%ymm10,%ymm10"); + asm volatile ("vpxor %ymm12,%ymm11,%ymm11"); + + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); + asm volatile ("vmovntdq %%ymm8,%0" : "=m" (p[i + 32])); + asm volatile ("vmovntdq %%ymm1,%0" : "=m" (q[i])); + asm volatile ("vmovntdq %%ymm9,%0" : "=m" (q[i + 32])); + asm volatile ("vmovntdq %%ymm2,%0" : "=m" (r[i])); + asm volatile ("vmovntdq %%ymm10,%0" : "=m" (r[i + 32])); + asm volatile ("vmovntdq %%ymm3,%0" : "=m" (s[i])); + asm volatile ("vmovntdq %%ymm11,%0" : "=m" (s[i + 32])); + } + + raid_avx_end(); +} + +/* + * GEN4 (quad parity with Cauchy matrix) AVX2 implementation + * using the AES G23 Q transition. + * + * Note that it uses 16 registers, meaning that x64 is required. + */ +static __always_inline void raid_gen4_avx2ext_g23(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + uint8_t *s; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + s = v[nd + 3]; + + if (l == 0) { + for (i = 0; i < 4; ++i) + memcpy(v[1 + i], v[0], size); + return; + } + + int g23_boundary = raid_g23_boundary(l - 1); + + raid_avx_begin(); + + asm volatile ("vpbroadcastb %0,%%ymm6" : : "m" (gfconst16.poly[0])); + asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.low4[0])); + + for (i = 0; i < size; i += 64) { + int boundary = g23_boundary; + + /* last disk */ + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[l][i])); + asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[l][i + 32])); + + asm volatile ("vmovdqa %ymm4,%ymm0"); + asm volatile ("vmovdqa %ymm4,%ymm1"); + asm volatile ("vmovdqa %ymm12,%ymm8"); + asm volatile ("vmovdqa %ymm12,%ymm9"); + + asm volatile ("vpsrlw $4,%ymm4,%ymm5"); + asm volatile ("vpand %ymm7,%ymm4,%ymm4"); + asm volatile ("vpand %ymm7,%ymm5,%ymm5"); + asm volatile ("vpsrlw $4,%ymm12,%ymm13"); + asm volatile ("vpand %ymm7,%ymm12,%ymm12"); + asm volatile ("vpand %ymm7,%ymm13,%ymm13"); + + /* R */ + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[l][1][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[l][1][1][0])); + asm volatile ("vpshufb %ymm4,%ymm14,%ymm2"); + asm volatile ("vpshufb %ymm5,%ymm15,%ymm3"); + asm volatile ("vpxor %ymm3,%ymm2,%ymm2"); + asm volatile ("vpshufb %ymm12,%ymm14,%ymm10"); + asm volatile ("vpshufb %ymm13,%ymm15,%ymm11"); + asm volatile ("vpxor %ymm11,%ymm10,%ymm10"); + + /* S */ + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[l][2][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[l][2][1][0])); + asm volatile ("vpshufb %ymm4,%ymm14,%ymm3"); + asm volatile ("vpshufb %ymm5,%ymm15,%ymm5"); + asm volatile ("vpxor %ymm5,%ymm3,%ymm3"); + asm volatile ("vpshufb %ymm12,%ymm14,%ymm11"); + asm volatile ("vpshufb %ymm13,%ymm15,%ymm13"); + asm volatile ("vpxor %ymm13,%ymm11,%ymm11"); + + d = l - 1; + + while (boundary >= 0) { + /* all transitions above boundary are x2 */ + for (; d > boundary; --d) { + asm volatile ("vpxor %ymm5,%ymm5,%ymm5"); + asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5"); + asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); + asm volatile ("vpand %ymm6,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); + + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i])); + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); + asm volatile ("vpsrlw $4,%ymm4,%ymm5"); + asm volatile ("vpand %ymm7,%ymm4,%ymm4"); + asm volatile ("vpand %ymm7,%ymm5,%ymm5"); + + asm volatile ("vpxor %ymm13,%ymm13,%ymm13"); + asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13"); + asm volatile ("vpaddb %ymm9,%ymm9,%ymm9"); + asm volatile ("vpand %ymm6,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm13,%ymm9,%ymm9"); + + asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i + 32])); + asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); + asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); + asm volatile ("vpsrlw $4,%ymm12,%ymm13"); + asm volatile ("vpand %ymm7,%ymm12,%ymm12"); + asm volatile ("vpand %ymm7,%ymm13,%ymm13"); + + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("vpshufb %ymm4,%ymm14,%ymm15"); + asm volatile ("vpshufb %ymm12,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm15,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm14,%ymm10,%ymm10"); + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("vpshufb %ymm5,%ymm14,%ymm15"); + asm volatile ("vpshufb %ymm13,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm15,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm14,%ymm10,%ymm10"); + + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][2][0][0])); + asm volatile ("vpshufb %ymm4,%ymm14,%ymm15"); + asm volatile ("vpshufb %ymm12,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm15,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm14,%ymm11,%ymm11"); + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("vpshufb %ymm5,%ymm14,%ymm15"); + asm volatile ("vpshufb %ymm13,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm15,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm14,%ymm11,%ymm11"); + } + + /* + * d == boundary: Q *= 3. + */ + asm volatile ("vmovdqa %ymm1,%ymm4"); + asm volatile ("vmovdqa %ymm9,%ymm12"); + + asm volatile ("vpxor %ymm5,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm13,%ymm13,%ymm13"); + asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5"); + asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13"); + asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); + asm volatile ("vpaddb %ymm9,%ymm9,%ymm9"); + asm volatile ("vpand %ymm6,%ymm5,%ymm5"); + asm volatile ("vpand %ymm6,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm13,%ymm9,%ymm9"); + asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); + + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i])); + asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i + 32])); + + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); + asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); + + asm volatile ("vpsrlw $4,%ymm4,%ymm5"); + asm volatile ("vpand %ymm7,%ymm4,%ymm4"); + asm volatile ("vpand %ymm7,%ymm5,%ymm5"); + asm volatile ("vpsrlw $4,%ymm12,%ymm13"); + asm volatile ("vpand %ymm7,%ymm12,%ymm12"); + asm volatile ("vpand %ymm7,%ymm13,%ymm13"); + + /* R */ + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("vpshufb %ymm4,%ymm14,%ymm15"); + asm volatile ("vpshufb %ymm12,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm15,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm14,%ymm10,%ymm10"); + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("vpshufb %ymm5,%ymm14,%ymm15"); + asm volatile ("vpshufb %ymm13,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm15,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm14,%ymm10,%ymm10"); + + /* S */ + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][2][0][0])); + asm volatile ("vpshufb %ymm4,%ymm14,%ymm15"); + asm volatile ("vpshufb %ymm12,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm15,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm14,%ymm11,%ymm11"); + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("vpshufb %ymm5,%ymm14,%ymm15"); + asm volatile ("vpshufb %ymm13,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm15,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm14,%ymm11,%ymm11"); + + --d; + boundary -= 51; + if (boundary < 50) + boundary = -1; + } + + /* + * No G23 boundaries remain. Process intermediate disks + * down to D1 using only Q *= 2. + */ + for (; d > 0; --d) { + asm volatile ("vpxor %ymm5,%ymm5,%ymm5"); + asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5"); + asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); + asm volatile ("vpand %ymm6,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); + + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i])); + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); + asm volatile ("vpsrlw $4,%ymm4,%ymm5"); + asm volatile ("vpand %ymm7,%ymm4,%ymm4"); + asm volatile ("vpand %ymm7,%ymm5,%ymm5"); + + asm volatile ("vpxor %ymm13,%ymm13,%ymm13"); + asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13"); + asm volatile ("vpaddb %ymm9,%ymm9,%ymm9"); + asm volatile ("vpand %ymm6,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm13,%ymm9,%ymm9"); + + asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i + 32])); asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); asm volatile ("vpsrlw $4,%ymm12,%ymm13"); @@ -798,37 +1499,26 @@ static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv asm volatile ("vpxor %ymm14,%ymm11,%ymm11"); } - /* first disk with all coefficients at 1 */ - g23_x3 = 0; - if (g23) { - g23_x3 = --g23_count == 0; - if (g23_x3) - g23_count = 51; - } - - if (g23 && g23_x3) - asm volatile ("vmovdqa %ymm1,%ymm4"); + /* + * D0 always follows an x2 transition. + */ asm volatile ("vpxor %ymm5,%ymm5,%ymm5"); asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5"); asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); asm volatile ("vpand %ymm6,%ymm5,%ymm5"); asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); - if (g23 && g23_x3) - asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[0][i])); asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); asm volatile ("vpxor %ymm4,%ymm2,%ymm2"); asm volatile ("vpxor %ymm4,%ymm3,%ymm3"); - if (g23 && g23_x3) - asm volatile ("vmovdqa %ymm9,%ymm12"); + asm volatile ("vpxor %ymm13,%ymm13,%ymm13"); asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13"); asm volatile ("vpaddb %ymm9,%ymm9,%ymm9"); asm volatile ("vpand %ymm6,%ymm13,%ymm13"); asm volatile ("vpxor %ymm13,%ymm9,%ymm9"); - if (g23 && g23_x3) - asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[0][i + 32])); asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); @@ -851,20 +1541,24 @@ static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv void raid_gen4_avx2ext_raid(int nd, size_t size, void **vv) { - raid_gen4_avx2ext_gen(nd, size, vv, 0); + raid_gen4_avx2ext_x2(nd, size, vv); } void raid_gen4_avx2ext_aes(int nd, size_t size, void **vv) { - raid_gen4_avx2ext_gen(nd, size, vv, 1); + if (raid_g23_x2_only(nd)) + raid_gen4_avx2ext_x2(nd, size, vv); + else + raid_gen4_avx2ext_g23(nd, size, vv); } #endif #ifdef CONFIG_X86_64 /* - * GENX AVX2EXT implementation + * GENX AVX2EXT implementation using only Q *= 2. */ -static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, int np, int g23) +static __always_inline void raid_genX_avx2ext_x2(int nd, size_t size, + void **vv, int np) { uint8_t **v = (uint8_t **)vv; size_t i; @@ -878,136 +1572,452 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in memcpy(v[1 + d], v[0], size); return; } - int g23_start = raid_g23_count(l - 1); raid_avx_begin(); - /* generic case with at least two data disks */ asm volatile ("vpxor %ymm6,%ymm6,%ymm6"); asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfconst16.poly[0])); asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 32) { - int g23_count = g23_start; - int g23_x3; - /* last disk without the generator multiplication */ asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[l][i])); asm volatile ("vmovdqa %ymm10,%ymm0"); asm volatile ("vmovdqa %ymm10,%ymm1"); - asm volatile ("vpsrlw $4,%ymm10,%ymm11"); - asm volatile ("vpand %ymm15,%ymm10,%ymm10"); - asm volatile ("vpand %ymm15,%ymm11,%ymm11"); + asm volatile ("vpsrlw $4,%ymm10,%ymm11"); + asm volatile ("vpand %ymm15,%ymm10,%ymm10"); + asm volatile ("vpand %ymm15,%ymm11,%ymm11"); + if (np >= 3) { - asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (raid_gfcauchypshufb[l][1][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[l][1][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm2" : : + "m" (raid_gfcauchypshufb[l][1][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : + "m" (raid_gfcauchypshufb[l][1][1][0])); asm volatile ("vpshufb %ymm10,%ymm2,%ymm2"); asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm13,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm13,%ymm2,%ymm2"); } + if (np >= 4) { - asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (raid_gfcauchypshufb[l][2][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[l][2][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm3" : : + "m" (raid_gfcauchypshufb[l][2][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : + "m" (raid_gfcauchypshufb[l][2][1][0])); asm volatile ("vpshufb %ymm10,%ymm3,%ymm3"); asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm13,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm13,%ymm3,%ymm3"); } + if (np >= 5) { - asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (raid_gfcauchypshufb[l][3][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[l][3][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm4" : : + "m" (raid_gfcauchypshufb[l][3][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : + "m" (raid_gfcauchypshufb[l][3][1][0])); asm volatile ("vpshufb %ymm10,%ymm4,%ymm4"); asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm13,%ymm4,%ymm4"); + asm volatile ("vpxor %ymm13,%ymm4,%ymm4"); } + if (np >= 6) { - asm volatile ("vbroadcasti128 %0,%%ymm5" : : "m" (raid_gfcauchypshufb[l][4][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[l][4][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm5" : : + "m" (raid_gfcauchypshufb[l][4][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : + "m" (raid_gfcauchypshufb[l][4][1][0])); asm volatile ("vpshufb %ymm10,%ymm5,%ymm5"); asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm13,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm13,%ymm5,%ymm5"); } /* intermediate disks */ for (d = l - 1; d > 0; --d) { - g23_x3 = 0; - if (g23) { - g23_x3 = --g23_count == 0; - if (g23_x3) - g23_count = 51; - } - asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[d][i])); - if (g23 && g23_x3) - asm volatile ("vmovdqa %ymm1,%ymm12"); + /* Q *= 2 */ asm volatile ("vpcmpgtb %ymm1,%ymm6,%ymm11"); asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); asm volatile ("vpand %ymm14,%ymm11,%ymm11"); asm volatile ("vpxor %ymm11,%ymm1,%ymm1"); - if (g23 && g23_x3) - asm volatile ("vpxor %ymm12,%ymm1,%ymm1"); asm volatile ("vpxor %ymm10,%ymm0,%ymm0"); asm volatile ("vpxor %ymm10,%ymm1,%ymm1"); - asm volatile ("vpsrlw $4,%ymm10,%ymm11"); - asm volatile ("vpand %ymm15,%ymm10,%ymm10"); - asm volatile ("vpand %ymm15,%ymm11,%ymm11"); + asm volatile ("vpsrlw $4,%ymm10,%ymm11"); + asm volatile ("vpand %ymm15,%ymm10,%ymm10"); + asm volatile ("vpand %ymm15,%ymm11,%ymm11"); + if (np >= 3) { - asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfcauchypshufb[d][1][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm12" : : + "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : + "m" (raid_gfcauchypshufb[d][1][1][0])); asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm12,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm13,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm12,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm13,%ymm2,%ymm2"); } + if (np >= 4) { - asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfcauchypshufb[d][2][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm12" : : + "m" (raid_gfcauchypshufb[d][2][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : + "m" (raid_gfcauchypshufb[d][2][1][0])); asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm12,%ymm3,%ymm3"); - asm volatile ("vpxor %ymm13,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm12,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm13,%ymm3,%ymm3"); } + if (np >= 5) { - asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfcauchypshufb[d][3][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[d][3][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm12" : : + "m" (raid_gfcauchypshufb[d][3][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : + "m" (raid_gfcauchypshufb[d][3][1][0])); asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm12,%ymm4,%ymm4"); - asm volatile ("vpxor %ymm13,%ymm4,%ymm4"); + asm volatile ("vpxor %ymm12,%ymm4,%ymm4"); + asm volatile ("vpxor %ymm13,%ymm4,%ymm4"); } + if (np >= 6) { - asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfcauchypshufb[d][4][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[d][4][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm12" : : + "m" (raid_gfcauchypshufb[d][4][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : + "m" (raid_gfcauchypshufb[d][4][1][0])); asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm12,%ymm5,%ymm5"); - asm volatile ("vpxor %ymm13,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm12,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm13,%ymm5,%ymm5"); } } /* first disk with all coefficients at 1 */ asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[0][i])); - g23_x3 = 0; - if (g23) { - g23_x3 = --g23_count == 0; - if (g23_x3) - g23_count = 51; - } - - if (g23 && g23_x3) - asm volatile ("vmovdqa %ymm1,%ymm12"); + /* final Q *= 2 */ asm volatile ("vpcmpgtb %ymm1,%ymm6,%ymm11"); asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); asm volatile ("vpand %ymm14,%ymm11,%ymm11"); asm volatile ("vpxor %ymm11,%ymm1,%ymm1"); - if (g23 && g23_x3) + + asm volatile ("vpxor %ymm10,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm10,%ymm1,%ymm1"); + if (np >= 3) + asm volatile ("vpxor %ymm10,%ymm2,%ymm2"); + if (np >= 4) + asm volatile ("vpxor %ymm10,%ymm3,%ymm3"); + if (np >= 5) + asm volatile ("vpxor %ymm10,%ymm4,%ymm4"); + if (np >= 6) + asm volatile ("vpxor %ymm10,%ymm5,%ymm5"); + + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovntdq %%ymm1,%0" : "=m" (v[nd + 1][i])); + if (np >= 3) + asm volatile ("vmovntdq %%ymm2,%0" : "=m" (v[nd + 2][i])); + if (np >= 4) + asm volatile ("vmovntdq %%ymm3,%0" : "=m" (v[nd + 3][i])); + if (np >= 5) + asm volatile ("vmovntdq %%ymm4,%0" : "=m" (v[nd + 4][i])); + if (np >= 6) + asm volatile ("vmovntdq %%ymm5,%0" : "=m" (v[nd + 5][i])); + } + + raid_avx_end(); +} + +/* + * GENX AVX2EXT implementation using the AES G23 Q transition. + */ +static __always_inline void raid_genX_avx2ext_g23(int nd, size_t size, + void **vv, int np) +{ + uint8_t **v = (uint8_t **)vv; + size_t i; + int d, l; + + l = nd - 1; + + if (l == 0) { + for (d = 0; d < np; ++d) + memcpy(v[1 + d], v[0], size); + return; + } + + int g23_boundary = raid_g23_boundary(l - 1); + + raid_avx_begin(); + + asm volatile ("vpxor %ymm6,%ymm6,%ymm6"); + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfconst16.poly[0])); + asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0])); + + for (i = 0; i < size; i += 32) { + int boundary = g23_boundary; + + /* last disk */ + asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[l][i])); + + asm volatile ("vmovdqa %ymm10,%ymm0"); + asm volatile ("vmovdqa %ymm10,%ymm1"); + + asm volatile ("vpsrlw $4,%ymm10,%ymm11"); + asm volatile ("vpand %ymm15,%ymm10,%ymm10"); + asm volatile ("vpand %ymm15,%ymm11,%ymm11"); + + if (np >= 3) { + asm volatile ("vbroadcasti128 %0,%%ymm2" : : + "m" (raid_gfcauchypshufb[l][1][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : + "m" (raid_gfcauchypshufb[l][1][1][0])); + asm volatile ("vpshufb %ymm10,%ymm2,%ymm2"); + asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm13,%ymm2,%ymm2"); + } + + if (np >= 4) { + asm volatile ("vbroadcasti128 %0,%%ymm3" : : + "m" (raid_gfcauchypshufb[l][2][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : + "m" (raid_gfcauchypshufb[l][2][1][0])); + asm volatile ("vpshufb %ymm10,%ymm3,%ymm3"); + asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm13,%ymm3,%ymm3"); + } + + if (np >= 5) { + asm volatile ("vbroadcasti128 %0,%%ymm4" : : + "m" (raid_gfcauchypshufb[l][3][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : + "m" (raid_gfcauchypshufb[l][3][1][0])); + asm volatile ("vpshufb %ymm10,%ymm4,%ymm4"); + asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm13,%ymm4,%ymm4"); + } + + if (np >= 6) { + asm volatile ("vbroadcasti128 %0,%%ymm5" : : + "m" (raid_gfcauchypshufb[l][4][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : + "m" (raid_gfcauchypshufb[l][4][1][0])); + asm volatile ("vpshufb %ymm10,%ymm5,%ymm5"); + asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm13,%ymm5,%ymm5"); + } + + d = l - 1; + + while (boundary >= 0) { + /* x2-only part of this segment */ + for (; d > boundary; --d) { + asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[d][i])); + + asm volatile ("vpcmpgtb %ymm1,%ymm6,%ymm11"); + asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); + asm volatile ("vpand %ymm14,%ymm11,%ymm11"); + asm volatile ("vpxor %ymm11,%ymm1,%ymm1"); + + asm volatile ("vpxor %ymm10,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm10,%ymm1,%ymm1"); + + asm volatile ("vpsrlw $4,%ymm10,%ymm11"); + asm volatile ("vpand %ymm15,%ymm10,%ymm10"); + asm volatile ("vpand %ymm15,%ymm11,%ymm11"); + + if (np >= 3) { + asm volatile ("vbroadcasti128 %0,%%ymm12" : : + "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : + "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); + asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm12,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm13,%ymm2,%ymm2"); + } + + if (np >= 4) { + asm volatile ("vbroadcasti128 %0,%%ymm12" : : + "m" (raid_gfcauchypshufb[d][2][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : + "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); + asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm12,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm13,%ymm3,%ymm3"); + } + + if (np >= 5) { + asm volatile ("vbroadcasti128 %0,%%ymm12" : : + "m" (raid_gfcauchypshufb[d][3][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : + "m" (raid_gfcauchypshufb[d][3][1][0])); + asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); + asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm12,%ymm4,%ymm4"); + asm volatile ("vpxor %ymm13,%ymm4,%ymm4"); + } + + if (np >= 6) { + asm volatile ("vbroadcasti128 %0,%%ymm12" : : + "m" (raid_gfcauchypshufb[d][4][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : + "m" (raid_gfcauchypshufb[d][4][1][0])); + asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); + asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm12,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm13,%ymm5,%ymm5"); + } + } + + /* + * d == boundary: Q *= 3. + * + * ymm12 is free here and can temporarily preserve + * the old Q value. + */ + asm volatile ("vmovdqa %ymm1,%ymm12"); + + /* Q *= 2 */ + asm volatile ("vpcmpgtb %ymm1,%ymm6,%ymm11"); + asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); + asm volatile ("vpand %ymm14,%ymm11,%ymm11"); + asm volatile ("vpxor %ymm11,%ymm1,%ymm1"); + + /* Q *= 3 */ asm volatile ("vpxor %ymm12,%ymm1,%ymm1"); + asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[d][i])); + + asm volatile ("vpxor %ymm10,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm10,%ymm1,%ymm1"); + + asm volatile ("vpsrlw $4,%ymm10,%ymm11"); + asm volatile ("vpand %ymm15,%ymm10,%ymm10"); + asm volatile ("vpand %ymm15,%ymm11,%ymm11"); + + if (np >= 3) { + asm volatile ("vbroadcasti128 %0,%%ymm12" : : + "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : + "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); + asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm12,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm13,%ymm2,%ymm2"); + } + + if (np >= 4) { + asm volatile ("vbroadcasti128 %0,%%ymm12" : : + "m" (raid_gfcauchypshufb[d][2][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : + "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); + asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm12,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm13,%ymm3,%ymm3"); + } + + if (np >= 5) { + asm volatile ("vbroadcasti128 %0,%%ymm12" : : + "m" (raid_gfcauchypshufb[d][3][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : + "m" (raid_gfcauchypshufb[d][3][1][0])); + asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); + asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm12,%ymm4,%ymm4"); + asm volatile ("vpxor %ymm13,%ymm4,%ymm4"); + } + + if (np >= 6) { + asm volatile ("vbroadcasti128 %0,%%ymm12" : : + "m" (raid_gfcauchypshufb[d][4][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : + "m" (raid_gfcauchypshufb[d][4][1][0])); + asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); + asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm12,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm13,%ymm5,%ymm5"); + } + + --d; + + boundary -= 51; + if (boundary < 50) + boundary = -1; + } + + /* final x2-only segment */ + for (; d > 0; --d) { + asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[d][i])); + + asm volatile ("vpcmpgtb %ymm1,%ymm6,%ymm11"); + asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); + asm volatile ("vpand %ymm14,%ymm11,%ymm11"); + asm volatile ("vpxor %ymm11,%ymm1,%ymm1"); + + asm volatile ("vpxor %ymm10,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm10,%ymm1,%ymm1"); + + asm volatile ("vpsrlw $4,%ymm10,%ymm11"); + asm volatile ("vpand %ymm15,%ymm10,%ymm10"); + asm volatile ("vpand %ymm15,%ymm11,%ymm11"); + + if (np >= 3) { + asm volatile ("vbroadcasti128 %0,%%ymm12" : : + "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : + "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); + asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm12,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm13,%ymm2,%ymm2"); + } + + if (np >= 4) { + asm volatile ("vbroadcasti128 %0,%%ymm12" : : + "m" (raid_gfcauchypshufb[d][2][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : + "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); + asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm12,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm13,%ymm3,%ymm3"); + } + + if (np >= 5) { + asm volatile ("vbroadcasti128 %0,%%ymm12" : : + "m" (raid_gfcauchypshufb[d][3][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : + "m" (raid_gfcauchypshufb[d][3][1][0])); + asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); + asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm12,%ymm4,%ymm4"); + asm volatile ("vpxor %ymm13,%ymm4,%ymm4"); + } + + if (np >= 6) { + asm volatile ("vbroadcasti128 %0,%%ymm12" : : + "m" (raid_gfcauchypshufb[d][4][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : + "m" (raid_gfcauchypshufb[d][4][1][0])); + asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); + asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm12,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm13,%ymm5,%ymm5"); + } + } + + /* + * D0 always follows Q *= 2. + */ + asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[0][i])); + + asm volatile ("vpcmpgtb %ymm1,%ymm6,%ymm11"); + asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); + asm volatile ("vpand %ymm14,%ymm11,%ymm11"); + asm volatile ("vpxor %ymm11,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm10,%ymm0,%ymm0"); asm volatile ("vpxor %ymm10,%ymm1,%ymm1"); if (np >= 3) @@ -1036,36 +2046,32 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in #endif #ifdef CONFIG_X86_64 -/* - * GEN5 (penta parity with Cauchy matrix) AVX2 implementation - * - * Note that it uses 16 registers, meaning that x64 is required. - */ void raid_gen5_avx2ext_raid(int nd, size_t size, void **vv) { - raid_genX_avx2ext(nd, size, vv, 5, 0); + raid_genX_avx2ext_x2(nd, size, vv, 5); } void raid_gen5_avx2ext_aes(int nd, size_t size, void **vv) { - raid_genX_avx2ext(nd, size, vv, 5, 1); + if (raid_g23_x2_only(nd)) + raid_genX_avx2ext_x2(nd, size, vv, 5); + else + raid_genX_avx2ext_g23(nd, size, vv, 5); } #endif #ifdef CONFIG_X86_64 -/* - * GEN6 (hexa parity with Cauchy matrix) AVX2 implementation - * - * Note that it uses 16 registers, meaning that x64 is required. - */ void raid_gen6_avx2ext_raid(int nd, size_t size, void **vv) { - raid_genX_avx2ext(nd, size, vv, 6, 0); + raid_genX_avx2ext_x2(nd, size, vv, 6); } void raid_gen6_avx2ext_aes(int nd, size_t size, void **vv) { - raid_genX_avx2ext(nd, size, vv, 6, 1); + if (raid_g23_x2_only(nd)) + raid_genX_avx2ext_x2(nd, size, vv, 6); + else + raid_genX_avx2ext_g23(nd, size, vv, 6); } #endif diff --git a/raid/int.c b/raid/int.c index 97c00ee0..3fb0ccdb 100644 --- a/raid/int.c +++ b/raid/int.c @@ -61,9 +61,53 @@ void raid_gen1_int64(int nd, size_t size, void **vv) } /* - * GEN2 (double parity) 32bit C implementation + * GEN2 (double parity) 32bit C implementation using only Q *= 2. */ -static __always_inline void raid_gen2_int32_gen(int nd, size_t size, void **vv, int g23) +static __always_inline void raid_gen2_int32_x2(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + int d, l; + size_t i; + const uint32_t poly_32 = raid_poly_32; + + uint32_t d0, q0, p0; + uint32_t d1, q1, p1; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + + for (i = 0; i < size; i += 8) { + q0 = p0 = v_read32(&v[l][i]); + q1 = p1 = v_read32(&v[l][i + 4]); + + for (d = l - 1; d >= 0; --d) { + d0 = v_read32(&v[d][i]); + d1 = v_read32(&v[d][i + 4]); + + p0 ^= d0; + p1 ^= d1; + + q0 = x2_32(q0, poly_32); + q1 = x2_32(q1, poly_32); + + q0 ^= d0; + q1 ^= d1; + } + + v_write32(&p[i], p0); + v_write32(&p[i + 4], p1); + v_write32(&q[i], q0); + v_write32(&q[i + 4], q1); + } +} + +/* + * GEN2 (double parity) 32bit C implementation using the AES G23 Q transition. + */ +static __always_inline void raid_gen2_int32_g23(int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -79,43 +123,76 @@ static __always_inline void raid_gen2_int32_gen(int nd, size_t size, void **vv, l = nd - 1; p = v[nd]; q = v[nd + 1]; - int g23_start = raid_g23_count(l - 1); + + int g23_boundary = raid_g23_boundary(l - 1); for (i = 0; i < size; i += 8) { - int g23_count = g23_start; - int g23_x3; + int boundary = g23_boundary; q0 = p0 = v_read32(&v[l][i]); q1 = p1 = v_read32(&v[l][i + 4]); - for (d = l - 1; d >= 0; --d) { - g23_x3 = 0; - if (g23) { - g23_x3 = --g23_count == 0; - if (g23_x3) - g23_count = 51; + + d = l - 1; + + for (;;) { + /* + * All transitions above 'boundary' are x2. + */ + for (; d > boundary; --d) { + d0 = v_read32(&v[d][i]); + d1 = v_read32(&v[d][i + 4]); + + p0 ^= d0; + p1 ^= d1; + + q0 = x2_32(q0, poly_32); + q1 = x2_32(q1, poly_32); + + q0 ^= d0; + q1 ^= d1; } + /* + * boundary == -1 identifies the final x2-only segment. + * At this point all remaining disks have been consumed. + */ + if (boundary < 0) + break; + + /* + * d == boundary. + * + * Perform the fixed G23 x3 transition and then add + * the current disk to P and Q. + */ d0 = v_read32(&v[d][i]); d1 = v_read32(&v[d][i + 4]); p0 ^= d0; p1 ^= d1; - if (g23 && g23_x3) { - qold = q0; - q0 = x2_32(q0, poly_32); - q0 ^= qold; - qold = q1; - q1 = x2_32(q1, poly_32); - q1 ^= qold; - } else { - q0 = x2_32(q0, poly_32); - q1 = x2_32(q1, poly_32); - } + qold = q0; + q0 = x2_32(q0, poly_32); + q0 ^= qold; + + qold = q1; + q1 = x2_32(q1, poly_32); + q1 ^= qold; q0 ^= d0; q1 ^= d1; + + --d; + + /* + * G23 boundaries are exactly 51 transitions apart. + * After d = 50 there are no more x3 transitions. + */ + boundary -= 51; + if (boundary < 50) + boundary = -1; } + v_write32(&p[i], p0); v_write32(&p[i + 4], p1); v_write32(&q[i], q0); @@ -125,18 +202,65 @@ static __always_inline void raid_gen2_int32_gen(int nd, size_t size, void **vv, void raid_gen2_int32_raid(int nd, size_t size, void **vv) { - raid_gen2_int32_gen(nd, size, vv, 0); + raid_gen2_int32_x2(nd, size, vv); } void raid_gen2_int32_aes(int nd, size_t size, void **vv) { - raid_gen2_int32_gen(nd, size, vv, 1); + if (raid_g23_x2_only(nd)) + raid_gen2_int32_x2(nd, size, vv); + else + raid_gen2_int32_g23(nd, size, vv); } /* - * GEN2 (double parity) 64bit C implementation + * GEN2 (double parity) 64bit C implementation using only Q *= 2. */ -static __always_inline void raid_gen2_int64_gen(int nd, size_t size, void **vv, int g23) +static __always_inline void raid_gen2_int64_x2(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + int d, l; + size_t i; + const uint64_t poly_64 = raid_poly_64; + + uint64_t d0, q0, p0; + uint64_t d1, q1, p1; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + + for (i = 0; i < size; i += 16) { + q0 = p0 = v_read64(&v[l][i]); + q1 = p1 = v_read64(&v[l][i + 8]); + + for (d = l - 1; d >= 0; --d) { + d0 = v_read64(&v[d][i]); + d1 = v_read64(&v[d][i + 8]); + + p0 ^= d0; + p1 ^= d1; + + q0 = x2_64(q0, poly_64); + q1 = x2_64(q1, poly_64); + + q0 ^= d0; + q1 ^= d1; + } + + v_write64(&p[i], p0); + v_write64(&p[i + 8], p1); + v_write64(&q[i], q0); + v_write64(&q[i + 8], q1); + } +} + +/* + * GEN2 (double parity) 64bit C implementation using the AES G23 Q transition. + */ +static __always_inline void raid_gen2_int64_g23(int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -152,43 +276,76 @@ static __always_inline void raid_gen2_int64_gen(int nd, size_t size, void **vv, l = nd - 1; p = v[nd]; q = v[nd + 1]; - int g23_start = raid_g23_count(l - 1); + + int g23_boundary = raid_g23_boundary(l - 1); for (i = 0; i < size; i += 16) { - int g23_count = g23_start; - int g23_x3; + int boundary = g23_boundary; q0 = p0 = v_read64(&v[l][i]); q1 = p1 = v_read64(&v[l][i + 8]); - for (d = l - 1; d >= 0; --d) { - g23_x3 = 0; - if (g23) { - g23_x3 = --g23_count == 0; - if (g23_x3) - g23_count = 51; + + d = l - 1; + + for (;;) { + /* + * All transitions above 'boundary' are x2. + */ + for (; d > boundary; --d) { + d0 = v_read64(&v[d][i]); + d1 = v_read64(&v[d][i + 8]); + + p0 ^= d0; + p1 ^= d1; + + q0 = x2_64(q0, poly_64); + q1 = x2_64(q1, poly_64); + + q0 ^= d0; + q1 ^= d1; } + /* + * boundary == -1 identifies the final x2-only segment. + * At this point all remaining disks have been consumed. + */ + if (boundary < 0) + break; + + /* + * d == boundary. + * + * Perform the fixed G23 x3 transition and then add + * the current disk to P and Q. + */ d0 = v_read64(&v[d][i]); d1 = v_read64(&v[d][i + 8]); p0 ^= d0; p1 ^= d1; - if (g23 && g23_x3) { - qold = q0; - q0 = x2_64(q0, poly_64); - q0 ^= qold; - qold = q1; - q1 = x2_64(q1, poly_64); - q1 ^= qold; - } else { - q0 = x2_64(q0, poly_64); - q1 = x2_64(q1, poly_64); - } + qold = q0; + q0 = x2_64(q0, poly_64); + q0 ^= qold; + + qold = q1; + q1 = x2_64(q1, poly_64); + q1 ^= qold; q0 ^= d0; q1 ^= d1; + + --d; + + /* + * G23 boundaries are exactly 51 transitions apart. + * After d = 50 there are no more x3 transitions. + */ + boundary -= 51; + if (boundary < 50) + boundary = -1; } + v_write64(&p[i], p0); v_write64(&p[i + 8], p1); v_write64(&q[i], q0); @@ -198,17 +355,15 @@ static __always_inline void raid_gen2_int64_gen(int nd, size_t size, void **vv, void raid_gen2_int64_raid(int nd, size_t size, void **vv) { - raid_gen2_int64_gen(nd, size, vv, 0); + raid_gen2_int64_x2(nd, size, vv); } void raid_gen2_int64_aes(int nd, size_t size, void **vv) { - raid_gen2_int64_gen(nd, size, vv, 1); -} - -void raid_gen2_int8(int nd, size_t size, void **vv) -{ - raid_gen_ref(nd, 2, size, vv); + if (raid_g23_x2_only(nd)) + raid_gen2_int64_x2(nd, size, vv); + else + raid_gen2_int64_g23(nd, size, vv); } /* @@ -720,7 +875,6 @@ void raid_recX_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv) void raid_register_int(void) { - raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "int8", raid_gen2_int8, RAID_POLY_ANY); raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "int8", raid_gen3_int8, RAID_POLY_ANY); raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "int8", raid_gen4_int8, RAID_POLY_ANY); raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "int8", raid_gen5_int8, RAID_POLY_ANY); diff --git a/raid/internal.h b/raid/internal.h index b16b8515..16a871f1 100644 --- a/raid/internal.h +++ b/raid/internal.h @@ -75,7 +75,15 @@ static __always_inline void * __align_ptr(void *ptr, uintptr_t size) } /* - * Initialize the countdown used by the AES G23 Horner recurrence. + * Check if the number of data disks requires only x2 multipliers in AES G23. + */ +static __always_inline int raid_g23_x2_only(int nd) +{ + return nd <= 51; +} + +/* + * Find the next AES G23 x3 boundary in descending Horner order. * * The G23 Q coefficients are generated by multiplying by 2 at every step, * except at the four boundaries between the five cosets, where the next @@ -92,43 +100,53 @@ static __always_inline void * __align_ptr(void *ptr, uintptr_t size) * * i.e. every 51 descending steps. * - * Rather than comparing d against those four values in the hot loop, we keep - * a countdown. Before processing each Horner step the generator does: + * The generators can process all the x2 transitions between two consecutive + * boundaries in a branch-free loop, and handle the x3 transition separately. + * This function returns the first x3 boundary encountered by a descending + * traversal whose first processed step has index d. * - * if (--g23_count == 0) { - * Q = 3 * Q; - * g23_count = 51; - * } else { - * Q = 2 * Q; - * } + * For d >= 50, (d - 50) % 51 gives the distance from the previous x3 + * boundary in ascending index order. Subtracting this distance from d gives + * that boundary directly: * - * This function returns the initial countdown for a traversal whose first - * processed step has index d. For d >= 50, (d - 50) % 51 gives the distance - * from the previous x3 boundary, so adding 1 gives the number of decrements - * required before the next descending x3 transition is reached. + * boundary = d - (d - 50) % 51 * - * For d < 50 there is no remaining x3 transition. Returning 51 is safe - * because fewer than 51 Horner steps remain, so the countdown cannot reach - * zero before the traversal ends. + * Since the Horner traversal proceeds toward decreasing indexes, this is also + * the next x3 transition that will be encountered. + * + * For d < 50 there is no remaining x3 transition, so -1 is returned. This + * value can also be used as the lower bound of the final x2-only segment, + * allowing all remaining steps down to d = 0 to be processed without a + * special G23 test. * * Examples: * - * d = 203 -> 1 -> x3 immediately - * d = 204 -> 2 -> x2 at 204, x3 at 203 - * d = 250 -> 48 -> x3 at 203 - * d = 152 -> 1 -> x3 immediately - * d = 50 -> 1 -> x3 immediately - * d = 49 -> 51 -> no further x3 transition + * d = 249 -> 203 -> x2 down to 204, then x3 at 203 + * d = 203 -> 203 -> x3 immediately + * d = 202 -> 152 -> x2 down to 153, then x3 at 152 + * d = 152 -> 152 -> x3 immediately + * d = 151 -> 101 -> x2 down to 102, then x3 at 101 + * d = 101 -> 101 -> x3 immediately + * d = 100 -> 50 -> x2 down to 51, then x3 at 50 + * d = 50 -> 50 -> x3 immediately + * d = 49 -> -1 -> no further x3 transition * - * The returned value is normally computed once as g23_start and copied into - * g23_count at the beginning of each independent Horner traversal. + * After processing a boundary, the following one in descending order is + * exactly 51 positions lower. Thus generators can continue with: + * + * boundary -= 51; + * if (boundary < 50) + * boundary = -1; + * + * The returned value is normally computed once before each independent + * Horner traversal and then advanced between the fixed G23 segments. */ -static __always_inline int raid_g23_count(int d) +static __always_inline int raid_g23_boundary(int d) { if (d < 50) - return 51; + return -1; - return (d - 50) % 51 + 1; + return d - (d - 50) % 51; } /* @@ -152,7 +170,6 @@ void raid_gen1_int64(int nd, size_t size, void **vv); void raid_gen1_sse2(int nd, size_t size, void **vv); void raid_gen1_avx2(int nd, size_t size, void **vv); void raid_gen1_avx512bw(int nd, size_t size, void **vv); -void raid_gen2_int8(int nd, size_t size, void **vv); void raid_gen2_int32_raid(int nd, size_t size, void **vv); void raid_gen2_int32_aes(int nd, size_t size, void **vv); void raid_gen2_int64_raid(int nd, size_t size, void **vv); @@ -201,6 +218,7 @@ void raid_gen4_avx2gfni_aes(int nd, size_t size, void **vv); void raid_gen4_avx512gfni_aes(int nd, size_t size, void **vv); void raid_gen5_int8(int nd, size_t size, void **vv); void raid_gen5_ssse3_raid(int nd, size_t size, void **vv); +void raid_gen5_ssse3_aes(int nd, size_t size, void **vv); void raid_gen5_ssse3ext_raid(int nd, size_t size, void **vv); void raid_gen5_ssse3ext_aes(int nd, size_t size, void **vv); void raid_gen5_avx2ext_raid(int nd, size_t size, void **vv); @@ -212,6 +230,7 @@ void raid_gen5_avx2gfni_aes(int nd, size_t size, void **vv); void raid_gen5_avx512gfni_aes(int nd, size_t size, void **vv); void raid_gen6_int8(int nd, size_t size, void **vv); void raid_gen6_ssse3_raid(int nd, size_t size, void **vv); +void raid_gen6_ssse3_aes(int nd, size_t size, void **vv); void raid_gen6_ssse3ext_raid(int nd, size_t size, void **vv); void raid_gen6_ssse3ext_aes(int nd, size_t size, void **vv); void raid_gen6_avx2ext_raid(int nd, size_t size, void **vv); diff --git a/raid/neon.c b/raid/neon.c index 6afd2177..c11bd51c 100644 --- a/raid/neon.c +++ b/raid/neon.c @@ -60,9 +60,9 @@ void raid_gen1_neon(int nd, size_t size, void **vv) } /* - * GEN2 Cauchy NEON implementation using the active Q transition + * GEN2 Cauchy NEON implementation using only Q *= 2. */ -static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, int g23) +static __always_inline void raid_gen2_neon_x2(int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -81,12 +81,8 @@ static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, i : : "m" (gfconst16.poly[0]) ); - int g23_start = raid_g23_count(l - 1); for (i = 0; i < size; i += 32) { - int g23_count = g23_start; - int g23_x3; - asm volatile ( "ldr q0, %0\n" "ldr q1, %1\n" @@ -97,20 +93,6 @@ static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, i ); for (d = l - 1; d >= 0; --d) { - g23_x3 = 0; - if (g23) { - g23_x3 = --g23_count == 0; - if (g23_x3) - g23_count = 51; - } - - if (g23 && g23_x3) { - /* v14-v15 are otherwise unused and preserve Q across the AES xtime. */ - asm volatile ( - "mov v14.16b, v2.16b\n" - "mov v15.16b, v3.16b\n" - ); - } asm volatile ( /* double Q0-Q1 */ "sshr v16.16b, v2.16b, #7\n" @@ -133,12 +115,142 @@ static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, i : : "m" (v[d][i]), "m" (v[d][i + 16]) ); - if (g23 && g23_x3) { + } + + asm volatile ( + "str q0, %0\n" + "str q1, %1\n" + "str q2, %2\n" + "str q3, %3\n" + : "=m" (p[i]), "=m" (p[i + 16]), + "=m" (q[i]), "=m" (q[i + 16]) + ); + } + + raid_neon_end(); +} + +/* + * GEN2 Cauchy NEON implementation using the AES G23 Q transition. + */ +static __always_inline void raid_gen2_neon_g23(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + + int g23_boundary = raid_g23_boundary(l - 1); + + raid_neon_begin(); + + asm volatile ( + "ldr q29, %0\n" + : + : "m" (gfconst16.poly[0]) + ); + + for (i = 0; i < size; i += 32) { + int boundary = g23_boundary; + + asm volatile ( + "ldr q0, %0\n" + "ldr q1, %1\n" + "mov v2.16b, v0.16b\n" + "mov v3.16b, v1.16b\n" + : + : "m" (v[l][i]), "m" (v[l][i + 16]) + ); + + d = l - 1; + + for (;;) { + /* + * All transitions above 'boundary' are x2. + */ + for (; d > boundary; --d) { asm volatile ( - "eor v2.16b, v2.16b, v14.16b\n" - "eor v3.16b, v3.16b, v15.16b\n" + /* double Q0-Q1 */ + "sshr v16.16b, v2.16b, #7\n" + "shl v2.16b, v2.16b, #1\n" + "and v16.16b, v16.16b, v29.16b\n" + "eor v2.16b, v2.16b, v16.16b\n" + + "sshr v16.16b, v3.16b, #7\n" + "shl v3.16b, v3.16b, #1\n" + "and v16.16b, v16.16b, v29.16b\n" + "eor v3.16b, v3.16b, v16.16b\n" + + /* load and XOR */ + "ldr q12, %0\n" + "ldr q13, %1\n" + "eor v0.16b, v0.16b, v12.16b\n" + "eor v1.16b, v1.16b, v13.16b\n" + "eor v2.16b, v2.16b, v12.16b\n" + "eor v3.16b, v3.16b, v13.16b\n" + : + : "m" (v[d][i]), "m" (v[d][i + 16]) ); } + + /* + * boundary == -1 identifies the final x2-only segment. + * At this point all remaining disks have been consumed. + */ + if (boundary < 0) + break; + + /* + * d == boundary. + * + * Perform the fixed G23 x3 transition and then add + * the current disk to P and Q. + */ + asm volatile ( + /* preserve Q for Q *= 3 */ + "mov v14.16b, v2.16b\n" + "mov v15.16b, v3.16b\n" + + /* double Q0-Q1 */ + "sshr v16.16b, v2.16b, #7\n" + "shl v2.16b, v2.16b, #1\n" + "and v16.16b, v16.16b, v29.16b\n" + "eor v2.16b, v2.16b, v16.16b\n" + + "sshr v16.16b, v3.16b, #7\n" + "shl v3.16b, v3.16b, #1\n" + "and v16.16b, v16.16b, v29.16b\n" + "eor v3.16b, v3.16b, v16.16b\n" + + /* load and XOR */ + "ldr q12, %0\n" + "ldr q13, %1\n" + "eor v0.16b, v0.16b, v12.16b\n" + "eor v1.16b, v1.16b, v13.16b\n" + "eor v2.16b, v2.16b, v12.16b\n" + "eor v3.16b, v3.16b, v13.16b\n" + + /* complete Q *= 3 */ + "eor v2.16b, v2.16b, v14.16b\n" + "eor v3.16b, v3.16b, v15.16b\n" + : + : "m" (v[d][i]), "m" (v[d][i + 16]) + ); + + --d; + + /* + * G23 boundaries are exactly 51 transitions apart. + * After d = 50 there are no more x3 transitions. + */ + boundary -= 51; + if (boundary < 50) + boundary = -1; } asm volatile ( @@ -156,12 +268,15 @@ static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, i void raid_gen2_neon_raid(int nd, size_t size, void **vv) { - raid_gen2_neon_gen(nd, size, vv, 0); + raid_gen2_neon_x2(nd, size, vv); } void raid_gen2_neon_aes(int nd, size_t size, void **vv) { - raid_gen2_neon_gen(nd, size, vv, 1); + if (raid_g23_x2_only(nd)) + raid_gen2_neon_x2(nd, size, vv); + else + raid_gen2_neon_g23(nd, size, vv); } /* @@ -259,9 +374,10 @@ void raid_genz_neon_raid(int nd, size_t size, void **vv) } /* - * GENX NEON implementation + * GENX NEON implementation using only Q *= 2. */ -static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int np, int g23) +static __always_inline void raid_genX_neon_x2(int nd, size_t size, + void **vv, int np) { uint8_t **v = (uint8_t **)vv; size_t i; @@ -275,7 +391,6 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n memcpy(v[1 + d], v[0], size); return; } - int g23_start = raid_g23_count(l - 1); raid_neon_begin(); @@ -288,9 +403,6 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n ); for (i = 0; i < size; i += 32) { - int g23_count = g23_start; - int g23_x3; - /* last disk without the generator multiplication */ asm volatile ( "ldr q0, %0\n" @@ -323,9 +435,11 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n "tbl v20.16b, {v23.16b}, v19.16b\n" "eor v5.16b, v5.16b, v20.16b\n" : - : "m" (raid_gfcauchypshufb[l][1][0][0]), "m" (raid_gfcauchypshufb[l][1][1][0]) + : "m" (raid_gfcauchypshufb[l][1][0][0]), + "m" (raid_gfcauchypshufb[l][1][1][0]) ); } + if (np >= 4) { asm volatile ( "ldr q22, %0\n" @@ -338,9 +452,11 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n "tbl v20.16b, {v23.16b}, v19.16b\n" "eor v7.16b, v7.16b, v20.16b\n" : - : "m" (raid_gfcauchypshufb[l][2][0][0]), "m" (raid_gfcauchypshufb[l][2][1][0]) + : "m" (raid_gfcauchypshufb[l][2][0][0]), + "m" (raid_gfcauchypshufb[l][2][1][0]) ); } + if (np >= 5) { asm volatile ( "ldr q22, %0\n" @@ -353,9 +469,11 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n "tbl v20.16b, {v23.16b}, v19.16b\n" "eor v9.16b, v9.16b, v20.16b\n" : - : "m" (raid_gfcauchypshufb[l][3][0][0]), "m" (raid_gfcauchypshufb[l][3][1][0]) + : "m" (raid_gfcauchypshufb[l][3][0][0]), + "m" (raid_gfcauchypshufb[l][3][1][0]) ); } + if (np >= 6) { asm volatile ( "ldr q22, %0\n" @@ -368,25 +486,13 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n "tbl v20.16b, {v23.16b}, v19.16b\n" "eor v11.16b, v11.16b, v20.16b\n" : - : "m" (raid_gfcauchypshufb[l][4][0][0]), "m" (raid_gfcauchypshufb[l][4][1][0]) + : "m" (raid_gfcauchypshufb[l][4][0][0]), + "m" (raid_gfcauchypshufb[l][4][1][0]) ); } /* intermediate disks */ for (d = l - 1; d > 0; --d) { - g23_x3 = 0; - if (g23) { - g23_x3 = --g23_count == 0; - if (g23_x3) - g23_count = 51; - } - - if (g23 && g23_x3) { - asm volatile ( - "mov v14.16b, v2.16b\n" - "mov v15.16b, v3.16b\n" - ); - } asm volatile ( /* double Q0-Q1 */ "sshr v20.16b, v2.16b, #7\n" @@ -409,12 +515,6 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n : : "m" (v[d][i]), "m" (v[d][i + 16]) ); - if (g23 && g23_x3) { - asm volatile ( - "eor v2.16b, v2.16b, v14.16b\n" - "eor v3.16b, v3.16b, v15.16b\n" - ); - } if (np >= 3) { asm volatile ( @@ -440,9 +540,11 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n "eor v20.16b, v20.16b, v21.16b\n" "eor v5.16b, v5.16b, v20.16b\n" : - : "m" (raid_gfcauchypshufb[d][1][0][0]), "m" (raid_gfcauchypshufb[d][1][1][0]) + : "m" (raid_gfcauchypshufb[d][1][0][0]), + "m" (raid_gfcauchypshufb[d][1][1][0]) ); } + if (np >= 4) { asm volatile ( "ldr q22, %0\n" @@ -457,9 +559,11 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n "eor v20.16b, v20.16b, v21.16b\n" "eor v7.16b, v7.16b, v20.16b\n" : - : "m" (raid_gfcauchypshufb[d][2][0][0]), "m" (raid_gfcauchypshufb[d][2][1][0]) + : "m" (raid_gfcauchypshufb[d][2][0][0]), + "m" (raid_gfcauchypshufb[d][2][1][0]) ); } + if (np >= 5) { asm volatile ( "ldr q22, %0\n" @@ -474,9 +578,11 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n "eor v20.16b, v20.16b, v21.16b\n" "eor v9.16b, v9.16b, v20.16b\n" : - : "m" (raid_gfcauchypshufb[d][3][0][0]), "m" (raid_gfcauchypshufb[d][3][1][0]) + : "m" (raid_gfcauchypshufb[d][3][0][0]), + "m" (raid_gfcauchypshufb[d][3][1][0]) ); } + if (np >= 6) { asm volatile ( "ldr q22, %0\n" @@ -491,25 +597,13 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n "eor v20.16b, v20.16b, v21.16b\n" "eor v11.16b, v11.16b, v20.16b\n" : - : "m" (raid_gfcauchypshufb[d][4][0][0]), "m" (raid_gfcauchypshufb[d][4][1][0]) + : "m" (raid_gfcauchypshufb[d][4][0][0]), + "m" (raid_gfcauchypshufb[d][4][1][0]) ); } } /* first disk with all coefficients at 1 */ - g23_x3 = 0; - if (g23) { - g23_x3 = --g23_count == 0; - if (g23_x3) - g23_count = 51; - } - - if (g23 && g23_x3) { - asm volatile ( - "mov v14.16b, v2.16b\n" - "mov v15.16b, v3.16b\n" - ); - } asm volatile ( /* double Q0-Q1 */ "sshr v20.16b, v2.16b, #7\n" @@ -532,12 +626,6 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n : : "m" (v[0][i]), "m" (v[0][i + 16]) ); - if (g23 && g23_x3) { - asm volatile ( - "eor v2.16b, v2.16b, v14.16b\n" - "eor v3.16b, v3.16b, v15.16b\n" - ); - } if (np >= 3) { asm volatile ( @@ -564,13 +652,560 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n ); } + /* write parity in increasing order */ asm volatile ( "str q0, %0\n" "str q1, %1\n" "str q2, %2\n" "str q3, %3\n" : "=m" (v[nd][i]), "=m" (v[nd][i + 16]), - "=m" (v[nd + 1][i]), "=m" (v[nd + 1][i + 16]) + "=m" (v[nd + 1][i]), "=m" (v[nd + 1][i + 16]) + ); + + if (np >= 3) { + asm volatile ( + "str q4, %0\n" + "str q5, %1\n" + : "=m" (v[nd + 2][i]), "=m" (v[nd + 2][i + 16]) + ); + } + if (np >= 4) { + asm volatile ( + "str q6, %0\n" + "str q7, %1\n" + : "=m" (v[nd + 3][i]), "=m" (v[nd + 3][i + 16]) + ); + } + if (np >= 5) { + asm volatile ( + "str q8, %0\n" + "str q9, %1\n" + : "=m" (v[nd + 4][i]), "=m" (v[nd + 4][i + 16]) + ); + } + if (np >= 6) { + asm volatile ( + "str q10, %0\n" + "str q11, %1\n" + : "=m" (v[nd + 5][i]), "=m" (v[nd + 5][i + 16]) + ); + } + } + + raid_neon_end(); +} + +/* + * GENX NEON implementation using the AES G23 Q transition. + */ +static __always_inline void raid_genX_neon_g23(int nd, size_t size, + void **vv, int np) +{ + uint8_t **v = (uint8_t **)vv; + size_t i; + int d, l; + + l = nd - 1; + + /* special case with only one data disk */ + if (l == 0) { + for (d = 0; d < np; ++d) + memcpy(v[1 + d], v[0], size); + return; + } + + int g23_boundary = raid_g23_boundary(l - 1); + + raid_neon_begin(); + + asm volatile ( + "ldr q29, %0\n" + "ldr q28, %1\n" + : + : "m" (gfconst16.poly[0]), "m" (gfconst16.low4[0]) + ); + + for (i = 0; i < size; i += 32) { + int boundary = g23_boundary; + + /* last disk without the generator multiplication */ + asm volatile ( + "ldr q0, %0\n" + "ldr q1, %1\n" + "mov v2.16b, v0.16b\n" + "mov v3.16b, v1.16b\n" + : + : "m" (v[l][i]), "m" (v[l][i + 16]) + ); + + if (np >= 3) { + asm volatile ( + "ushr v17.16b, v0.16b, #4\n" + "and v16.16b, v0.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "ushr v19.16b, v1.16b, #4\n" + "and v18.16b, v1.16b, v28.16b\n" + "and v19.16b, v19.16b, v28.16b\n" + ); + asm volatile ( + "ldr q22, %0\n" + "ldr q23, %1\n" + "tbl v4.16b, {v22.16b}, v16.16b\n" + "tbl v20.16b, {v23.16b}, v17.16b\n" + "eor v4.16b, v4.16b, v20.16b\n" + "tbl v5.16b, {v22.16b}, v18.16b\n" + "tbl v20.16b, {v23.16b}, v19.16b\n" + "eor v5.16b, v5.16b, v20.16b\n" + : + : "m" (raid_gfcauchypshufb[l][1][0][0]), + "m" (raid_gfcauchypshufb[l][1][1][0]) + ); + } + + if (np >= 4) { + asm volatile ( + "ldr q22, %0\n" + "ldr q23, %1\n" + "tbl v6.16b, {v22.16b}, v16.16b\n" + "tbl v20.16b, {v23.16b}, v17.16b\n" + "eor v6.16b, v6.16b, v20.16b\n" + "tbl v7.16b, {v22.16b}, v18.16b\n" + "tbl v20.16b, {v23.16b}, v19.16b\n" + "eor v7.16b, v7.16b, v20.16b\n" + : + : "m" (raid_gfcauchypshufb[l][2][0][0]), + "m" (raid_gfcauchypshufb[l][2][1][0]) + ); + } + + if (np >= 5) { + asm volatile ( + "ldr q22, %0\n" + "ldr q23, %1\n" + "tbl v8.16b, {v22.16b}, v16.16b\n" + "tbl v20.16b, {v23.16b}, v17.16b\n" + "eor v8.16b, v8.16b, v20.16b\n" + "tbl v9.16b, {v22.16b}, v18.16b\n" + "tbl v20.16b, {v23.16b}, v19.16b\n" + "eor v9.16b, v9.16b, v20.16b\n" + : + : "m" (raid_gfcauchypshufb[l][3][0][0]), + "m" (raid_gfcauchypshufb[l][3][1][0]) + ); + } + + if (np >= 6) { + asm volatile ( + "ldr q22, %0\n" + "ldr q23, %1\n" + "tbl v10.16b, {v22.16b}, v16.16b\n" + "tbl v20.16b, {v23.16b}, v17.16b\n" + "eor v10.16b, v10.16b, v20.16b\n" + "tbl v11.16b, {v22.16b}, v18.16b\n" + "tbl v20.16b, {v23.16b}, v19.16b\n" + "eor v11.16b, v11.16b, v20.16b\n" + : + : "m" (raid_gfcauchypshufb[l][4][0][0]), + "m" (raid_gfcauchypshufb[l][4][1][0]) + ); + } + + d = l - 1; + + while (boundary >= 0) { + /* + * All transitions above the next G23 boundary use Q *= 2. + */ + for (; d > boundary; --d) { + asm volatile ( + "sshr v20.16b, v2.16b, #7\n" + "shl v2.16b, v2.16b, #1\n" + "and v20.16b, v20.16b, v29.16b\n" + "eor v2.16b, v2.16b, v20.16b\n" + + "sshr v20.16b, v3.16b, #7\n" + "shl v3.16b, v3.16b, #1\n" + "and v20.16b, v20.16b, v29.16b\n" + "eor v3.16b, v3.16b, v20.16b\n" + + "ldr q12, %0\n" + "ldr q13, %1\n" + "eor v0.16b, v0.16b, v12.16b\n" + "eor v1.16b, v1.16b, v13.16b\n" + "eor v2.16b, v2.16b, v12.16b\n" + "eor v3.16b, v3.16b, v13.16b\n" + : + : "m" (v[d][i]), "m" (v[d][i + 16]) + ); + + if (np >= 3) { + asm volatile ( + "ushr v17.16b, v12.16b, #4\n" + "and v16.16b, v12.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "ushr v19.16b, v13.16b, #4\n" + "and v18.16b, v13.16b, v28.16b\n" + "and v19.16b, v19.16b, v28.16b\n" + ); + + asm volatile ( + "ldr q22, %0\n" + "ldr q23, %1\n" + "tbl v20.16b, {v22.16b}, v16.16b\n" + "tbl v21.16b, {v23.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v4.16b, v4.16b, v20.16b\n" + "tbl v20.16b, {v22.16b}, v18.16b\n" + "tbl v21.16b, {v23.16b}, v19.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v5.16b, v5.16b, v20.16b\n" + : + : "m" (raid_gfcauchypshufb[d][1][0][0]), + "m" (raid_gfcauchypshufb[d][1][1][0]) + ); + } + + if (np >= 4) { + asm volatile ( + "ldr q22, %0\n" + "ldr q23, %1\n" + "tbl v20.16b, {v22.16b}, v16.16b\n" + "tbl v21.16b, {v23.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v6.16b, v6.16b, v20.16b\n" + "tbl v20.16b, {v22.16b}, v18.16b\n" + "tbl v21.16b, {v23.16b}, v19.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v7.16b, v7.16b, v20.16b\n" + : + : "m" (raid_gfcauchypshufb[d][2][0][0]), + "m" (raid_gfcauchypshufb[d][2][1][0]) + ); + } + + if (np >= 5) { + asm volatile ( + "ldr q22, %0\n" + "ldr q23, %1\n" + "tbl v20.16b, {v22.16b}, v16.16b\n" + "tbl v21.16b, {v23.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v8.16b, v8.16b, v20.16b\n" + "tbl v20.16b, {v22.16b}, v18.16b\n" + "tbl v21.16b, {v23.16b}, v19.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v9.16b, v9.16b, v20.16b\n" + : + : "m" (raid_gfcauchypshufb[d][3][0][0]), + "m" (raid_gfcauchypshufb[d][3][1][0]) + ); + } + + if (np >= 6) { + asm volatile ( + "ldr q22, %0\n" + "ldr q23, %1\n" + "tbl v20.16b, {v22.16b}, v16.16b\n" + "tbl v21.16b, {v23.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v10.16b, v10.16b, v20.16b\n" + "tbl v20.16b, {v22.16b}, v18.16b\n" + "tbl v21.16b, {v23.16b}, v19.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v11.16b, v11.16b, v20.16b\n" + : + : "m" (raid_gfcauchypshufb[d][4][0][0]), + "m" (raid_gfcauchypshufb[d][4][1][0]) + ); + } + } + + /* + * d == boundary. + * + * Preserve Q in v14/v15, perform Q *= 2, then XOR + * the old value back to obtain Q *= 3. + */ + asm volatile ( + "mov v14.16b, v2.16b\n" + "mov v15.16b, v3.16b\n" + + "sshr v20.16b, v2.16b, #7\n" + "shl v2.16b, v2.16b, #1\n" + "and v20.16b, v20.16b, v29.16b\n" + "eor v2.16b, v2.16b, v20.16b\n" + + "sshr v20.16b, v3.16b, #7\n" + "shl v3.16b, v3.16b, #1\n" + "and v20.16b, v20.16b, v29.16b\n" + "eor v3.16b, v3.16b, v20.16b\n" + + "ldr q12, %0\n" + "ldr q13, %1\n" + "eor v0.16b, v0.16b, v12.16b\n" + "eor v1.16b, v1.16b, v13.16b\n" + "eor v2.16b, v2.16b, v12.16b\n" + "eor v3.16b, v3.16b, v13.16b\n" + + "eor v2.16b, v2.16b, v14.16b\n" + "eor v3.16b, v3.16b, v15.16b\n" + : + : "m" (v[d][i]), "m" (v[d][i + 16]) + ); + + if (np >= 3) { + asm volatile ( + "ushr v17.16b, v12.16b, #4\n" + "and v16.16b, v12.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "ushr v19.16b, v13.16b, #4\n" + "and v18.16b, v13.16b, v28.16b\n" + "and v19.16b, v19.16b, v28.16b\n" + ); + + asm volatile ( + "ldr q22, %0\n" + "ldr q23, %1\n" + "tbl v20.16b, {v22.16b}, v16.16b\n" + "tbl v21.16b, {v23.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v4.16b, v4.16b, v20.16b\n" + "tbl v20.16b, {v22.16b}, v18.16b\n" + "tbl v21.16b, {v23.16b}, v19.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v5.16b, v5.16b, v20.16b\n" + : + : "m" (raid_gfcauchypshufb[d][1][0][0]), + "m" (raid_gfcauchypshufb[d][1][1][0]) + ); + } + + if (np >= 4) { + asm volatile ( + "ldr q22, %0\n" + "ldr q23, %1\n" + "tbl v20.16b, {v22.16b}, v16.16b\n" + "tbl v21.16b, {v23.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v6.16b, v6.16b, v20.16b\n" + "tbl v20.16b, {v22.16b}, v18.16b\n" + "tbl v21.16b, {v23.16b}, v19.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v7.16b, v7.16b, v20.16b\n" + : + : "m" (raid_gfcauchypshufb[d][2][0][0]), + "m" (raid_gfcauchypshufb[d][2][1][0]) + ); + } + + if (np >= 5) { + asm volatile ( + "ldr q22, %0\n" + "ldr q23, %1\n" + "tbl v20.16b, {v22.16b}, v16.16b\n" + "tbl v21.16b, {v23.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v8.16b, v8.16b, v20.16b\n" + "tbl v20.16b, {v22.16b}, v18.16b\n" + "tbl v21.16b, {v23.16b}, v19.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v9.16b, v9.16b, v20.16b\n" + : + : "m" (raid_gfcauchypshufb[d][3][0][0]), + "m" (raid_gfcauchypshufb[d][3][1][0]) + ); + } + + if (np >= 6) { + asm volatile ( + "ldr q22, %0\n" + "ldr q23, %1\n" + "tbl v20.16b, {v22.16b}, v16.16b\n" + "tbl v21.16b, {v23.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v10.16b, v10.16b, v20.16b\n" + "tbl v20.16b, {v22.16b}, v18.16b\n" + "tbl v21.16b, {v23.16b}, v19.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v11.16b, v11.16b, v20.16b\n" + : + : "m" (raid_gfcauchypshufb[d][4][0][0]), + "m" (raid_gfcauchypshufb[d][4][1][0]) + ); + } + + --d; + + boundary -= 51; + if (boundary < 50) + boundary = -1; + } + + /* + * No G23 boundaries remain. Process D[d] ... D1 using + * only Q *= 2. + */ + for (; d > 0; --d) { + asm volatile ( + "sshr v20.16b, v2.16b, #7\n" + "shl v2.16b, v2.16b, #1\n" + "and v20.16b, v20.16b, v29.16b\n" + "eor v2.16b, v2.16b, v20.16b\n" + + "sshr v20.16b, v3.16b, #7\n" + "shl v3.16b, v3.16b, #1\n" + "and v20.16b, v20.16b, v29.16b\n" + "eor v3.16b, v3.16b, v20.16b\n" + + "ldr q12, %0\n" + "ldr q13, %1\n" + "eor v0.16b, v0.16b, v12.16b\n" + "eor v1.16b, v1.16b, v13.16b\n" + "eor v2.16b, v2.16b, v12.16b\n" + "eor v3.16b, v3.16b, v13.16b\n" + : + : "m" (v[d][i]), "m" (v[d][i + 16]) + ); + + if (np >= 3) { + asm volatile ( + "ushr v17.16b, v12.16b, #4\n" + "and v16.16b, v12.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "ushr v19.16b, v13.16b, #4\n" + "and v18.16b, v13.16b, v28.16b\n" + "and v19.16b, v19.16b, v28.16b\n" + ); + + asm volatile ( + "ldr q22, %0\n" + "ldr q23, %1\n" + "tbl v20.16b, {v22.16b}, v16.16b\n" + "tbl v21.16b, {v23.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v4.16b, v4.16b, v20.16b\n" + "tbl v20.16b, {v22.16b}, v18.16b\n" + "tbl v21.16b, {v23.16b}, v19.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v5.16b, v5.16b, v20.16b\n" + : + : "m" (raid_gfcauchypshufb[d][1][0][0]), + "m" (raid_gfcauchypshufb[d][1][1][0]) + ); + } + + if (np >= 4) { + asm volatile ( + "ldr q22, %0\n" + "ldr q23, %1\n" + "tbl v20.16b, {v22.16b}, v16.16b\n" + "tbl v21.16b, {v23.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v6.16b, v6.16b, v20.16b\n" + "tbl v20.16b, {v22.16b}, v18.16b\n" + "tbl v21.16b, {v23.16b}, v19.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v7.16b, v7.16b, v20.16b\n" + : + : "m" (raid_gfcauchypshufb[d][2][0][0]), + "m" (raid_gfcauchypshufb[d][2][1][0]) + ); + } + + if (np >= 5) { + asm volatile ( + "ldr q22, %0\n" + "ldr q23, %1\n" + "tbl v20.16b, {v22.16b}, v16.16b\n" + "tbl v21.16b, {v23.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v8.16b, v8.16b, v20.16b\n" + "tbl v20.16b, {v22.16b}, v18.16b\n" + "tbl v21.16b, {v23.16b}, v19.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v9.16b, v9.16b, v20.16b\n" + : + : "m" (raid_gfcauchypshufb[d][3][0][0]), + "m" (raid_gfcauchypshufb[d][3][1][0]) + ); + } + + if (np >= 6) { + asm volatile ( + "ldr q22, %0\n" + "ldr q23, %1\n" + "tbl v20.16b, {v22.16b}, v16.16b\n" + "tbl v21.16b, {v23.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v10.16b, v10.16b, v20.16b\n" + "tbl v20.16b, {v22.16b}, v18.16b\n" + "tbl v21.16b, {v23.16b}, v19.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v11.16b, v11.16b, v20.16b\n" + : + : "m" (raid_gfcauchypshufb[d][4][0][0]), + "m" (raid_gfcauchypshufb[d][4][1][0]) + ); + } + } + + /* + * D0 always follows Q *= 2. The lowest G23 boundary is 50. + */ + asm volatile ( + "sshr v20.16b, v2.16b, #7\n" + "shl v2.16b, v2.16b, #1\n" + "and v20.16b, v20.16b, v29.16b\n" + "eor v2.16b, v2.16b, v20.16b\n" + + "sshr v20.16b, v3.16b, #7\n" + "shl v3.16b, v3.16b, #1\n" + "and v20.16b, v20.16b, v29.16b\n" + "eor v3.16b, v3.16b, v20.16b\n" + + "ldr q12, %0\n" + "ldr q13, %1\n" + "eor v0.16b, v0.16b, v12.16b\n" + "eor v1.16b, v1.16b, v13.16b\n" + "eor v2.16b, v2.16b, v12.16b\n" + "eor v3.16b, v3.16b, v13.16b\n" + : + : "m" (v[0][i]), "m" (v[0][i + 16]) + ); + + if (np >= 3) { + asm volatile ( + "eor v4.16b, v4.16b, v12.16b\n" + "eor v5.16b, v5.16b, v13.16b\n" + ); + } + if (np >= 4) { + asm volatile ( + "eor v6.16b, v6.16b, v12.16b\n" + "eor v7.16b, v7.16b, v13.16b\n" + ); + } + if (np >= 5) { + asm volatile ( + "eor v8.16b, v8.16b, v12.16b\n" + "eor v9.16b, v9.16b, v13.16b\n" + ); + } + if (np >= 6) { + asm volatile ( + "eor v10.16b, v10.16b, v12.16b\n" + "eor v11.16b, v11.16b, v13.16b\n" + ); + } + + /* write parity in increasing order */ + asm volatile ( + "str q0, %0\n" + "str q1, %1\n" + "str q2, %2\n" + "str q3, %3\n" + : "=m" (v[nd][i]), "=m" (v[nd][i + 16]), + "=m" (v[nd + 1][i]), "=m" (v[nd + 1][i + 16]) ); if (np >= 3) { @@ -611,12 +1246,15 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n */ void raid_gen3_neon_raid(int nd, size_t size, void **vv) { - raid_genX_neon(nd, size, vv, 3, 0); + raid_genX_neon_x2(nd, size, vv, 3); } void raid_gen3_neon_aes(int nd, size_t size, void **vv) { - raid_genX_neon(nd, size, vv, 3, 1); + if (raid_g23_x2_only(nd)) + raid_genX_neon_x2(nd, size, vv, 3); + else + raid_genX_neon_g23(nd, size, vv, 3); } /* @@ -624,12 +1262,15 @@ void raid_gen3_neon_aes(int nd, size_t size, void **vv) */ void raid_gen4_neon_raid(int nd, size_t size, void **vv) { - raid_genX_neon(nd, size, vv, 4, 0); + raid_genX_neon_x2(nd, size, vv, 4); } void raid_gen4_neon_aes(int nd, size_t size, void **vv) { - raid_genX_neon(nd, size, vv, 4, 1); + if (raid_g23_x2_only(nd)) + raid_genX_neon_x2(nd, size, vv, 4); + else + raid_genX_neon_g23(nd, size, vv, 4); } /* @@ -637,12 +1278,15 @@ void raid_gen4_neon_aes(int nd, size_t size, void **vv) */ void raid_gen5_neon_raid(int nd, size_t size, void **vv) { - raid_genX_neon(nd, size, vv, 5, 0); + raid_genX_neon_x2(nd, size, vv, 5); } void raid_gen5_neon_aes(int nd, size_t size, void **vv) { - raid_genX_neon(nd, size, vv, 5, 1); + if (raid_g23_x2_only(nd)) + raid_genX_neon_x2(nd, size, vv, 5); + else + raid_genX_neon_g23(nd, size, vv, 5); } /* @@ -650,12 +1294,15 @@ void raid_gen5_neon_aes(int nd, size_t size, void **vv) */ void raid_gen6_neon_raid(int nd, size_t size, void **vv) { - raid_genX_neon(nd, size, vv, 6, 0); + raid_genX_neon_x2(nd, size, vv, 6); } void raid_gen6_neon_aes(int nd, size_t size, void **vv) { - raid_genX_neon(nd, size, vv, 6, 1); + if (raid_g23_x2_only(nd)) + raid_genX_neon_x2(nd, size, vv, 6); + else + raid_genX_neon_g23(nd, size, vv, 6); } /* diff --git a/raid/neon32.c b/raid/neon32.c index 11d29a56..fde390c4 100644 --- a/raid/neon32.c +++ b/raid/neon32.c @@ -63,10 +63,9 @@ void raid_gen1_neon32(int nd, size_t size, void **vv) } /* - * GEN2 Cauchy AArch32 NEON implementation using the active Q transition. + * GEN2 Cauchy AArch32 NEON implementation using only Q *= 2. */ -static __always_inline void raid_gen2_neon32_gen(int nd, size_t size, - void **vv, int g23) +static __always_inline void raid_gen2_neon32_x2(int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -86,12 +85,8 @@ static __always_inline void raid_gen2_neon32_gen(int nd, size_t size, : : "Q" (gfconst16.poly[0]) ); - int g23_start = raid_g23_count(l - 1); for (i = 0; i < size; i += 32) { - int g23_count = g23_start; - int g23_x3; - asm volatile ( "vld1.8 {q0}, %0\n" "vld1.8 {q1}, %1\n" @@ -102,28 +97,19 @@ static __always_inline void raid_gen2_neon32_gen(int nd, size_t size, ); for (d = l - 1; d >= 0; --d) { - g23_x3 = 0; - if (g23) { - g23_x3 = --g23_count == 0; - if (g23_x3) - g23_count = 51; - } - - if (g23 && g23_x3) { - asm volatile ( - "vmov q10, q2\n" - "vmov q11, q3\n" - ); - } asm volatile ( + /* double Q0-Q1 */ "vshr.s8 q8, q2, #7\n" "vshl.i8 q2, q2, #1\n" "vand q8, q8, q14\n" "veor q2, q2, q8\n" + "vshr.s8 q8, q3, #7\n" "vshl.i8 q3, q3, #1\n" "vand q8, q8, q14\n" "veor q3, q3, q8\n" + + /* load and XOR */ "vld1.8 {q12}, %0\n" "vld1.8 {q13}, %1\n" "veor q0, q0, q12\n" @@ -133,12 +119,143 @@ static __always_inline void raid_gen2_neon32_gen(int nd, size_t size, : : "Q" (v[d][i]), "Q" (v[d][i + 16]) ); - if (g23 && g23_x3) { + } + + asm volatile ( + "vst1.8 {q0}, %0\n" + "vst1.8 {q1}, %1\n" + "vst1.8 {q2}, %2\n" + "vst1.8 {q3}, %3\n" + : "=Q" (p[i]), "=Q" (p[i + 16]), + "=Q" (q[i]), "=Q" (q[i + 16]) + ); + } + + raid_neon32_end(); +} + +/* + * GEN2 Cauchy AArch32 NEON implementation using the AES G23 Q transition. + */ +static __always_inline void raid_gen2_neon32_g23(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + + int g23_boundary = raid_g23_boundary(l - 1); + + raid_neon32_begin(); + + /* q14 contains the active reduction polynomial. */ + asm volatile ( + "vld1.8 {q14}, %0\n" + : + : "Q" (gfconst16.poly[0]) + ); + + for (i = 0; i < size; i += 32) { + int boundary = g23_boundary; + + asm volatile ( + "vld1.8 {q0}, %0\n" + "vld1.8 {q1}, %1\n" + "vmov q2, q0\n" + "vmov q3, q1\n" + : + : "Q" (v[l][i]), "Q" (v[l][i + 16]) + ); + + d = l - 1; + + for (;;) { + /* + * All transitions above 'boundary' are x2. + */ + for (; d > boundary; --d) { asm volatile ( - "veor q2, q2, q10\n" - "veor q3, q3, q11\n" + /* double Q0-Q1 */ + "vshr.s8 q8, q2, #7\n" + "vshl.i8 q2, q2, #1\n" + "vand q8, q8, q14\n" + "veor q2, q2, q8\n" + + "vshr.s8 q8, q3, #7\n" + "vshl.i8 q3, q3, #1\n" + "vand q8, q8, q14\n" + "veor q3, q3, q8\n" + + /* load and XOR */ + "vld1.8 {q12}, %0\n" + "vld1.8 {q13}, %1\n" + "veor q0, q0, q12\n" + "veor q1, q1, q13\n" + "veor q2, q2, q12\n" + "veor q3, q3, q13\n" + : + : "Q" (v[d][i]), "Q" (v[d][i + 16]) ); } + + /* + * boundary == -1 identifies the final x2-only segment. + * At this point all remaining disks have been consumed. + */ + if (boundary < 0) + break; + + /* + * d == boundary. + * + * Perform the fixed G23 x3 transition and then add + * the current disk to P and Q. + */ + asm volatile ( + /* preserve Q for Q *= 3 */ + "vmov q10, q2\n" + "vmov q11, q3\n" + + /* double Q0-Q1 */ + "vshr.s8 q8, q2, #7\n" + "vshl.i8 q2, q2, #1\n" + "vand q8, q8, q14\n" + "veor q2, q2, q8\n" + + "vshr.s8 q8, q3, #7\n" + "vshl.i8 q3, q3, #1\n" + "vand q8, q8, q14\n" + "veor q3, q3, q8\n" + + /* load and XOR */ + "vld1.8 {q12}, %0\n" + "vld1.8 {q13}, %1\n" + "veor q0, q0, q12\n" + "veor q1, q1, q13\n" + "veor q2, q2, q12\n" + "veor q3, q3, q13\n" + + /* complete Q *= 3 */ + "veor q2, q2, q10\n" + "veor q3, q3, q11\n" + : + : "Q" (v[d][i]), "Q" (v[d][i + 16]) + ); + + --d; + + /* + * G23 boundaries are exactly 51 transitions apart. + * After d = 50 there are no more x3 transitions. + */ + boundary -= 51; + if (boundary < 50) + boundary = -1; } asm volatile ( @@ -156,12 +273,15 @@ static __always_inline void raid_gen2_neon32_gen(int nd, size_t size, void raid_gen2_neon32_raid(int nd, size_t size, void **vv) { - raid_gen2_neon32_gen(nd, size, vv, 0); + raid_gen2_neon32_x2(nd, size, vv); } void raid_gen2_neon32_aes(int nd, size_t size, void **vv) { - raid_gen2_neon32_gen(nd, size, vv, 1); + if (raid_g23_x2_only(nd)) + raid_gen2_neon32_x2(nd, size, vv); + else + raid_gen2_neon32_g23(nd, size, vv); } /* @@ -252,10 +372,10 @@ void raid_genz_neon32_raid(int nd, size_t size, void **vv) } /* - * GENX AArch32 NEON implementation. + * GENX AArch32 NEON implementation using only Q *= 2. */ -static __always_inline void raid_genX_neon32(int nd, size_t size, - void **vv, int np, int g23) +static __always_inline void raid_genX_neon32_x2(int nd, size_t size, + void **vv, int np) { uint8_t **v = (uint8_t **)vv; size_t i; @@ -269,7 +389,6 @@ static __always_inline void raid_genX_neon32(int nd, size_t size, memcpy(v[1 + d], v[0], size); return; } - int g23_start = raid_g23_count(l - 1); raid_neon32_begin(); @@ -281,9 +400,6 @@ static __always_inline void raid_genX_neon32(int nd, size_t size, ); for (i = 0; i < size; i += 16) { - int g23_count = g23_start; - int g23_x3; - /* last disk without the generator multiplication */ asm volatile ( "vld1.8 {q0}, %0\n" @@ -311,7 +427,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size, "veor q2, q11, q12\n" : : "Q" (raid_gfcauchypshufb[l][1][0][0]), - "Q" (raid_gfcauchypshufb[l][1][1][0]) + "Q" (raid_gfcauchypshufb[l][1][1][0]) ); } @@ -328,7 +444,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size, "veor q3, q11, q12\n" : : "Q" (raid_gfcauchypshufb[l][2][0][0]), - "Q" (raid_gfcauchypshufb[l][2][1][0]) + "Q" (raid_gfcauchypshufb[l][2][1][0]) ); } @@ -345,7 +461,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size, "veor q4, q11, q12\n" : : "Q" (raid_gfcauchypshufb[l][3][0][0]), - "Q" (raid_gfcauchypshufb[l][3][1][0]) + "Q" (raid_gfcauchypshufb[l][3][1][0]) ); } @@ -362,40 +478,26 @@ static __always_inline void raid_genX_neon32(int nd, size_t size, "veor q5, q11, q12\n" : : "Q" (raid_gfcauchypshufb[l][4][0][0]), - "Q" (raid_gfcauchypshufb[l][4][1][0]) + "Q" (raid_gfcauchypshufb[l][4][1][0]) ); } /* intermediate disks */ for (d = l - 1; d > 0; --d) { - g23_x3 = 0; - if (g23) { - g23_x3 = --g23_count == 0; - if (g23_x3) - g23_count = 51; - } - - if (g23 && g23_x3) { - asm volatile ( - "vmov q13, q1\n" - ); - } asm volatile ( + /* Q *= 2 */ "vshr.s8 q11, q1, #7\n" "vshl.i8 q1, q1, #1\n" "vand q11, q11, q14\n" "veor q1, q1, q11\n" + + /* load disk and update P/Q */ "vld1.8 {q6}, %0\n" "veor q0, q0, q6\n" "veor q1, q1, q6\n" : : "Q" (v[d][i]) ); - if (g23 && g23_x3) { - asm volatile ( - "veor q1, q1, q13\n" - ); - } if (np >= 3) { asm volatile ( @@ -403,6 +505,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size, "vand q7, q6, q15\n" "vand q8, q8, q15\n" ); + asm volatile ( "vld1.8 {q9}, %0\n" "vld1.8 {q10}, %1\n" @@ -416,7 +519,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size, "veor q2, q2, q11\n" : : "Q" (raid_gfcauchypshufb[d][1][0][0]), - "Q" (raid_gfcauchypshufb[d][1][1][0]) + "Q" (raid_gfcauchypshufb[d][1][1][0]) ); } @@ -432,7 +535,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size, "veor q3, q3, q11\n" : : "Q" (raid_gfcauchypshufb[d][2][0][0]), - "Q" (raid_gfcauchypshufb[d][2][1][0]) + "Q" (raid_gfcauchypshufb[d][2][1][0]) ); } @@ -448,7 +551,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size, "veor q4, q4, q11\n" : : "Q" (raid_gfcauchypshufb[d][3][0][0]), - "Q" (raid_gfcauchypshufb[d][3][1][0]) + "Q" (raid_gfcauchypshufb[d][3][1][0]) ); } @@ -464,94 +567,484 @@ static __always_inline void raid_genX_neon32(int nd, size_t size, "veor q5, q5, q11\n" : : "Q" (raid_gfcauchypshufb[d][4][0][0]), - "Q" (raid_gfcauchypshufb[d][4][1][0]) + "Q" (raid_gfcauchypshufb[d][4][1][0]) ); } } /* first disk with all coefficients at 1 */ - g23_x3 = 0; - if (g23) { - g23_x3 = --g23_count == 0; - if (g23_x3) - g23_count = 51; - } - - if (g23 && g23_x3) { - asm volatile ( - "vmov q13, q1\n" - ); - } asm volatile ( "vshr.s8 q11, q1, #7\n" "vshl.i8 q1, q1, #1\n" "vand q11, q11, q14\n" "veor q1, q1, q11\n" + "vld1.8 {q6}, %0\n" "veor q0, q0, q6\n" "veor q1, q1, q6\n" : : "Q" (v[0][i]) ); - if (g23 && g23_x3) { - asm volatile ( - "veor q1, q1, q13\n" - ); - } - if (np >= 3) { - asm volatile ( - "veor q2, q2, q6\n" - ); - } - if (np >= 4) { - asm volatile ( - "veor q3, q3, q6\n" - ); - } - if (np >= 5) { - asm volatile ( - "veor q4, q4, q6\n" - ); - } - if (np >= 6) { - asm volatile ( - "veor q5, q5, q6\n" - ); - } + if (np >= 3) + asm volatile ("veor q2, q2, q6\n"); + if (np >= 4) + asm volatile ("veor q3, q3, q6\n"); + if (np >= 5) + asm volatile ("veor q4, q4, q6\n"); + if (np >= 6) + asm volatile ("veor q5, q5, q6\n"); /* write parity in increasing order */ asm volatile ( "vst1.8 {q0}, %0\n" "vst1.8 {q1}, %1\n" : "=Q" (v[nd][i]), - "=Q" (v[nd + 1][i]) + "=Q" (v[nd + 1][i]) + ); + + if (np >= 3) + asm volatile ("vst1.8 {q2}, %0\n" : "=Q" (v[nd + 2][i])); + if (np >= 4) + asm volatile ("vst1.8 {q3}, %0\n" : "=Q" (v[nd + 3][i])); + if (np >= 5) + asm volatile ("vst1.8 {q4}, %0\n" : "=Q" (v[nd + 4][i])); + if (np >= 6) + asm volatile ("vst1.8 {q5}, %0\n" : "=Q" (v[nd + 5][i])); + } + + raid_neon32_end(); +} + +/* + * GENX AArch32 NEON implementation using the AES G23 Q transition. + */ +static __always_inline void raid_genX_neon32_g23(int nd, size_t size, + void **vv, int np) +{ + uint8_t **v = (uint8_t **)vv; + size_t i; + int d, l; + + l = nd - 1; + + /* special case with only one data disk */ + if (l == 0) { + for (d = 0; d < np; ++d) + memcpy(v[1 + d], v[0], size); + return; + } + + int g23_boundary = raid_g23_boundary(l - 1); + + raid_neon32_begin(); + + asm volatile ( + "vld1.8 {q14}, %0\n" + "vld1.8 {q15}, %1\n" + : + : "Q" (gfconst16.poly[0]), "Q" (gfconst16.low4[0]) + ); + + for (i = 0; i < size; i += 16) { + int boundary = g23_boundary; + + /* last disk without the generator multiplication */ + asm volatile ( + "vld1.8 {q0}, %0\n" + "vld1.8 {q1}, %0\n" + : + : "Q" (v[l][i]) ); if (np >= 3) { asm volatile ( - "vst1.8 {q2}, %0\n" - : "=Q" (v[nd + 2][i]) + "vshr.u8 q8, q0, #4\n" + "vand q7, q0, q15\n" + "vand q8, q8, q15\n" + ); + + asm volatile ( + "vld1.8 {q9}, %0\n" + "vld1.8 {q10}, %1\n" + "vtbl.8 d22, {d18-d19}, d14\n" + "vtbl.8 d23, {d18-d19}, d15\n" + "vtbl.8 d24, {d20-d21}, d16\n" + "vtbl.8 d25, {d20-d21}, d17\n" + "veor q2, q11, q12\n" + : + : "Q" (raid_gfcauchypshufb[l][1][0][0]), + "Q" (raid_gfcauchypshufb[l][1][1][0]) ); } + if (np >= 4) { asm volatile ( - "vst1.8 {q3}, %0\n" - : "=Q" (v[nd + 3][i]) + "vld1.8 {q9}, %0\n" + "vld1.8 {q10}, %1\n" + "vtbl.8 d22, {d18-d19}, d14\n" + "vtbl.8 d23, {d18-d19}, d15\n" + "vtbl.8 d24, {d20-d21}, d16\n" + "vtbl.8 d25, {d20-d21}, d17\n" + "veor q3, q11, q12\n" + : + : "Q" (raid_gfcauchypshufb[l][2][0][0]), + "Q" (raid_gfcauchypshufb[l][2][1][0]) ); } + if (np >= 5) { asm volatile ( - "vst1.8 {q4}, %0\n" - : "=Q" (v[nd + 4][i]) + "vld1.8 {q9}, %0\n" + "vld1.8 {q10}, %1\n" + "vtbl.8 d22, {d18-d19}, d14\n" + "vtbl.8 d23, {d18-d19}, d15\n" + "vtbl.8 d24, {d20-d21}, d16\n" + "vtbl.8 d25, {d20-d21}, d17\n" + "veor q4, q11, q12\n" + : + : "Q" (raid_gfcauchypshufb[l][3][0][0]), + "Q" (raid_gfcauchypshufb[l][3][1][0]) ); } + if (np >= 6) { asm volatile ( - "vst1.8 {q5}, %0\n" - : "=Q" (v[nd + 5][i]) + "vld1.8 {q9}, %0\n" + "vld1.8 {q10}, %1\n" + "vtbl.8 d22, {d18-d19}, d14\n" + "vtbl.8 d23, {d18-d19}, d15\n" + "vtbl.8 d24, {d20-d21}, d16\n" + "vtbl.8 d25, {d20-d21}, d17\n" + "veor q5, q11, q12\n" + : + : "Q" (raid_gfcauchypshufb[l][4][0][0]), + "Q" (raid_gfcauchypshufb[l][4][1][0]) ); } + + d = l - 1; + + while (boundary >= 0) { + /* + * Everything above the next G23 boundary is Q *= 2. + */ + for (; d > boundary; --d) { + asm volatile ( + "vshr.s8 q11, q1, #7\n" + "vshl.i8 q1, q1, #1\n" + "vand q11, q11, q14\n" + "veor q1, q1, q11\n" + + "vld1.8 {q6}, %0\n" + "veor q0, q0, q6\n" + "veor q1, q1, q6\n" + : + : "Q" (v[d][i]) + ); + + if (np >= 3) { + asm volatile ( + "vshr.u8 q8, q6, #4\n" + "vand q7, q6, q15\n" + "vand q8, q8, q15\n" + ); + + asm volatile ( + "vld1.8 {q9}, %0\n" + "vld1.8 {q10}, %1\n" + "vtbl.8 d22, {d18-d19}, d14\n" + "vtbl.8 d23, {d18-d19}, d15\n" + "vtbl.8 d24, {d20-d21}, d16\n" + "vtbl.8 d25, {d20-d21}, d17\n" + "veor q11, q11, q12\n" + "veor q2, q2, q11\n" + : + : "Q" (raid_gfcauchypshufb[d][1][0][0]), + "Q" (raid_gfcauchypshufb[d][1][1][0]) + ); + } + + if (np >= 4) { + asm volatile ( + "vld1.8 {q9}, %0\n" + "vld1.8 {q10}, %1\n" + "vtbl.8 d22, {d18-d19}, d14\n" + "vtbl.8 d23, {d18-d19}, d15\n" + "vtbl.8 d24, {d20-d21}, d16\n" + "vtbl.8 d25, {d20-d21}, d17\n" + "veor q11, q11, q12\n" + "veor q3, q3, q11\n" + : + : "Q" (raid_gfcauchypshufb[d][2][0][0]), + "Q" (raid_gfcauchypshufb[d][2][1][0]) + ); + } + + if (np >= 5) { + asm volatile ( + "vld1.8 {q9}, %0\n" + "vld1.8 {q10}, %1\n" + "vtbl.8 d22, {d18-d19}, d14\n" + "vtbl.8 d23, {d18-d19}, d15\n" + "vtbl.8 d24, {d20-d21}, d16\n" + "vtbl.8 d25, {d20-d21}, d17\n" + "veor q11, q11, q12\n" + "veor q4, q4, q11\n" + : + : "Q" (raid_gfcauchypshufb[d][3][0][0]), + "Q" (raid_gfcauchypshufb[d][3][1][0]) + ); + } + + if (np >= 6) { + asm volatile ( + "vld1.8 {q9}, %0\n" + "vld1.8 {q10}, %1\n" + "vtbl.8 d22, {d18-d19}, d14\n" + "vtbl.8 d23, {d18-d19}, d15\n" + "vtbl.8 d24, {d20-d21}, d16\n" + "vtbl.8 d25, {d20-d21}, d17\n" + "veor q11, q11, q12\n" + "veor q5, q5, q11\n" + : + : "Q" (raid_gfcauchypshufb[d][4][0][0]), + "Q" (raid_gfcauchypshufb[d][4][1][0]) + ); + } + } + + /* + * d == boundary: Q *= 3. + * + * q13 is otherwise unused here, so preserve the old Q + * there and XOR it back after the normal x2 transition. + */ + asm volatile ( + "vmov q13, q1\n" + + "vshr.s8 q11, q1, #7\n" + "vshl.i8 q1, q1, #1\n" + "vand q11, q11, q14\n" + "veor q1, q1, q11\n" + + "vld1.8 {q6}, %0\n" + "veor q0, q0, q6\n" + "veor q1, q1, q6\n" + + /* complete Q *= 3 */ + "veor q1, q1, q13\n" + : + : "Q" (v[d][i]) + ); + + if (np >= 3) { + asm volatile ( + "vshr.u8 q8, q6, #4\n" + "vand q7, q6, q15\n" + "vand q8, q8, q15\n" + ); + + asm volatile ( + "vld1.8 {q9}, %0\n" + "vld1.8 {q10}, %1\n" + "vtbl.8 d22, {d18-d19}, d14\n" + "vtbl.8 d23, {d18-d19}, d15\n" + "vtbl.8 d24, {d20-d21}, d16\n" + "vtbl.8 d25, {d20-d21}, d17\n" + "veor q11, q11, q12\n" + "veor q2, q2, q11\n" + : + : "Q" (raid_gfcauchypshufb[d][1][0][0]), + "Q" (raid_gfcauchypshufb[d][1][1][0]) + ); + } + + if (np >= 4) { + asm volatile ( + "vld1.8 {q9}, %0\n" + "vld1.8 {q10}, %1\n" + "vtbl.8 d22, {d18-d19}, d14\n" + "vtbl.8 d23, {d18-d19}, d15\n" + "vtbl.8 d24, {d20-d21}, d16\n" + "vtbl.8 d25, {d20-d21}, d17\n" + "veor q11, q11, q12\n" + "veor q3, q3, q11\n" + : + : "Q" (raid_gfcauchypshufb[d][2][0][0]), + "Q" (raid_gfcauchypshufb[d][2][1][0]) + ); + } + + if (np >= 5) { + asm volatile ( + "vld1.8 {q9}, %0\n" + "vld1.8 {q10}, %1\n" + "vtbl.8 d22, {d18-d19}, d14\n" + "vtbl.8 d23, {d18-d19}, d15\n" + "vtbl.8 d24, {d20-d21}, d16\n" + "vtbl.8 d25, {d20-d21}, d17\n" + "veor q11, q11, q12\n" + "veor q4, q4, q11\n" + : + : "Q" (raid_gfcauchypshufb[d][3][0][0]), + "Q" (raid_gfcauchypshufb[d][3][1][0]) + ); + } + + if (np >= 6) { + asm volatile ( + "vld1.8 {q9}, %0\n" + "vld1.8 {q10}, %1\n" + "vtbl.8 d22, {d18-d19}, d14\n" + "vtbl.8 d23, {d18-d19}, d15\n" + "vtbl.8 d24, {d20-d21}, d16\n" + "vtbl.8 d25, {d20-d21}, d17\n" + "veor q11, q11, q12\n" + "veor q5, q5, q11\n" + : + : "Q" (raid_gfcauchypshufb[d][4][0][0]), + "Q" (raid_gfcauchypshufb[d][4][1][0]) + ); + } + + --d; + + boundary -= 51; + if (boundary < 50) + boundary = -1; + } + + /* + * Remaining intermediate disks contain no more G23 + * boundaries, so this loop is pure x2. + */ + for (; d > 0; --d) { + asm volatile ( + "vshr.s8 q11, q1, #7\n" + "vshl.i8 q1, q1, #1\n" + "vand q11, q11, q14\n" + "veor q1, q1, q11\n" + + "vld1.8 {q6}, %0\n" + "veor q0, q0, q6\n" + "veor q1, q1, q6\n" + : + : "Q" (v[d][i]) + ); + + if (np >= 3) { + asm volatile ( + "vshr.u8 q8, q6, #4\n" + "vand q7, q6, q15\n" + "vand q8, q8, q15\n" + ); + + asm volatile ( + "vld1.8 {q9}, %0\n" + "vld1.8 {q10}, %1\n" + "vtbl.8 d22, {d18-d19}, d14\n" + "vtbl.8 d23, {d18-d19}, d15\n" + "vtbl.8 d24, {d20-d21}, d16\n" + "vtbl.8 d25, {d20-d21}, d17\n" + "veor q11, q11, q12\n" + "veor q2, q2, q11\n" + : + : "Q" (raid_gfcauchypshufb[d][1][0][0]), + "Q" (raid_gfcauchypshufb[d][1][1][0]) + ); + } + + if (np >= 4) { + asm volatile ( + "vld1.8 {q9}, %0\n" + "vld1.8 {q10}, %1\n" + "vtbl.8 d22, {d18-d19}, d14\n" + "vtbl.8 d23, {d18-d19}, d15\n" + "vtbl.8 d24, {d20-d21}, d16\n" + "vtbl.8 d25, {d20-d21}, d17\n" + "veor q11, q11, q12\n" + "veor q3, q3, q11\n" + : + : "Q" (raid_gfcauchypshufb[d][2][0][0]), + "Q" (raid_gfcauchypshufb[d][2][1][0]) + ); + } + + if (np >= 5) { + asm volatile ( + "vld1.8 {q9}, %0\n" + "vld1.8 {q10}, %1\n" + "vtbl.8 d22, {d18-d19}, d14\n" + "vtbl.8 d23, {d18-d19}, d15\n" + "vtbl.8 d24, {d20-d21}, d16\n" + "vtbl.8 d25, {d20-d21}, d17\n" + "veor q11, q11, q12\n" + "veor q4, q4, q11\n" + : + : "Q" (raid_gfcauchypshufb[d][3][0][0]), + "Q" (raid_gfcauchypshufb[d][3][1][0]) + ); + } + + if (np >= 6) { + asm volatile ( + "vld1.8 {q9}, %0\n" + "vld1.8 {q10}, %1\n" + "vtbl.8 d22, {d18-d19}, d14\n" + "vtbl.8 d23, {d18-d19}, d15\n" + "vtbl.8 d24, {d20-d21}, d16\n" + "vtbl.8 d25, {d20-d21}, d17\n" + "veor q11, q11, q12\n" + "veor q5, q5, q11\n" + : + : "Q" (raid_gfcauchypshufb[d][4][0][0]), + "Q" (raid_gfcauchypshufb[d][4][1][0]) + ); + } + } + + /* + * D0 always follows Q *= 2. The lowest G23 boundary is 50. + */ + asm volatile ( + "vshr.s8 q11, q1, #7\n" + "vshl.i8 q1, q1, #1\n" + "vand q11, q11, q14\n" + "veor q1, q1, q11\n" + + "vld1.8 {q6}, %0\n" + "veor q0, q0, q6\n" + "veor q1, q1, q6\n" + : + : "Q" (v[0][i]) + ); + + if (np >= 3) + asm volatile ("veor q2, q2, q6\n"); + if (np >= 4) + asm volatile ("veor q3, q3, q6\n"); + if (np >= 5) + asm volatile ("veor q4, q4, q6\n"); + if (np >= 6) + asm volatile ("veor q5, q5, q6\n"); + + /* write parity in increasing order */ + asm volatile ( + "vst1.8 {q0}, %0\n" + "vst1.8 {q1}, %1\n" + : "=Q" (v[nd][i]), + "=Q" (v[nd + 1][i]) + ); + + if (np >= 3) + asm volatile ("vst1.8 {q2}, %0\n" : "=Q" (v[nd + 2][i])); + if (np >= 4) + asm volatile ("vst1.8 {q3}, %0\n" : "=Q" (v[nd + 3][i])); + if (np >= 5) + asm volatile ("vst1.8 {q4}, %0\n" : "=Q" (v[nd + 4][i])); + if (np >= 6) + asm volatile ("vst1.8 {q5}, %0\n" : "=Q" (v[nd + 5][i])); } raid_neon32_end(); @@ -562,12 +1055,15 @@ static __always_inline void raid_genX_neon32(int nd, size_t size, */ void raid_gen3_neon32_raid(int nd, size_t size, void **vv) { - raid_genX_neon32(nd, size, vv, 3, 0); + raid_genX_neon32_x2(nd, size, vv, 3); } void raid_gen3_neon32_aes(int nd, size_t size, void **vv) { - raid_genX_neon32(nd, size, vv, 3, 1); + if (raid_g23_x2_only(nd)) + raid_genX_neon32_x2(nd, size, vv, 3); + else + raid_genX_neon32_g23(nd, size, vv, 3); } /* @@ -575,12 +1071,15 @@ void raid_gen3_neon32_aes(int nd, size_t size, void **vv) */ void raid_gen4_neon32_raid(int nd, size_t size, void **vv) { - raid_genX_neon32(nd, size, vv, 4, 0); + raid_genX_neon32_x2(nd, size, vv, 4); } void raid_gen4_neon32_aes(int nd, size_t size, void **vv) { - raid_genX_neon32(nd, size, vv, 4, 1); + if (raid_g23_x2_only(nd)) + raid_genX_neon32_x2(nd, size, vv, 4); + else + raid_genX_neon32_g23(nd, size, vv, 4); } /* @@ -588,12 +1087,15 @@ void raid_gen4_neon32_aes(int nd, size_t size, void **vv) */ void raid_gen5_neon32_raid(int nd, size_t size, void **vv) { - raid_genX_neon32(nd, size, vv, 5, 0); + raid_genX_neon32_x2(nd, size, vv, 5); } void raid_gen5_neon32_aes(int nd, size_t size, void **vv) { - raid_genX_neon32(nd, size, vv, 5, 1); + if (raid_g23_x2_only(nd)) + raid_genX_neon32_x2(nd, size, vv, 5); + else + raid_genX_neon32_g23(nd, size, vv, 5); } /* @@ -601,12 +1103,15 @@ void raid_gen5_neon32_aes(int nd, size_t size, void **vv) */ void raid_gen6_neon32_raid(int nd, size_t size, void **vv) { - raid_genX_neon32(nd, size, vv, 6, 0); + raid_genX_neon32_x2(nd, size, vv, 6); } void raid_gen6_neon32_aes(int nd, size_t size, void **vv) { - raid_genX_neon32(nd, size, vv, 6, 1); + if (raid_g23_x2_only(nd)) + raid_genX_neon32_x2(nd, size, vv, 6); + else + raid_genX_neon32_g23(nd, size, vv, 6); } /* diff --git a/raid/sse2.c b/raid/sse2.c index 030499a2..b9583778 100644 --- a/raid/sse2.c +++ b/raid/sse2.c @@ -137,9 +137,9 @@ void raid_gen1_sse2(int nd, size_t size, void **vv) } /* - * GEN2 Cauchy SSE2 implementation using the active Q transition + * GEN2 Cauchy SSE2 implementation using only Q *= 2. */ -static __always_inline void raid_gen2_sse2_gen(int nd, size_t size, void **vv, int g23) +static __always_inline void raid_gen2_sse2_x2(int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -154,53 +154,25 @@ static __always_inline void raid_gen2_sse2_gen(int nd, size_t size, void **vv, i raid_sse_begin(); asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); - int g23_start = raid_g23_count(l - 1); for (i = 0; i < size; i += 32) { - int g23_count = g23_start; - int g23_x3; - asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i])); asm volatile ("movdqa %0,%%xmm1" : : "m" (v[l][i + 16])); asm volatile ("movdqa %xmm0,%xmm2"); asm volatile ("movdqa %xmm1,%xmm3"); for (d = l - 1; d >= 0; --d) { - g23_x3 = 0; - if (g23) { - g23_x3 = --g23_count == 0; - if (g23_x3) - g23_count = 51; - } - /* scale Q before adding the current disk */ - if (g23 && g23_x3) { - asm volatile ("movdqa %xmm2,%xmm4"); - asm volatile ("movdqa %xmm3,%xmm5"); - asm volatile ("pxor %xmm6,%xmm6"); - asm volatile ("pcmpgtb %xmm4,%xmm6"); - asm volatile ("paddb %xmm2,%xmm2"); - asm volatile ("paddb %xmm3,%xmm3"); - asm volatile ("pxor %xmm4,%xmm2"); - asm volatile ("pxor %xmm4,%xmm4"); - asm volatile ("pcmpgtb %xmm5,%xmm4"); - asm volatile ("pand %xmm7,%xmm6"); - asm volatile ("pand %xmm7,%xmm4"); - asm volatile ("pxor %xmm6,%xmm2"); - asm volatile ("pxor %xmm4,%xmm3"); - asm volatile ("pxor %xmm5,%xmm3"); - } else { - asm volatile ("pxor %xmm4,%xmm4"); - asm volatile ("pxor %xmm5,%xmm5"); - asm volatile ("pcmpgtb %xmm2,%xmm4"); - asm volatile ("pcmpgtb %xmm3,%xmm5"); - asm volatile ("paddb %xmm2,%xmm2"); - asm volatile ("paddb %xmm3,%xmm3"); - asm volatile ("pand %xmm7,%xmm4"); - asm volatile ("pand %xmm7,%xmm5"); - asm volatile ("pxor %xmm4,%xmm2"); - asm volatile ("pxor %xmm5,%xmm3"); - } + asm volatile ("pxor %xmm4,%xmm4"); + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pcmpgtb %xmm2,%xmm4"); + asm volatile ("pcmpgtb %xmm3,%xmm5"); + asm volatile ("paddb %xmm2,%xmm2"); + asm volatile ("paddb %xmm3,%xmm3"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + asm volatile ("pxor %xmm4,%xmm2"); + asm volatile ("pxor %xmm5,%xmm3"); asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); asm volatile ("movdqa %0,%%xmm5" : : "m" (v[d][i + 16])); @@ -219,23 +191,137 @@ static __always_inline void raid_gen2_sse2_gen(int nd, size_t size, void **vv, i raid_sse_end(); } +/* + * GEN2 Cauchy SSE2 implementation using the AES G23 Q transition. + */ +static __always_inline void raid_gen2_sse2_g23(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + + int g23_boundary = raid_g23_boundary(l - 1); + + raid_sse_begin(); + + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + + for (i = 0; i < size; i += 32) { + int boundary = g23_boundary; + + asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i])); + asm volatile ("movdqa %0,%%xmm1" : : "m" (v[l][i + 16])); + asm volatile ("movdqa %xmm0,%xmm2"); + asm volatile ("movdqa %xmm1,%xmm3"); + + d = l - 1; + + for (;;) { + /* + * All transitions above 'boundary' are x2. + */ + for (; d > boundary; --d) { + /* scale Q before adding the current disk */ + asm volatile ("pxor %xmm4,%xmm4"); + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pcmpgtb %xmm2,%xmm4"); + asm volatile ("pcmpgtb %xmm3,%xmm5"); + asm volatile ("paddb %xmm2,%xmm2"); + asm volatile ("paddb %xmm3,%xmm3"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + asm volatile ("pxor %xmm4,%xmm2"); + asm volatile ("pxor %xmm5,%xmm3"); + + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); + asm volatile ("movdqa %0,%%xmm5" : : "m" (v[d][i + 16])); + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm5,%xmm1"); + asm volatile ("pxor %xmm4,%xmm2"); + asm volatile ("pxor %xmm5,%xmm3"); + } + + /* + * boundary == -1 identifies the final x2-only segment. + * At this point all remaining disks have been consumed. + */ + if (boundary < 0) + break; + + /* + * d == boundary. + * + * Perform the fixed G23 x3 transition and then add + * the current disk to P and Q. + */ + asm volatile ("movdqa %xmm2,%xmm4"); + asm volatile ("movdqa %xmm3,%xmm5"); + asm volatile ("pxor %xmm6,%xmm6"); + asm volatile ("pcmpgtb %xmm4,%xmm6"); + asm volatile ("paddb %xmm2,%xmm2"); + asm volatile ("paddb %xmm3,%xmm3"); + asm volatile ("pxor %xmm4,%xmm2"); + asm volatile ("pxor %xmm4,%xmm4"); + asm volatile ("pcmpgtb %xmm5,%xmm4"); + asm volatile ("pand %xmm7,%xmm6"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pxor %xmm6,%xmm2"); + asm volatile ("pxor %xmm4,%xmm3"); + asm volatile ("pxor %xmm5,%xmm3"); + + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); + asm volatile ("movdqa %0,%%xmm5" : : "m" (v[d][i + 16])); + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm5,%xmm1"); + asm volatile ("pxor %xmm4,%xmm2"); + asm volatile ("pxor %xmm5,%xmm3"); + + --d; + + /* + * G23 boundaries are exactly 51 transitions apart. + * After d = 50 there are no more x3 transitions. + */ + boundary -= 51; + if (boundary < 50) + boundary = -1; + } + + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (p[i + 16])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (q[i])); + asm volatile ("movntdq %%xmm3,%0" : "=m" (q[i + 16])); + } + + raid_sse_end(); +} + void raid_gen2_sse2_raid(int nd, size_t size, void **vv) { - raid_gen2_sse2_gen(nd, size, vv, 0); + raid_gen2_sse2_x2(nd, size, vv); } void raid_gen2_sse2_aes(int nd, size_t size, void **vv) { - raid_gen2_sse2_gen(nd, size, vv, 1); + if (raid_g23_x2_only(nd)) + raid_gen2_sse2_x2(nd, size, vv); + else + raid_gen2_sse2_g23(nd, size, vv); } #ifdef CONFIG_X86_64 /* - * GEN2 Cauchy SSE2 implementation using the active Q transition + * GEN2 Cauchy SSE2 implementation using only Q *= 2. * * Note that it uses 16 registers, meaning that x64 is required. */ -static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv, int g23) +static __always_inline void raid_gen2_sse2ext_x2(int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -250,13 +336,8 @@ static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv raid_sse_begin(); asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.poly[0])); - int g23_start = raid_g23_count(l - 1); for (i = 0; i < size; i += 64) { - int g23_count = g23_start; - int g23_x3_d; - int g23_x3_d1; - asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i])); asm volatile ("movdqa %0,%%xmm1" : : "m" (v[l][i + 16])); asm volatile ("movdqa %0,%%xmm2" : : "m" (v[l][i + 32])); @@ -268,41 +349,22 @@ static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv /* process two disks per iteration */ for (d = l - 1; d >= 1; d -= 2) { - g23_x3_d = 0; - g23_x3_d1 = 0; - if (g23) { - g23_x3_d = --g23_count == 0; - if (g23_x3_d) - g23_count = 51; - g23_x3_d1 = --g23_count == 0; - if (g23_x3_d1) - g23_count = 51; - } - /* lane 0 */ asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i])); asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i])); - if (g23 && g23_x3_d) - asm volatile ("movdqa %xmm4,%xmm10"); asm volatile ("pxor %xmm11,%xmm11"); asm volatile ("pcmpgtb %xmm4,%xmm11"); asm volatile ("paddb %xmm4,%xmm4"); asm volatile ("pand %xmm15,%xmm11"); asm volatile ("pxor %xmm11,%xmm4"); - if (g23 && g23_x3_d) - asm volatile ("pxor %xmm10,%xmm4"); asm volatile ("pxor %xmm8,%xmm4"); - if (g23 && g23_x3_d1) - asm volatile ("movdqa %xmm4,%xmm10"); asm volatile ("pxor %xmm11,%xmm11"); asm volatile ("pcmpgtb %xmm4,%xmm11"); asm volatile ("paddb %xmm4,%xmm4"); asm volatile ("pand %xmm15,%xmm11"); asm volatile ("pxor %xmm11,%xmm4"); - if (g23 && g23_x3_d1) - asm volatile ("pxor %xmm10,%xmm4"); asm volatile ("pxor %xmm9,%xmm4"); asm volatile ("pxor %xmm8,%xmm0"); @@ -312,26 +374,18 @@ static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i + 16])); asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i + 16])); - if (g23 && g23_x3_d) - asm volatile ("movdqa %xmm5,%xmm10"); asm volatile ("pxor %xmm11,%xmm11"); asm volatile ("pcmpgtb %xmm5,%xmm11"); asm volatile ("paddb %xmm5,%xmm5"); asm volatile ("pand %xmm15,%xmm11"); asm volatile ("pxor %xmm11,%xmm5"); - if (g23 && g23_x3_d) - asm volatile ("pxor %xmm10,%xmm5"); asm volatile ("pxor %xmm8,%xmm5"); - if (g23 && g23_x3_d1) - asm volatile ("movdqa %xmm5,%xmm10"); asm volatile ("pxor %xmm11,%xmm11"); asm volatile ("pcmpgtb %xmm5,%xmm11"); asm volatile ("paddb %xmm5,%xmm5"); asm volatile ("pand %xmm15,%xmm11"); asm volatile ("pxor %xmm11,%xmm5"); - if (g23 && g23_x3_d1) - asm volatile ("pxor %xmm10,%xmm5"); asm volatile ("pxor %xmm9,%xmm5"); asm volatile ("pxor %xmm8,%xmm1"); @@ -341,26 +395,18 @@ static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i + 32])); asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i + 32])); - if (g23 && g23_x3_d) - asm volatile ("movdqa %xmm6,%xmm10"); asm volatile ("pxor %xmm11,%xmm11"); asm volatile ("pcmpgtb %xmm6,%xmm11"); asm volatile ("paddb %xmm6,%xmm6"); asm volatile ("pand %xmm15,%xmm11"); asm volatile ("pxor %xmm11,%xmm6"); - if (g23 && g23_x3_d) - asm volatile ("pxor %xmm10,%xmm6"); asm volatile ("pxor %xmm8,%xmm6"); - if (g23 && g23_x3_d1) - asm volatile ("movdqa %xmm6,%xmm10"); asm volatile ("pxor %xmm11,%xmm11"); asm volatile ("pcmpgtb %xmm6,%xmm11"); asm volatile ("paddb %xmm6,%xmm6"); asm volatile ("pand %xmm15,%xmm11"); asm volatile ("pxor %xmm11,%xmm6"); - if (g23 && g23_x3_d1) - asm volatile ("pxor %xmm10,%xmm6"); asm volatile ("pxor %xmm9,%xmm6"); asm volatile ("pxor %xmm8,%xmm2"); @@ -370,26 +416,18 @@ static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i + 48])); asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i + 48])); - if (g23 && g23_x3_d) - asm volatile ("movdqa %xmm7,%xmm10"); asm volatile ("pxor %xmm11,%xmm11"); asm volatile ("pcmpgtb %xmm7,%xmm11"); asm volatile ("paddb %xmm7,%xmm7"); asm volatile ("pand %xmm15,%xmm11"); asm volatile ("pxor %xmm11,%xmm7"); - if (g23 && g23_x3_d) - asm volatile ("pxor %xmm10,%xmm7"); asm volatile ("pxor %xmm8,%xmm7"); - if (g23 && g23_x3_d1) - asm volatile ("movdqa %xmm7,%xmm10"); asm volatile ("pxor %xmm11,%xmm11"); asm volatile ("pcmpgtb %xmm7,%xmm11"); asm volatile ("paddb %xmm7,%xmm7"); asm volatile ("pand %xmm15,%xmm11"); asm volatile ("pxor %xmm11,%xmm7"); - if (g23 && g23_x3_d1) - asm volatile ("pxor %xmm10,%xmm7"); asm volatile ("pxor %xmm9,%xmm7"); asm volatile ("pxor %xmm8,%xmm3"); @@ -398,66 +436,26 @@ static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv /* single remaining disk */ if (d == 0) { - int g23_x3; - - g23_x3 = 0; - if (g23) { - g23_x3 = --g23_count == 0; - if (g23_x3) - g23_count = 51; - } - - if (g23 && g23_x3) { - asm volatile ("movdqa %xmm4,%xmm8"); - asm volatile ("movdqa %xmm5,%xmm9"); - asm volatile ("movdqa %xmm6,%xmm10"); - asm volatile ("movdqa %xmm7,%xmm11"); - asm volatile ("pxor %xmm12,%xmm12"); - asm volatile ("pxor %xmm13,%xmm13"); - asm volatile ("pxor %xmm14,%xmm14"); - asm volatile ("pcmpgtb %xmm8,%xmm12"); - asm volatile ("pcmpgtb %xmm9,%xmm13"); - asm volatile ("pcmpgtb %xmm10,%xmm14"); - asm volatile ("paddb %xmm4,%xmm4"); - asm volatile ("paddb %xmm5,%xmm5"); - asm volatile ("paddb %xmm6,%xmm6"); - asm volatile ("paddb %xmm7,%xmm7"); - asm volatile ("pxor %xmm8,%xmm4"); - asm volatile ("pxor %xmm8,%xmm8"); - asm volatile ("pcmpgtb %xmm11,%xmm8"); - asm volatile ("pand %xmm15,%xmm12"); - asm volatile ("pand %xmm15,%xmm13"); - asm volatile ("pand %xmm15,%xmm14"); - asm volatile ("pand %xmm15,%xmm8"); - asm volatile ("pxor %xmm12,%xmm4"); - asm volatile ("pxor %xmm13,%xmm5"); - asm volatile ("pxor %xmm14,%xmm6"); - asm volatile ("pxor %xmm8,%xmm7"); - asm volatile ("pxor %xmm9,%xmm5"); - asm volatile ("pxor %xmm10,%xmm6"); - asm volatile ("pxor %xmm11,%xmm7"); - } else { - asm volatile ("pxor %xmm8,%xmm8"); - asm volatile ("pxor %xmm9,%xmm9"); - asm volatile ("pxor %xmm10,%xmm10"); - asm volatile ("pxor %xmm11,%xmm11"); - asm volatile ("pcmpgtb %xmm4,%xmm8"); - asm volatile ("pcmpgtb %xmm5,%xmm9"); - asm volatile ("pcmpgtb %xmm6,%xmm10"); - asm volatile ("pcmpgtb %xmm7,%xmm11"); - asm volatile ("paddb %xmm4,%xmm4"); - asm volatile ("paddb %xmm5,%xmm5"); - asm volatile ("paddb %xmm6,%xmm6"); - asm volatile ("paddb %xmm7,%xmm7"); - asm volatile ("pand %xmm15,%xmm8"); - asm volatile ("pand %xmm15,%xmm9"); - asm volatile ("pand %xmm15,%xmm10"); - asm volatile ("pand %xmm15,%xmm11"); - asm volatile ("pxor %xmm8,%xmm4"); - asm volatile ("pxor %xmm9,%xmm5"); - asm volatile ("pxor %xmm10,%xmm6"); - asm volatile ("pxor %xmm11,%xmm7"); - } + asm volatile ("pxor %xmm8,%xmm8"); + asm volatile ("pxor %xmm9,%xmm9"); + asm volatile ("pxor %xmm10,%xmm10"); + asm volatile ("pxor %xmm11,%xmm11"); + asm volatile ("pcmpgtb %xmm4,%xmm8"); + asm volatile ("pcmpgtb %xmm5,%xmm9"); + asm volatile ("pcmpgtb %xmm6,%xmm10"); + asm volatile ("pcmpgtb %xmm7,%xmm11"); + asm volatile ("paddb %xmm4,%xmm4"); + asm volatile ("paddb %xmm5,%xmm5"); + asm volatile ("paddb %xmm6,%xmm6"); + asm volatile ("paddb %xmm7,%xmm7"); + asm volatile ("pand %xmm15,%xmm8"); + asm volatile ("pand %xmm15,%xmm9"); + asm volatile ("pand %xmm15,%xmm10"); + asm volatile ("pand %xmm15,%xmm11"); + asm volatile ("pxor %xmm8,%xmm4"); + asm volatile ("pxor %xmm9,%xmm5"); + asm volatile ("pxor %xmm10,%xmm6"); + asm volatile ("pxor %xmm11,%xmm7"); asm volatile ("movdqa %0,%%xmm8" : : "m" (v[0][i])); asm volatile ("movdqa %0,%%xmm9" : : "m" (v[0][i + 16])); @@ -486,14 +484,266 @@ static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv raid_sse_end(); } +/* + * GEN2 Cauchy SSE2 implementation using the AES G23 Q transition. + * + * Note that it uses 16 registers, meaning that x64 is required. + */ +static __always_inline void raid_gen2_sse2ext_g23(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + + int g23_boundary = raid_g23_boundary(l - 1); + + raid_sse_begin(); + + asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.poly[0])); + + for (i = 0; i < size; i += 64) { + int boundary = g23_boundary; + + asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i])); + asm volatile ("movdqa %0,%%xmm1" : : "m" (v[l][i + 16])); + asm volatile ("movdqa %0,%%xmm2" : : "m" (v[l][i + 32])); + asm volatile ("movdqa %0,%%xmm3" : : "m" (v[l][i + 48])); + asm volatile ("movdqa %xmm0,%xmm4"); + asm volatile ("movdqa %xmm1,%xmm5"); + asm volatile ("movdqa %xmm2,%xmm6"); + asm volatile ("movdqa %xmm3,%xmm7"); + + d = l - 1; + + for (;;) { + /* + * All transitions above 'boundary' are x2. + * Do not let a two-disk iteration cross the boundary. + */ + for (; d >= boundary + 2; d -= 2) { + /* lane 0 */ + asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i])); + asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i])); + + asm volatile ("pxor %xmm11,%xmm11"); + asm volatile ("pcmpgtb %xmm4,%xmm11"); + asm volatile ("paddb %xmm4,%xmm4"); + asm volatile ("pand %xmm15,%xmm11"); + asm volatile ("pxor %xmm11,%xmm4"); + asm volatile ("pxor %xmm8,%xmm4"); + + asm volatile ("pxor %xmm11,%xmm11"); + asm volatile ("pcmpgtb %xmm4,%xmm11"); + asm volatile ("paddb %xmm4,%xmm4"); + asm volatile ("pand %xmm15,%xmm11"); + asm volatile ("pxor %xmm11,%xmm4"); + asm volatile ("pxor %xmm9,%xmm4"); + + asm volatile ("pxor %xmm8,%xmm0"); + asm volatile ("pxor %xmm9,%xmm0"); + + /* lane 1 */ + asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i + 16])); + asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i + 16])); + + asm volatile ("pxor %xmm11,%xmm11"); + asm volatile ("pcmpgtb %xmm5,%xmm11"); + asm volatile ("paddb %xmm5,%xmm5"); + asm volatile ("pand %xmm15,%xmm11"); + asm volatile ("pxor %xmm11,%xmm5"); + asm volatile ("pxor %xmm8,%xmm5"); + + asm volatile ("pxor %xmm11,%xmm11"); + asm volatile ("pcmpgtb %xmm5,%xmm11"); + asm volatile ("paddb %xmm5,%xmm5"); + asm volatile ("pand %xmm15,%xmm11"); + asm volatile ("pxor %xmm11,%xmm5"); + asm volatile ("pxor %xmm9,%xmm5"); + + asm volatile ("pxor %xmm8,%xmm1"); + asm volatile ("pxor %xmm9,%xmm1"); + + /* lane 2 */ + asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i + 32])); + asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i + 32])); + + asm volatile ("pxor %xmm11,%xmm11"); + asm volatile ("pcmpgtb %xmm6,%xmm11"); + asm volatile ("paddb %xmm6,%xmm6"); + asm volatile ("pand %xmm15,%xmm11"); + asm volatile ("pxor %xmm11,%xmm6"); + asm volatile ("pxor %xmm8,%xmm6"); + + asm volatile ("pxor %xmm11,%xmm11"); + asm volatile ("pcmpgtb %xmm6,%xmm11"); + asm volatile ("paddb %xmm6,%xmm6"); + asm volatile ("pand %xmm15,%xmm11"); + asm volatile ("pxor %xmm11,%xmm6"); + asm volatile ("pxor %xmm9,%xmm6"); + + asm volatile ("pxor %xmm8,%xmm2"); + asm volatile ("pxor %xmm9,%xmm2"); + + /* lane 3 */ + asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i + 48])); + asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i + 48])); + + asm volatile ("pxor %xmm11,%xmm11"); + asm volatile ("pcmpgtb %xmm7,%xmm11"); + asm volatile ("paddb %xmm7,%xmm7"); + asm volatile ("pand %xmm15,%xmm11"); + asm volatile ("pxor %xmm11,%xmm7"); + asm volatile ("pxor %xmm8,%xmm7"); + + asm volatile ("pxor %xmm11,%xmm11"); + asm volatile ("pcmpgtb %xmm7,%xmm11"); + asm volatile ("paddb %xmm7,%xmm7"); + asm volatile ("pand %xmm15,%xmm11"); + asm volatile ("pxor %xmm11,%xmm7"); + asm volatile ("pxor %xmm9,%xmm7"); + + asm volatile ("pxor %xmm8,%xmm3"); + asm volatile ("pxor %xmm9,%xmm3"); + } + + /* + * The first x2 segment can have odd length. + * Consume its last x2 transition separately. + */ + if (d > boundary) { + asm volatile ("pxor %xmm8,%xmm8"); + asm volatile ("pxor %xmm9,%xmm9"); + asm volatile ("pxor %xmm10,%xmm10"); + asm volatile ("pxor %xmm11,%xmm11"); + asm volatile ("pcmpgtb %xmm4,%xmm8"); + asm volatile ("pcmpgtb %xmm5,%xmm9"); + asm volatile ("pcmpgtb %xmm6,%xmm10"); + asm volatile ("pcmpgtb %xmm7,%xmm11"); + asm volatile ("paddb %xmm4,%xmm4"); + asm volatile ("paddb %xmm5,%xmm5"); + asm volatile ("paddb %xmm6,%xmm6"); + asm volatile ("paddb %xmm7,%xmm7"); + asm volatile ("pand %xmm15,%xmm8"); + asm volatile ("pand %xmm15,%xmm9"); + asm volatile ("pand %xmm15,%xmm10"); + asm volatile ("pand %xmm15,%xmm11"); + asm volatile ("pxor %xmm8,%xmm4"); + asm volatile ("pxor %xmm9,%xmm5"); + asm volatile ("pxor %xmm10,%xmm6"); + asm volatile ("pxor %xmm11,%xmm7"); + + asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i])); + asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d][i + 16])); + asm volatile ("movdqa %0,%%xmm10" : : "m" (v[d][i + 32])); + asm volatile ("movdqa %0,%%xmm11" : : "m" (v[d][i + 48])); + asm volatile ("pxor %xmm8,%xmm0"); + asm volatile ("pxor %xmm9,%xmm1"); + asm volatile ("pxor %xmm10,%xmm2"); + asm volatile ("pxor %xmm11,%xmm3"); + asm volatile ("pxor %xmm8,%xmm4"); + asm volatile ("pxor %xmm9,%xmm5"); + asm volatile ("pxor %xmm10,%xmm6"); + asm volatile ("pxor %xmm11,%xmm7"); + + --d; + } + + /* + * boundary == -1 identifies the final x2-only segment. + * At this point all remaining disks have been consumed. + */ + if (boundary < 0) + break; + + /* + * d == boundary. + * + * Perform the fixed G23 x3 transition and then add + * the current disk to P and Q. + */ + asm volatile ("movdqa %xmm4,%xmm8"); + asm volatile ("movdqa %xmm5,%xmm9"); + asm volatile ("movdqa %xmm6,%xmm10"); + asm volatile ("movdqa %xmm7,%xmm11"); + asm volatile ("pxor %xmm12,%xmm12"); + asm volatile ("pxor %xmm13,%xmm13"); + asm volatile ("pxor %xmm14,%xmm14"); + asm volatile ("pcmpgtb %xmm8,%xmm12"); + asm volatile ("pcmpgtb %xmm9,%xmm13"); + asm volatile ("pcmpgtb %xmm10,%xmm14"); + asm volatile ("paddb %xmm4,%xmm4"); + asm volatile ("paddb %xmm5,%xmm5"); + asm volatile ("paddb %xmm6,%xmm6"); + asm volatile ("paddb %xmm7,%xmm7"); + asm volatile ("pxor %xmm8,%xmm4"); + asm volatile ("pxor %xmm8,%xmm8"); + asm volatile ("pcmpgtb %xmm11,%xmm8"); + asm volatile ("pand %xmm15,%xmm12"); + asm volatile ("pand %xmm15,%xmm13"); + asm volatile ("pand %xmm15,%xmm14"); + asm volatile ("pand %xmm15,%xmm8"); + asm volatile ("pxor %xmm12,%xmm4"); + asm volatile ("pxor %xmm13,%xmm5"); + asm volatile ("pxor %xmm14,%xmm6"); + asm volatile ("pxor %xmm8,%xmm7"); + asm volatile ("pxor %xmm9,%xmm5"); + asm volatile ("pxor %xmm10,%xmm6"); + asm volatile ("pxor %xmm11,%xmm7"); + + asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i])); + asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d][i + 16])); + asm volatile ("movdqa %0,%%xmm10" : : "m" (v[d][i + 32])); + asm volatile ("movdqa %0,%%xmm11" : : "m" (v[d][i + 48])); + asm volatile ("pxor %xmm8,%xmm0"); + asm volatile ("pxor %xmm9,%xmm1"); + asm volatile ("pxor %xmm10,%xmm2"); + asm volatile ("pxor %xmm11,%xmm3"); + asm volatile ("pxor %xmm8,%xmm4"); + asm volatile ("pxor %xmm9,%xmm5"); + asm volatile ("pxor %xmm10,%xmm6"); + asm volatile ("pxor %xmm11,%xmm7"); + + --d; + + /* + * G23 boundaries are exactly 51 transitions apart. + * After d = 50 there are no more x3 transitions. + */ + boundary -= 51; + if (boundary < 50) + boundary = -1; + } + + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (p[i + 16])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (p[i + 32])); + asm volatile ("movntdq %%xmm3,%0" : "=m" (p[i + 48])); + asm volatile ("movntdq %%xmm4,%0" : "=m" (q[i])); + asm volatile ("movntdq %%xmm5,%0" : "=m" (q[i + 16])); + asm volatile ("movntdq %%xmm6,%0" : "=m" (q[i + 32])); + asm volatile ("movntdq %%xmm7,%0" : "=m" (q[i + 48])); + } + + raid_sse_end(); +} + void raid_gen2_sse2ext_raid(int nd, size_t size, void **vv) { - raid_gen2_sse2ext_gen(nd, size, vv, 0); + raid_gen2_sse2ext_x2(nd, size, vv); } void raid_gen2_sse2ext_aes(int nd, size_t size, void **vv) { - raid_gen2_sse2ext_gen(nd, size, vv, 1); + if (raid_g23_x2_only(nd)) + raid_gen2_sse2ext_x2(nd, size, vv); + else + raid_gen2_sse2ext_g23(nd, size, vv); } #endif diff --git a/raid/ssse3.c b/raid/ssse3.c index 909d3f17..0af1ee9b 100644 --- a/raid/ssse3.c +++ b/raid/ssse3.c @@ -8,8 +8,9 @@ #ifdef CONFIG_X86 /* * GEN3 (triple parity with Cauchy matrix) SSSE3 implementation + * using only Q *= 2. */ -static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv, int g23) +static __always_inline void raid_gen3_ssse3_x2(int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -29,7 +30,6 @@ static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv, memcpy(v[1 + i], v[0], size); return; } - int g23_start = raid_g23_count(l - 1); raid_sse_begin(); @@ -38,9 +38,6 @@ static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv, asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 16) { - int g23_count = g23_start; - int g23_x3; - /* last disk without the generator multiplication */ asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i])); @@ -60,24 +57,14 @@ static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv, /* intermediate disks */ for (d = l - 1; d > 0; --d) { - g23_x3 = 0; - if (g23) { - g23_x3 = --g23_count == 0; - if (g23_x3) - g23_count = 51; - } - asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); - if (g23 && g23_x3) - asm volatile ("movdqa %xmm1,%xmm6"); + /* Q *= 2 */ asm volatile ("pxor %xmm5,%xmm5"); asm volatile ("pcmpgtb %xmm1,%xmm5"); asm volatile ("paddb %xmm1,%xmm1"); asm volatile ("pand %xmm3,%xmm5"); asm volatile ("pxor %xmm5,%xmm1"); - if (g23 && g23_x3) - asm volatile ("pxor %xmm6,%xmm1"); asm volatile ("pxor %xmm4,%xmm0"); asm volatile ("pxor %xmm4,%xmm1"); @@ -98,22 +85,12 @@ static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv, /* first disk with all coefficients at 1 */ asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i])); - g23_x3 = 0; - if (g23) { - g23_x3 = --g23_count == 0; - if (g23_x3) - g23_count = 51; - } - - if (g23 && g23_x3) - asm volatile ("movdqa %xmm1,%xmm6"); + /* final Q *= 2 */ asm volatile ("pxor %xmm5,%xmm5"); asm volatile ("pcmpgtb %xmm1,%xmm5"); asm volatile ("paddb %xmm1,%xmm1"); asm volatile ("pand %xmm3,%xmm5"); asm volatile ("pxor %xmm5,%xmm1"); - if (g23 && g23_x3) - asm volatile ("pxor %xmm6,%xmm1"); asm volatile ("pxor %xmm4,%xmm0"); asm volatile ("pxor %xmm4,%xmm1"); @@ -127,23 +104,205 @@ static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv, raid_sse_end(); } -void raid_gen3_ssse3_raid(int nd, size_t size, void **vv) -{ - raid_gen3_ssse3_gen(nd, size, vv, 0); -} - -void raid_gen3_ssse3_aes(int nd, size_t size, void **vv) -{ - raid_gen3_ssse3_gen(nd, size, vv, 1); -} - -#ifdef CONFIG_X86_64 /* * GEN3 (triple parity with Cauchy matrix) SSSE3 implementation - * - * Note that it uses 16 registers, meaning that x64 is required. + * using the AES G23 Q transition. */ -static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **vv, int g23) +static __always_inline void raid_gen3_ssse3_g23(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + + /* special case with only one data disk */ + if (l == 0) { + for (i = 0; i < 3; ++i) + memcpy(v[1 + i], v[0], size); + return; + } + + int g23_boundary = raid_g23_boundary(l - 1); + + raid_sse_begin(); + + /* generic case with at least two data disks */ + asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + + for (i = 0; i < size; i += 16) { + int boundary = g23_boundary; + + /* last disk without the generator multiplication */ + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i])); + + asm volatile ("movdqa %xmm4,%xmm0"); + asm volatile ("movdqa %xmm4,%xmm1"); + + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + + asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0][0])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[l][1][1][0])); + asm volatile ("pshufb %xmm4,%xmm2"); + asm volatile ("pshufb %xmm5,%xmm6"); + asm volatile ("pxor %xmm6,%xmm2"); + + d = l - 1; + + /* + * Process each G23 segment. Everything above a boundary uses + * Q *= 2, while the boundary itself uses Q *= 3. + */ + while (boundary >= 0) { + for (; d > boundary; --d) { + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); + + /* Q *= 2 */ + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pcmpgtb %xmm1,%xmm5"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("pand %xmm3,%xmm5"); + asm volatile ("pxor %xmm5,%xmm1"); + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pxor %xmm6,%xmm2"); + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("pshufb %xmm5,%xmm6"); + asm volatile ("pxor %xmm6,%xmm2"); + } + + /* + * d == boundary. + * + * Save the old Q, perform Q *= 2, then XOR the old + * value to obtain Q *= 3. + */ + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); + + asm volatile ("movdqa %xmm1,%xmm6"); + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pcmpgtb %xmm1,%xmm5"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("pand %xmm3,%xmm5"); + asm volatile ("pxor %xmm5,%xmm1"); + asm volatile ("pxor %xmm6,%xmm1"); + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pxor %xmm6,%xmm2"); + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("pshufb %xmm5,%xmm6"); + asm volatile ("pxor %xmm6,%xmm2"); + + --d; + + boundary -= 51; + if (boundary < 50) + boundary = -1; + } + + /* + * No G23 boundaries remain. Process the remaining intermediate + * disks using only Q *= 2. + */ + for (; d > 0; --d) { + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); + + /* Q *= 2 */ + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pcmpgtb %xmm1,%xmm5"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("pand %xmm3,%xmm5"); + asm volatile ("pxor %xmm5,%xmm1"); + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pxor %xmm6,%xmm2"); + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("pshufb %xmm5,%xmm6"); + asm volatile ("pxor %xmm6,%xmm2"); + } + + /* + * D0 always follows an x2 transition. G23 boundaries never + * occur below d = 50. + */ + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i])); + + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pcmpgtb %xmm1,%xmm5"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("pand %xmm3,%xmm5"); + asm volatile ("pxor %xmm5,%xmm1"); + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + asm volatile ("pxor %xmm4,%xmm2"); + + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); + } + + raid_sse_end(); +} + +void raid_gen3_ssse3_raid(int nd, size_t size, void **vv) +{ + raid_gen3_ssse3_x2(nd, size, vv); +} + +void raid_gen3_ssse3_aes(int nd, size_t size, void **vv) +{ + if (raid_g23_x2_only(nd)) + raid_gen3_ssse3_x2(nd, size, vv); + else + raid_gen3_ssse3_g23(nd, size, vv); +} + +#ifdef CONFIG_X86_64 +/* + * GEN3 (triple parity with Cauchy matrix) SSSE3 implementation + * using only Q *= 2. + * + * Note that it uses 16 registers, meaning that x64 is required. + */ +static __always_inline void raid_gen3_ssse3ext_x2(int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -163,7 +322,6 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v memcpy(v[1 + i], v[0], size); return; } - int g23_start = raid_g23_count(l - 1); raid_sse_begin(); @@ -172,10 +330,6 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 32) { - int g23_count = g23_start; - int g23_x3_d; - int g23_x3_d1; - /* last disk without the generator multiplication */ asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i])); asm volatile ("movdqa %0,%%xmm12" : : "m" (v[l][i + 16])); @@ -207,17 +361,6 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v /* process two intermediate disks per iteration */ for (d = l - 1; d > 1; d -= 2) { - g23_x3_d = 0; - g23_x3_d1 = 0; - if (g23) { - g23_x3_d = --g23_count == 0; - if (g23_x3_d) - g23_count = 51; - g23_x3_d1 = --g23_count == 0; - if (g23_x3_d1) - g23_count = 51; - } - /* disk d */ asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16])); @@ -225,10 +368,7 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][0][0])); asm volatile ("movdqa %0,%%xmm15" : : "m" (raid_gfcauchypshufb[d][1][1][0])); - if (g23 && g23_x3_d) { - asm volatile ("movdqa %xmm1,%xmm6"); - asm volatile ("movdqa %xmm9,%xmm14"); - } + /* Q *= 2 */ asm volatile ("pxor %xmm5,%xmm5"); asm volatile ("pxor %xmm13,%xmm13"); asm volatile ("pcmpgtb %xmm1,%xmm5"); @@ -239,10 +379,6 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v asm volatile ("pand %xmm3,%xmm13"); asm volatile ("pxor %xmm5,%xmm1"); asm volatile ("pxor %xmm13,%xmm9"); - if (g23 && g23_x3_d) { - asm volatile ("pxor %xmm6,%xmm1"); - asm volatile ("pxor %xmm14,%xmm9"); - } asm volatile ("pxor %xmm4,%xmm0"); asm volatile ("pxor %xmm4,%xmm1"); @@ -269,16 +405,14 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v asm volatile ("pxor %xmm15,%xmm2"); asm volatile ("pxor %xmm14,%xmm10"); + /* disk d - 1 */ asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d - 1][i])); asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d - 1][i + 16])); asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d - 1][1][0][0])); asm volatile ("movdqa %0,%%xmm15" : : "m" (raid_gfcauchypshufb[d - 1][1][1][0])); - if (g23 && g23_x3_d1) { - asm volatile ("movdqa %xmm1,%xmm6"); - asm volatile ("movdqa %xmm9,%xmm14"); - } + /* Q *= 2 */ asm volatile ("pxor %xmm5,%xmm5"); asm volatile ("pxor %xmm13,%xmm13"); asm volatile ("pcmpgtb %xmm1,%xmm5"); @@ -289,10 +423,6 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v asm volatile ("pand %xmm3,%xmm13"); asm volatile ("pxor %xmm5,%xmm1"); asm volatile ("pxor %xmm13,%xmm9"); - if (g23 && g23_x3_d1) { - asm volatile ("pxor %xmm6,%xmm1"); - asm volatile ("pxor %xmm14,%xmm9"); - } asm volatile ("pxor %xmm4,%xmm0"); asm volatile ("pxor %xmm4,%xmm1"); @@ -320,21 +450,11 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v asm volatile ("pxor %xmm14,%xmm10"); } + /* single remaining intermediate disk */ if (d == 1) { - g23_x3_d = 0; - if (g23) { - g23_x3_d = --g23_count == 0; - if (g23_x3_d) - g23_count = 51; - } - asm volatile ("movdqa %0,%%xmm4" : : "m" (v[1][i])); asm volatile ("movdqa %0,%%xmm12" : : "m" (v[1][i + 16])); - if (g23 && g23_x3_d) { - asm volatile ("movdqa %xmm1,%xmm6"); - asm volatile ("movdqa %xmm9,%xmm14"); - } asm volatile ("pxor %xmm5,%xmm5"); asm volatile ("pxor %xmm13,%xmm13"); asm volatile ("pcmpgtb %xmm1,%xmm5"); @@ -345,10 +465,6 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v asm volatile ("pand %xmm3,%xmm13"); asm volatile ("pxor %xmm5,%xmm1"); asm volatile ("pxor %xmm13,%xmm9"); - if (g23 && g23_x3_d) { - asm volatile ("pxor %xmm6,%xmm1"); - asm volatile ("pxor %xmm14,%xmm9"); - } asm volatile ("pxor %xmm4,%xmm0"); asm volatile ("pxor %xmm4,%xmm1"); @@ -382,17 +498,7 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i])); asm volatile ("movdqa %0,%%xmm12" : : "m" (v[0][i + 16])); - g23_x3_d = 0; - if (g23) { - g23_x3_d = --g23_count == 0; - if (g23_x3_d) - g23_count = 51; - } - - if (g23 && g23_x3_d) { - asm volatile ("movdqa %xmm1,%xmm6"); - asm volatile ("movdqa %xmm9,%xmm14"); - } + /* final Q *= 2 */ asm volatile ("pxor %xmm5,%xmm5"); asm volatile ("pxor %xmm13,%xmm13"); asm volatile ("pcmpgtb %xmm1,%xmm5"); @@ -403,11 +509,312 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v asm volatile ("pand %xmm3,%xmm13"); asm volatile ("pxor %xmm5,%xmm1"); asm volatile ("pxor %xmm13,%xmm9"); - if (g23 && g23_x3_d) { + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + asm volatile ("pxor %xmm4,%xmm2"); + asm volatile ("pxor %xmm12,%xmm8"); + asm volatile ("pxor %xmm12,%xmm9"); + asm volatile ("pxor %xmm12,%xmm10"); + + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm8,%0" : "=m" (p[i + 16])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); + asm volatile ("movntdq %%xmm9,%0" : "=m" (q[i + 16])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); + asm volatile ("movntdq %%xmm10,%0" : "=m" (r[i + 16])); + } + + raid_sse_end(); +} + +/* + * GEN3 (triple parity with Cauchy matrix) SSSE3 implementation + * using the AES G23 Q transition. + * + * Note that it uses 16 registers, meaning that x64 is required. + */ +static __always_inline void raid_gen3_ssse3ext_g23(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + + /* special case with only one data disk */ + if (l == 0) { + for (i = 0; i < 3; ++i) + memcpy(v[1 + i], v[0], size); + return; + } + + int g23_boundary = raid_g23_boundary(l - 1); + + raid_sse_begin(); + + asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.low4[0])); + + for (i = 0; i < size; i += 32) { + int boundary = g23_boundary; + + /* last disk without the generator multiplication */ + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i])); + asm volatile ("movdqa %0,%%xmm12" : : "m" (v[l][i + 16])); + + asm volatile ("movdqa %xmm4,%xmm0"); + asm volatile ("movdqa %xmm4,%xmm1"); + asm volatile ("movdqa %xmm12,%xmm8"); + asm volatile ("movdqa %xmm12,%xmm9"); + + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("movdqa %xmm12,%xmm13"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("psrlw $4,%xmm13"); + asm volatile ("pand %xmm11,%xmm4"); + asm volatile ("pand %xmm11,%xmm12"); + asm volatile ("pand %xmm11,%xmm5"); + asm volatile ("pand %xmm11,%xmm13"); + + asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1][0])); + asm volatile ("movdqa %xmm2,%xmm10"); + asm volatile ("movdqa %xmm7,%xmm15"); + asm volatile ("pshufb %xmm4,%xmm2"); + asm volatile ("pshufb %xmm12,%xmm10"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pshufb %xmm13,%xmm15"); + asm volatile ("pxor %xmm7,%xmm2"); + asm volatile ("pxor %xmm15,%xmm10"); + + d = l - 1; + + for (;;) { + /* + * Process pairs only when both d and d - 1 are above + * the next G23 boundary. + */ + for (; d > 1 && d >= boundary + 2; d -= 2) { + /* disk d */ + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); + asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16])); + + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("movdqa %0,%%xmm15" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + + /* Q *= 2 */ + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pxor %xmm13,%xmm13"); + asm volatile ("pcmpgtb %xmm1,%xmm5"); + asm volatile ("pcmpgtb %xmm9,%xmm13"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("paddb %xmm9,%xmm9"); + asm volatile ("pand %xmm3,%xmm5"); + asm volatile ("pand %xmm3,%xmm13"); + asm volatile ("pxor %xmm5,%xmm1"); + asm volatile ("pxor %xmm13,%xmm9"); + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + asm volatile ("pxor %xmm12,%xmm8"); + asm volatile ("pxor %xmm12,%xmm9"); + + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("movdqa %xmm12,%xmm13"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("psrlw $4,%xmm13"); + asm volatile ("pand %xmm11,%xmm4"); + asm volatile ("pand %xmm11,%xmm12"); + asm volatile ("pand %xmm11,%xmm5"); + asm volatile ("pand %xmm11,%xmm13"); + + asm volatile ("movdqa %xmm7,%xmm6"); + asm volatile ("movdqa %xmm15,%xmm14"); + asm volatile ("pshufb %xmm4,%xmm7"); + asm volatile ("pshufb %xmm12,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm15"); + asm volatile ("pshufb %xmm13,%xmm14"); + asm volatile ("pxor %xmm7,%xmm2"); + asm volatile ("pxor %xmm6,%xmm10"); + asm volatile ("pxor %xmm15,%xmm2"); + asm volatile ("pxor %xmm14,%xmm10"); + + /* disk d - 1 */ + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d - 1][i])); + asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d - 1][i + 16])); + + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d - 1][1][0][0])); + asm volatile ("movdqa %0,%%xmm15" : : "m" (raid_gfcauchypshufb[d - 1][1][1][0])); + + /* Q *= 2 */ + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pxor %xmm13,%xmm13"); + asm volatile ("pcmpgtb %xmm1,%xmm5"); + asm volatile ("pcmpgtb %xmm9,%xmm13"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("paddb %xmm9,%xmm9"); + asm volatile ("pand %xmm3,%xmm5"); + asm volatile ("pand %xmm3,%xmm13"); + asm volatile ("pxor %xmm5,%xmm1"); + asm volatile ("pxor %xmm13,%xmm9"); + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + asm volatile ("pxor %xmm12,%xmm8"); + asm volatile ("pxor %xmm12,%xmm9"); + + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("movdqa %xmm12,%xmm13"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("psrlw $4,%xmm13"); + asm volatile ("pand %xmm11,%xmm4"); + asm volatile ("pand %xmm11,%xmm12"); + asm volatile ("pand %xmm11,%xmm5"); + asm volatile ("pand %xmm11,%xmm13"); + + asm volatile ("movdqa %xmm7,%xmm6"); + asm volatile ("movdqa %xmm15,%xmm14"); + asm volatile ("pshufb %xmm4,%xmm7"); + asm volatile ("pshufb %xmm12,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm15"); + asm volatile ("pshufb %xmm13,%xmm14"); + asm volatile ("pxor %xmm7,%xmm2"); + asm volatile ("pxor %xmm6,%xmm10"); + asm volatile ("pxor %xmm15,%xmm2"); + asm volatile ("pxor %xmm14,%xmm10"); + } + + /* + * An x2 segment may contain one unpaired disk. + */ + if (d > boundary && d > 0) { + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); + asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16])); + + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pxor %xmm13,%xmm13"); + asm volatile ("pcmpgtb %xmm1,%xmm5"); + asm volatile ("pcmpgtb %xmm9,%xmm13"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("paddb %xmm9,%xmm9"); + asm volatile ("pand %xmm3,%xmm5"); + asm volatile ("pand %xmm3,%xmm13"); + asm volatile ("pxor %xmm5,%xmm1"); + asm volatile ("pxor %xmm13,%xmm9"); + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + asm volatile ("pxor %xmm12,%xmm8"); + asm volatile ("pxor %xmm12,%xmm9"); + + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("movdqa %xmm12,%xmm13"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("psrlw $4,%xmm13"); + asm volatile ("pand %xmm11,%xmm4"); + asm volatile ("pand %xmm11,%xmm12"); + asm volatile ("pand %xmm11,%xmm5"); + asm volatile ("pand %xmm11,%xmm13"); + + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("movdqa %xmm6,%xmm14"); + asm volatile ("movdqa %xmm7,%xmm15"); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm12,%xmm14"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pshufb %xmm13,%xmm15"); + asm volatile ("pxor %xmm6,%xmm2"); + asm volatile ("pxor %xmm14,%xmm10"); + asm volatile ("pxor %xmm7,%xmm2"); + asm volatile ("pxor %xmm15,%xmm10"); + + --d; + } + + if (boundary < 0) + break; + + /* + * d == boundary: Q *= 3. + */ + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); + asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16])); + + asm volatile ("movdqa %xmm1,%xmm6"); + asm volatile ("movdqa %xmm9,%xmm14"); + + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pxor %xmm13,%xmm13"); + asm volatile ("pcmpgtb %xmm1,%xmm5"); + asm volatile ("pcmpgtb %xmm9,%xmm13"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("paddb %xmm9,%xmm9"); + asm volatile ("pand %xmm3,%xmm5"); + asm volatile ("pand %xmm3,%xmm13"); + asm volatile ("pxor %xmm5,%xmm1"); + asm volatile ("pxor %xmm13,%xmm9"); asm volatile ("pxor %xmm6,%xmm1"); asm volatile ("pxor %xmm14,%xmm9"); + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + asm volatile ("pxor %xmm12,%xmm8"); + asm volatile ("pxor %xmm12,%xmm9"); + + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("movdqa %xmm12,%xmm13"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("psrlw $4,%xmm13"); + asm volatile ("pand %xmm11,%xmm4"); + asm volatile ("pand %xmm11,%xmm12"); + asm volatile ("pand %xmm11,%xmm5"); + asm volatile ("pand %xmm11,%xmm13"); + + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("movdqa %xmm6,%xmm14"); + asm volatile ("movdqa %xmm7,%xmm15"); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm12,%xmm14"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pshufb %xmm13,%xmm15"); + asm volatile ("pxor %xmm6,%xmm2"); + asm volatile ("pxor %xmm14,%xmm10"); + asm volatile ("pxor %xmm7,%xmm2"); + asm volatile ("pxor %xmm15,%xmm10"); + + --d; + + boundary -= 51; + if (boundary < 50) + boundary = -1; } + /* + * D0 always follows an x2 transition. + */ + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i])); + asm volatile ("movdqa %0,%%xmm12" : : "m" (v[0][i + 16])); + + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pxor %xmm13,%xmm13"); + asm volatile ("pcmpgtb %xmm1,%xmm5"); + asm volatile ("pcmpgtb %xmm9,%xmm13"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("paddb %xmm9,%xmm9"); + asm volatile ("pand %xmm3,%xmm5"); + asm volatile ("pand %xmm3,%xmm13"); + asm volatile ("pxor %xmm5,%xmm1"); + asm volatile ("pxor %xmm13,%xmm9"); + asm volatile ("pxor %xmm4,%xmm0"); asm volatile ("pxor %xmm4,%xmm1"); asm volatile ("pxor %xmm4,%xmm2"); @@ -428,19 +835,23 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v void raid_gen3_ssse3ext_raid(int nd, size_t size, void **vv) { - raid_gen3_ssse3ext_gen(nd, size, vv, 0); + raid_gen3_ssse3ext_x2(nd, size, vv); } void raid_gen3_ssse3ext_aes(int nd, size_t size, void **vv) { - raid_gen3_ssse3ext_gen(nd, size, vv, 1); + if (raid_g23_x2_only(nd)) + raid_gen3_ssse3ext_x2(nd, size, vv); + else + raid_gen3_ssse3ext_g23(nd, size, vv); } #endif /* * GEN4 (quad parity with Cauchy matrix) SSSE3 implementation + * using only Q *= 2. */ -static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv, int g23) +static __always_inline void raid_gen4_ssse3_x2(int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -462,15 +873,11 @@ static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv, memcpy(v[1 + i], v[0], size); return; } - int g23_start = raid_g23_count(l - 1); raid_sse_begin(); /* generic case with at least two data disks */ for (i = 0; i < size; i += 16) { - int g23_count = g23_start; - int g23_x3; - /* last disk without the generator multiplication */ asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i])); @@ -497,25 +904,230 @@ static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv, /* intermediate disks */ for (d = l - 1; d > 0; --d) { - g23_x3 = 0; - if (g23) { - g23_x3 = --g23_count == 0; - if (g23_x3) - g23_count = 51; - } - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); - if (g23 && g23_x3) - asm volatile ("movdqa %xmm1,%xmm6"); + /* Q *= 2 */ + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pcmpgtb %xmm1,%xmm5"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("pand %xmm7,%xmm5"); + asm volatile ("pxor %xmm5,%xmm1"); + + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + + /* R */ + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm2"); + asm volatile ("pxor %xmm7,%xmm2"); + + /* S */ + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm3"); + asm volatile ("pxor %xmm7,%xmm3"); + } + + /* first disk with all coefficients at 1 */ + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i])); + + /* final Q *= 2 */ + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pcmpgtb %xmm1,%xmm5"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("pand %xmm7,%xmm5"); + asm volatile ("pxor %xmm5,%xmm1"); + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + asm volatile ("pxor %xmm4,%xmm2"); + asm volatile ("pxor %xmm4,%xmm3"); + + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); + asm volatile ("movntdq %%xmm3,%0" : "=m" (s[i])); + } + + raid_sse_end(); +} + +/* + * GEN4 (quad parity with Cauchy matrix) SSSE3 implementation + * using the AES G23 Q transition. + */ +static __always_inline void raid_gen4_ssse3_g23(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + uint8_t *s; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + s = v[nd + 3]; + + /* special case with only one data disk */ + if (l == 0) { + for (i = 0; i < 4; ++i) + memcpy(v[1 + i], v[0], size); + return; + } + + int g23_boundary = raid_g23_boundary(l - 1); + + raid_sse_begin(); + + for (i = 0; i < size; i += 16) { + int boundary = g23_boundary; + + /* last disk without the generator multiplication */ + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i])); + + asm volatile ("movdqa %xmm4,%xmm0"); + asm volatile ("movdqa %xmm4,%xmm1"); + + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + + asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1][0])); + asm volatile ("pshufb %xmm4,%xmm2"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm7,%xmm2"); + + asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][2][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1][0])); + asm volatile ("pshufb %xmm4,%xmm3"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm7,%xmm3"); + + d = l - 1; + + /* + * Process each G23 segment. Everything above a boundary + * uses Q *= 2, while the boundary itself uses Q *= 3. + */ + while (boundary >= 0) { + for (; d > boundary; --d) { + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); + + /* Q *= 2 */ + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pcmpgtb %xmm1,%xmm5"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("pand %xmm7,%xmm5"); + asm volatile ("pxor %xmm5,%xmm1"); + + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm2"); + asm volatile ("pxor %xmm7,%xmm2"); + + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm3"); + asm volatile ("pxor %xmm7,%xmm3"); + } + + /* + * d == boundary: Q *= 3. + * + * Save the old Q, perform the normal x2 transition, + * then XOR the old value back. + */ + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); + + asm volatile ("movdqa %xmm1,%xmm6"); + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pcmpgtb %xmm1,%xmm5"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("pand %xmm7,%xmm5"); + asm volatile ("pxor %xmm5,%xmm1"); + asm volatile ("pxor %xmm6,%xmm1"); + + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm2"); + asm volatile ("pxor %xmm7,%xmm2"); + + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm3"); + asm volatile ("pxor %xmm7,%xmm3"); + + --d; + + boundary -= 51; + if (boundary < 50) + boundary = -1; + } + + /* + * No G23 boundaries remain. Process the remaining + * intermediate disks using only Q *= 2. + */ + for (; d > 0; --d) { + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); + asm volatile ("pxor %xmm5,%xmm5"); asm volatile ("pcmpgtb %xmm1,%xmm5"); asm volatile ("paddb %xmm1,%xmm1"); asm volatile ("pand %xmm7,%xmm5"); asm volatile ("pxor %xmm5,%xmm1"); - if (g23 && g23_x3) - asm volatile ("pxor %xmm6,%xmm1"); asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); @@ -542,26 +1154,17 @@ static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv, asm volatile ("pxor %xmm7,%xmm3"); } - /* first disk with all coefficients at 1 */ + /* + * D0 always follows an x2 transition. + */ asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i])); - g23_x3 = 0; - if (g23) { - g23_x3 = --g23_count == 0; - if (g23_x3) - g23_count = 51; - } - - if (g23 && g23_x3) - asm volatile ("movdqa %xmm1,%xmm6"); asm volatile ("pxor %xmm5,%xmm5"); asm volatile ("pcmpgtb %xmm1,%xmm5"); asm volatile ("paddb %xmm1,%xmm1"); asm volatile ("pand %xmm7,%xmm5"); asm volatile ("pxor %xmm5,%xmm1"); - if (g23 && g23_x3) - asm volatile ("pxor %xmm6,%xmm1"); asm volatile ("pxor %xmm4,%xmm0"); asm volatile ("pxor %xmm4,%xmm1"); @@ -579,21 +1182,25 @@ static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv, void raid_gen4_ssse3_raid(int nd, size_t size, void **vv) { - raid_gen4_ssse3_gen(nd, size, vv, 0); + raid_gen4_ssse3_x2(nd, size, vv); } void raid_gen4_ssse3_aes(int nd, size_t size, void **vv) { - raid_gen4_ssse3_gen(nd, size, vv, 1); + if (raid_g23_x2_only(nd)) + raid_gen4_ssse3_x2(nd, size, vv); + else + raid_gen4_ssse3_g23(nd, size, vv); } #ifdef CONFIG_X86_64 /* * GEN4 (quad parity with Cauchy matrix) SSSE3 implementation + * using only Q *= 2. * * Note that it uses 16 registers, meaning that x64 is required. */ -static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **vv, int g23) +static __always_inline void raid_gen4_ssse3ext_x2(int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -615,14 +1222,190 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v memcpy(v[1 + i], v[0], size); return; } - int g23_start = raid_g23_count(l - 1); raid_sse_begin(); /* generic case with at least two data disks */ for (i = 0; i < size; i += 32) { - int g23_count = g23_start; - int g23_x3; + /* last disk without the generator multiplication */ + asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i])); + asm volatile ("movdqa %0,%%xmm12" : : "m" (v[l][i + 16])); + + asm volatile ("movdqa %xmm4,%xmm0"); + asm volatile ("movdqa %xmm4,%xmm1"); + asm volatile ("movdqa %xmm12,%xmm8"); + asm volatile ("movdqa %xmm12,%xmm9"); + + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("movdqa %xmm12,%xmm13"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("psrlw $4,%xmm13"); + asm volatile ("pand %xmm15,%xmm4"); + asm volatile ("pand %xmm15,%xmm12"); + asm volatile ("pand %xmm15,%xmm5"); + asm volatile ("pand %xmm15,%xmm13"); + + /* R */ + asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1][0])); + asm volatile ("movdqa %xmm2,%xmm10"); + asm volatile ("movdqa %xmm7,%xmm15"); + asm volatile ("pshufb %xmm4,%xmm2"); + asm volatile ("pshufb %xmm12,%xmm10"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pshufb %xmm13,%xmm15"); + asm volatile ("pxor %xmm7,%xmm2"); + asm volatile ("pxor %xmm15,%xmm10"); + + /* S */ + asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][2][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1][0])); + asm volatile ("movdqa %xmm3,%xmm11"); + asm volatile ("movdqa %xmm7,%xmm15"); + asm volatile ("pshufb %xmm4,%xmm3"); + asm volatile ("pshufb %xmm12,%xmm11"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pshufb %xmm13,%xmm15"); + asm volatile ("pxor %xmm7,%xmm3"); + asm volatile ("pxor %xmm15,%xmm11"); + + /* intermediate disks */ + for (d = l - 1; d > 0; --d) { + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); + asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16])); + + /* Q *= 2 */ + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pxor %xmm13,%xmm13"); + asm volatile ("pcmpgtb %xmm1,%xmm5"); + asm volatile ("pcmpgtb %xmm9,%xmm13"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("paddb %xmm9,%xmm9"); + asm volatile ("pand %xmm7,%xmm5"); + asm volatile ("pand %xmm7,%xmm13"); + asm volatile ("pxor %xmm5,%xmm1"); + asm volatile ("pxor %xmm13,%xmm9"); + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + asm volatile ("pxor %xmm12,%xmm8"); + asm volatile ("pxor %xmm12,%xmm9"); + + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("movdqa %xmm12,%xmm13"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("psrlw $4,%xmm13"); + asm volatile ("pand %xmm15,%xmm4"); + asm volatile ("pand %xmm15,%xmm12"); + asm volatile ("pand %xmm15,%xmm5"); + asm volatile ("pand %xmm15,%xmm13"); + + /* R */ + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("movdqa %xmm6,%xmm14"); + asm volatile ("movdqa %xmm7,%xmm15"); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm12,%xmm14"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pshufb %xmm13,%xmm15"); + asm volatile ("pxor %xmm6,%xmm2"); + asm volatile ("pxor %xmm14,%xmm10"); + asm volatile ("pxor %xmm7,%xmm2"); + asm volatile ("pxor %xmm15,%xmm10"); + + /* S */ + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("movdqa %xmm6,%xmm14"); + asm volatile ("movdqa %xmm7,%xmm15"); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm12,%xmm14"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pshufb %xmm13,%xmm15"); + asm volatile ("pxor %xmm6,%xmm3"); + asm volatile ("pxor %xmm14,%xmm11"); + asm volatile ("pxor %xmm7,%xmm3"); + asm volatile ("pxor %xmm15,%xmm11"); + } + + /* first disk with all coefficients at 1 */ + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i])); + asm volatile ("movdqa %0,%%xmm12" : : "m" (v[0][i + 16])); + + /* final Q *= 2 */ + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pxor %xmm13,%xmm13"); + asm volatile ("pcmpgtb %xmm1,%xmm5"); + asm volatile ("pcmpgtb %xmm9,%xmm13"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("paddb %xmm9,%xmm9"); + asm volatile ("pand %xmm7,%xmm5"); + asm volatile ("pand %xmm7,%xmm13"); + asm volatile ("pxor %xmm5,%xmm1"); + asm volatile ("pxor %xmm13,%xmm9"); + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + asm volatile ("pxor %xmm4,%xmm2"); + asm volatile ("pxor %xmm4,%xmm3"); + asm volatile ("pxor %xmm12,%xmm8"); + asm volatile ("pxor %xmm12,%xmm9"); + asm volatile ("pxor %xmm12,%xmm10"); + asm volatile ("pxor %xmm12,%xmm11"); + + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm8,%0" : "=m" (p[i + 16])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); + asm volatile ("movntdq %%xmm9,%0" : "=m" (q[i + 16])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); + asm volatile ("movntdq %%xmm10,%0" : "=m" (r[i + 16])); + asm volatile ("movntdq %%xmm3,%0" : "=m" (s[i])); + asm volatile ("movntdq %%xmm11,%0" : "=m" (s[i + 16])); + } + + raid_sse_end(); +} + +/* + * GEN4 (quad parity with Cauchy matrix) SSSE3 implementation + * using the AES G23 Q transition. + * + * Note that it uses 16 registers, meaning that x64 is required. + */ +static __always_inline void raid_gen4_ssse3ext_g23(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + uint8_t *s; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + s = v[nd + 3]; + + /* special case with only one data disk */ + if (l == 0) { + for (i = 0; i < 4; ++i) + memcpy(v[1 + i], v[0], size); + return; + } + + int g23_boundary = raid_g23_boundary(l - 1); + + raid_sse_begin(); + + for (i = 0; i < size; i += 32) { + int boundary = g23_boundary; /* last disk without the generator multiplication */ asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); @@ -643,6 +1426,7 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v asm volatile ("pand %xmm15,%xmm5"); asm volatile ("pand %xmm15,%xmm13"); + /* R */ asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0][0])); asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1][0])); asm volatile ("movdqa %xmm2,%xmm10"); @@ -654,6 +1438,7 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v asm volatile ("pxor %xmm7,%xmm2"); asm volatile ("pxor %xmm15,%xmm10"); + /* S */ asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][2][0][0])); asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1][0])); asm volatile ("movdqa %xmm3,%xmm11"); @@ -665,24 +1450,88 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v asm volatile ("pxor %xmm7,%xmm3"); asm volatile ("pxor %xmm15,%xmm11"); - /* intermediate disks */ - for (d = l - 1; d > 0; --d) { - g23_x3 = 0; - if (g23) { - g23_x3 = --g23_count == 0; - if (g23_x3) - g23_count = 51; + d = l - 1; + + while (boundary >= 0) { + /* + * All transitions above the next G23 boundary use Q *= 2. + */ + for (; d > boundary; --d) { + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); + asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16])); + + /* Q *= 2 */ + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pxor %xmm13,%xmm13"); + asm volatile ("pcmpgtb %xmm1,%xmm5"); + asm volatile ("pcmpgtb %xmm9,%xmm13"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("paddb %xmm9,%xmm9"); + asm volatile ("pand %xmm7,%xmm5"); + asm volatile ("pand %xmm7,%xmm13"); + asm volatile ("pxor %xmm5,%xmm1"); + asm volatile ("pxor %xmm13,%xmm9"); + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + asm volatile ("pxor %xmm12,%xmm8"); + asm volatile ("pxor %xmm12,%xmm9"); + + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("movdqa %xmm12,%xmm13"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("psrlw $4,%xmm13"); + asm volatile ("pand %xmm15,%xmm4"); + asm volatile ("pand %xmm15,%xmm12"); + asm volatile ("pand %xmm15,%xmm5"); + asm volatile ("pand %xmm15,%xmm13"); + + /* R */ + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("movdqa %xmm6,%xmm14"); + asm volatile ("movdqa %xmm7,%xmm15"); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm12,%xmm14"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pshufb %xmm13,%xmm15"); + asm volatile ("pxor %xmm6,%xmm2"); + asm volatile ("pxor %xmm14,%xmm10"); + asm volatile ("pxor %xmm7,%xmm2"); + asm volatile ("pxor %xmm15,%xmm10"); + + /* S */ + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("movdqa %xmm6,%xmm14"); + asm volatile ("movdqa %xmm7,%xmm15"); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm12,%xmm14"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pshufb %xmm13,%xmm15"); + asm volatile ("pxor %xmm6,%xmm3"); + asm volatile ("pxor %xmm14,%xmm11"); + asm volatile ("pxor %xmm7,%xmm3"); + asm volatile ("pxor %xmm15,%xmm11"); } + /* + * d == boundary: Q *= 3. + */ asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16])); - if (g23 && g23_x3) { - asm volatile ("movdqa %xmm1,%xmm6"); - asm volatile ("movdqa %xmm9,%xmm14"); - } + /* + * Save the old Q values before the normal x2 + * transition and XOR them back to obtain x3. + */ + asm volatile ("movdqa %xmm1,%xmm6"); + asm volatile ("movdqa %xmm9,%xmm14"); + asm volatile ("pxor %xmm5,%xmm5"); asm volatile ("pxor %xmm13,%xmm13"); asm volatile ("pcmpgtb %xmm1,%xmm5"); @@ -693,10 +1542,8 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v asm volatile ("pand %xmm7,%xmm13"); asm volatile ("pxor %xmm5,%xmm1"); asm volatile ("pxor %xmm13,%xmm9"); - if (g23 && g23_x3) { - asm volatile ("pxor %xmm6,%xmm1"); - asm volatile ("pxor %xmm14,%xmm9"); - } + asm volatile ("pxor %xmm6,%xmm1"); + asm volatile ("pxor %xmm14,%xmm9"); asm volatile ("pxor %xmm4,%xmm0"); asm volatile ("pxor %xmm4,%xmm1"); @@ -712,6 +1559,7 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v asm volatile ("pand %xmm15,%xmm5"); asm volatile ("pand %xmm15,%xmm13"); + /* R */ asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0])); asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0])); asm volatile ("movdqa %xmm6,%xmm14"); @@ -725,6 +1573,78 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v asm volatile ("pxor %xmm7,%xmm2"); asm volatile ("pxor %xmm15,%xmm10"); + /* S */ + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("movdqa %xmm6,%xmm14"); + asm volatile ("movdqa %xmm7,%xmm15"); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm12,%xmm14"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pshufb %xmm13,%xmm15"); + asm volatile ("pxor %xmm6,%xmm3"); + asm volatile ("pxor %xmm14,%xmm11"); + asm volatile ("pxor %xmm7,%xmm3"); + asm volatile ("pxor %xmm15,%xmm11"); + + --d; + + boundary -= 51; + if (boundary < 50) + boundary = -1; + } + + /* + * No G23 boundaries remain. Process the remaining + * intermediate disks using only Q *= 2. + */ + for (; d > 0; --d) { + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); + asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16])); + + /* Q *= 2 */ + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pxor %xmm13,%xmm13"); + asm volatile ("pcmpgtb %xmm1,%xmm5"); + asm volatile ("pcmpgtb %xmm9,%xmm13"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("paddb %xmm9,%xmm9"); + asm volatile ("pand %xmm7,%xmm5"); + asm volatile ("pand %xmm7,%xmm13"); + asm volatile ("pxor %xmm5,%xmm1"); + asm volatile ("pxor %xmm13,%xmm9"); + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + asm volatile ("pxor %xmm12,%xmm8"); + asm volatile ("pxor %xmm12,%xmm9"); + + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("movdqa %xmm12,%xmm13"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("psrlw $4,%xmm13"); + asm volatile ("pand %xmm15,%xmm4"); + asm volatile ("pand %xmm15,%xmm12"); + asm volatile ("pand %xmm15,%xmm5"); + asm volatile ("pand %xmm15,%xmm13"); + + /* R */ + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("movdqa %xmm6,%xmm14"); + asm volatile ("movdqa %xmm7,%xmm15"); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm12,%xmm14"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pshufb %xmm13,%xmm15"); + asm volatile ("pxor %xmm6,%xmm2"); + asm volatile ("pxor %xmm14,%xmm10"); + asm volatile ("pxor %xmm7,%xmm2"); + asm volatile ("pxor %xmm15,%xmm10"); + + /* S */ asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0][0])); asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1][0])); asm volatile ("movdqa %xmm6,%xmm14"); @@ -739,23 +1659,13 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v asm volatile ("pxor %xmm15,%xmm11"); } - /* first disk with all coefficients at 1 */ + /* + * D0 always follows Q *= 2. The lowest G23 boundary is 50. + */ asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); - asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i])); asm volatile ("movdqa %0,%%xmm12" : : "m" (v[0][i + 16])); - g23_x3 = 0; - if (g23) { - g23_x3 = --g23_count == 0; - if (g23_x3) - g23_count = 51; - } - - if (g23 && g23_x3) { - asm volatile ("movdqa %xmm1,%xmm6"); - asm volatile ("movdqa %xmm9,%xmm14"); - } asm volatile ("pxor %xmm5,%xmm5"); asm volatile ("pxor %xmm13,%xmm13"); asm volatile ("pcmpgtb %xmm1,%xmm5"); @@ -766,10 +1676,6 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v asm volatile ("pand %xmm7,%xmm13"); asm volatile ("pxor %xmm5,%xmm1"); asm volatile ("pxor %xmm13,%xmm9"); - if (g23 && g23_x3) { - asm volatile ("pxor %xmm6,%xmm1"); - asm volatile ("pxor %xmm14,%xmm9"); - } asm volatile ("pxor %xmm4,%xmm0"); asm volatile ("pxor %xmm4,%xmm1"); @@ -795,19 +1701,23 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v void raid_gen4_ssse3ext_raid(int nd, size_t size, void **vv) { - raid_gen4_ssse3ext_gen(nd, size, vv, 0); + raid_gen4_ssse3ext_x2(nd, size, vv); } void raid_gen4_ssse3ext_aes(int nd, size_t size, void **vv) { - raid_gen4_ssse3ext_gen(nd, size, vv, 1); + if (raid_g23_x2_only(nd)) + raid_gen4_ssse3ext_x2(nd, size, vv); + else + raid_gen4_ssse3ext_g23(nd, size, vv); } #endif /* * GEN5 (penta parity with Cauchy matrix) SSSE3 implementation + * using only Q *= 2. */ -void raid_gen5_ssse3_raid(int nd, size_t size, void **vv) +void raid_gen5_ssse3_x2(int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -939,11 +1849,305 @@ void raid_gen5_ssse3_raid(int nd, size_t size, void **vv) raid_sse_end(); } +/* + * GEN5 (penta parity with Cauchy matrix) SSSE3 implementation + * using the AES G23 Q transition. + */ +static __always_inline void raid_gen5_ssse3_g23(int nd, size_t size, + void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + uint8_t *s; + uint8_t *t; + int d, l; + size_t i; + uint8_t buffer[16 + 16]; + uint8_t *pd = __align_ptr(buffer, 16); + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + s = v[nd + 3]; + t = v[nd + 4]; + + /* special case with only one data disk */ + if (l == 0) { + for (i = 0; i < 5; ++i) + memcpy(v[1 + i], v[0], size); + return; + } + + int g23_boundary = raid_g23_boundary(l - 1); + + raid_sse_begin(); + + /* generic case with at least two data disks */ + for (i = 0; i < size; i += 16) { + int boundary = g23_boundary; + + /* last disk without the generator multiplication */ + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i])); + + /* + * xmm0 = Q + * pd[0] = P + */ + asm volatile ("movdqa %xmm4,%xmm0"); + asm volatile ("movdqa %%xmm4,%0" : "=m" (pd[0])); + + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + + /* R */ + asm volatile ("movdqa %0,%%xmm1" : : "m" (raid_gfcauchypshufb[l][1][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1][0])); + asm volatile ("pshufb %xmm4,%xmm1"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm7,%xmm1"); + + /* S */ + asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][2][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1][0])); + asm volatile ("pshufb %xmm4,%xmm2"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm7,%xmm2"); + + /* T */ + asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][3][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][3][1][0])); + asm volatile ("pshufb %xmm4,%xmm3"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm7,%xmm3"); + + d = l - 1; + + /* + * Process all complete G23 segments. + */ + while (boundary >= 0) { + /* + * All transitions above the boundary use Q *= 2. + */ + for (; d > boundary; --d) { + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + + /* Q *= 2 */ + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pcmpgtb %xmm0,%xmm5"); + asm volatile ("paddb %xmm0,%xmm0"); + asm volatile ("pand %xmm7,%xmm5"); + asm volatile ("pxor %xmm5,%xmm0"); + + /* P ^= D, Q ^= D */ + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm6"); + asm volatile ("movdqa %%xmm6,%0" : "=m" (pd[0])); + + /* split D into nibbles */ + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + + /* R */ + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm1"); + asm volatile ("pxor %xmm7,%xmm1"); + + /* S */ + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm2"); + asm volatile ("pxor %xmm7,%xmm2"); + + /* T */ + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][3][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][3][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm3"); + asm volatile ("pxor %xmm7,%xmm3"); + } + + /* + * d == boundary: Q *= 3. + * + * xmm6 can temporarily hold the old Q. After the x3 + * transition it is free again and can be reused to + * load the spilled P accumulator. + */ + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); + + /* save old Q */ + asm volatile ("movdqa %xmm0,%xmm6"); + + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + + /* Q *= 2 */ + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pcmpgtb %xmm0,%xmm5"); + asm volatile ("paddb %xmm0,%xmm0"); + asm volatile ("pand %xmm7,%xmm5"); + asm volatile ("pxor %xmm5,%xmm0"); + + /* Q *= 3 */ + asm volatile ("pxor %xmm6,%xmm0"); + + /* now xmm6 can be reused for P */ + asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[0])); + + /* P ^= D, Q ^= D */ + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm6"); + asm volatile ("movdqa %%xmm6,%0" : "=m" (pd[0])); + + /* split D into nibbles */ + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + + /* R */ + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm1"); + asm volatile ("pxor %xmm7,%xmm1"); + + /* S */ + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm2"); + asm volatile ("pxor %xmm7,%xmm2"); + + /* T */ + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][3][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][3][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm3"); + asm volatile ("pxor %xmm7,%xmm3"); + + --d; + + boundary -= 51; + if (boundary < 50) + boundary = -1; + } + + /* + * No G23 boundaries remain. + */ + for (; d > 0; --d) { + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + + /* Q *= 2 */ + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pcmpgtb %xmm0,%xmm5"); + asm volatile ("paddb %xmm0,%xmm0"); + asm volatile ("pand %xmm7,%xmm5"); + asm volatile ("pxor %xmm5,%xmm0"); + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm6"); + asm volatile ("movdqa %%xmm6,%0" : "=m" (pd[0])); + + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm1"); + asm volatile ("pxor %xmm7,%xmm1"); + + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm2"); + asm volatile ("pxor %xmm7,%xmm2"); + + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][3][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][3][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm3"); + asm volatile ("pxor %xmm7,%xmm3"); + } + + /* + * D0 always uses Q *= 2. The lowest G23 boundary is 50. + */ + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pcmpgtb %xmm0,%xmm5"); + asm volatile ("paddb %xmm0,%xmm0"); + asm volatile ("pand %xmm7,%xmm5"); + asm volatile ("pxor %xmm5,%xmm0"); + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + asm volatile ("pxor %xmm4,%xmm2"); + asm volatile ("pxor %xmm4,%xmm3"); + asm volatile ("pxor %xmm4,%xmm6"); + + asm volatile ("movntdq %%xmm6,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm0,%0" : "=m" (q[i])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (r[i])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (s[i])); + asm volatile ("movntdq %%xmm3,%0" : "=m" (t[i])); + } + + raid_sse_end(); +} + +void raid_gen5_ssse3_raid(int nd, size_t size, void **vv) +{ + raid_gen5_ssse3_x2(nd, size, vv); +} + +void raid_gen5_ssse3_aes(int nd, size_t size, void **vv) +{ + if (raid_g23_x2_only(nd)) + raid_gen5_ssse3_x2(nd, size, vv); + else + raid_gen5_ssse3_g23(nd, size, vv); +} + #ifdef CONFIG_X86_64 /* - * GENX SSSE3ext implementation + * GENX SSSE3ext implementation using only Q *= 2. */ -static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, int np, int g23) +static __always_inline void raid_genX_ssse3ext_x2(int nd, size_t size, void **vv, int np) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -970,7 +2174,6 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i memcpy(v[1 + d], v[0], size); return; } - int g23_start = raid_g23_count(l - 1); raid_sse_begin(); @@ -979,9 +2182,6 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 16) { - int g23_count = g23_start; - int g23_x3; - /* last disk without the generator multiplication */ asm volatile ("movdqa %0,%%xmm10" : : "m" (v[l][i])); @@ -1027,24 +2227,14 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i /* intermediate disks */ for (d = l - 1; d > 0; --d) { - g23_x3 = 0; - if (g23) { - g23_x3 = --g23_count == 0; - if (g23_x3) - g23_count = 51; - } - asm volatile ("movdqa %0,%%xmm10" : : "m" (v[d][i])); - if (g23 && g23_x3) - asm volatile ("movdqa %xmm1,%xmm12"); + /* Q *= 2 */ asm volatile ("pxor %xmm11,%xmm11"); asm volatile ("pcmpgtb %xmm1,%xmm11"); asm volatile ("paddb %xmm1,%xmm1"); asm volatile ("pand %xmm14,%xmm11"); asm volatile ("pxor %xmm11,%xmm1"); - if (g23 && g23_x3) - asm volatile ("pxor %xmm12,%xmm1"); asm volatile ("pxor %xmm10,%xmm0"); asm volatile ("pxor %xmm10,%xmm1"); @@ -1094,22 +2284,12 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i /* first disk with all coefficients at 1 */ asm volatile ("movdqa %0,%%xmm10" : : "m" (v[0][i])); - g23_x3 = 0; - if (g23) { - g23_x3 = --g23_count == 0; - if (g23_x3) - g23_count = 51; - } - - if (g23 && g23_x3) - asm volatile ("movdqa %xmm1,%xmm12"); + /* final Q *= 2 */ asm volatile ("pxor %xmm11,%xmm11"); asm volatile ("pcmpgtb %xmm1,%xmm11"); asm volatile ("paddb %xmm1,%xmm1"); asm volatile ("pand %xmm14,%xmm11"); asm volatile ("pxor %xmm11,%xmm1"); - if (g23 && g23_x3) - asm volatile ("pxor %xmm12,%xmm1"); asm volatile ("pxor %xmm10,%xmm0"); asm volatile ("pxor %xmm10,%xmm1"); @@ -1138,25 +2318,332 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i } /* - * GEN5 (penta parity with Cauchy matrix) SSSE3 implementation - * - * Note that it uses 16 registers, meaning that x64 is required. + * GENX SSSE3ext implementation using the AES G23 Q transition. */ +static __always_inline void raid_genX_ssse3ext_g23(int nd, size_t size, void **vv, int np) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + uint8_t *s; + uint8_t *t; + uint8_t *u; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + s = v[nd + 3]; + t = v[nd + 4]; + if (np >= 6) + u = v[nd + 5]; + + /* special case with only one data disk */ + if (l == 0) { + for (d = 0; d < np; ++d) + memcpy(v[1 + d], v[0], size); + return; + } + + int g23_boundary = raid_g23_boundary(l - 1); + + raid_sse_begin(); + + asm volatile ("movdqa %0,%%xmm14" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); + + for (i = 0; i < size; i += 16) { + int boundary = g23_boundary; + + /* last disk without the generator multiplication */ + asm volatile ("movdqa %0,%%xmm10" : : "m" (v[l][i])); + + asm volatile ("movdqa %xmm10,%xmm0"); + asm volatile ("movdqa %xmm10,%xmm1"); + + asm volatile ("movdqa %xmm10,%xmm11"); + asm volatile ("psrlw $4,%xmm11"); + asm volatile ("pand %xmm15,%xmm10"); + asm volatile ("pand %xmm15,%xmm11"); + + if (np >= 3) { + asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[l][1][1][0])); + asm volatile ("pshufb %xmm10,%xmm2"); + asm volatile ("pshufb %xmm11,%xmm13"); + asm volatile ("pxor %xmm13,%xmm2"); + } + + if (np >= 4) { + asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][2][0][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[l][2][1][0])); + asm volatile ("pshufb %xmm10,%xmm3"); + asm volatile ("pshufb %xmm11,%xmm13"); + asm volatile ("pxor %xmm13,%xmm3"); + } + + if (np >= 5) { + asm volatile ("movdqa %0,%%xmm4" : : "m" (raid_gfcauchypshufb[l][3][0][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[l][3][1][0])); + asm volatile ("pshufb %xmm10,%xmm4"); + asm volatile ("pshufb %xmm11,%xmm13"); + asm volatile ("pxor %xmm13,%xmm4"); + } + + if (np >= 6) { + asm volatile ("movdqa %0,%%xmm5" : : "m" (raid_gfcauchypshufb[l][4][0][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[l][4][1][0])); + asm volatile ("pshufb %xmm10,%xmm5"); + asm volatile ("pshufb %xmm11,%xmm13"); + asm volatile ("pxor %xmm13,%xmm5"); + } + + d = l - 1; + + while (boundary >= 0) { + /* + * All transitions above the next G23 boundary use Q *= 2. + */ + for (; d > boundary; --d) { + asm volatile ("movdqa %0,%%xmm10" : : "m" (v[d][i])); + + asm volatile ("pxor %xmm11,%xmm11"); + asm volatile ("pcmpgtb %xmm1,%xmm11"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("pand %xmm14,%xmm11"); + asm volatile ("pxor %xmm11,%xmm1"); + + asm volatile ("pxor %xmm10,%xmm0"); + asm volatile ("pxor %xmm10,%xmm1"); + + asm volatile ("movdqa %xmm10,%xmm11"); + asm volatile ("psrlw $4,%xmm11"); + asm volatile ("pand %xmm15,%xmm10"); + asm volatile ("pand %xmm15,%xmm11"); + + if (np >= 3) { + asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("pshufb %xmm10,%xmm12"); + asm volatile ("pshufb %xmm11,%xmm13"); + asm volatile ("pxor %xmm12,%xmm2"); + asm volatile ("pxor %xmm13,%xmm2"); + } + + if (np >= 4) { + asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][2][0][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("pshufb %xmm10,%xmm12"); + asm volatile ("pshufb %xmm11,%xmm13"); + asm volatile ("pxor %xmm12,%xmm3"); + asm volatile ("pxor %xmm13,%xmm3"); + } + + if (np >= 5) { + asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][3][0][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][3][1][0])); + asm volatile ("pshufb %xmm10,%xmm12"); + asm volatile ("pshufb %xmm11,%xmm13"); + asm volatile ("pxor %xmm12,%xmm4"); + asm volatile ("pxor %xmm13,%xmm4"); + } + + if (np >= 6) { + asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][4][0][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][4][1][0])); + asm volatile ("pshufb %xmm10,%xmm12"); + asm volatile ("pshufb %xmm11,%xmm13"); + asm volatile ("pxor %xmm12,%xmm5"); + asm volatile ("pxor %xmm13,%xmm5"); + } + } + + /* + * d == boundary: Q *= 3. + */ + asm volatile ("movdqa %0,%%xmm10" : : "m" (v[d][i])); + + /* preserve old Q */ + asm volatile ("movdqa %xmm1,%xmm12"); + + /* Q *= 2 */ + asm volatile ("pxor %xmm11,%xmm11"); + asm volatile ("pcmpgtb %xmm1,%xmm11"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("pand %xmm14,%xmm11"); + asm volatile ("pxor %xmm11,%xmm1"); + + /* Q *= 3 */ + asm volatile ("pxor %xmm12,%xmm1"); + + asm volatile ("pxor %xmm10,%xmm0"); + asm volatile ("pxor %xmm10,%xmm1"); + + asm volatile ("movdqa %xmm10,%xmm11"); + asm volatile ("psrlw $4,%xmm11"); + asm volatile ("pand %xmm15,%xmm10"); + asm volatile ("pand %xmm15,%xmm11"); + + if (np >= 3) { + asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("pshufb %xmm10,%xmm12"); + asm volatile ("pshufb %xmm11,%xmm13"); + asm volatile ("pxor %xmm12,%xmm2"); + asm volatile ("pxor %xmm13,%xmm2"); + } + + if (np >= 4) { + asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][2][0][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("pshufb %xmm10,%xmm12"); + asm volatile ("pshufb %xmm11,%xmm13"); + asm volatile ("pxor %xmm12,%xmm3"); + asm volatile ("pxor %xmm13,%xmm3"); + } + + if (np >= 5) { + asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][3][0][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][3][1][0])); + asm volatile ("pshufb %xmm10,%xmm12"); + asm volatile ("pshufb %xmm11,%xmm13"); + asm volatile ("pxor %xmm12,%xmm4"); + asm volatile ("pxor %xmm13,%xmm4"); + } + + if (np >= 6) { + asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][4][0][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][4][1][0])); + asm volatile ("pshufb %xmm10,%xmm12"); + asm volatile ("pshufb %xmm11,%xmm13"); + asm volatile ("pxor %xmm12,%xmm5"); + asm volatile ("pxor %xmm13,%xmm5"); + } + + --d; + + boundary -= 51; + if (boundary < 50) + boundary = -1; + } + + /* + * No G23 boundaries remain. Process all remaining + * intermediate disks with Q *= 2. + */ + for (; d > 0; --d) { + asm volatile ("movdqa %0,%%xmm10" : : "m" (v[d][i])); + + asm volatile ("pxor %xmm11,%xmm11"); + asm volatile ("pcmpgtb %xmm1,%xmm11"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("pand %xmm14,%xmm11"); + asm volatile ("pxor %xmm11,%xmm1"); + + asm volatile ("pxor %xmm10,%xmm0"); + asm volatile ("pxor %xmm10,%xmm1"); + + asm volatile ("movdqa %xmm10,%xmm11"); + asm volatile ("psrlw $4,%xmm11"); + asm volatile ("pand %xmm15,%xmm10"); + asm volatile ("pand %xmm15,%xmm11"); + + if (np >= 3) { + asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("pshufb %xmm10,%xmm12"); + asm volatile ("pshufb %xmm11,%xmm13"); + asm volatile ("pxor %xmm12,%xmm2"); + asm volatile ("pxor %xmm13,%xmm2"); + } + + if (np >= 4) { + asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][2][0][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("pshufb %xmm10,%xmm12"); + asm volatile ("pshufb %xmm11,%xmm13"); + asm volatile ("pxor %xmm12,%xmm3"); + asm volatile ("pxor %xmm13,%xmm3"); + } + + if (np >= 5) { + asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][3][0][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][3][1][0])); + asm volatile ("pshufb %xmm10,%xmm12"); + asm volatile ("pshufb %xmm11,%xmm13"); + asm volatile ("pxor %xmm12,%xmm4"); + asm volatile ("pxor %xmm13,%xmm4"); + } + + if (np >= 6) { + asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][4][0][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][4][1][0])); + asm volatile ("pshufb %xmm10,%xmm12"); + asm volatile ("pshufb %xmm11,%xmm13"); + asm volatile ("pxor %xmm12,%xmm5"); + asm volatile ("pxor %xmm13,%xmm5"); + } + } + + /* + * D0 always follows Q *= 2. + */ + asm volatile ("movdqa %0,%%xmm10" : : "m" (v[0][i])); + + asm volatile ("pxor %xmm11,%xmm11"); + asm volatile ("pcmpgtb %xmm1,%xmm11"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("pand %xmm14,%xmm11"); + asm volatile ("pxor %xmm11,%xmm1"); + + asm volatile ("pxor %xmm10,%xmm0"); + asm volatile ("pxor %xmm10,%xmm1"); + if (np >= 3) + asm volatile ("pxor %xmm10,%xmm2"); + if (np >= 4) + asm volatile ("pxor %xmm10,%xmm3"); + if (np >= 5) + asm volatile ("pxor %xmm10,%xmm4"); + if (np >= 6) + asm volatile ("pxor %xmm10,%xmm5"); + + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); + if (np >= 3) + asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); + if (np >= 4) + asm volatile ("movntdq %%xmm3,%0" : "=m" (s[i])); + if (np >= 5) + asm volatile ("movntdq %%xmm4,%0" : "=m" (t[i])); + if (np >= 6) + asm volatile ("movntdq %%xmm5,%0" : "=m" (u[i])); + } + + raid_sse_end(); +} + void raid_gen5_ssse3ext_raid(int nd, size_t size, void **vv) { - raid_genX_ssse3ext(nd, size, vv, 5, 0); + raid_genX_ssse3ext_x2(nd, size, vv, 5); } void raid_gen5_ssse3ext_aes(int nd, size_t size, void **vv) { - raid_genX_ssse3ext(nd, size, vv, 5, 1); + if (raid_g23_x2_only(nd)) + raid_genX_ssse3ext_x2(nd, size, vv, 5); + else + raid_genX_ssse3ext_g23(nd, size, vv, 5); } #endif /* * GEN6 (hexa parity with Cauchy matrix) SSSE3 implementation + * using only Q *= 2. */ -void raid_gen6_ssse3_raid(int nd, size_t size, void **vv) +void raid_gen6_ssse3_x2(int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -1309,6 +2796,326 @@ void raid_gen6_ssse3_raid(int nd, size_t size, void **vv) raid_sse_end(); } +/* + * GEN6 (hexa parity with Cauchy matrix) SSSE3 implementation + * using the AES G23 Q transition. + */ +static __always_inline void raid_gen6_ssse3_g23(int nd, size_t size, + void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + uint8_t *s; + uint8_t *t; + uint8_t *u; + int d, l; + size_t i; + uint8_t buffer[2 * 16 + 16]; + uint8_t *pd = __align_ptr(buffer, 16); + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + s = v[nd + 3]; + t = v[nd + 4]; + u = v[nd + 5]; + + /* special case with only one data disk */ + if (l == 0) { + for (i = 0; i < 6; ++i) + memcpy(v[1 + i], v[0], size); + return; + } + + int g23_boundary = raid_g23_boundary(l - 1); + + raid_sse_begin(); + + for (i = 0; i < size; i += 16) { + int boundary = g23_boundary; + + /* last disk without the generator multiplication */ + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i])); + + /* P and Q are spilled */ + asm volatile ("movdqa %%xmm4,%0" : "=m" (pd[0])); + asm volatile ("movdqa %%xmm4,%0" : "=m" (pd[16])); + + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + + /* R */ + asm volatile ("movdqa %0,%%xmm0" : : "m" (raid_gfcauchypshufb[l][1][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1][0])); + asm volatile ("pshufb %xmm4,%xmm0"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm7,%xmm0"); + + /* S */ + asm volatile ("movdqa %0,%%xmm1" : : "m" (raid_gfcauchypshufb[l][2][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1][0])); + asm volatile ("pshufb %xmm4,%xmm1"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm7,%xmm1"); + + /* T */ + asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][3][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][3][1][0])); + asm volatile ("pshufb %xmm4,%xmm2"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm7,%xmm2"); + + /* U */ + asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][4][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][4][1][0])); + asm volatile ("pshufb %xmm4,%xmm3"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm7,%xmm3"); + + d = l - 1; + + while (boundary >= 0) { + /* pure x2 segment */ + for (; d > boundary; --d) { + asm volatile ("movdqa %0,%%xmm5" : : "m" (pd[0])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[16])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + + /* Q *= 2 */ + asm volatile ("pxor %xmm4,%xmm4"); + asm volatile ("pcmpgtb %xmm6,%xmm4"); + asm volatile ("paddb %xmm6,%xmm6"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pxor %xmm4,%xmm6"); + + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); + + asm volatile ("pxor %xmm4,%xmm5"); + asm volatile ("pxor %xmm4,%xmm6"); + asm volatile ("movdqa %%xmm5,%0" : "=m" (pd[0])); + asm volatile ("movdqa %%xmm6,%0" : "=m" (pd[16])); + + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + + /* R */ + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm0"); + asm volatile ("pxor %xmm7,%xmm0"); + + /* S */ + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm1"); + asm volatile ("pxor %xmm7,%xmm1"); + + /* T */ + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][3][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][3][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm2"); + asm volatile ("pxor %xmm7,%xmm2"); + + /* U */ + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][4][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][4][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm3"); + asm volatile ("pxor %xmm7,%xmm3"); + } + + /* + * d == boundary: Q *= 3. + * + * xmm6 = current Q + * xmm5 = saved old Q + */ + asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[16])); + asm volatile ("movdqa %xmm6,%xmm5"); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + + /* Q *= 2 */ + asm volatile ("pxor %xmm4,%xmm4"); + asm volatile ("pcmpgtb %xmm6,%xmm4"); + asm volatile ("paddb %xmm6,%xmm6"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pxor %xmm4,%xmm6"); + + /* Q *= 3 */ + asm volatile ("pxor %xmm5,%xmm6"); + + /* xmm5 is free again: load P */ + asm volatile ("movdqa %0,%%xmm5" : : "m" (pd[0])); + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); + + asm volatile ("pxor %xmm4,%xmm5"); + asm volatile ("pxor %xmm4,%xmm6"); + asm volatile ("movdqa %%xmm5,%0" : "=m" (pd[0])); + asm volatile ("movdqa %%xmm6,%0" : "=m" (pd[16])); + + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + + /* R */ + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm0"); + asm volatile ("pxor %xmm7,%xmm0"); + + /* S */ + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm1"); + asm volatile ("pxor %xmm7,%xmm1"); + + /* T */ + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][3][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][3][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm2"); + asm volatile ("pxor %xmm7,%xmm2"); + + /* U */ + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][4][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][4][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm3"); + asm volatile ("pxor %xmm7,%xmm3"); + + --d; + + boundary -= 51; + if (boundary < 50) + boundary = -1; + } + + /* + * Remaining intermediate disks contain no G23 boundary. + */ + for (; d > 0; --d) { + asm volatile ("movdqa %0,%%xmm5" : : "m" (pd[0])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[16])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + + asm volatile ("pxor %xmm4,%xmm4"); + asm volatile ("pcmpgtb %xmm6,%xmm4"); + asm volatile ("paddb %xmm6,%xmm6"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pxor %xmm4,%xmm6"); + + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); + + asm volatile ("pxor %xmm4,%xmm5"); + asm volatile ("pxor %xmm4,%xmm6"); + asm volatile ("movdqa %%xmm5,%0" : "=m" (pd[0])); + asm volatile ("movdqa %%xmm6,%0" : "=m" (pd[16])); + + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm0"); + asm volatile ("pxor %xmm7,%xmm0"); + + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm1"); + asm volatile ("pxor %xmm7,%xmm1"); + + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][3][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][3][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm2"); + asm volatile ("pxor %xmm7,%xmm2"); + + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][4][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][4][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm3"); + asm volatile ("pxor %xmm7,%xmm3"); + } + + /* + * D0 always follows Q *= 2. + */ + asm volatile ("movdqa %0,%%xmm5" : : "m" (pd[0])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[16])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + + asm volatile ("pxor %xmm4,%xmm4"); + asm volatile ("pcmpgtb %xmm6,%xmm4"); + asm volatile ("paddb %xmm6,%xmm6"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pxor %xmm4,%xmm6"); + + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i])); + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + asm volatile ("pxor %xmm4,%xmm2"); + asm volatile ("pxor %xmm4,%xmm3"); + asm volatile ("pxor %xmm4,%xmm5"); + asm volatile ("pxor %xmm4,%xmm6"); + + asm volatile ("movntdq %%xmm5,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm6,%0" : "=m" (q[i])); + asm volatile ("movntdq %%xmm0,%0" : "=m" (r[i])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (s[i])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (t[i])); + asm volatile ("movntdq %%xmm3,%0" : "=m" (u[i])); + } + + raid_sse_end(); +} + +void raid_gen6_ssse3_raid(int nd, size_t size, void **vv) +{ + raid_gen6_ssse3_x2(nd, size, vv); +} + +void raid_gen6_ssse3_aes(int nd, size_t size, void **vv) +{ + if (raid_g23_x2_only(nd)) + raid_gen6_ssse3_x2(nd, size, vv); + else + raid_gen6_ssse3_g23(nd, size, vv); +} + #ifdef CONFIG_X86_64 /* * GEN6 (hexa parity with Cauchy matrix) SSSE3 implementation @@ -1317,12 +3124,15 @@ void raid_gen6_ssse3_raid(int nd, size_t size, void **vv) */ void raid_gen6_ssse3ext_raid(int nd, size_t size, void **vv) { - raid_genX_ssse3ext(nd, size, vv, 6, 0); + raid_genX_ssse3ext_x2(nd, size, vv, 6); } void raid_gen6_ssse3ext_aes(int nd, size_t size, void **vv) { - raid_genX_ssse3ext(nd, size, vv, 6, 1); + if (raid_g23_x2_only(nd)) + raid_genX_ssse3ext_x2(nd, size, vv, 6); + else + raid_genX_ssse3ext_g23(nd, size, vv, 6); } #endif @@ -1905,7 +3715,9 @@ void raid_register_ssse3(void) raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3", raid_gen4_ssse3_raid, RAID_POLY_RAID); raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3", raid_gen4_ssse3_aes, RAID_POLY_AES); raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3", raid_gen5_ssse3_raid, RAID_POLY_RAID); + raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3", raid_gen5_ssse3_aes, RAID_POLY_AES); raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3", raid_gen6_ssse3_raid, RAID_POLY_RAID); + raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3", raid_gen6_ssse3_aes, RAID_POLY_AES); #ifdef CONFIG_X86_64 if (!raid_cpu_has_slow_extendedreg()) { raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3e", raid_gen3_ssse3ext_raid, RAID_POLY_RAID); diff --git a/raid/test.c b/raid/test.c index d3f06711..256ddc46 100644 --- a/raid/test.c +++ b/raid/test.c @@ -887,7 +887,6 @@ int raid_test_par(int mode, int nd, size_t size) /* setup all the available functions */ test_setup(1) = raid_gen1_int32; test_setup(1) = raid_gen1_int64; - test_setup(2) = raid_gen2_int8; if (mode == RAID_MODE_CAUCHY_AES) { test_setup(2) = raid_gen2_int32_aes; test_setup(2) = raid_gen2_int64_aes; diff --git a/raid/test/speedtest.c b/raid/test/speedtest.c index 591ff39a..f29d4eb6 100644 --- a/raid/test/speedtest.c +++ b/raid/test/speedtest.c @@ -232,12 +232,7 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8s", raid_gen_tag(RAID_ALGO_CAUCHY_PAR2)); fflush(stdout); - SPEED_START { - raid_gen2_int8(nd, size, v); - } SPEED_STOP - - printf("%8" PRIu64, ds / dt); - fflush(stdout); + printf("%8s", ""); if (mode == RAID_MODE_CAUCHY_AES) { SPEED_START { @@ -742,15 +737,17 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #endif } if (raid_cpu_has_ssse3()) { - if (mode == RAID_MODE_CAUCHY_RAID) { + if (mode == RAID_MODE_CAUCHY_AES) { + SPEED_START { + raid_gen5_ssse3_aes(nd, size, v); + } SPEED_STOP + } else { SPEED_START { raid_gen5_ssse3_raid(nd, size, v); } SPEED_STOP - - printf("%8" PRIu64, ds / dt); - } else { - printf("%8s", ""); } + + printf("%8" PRIu64, ds / dt); fflush(stdout); #ifdef CONFIG_X86_64 if (mode == RAID_MODE_CAUCHY_AES) { @@ -881,15 +878,17 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #endif } if (raid_cpu_has_ssse3()) { - if (mode == RAID_MODE_CAUCHY_RAID) { + if (mode == RAID_MODE_CAUCHY_AES) { + SPEED_START { + raid_gen6_ssse3_aes(nd, size, v); + } SPEED_STOP + } else { SPEED_START { raid_gen6_ssse3_raid(nd, size, v); } SPEED_STOP - - printf("%8" PRIu64, ds / dt); - } else { - printf("%8s", ""); } + + printf("%8" PRIu64, ds / dt); fflush(stdout); #ifdef CONFIG_X86_64 if (mode == RAID_MODE_CAUCHY_AES) {