From 7f6ef3b4cd5d023c0c33d774d4ecccec97b86153 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Thu, 20 Aug 2026 16:18:59 +0200 Subject: [PATCH] Eliminate stack buffers, pre-split delta nibbles and and inner loop in recX --- raid/avx2.c | 212 ++++++++++++++++++++++++++++--- raid/avx512.c | 169 ++++++++++++++++++++---- raid/gfni.c | 346 +++++++++++++++++++++++++++++++++++++++----------- raid/neon.c | 167 ++++++++++++++++++------ raid/ssse3.c | 305 +++++++++++++++++++++++++++++++++++++++++--- 5 files changed, 1022 insertions(+), 177 deletions(-) diff --git a/raid/avx2.c b/raid/avx2.c index b6de7caa..e3db58fd 100644 --- a/raid/avx2.c +++ b/raid/avx2.c @@ -1190,8 +1190,6 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) uint8_t *pa[RAID_PARITY_MAX]; uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; - uint8_t buffer[RAID_PARITY_MAX * 32 + 32]; - uint8_t *pd = __align_ptr(buffer, 32); size_t i; int j, k; @@ -1213,41 +1211,215 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) raid_avx_begin(); +#ifdef CONFIG_X86_64 + asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0])); + + for (i = 0; i < size; i += 32) { + /* delta */ + asm volatile ( + "movq 0(%2), %%rax\n" + "movq 0(%3), %%rbx\n" + "vmovdqa (%%rax, %1), %%ymm12\n" + "vmovdqa (%%rbx, %1), %%ymm13\n" + "vpxor %%ymm13, %%ymm12, %%ymm12\n" + "vpsrlw $4, %%ymm12, %%ymm1\n" + "vpand %%ymm15, %%ymm12, %%ymm0\n" + "vpand %%ymm15, %%ymm1, %%ymm1\n" + "cmpq $1, %0\n" + "jbe 1f\n" + + "movq 8(%2), %%rax\n" + "movq 8(%3), %%rbx\n" + "vmovdqa (%%rax, %1), %%ymm12\n" + "vmovdqa (%%rbx, %1), %%ymm13\n" + "vpxor %%ymm13, %%ymm12, %%ymm12\n" + "vpsrlw $4, %%ymm12, %%ymm3\n" + "vpand %%ymm15, %%ymm12, %%ymm2\n" + "vpand %%ymm15, %%ymm3, %%ymm3\n" + "cmpq $2, %0\n" + "jbe 1f\n" + + "movq 16(%2), %%rax\n" + "movq 16(%3), %%rbx\n" + "vmovdqa (%%rax, %1), %%ymm12\n" + "vmovdqa (%%rbx, %1), %%ymm13\n" + "vpxor %%ymm13, %%ymm12, %%ymm12\n" + "vpsrlw $4, %%ymm12, %%ymm5\n" + "vpand %%ymm15, %%ymm12, %%ymm4\n" + "vpand %%ymm15, %%ymm5, %%ymm5\n" + "cmpq $3, %0\n" + "jbe 1f\n" + + "movq 24(%2), %%rax\n" + "movq 24(%3), %%rbx\n" + "vmovdqa (%%rax, %1), %%ymm12\n" + "vmovdqa (%%rbx, %1), %%ymm13\n" + "vpxor %%ymm13, %%ymm12, %%ymm12\n" + "vpsrlw $4, %%ymm12, %%ymm7\n" + "vpand %%ymm15, %%ymm12, %%ymm6\n" + "vpand %%ymm15, %%ymm7, %%ymm7\n" + "cmpq $4, %0\n" + "jbe 1f\n" + + "movq 32(%2), %%rax\n" + "movq 32(%3), %%rbx\n" + "vmovdqa (%%rax, %1), %%ymm12\n" + "vmovdqa (%%rbx, %1), %%ymm13\n" + "vpxor %%ymm13, %%ymm12, %%ymm12\n" + "vpsrlw $4, %%ymm12, %%ymm9\n" + "vpand %%ymm15, %%ymm12, %%ymm8\n" + "vpand %%ymm15, %%ymm9, %%ymm9\n" + "cmpq $5, %0\n" + "jbe 1f\n" + + "movq 40(%2), %%rax\n" + "movq 40(%3), %%rbx\n" + "vmovdqa (%%rax, %1), %%ymm12\n" + "vmovdqa (%%rbx, %1), %%ymm13\n" + "vpxor %%ymm13, %%ymm12, %%ymm12\n" + "vpsrlw $4, %%ymm12, %%ymm11\n" + "vpand %%ymm15, %%ymm12, %%ymm10\n" + "vpand %%ymm15, %%ymm11, %%ymm11\n" + + "1:\n" + : + : "r" ((uint64_t)N), "r" (i), "r" (p), "r" (pa) + : "rax", "rbx", "cc", "memory" + ); + + /* reconstruct */ + for (j = 0; j < N; ++j) { + asm volatile ( + "movzbq 0(%2), %%rcx\n" + "shlq $5, %%rcx\n" + "vbroadcasti128 0(%4, %%rcx), %%ymm13\n" + "vbroadcasti128 16(%4, %%rcx), %%ymm14\n" + "vpshufb %%ymm0, %%ymm13, %%ymm13\n" + "vpshufb %%ymm1, %%ymm14, %%ymm14\n" + "cmpq $1, %0\n" + "jbe 1f\n" + + "movzbq 1(%2), %%rcx\n" + "shlq $5, %%rcx\n" + "vbroadcasti128 0(%4, %%rcx), %%ymm12\n" + "vpshufb %%ymm2, %%ymm12, %%ymm12\n" + "vpxor %%ymm12, %%ymm13, %%ymm13\n" + "vbroadcasti128 16(%4, %%rcx), %%ymm12\n" + "vpshufb %%ymm3, %%ymm12, %%ymm12\n" + "vpxor %%ymm12, %%ymm14, %%ymm14\n" + "cmpq $2, %0\n" + "jbe 1f\n" + + "movzbq 2(%2), %%rcx\n" + "shlq $5, %%rcx\n" + "vbroadcasti128 0(%4, %%rcx), %%ymm12\n" + "vpshufb %%ymm4, %%ymm12, %%ymm12\n" + "vpxor %%ymm12, %%ymm13, %%ymm13\n" + "vbroadcasti128 16(%4, %%rcx), %%ymm12\n" + "vpshufb %%ymm5, %%ymm12, %%ymm12\n" + "vpxor %%ymm12, %%ymm14, %%ymm14\n" + "cmpq $3, %0\n" + "jbe 1f\n" + + "movzbq 3(%2), %%rcx\n" + "shlq $5, %%rcx\n" + "vbroadcasti128 0(%4, %%rcx), %%ymm12\n" + "vpshufb %%ymm6, %%ymm12, %%ymm12\n" + "vpxor %%ymm12, %%ymm13, %%ymm13\n" + "vbroadcasti128 16(%4, %%rcx), %%ymm12\n" + "vpshufb %%ymm7, %%ymm12, %%ymm12\n" + "vpxor %%ymm12, %%ymm14, %%ymm14\n" + "cmpq $4, %0\n" + "jbe 1f\n" + + "movzbq 4(%2), %%rcx\n" + "shlq $5, %%rcx\n" + "vbroadcasti128 0(%4, %%rcx), %%ymm12\n" + "vpshufb %%ymm8, %%ymm12, %%ymm12\n" + "vpxor %%ymm12, %%ymm13, %%ymm13\n" + "vbroadcasti128 16(%4, %%rcx), %%ymm12\n" + "vpshufb %%ymm9, %%ymm12, %%ymm12\n" + "vpxor %%ymm12, %%ymm14, %%ymm14\n" + "cmpq $5, %0\n" + "jbe 1f\n" + + "movzbq 5(%2), %%rcx\n" + "shlq $5, %%rcx\n" + "vbroadcasti128 0(%4, %%rcx), %%ymm12\n" + "vpshufb %%ymm10, %%ymm12, %%ymm12\n" + "vpxor %%ymm12, %%ymm13, %%ymm13\n" + "vbroadcasti128 16(%4, %%rcx), %%ymm12\n" + "vpshufb %%ymm11, %%ymm12, %%ymm12\n" + "vpxor %%ymm12, %%ymm14, %%ymm14\n" + + "1:\n" + "vpxor %%ymm14, %%ymm13, %%ymm13\n" + "movq %3, %%rax\n" + "vmovdqa %%ymm13, (%%rax, %1)\n" + : + : "r" ((uint64_t)N), "r" (i), "r" (&V[j * N]), "r" (pa[j]), "r" (raid_gfmulpshufb) + : "rax", "rcx", "cc", "memory" + ); + } + } +#else /* CONFIG_X86_32 */ + uint8_t buffer_low[RAID_PARITY_MAX * 32 + 32]; + uint8_t buffer_high[RAID_PARITY_MAX * 32 + 32]; + uint8_t *pd_low = __align_ptr(buffer_low, 32); + uint8_t *pd_high = __align_ptr(buffer_high, 32); + asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 32) { /* delta */ for (j = 0; j < N; ++j) { - asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[j][i])); - asm volatile ("vmovdqa %0,%%ymm1" : : "m" (pa[j][i])); - asm volatile ("vpxor %ymm1,%ymm0,%ymm0"); - asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pd[j * 32])); + asm volatile ( + "vmovdqa %2, %%ymm0\n" + "vmovdqa %3, %%ymm1\n" + "vpxor %%ymm1, %%ymm0, %%ymm0\n" + "vpsrlw $4, %%ymm0, %%ymm1\n" + "vpand %%ymm7, %%ymm0, %%ymm0\n" + "vpand %%ymm7, %%ymm1, %%ymm1\n" + "vmovdqa %%ymm0, %0\n" + "vmovdqa %%ymm1, %1\n" + : "=m" (pd_low[j * 32]), "=m" (pd_high[j * 32]) + : "m" (p[j][i]), "m" (pa[j][i]) + ); } /* reconstruct */ for (j = 0; j < N; ++j) { - asm volatile ("vpxor %ymm0,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm1,%ymm1,%ymm1"); + asm volatile ( + "vpxor %%ymm0, %%ymm0, %%ymm0\n" + "vpxor %%ymm1, %%ymm1, %%ymm1\n" + ); for (k = 0; k < N; ++k) { uint8_t m = V[j * N + k]; - asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (raid_gfmulpshufb[m][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (raid_gfmulpshufb[m][1][0])); - asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pd[k * 32])); - asm volatile ("vpsrlw $4,%ymm4,%ymm5"); - asm volatile ("vpand %ymm7,%ymm4,%ymm4"); - asm volatile ("vpand %ymm7,%ymm5,%ymm5"); - asm volatile ("vpshufb %ymm4,%ymm2,%ymm2"); - asm volatile ("vpshufb %ymm5,%ymm3,%ymm3"); - asm volatile ("vpxor %ymm2,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm3,%ymm1,%ymm1"); + asm volatile ( + "vbroadcasti128 %0, %%ymm2\n" + "vbroadcasti128 %1, %%ymm3\n" + "vmovdqa %2, %%ymm4\n" + "vmovdqa %3, %%ymm5\n" + "vpshufb %%ymm4, %%ymm2, %%ymm2\n" + "vpshufb %%ymm5, %%ymm3, %%ymm3\n" + "vpxor %%ymm2, %%ymm0, %%ymm0\n" + "vpxor %%ymm3, %%ymm1, %%ymm1\n" + : + : "m" (raid_gfmulpshufb[m][0][0]), "m" (raid_gfmulpshufb[m][1][0]), + "m" (pd_low[k * 32]), "m" (pd_high[k * 32]) + ); } - asm volatile ("vpxor %ymm1,%ymm0,%ymm0"); - asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[j][i])); + asm volatile ( + "vpxor %%ymm1, %%ymm0, %%ymm0\n" + "vmovdqa %%ymm0, %0\n" + : "=m" (pa[j][i]) + ); } } +#endif raid_avx_end(); } diff --git a/raid/avx512.c b/raid/avx512.c index adb046f8..84c10df4 100644 --- a/raid/avx512.c +++ b/raid/avx512.c @@ -369,8 +369,6 @@ void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv uint8_t *pa[RAID_PARITY_MAX]; uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; - uint8_t buffer[RAID_PARITY_MAX * 64 + 64]; - uint8_t *pd = __align_ptr(buffer, 64); size_t i; int j, k; @@ -392,35 +390,154 @@ void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv raid_avx_begin(); - asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 64) { /* delta */ - for (j = 0; j < N; ++j) { - asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[j][i])); - asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (pa[j][i])); - asm volatile ("vpxorq %zmm1,%zmm0,%zmm0"); - asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pd[j * 64])); - } + asm volatile ( + "movq 0(%2), %%rax\n" + "movq 0(%3), %%rbx\n" + "vmovdqa64 (%%rax, %1), %%zmm12\n" + "vmovdqa64 (%%rbx, %1), %%zmm13\n" + "vpxorq %%zmm13, %%zmm12, %%zmm12\n" + "vpsrlw $4, %%zmm12, %%zmm1\n" + "vpandq %%zmm31, %%zmm12, %%zmm0\n" + "vpandq %%zmm31, %%zmm1, %%zmm1\n" + "cmpq $1, %0\n" + "jbe 1f\n" + + "movq 8(%2), %%rax\n" + "movq 8(%3), %%rbx\n" + "vmovdqa64 (%%rax, %1), %%zmm12\n" + "vmovdqa64 (%%rbx, %1), %%zmm13\n" + "vpxorq %%zmm13, %%zmm12, %%zmm12\n" + "vpsrlw $4, %%zmm12, %%zmm3\n" + "vpandq %%zmm31, %%zmm12, %%zmm2\n" + "vpandq %%zmm31, %%zmm3, %%zmm3\n" + "cmpq $2, %0\n" + "jbe 1f\n" + + "movq 16(%2), %%rax\n" + "movq 16(%3), %%rbx\n" + "vmovdqa64 (%%rax, %1), %%zmm12\n" + "vmovdqa64 (%%rbx, %1), %%zmm13\n" + "vpxorq %%zmm13, %%zmm12, %%zmm12\n" + "vpsrlw $4, %%zmm12, %%zmm5\n" + "vpandq %%zmm31, %%zmm12, %%zmm4\n" + "vpandq %%zmm31, %%zmm5, %%zmm5\n" + "cmpq $3, %0\n" + "jbe 1f\n" + + "movq 24(%2), %%rax\n" + "movq 24(%3), %%rbx\n" + "vmovdqa64 (%%rax, %1), %%zmm12\n" + "vmovdqa64 (%%rbx, %1), %%zmm13\n" + "vpxorq %%zmm13, %%zmm12, %%zmm12\n" + "vpsrlw $4, %%zmm12, %%zmm7\n" + "vpandq %%zmm31, %%zmm12, %%zmm6\n" + "vpandq %%zmm31, %%zmm7, %%zmm7\n" + "cmpq $4, %0\n" + "jbe 1f\n" + + "movq 32(%2), %%rax\n" + "movq 32(%3), %%rbx\n" + "vmovdqa64 (%%rax, %1), %%zmm12\n" + "vmovdqa64 (%%rbx, %1), %%zmm13\n" + "vpxorq %%zmm13, %%zmm12, %%zmm12\n" + "vpsrlw $4, %%zmm12, %%zmm9\n" + "vpandq %%zmm31, %%zmm12, %%zmm8\n" + "vpandq %%zmm31, %%zmm9, %%zmm9\n" + "cmpq $5, %0\n" + "jbe 1f\n" + + "movq 40(%2), %%rax\n" + "movq 40(%3), %%rbx\n" + "vmovdqa64 (%%rax, %1), %%zmm12\n" + "vmovdqa64 (%%rbx, %1), %%zmm13\n" + "vpxorq %%zmm13, %%zmm12, %%zmm12\n" + "vpsrlw $4, %%zmm12, %%zmm11\n" + "vpandq %%zmm31, %%zmm12, %%zmm10\n" + "vpandq %%zmm31, %%zmm11, %%zmm11\n" + + "1:\n" + : + : "r" ((uint64_t)N), "r" (i), "r" (p), "r" (pa) + : "rax", "rbx", "cc", "memory" + ); + /* reconstruct */ for (j = 0; j < N; ++j) { - asm volatile ("vpxorq %zmm0,%zmm0,%zmm0"); - asm volatile ("vpxorq %zmm1,%zmm1,%zmm1"); - for (k = 0; k < N; ++k) { - uint8_t m = V[j * N + k]; - asm volatile ("vbroadcasti32x4 %0,%%zmm2" : : "m" (raid_gfmulpshufb[m][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (raid_gfmulpshufb[m][1][0])); - asm volatile ("vmovdqa64 %0,%%zmm4" : : "m" (pd[k * 64])); - asm volatile ("vpsrlw $4,%zmm4,%zmm5"); - asm volatile ("vpandq %zmm7,%zmm4,%zmm4"); - asm volatile ("vpandq %zmm7,%zmm5,%zmm5"); - asm volatile ("vpshufb %zmm4,%zmm2,%zmm2"); - asm volatile ("vpshufb %zmm5,%zmm3,%zmm3"); - asm volatile ("vpxorq %zmm2,%zmm0,%zmm0"); - asm volatile ("vpxorq %zmm3,%zmm1,%zmm1"); - } - asm volatile ("vpxorq %zmm1,%zmm0,%zmm0"); - asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[j][i])); + asm volatile ( + "movzbq 0(%2), %%rcx\n" + "shlq $5, %%rcx\n" + "vbroadcasti32x4 0(%4, %%rcx), %%zmm14\n" + "vbroadcasti32x4 16(%4, %%rcx), %%zmm15\n" + "vpshufb %%zmm0, %%zmm14, %%zmm14\n" + "vpshufb %%zmm1, %%zmm15, %%zmm15\n" + "cmpq $1, %0\n" + "jbe 1f\n" + + "movzbq 1(%2), %%rcx\n" + "shlq $5, %%rcx\n" + "vbroadcasti32x4 0(%4, %%rcx), %%zmm12\n" + "vbroadcasti32x4 16(%4, %%rcx), %%zmm13\n" + "vpshufb %%zmm2, %%zmm12, %%zmm12\n" + "vpshufb %%zmm3, %%zmm13, %%zmm13\n" + "vpxorq %%zmm12, %%zmm14, %%zmm14\n" + "vpxorq %%zmm13, %%zmm15, %%zmm15\n" + "cmpq $2, %0\n" + "jbe 1f\n" + + "movzbq 2(%2), %%rcx\n" + "shlq $5, %%rcx\n" + "vbroadcasti32x4 0(%4, %%rcx), %%zmm12\n" + "vbroadcasti32x4 16(%4, %%rcx), %%zmm13\n" + "vpshufb %%zmm4, %%zmm12, %%zmm12\n" + "vpshufb %%zmm5, %%zmm13, %%zmm13\n" + "vpxorq %%zmm12, %%zmm14, %%zmm14\n" + "vpxorq %%zmm13, %%zmm15, %%zmm15\n" + "cmpq $3, %0\n" + "jbe 1f\n" + + "movzbq 3(%2), %%rcx\n" + "shlq $5, %%rcx\n" + "vbroadcasti32x4 0(%4, %%rcx), %%zmm12\n" + "vbroadcasti32x4 16(%4, %%rcx), %%zmm13\n" + "vpshufb %%zmm6, %%zmm12, %%zmm12\n" + "vpshufb %%zmm7, %%zmm13, %%zmm13\n" + "vpxorq %%zmm12, %%zmm14, %%zmm14\n" + "vpxorq %%zmm13, %%zmm15, %%zmm15\n" + "cmpq $4, %0\n" + "jbe 1f\n" + + "movzbq 4(%2), %%rcx\n" + "shlq $5, %%rcx\n" + "vbroadcasti32x4 0(%4, %%rcx), %%zmm12\n" + "vbroadcasti32x4 16(%4, %%rcx), %%zmm13\n" + "vpshufb %%zmm8, %%zmm12, %%zmm12\n" + "vpshufb %%zmm9, %%zmm13, %%zmm13\n" + "vpxorq %%zmm12, %%zmm14, %%zmm14\n" + "vpxorq %%zmm13, %%zmm15, %%zmm15\n" + "cmpq $5, %0\n" + "jbe 1f\n" + + "movzbq 5(%2), %%rcx\n" + "shlq $5, %%rcx\n" + "vbroadcasti32x4 0(%4, %%rcx), %%zmm12\n" + "vbroadcasti32x4 16(%4, %%rcx), %%zmm13\n" + "vpshufb %%zmm10, %%zmm12, %%zmm12\n" + "vpshufb %%zmm11, %%zmm13, %%zmm13\n" + "vpxorq %%zmm12, %%zmm14, %%zmm14\n" + "vpxorq %%zmm13, %%zmm15, %%zmm15\n" + + "1:\n" + "vpxorq %%zmm15, %%zmm14, %%zmm14\n" + "movq %3, %%rax\n" + "vmovdqa64 %%zmm14, (%%rax, %1)\n" + : + : "r" ((uint64_t)N), "r" (i), "r" (&V[j * N]), "r" (pa[j]), "r" (raid_gfmulpshufb) + : "rax", "rcx", "cc", "memory" + ); } } diff --git a/raid/gfni.c b/raid/gfni.c index 0d05b297..81d47dff 100644 --- a/raid/gfni.c +++ b/raid/gfni.c @@ -978,16 +978,18 @@ void raid_recX_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void uint8_t *pa[RAID_PARITY_MAX]; uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; - uint8_t buffer[RAID_PARITY_MAX * 64 + 64]; - uint8_t *pd = __align_ptr(buffer, 64); size_t i; int j, k; + /* setup the coefficients matrix */ for (j = 0; j < N; ++j) for (k = 0; k < N; ++k) G[j * N + k] = A(ip[j], id[k]); + /* invert it to solve the system of linear equations */ raid_invert(G, V, N); + + /* compute delta parity */ raid_delta_gen(N, id, ip, nd, size, vv); for (j = 0; j < N; ++j) { @@ -999,38 +1001,138 @@ void raid_recX_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void for (i = 0; i < size; i += 64) { /* delta */ - for (j = 0; j < N; ++j) { - asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[j][i])); - asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[j][i + 32])); + asm volatile ( + "movq 0(%2), %%rax\n" + "movq 0(%3), %%rbx\n" + "vmovdqa 0(%%rax, %1), %%ymm0\n" + "vmovdqa 32(%%rax, %1), %%ymm1\n" + "vmovdqa 0(%%rbx, %1), %%ymm14\n" + "vmovdqa 32(%%rbx, %1), %%ymm15\n" + "vpxor %%ymm14, %%ymm0, %%ymm0\n" + "vpxor %%ymm15, %%ymm1, %%ymm1\n" + "cmpq $1, %0\n" + "jbe 1f\n" - asm volatile ("vmovdqa %0,%%ymm2" : : "m" (pa[j][i])); - asm volatile ("vmovdqa %0,%%ymm3" : : "m" (pa[j][i + 32])); + "movq 8(%2), %%rax\n" + "movq 8(%3), %%rbx\n" + "vmovdqa 0(%%rax, %1), %%ymm2\n" + "vmovdqa 32(%%rax, %1), %%ymm3\n" + "vmovdqa 0(%%rbx, %1), %%ymm14\n" + "vmovdqa 32(%%rbx, %1), %%ymm15\n" + "vpxor %%ymm14, %%ymm2, %%ymm2\n" + "vpxor %%ymm15, %%ymm3, %%ymm3\n" + "cmpq $2, %0\n" + "jbe 1f\n" - asm volatile ("vpxor %ymm2,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm3,%ymm1,%ymm1"); + "movq 16(%2), %%rax\n" + "movq 16(%3), %%rbx\n" + "vmovdqa 0(%%rax, %1), %%ymm4\n" + "vmovdqa 32(%%rax, %1), %%ymm5\n" + "vmovdqa 0(%%rbx, %1), %%ymm14\n" + "vmovdqa 32(%%rbx, %1), %%ymm15\n" + "vpxor %%ymm14, %%ymm4, %%ymm4\n" + "vpxor %%ymm15, %%ymm5, %%ymm5\n" + "cmpq $3, %0\n" + "jbe 1f\n" - asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pd[j * 64])); - asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pd[j * 64 + 32])); - } + "movq 24(%2), %%rax\n" + "movq 24(%3), %%rbx\n" + "vmovdqa 0(%%rax, %1), %%ymm6\n" + "vmovdqa 32(%%rax, %1), %%ymm7\n" + "vmovdqa 0(%%rbx, %1), %%ymm14\n" + "vmovdqa 32(%%rbx, %1), %%ymm15\n" + "vpxor %%ymm14, %%ymm6, %%ymm6\n" + "vpxor %%ymm15, %%ymm7, %%ymm7\n" + "cmpq $4, %0\n" + "jbe 1f\n" + + "movq 32(%2), %%rax\n" + "movq 32(%3), %%rbx\n" + "vmovdqa 0(%%rax, %1), %%ymm8\n" + "vmovdqa 32(%%rax, %1), %%ymm9\n" + "vmovdqa 0(%%rbx, %1), %%ymm14\n" + "vmovdqa 32(%%rbx, %1), %%ymm15\n" + "vpxor %%ymm14, %%ymm8, %%ymm8\n" + "vpxor %%ymm15, %%ymm9, %%ymm9\n" + "cmpq $5, %0\n" + "jbe 1f\n" + + "movq 40(%2), %%rax\n" + "movq 40(%3), %%rbx\n" + "vmovdqa 0(%%rax, %1), %%ymm10\n" + "vmovdqa 32(%%rax, %1), %%ymm11\n" + "vmovdqa 0(%%rbx, %1), %%ymm14\n" + "vmovdqa 32(%%rbx, %1), %%ymm15\n" + "vpxor %%ymm14, %%ymm10, %%ymm10\n" + "vpxor %%ymm15, %%ymm11, %%ymm11\n" + + "1:\n" + : + : "r" ((uint64_t)N), "r" (i), "r" (p), "r" (pa) + : "rax", "rbx", "cc", "memory" + ); /* reconstruct */ for (j = 0; j < N; ++j) { - asm volatile ("vpxor %ymm0,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm1,%ymm1,%ymm1"); - for (k = 0; k < N; ++k) { - asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pd[k * 64])); - asm volatile ("vmovdqa %0,%%ymm5" : : "m" (pd[k * 64 + 32])); + asm volatile ( + "movzbq 0(%2), %%rcx\n" + "vpbroadcastq (%4, %%rcx, 8), %%ymm14\n" + "vgf2p8affineqb $0, %%ymm14, %%ymm0, %%ymm12\n" + "vgf2p8affineqb $0, %%ymm14, %%ymm1, %%ymm13\n" + "cmpq $1, %0\n" + "jbe 1f\n" - asm volatile ("vpbroadcastq %0,%%ymm2" : : "m" (raid_gfaffine_raid[V[j * N + k]][0])); + "movzbq 1(%2), %%rcx\n" + "vpbroadcastq (%4, %%rcx, 8), %%ymm14\n" + "vgf2p8affineqb $0, %%ymm14, %%ymm2, %%ymm15\n" + "vpxor %%ymm15, %%ymm12, %%ymm12\n" + "vgf2p8affineqb $0, %%ymm14, %%ymm3, %%ymm15\n" + "vpxor %%ymm15, %%ymm13, %%ymm13\n" + "cmpq $2, %0\n" + "jbe 1f\n" - asm volatile ("vgf2p8affineqb $0,%ymm2,%ymm4,%ymm3"); - asm volatile ("vpxor %ymm3,%ymm0,%ymm0"); + "movzbq 2(%2), %%rcx\n" + "vpbroadcastq (%4, %%rcx, 8), %%ymm14\n" + "vgf2p8affineqb $0, %%ymm14, %%ymm4, %%ymm15\n" + "vpxor %%ymm15, %%ymm12, %%ymm12\n" + "vgf2p8affineqb $0, %%ymm14, %%ymm5, %%ymm15\n" + "vpxor %%ymm15, %%ymm13, %%ymm13\n" + "cmpq $3, %0\n" + "jbe 1f\n" - asm volatile ("vgf2p8affineqb $0,%ymm2,%ymm5,%ymm3"); - asm volatile ("vpxor %ymm3,%ymm1,%ymm1"); - } - asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[j][i])); - asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[j][i + 32])); + "movzbq 3(%2), %%rcx\n" + "vpbroadcastq (%4, %%rcx, 8), %%ymm14\n" + "vgf2p8affineqb $0, %%ymm14, %%ymm6, %%ymm15\n" + "vpxor %%ymm15, %%ymm12, %%ymm12\n" + "vgf2p8affineqb $0, %%ymm14, %%ymm7, %%ymm15\n" + "vpxor %%ymm15, %%ymm13, %%ymm13\n" + "cmpq $4, %0\n" + "jbe 1f\n" + + "movzbq 4(%2), %%rcx\n" + "vpbroadcastq (%4, %%rcx, 8), %%ymm14\n" + "vgf2p8affineqb $0, %%ymm14, %%ymm8, %%ymm15\n" + "vpxor %%ymm15, %%ymm12, %%ymm12\n" + "vgf2p8affineqb $0, %%ymm14, %%ymm9, %%ymm15\n" + "vpxor %%ymm15, %%ymm13, %%ymm13\n" + "cmpq $5, %0\n" + "jbe 1f\n" + + "movzbq 5(%2), %%rcx\n" + "vpbroadcastq (%4, %%rcx, 8), %%ymm14\n" + "vgf2p8affineqb $0, %%ymm14, %%ymm10, %%ymm15\n" + "vpxor %%ymm15, %%ymm12, %%ymm12\n" + "vgf2p8affineqb $0, %%ymm14, %%ymm11, %%ymm15\n" + "vpxor %%ymm15, %%ymm13, %%ymm13\n" + + "1:\n" + "movq %3, %%rax\n" + "vmovdqa %%ymm12, 0(%%rax, %1)\n" + "vmovdqa %%ymm13, 32(%%rax, %1)\n" + : + : "r" ((uint64_t)N), "r" (i), "r" (&V[j * N]), "r" (pa[j]), "r" (raid_gfaffine_raid) + : "rax", "rcx", "cc", "memory" + ); } } @@ -1121,7 +1223,7 @@ void raid_recX_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, vo "1:\n" : : "r" ((uint64_t)N), "r" (i), "r" (p), "r" (pa) - : "rax", "rbx" + : "rax", "rbx", "cc", "memory" ); /* reconstruct */ @@ -1171,7 +1273,7 @@ void raid_recX_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, vo "vmovdqa64 %%zmm7, (%%rax, %1)\n" : : "r" ((uint64_t)N), "r" (i), "r" (&V[j * N]), "r" (pa[j]), "r" (raid_gfaffine_raid) - : "rax", "rcx" + : "rax", "rcx", "cc", "memory" ); } } @@ -1295,6 +1397,10 @@ void raid_rec2_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void } raid_avx_begin(); + asm volatile ("vpbroadcastb %0,%%ymm10" : : "m" (V[0])); + asm volatile ("vpbroadcastb %0,%%ymm11" : : "m" (V[1])); + asm volatile ("vpbroadcastb %0,%%ymm12" : : "m" (V[2])); + asm volatile ("vpbroadcastb %0,%%ymm13" : : "m" (V[3])); for (i = 0; i < size; i += 64) { asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i])); @@ -1315,27 +1421,23 @@ void raid_rec2_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void asm volatile ("vpxor %ymm6,%ymm2,%ymm2"); asm volatile ("vpxor %ymm7,%ymm3,%ymm3"); - asm volatile ("vpbroadcastb %0,%%ymm8" : : "m" (V[0])); - asm volatile ("vgf2p8mulb %ymm0,%ymm8,%ymm4"); - asm volatile ("vgf2p8mulb %ymm1,%ymm8,%ymm5"); + asm volatile ("vgf2p8mulb %ymm0,%ymm10,%ymm4"); + asm volatile ("vgf2p8mulb %ymm1,%ymm10,%ymm5"); - asm volatile ("vpbroadcastb %0,%%ymm8" : : "m" (V[1])); - asm volatile ("vgf2p8mulb %ymm2,%ymm8,%ymm9"); + asm volatile ("vgf2p8mulb %ymm2,%ymm11,%ymm9"); asm volatile ("vpxor %ymm9,%ymm4,%ymm4"); - asm volatile ("vgf2p8mulb %ymm3,%ymm8,%ymm9"); + asm volatile ("vgf2p8mulb %ymm3,%ymm11,%ymm9"); asm volatile ("vpxor %ymm9,%ymm5,%ymm5"); asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pa[0][i])); asm volatile ("vmovdqa %%ymm5,%0" : "=m" (pa[0][i + 32])); - asm volatile ("vpbroadcastb %0,%%ymm8" : : "m" (V[2])); - asm volatile ("vgf2p8mulb %ymm0,%ymm8,%ymm6"); - asm volatile ("vgf2p8mulb %ymm1,%ymm8,%ymm7"); + asm volatile ("vgf2p8mulb %ymm0,%ymm12,%ymm6"); + asm volatile ("vgf2p8mulb %ymm1,%ymm12,%ymm7"); - asm volatile ("vpbroadcastb %0,%%ymm8" : : "m" (V[3])); - asm volatile ("vgf2p8mulb %ymm2,%ymm8,%ymm9"); + asm volatile ("vgf2p8mulb %ymm2,%ymm13,%ymm9"); asm volatile ("vpxor %ymm9,%ymm6,%ymm6"); - asm volatile ("vgf2p8mulb %ymm3,%ymm8,%ymm9"); + asm volatile ("vgf2p8mulb %ymm3,%ymm13,%ymm9"); asm volatile ("vpxor %ymm9,%ymm7,%ymm7"); asm volatile ("vmovdqa %%ymm6,%0" : "=m" (pa[1][i])); @@ -1372,6 +1474,10 @@ void raid_rec2_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, voi } raid_avx_begin(); + asm volatile ("vpbroadcastb %0,%%zmm8" : : "m" (V[0])); + asm volatile ("vpbroadcastb %0,%%zmm9" : : "m" (V[1])); + asm volatile ("vpbroadcastb %0,%%zmm10" : : "m" (V[2])); + asm volatile ("vpbroadcastb %0,%%zmm11" : : "m" (V[3])); for (i = 0; i < size; i += 64) { asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[0][i])); @@ -1383,24 +1489,20 @@ void raid_rec2_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, voi asm volatile ("vpxorq %zmm6,%zmm6,%zmm6"); - asm volatile ("vpbroadcastb %0,%%zmm2" : : "m" (V[0])); - asm volatile ("vgf2p8mulb %zmm0,%zmm2,%zmm2"); + asm volatile ("vgf2p8mulb %zmm0,%zmm8,%zmm2"); asm volatile ("vpxorq %zmm2,%zmm6,%zmm6"); - asm volatile ("vpbroadcastb %0,%%zmm3" : : "m" (V[1])); - asm volatile ("vgf2p8mulb %zmm1,%zmm3,%zmm3"); + asm volatile ("vgf2p8mulb %zmm1,%zmm9,%zmm3"); asm volatile ("vpxorq %zmm3,%zmm6,%zmm6"); asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (pa[0][i])); asm volatile ("vpxorq %zmm6,%zmm6,%zmm6"); - asm volatile ("vpbroadcastb %0,%%zmm2" : : "m" (V[2])); - asm volatile ("vgf2p8mulb %zmm0,%zmm2,%zmm2"); + asm volatile ("vgf2p8mulb %zmm0,%zmm10,%zmm2"); asm volatile ("vpxorq %zmm2,%zmm6,%zmm6"); - asm volatile ("vpbroadcastb %0,%%zmm3" : : "m" (V[3])); - asm volatile ("vgf2p8mulb %zmm1,%zmm3,%zmm3"); + asm volatile ("vgf2p8mulb %zmm1,%zmm11,%zmm3"); asm volatile ("vpxorq %zmm3,%zmm6,%zmm6"); asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (pa[1][i])); @@ -1420,16 +1522,18 @@ void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void uint8_t *pa[RAID_PARITY_MAX]; uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; - uint8_t buffer[RAID_PARITY_MAX * 64 + 64]; - uint8_t *pd = __align_ptr(buffer, 64); size_t i; int j, k; + /* setup the coefficients matrix */ for (j = 0; j < N; ++j) for (k = 0; k < N; ++k) G[j * N + k] = A(ip[j], id[k]); + /* invert it to solve the system of linear equations */ raid_invert(G, V, N); + + /* compute delta parity */ raid_delta_gen(N, id, ip, nd, size, vv); for (j = 0; j < N; ++j) { @@ -1441,38 +1545,132 @@ void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void for (i = 0; i < size; i += 64) { /* delta */ - for (j = 0; j < N; ++j) { - asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[j][i])); - asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[j][i + 32])); + asm volatile ( + "movq 0(%2), %%rax\n" + "movq 0(%3), %%rbx\n" + "vmovdqa 0(%%rax, %1), %%ymm0\n" + "vmovdqa 32(%%rax, %1), %%ymm1\n" + "vmovdqa 0(%%rbx, %1), %%ymm14\n" + "vmovdqa 32(%%rbx, %1), %%ymm15\n" + "vpxor %%ymm14, %%ymm0, %%ymm0\n" + "vpxor %%ymm15, %%ymm1, %%ymm1\n" + "cmpq $1, %0\n" + "jbe 1f\n" - asm volatile ("vmovdqa %0,%%ymm2" : : "m" (pa[j][i])); - asm volatile ("vmovdqa %0,%%ymm3" : : "m" (pa[j][i + 32])); + "movq 8(%2), %%rax\n" + "movq 8(%3), %%rbx\n" + "vmovdqa 0(%%rax, %1), %%ymm2\n" + "vmovdqa 32(%%rax, %1), %%ymm3\n" + "vmovdqa 0(%%rbx, %1), %%ymm14\n" + "vmovdqa 32(%%rbx, %1), %%ymm15\n" + "vpxor %%ymm14, %%ymm2, %%ymm2\n" + "vpxor %%ymm15, %%ymm3, %%ymm3\n" + "cmpq $2, %0\n" + "jbe 1f\n" - asm volatile ("vpxor %ymm2,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm3,%ymm1,%ymm1"); + "movq 16(%2), %%rax\n" + "movq 16(%3), %%rbx\n" + "vmovdqa 0(%%rax, %1), %%ymm4\n" + "vmovdqa 32(%%rax, %1), %%ymm5\n" + "vmovdqa 0(%%rbx, %1), %%ymm14\n" + "vmovdqa 32(%%rbx, %1), %%ymm15\n" + "vpxor %%ymm14, %%ymm4, %%ymm4\n" + "vpxor %%ymm15, %%ymm5, %%ymm5\n" + "cmpq $3, %0\n" + "jbe 1f\n" - asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pd[j * 64])); - asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pd[j * 64 + 32])); - } + "movq 24(%2), %%rax\n" + "movq 24(%3), %%rbx\n" + "vmovdqa 0(%%rax, %1), %%ymm6\n" + "vmovdqa 32(%%rax, %1), %%ymm7\n" + "vmovdqa 0(%%rbx, %1), %%ymm14\n" + "vmovdqa 32(%%rbx, %1), %%ymm15\n" + "vpxor %%ymm14, %%ymm6, %%ymm6\n" + "vpxor %%ymm15, %%ymm7, %%ymm7\n" + "cmpq $4, %0\n" + "jbe 1f\n" + + "movq 32(%2), %%rax\n" + "movq 32(%3), %%rbx\n" + "vmovdqa 0(%%rax, %1), %%ymm8\n" + "vmovdqa 32(%%rax, %1), %%ymm9\n" + "vmovdqa 0(%%rbx, %1), %%ymm14\n" + "vmovdqa 32(%%rbx, %1), %%ymm15\n" + "vpxor %%ymm14, %%ymm8, %%ymm8\n" + "vpxor %%ymm15, %%ymm9, %%ymm9\n" + "cmpq $5, %0\n" + "jbe 1f\n" + + "movq 40(%2), %%rax\n" + "movq 40(%3), %%rbx\n" + "vmovdqa 0(%%rax, %1), %%ymm10\n" + "vmovdqa 32(%%rax, %1), %%ymm11\n" + "vmovdqa 0(%%rbx, %1), %%ymm14\n" + "vmovdqa 32(%%rbx, %1), %%ymm15\n" + "vpxor %%ymm14, %%ymm10, %%ymm10\n" + "vpxor %%ymm15, %%ymm11, %%ymm11\n" + + "1:\n" + : + : "r" ((uint64_t)N), "r" (i), "r" (p), "r" (pa) + : "rax", "rbx", "cc", "memory" + ); /* reconstruct */ for (j = 0; j < N; ++j) { - asm volatile ("vpxor %ymm0,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm1,%ymm1,%ymm1"); - for (k = 0; k < N; ++k) { - asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pd[k * 64])); - asm volatile ("vmovdqa %0,%%ymm5" : : "m" (pd[k * 64 + 32])); + asm volatile ( + "vpbroadcastb 0(%2), %%ymm14\n" + "vgf2p8mulb %%ymm0, %%ymm14, %%ymm12\n" + "vgf2p8mulb %%ymm1, %%ymm14, %%ymm13\n" + "cmpq $1, %0\n" + "jbe 1f\n" - asm volatile ("vpbroadcastb %0,%%ymm2" : : "m" (V[j * N + k])); + "vpbroadcastb 1(%2), %%ymm14\n" + "vgf2p8mulb %%ymm2, %%ymm14, %%ymm15\n" + "vpxor %%ymm15, %%ymm12, %%ymm12\n" + "vgf2p8mulb %%ymm3, %%ymm14, %%ymm15\n" + "vpxor %%ymm15, %%ymm13, %%ymm13\n" + "cmpq $2, %0\n" + "jbe 1f\n" - asm volatile ("vgf2p8mulb %ymm4,%ymm2,%ymm3"); - asm volatile ("vpxor %ymm3,%ymm0,%ymm0"); + "vpbroadcastb 2(%2), %%ymm14\n" + "vgf2p8mulb %%ymm4, %%ymm14, %%ymm15\n" + "vpxor %%ymm15, %%ymm12, %%ymm12\n" + "vgf2p8mulb %%ymm5, %%ymm14, %%ymm15\n" + "vpxor %%ymm15, %%ymm13, %%ymm13\n" + "cmpq $3, %0\n" + "jbe 1f\n" - asm volatile ("vgf2p8mulb %ymm5,%ymm2,%ymm3"); - asm volatile ("vpxor %ymm3,%ymm1,%ymm1"); - } - asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[j][i])); - asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[j][i + 32])); + "vpbroadcastb 3(%2), %%ymm14\n" + "vgf2p8mulb %%ymm6, %%ymm14, %%ymm15\n" + "vpxor %%ymm15, %%ymm12, %%ymm12\n" + "vgf2p8mulb %%ymm7, %%ymm14, %%ymm15\n" + "vpxor %%ymm15, %%ymm13, %%ymm13\n" + "cmpq $4, %0\n" + "jbe 1f\n" + + "vpbroadcastb 4(%2), %%ymm14\n" + "vgf2p8mulb %%ymm8, %%ymm14, %%ymm15\n" + "vpxor %%ymm15, %%ymm12, %%ymm12\n" + "vgf2p8mulb %%ymm9, %%ymm14, %%ymm15\n" + "vpxor %%ymm15, %%ymm13, %%ymm13\n" + "cmpq $5, %0\n" + "jbe 1f\n" + + "vpbroadcastb 5(%2), %%ymm14\n" + "vgf2p8mulb %%ymm10, %%ymm14, %%ymm15\n" + "vpxor %%ymm15, %%ymm12, %%ymm12\n" + "vgf2p8mulb %%ymm11, %%ymm14, %%ymm15\n" + "vpxor %%ymm15, %%ymm13, %%ymm13\n" + + "1:\n" + "movq %3, %%rax\n" + "vmovdqa %%ymm12, 0(%%rax, %1)\n" + "vmovdqa %%ymm13, 32(%%rax, %1)\n" + : + : "r" ((uint64_t)N), "r" (i), "r" (&V[j * N]), "r" (pa[j]) + : "rax", "cc", "memory" + ); } } @@ -1563,7 +1761,7 @@ void raid_recX_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, voi "1:\n" : : "r" ((uint64_t)N), "r" (i), "r" (p), "r" (pa) - : "rax", "rbx" + : "rax", "rbx", "cc", "memory" ); /* reconstruct */ @@ -1607,7 +1805,7 @@ void raid_recX_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, voi "vmovdqa64 %%zmm7, (%%rax, %1)\n" : : "r" ((uint64_t)N), "r" (i), "r" (&V[j * N]), "r" (pa[j]) - : "rax" + : "rax", "cc", "memory" ); } } diff --git a/raid/neon.c b/raid/neon.c index 4e008332..b4247bf1 100644 --- a/raid/neon.c +++ b/raid/neon.c @@ -807,8 +807,6 @@ void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) uint8_t *pa[RAID_PARITY_MAX]; uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; - uint8_t buffer[RAID_PARITY_MAX * 16 + 16]; - uint8_t *pd = __align_ptr(buffer, 16); size_t i; int j, k; @@ -830,7 +828,7 @@ void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) raid_neon_begin(); - /* preload tables */ + /* preload mask */ asm volatile ( "ldr q28, %0\n" /* low4 */ : @@ -839,45 +837,142 @@ void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) for (i = 0; i < size; i += 16) { /* delta */ - for (j = 0; j < N; ++j) { - asm volatile ( - "ldr q0, %1\n" - "ldr q1, %2\n" - "eor v0.16b, v0.16b, v1.16b\n" - "str q0, %0\n" - : "=m" (pd[j * 16]) - : "m" (p[j][i]), "m" (pa[j][i]) - ); - } + asm volatile ( + "ldr x4, [%2, #0]\n" + "ldr x5, [%3, #0]\n" + "ldr q0, [x4, %1]\n" + "ldr q14, [x5, %1]\n" + "eor v0.16b, v0.16b, v14.16b\n" + "ushr v1.16b, v0.16b, #4\n" + "and v0.16b, v0.16b, v28.16b\n" + "and v1.16b, v1.16b, v28.16b\n" + "cmp %0, #1\n" + "b.ls 1f\n" + + "ldr x4, [%2, #8]\n" + "ldr x5, [%3, #8]\n" + "ldr q2, [x4, %1]\n" + "ldr q14, [x5, %1]\n" + "eor v2.16b, v2.16b, v14.16b\n" + "ushr v3.16b, v2.16b, #4\n" + "and v2.16b, v2.16b, v28.16b\n" + "and v3.16b, v3.16b, v28.16b\n" + "cmp %0, #2\n" + "b.ls 1f\n" + + "ldr x4, [%2, #16]\n" + "ldr x5, [%3, #16]\n" + "ldr q4, [x4, %1]\n" + "ldr q14, [x5, %1]\n" + "eor v4.16b, v4.16b, v14.16b\n" + "ushr v5.16b, v4.16b, #4\n" + "and v4.16b, v4.16b, v28.16b\n" + "and v5.16b, v5.16b, v28.16b\n" + "cmp %0, #3\n" + "b.ls 1f\n" + + "ldr x4, [%2, #24]\n" + "ldr x5, [%3, #24]\n" + "ldr q6, [x4, %1]\n" + "ldr q14, [x5, %1]\n" + "eor v6.16b, v6.16b, v14.16b\n" + "ushr v7.16b, v6.16b, #4\n" + "and v6.16b, v6.16b, v28.16b\n" + "and v7.16b, v7.16b, v28.16b\n" + "cmp %0, #4\n" + "b.ls 1f\n" + + "ldr x4, [%2, #32]\n" + "ldr x5, [%3, #32]\n" + "ldr q8, [x4, %1]\n" + "ldr q14, [x5, %1]\n" + "eor v8.16b, v8.16b, v14.16b\n" + "ushr v9.16b, v8.16b, #4\n" + "and v8.16b, v8.16b, v28.16b\n" + "and v9.16b, v9.16b, v28.16b\n" + "cmp %0, #5\n" + "b.ls 1f\n" + + "ldr x4, [%2, #40]\n" + "ldr x5, [%3, #40]\n" + "ldr q10, [x4, %1]\n" + "ldr q14, [x5, %1]\n" + "eor v10.16b, v10.16b, v14.16b\n" + "ushr v11.16b, v10.16b, #4\n" + "and v10.16b, v10.16b, v28.16b\n" + "and v11.16b, v11.16b, v28.16b\n" + + "1:\n" + : + : "r" ((uint64_t)N), "r" (i), "r" (p), "r" (pa) + : "x4", "x5", "cc", "memory" + ); /* reconstruct */ for (j = 0; j < N; ++j) { asm volatile ( - "eor v0.16b, v0.16b, v0.16b\n" - ); + "ldrb w4, [%2, #0]\n" + "add x5, %4, x4, lsl #5\n" + "ldp q24, q25, [x5]\n" + "tbl v12.16b, {v24.16b}, v0.16b\n" + "tbl v13.16b, {v25.16b}, v1.16b\n" + "cmp %0, #1\n" + "b.ls 1f\n" - for (k = 0; k < N; ++k) { - uint8_t m = V[j * N + k]; - asm volatile ( - "ldr q24, %1\n" - "ldr q25, %2\n" - "ldr q4, %0\n" - "ushr v17.16b, v4.16b, #4\n" - "and v16.16b, v4.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v2.16b, {v24.16b}, v16.16b\n" - "tbl v3.16b, {v25.16b}, v17.16b\n" - "eor v2.16b, v2.16b, v3.16b\n" - "eor v0.16b, v0.16b, v2.16b\n" - : - : "m" (pd[k * 16]), - "m" (raid_gfmulpshufb[m][0][0]), "m" (raid_gfmulpshufb[m][1][0]) - ); - } + "ldrb w4, [%2, #1]\n" + "add x5, %4, x4, lsl #5\n" + "ldp q24, q25, [x5]\n" + "tbl v14.16b, {v24.16b}, v2.16b\n" + "tbl v15.16b, {v25.16b}, v3.16b\n" + "eor v12.16b, v12.16b, v14.16b\n" + "eor v13.16b, v13.16b, v15.16b\n" + "cmp %0, #2\n" + "b.ls 1f\n" - asm volatile ( - "str q0, %0\n" - : "=m" (pa[j][i]) + "ldrb w4, [%2, #2]\n" + "add x5, %4, x4, lsl #5\n" + "ldp q24, q25, [x5]\n" + "tbl v14.16b, {v24.16b}, v4.16b\n" + "tbl v15.16b, {v25.16b}, v5.16b\n" + "eor v12.16b, v12.16b, v14.16b\n" + "eor v13.16b, v13.16b, v15.16b\n" + "cmp %0, #3\n" + "b.ls 1f\n" + + "ldrb w4, [%2, #3]\n" + "add x5, %4, x4, lsl #5\n" + "ldp q24, q25, [x5]\n" + "tbl v14.16b, {v24.16b}, v6.16b\n" + "tbl v15.16b, {v25.16b}, v7.16b\n" + "eor v12.16b, v12.16b, v14.16b\n" + "eor v13.16b, v13.16b, v15.16b\n" + "cmp %0, #4\n" + "b.ls 1f\n" + + "ldrb w4, [%2, #4]\n" + "add x5, %4, x4, lsl #5\n" + "ldp q24, q25, [x5]\n" + "tbl v14.16b, {v24.16b}, v8.16b\n" + "tbl v15.16b, {v25.16b}, v9.16b\n" + "eor v12.16b, v12.16b, v14.16b\n" + "eor v13.16b, v13.16b, v15.16b\n" + "cmp %0, #5\n" + "b.ls 1f\n" + + "ldrb w4, [%2, #5]\n" + "add x5, %4, x4, lsl #5\n" + "ldp q24, q25, [x5]\n" + "tbl v14.16b, {v24.16b}, v10.16b\n" + "tbl v15.16b, {v25.16b}, v11.16b\n" + "eor v12.16b, v12.16b, v14.16b\n" + "eor v13.16b, v13.16b, v15.16b\n" + + "1:\n" + "eor v12.16b, v12.16b, v13.16b\n" + "str q12, [%3, %1]\n" + : + : "r" ((uint64_t)N), "r" (i), "r" (&V[j * N]), "r" (pa[j]), "r" (raid_gfmulpshufb) + : "x4", "x5", "cc", "memory" ); } } diff --git a/raid/ssse3.c b/raid/ssse3.c index 57c524b2..0787c80d 100644 --- a/raid/ssse3.c +++ b/raid/ssse3.c @@ -1315,6 +1315,84 @@ void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) raid_sse_begin(); +#ifdef CONFIG_X86_64 + asm volatile ("movdqa %0,%%xmm6" : : "m" (gfconst16.low4[0])); + + /* the inverse matrix V[] is constant for the whole recovery. */ + asm volatile ("movdqa %0,%%xmm8" : : "m" (raid_gfmulpshufb[V[0]][0][0])); + asm volatile ("movdqa %0,%%xmm9" : : "m" (raid_gfmulpshufb[V[0]][1][0])); + asm volatile ("movdqa %0,%%xmm10" : : "m" (raid_gfmulpshufb[V[1]][0][0])); + asm volatile ("movdqa %0,%%xmm11" : : "m" (raid_gfmulpshufb[V[1]][1][0])); + asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfmulpshufb[V[2]][0][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfmulpshufb[V[2]][1][0])); + asm volatile ("movdqa %0,%%xmm14" : : "m" (raid_gfmulpshufb[V[3]][0][0])); + asm volatile ("movdqa %0,%%xmm15" : : "m" (raid_gfmulpshufb[V[3]][1][0])); + + for (i = 0; i < size; i += 16) { + /* d0 = p[0] ^ pa[0] */ + asm volatile ("movdqa %0,%%xmm0" : : "m" (p[0][i])); + asm volatile ("movdqa %0,%%xmm4" : : "m" (pa[0][i])); + asm volatile ("pxor %xmm4,%xmm0"); + + /* d1 = p[1] ^ pa[1] */ + asm volatile ("movdqa %0,%%xmm1" : : "m" (p[1][i])); + asm volatile ("movdqa %0,%%xmm4" : : "m" (pa[1][i])); + asm volatile ("pxor %xmm4,%xmm1"); + + /* + * Split both deltas into low/high nibbles once. + * + * xmm0 = d0 low + * xmm2 = d0 high + * xmm1 = d1 low + * xmm3 = d1 high + */ + asm volatile ("movdqa %xmm0,%xmm2"); + asm volatile ("movdqa %xmm1,%xmm3"); + asm volatile ("psrlw $4,%xmm2"); + asm volatile ("psrlw $4,%xmm3"); + asm volatile ("pand %xmm6,%xmm0"); + asm volatile ("pand %xmm6,%xmm2"); + asm volatile ("pand %xmm6,%xmm1"); + asm volatile ("pand %xmm6,%xmm3"); + + /* + * pa[0] = V[0] * d0 ^ V[1] * d1 + */ + asm volatile ("movdqa %xmm8,%xmm4"); + asm volatile ("pshufb %xmm0,%xmm4"); + asm volatile ("movdqa %xmm9,%xmm5"); + asm volatile ("pshufb %xmm2,%xmm5"); + asm volatile ("pxor %xmm5,%xmm4"); + + asm volatile ("movdqa %xmm10,%xmm5"); + asm volatile ("pshufb %xmm1,%xmm5"); + asm volatile ("pxor %xmm5,%xmm4"); + asm volatile ("movdqa %xmm11,%xmm5"); + asm volatile ("pshufb %xmm3,%xmm5"); + asm volatile ("pxor %xmm5,%xmm4"); + + asm volatile ("movdqa %%xmm4,%0" : "=m" (pa[0][i])); + + /* + * pa[1] = V[2] * d0 ^ V[3] * d1 + */ + asm volatile ("movdqa %xmm12,%xmm4"); + asm volatile ("pshufb %xmm0,%xmm4"); + asm volatile ("movdqa %xmm13,%xmm5"); + asm volatile ("pshufb %xmm2,%xmm5"); + asm volatile ("pxor %xmm5,%xmm4"); + + asm volatile ("movdqa %xmm14,%xmm5"); + asm volatile ("pshufb %xmm1,%xmm5"); + asm volatile ("pxor %xmm5,%xmm4"); + asm volatile ("movdqa %xmm15,%xmm5"); + asm volatile ("pshufb %xmm3,%xmm5"); + asm volatile ("pxor %xmm5,%xmm4"); + + asm volatile ("movdqa %%xmm4,%0" : "=m" (pa[1][i])); + } +#else /* CONFIG_X86_32 */ asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 16) { @@ -1381,6 +1459,7 @@ void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) asm volatile ("movdqa %%xmm6,%0" : "=m" (pa[1][i])); } +#endif raid_sse_end(); } @@ -1396,8 +1475,6 @@ void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) uint8_t *pa[RAID_PARITY_MAX]; uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; - uint8_t buffer[RAID_PARITY_MAX * 16 + 16]; - uint8_t *pd = __align_ptr(buffer, 16); size_t i; int j, k; @@ -1419,42 +1496,228 @@ void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) raid_sse_begin(); +#ifdef CONFIG_X86_64 + asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); + + for (i = 0; i < size; i += 16) { + /* delta */ + asm volatile ( + "movq 0(%2), %%rax\n" + "movq 0(%3), %%rbx\n" + "movdqa (%%rax, %1), %%xmm12\n" + "movdqa (%%rbx, %1), %%xmm13\n" + "pxor %%xmm13, %%xmm12\n" + "movdqa %%xmm12, %%xmm1\n" + "psrlw $4, %%xmm1\n" + "pand %%xmm15, %%xmm12\n" + "pand %%xmm15, %%xmm1\n" + "movdqa %%xmm12, %%xmm0\n" + "cmpq $1, %0\n" + "jbe 1f\n" + + "movq 8(%2), %%rax\n" + "movq 8(%3), %%rbx\n" + "movdqa (%%rax, %1), %%xmm12\n" + "movdqa (%%rbx, %1), %%xmm13\n" + "pxor %%xmm13, %%xmm12\n" + "movdqa %%xmm12, %%xmm3\n" + "psrlw $4, %%xmm3\n" + "pand %%xmm15, %%xmm12\n" + "pand %%xmm15, %%xmm3\n" + "movdqa %%xmm12, %%xmm2\n" + "cmpq $2, %0\n" + "jbe 1f\n" + + "movq 16(%2), %%rax\n" + "movq 16(%3), %%rbx\n" + "movdqa (%%rax, %1), %%xmm12\n" + "movdqa (%%rbx, %1), %%xmm13\n" + "pxor %%xmm13, %%xmm12\n" + "movdqa %%xmm12, %%xmm5\n" + "psrlw $4, %%xmm5\n" + "pand %%xmm15, %%xmm12\n" + "pand %%xmm15, %%xmm5\n" + "movdqa %%xmm12, %%xmm4\n" + "cmpq $3, %0\n" + "jbe 1f\n" + + "movq 24(%2), %%rax\n" + "movq 24(%3), %%rbx\n" + "movdqa (%%rax, %1), %%xmm12\n" + "movdqa (%%rbx, %1), %%xmm13\n" + "pxor %%xmm13, %%xmm12\n" + "movdqa %%xmm12, %%xmm7\n" + "psrlw $4, %%xmm7\n" + "pand %%xmm15, %%xmm12\n" + "pand %%xmm15, %%xmm7\n" + "movdqa %%xmm12, %%xmm6\n" + "cmpq $4, %0\n" + "jbe 1f\n" + + "movq 32(%2), %%rax\n" + "movq 32(%3), %%rbx\n" + "movdqa (%%rax, %1), %%xmm12\n" + "movdqa (%%rbx, %1), %%xmm13\n" + "pxor %%xmm13, %%xmm12\n" + "movdqa %%xmm12, %%xmm9\n" + "psrlw $4, %%xmm9\n" + "pand %%xmm15, %%xmm12\n" + "pand %%xmm15, %%xmm9\n" + "movdqa %%xmm12, %%xmm8\n" + "cmpq $5, %0\n" + "jbe 1f\n" + + "movq 40(%2), %%rax\n" + "movq 40(%3), %%rbx\n" + "movdqa (%%rax, %1), %%xmm12\n" + "movdqa (%%rbx, %1), %%xmm13\n" + "pxor %%xmm13, %%xmm12\n" + "movdqa %%xmm12, %%xmm11\n" + "psrlw $4, %%xmm11\n" + "pand %%xmm15, %%xmm12\n" + "pand %%xmm15, %%xmm11\n" + "movdqa %%xmm12, %%xmm10\n" + + "1:\n" + : + : "r" ((uint64_t)N), "r" (i), "r" (p), "r" (pa) + : "rax", "rbx", "cc", "memory" + ); + + /* reconstruct */ + for (j = 0; j < N; ++j) { + asm volatile ( + "movzbq 0(%2), %%rcx\n" + "shlq $5, %%rcx\n" + "movdqa 0(%4, %%rcx), %%xmm13\n" + "movdqa 16(%4, %%rcx), %%xmm14\n" + "pshufb %%xmm0, %%xmm13\n" + "pshufb %%xmm1, %%xmm14\n" + "cmpq $1, %0\n" + "jbe 1f\n" + + "movzbq 1(%2), %%rcx\n" + "shlq $5, %%rcx\n" + "movdqa 0(%4, %%rcx), %%xmm12\n" + "pshufb %%xmm2, %%xmm12\n" + "pxor %%xmm12, %%xmm13\n" + "movdqa 16(%4, %%rcx), %%xmm12\n" + "pshufb %%xmm3, %%xmm12\n" + "pxor %%xmm12, %%xmm14\n" + "cmpq $2, %0\n" + "jbe 1f\n" + + "movzbq 2(%2), %%rcx\n" + "shlq $5, %%rcx\n" + "movdqa 0(%4, %%rcx), %%xmm12\n" + "pshufb %%xmm4, %%xmm12\n" + "pxor %%xmm12, %%xmm13\n" + "movdqa 16(%4, %%rcx), %%xmm12\n" + "pshufb %%xmm5, %%xmm12\n" + "pxor %%xmm12, %%xmm14\n" + "cmpq $3, %0\n" + "jbe 1f\n" + + "movzbq 3(%2), %%rcx\n" + "shlq $5, %%rcx\n" + "movdqa 0(%4, %%rcx), %%xmm12\n" + "pshufb %%xmm6, %%xmm12\n" + "pxor %%xmm12, %%xmm13\n" + "movdqa 16(%4, %%rcx), %%xmm12\n" + "pshufb %%xmm7, %%xmm12\n" + "pxor %%xmm12, %%xmm14\n" + "cmpq $4, %0\n" + "jbe 1f\n" + + "movzbq 4(%2), %%rcx\n" + "shlq $5, %%rcx\n" + "movdqa 0(%4, %%rcx), %%xmm12\n" + "pshufb %%xmm8, %%xmm12\n" + "pxor %%xmm12, %%xmm13\n" + "movdqa 16(%4, %%rcx), %%xmm12\n" + "pshufb %%xmm9, %%xmm12\n" + "pxor %%xmm12, %%xmm14\n" + "cmpq $5, %0\n" + "jbe 1f\n" + + "movzbq 5(%2), %%rcx\n" + "shlq $5, %%rcx\n" + "movdqa 0(%4, %%rcx), %%xmm12\n" + "pshufb %%xmm10, %%xmm12\n" + "pxor %%xmm12, %%xmm13\n" + "movdqa 16(%4, %%rcx), %%xmm12\n" + "pshufb %%xmm11, %%xmm12\n" + "pxor %%xmm12, %%xmm14\n" + + "1:\n" + "pxor %%xmm14, %%xmm13\n" + "movq %3, %%rax\n" + "movdqa %%xmm13, (%%rax, %1)\n" + : + : "r" ((uint64_t)N), "r" (i), "r" (&V[j * N]), "r" (pa[j]), "r" (raid_gfmulpshufb) + : "rax", "rcx", "cc", "memory" + ); + } + } +#else /* CONFIG_X86_32 */ + uint8_t buffer_low[RAID_PARITY_MAX * 16 + 16]; + uint8_t buffer_high[RAID_PARITY_MAX * 16 + 16]; + uint8_t *pd_low = __align_ptr(buffer_low, 16); + uint8_t *pd_high = __align_ptr(buffer_high, 16); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 16) { /* delta */ for (j = 0; j < N; ++j) { - asm volatile ("movdqa %0,%%xmm0" : : "m" (p[j][i])); - asm volatile ("movdqa %0,%%xmm1" : : "m" (pa[j][i])); - asm volatile ("pxor %xmm1,%xmm0"); - asm volatile ("movdqa %%xmm0,%0" : "=m" (pd[j * 16])); + asm volatile ( + "movdqa %2, %%xmm0\n" + "movdqa %3, %%xmm1\n" + "pxor %%xmm1, %%xmm0\n" + "movdqa %%xmm0, %%xmm1\n" + "psrlw $4, %%xmm1\n" + "pand %%xmm7, %%xmm0\n" + "pand %%xmm7, %%xmm1\n" + "movdqa %%xmm0, %0\n" + "movdqa %%xmm1, %1\n" + : "=m" (pd_low[j * 16]), "=m" (pd_high[j * 16]) + : "m" (p[j][i]), "m" (pa[j][i]) + ); } /* reconstruct */ for (j = 0; j < N; ++j) { - asm volatile ("pxor %xmm0,%xmm0"); - asm volatile ("pxor %xmm1,%xmm1"); + asm volatile ( + "pxor %%xmm0, %%xmm0\n" + "pxor %%xmm1, %%xmm1\n" + ); for (k = 0; k < N; ++k) { uint8_t m = V[j * N + k]; - asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfmulpshufb[m][0][0])); - asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfmulpshufb[m][1][0])); - asm volatile ("movdqa %0,%%xmm4" : : "m" (pd[k * 16])); - asm volatile ("movdqa %xmm4,%xmm5"); - asm volatile ("psrlw $4,%xmm5"); - asm volatile ("pand %xmm7,%xmm4"); - asm volatile ("pand %xmm7,%xmm5"); - asm volatile ("pshufb %xmm4,%xmm2"); - asm volatile ("pshufb %xmm5,%xmm3"); - asm volatile ("pxor %xmm2,%xmm0"); - asm volatile ("pxor %xmm3,%xmm1"); + asm volatile ( + "movdqa %0, %%xmm2\n" + "movdqa %1, %%xmm3\n" + "movdqa %2, %%xmm4\n" + "movdqa %3, %%xmm5\n" + "pshufb %%xmm4, %%xmm2\n" + "pshufb %%xmm5, %%xmm3\n" + "pxor %%xmm2, %%xmm0\n" + "pxor %%xmm3, %%xmm1\n" + : + : "m" (raid_gfmulpshufb[m][0][0]), "m" (raid_gfmulpshufb[m][1][0]), + "m" (pd_low[k * 16]), "m" (pd_high[k * 16]) + ); } - asm volatile ("pxor %xmm1,%xmm0"); - asm volatile ("movdqa %%xmm0,%0" : "=m" (pa[j][i])); + asm volatile ( + "pxor %%xmm1, %%xmm0\n" + "movdqa %%xmm0, %0\n" + : "=m" (pa[j][i]) + ); } } +#endif raid_sse_end(); }