diff --git a/raid/avx2.c b/raid/avx2.c index 10c53226..3271724b 100644 --- a/raid/avx2.c +++ b/raid/avx2.c @@ -1228,6 +1228,12 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) raid_avx_begin(); #ifdef CONFIG_X86_64 + const uint8_t *T[RAID_PARITY_MAX * RAID_PARITY_MAX]; + + /* precompute shuffle table pointers */ + for (j = 0; j < N * N; ++j) + T[j] = &raid_gfmulpshufb[V[j]][0][0]; + asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 32) { @@ -1306,65 +1312,59 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) /* reconstruct */ for (j = 0; j < N; ++j) { asm volatile ( - "movzbq 0(%2), %%rcx\n" - "shlq $5, %%rcx\n" - "vbroadcasti128 0(%4, %%rcx), %%ymm13\n" - "vbroadcasti128 16(%4, %%rcx), %%ymm14\n" + "movq 0(%2), %%rcx\n" + "vbroadcasti128 0(%%rcx), %%ymm13\n" + "vbroadcasti128 16(%%rcx), %%ymm14\n" "vpshufb %%ymm0, %%ymm13, %%ymm13\n" "vpshufb %%ymm1, %%ymm14, %%ymm14\n" "cmpq $1, %0\n" "jbe 1f\n" - "movzbq 1(%2), %%rcx\n" - "shlq $5, %%rcx\n" - "vbroadcasti128 0(%4, %%rcx), %%ymm12\n" + "movq 8(%2), %%rcx\n" + "vbroadcasti128 0(%%rcx), %%ymm12\n" "vpshufb %%ymm2, %%ymm12, %%ymm12\n" "vpxor %%ymm12, %%ymm13, %%ymm13\n" - "vbroadcasti128 16(%4, %%rcx), %%ymm12\n" + "vbroadcasti128 16(%%rcx), %%ymm12\n" "vpshufb %%ymm3, %%ymm12, %%ymm12\n" "vpxor %%ymm12, %%ymm14, %%ymm14\n" "cmpq $2, %0\n" "jbe 1f\n" - "movzbq 2(%2), %%rcx\n" - "shlq $5, %%rcx\n" - "vbroadcasti128 0(%4, %%rcx), %%ymm12\n" + "movq 16(%2), %%rcx\n" + "vbroadcasti128 0(%%rcx), %%ymm12\n" "vpshufb %%ymm4, %%ymm12, %%ymm12\n" "vpxor %%ymm12, %%ymm13, %%ymm13\n" - "vbroadcasti128 16(%4, %%rcx), %%ymm12\n" + "vbroadcasti128 16(%%rcx), %%ymm12\n" "vpshufb %%ymm5, %%ymm12, %%ymm12\n" "vpxor %%ymm12, %%ymm14, %%ymm14\n" "cmpq $3, %0\n" "jbe 1f\n" - "movzbq 3(%2), %%rcx\n" - "shlq $5, %%rcx\n" - "vbroadcasti128 0(%4, %%rcx), %%ymm12\n" + "movq 24(%2), %%rcx\n" + "vbroadcasti128 0(%%rcx), %%ymm12\n" "vpshufb %%ymm6, %%ymm12, %%ymm12\n" "vpxor %%ymm12, %%ymm13, %%ymm13\n" - "vbroadcasti128 16(%4, %%rcx), %%ymm12\n" + "vbroadcasti128 16(%%rcx), %%ymm12\n" "vpshufb %%ymm7, %%ymm12, %%ymm12\n" "vpxor %%ymm12, %%ymm14, %%ymm14\n" "cmpq $4, %0\n" "jbe 1f\n" - "movzbq 4(%2), %%rcx\n" - "shlq $5, %%rcx\n" - "vbroadcasti128 0(%4, %%rcx), %%ymm12\n" + "movq 32(%2), %%rcx\n" + "vbroadcasti128 0(%%rcx), %%ymm12\n" "vpshufb %%ymm8, %%ymm12, %%ymm12\n" "vpxor %%ymm12, %%ymm13, %%ymm13\n" - "vbroadcasti128 16(%4, %%rcx), %%ymm12\n" + "vbroadcasti128 16(%%rcx), %%ymm12\n" "vpshufb %%ymm9, %%ymm12, %%ymm12\n" "vpxor %%ymm12, %%ymm14, %%ymm14\n" "cmpq $5, %0\n" "jbe 1f\n" - "movzbq 5(%2), %%rcx\n" - "shlq $5, %%rcx\n" - "vbroadcasti128 0(%4, %%rcx), %%ymm12\n" + "movq 40(%2), %%rcx\n" + "vbroadcasti128 0(%%rcx), %%ymm12\n" "vpshufb %%ymm10, %%ymm12, %%ymm12\n" "vpxor %%ymm12, %%ymm13, %%ymm13\n" - "vbroadcasti128 16(%4, %%rcx), %%ymm12\n" + "vbroadcasti128 16(%%rcx), %%ymm12\n" "vpshufb %%ymm11, %%ymm12, %%ymm12\n" "vpxor %%ymm12, %%ymm14, %%ymm14\n" @@ -1373,7 +1373,7 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) "movq %3, %%rax\n" "vmovdqa %%ymm13, (%%rax, %1)\n" : - : "r" ((uint64_t)N), "r" (i), "r" (&V[j * N]), "r" (pa[j]), "r" (raid_gfmulpshufb) + : "r" ((uint64_t)N), "r" (i), "r" (&T[j * N]), "r" (pa[j]) : "rax", "rcx", "cc", "memory" ); } diff --git a/raid/avx512.c b/raid/avx512.c index 84c10df4..ea1fa5c4 100644 --- a/raid/avx512.c +++ b/raid/avx512.c @@ -369,6 +369,7 @@ void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv uint8_t *pa[RAID_PARITY_MAX]; uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; + const uint8_t *T[RAID_PARITY_MAX * RAID_PARITY_MAX]; size_t i; int j, k; @@ -380,6 +381,10 @@ void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv /* invert it to solve the system of linear equations */ raid_invert(G, V, N); + /* precompute shuffle table pointers */ + for (j = 0; j < N * N; ++j) + T[j] = &raid_gfmulpshufb[V[j]][0][0]; + /* compute delta parity */ raid_delta_gen(N, id, ip, nd, size, vv); @@ -468,19 +473,17 @@ void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv /* reconstruct */ for (j = 0; j < N; ++j) { asm volatile ( - "movzbq 0(%2), %%rcx\n" - "shlq $5, %%rcx\n" - "vbroadcasti32x4 0(%4, %%rcx), %%zmm14\n" - "vbroadcasti32x4 16(%4, %%rcx), %%zmm15\n" + "movq 0(%2), %%rcx\n" + "vbroadcasti32x4 0(%%rcx), %%zmm14\n" + "vbroadcasti32x4 16(%%rcx), %%zmm15\n" "vpshufb %%zmm0, %%zmm14, %%zmm14\n" "vpshufb %%zmm1, %%zmm15, %%zmm15\n" "cmpq $1, %0\n" "jbe 1f\n" - "movzbq 1(%2), %%rcx\n" - "shlq $5, %%rcx\n" - "vbroadcasti32x4 0(%4, %%rcx), %%zmm12\n" - "vbroadcasti32x4 16(%4, %%rcx), %%zmm13\n" + "movq 8(%2), %%rcx\n" + "vbroadcasti32x4 0(%%rcx), %%zmm12\n" + "vbroadcasti32x4 16(%%rcx), %%zmm13\n" "vpshufb %%zmm2, %%zmm12, %%zmm12\n" "vpshufb %%zmm3, %%zmm13, %%zmm13\n" "vpxorq %%zmm12, %%zmm14, %%zmm14\n" @@ -488,10 +491,9 @@ void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv "cmpq $2, %0\n" "jbe 1f\n" - "movzbq 2(%2), %%rcx\n" - "shlq $5, %%rcx\n" - "vbroadcasti32x4 0(%4, %%rcx), %%zmm12\n" - "vbroadcasti32x4 16(%4, %%rcx), %%zmm13\n" + "movq 16(%2), %%rcx\n" + "vbroadcasti32x4 0(%%rcx), %%zmm12\n" + "vbroadcasti32x4 16(%%rcx), %%zmm13\n" "vpshufb %%zmm4, %%zmm12, %%zmm12\n" "vpshufb %%zmm5, %%zmm13, %%zmm13\n" "vpxorq %%zmm12, %%zmm14, %%zmm14\n" @@ -499,10 +501,9 @@ void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv "cmpq $3, %0\n" "jbe 1f\n" - "movzbq 3(%2), %%rcx\n" - "shlq $5, %%rcx\n" - "vbroadcasti32x4 0(%4, %%rcx), %%zmm12\n" - "vbroadcasti32x4 16(%4, %%rcx), %%zmm13\n" + "movq 24(%2), %%rcx\n" + "vbroadcasti32x4 0(%%rcx), %%zmm12\n" + "vbroadcasti32x4 16(%%rcx), %%zmm13\n" "vpshufb %%zmm6, %%zmm12, %%zmm12\n" "vpshufb %%zmm7, %%zmm13, %%zmm13\n" "vpxorq %%zmm12, %%zmm14, %%zmm14\n" @@ -510,10 +511,9 @@ void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv "cmpq $4, %0\n" "jbe 1f\n" - "movzbq 4(%2), %%rcx\n" - "shlq $5, %%rcx\n" - "vbroadcasti32x4 0(%4, %%rcx), %%zmm12\n" - "vbroadcasti32x4 16(%4, %%rcx), %%zmm13\n" + "movq 32(%2), %%rcx\n" + "vbroadcasti32x4 0(%%rcx), %%zmm12\n" + "vbroadcasti32x4 16(%%rcx), %%zmm13\n" "vpshufb %%zmm8, %%zmm12, %%zmm12\n" "vpshufb %%zmm9, %%zmm13, %%zmm13\n" "vpxorq %%zmm12, %%zmm14, %%zmm14\n" @@ -521,10 +521,9 @@ void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv "cmpq $5, %0\n" "jbe 1f\n" - "movzbq 5(%2), %%rcx\n" - "shlq $5, %%rcx\n" - "vbroadcasti32x4 0(%4, %%rcx), %%zmm12\n" - "vbroadcasti32x4 16(%4, %%rcx), %%zmm13\n" + "movq 40(%2), %%rcx\n" + "vbroadcasti32x4 0(%%rcx), %%zmm12\n" + "vbroadcasti32x4 16(%%rcx), %%zmm13\n" "vpshufb %%zmm10, %%zmm12, %%zmm12\n" "vpshufb %%zmm11, %%zmm13, %%zmm13\n" "vpxorq %%zmm12, %%zmm14, %%zmm14\n" @@ -535,7 +534,7 @@ void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv "movq %3, %%rax\n" "vmovdqa64 %%zmm14, (%%rax, %1)\n" : - : "r" ((uint64_t)N), "r" (i), "r" (&V[j * N]), "r" (pa[j]), "r" (raid_gfmulpshufb) + : "r" ((uint64_t)N), "r" (i), "r" (&T[j * N]), "r" (pa[j]) : "rax", "rcx", "cc", "memory" ); } diff --git a/raid/neon.c b/raid/neon.c index 38efe6ee..0e78269b 100644 --- a/raid/neon.c +++ b/raid/neon.c @@ -884,6 +884,7 @@ void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) uint8_t *pa[RAID_PARITY_MAX]; uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; + const uint8_t *T[RAID_PARITY_MAX * RAID_PARITY_MAX]; size_t i; int j, k; @@ -895,6 +896,10 @@ void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) /* invert it to solve the system of linear equations */ raid_invert(G, V, N); + /* precompute shuffle table pointers */ + for (j = 0; j < N * N; ++j) + T[j] = &raid_gfmulpshufb[V[j]][0][0]; + /* compute delta parity */ raid_delta_gen(N, id, ip, nd, size, vv); @@ -988,16 +993,14 @@ void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) /* reconstruct */ for (j = 0; j < N; ++j) { asm volatile ( - "ldrb w4, [%2, #0]\n" - "add x5, %4, x4, lsl #5\n" + "ldr x5, [%2, #0]\n" "ldp q24, q25, [x5]\n" "tbl v12.16b, {v24.16b}, v0.16b\n" "tbl v13.16b, {v25.16b}, v1.16b\n" "cmp %0, #1\n" "b.ls 1f\n" - "ldrb w4, [%2, #1]\n" - "add x5, %4, x4, lsl #5\n" + "ldr x5, [%2, #8]\n" "ldp q24, q25, [x5]\n" "tbl v14.16b, {v24.16b}, v2.16b\n" "tbl v15.16b, {v25.16b}, v3.16b\n" @@ -1006,8 +1009,7 @@ void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) "cmp %0, #2\n" "b.ls 1f\n" - "ldrb w4, [%2, #2]\n" - "add x5, %4, x4, lsl #5\n" + "ldr x5, [%2, #16]\n" "ldp q24, q25, [x5]\n" "tbl v14.16b, {v24.16b}, v4.16b\n" "tbl v15.16b, {v25.16b}, v5.16b\n" @@ -1016,8 +1018,7 @@ void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) "cmp %0, #3\n" "b.ls 1f\n" - "ldrb w4, [%2, #3]\n" - "add x5, %4, x4, lsl #5\n" + "ldr x5, [%2, #24]\n" "ldp q24, q25, [x5]\n" "tbl v14.16b, {v24.16b}, v6.16b\n" "tbl v15.16b, {v25.16b}, v7.16b\n" @@ -1026,8 +1027,7 @@ void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) "cmp %0, #4\n" "b.ls 1f\n" - "ldrb w4, [%2, #4]\n" - "add x5, %4, x4, lsl #5\n" + "ldr x5, [%2, #32]\n" "ldp q24, q25, [x5]\n" "tbl v14.16b, {v24.16b}, v8.16b\n" "tbl v15.16b, {v25.16b}, v9.16b\n" @@ -1036,8 +1036,7 @@ void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) "cmp %0, #5\n" "b.ls 1f\n" - "ldrb w4, [%2, #5]\n" - "add x5, %4, x4, lsl #5\n" + "ldr x5, [%2, #40]\n" "ldp q24, q25, [x5]\n" "tbl v14.16b, {v24.16b}, v10.16b\n" "tbl v15.16b, {v25.16b}, v11.16b\n" @@ -1048,7 +1047,7 @@ void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) "eor v12.16b, v12.16b, v13.16b\n" "str q12, [%3, %1]\n" : - : "r" ((uint64_t)N), "r" (i), "r" (&V[j * N]), "r" (pa[j]), "r" (raid_gfmulpshufb) + : "r" ((uint64_t)N), "r" (i), "r" (&T[j * N]), "r" (pa[j]) : "x4", "x5", "cc", "memory" ); } diff --git a/raid/ssse3.c b/raid/ssse3.c index 394efd51..d34dc310 100644 --- a/raid/ssse3.c +++ b/raid/ssse3.c @@ -1570,6 +1570,12 @@ void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) raid_sse_begin(); #ifdef CONFIG_X86_64 + const uint8_t *T[RAID_PARITY_MAX * RAID_PARITY_MAX]; + + /* precompute shuffle table pointers */ + for (j = 0; j < N * N; ++j) + T[j] = &raid_gfmulpshufb[V[j]][0][0]; + asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 16) { @@ -1660,65 +1666,59 @@ void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) /* reconstruct */ for (j = 0; j < N; ++j) { asm volatile ( - "movzbq 0(%2), %%rcx\n" - "shlq $5, %%rcx\n" - "movdqa 0(%4, %%rcx), %%xmm13\n" - "movdqa 16(%4, %%rcx), %%xmm14\n" + "movq 0(%2), %%rcx\n" + "movdqa 0(%%rcx), %%xmm13\n" + "movdqa 16(%%rcx), %%xmm14\n" "pshufb %%xmm0, %%xmm13\n" "pshufb %%xmm1, %%xmm14\n" "cmpq $1, %0\n" "jbe 1f\n" - "movzbq 1(%2), %%rcx\n" - "shlq $5, %%rcx\n" - "movdqa 0(%4, %%rcx), %%xmm12\n" + "movq 8(%2), %%rcx\n" + "movdqa 0(%%rcx), %%xmm12\n" "pshufb %%xmm2, %%xmm12\n" "pxor %%xmm12, %%xmm13\n" - "movdqa 16(%4, %%rcx), %%xmm12\n" + "movdqa 16(%%rcx), %%xmm12\n" "pshufb %%xmm3, %%xmm12\n" "pxor %%xmm12, %%xmm14\n" "cmpq $2, %0\n" "jbe 1f\n" - "movzbq 2(%2), %%rcx\n" - "shlq $5, %%rcx\n" - "movdqa 0(%4, %%rcx), %%xmm12\n" + "movq 16(%2), %%rcx\n" + "movdqa 0(%%rcx), %%xmm12\n" "pshufb %%xmm4, %%xmm12\n" "pxor %%xmm12, %%xmm13\n" - "movdqa 16(%4, %%rcx), %%xmm12\n" + "movdqa 16(%%rcx), %%xmm12\n" "pshufb %%xmm5, %%xmm12\n" "pxor %%xmm12, %%xmm14\n" "cmpq $3, %0\n" "jbe 1f\n" - "movzbq 3(%2), %%rcx\n" - "shlq $5, %%rcx\n" - "movdqa 0(%4, %%rcx), %%xmm12\n" + "movq 24(%2), %%rcx\n" + "movdqa 0(%%rcx), %%xmm12\n" "pshufb %%xmm6, %%xmm12\n" "pxor %%xmm12, %%xmm13\n" - "movdqa 16(%4, %%rcx), %%xmm12\n" + "movdqa 16(%%rcx), %%xmm12\n" "pshufb %%xmm7, %%xmm12\n" "pxor %%xmm12, %%xmm14\n" "cmpq $4, %0\n" "jbe 1f\n" - "movzbq 4(%2), %%rcx\n" - "shlq $5, %%rcx\n" - "movdqa 0(%4, %%rcx), %%xmm12\n" + "movq 32(%2), %%rcx\n" + "movdqa 0(%%rcx), %%xmm12\n" "pshufb %%xmm8, %%xmm12\n" "pxor %%xmm12, %%xmm13\n" - "movdqa 16(%4, %%rcx), %%xmm12\n" + "movdqa 16(%%rcx), %%xmm12\n" "pshufb %%xmm9, %%xmm12\n" "pxor %%xmm12, %%xmm14\n" "cmpq $5, %0\n" "jbe 1f\n" - "movzbq 5(%2), %%rcx\n" - "shlq $5, %%rcx\n" - "movdqa 0(%4, %%rcx), %%xmm12\n" + "movq 40(%2), %%rcx\n" + "movdqa 0(%%rcx), %%xmm12\n" "pshufb %%xmm10, %%xmm12\n" "pxor %%xmm12, %%xmm13\n" - "movdqa 16(%4, %%rcx), %%xmm12\n" + "movdqa 16(%%rcx), %%xmm12\n" "pshufb %%xmm11, %%xmm12\n" "pxor %%xmm12, %%xmm14\n" @@ -1727,7 +1727,7 @@ void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) "movq %3, %%rax\n" "movdqa %%xmm13, (%%rax, %1)\n" : - : "r" ((uint64_t)N), "r" (i), "r" (&V[j * N]), "r" (pa[j]), "r" (raid_gfmulpshufb) + : "r" ((uint64_t)N), "r" (i), "r" (&T[j * N]), "r" (pa[j]) : "rax", "rcx", "cc", "memory" ); }