From 1e63403bf2443a4b87a0fef86f5a8d04bd4e19a4 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sat, 30 May 2026 10:11:00 +0200 Subject: [PATCH] Change RAID AVX512/GFNI functions to use forward iterations --- raid/gfni.c | 280 +++++++++++++++++--------------------------------- raid/memory.c | 11 -- raid/x86.c | 130 +++++++++-------------- 3 files changed, 145 insertions(+), 276 deletions(-) diff --git a/raid/gfni.c b/raid/gfni.c index 9e6220ad..187ca8de 100644 --- a/raid/gfni.c +++ b/raid/gfni.c @@ -17,33 +17,34 @@ void raid_gen2_avx2gfni(int nd, size_t size, void **vv) uint8_t *q = v[nd + 1]; int d, l = nd - 1; size_t i; - uint8_t gf2 = 0x02; raid_avx_begin(); - asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gf2)); - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa %0, %%ymm0" : : "m" (v[l][i])); - asm volatile ("vmovdqa %0, %%ymm1" : : "m" (v[l][i + 32])); - asm volatile ("vmovdqa %ymm0, %ymm2"); - asm volatile ("vmovdqa %ymm1, %ymm3"); + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[0][i])); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[0][i + 32])); + asm volatile ("vmovdqa %ymm0,%ymm2"); + asm volatile ("vmovdqa %ymm1,%ymm3"); - for (d = l - 1; d >= 0; --d) { - asm volatile ("vgf2p8mulb %ymm15, %ymm2, %ymm2"); - asm volatile ("vgf2p8mulb %ymm15, %ymm3, %ymm3"); - asm volatile ("vmovdqa %0, %%ymm4" : : "m" (v[d][i])); - asm volatile ("vmovdqa %0, %%ymm5" : : "m" (v[d][i + 32])); - asm volatile ("vpxor %ymm4, %ymm0, %ymm0"); - asm volatile ("vpxor %ymm5, %ymm1, %ymm1"); - asm volatile ("vpxor %ymm4, %ymm2, %ymm2"); - asm volatile ("vpxor %ymm5, %ymm3, %ymm3"); + for (d = 1; d <= l; ++d) { + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i])); + asm volatile ("vmovdqa %0,%%ymm5" : : "m" (v[d][i + 32])); + + asm volatile ("vpbroadcastb %0,%%ymm6" : : "m" (gfcauchy[1][d])); + + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); + + asm volatile ("vgf2p8mulb %ymm4,%ymm6,%ymm8"); + asm volatile ("vgf2p8mulb %ymm5,%ymm6,%ymm9"); + asm volatile ("vpxor %ymm8,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm9,%ymm3,%ymm3"); } - asm volatile ("vmovntdq %%ymm0, %0" : "=m" (p[i])); - asm volatile ("vmovntdq %%ymm1, %0" : "=m" (p[i + 32])); - asm volatile ("vmovntdq %%ymm2, %0" : "=m" (q[i])); - asm volatile ("vmovntdq %%ymm3, %0" : "=m" (q[i + 32])); + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); + asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); + asm volatile ("vmovntdq %%ymm2,%0" : "=m" (q[i])); + asm volatile ("vmovntdq %%ymm3,%0" : "=m" (q[i + 32])); } raid_avx_end(); @@ -59,25 +60,26 @@ void raid_gen2_avx512gfni(int nd, size_t size, void **vv) uint8_t *q = v[nd + 1]; int d, l = nd - 1; size_t i; - uint8_t gf2 = 0x02; raid_avx_begin(); - asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gf2)); - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa64 %0, %%zmm0" : : "m" (v[l][i])); - asm volatile ("vmovdqa64 %zmm0, %zmm1"); + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); + asm volatile ("vmovdqa64 %zmm0,%zmm1"); - for (d = l - 1; d >= 0; --d) { - asm volatile ("vgf2p8mulb %zmm31, %zmm1, %zmm1"); - asm volatile ("vmovdqa64 %0, %%zmm2" : : "m" (v[d][i])); - asm volatile ("vpxorq %zmm2, %zmm0, %zmm0"); - asm volatile ("vpxorq %zmm2, %zmm1, %zmm1"); + for (d = 1; d <= l; ++d) { + asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[d][i])); + + asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfcauchy[1][d])); + + asm volatile ("vgf2p8mulb %zmm6,%zmm7,%zmm12"); + + asm volatile ("vpxorq %zmm6,%zmm0,%zmm0"); + asm volatile ("vpxorq %zmm12,%zmm1,%zmm1"); } - asm volatile ("vmovntdq %%zmm0, %0" : "=m" (p[i])); - asm volatile ("vmovntdq %%zmm1, %0" : "=m" (q[i])); + asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i])); + asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i])); } raid_avx_end(); @@ -98,14 +100,14 @@ void raid_gen3_avx2gfni(int nd, size_t size, void **vv) raid_avx_begin(); for (i = 0; i < size; i += 64) { - asm volatile ("vpxor %ymm0,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm1,%ymm1,%ymm1"); - asm volatile ("vpxor %ymm2,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm3,%ymm3,%ymm3"); - asm volatile ("vpxor %ymm4,%ymm4,%ymm4"); - asm volatile ("vpxor %ymm5,%ymm5,%ymm5"); + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[0][i])); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[0][i + 32])); + asm volatile ("vmovdqa %ymm0,%ymm2"); + asm volatile ("vmovdqa %ymm1,%ymm3"); + asm volatile ("vmovdqa %ymm0,%ymm4"); + asm volatile ("vmovdqa %ymm1,%ymm5"); - for (d = l; d > 0; --d) { + for (d = 1; d <= l; ++d) { asm volatile ("vmovdqa %0,%%ymm6" : : "m" (v[d][i])); asm volatile ("vmovdqa %0,%%ymm7" : : "m" (v[d][i + 32])); @@ -126,17 +128,6 @@ void raid_gen3_avx2gfni(int nd, size_t size, void **vv) asm volatile ("vpxor %ymm11,%ymm5,%ymm5"); } - /* first disk with all coefficients at 1 */ - asm volatile ("vmovdqa %0,%%ymm6" : : "m" (v[0][i])); - asm volatile ("vmovdqa %0,%%ymm7" : : "m" (v[0][i + 32])); - - asm volatile ("vpxor %ymm6,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm7,%ymm1,%ymm1"); - asm volatile ("vpxor %ymm6,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm7,%ymm3,%ymm3"); - asm volatile ("vpxor %ymm6,%ymm4,%ymm4"); - asm volatile ("vpxor %ymm7,%ymm5,%ymm5"); - asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); asm volatile ("vmovntdq %%ymm2,%0" : "=m" (q[i])); @@ -163,11 +154,11 @@ void raid_gen3_avx512gfni(int nd, size_t size, void **vv) raid_avx_begin(); for (i = 0; i < size; i += 64) { - asm volatile ("vpxorq %zmm0,%zmm0,%zmm0"); - asm volatile ("vpxorq %zmm1,%zmm1,%zmm1"); - asm volatile ("vpxorq %zmm2,%zmm2,%zmm2"); + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); + asm volatile ("vmovdqa64 %zmm0,%zmm1"); + asm volatile ("vmovdqa64 %zmm0,%zmm2"); - for (d = l; d > 0; --d) { + for (d = 1; d <= l; ++d) { asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[d][i])); asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfcauchy[1][d])); @@ -181,13 +172,6 @@ void raid_gen3_avx512gfni(int nd, size_t size, void **vv) asm volatile ("vpxorq %zmm13,%zmm2,%zmm2"); } - /* first disk with all coefficients at 1 */ - asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[0][i])); - - asm volatile ("vpxorq %zmm6,%zmm0,%zmm0"); - asm volatile ("vpxorq %zmm6,%zmm1,%zmm1"); - asm volatile ("vpxorq %zmm6,%zmm2,%zmm2"); - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i])); asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i])); asm volatile ("vmovntdq %%zmm2,%0" : "=m" (r[i])); @@ -212,16 +196,16 @@ void raid_gen4_avx2gfni(int nd, size_t size, void **vv) raid_avx_begin(); for (i = 0; i < size; i += 64) { - asm volatile ("vpxor %ymm0,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm1,%ymm1,%ymm1"); - asm volatile ("vpxor %ymm2,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm3,%ymm3,%ymm3"); - asm volatile ("vpxor %ymm4,%ymm4,%ymm4"); - asm volatile ("vpxor %ymm5,%ymm5,%ymm5"); - asm volatile ("vpxor %ymm6,%ymm6,%ymm6"); - asm volatile ("vpxor %ymm7,%ymm7,%ymm7"); + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[0][i])); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[0][i + 32])); + asm volatile ("vmovdqa %ymm0,%ymm2"); + asm volatile ("vmovdqa %ymm1,%ymm3"); + asm volatile ("vmovdqa %ymm0,%ymm4"); + asm volatile ("vmovdqa %ymm1,%ymm5"); + asm volatile ("vmovdqa %ymm0,%ymm6"); + asm volatile ("vmovdqa %ymm1,%ymm7"); - for (d = l; d > 0; --d) { + for (d = 1; d <= l; ++d) { asm volatile ("vmovdqa %0,%%ymm8" : : "m" (v[d][i])); asm volatile ("vmovdqa %0,%%ymm9" : : "m" (v[d][i + 32])); @@ -248,19 +232,6 @@ void raid_gen4_avx2gfni(int nd, size_t size, void **vv) asm volatile ("vpxor %ymm14,%ymm7,%ymm7"); } - /* first disk with all coefficients at 1 */ - asm volatile ("vmovdqa %0,%%ymm8" : : "m" (v[0][i])); - asm volatile ("vmovdqa %0,%%ymm9" : : "m" (v[0][i + 32])); - - asm volatile ("vpxor %ymm8,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm9,%ymm1,%ymm1"); - asm volatile ("vpxor %ymm8,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm9,%ymm3,%ymm3"); - asm volatile ("vpxor %ymm8,%ymm4,%ymm4"); - asm volatile ("vpxor %ymm9,%ymm5,%ymm5"); - asm volatile ("vpxor %ymm8,%ymm6,%ymm6"); - asm volatile ("vpxor %ymm9,%ymm7,%ymm7"); - asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); asm volatile ("vmovntdq %%ymm2,%0" : "=m" (q[i])); @@ -290,12 +261,12 @@ void raid_gen4_avx512gfni(int nd, size_t size, void **vv) raid_avx_begin(); for (i = 0; i < size; i += 64) { - asm volatile ("vpxorq %zmm0,%zmm0,%zmm0"); - asm volatile ("vpxorq %zmm1,%zmm1,%zmm1"); - asm volatile ("vpxorq %zmm2,%zmm2,%zmm2"); - asm volatile ("vpxorq %zmm3,%zmm3,%zmm3"); + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); + asm volatile ("vmovdqa64 %zmm0,%zmm1"); + asm volatile ("vmovdqa64 %zmm0,%zmm2"); + asm volatile ("vmovdqa64 %zmm0,%zmm3"); - for (d = l; d > 0; --d) { + for (d = 1; d <= l; ++d) { asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[d][i])); asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfcauchy[1][d])); @@ -312,14 +283,6 @@ void raid_gen4_avx512gfni(int nd, size_t size, void **vv) asm volatile ("vpxorq %zmm14,%zmm3,%zmm3"); } - /* first disk with all coefficients at 1 */ - asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[0][i])); - - asm volatile ("vpxorq %zmm6,%zmm0,%zmm0"); - asm volatile ("vpxorq %zmm6,%zmm1,%zmm1"); - asm volatile ("vpxorq %zmm6,%zmm2,%zmm2"); - asm volatile ("vpxorq %zmm6,%zmm3,%zmm3"); - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i])); asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i])); asm volatile ("vmovntdq %%zmm2,%0" : "=m" (r[i])); @@ -346,18 +309,18 @@ void raid_gen5_avx2gfni(int nd, size_t size, void **vv) raid_avx_begin(); for (i = 0; i < size; i += 64) { - asm volatile ("vpxor %ymm0,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm1,%ymm1,%ymm1"); - asm volatile ("vpxor %ymm2,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm3,%ymm3,%ymm3"); - asm volatile ("vpxor %ymm4,%ymm4,%ymm4"); - asm volatile ("vpxor %ymm5,%ymm5,%ymm5"); - asm volatile ("vpxor %ymm6,%ymm6,%ymm6"); - asm volatile ("vpxor %ymm7,%ymm7,%ymm7"); - asm volatile ("vpxor %ymm8,%ymm8,%ymm8"); - asm volatile ("vpxor %ymm9,%ymm9,%ymm9"); + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[0][i])); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[0][i + 32])); + asm volatile ("vmovdqa %ymm0,%ymm2"); + asm volatile ("vmovdqa %ymm1,%ymm3"); + asm volatile ("vmovdqa %ymm0,%ymm4"); + asm volatile ("vmovdqa %ymm1,%ymm5"); + asm volatile ("vmovdqa %ymm0,%ymm6"); + asm volatile ("vmovdqa %ymm1,%ymm7"); + asm volatile ("vmovdqa %ymm0,%ymm8"); + asm volatile ("vmovdqa %ymm1,%ymm9"); - for (d = l; d > 0; --d) { + for (d = 1; d <= l; ++d) { asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[d][i])); asm volatile ("vmovdqa %0,%%ymm11" : : "m" (v[d][i + 32])); @@ -389,21 +352,6 @@ void raid_gen5_avx2gfni(int nd, size_t size, void **vv) asm volatile ("vpxor %ymm14,%ymm9,%ymm9"); } - /* first disk with all coefficients at 1 */ - asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[0][i])); - asm volatile ("vmovdqa %0,%%ymm11" : : "m" (v[0][i + 32])); - - asm volatile ("vpxor %ymm10,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm11,%ymm1,%ymm1"); - asm volatile ("vpxor %ymm10,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm11,%ymm3,%ymm3"); - asm volatile ("vpxor %ymm10,%ymm4,%ymm4"); - asm volatile ("vpxor %ymm11,%ymm5,%ymm5"); - asm volatile ("vpxor %ymm10,%ymm6,%ymm6"); - asm volatile ("vpxor %ymm11,%ymm7,%ymm7"); - asm volatile ("vpxor %ymm10,%ymm8,%ymm8"); - asm volatile ("vpxor %ymm11,%ymm9,%ymm9"); - asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); asm volatile ("vmovntdq %%ymm2,%0" : "=m" (q[i])); @@ -436,13 +384,13 @@ void raid_gen5_avx512gfni(int nd, size_t size, void **vv) raid_avx_begin(); for (i = 0; i < size; i += 64) { - asm volatile ("vpxorq %zmm0,%zmm0,%zmm0"); - asm volatile ("vpxorq %zmm1,%zmm1,%zmm1"); - asm volatile ("vpxorq %zmm2,%zmm2,%zmm2"); - asm volatile ("vpxorq %zmm3,%zmm3,%zmm3"); - asm volatile ("vpxorq %zmm4,%zmm4,%zmm4"); + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); + asm volatile ("vmovdqa64 %zmm0,%zmm1"); + asm volatile ("vmovdqa64 %zmm0,%zmm2"); + asm volatile ("vmovdqa64 %zmm0,%zmm3"); + asm volatile ("vmovdqa64 %zmm0,%zmm4"); - for (d = l; d > 0; --d) { + for (d = 1; d <= l; ++d) { asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[d][i])); asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfcauchy[1][d])); @@ -462,15 +410,6 @@ void raid_gen5_avx512gfni(int nd, size_t size, void **vv) asm volatile ("vpxorq %zmm15,%zmm4,%zmm4"); } - /* first disk with all coefficients at 1 */ - asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[0][i])); - - asm volatile ("vpxorq %zmm6,%zmm0,%zmm0"); - asm volatile ("vpxorq %zmm6,%zmm1,%zmm1"); - asm volatile ("vpxorq %zmm6,%zmm2,%zmm2"); - asm volatile ("vpxorq %zmm6,%zmm3,%zmm3"); - asm volatile ("vpxorq %zmm6,%zmm4,%zmm4"); - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i])); asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i])); asm volatile ("vmovntdq %%zmm2,%0" : "=m" (r[i])); @@ -499,20 +438,20 @@ void raid_gen6_avx2gfni(int nd, size_t size, void **vv) raid_avx_begin(); for (i = 0; i < size; i += 64) { - asm volatile ("vpxor %ymm0,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm1,%ymm1,%ymm1"); - asm volatile ("vpxor %ymm2,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm3,%ymm3,%ymm3"); - asm volatile ("vpxor %ymm4,%ymm4,%ymm4"); - asm volatile ("vpxor %ymm5,%ymm5,%ymm5"); - asm volatile ("vpxor %ymm6,%ymm6,%ymm6"); - asm volatile ("vpxor %ymm7,%ymm7,%ymm7"); - asm volatile ("vpxor %ymm8,%ymm8,%ymm8"); - asm volatile ("vpxor %ymm9,%ymm9,%ymm9"); - asm volatile ("vpxor %ymm10,%ymm10,%ymm10"); - asm volatile ("vpxor %ymm11,%ymm11,%ymm11"); + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[0][i])); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[0][i + 32])); + asm volatile ("vmovdqa %ymm0,%ymm2"); + asm volatile ("vmovdqa %ymm1,%ymm3"); + asm volatile ("vmovdqa %ymm0,%ymm4"); + asm volatile ("vmovdqa %ymm1,%ymm5"); + asm volatile ("vmovdqa %ymm0,%ymm6"); + asm volatile ("vmovdqa %ymm1,%ymm7"); + asm volatile ("vmovdqa %ymm0,%ymm8"); + asm volatile ("vmovdqa %ymm1,%ymm9"); + asm volatile ("vmovdqa %ymm0,%ymm10"); + asm volatile ("vmovdqa %ymm1,%ymm11"); - for (d = l; d > 0; --d) { + for (d = 1; d <= l; ++d) { asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i])); asm volatile ("vmovdqa %0,%%ymm13" : : "m" (v[d][i + 32])); @@ -550,23 +489,6 @@ void raid_gen6_avx2gfni(int nd, size_t size, void **vv) asm volatile ("vpxor %ymm15,%ymm11,%ymm11"); } - /* first disk with all coefficients at 1 */ - asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[0][i])); - asm volatile ("vmovdqa %0,%%ymm13" : : "m" (v[0][i + 32])); - - asm volatile ("vpxor %ymm12,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm13,%ymm1,%ymm1"); - asm volatile ("vpxor %ymm12,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm13,%ymm3,%ymm3"); - asm volatile ("vpxor %ymm12,%ymm4,%ymm4"); - asm volatile ("vpxor %ymm13,%ymm5,%ymm5"); - asm volatile ("vpxor %ymm12,%ymm6,%ymm6"); - asm volatile ("vpxor %ymm13,%ymm7,%ymm7"); - asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); - asm volatile ("vpxor %ymm13,%ymm9,%ymm9"); - asm volatile ("vpxor %ymm12,%ymm10,%ymm10"); - asm volatile ("vpxor %ymm13,%ymm11,%ymm11"); - asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); asm volatile ("vmovntdq %%ymm2,%0" : "=m" (q[i])); @@ -602,14 +524,14 @@ void raid_gen6_avx512gfni(int nd, size_t size, void **vv) raid_avx_begin(); for (i = 0; i < size; i += 64) { - asm volatile ("vpxorq %zmm0,%zmm0,%zmm0"); - asm volatile ("vpxorq %zmm1,%zmm1,%zmm1"); - asm volatile ("vpxorq %zmm2,%zmm2,%zmm2"); - asm volatile ("vpxorq %zmm3,%zmm3,%zmm3"); - asm volatile ("vpxorq %zmm4,%zmm4,%zmm4"); - asm volatile ("vpxorq %zmm5,%zmm5,%zmm5"); + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); + asm volatile ("vmovdqa64 %zmm0,%zmm1"); + asm volatile ("vmovdqa64 %zmm0,%zmm2"); + asm volatile ("vmovdqa64 %zmm0,%zmm3"); + asm volatile ("vmovdqa64 %zmm0,%zmm4"); + asm volatile ("vmovdqa64 %zmm0,%zmm5"); - for (d = l; d > 0; --d) { + for (d = 1; d <= l; ++d) { asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[d][i])); asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfcauchy[1][d])); @@ -632,16 +554,6 @@ void raid_gen6_avx512gfni(int nd, size_t size, void **vv) asm volatile ("vpxorq %zmm16,%zmm5,%zmm5"); } - /* first disk with all coefficients at 1 */ - asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[0][i])); - - asm volatile ("vpxorq %zmm6,%zmm0,%zmm0"); - asm volatile ("vpxorq %zmm6,%zmm1,%zmm1"); - asm volatile ("vpxorq %zmm6,%zmm2,%zmm2"); - asm volatile ("vpxorq %zmm6,%zmm3,%zmm3"); - asm volatile ("vpxorq %zmm6,%zmm4,%zmm4"); - asm volatile ("vpxorq %zmm6,%zmm5,%zmm5"); - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i])); asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i])); asm volatile ("vmovntdq %%zmm2,%0" : "=m" (r[i])); diff --git a/raid/memory.c b/raid/memory.c index 23b18420..b6476221 100644 --- a/raid/memory.c +++ b/raid/memory.c @@ -59,17 +59,6 @@ void **raid_malloc_vector_align(int nd, int n, size_t size, size_t align_size, s va += size + displacement_size; } - /* - * Reverse order of the data blocks - * because they are usually accessed from the last one - */ - for (i = 0; i < nd / 2; ++i) { - void *ptr = v[i]; - - v[i] = v[nd - 1 - i]; - v[nd - 1 - i] = ptr; - } - return v; } diff --git a/raid/x86.c b/raid/x86.c index 74e91c72..4d20a167 100644 --- a/raid/x86.c +++ b/raid/x86.c @@ -136,16 +136,18 @@ void raid_gen1_sse2(int nd, size_t size, void **vv) raid_sse_begin(); for (i = 0; i < size; i += 64) { - asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i])); - asm volatile ("movdqa %0,%%xmm1" : : "m" (v[l][i + 16])); - asm volatile ("movdqa %0,%%xmm2" : : "m" (v[l][i + 32])); - asm volatile ("movdqa %0,%%xmm3" : : "m" (v[l][i + 48])); - for (d = l - 1; d >= 0; --d) { + asm volatile ("movdqa %0,%%xmm0" : : "m" (v[0][i])); + asm volatile ("movdqa %0,%%xmm1" : : "m" (v[0][i + 16])); + asm volatile ("movdqa %0,%%xmm2" : : "m" (v[0][i + 32])); + asm volatile ("movdqa %0,%%xmm3" : : "m" (v[0][i + 48])); + + for (d = 1; d <= l; ++d) { asm volatile ("pxor %0,%%xmm0" : : "m" (v[d][i])); asm volatile ("pxor %0,%%xmm1" : : "m" (v[d][i + 16])); asm volatile ("pxor %0,%%xmm2" : : "m" (v[d][i + 32])); asm volatile ("pxor %0,%%xmm3" : : "m" (v[d][i + 48])); } + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); asm volatile ("movntdq %%xmm1,%0" : "=m" (p[i + 16])); asm volatile ("movntdq %%xmm2,%0" : "=m" (p[i + 32])); @@ -171,12 +173,14 @@ void raid_gen1_avx2(int nd, size_t size, void **vv) raid_avx_begin(); for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[l][i])); - asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[l][i + 32])); - for (d = l - 1; d >= 0; --d) { + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[0][i])); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[0][i + 32])); + + for (d = 1; d <= l; ++d) { asm volatile ("vpxor %0,%%ymm0,%%ymm0" : : "m" (v[d][i])); asm volatile ("vpxor %0,%%ymm1,%%ymm1" : : "m" (v[d][i + 32])); } + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); } @@ -205,19 +209,19 @@ void raid_gen1_avx512bw(int nd, size_t size, void **vv) raid_avx_begin(); for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[l][i])); - - for (d = l - 1; d >= 1; d -= 2) { + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); + + for (d = 1; d <= l - 1; d += 2) { asm volatile ( "vmovdqa64 %0,%%zmm1\n\t" "vpternlogq $0x96,%1,%%zmm1,%%zmm0" : - : "m" (v[d][i]), "m" (v[d-1][i]) + : "m" (v[d][i]), "m" (v[d+1][i]) ); } - - if (d == 0) { - asm volatile ("vpxorq %0, %%zmm0,%%zmm0" : : "m" (v[0][i])); + + if (d == l) { + asm volatile ("vpxorq %0,%%zmm0,%%zmm0" : : "m" (v[l][i])); } asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i])); @@ -265,6 +269,7 @@ void raid_gen2_sse2(int nd, size_t size, void **vv) asm volatile ("movdqa %0,%%xmm1" : : "m" (v[l][i + 16])); asm volatile ("movdqa %xmm0,%xmm2"); asm volatile ("movdqa %xmm1,%xmm3"); + for (d = l - 1; d >= 0; --d) { /* scale by 2 before adding the current disk so the first (last processed) disk with factor 1 avoids doubling */ asm volatile ("pxor %xmm4,%xmm4"); @@ -320,6 +325,7 @@ void raid_gen2_avx2(int nd, size_t size, void **vv) asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[l][i + 32])); asm volatile ("vmovdqa %ymm0,%ymm2"); asm volatile ("vmovdqa %ymm1,%ymm3"); + for (d = l - 1; d >= 0; --d) { /* scale by 2 before adding the current disk so the first (last processed) disk with factor 1 avoids doubling */ asm volatile ("vpcmpgtb %ymm2,%ymm6,%ymm4"); @@ -338,6 +344,7 @@ void raid_gen2_avx2(int nd, size_t size, void **vv) asm volatile ("vpxor %ymm4,%ymm2,%ymm2"); asm volatile ("vpxor %ymm5,%ymm3,%ymm3"); } + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); asm volatile ("vmovntdq %%ymm2,%0" : "=m" (q[i])); @@ -368,10 +375,10 @@ void raid_gen2_avx512bw(int nd, size_t size, void **vv) asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 64) { - asm volatile ("vpxorq %zmm0,%zmm0,%zmm0"); - asm volatile ("vpxorq %zmm1,%zmm1,%zmm1"); + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); + asm volatile ("vmovdqa64 %zmm0,%zmm1"); - for (d = l; d > 0; --d) { + for (d = 1; d <= l; ++d) { asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (v[d][i])); asm volatile ("vpxorq %zmm10,%zmm0,%zmm0"); @@ -388,12 +395,6 @@ void raid_gen2_avx512bw(int nd, size_t size, void **vv) asm volatile ("vpxorq %zmm13,%zmm1,%zmm1"); } - /* first disk with all coefficients at 1 */ - asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (v[0][i])); - - asm volatile ("vpxorq %zmm10,%zmm0,%zmm0"); - asm volatile ("vpxorq %zmm10,%zmm1,%zmm1"); - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i])); asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i])); } @@ -433,6 +434,7 @@ void raid_gen2_sse2ext(int nd, size_t size, void **vv) asm volatile ("movdqa %xmm1,%xmm5"); asm volatile ("movdqa %xmm2,%xmm6"); asm volatile ("movdqa %xmm3,%xmm7"); + for (d = l - 1; d >= 0; --d) { /* scale by 2 before adding the current disk so the first (last processed) disk with factor 1 avoids doubling */ asm volatile ("pxor %xmm8,%xmm8"); @@ -469,6 +471,7 @@ void raid_gen2_sse2ext(int nd, size_t size, void **vv) asm volatile ("pxor %xmm10,%xmm6"); asm volatile ("pxor %xmm11,%xmm7"); } + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); asm volatile ("movntdq %%xmm1,%0" : "=m" (p[i + 16])); asm volatile ("movntdq %%xmm2,%0" : "=m" (p[i + 32])); @@ -875,11 +878,11 @@ void raid_gen3_avx512bw(int nd, size_t size, void **vv) asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 64) { - asm volatile ("vpxorq %zmm0,%zmm0,%zmm0"); - asm volatile ("vpxorq %zmm1,%zmm1,%zmm1"); - asm volatile ("vpxorq %zmm2,%zmm2,%zmm2"); + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); + asm volatile ("vmovdqa64 %zmm0,%zmm1"); + asm volatile ("vmovdqa64 %zmm0,%zmm2"); - for (d = l; d > 0; --d) { + for (d = 1; d <= l; ++d) { asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (v[d][i])); asm volatile ("vpxorq %zmm10,%zmm0,%zmm0"); @@ -902,13 +905,6 @@ void raid_gen3_avx512bw(int nd, size_t size, void **vv) asm volatile ("vpxorq %zmm15,%zmm2,%zmm2"); } - /* first disk with all coefficients at 1 */ - asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (v[0][i])); - - asm volatile ("vpxorq %zmm10,%zmm0,%zmm0"); - asm volatile ("vpxorq %zmm10,%zmm1,%zmm1"); - asm volatile ("vpxorq %zmm10,%zmm2,%zmm2"); - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i])); asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i])); asm volatile ("vmovntdq %%zmm2,%0" : "=m" (r[i])); @@ -1391,12 +1387,12 @@ void raid_gen4_avx512bw(int nd, size_t size, void **vv) asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 64) { - asm volatile ("vpxorq %zmm0,%zmm0,%zmm0"); - asm volatile ("vpxorq %zmm1,%zmm1,%zmm1"); - asm volatile ("vpxorq %zmm2,%zmm2,%zmm2"); - asm volatile ("vpxorq %zmm3,%zmm3,%zmm3"); + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); + asm volatile ("vmovdqa64 %zmm0,%zmm1"); + asm volatile ("vmovdqa64 %zmm0,%zmm2"); + asm volatile ("vmovdqa64 %zmm0,%zmm3"); - for (d = l; d > 0; --d) { + for (d = 1; d <= l; ++d) { asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (v[d][i])); asm volatile ("vpxorq %zmm10,%zmm0,%zmm0"); @@ -1425,14 +1421,6 @@ void raid_gen4_avx512bw(int nd, size_t size, void **vv) asm volatile ("vpxorq %zmm17,%zmm3,%zmm3"); } - /* first disk with all coefficients at 1 */ - asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (v[0][i])); - - asm volatile ("vpxorq %zmm10,%zmm0,%zmm0"); - asm volatile ("vpxorq %zmm10,%zmm1,%zmm1"); - asm volatile ("vpxorq %zmm10,%zmm2,%zmm2"); - asm volatile ("vpxorq %zmm10,%zmm3,%zmm3"); - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i])); asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i])); asm volatile ("vmovntdq %%zmm2,%0" : "=m" (r[i])); @@ -1868,13 +1856,13 @@ void raid_gen5_avx512bw(int nd, size_t size, void **vv) asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 64) { - asm volatile ("vpxorq %zmm0,%zmm0,%zmm0"); - asm volatile ("vpxorq %zmm1,%zmm1,%zmm1"); - asm volatile ("vpxorq %zmm2,%zmm2,%zmm2"); - asm volatile ("vpxorq %zmm3,%zmm3,%zmm3"); - asm volatile ("vpxorq %zmm4,%zmm4,%zmm4"); + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); + asm volatile ("vmovdqa64 %zmm0,%zmm1"); + asm volatile ("vmovdqa64 %zmm0,%zmm2"); + asm volatile ("vmovdqa64 %zmm0,%zmm3"); + asm volatile ("vmovdqa64 %zmm0,%zmm4"); - for (d = l; d > 0; --d) { + for (d = 1; d <= l; ++d) { asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (v[d][i])); asm volatile ("vpxorq %zmm10,%zmm0,%zmm0"); @@ -1909,15 +1897,6 @@ void raid_gen5_avx512bw(int nd, size_t size, void **vv) asm volatile ("vpxorq %zmm19,%zmm4,%zmm4"); } - /* first disk with all coefficients at 1 */ - asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (v[0][i])); - - asm volatile ("vpxorq %zmm10,%zmm0,%zmm0"); - asm volatile ("vpxorq %zmm10,%zmm1,%zmm1"); - asm volatile ("vpxorq %zmm10,%zmm2,%zmm2"); - asm volatile ("vpxorq %zmm10,%zmm3,%zmm3"); - asm volatile ("vpxorq %zmm10,%zmm4,%zmm4"); - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i])); asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i])); asm volatile ("vmovntdq %%zmm2,%0" : "=m" (r[i])); @@ -2422,15 +2401,14 @@ void raid_gen6_avx512bw(int nd, size_t size, void **vv) asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 64) { - asm volatile ("vpxorq %zmm0,%zmm0,%zmm0"); - asm volatile ("vpxorq %zmm1,%zmm1,%zmm1"); - asm volatile ("vpxorq %zmm2,%zmm2,%zmm2"); - asm volatile ("vpxorq %zmm3,%zmm3,%zmm3"); - asm volatile ("vpxorq %zmm4,%zmm4,%zmm4"); - asm volatile ("vpxorq %zmm5,%zmm5,%zmm5"); + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); + asm volatile ("vmovdqa64 %zmm0,%zmm1"); + asm volatile ("vmovdqa64 %zmm0,%zmm2"); + asm volatile ("vmovdqa64 %zmm0,%zmm3"); + asm volatile ("vmovdqa64 %zmm0,%zmm4"); + asm volatile ("vmovdqa64 %zmm0,%zmm5"); - /* intermediate disks */ - for (d = l; d > 0; --d) { + for (d = 1; d <= l; ++d) { asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (v[d][i])); asm volatile ("vpxorq %zmm10,%zmm0,%zmm0"); @@ -2471,16 +2449,6 @@ void raid_gen6_avx512bw(int nd, size_t size, void **vv) asm volatile ("vpxorq %zmm21,%zmm5,%zmm5"); } - /* first disk with all coefficients at 1 */ - asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (v[0][i])); - - asm volatile ("vpxorq %zmm10,%zmm0,%zmm0"); - asm volatile ("vpxorq %zmm10,%zmm1,%zmm1"); - asm volatile ("vpxorq %zmm10,%zmm2,%zmm2"); - asm volatile ("vpxorq %zmm10,%zmm3,%zmm3"); - asm volatile ("vpxorq %zmm10,%zmm4,%zmm4"); - asm volatile ("vpxorq %zmm10,%zmm5,%zmm5"); - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i])); asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i])); asm volatile ("vmovntdq %%zmm2,%0" : "=m" (r[i]));