From b070cbb894b68d9516cb5180e73c1b51bb465884 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Fri, 25 Sep 2026 13:33:26 +0200 Subject: [PATCH] Fix inline asm memory operand sizes for array-backed constants --- raid/avx2.c | 72 +++++++------- raid/avx512.c | 132 ++++++++++++------------- raid/gfni.c | 134 +++++++++++++------------- raid/neon.c | 6 +- raid/neon32.c | 38 ++++---- raid/sse2.c | 16 +-- raid/ssse3.c | 262 +++++++++++++++++++++++++------------------------- 7 files changed, 330 insertions(+), 330 deletions(-) diff --git a/raid/avx2.c b/raid/avx2.c index 49254151..6ab2cb5d 100644 --- a/raid/avx2.c +++ b/raid/avx2.c @@ -380,8 +380,8 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv asm volatile ("vpand %ymm11,%ymm5,%ymm5"); asm volatile ("vpand %ymm11,%ymm13,%ymm13"); - asm volatile ("vbroadcasti128 %0,%%ymm10" : : "m" (raid_gfcauchypshufb[l][1][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[l][1][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm10" : : "m" (raid_gfcauchypshufb[l][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[l][1][1])); asm volatile ("vpshufb %ymm4,%ymm10,%ymm2"); asm volatile ("vpshufb %ymm12,%ymm10,%ymm10"); asm volatile ("vpshufb %ymm5,%ymm15,%ymm7"); @@ -394,8 +394,8 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i])); asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i + 32])); - asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[d][1][1])); if (generator == 3) asm volatile ("vmovdqa %ymm1,%ymm6"); @@ -444,8 +444,8 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d - 1][i])); asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d - 1][i + 32])); - asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d - 1][1][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[d - 1][1][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d - 1][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[d - 1][1][1])); if (generator == 3) asm volatile ("vmovdqa %ymm1,%ymm6"); @@ -514,8 +514,8 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv asm volatile ("vpand %ymm11,%ymm4,%ymm4"); asm volatile ("vpand %ymm11,%ymm5,%ymm5"); - asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[1][1][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[1][1][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[1][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[1][1][1])); asm volatile ("vpshufb %ymm4,%ymm14,%ymm6"); asm volatile ("vpshufb %ymm5,%ymm15,%ymm7"); @@ -651,8 +651,8 @@ static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv asm volatile ("vpand %ymm7,%ymm12,%ymm12"); asm volatile ("vpand %ymm7,%ymm13,%ymm13"); - asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[l][1][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[l][1][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[l][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[l][1][1])); asm volatile ("vpshufb %ymm4,%ymm14,%ymm2"); asm volatile ("vpshufb %ymm5,%ymm15,%ymm3"); asm volatile ("vpxor %ymm3,%ymm2,%ymm2"); @@ -660,8 +660,8 @@ static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv asm volatile ("vpshufb %ymm13,%ymm15,%ymm11"); asm volatile ("vpxor %ymm11,%ymm10,%ymm10"); - asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[l][2][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[l][2][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[l][2][0])); + asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[l][2][1])); asm volatile ("vpshufb %ymm4,%ymm14,%ymm3"); asm volatile ("vpshufb %ymm5,%ymm15,%ymm5"); asm volatile ("vpxor %ymm5,%ymm3,%ymm3"); @@ -704,23 +704,23 @@ static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv asm volatile ("vpand %ymm7,%ymm12,%ymm12"); asm volatile ("vpand %ymm7,%ymm13,%ymm13"); - asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][0])); asm volatile ("vpshufb %ymm4,%ymm14,%ymm15"); asm volatile ("vpshufb %ymm12,%ymm14,%ymm14"); asm volatile ("vpxor %ymm15,%ymm2,%ymm2"); asm volatile ("vpxor %ymm14,%ymm10,%ymm10"); - asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][1])); asm volatile ("vpshufb %ymm5,%ymm14,%ymm15"); asm volatile ("vpshufb %ymm13,%ymm14,%ymm14"); asm volatile ("vpxor %ymm15,%ymm2,%ymm2"); asm volatile ("vpxor %ymm14,%ymm10,%ymm10"); - asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][2][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][2][0])); asm volatile ("vpshufb %ymm4,%ymm14,%ymm15"); asm volatile ("vpshufb %ymm12,%ymm14,%ymm14"); asm volatile ("vpxor %ymm15,%ymm3,%ymm3"); asm volatile ("vpxor %ymm14,%ymm11,%ymm11"); - asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][2][1])); asm volatile ("vpshufb %ymm5,%ymm14,%ymm15"); asm volatile ("vpshufb %ymm13,%ymm14,%ymm14"); asm volatile ("vpxor %ymm15,%ymm3,%ymm3"); @@ -819,29 +819,29 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in asm volatile ("vpand %ymm15,%ymm10,%ymm10"); asm volatile ("vpand %ymm15,%ymm11,%ymm11"); if (np >= 3) { - asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (raid_gfcauchypshufb[l][1][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[l][1][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (raid_gfcauchypshufb[l][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[l][1][1])); asm volatile ("vpshufb %ymm10,%ymm2,%ymm2"); asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); asm volatile ("vpxor %ymm13,%ymm2,%ymm2"); } if (np >= 4) { - asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (raid_gfcauchypshufb[l][2][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[l][2][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (raid_gfcauchypshufb[l][2][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[l][2][1])); asm volatile ("vpshufb %ymm10,%ymm3,%ymm3"); asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); asm volatile ("vpxor %ymm13,%ymm3,%ymm3"); } if (np >= 5) { - asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (raid_gfcauchypshufb[l][3][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[l][3][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (raid_gfcauchypshufb[l][3][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[l][3][1])); asm volatile ("vpshufb %ymm10,%ymm4,%ymm4"); asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); asm volatile ("vpxor %ymm13,%ymm4,%ymm4"); } if (np >= 6) { - asm volatile ("vbroadcasti128 %0,%%ymm5" : : "m" (raid_gfcauchypshufb[l][4][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[l][4][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm5" : : "m" (raid_gfcauchypshufb[l][4][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[l][4][1])); asm volatile ("vpshufb %ymm10,%ymm5,%ymm5"); asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); asm volatile ("vpxor %ymm13,%ymm5,%ymm5"); @@ -867,32 +867,32 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in asm volatile ("vpand %ymm15,%ymm10,%ymm10"); asm volatile ("vpand %ymm15,%ymm11,%ymm11"); if (np >= 3) { - asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfcauchypshufb[d][1][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfcauchypshufb[d][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[d][1][1])); asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); asm volatile ("vpxor %ymm12,%ymm2,%ymm2"); asm volatile ("vpxor %ymm13,%ymm2,%ymm2"); } if (np >= 4) { - asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfcauchypshufb[d][2][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfcauchypshufb[d][2][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[d][2][1])); asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); asm volatile ("vpxor %ymm12,%ymm3,%ymm3"); asm volatile ("vpxor %ymm13,%ymm3,%ymm3"); } if (np >= 5) { - asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfcauchypshufb[d][3][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[d][3][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfcauchypshufb[d][3][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[d][3][1])); asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); asm volatile ("vpxor %ymm12,%ymm4,%ymm4"); asm volatile ("vpxor %ymm13,%ymm4,%ymm4"); } if (np >= 6) { - asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfcauchypshufb[d][4][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[d][4][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfcauchypshufb[d][4][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[d][4][1])); asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); asm volatile ("vpxor %ymm12,%ymm5,%ymm5"); @@ -1090,8 +1090,8 @@ static __always_inline void raid_rec1_avx2_q(int *id, int *ip, int nd, size_t si raid_avx_begin(); /* keep the inverse coefficient resident */ - asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (raid_gfmulpshufb[V][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (raid_gfmulpshufb[V][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (raid_gfmulpshufb[V][0])); + asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (raid_gfmulpshufb[V][1])); /* keep zero and the active reduction polynomial resident */ asm volatile ("vpxor %ymm6,%ymm6,%ymm6"); @@ -1637,8 +1637,8 @@ static __always_inline void raid_rec1_avx2ext_q(int *id, int *ip, int nd, size_t raid_avx_begin(); /* keep the inverse coefficient, zero, and active reduction polynomial resident */ - asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfmulpshufb[V][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfmulpshufb[V][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfmulpshufb[V][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfmulpshufb[V][1])); asm volatile ("vpxor %ymm14,%ymm14,%ymm14"); asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.poly[0])); diff --git a/raid/avx512.c b/raid/avx512.c index 54372507..1a76d1b2 100644 --- a/raid/avx512.c +++ b/raid/avx512.c @@ -77,32 +77,32 @@ static __always_inline void raid_gen2_avx512bw_gen(int nd, size_t size, void **v asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gfconst16.low4[0])); /* preload as many Q coefficient tables as possible */ - asm volatile ("vbroadcasti32x4 %0,%%zmm4" : : "m" (raid_gfcauchypshufb[1][0][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm5" : : "m" (raid_gfcauchypshufb[1][0][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm6" : : "m" (raid_gfcauchypshufb[2][0][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm7" : : "m" (raid_gfcauchypshufb[2][0][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm8" : : "m" (raid_gfcauchypshufb[3][0][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm9" : : "m" (raid_gfcauchypshufb[3][0][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm10" : : "m" (raid_gfcauchypshufb[4][0][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm11" : : "m" (raid_gfcauchypshufb[4][0][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (raid_gfcauchypshufb[5][0][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (raid_gfcauchypshufb[5][0][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (raid_gfcauchypshufb[6][0][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (raid_gfcauchypshufb[6][0][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm16" : : "m" (raid_gfcauchypshufb[7][0][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm17" : : "m" (raid_gfcauchypshufb[7][0][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm18" : : "m" (raid_gfcauchypshufb[8][0][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm19" : : "m" (raid_gfcauchypshufb[8][0][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm20" : : "m" (raid_gfcauchypshufb[9][0][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm21" : : "m" (raid_gfcauchypshufb[9][0][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm22" : : "m" (raid_gfcauchypshufb[10][0][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm23" : : "m" (raid_gfcauchypshufb[10][0][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm24" : : "m" (raid_gfcauchypshufb[11][0][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm25" : : "m" (raid_gfcauchypshufb[11][0][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm26" : : "m" (raid_gfcauchypshufb[12][0][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm27" : : "m" (raid_gfcauchypshufb[12][0][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm28" : : "m" (raid_gfcauchypshufb[13][0][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm29" : : "m" (raid_gfcauchypshufb[13][0][1][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm4" : : "m" (raid_gfcauchypshufb[1][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm5" : : "m" (raid_gfcauchypshufb[1][0][1])); + asm volatile ("vbroadcasti32x4 %0,%%zmm6" : : "m" (raid_gfcauchypshufb[2][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm7" : : "m" (raid_gfcauchypshufb[2][0][1])); + asm volatile ("vbroadcasti32x4 %0,%%zmm8" : : "m" (raid_gfcauchypshufb[3][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm9" : : "m" (raid_gfcauchypshufb[3][0][1])); + asm volatile ("vbroadcasti32x4 %0,%%zmm10" : : "m" (raid_gfcauchypshufb[4][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm11" : : "m" (raid_gfcauchypshufb[4][0][1])); + asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (raid_gfcauchypshufb[5][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (raid_gfcauchypshufb[5][0][1])); + asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (raid_gfcauchypshufb[6][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (raid_gfcauchypshufb[6][0][1])); + asm volatile ("vbroadcasti32x4 %0,%%zmm16" : : "m" (raid_gfcauchypshufb[7][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm17" : : "m" (raid_gfcauchypshufb[7][0][1])); + asm volatile ("vbroadcasti32x4 %0,%%zmm18" : : "m" (raid_gfcauchypshufb[8][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm19" : : "m" (raid_gfcauchypshufb[8][0][1])); + asm volatile ("vbroadcasti32x4 %0,%%zmm20" : : "m" (raid_gfcauchypshufb[9][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm21" : : "m" (raid_gfcauchypshufb[9][0][1])); + asm volatile ("vbroadcasti32x4 %0,%%zmm22" : : "m" (raid_gfcauchypshufb[10][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm23" : : "m" (raid_gfcauchypshufb[10][0][1])); + asm volatile ("vbroadcasti32x4 %0,%%zmm24" : : "m" (raid_gfcauchypshufb[11][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm25" : : "m" (raid_gfcauchypshufb[11][0][1])); + asm volatile ("vbroadcasti32x4 %0,%%zmm26" : : "m" (raid_gfcauchypshufb[12][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm27" : : "m" (raid_gfcauchypshufb[12][0][1])); + asm volatile ("vbroadcasti32x4 %0,%%zmm28" : : "m" (raid_gfcauchypshufb[13][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm29" : : "m" (raid_gfcauchypshufb[13][0][1])); for (i = 0; i < size; i += 64) { asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); @@ -264,10 +264,10 @@ static __always_inline void raid_gen2_avx512bw_gen(int nd, size_t size, void **v asm volatile ("vpandq %zmm31,%zmm2,%zmm2"); asm volatile ("vpandq %zmm31,%zmm3,%zmm3"); - asm volatile ("vbroadcasti32x4 %0,%%zmm30" : : "m" (raid_gfcauchypshufb[d][0][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm30" : : "m" (raid_gfcauchypshufb[d][0][0])); asm volatile ("vpshufb %zmm2,%zmm30,%zmm2"); - asm volatile ("vbroadcasti32x4 %0,%%zmm30" : : "m" (raid_gfcauchypshufb[d][0][1][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm30" : : "m" (raid_gfcauchypshufb[d][0][1])); asm volatile ("vpshufb %zmm3,%zmm30,%zmm3"); asm volatile ("vpternlogq $0x96,%zmm2,%zmm3,%zmm1"); @@ -313,30 +313,30 @@ static __always_inline void raid_gen3_avx512bw_gen(int nd, size_t size, void **v asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gfconst16.low4[0])); /* preload as many Q and R coefficient tables as possible */ - asm volatile ("vbroadcasti32x4 %0,%%zmm7" : : "m" (raid_gfcauchypshufb[1][0][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm8" : : "m" (raid_gfcauchypshufb[1][0][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm9" : : "m" (raid_gfcauchypshufb[1][1][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm10" : : "m" (raid_gfcauchypshufb[1][1][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm11" : : "m" (raid_gfcauchypshufb[2][0][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (raid_gfcauchypshufb[2][0][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (raid_gfcauchypshufb[2][1][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (raid_gfcauchypshufb[2][1][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (raid_gfcauchypshufb[3][0][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm16" : : "m" (raid_gfcauchypshufb[3][0][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm17" : : "m" (raid_gfcauchypshufb[3][1][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm18" : : "m" (raid_gfcauchypshufb[3][1][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm19" : : "m" (raid_gfcauchypshufb[4][0][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm20" : : "m" (raid_gfcauchypshufb[4][0][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm21" : : "m" (raid_gfcauchypshufb[4][1][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm22" : : "m" (raid_gfcauchypshufb[4][1][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm23" : : "m" (raid_gfcauchypshufb[5][0][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm24" : : "m" (raid_gfcauchypshufb[5][0][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm25" : : "m" (raid_gfcauchypshufb[5][1][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm26" : : "m" (raid_gfcauchypshufb[5][1][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm27" : : "m" (raid_gfcauchypshufb[6][0][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm28" : : "m" (raid_gfcauchypshufb[6][0][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm29" : : "m" (raid_gfcauchypshufb[6][1][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm30" : : "m" (raid_gfcauchypshufb[6][1][1][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm7" : : "m" (raid_gfcauchypshufb[1][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm8" : : "m" (raid_gfcauchypshufb[1][0][1])); + asm volatile ("vbroadcasti32x4 %0,%%zmm9" : : "m" (raid_gfcauchypshufb[1][1][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm10" : : "m" (raid_gfcauchypshufb[1][1][1])); + asm volatile ("vbroadcasti32x4 %0,%%zmm11" : : "m" (raid_gfcauchypshufb[2][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (raid_gfcauchypshufb[2][0][1])); + asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (raid_gfcauchypshufb[2][1][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (raid_gfcauchypshufb[2][1][1])); + asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (raid_gfcauchypshufb[3][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm16" : : "m" (raid_gfcauchypshufb[3][0][1])); + asm volatile ("vbroadcasti32x4 %0,%%zmm17" : : "m" (raid_gfcauchypshufb[3][1][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm18" : : "m" (raid_gfcauchypshufb[3][1][1])); + asm volatile ("vbroadcasti32x4 %0,%%zmm19" : : "m" (raid_gfcauchypshufb[4][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm20" : : "m" (raid_gfcauchypshufb[4][0][1])); + asm volatile ("vbroadcasti32x4 %0,%%zmm21" : : "m" (raid_gfcauchypshufb[4][1][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm22" : : "m" (raid_gfcauchypshufb[4][1][1])); + asm volatile ("vbroadcasti32x4 %0,%%zmm23" : : "m" (raid_gfcauchypshufb[5][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm24" : : "m" (raid_gfcauchypshufb[5][0][1])); + asm volatile ("vbroadcasti32x4 %0,%%zmm25" : : "m" (raid_gfcauchypshufb[5][1][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm26" : : "m" (raid_gfcauchypshufb[5][1][1])); + asm volatile ("vbroadcasti32x4 %0,%%zmm27" : : "m" (raid_gfcauchypshufb[6][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm28" : : "m" (raid_gfcauchypshufb[6][0][1])); + asm volatile ("vbroadcasti32x4 %0,%%zmm29" : : "m" (raid_gfcauchypshufb[6][1][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm30" : : "m" (raid_gfcauchypshufb[6][1][1])); for (i = 0; i < size; i += 64) { asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); @@ -438,15 +438,15 @@ static __always_inline void raid_gen3_avx512bw_gen(int nd, size_t size, void **v asm volatile ("vpandq %zmm31,%zmm3,%zmm4"); asm volatile ("vpandq %zmm31,%zmm5,%zmm5"); - asm volatile ("vbroadcasti32x4 %0,%%zmm6" : : "m" (raid_gfcauchypshufb[d][0][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm6" : : "m" (raid_gfcauchypshufb[d][0][0])); asm volatile ("vpshufb %zmm4,%zmm6,%zmm6"); - asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (raid_gfcauchypshufb[d][0][1][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (raid_gfcauchypshufb[d][0][1])); asm volatile ("vpshufb %zmm5,%zmm3,%zmm3"); asm volatile ("vpternlogq $0x96,%zmm3,%zmm6,%zmm1"); - asm volatile ("vbroadcasti32x4 %0,%%zmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm6" : : "m" (raid_gfcauchypshufb[d][1][0])); asm volatile ("vpshufb %zmm4,%zmm6,%zmm6"); - asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (raid_gfcauchypshufb[d][1][1])); asm volatile ("vpshufb %zmm5,%zmm3,%zmm3"); asm volatile ("vpternlogq $0x96,%zmm3,%zmm6,%zmm2"); } @@ -507,23 +507,23 @@ static __always_inline void raid_genX_avx512bw(int nd, size_t size, void **vv, i asm volatile ("vpandq %zmm31,%zmm10,%zmm10"); asm volatile ("vpandq %zmm31,%zmm11,%zmm11"); - asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (raid_gfcauchypshufb[d][0][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (raid_gfcauchypshufb[d][0][1][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (raid_gfcauchypshufb[d][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (raid_gfcauchypshufb[d][0][1])); if (np >= 3) { - asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (raid_gfcauchypshufb[d][1][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (raid_gfcauchypshufb[d][1][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (raid_gfcauchypshufb[d][1][1])); } if (np >= 4) { - asm volatile ("vbroadcasti32x4 %0,%%zmm16" : : "m" (raid_gfcauchypshufb[d][2][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm17" : : "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm16" : : "m" (raid_gfcauchypshufb[d][2][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm17" : : "m" (raid_gfcauchypshufb[d][2][1])); } if (np >= 5) { - asm volatile ("vbroadcasti32x4 %0,%%zmm18" : : "m" (raid_gfcauchypshufb[d][3][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm19" : : "m" (raid_gfcauchypshufb[d][3][1][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm18" : : "m" (raid_gfcauchypshufb[d][3][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm19" : : "m" (raid_gfcauchypshufb[d][3][1])); } if (np >= 6) { - asm volatile ("vbroadcasti32x4 %0,%%zmm20" : : "m" (raid_gfcauchypshufb[d][4][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm21" : : "m" (raid_gfcauchypshufb[d][4][1][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm20" : : "m" (raid_gfcauchypshufb[d][4][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm21" : : "m" (raid_gfcauchypshufb[d][4][1])); } asm volatile ("vpshufb %zmm10,%zmm12,%zmm12"); diff --git a/raid/gfni.c b/raid/gfni.c index 1119c088..29068432 100644 --- a/raid/gfni.c +++ b/raid/gfni.c @@ -56,35 +56,35 @@ static __always_inline void raid_genX_avx2gfni_raid(int nd, size_t size, void ** asm volatile ("vpxor %ymm12,%ymm0,%ymm0"); asm volatile ("vpxor %ymm13,%ymm1,%ymm1"); - asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfcauchyaffine_raid[d][0][0])); + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfcauchyaffine_raid[d][0])); asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15"); asm volatile ("vpxor %ymm15,%ymm2,%ymm2"); asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15"); asm volatile ("vpxor %ymm15,%ymm3,%ymm3"); if (np >= 3) { - asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfcauchyaffine_raid[d][1][0])); + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfcauchyaffine_raid[d][1])); asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15"); asm volatile ("vpxor %ymm15,%ymm4,%ymm4"); asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15"); asm volatile ("vpxor %ymm15,%ymm5,%ymm5"); } if (np >= 4) { - asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfcauchyaffine_raid[d][2][0])); + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfcauchyaffine_raid[d][2])); asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15"); asm volatile ("vpxor %ymm15,%ymm6,%ymm6"); asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15"); asm volatile ("vpxor %ymm15,%ymm7,%ymm7"); } if (np >= 5) { - asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfcauchyaffine_raid[d][3][0])); + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfcauchyaffine_raid[d][3])); asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15"); asm volatile ("vpxor %ymm15,%ymm8,%ymm8"); asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15"); asm volatile ("vpxor %ymm15,%ymm9,%ymm9"); } if (np >= 6) { - asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfcauchyaffine_raid[d][4][0])); + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfcauchyaffine_raid[d][4])); asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15"); asm volatile ("vpxor %ymm15,%ymm10,%ymm10"); asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15"); @@ -163,34 +163,34 @@ static __always_inline void raid_gen2_avx512gfni_raid_gen(int nd, size_t size, v raid_avx_begin(); /* preload as many Q coefficient matrices as possible */ - asm volatile ("vpbroadcastq %0,%%zmm4" : : "m" (raid_gfcauchyaffine_raid[1][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm5" : : "m" (raid_gfcauchyaffine_raid[2][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm6" : : "m" (raid_gfcauchyaffine_raid[3][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm7" : : "m" (raid_gfcauchyaffine_raid[4][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm8" : : "m" (raid_gfcauchyaffine_raid[5][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm9" : : "m" (raid_gfcauchyaffine_raid[6][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm10" : : "m" (raid_gfcauchyaffine_raid[7][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm11" : : "m" (raid_gfcauchyaffine_raid[8][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm12" : : "m" (raid_gfcauchyaffine_raid[9][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm13" : : "m" (raid_gfcauchyaffine_raid[10][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm14" : : "m" (raid_gfcauchyaffine_raid[11][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm15" : : "m" (raid_gfcauchyaffine_raid[12][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm16" : : "m" (raid_gfcauchyaffine_raid[13][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm17" : : "m" (raid_gfcauchyaffine_raid[14][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm18" : : "m" (raid_gfcauchyaffine_raid[15][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm19" : : "m" (raid_gfcauchyaffine_raid[16][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm20" : : "m" (raid_gfcauchyaffine_raid[17][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm21" : : "m" (raid_gfcauchyaffine_raid[18][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm22" : : "m" (raid_gfcauchyaffine_raid[19][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm23" : : "m" (raid_gfcauchyaffine_raid[20][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm24" : : "m" (raid_gfcauchyaffine_raid[21][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm25" : : "m" (raid_gfcauchyaffine_raid[22][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm26" : : "m" (raid_gfcauchyaffine_raid[23][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm27" : : "m" (raid_gfcauchyaffine_raid[24][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm28" : : "m" (raid_gfcauchyaffine_raid[25][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm29" : : "m" (raid_gfcauchyaffine_raid[26][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm30" : : "m" (raid_gfcauchyaffine_raid[27][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm31" : : "m" (raid_gfcauchyaffine_raid[28][0][0])); + asm volatile ("vpbroadcastq %0,%%zmm4" : : "m" (raid_gfcauchyaffine_raid[1][0])); + asm volatile ("vpbroadcastq %0,%%zmm5" : : "m" (raid_gfcauchyaffine_raid[2][0])); + asm volatile ("vpbroadcastq %0,%%zmm6" : : "m" (raid_gfcauchyaffine_raid[3][0])); + asm volatile ("vpbroadcastq %0,%%zmm7" : : "m" (raid_gfcauchyaffine_raid[4][0])); + asm volatile ("vpbroadcastq %0,%%zmm8" : : "m" (raid_gfcauchyaffine_raid[5][0])); + asm volatile ("vpbroadcastq %0,%%zmm9" : : "m" (raid_gfcauchyaffine_raid[6][0])); + asm volatile ("vpbroadcastq %0,%%zmm10" : : "m" (raid_gfcauchyaffine_raid[7][0])); + asm volatile ("vpbroadcastq %0,%%zmm11" : : "m" (raid_gfcauchyaffine_raid[8][0])); + asm volatile ("vpbroadcastq %0,%%zmm12" : : "m" (raid_gfcauchyaffine_raid[9][0])); + asm volatile ("vpbroadcastq %0,%%zmm13" : : "m" (raid_gfcauchyaffine_raid[10][0])); + asm volatile ("vpbroadcastq %0,%%zmm14" : : "m" (raid_gfcauchyaffine_raid[11][0])); + asm volatile ("vpbroadcastq %0,%%zmm15" : : "m" (raid_gfcauchyaffine_raid[12][0])); + asm volatile ("vpbroadcastq %0,%%zmm16" : : "m" (raid_gfcauchyaffine_raid[13][0])); + asm volatile ("vpbroadcastq %0,%%zmm17" : : "m" (raid_gfcauchyaffine_raid[14][0])); + asm volatile ("vpbroadcastq %0,%%zmm18" : : "m" (raid_gfcauchyaffine_raid[15][0])); + asm volatile ("vpbroadcastq %0,%%zmm19" : : "m" (raid_gfcauchyaffine_raid[16][0])); + asm volatile ("vpbroadcastq %0,%%zmm20" : : "m" (raid_gfcauchyaffine_raid[17][0])); + asm volatile ("vpbroadcastq %0,%%zmm21" : : "m" (raid_gfcauchyaffine_raid[18][0])); + asm volatile ("vpbroadcastq %0,%%zmm22" : : "m" (raid_gfcauchyaffine_raid[19][0])); + asm volatile ("vpbroadcastq %0,%%zmm23" : : "m" (raid_gfcauchyaffine_raid[20][0])); + asm volatile ("vpbroadcastq %0,%%zmm24" : : "m" (raid_gfcauchyaffine_raid[21][0])); + asm volatile ("vpbroadcastq %0,%%zmm25" : : "m" (raid_gfcauchyaffine_raid[22][0])); + asm volatile ("vpbroadcastq %0,%%zmm26" : : "m" (raid_gfcauchyaffine_raid[23][0])); + asm volatile ("vpbroadcastq %0,%%zmm27" : : "m" (raid_gfcauchyaffine_raid[24][0])); + asm volatile ("vpbroadcastq %0,%%zmm28" : : "m" (raid_gfcauchyaffine_raid[25][0])); + asm volatile ("vpbroadcastq %0,%%zmm29" : : "m" (raid_gfcauchyaffine_raid[26][0])); + asm volatile ("vpbroadcastq %0,%%zmm30" : : "m" (raid_gfcauchyaffine_raid[27][0])); + asm volatile ("vpbroadcastq %0,%%zmm31" : : "m" (raid_gfcauchyaffine_raid[28][0])); for (i = 0; i < size; i += 64) { asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); @@ -399,7 +399,7 @@ static __always_inline void raid_gen2_avx512gfni_raid_gen(int nd, size_t size, v for (d = 29; d < nd; ++d) { asm volatile ("vmovdqa64 %0,%%zmm2" : : "m" (v[d][i])); asm volatile ("vpxorq %zmm2,%zmm0,%zmm0"); - asm volatile ("vpbroadcastq %0,%%zmm3" : : "m" (raid_gfcauchyaffine_raid[d][0][0])); + asm volatile ("vpbroadcastq %0,%%zmm3" : : "m" (raid_gfcauchyaffine_raid[d][0])); asm volatile ("vgf2p8affineqb $0,%zmm3,%zmm2,%zmm3"); asm volatile ("vpxorq %zmm3,%zmm1,%zmm1"); } @@ -442,32 +442,32 @@ static __always_inline void raid_gen3_avx512gfni_raid_gen(int nd, size_t size, v raid_avx_begin(); /* preload as many Q and R coefficient matrices as possible */ - asm volatile ("vpbroadcastq %0,%%zmm6" : : "m" (raid_gfcauchyaffine_raid[1][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm7" : : "m" (raid_gfcauchyaffine_raid[1][1][0])); - asm volatile ("vpbroadcastq %0,%%zmm8" : : "m" (raid_gfcauchyaffine_raid[2][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm9" : : "m" (raid_gfcauchyaffine_raid[2][1][0])); - asm volatile ("vpbroadcastq %0,%%zmm10" : : "m" (raid_gfcauchyaffine_raid[3][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm11" : : "m" (raid_gfcauchyaffine_raid[3][1][0])); - asm volatile ("vpbroadcastq %0,%%zmm12" : : "m" (raid_gfcauchyaffine_raid[4][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm13" : : "m" (raid_gfcauchyaffine_raid[4][1][0])); - asm volatile ("vpbroadcastq %0,%%zmm14" : : "m" (raid_gfcauchyaffine_raid[5][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm15" : : "m" (raid_gfcauchyaffine_raid[5][1][0])); - asm volatile ("vpbroadcastq %0,%%zmm16" : : "m" (raid_gfcauchyaffine_raid[6][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm17" : : "m" (raid_gfcauchyaffine_raid[6][1][0])); - asm volatile ("vpbroadcastq %0,%%zmm18" : : "m" (raid_gfcauchyaffine_raid[7][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm19" : : "m" (raid_gfcauchyaffine_raid[7][1][0])); - asm volatile ("vpbroadcastq %0,%%zmm20" : : "m" (raid_gfcauchyaffine_raid[8][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm21" : : "m" (raid_gfcauchyaffine_raid[8][1][0])); - asm volatile ("vpbroadcastq %0,%%zmm22" : : "m" (raid_gfcauchyaffine_raid[9][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm23" : : "m" (raid_gfcauchyaffine_raid[9][1][0])); - asm volatile ("vpbroadcastq %0,%%zmm24" : : "m" (raid_gfcauchyaffine_raid[10][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm25" : : "m" (raid_gfcauchyaffine_raid[10][1][0])); - asm volatile ("vpbroadcastq %0,%%zmm26" : : "m" (raid_gfcauchyaffine_raid[11][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm27" : : "m" (raid_gfcauchyaffine_raid[11][1][0])); - asm volatile ("vpbroadcastq %0,%%zmm28" : : "m" (raid_gfcauchyaffine_raid[12][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm29" : : "m" (raid_gfcauchyaffine_raid[12][1][0])); - asm volatile ("vpbroadcastq %0,%%zmm30" : : "m" (raid_gfcauchyaffine_raid[13][0][0])); - asm volatile ("vpbroadcastq %0,%%zmm31" : : "m" (raid_gfcauchyaffine_raid[13][1][0])); + asm volatile ("vpbroadcastq %0,%%zmm6" : : "m" (raid_gfcauchyaffine_raid[1][0])); + asm volatile ("vpbroadcastq %0,%%zmm7" : : "m" (raid_gfcauchyaffine_raid[1][1])); + asm volatile ("vpbroadcastq %0,%%zmm8" : : "m" (raid_gfcauchyaffine_raid[2][0])); + asm volatile ("vpbroadcastq %0,%%zmm9" : : "m" (raid_gfcauchyaffine_raid[2][1])); + asm volatile ("vpbroadcastq %0,%%zmm10" : : "m" (raid_gfcauchyaffine_raid[3][0])); + asm volatile ("vpbroadcastq %0,%%zmm11" : : "m" (raid_gfcauchyaffine_raid[3][1])); + asm volatile ("vpbroadcastq %0,%%zmm12" : : "m" (raid_gfcauchyaffine_raid[4][0])); + asm volatile ("vpbroadcastq %0,%%zmm13" : : "m" (raid_gfcauchyaffine_raid[4][1])); + asm volatile ("vpbroadcastq %0,%%zmm14" : : "m" (raid_gfcauchyaffine_raid[5][0])); + asm volatile ("vpbroadcastq %0,%%zmm15" : : "m" (raid_gfcauchyaffine_raid[5][1])); + asm volatile ("vpbroadcastq %0,%%zmm16" : : "m" (raid_gfcauchyaffine_raid[6][0])); + asm volatile ("vpbroadcastq %0,%%zmm17" : : "m" (raid_gfcauchyaffine_raid[6][1])); + asm volatile ("vpbroadcastq %0,%%zmm18" : : "m" (raid_gfcauchyaffine_raid[7][0])); + asm volatile ("vpbroadcastq %0,%%zmm19" : : "m" (raid_gfcauchyaffine_raid[7][1])); + asm volatile ("vpbroadcastq %0,%%zmm20" : : "m" (raid_gfcauchyaffine_raid[8][0])); + asm volatile ("vpbroadcastq %0,%%zmm21" : : "m" (raid_gfcauchyaffine_raid[8][1])); + asm volatile ("vpbroadcastq %0,%%zmm22" : : "m" (raid_gfcauchyaffine_raid[9][0])); + asm volatile ("vpbroadcastq %0,%%zmm23" : : "m" (raid_gfcauchyaffine_raid[9][1])); + asm volatile ("vpbroadcastq %0,%%zmm24" : : "m" (raid_gfcauchyaffine_raid[10][0])); + asm volatile ("vpbroadcastq %0,%%zmm25" : : "m" (raid_gfcauchyaffine_raid[10][1])); + asm volatile ("vpbroadcastq %0,%%zmm26" : : "m" (raid_gfcauchyaffine_raid[11][0])); + asm volatile ("vpbroadcastq %0,%%zmm27" : : "m" (raid_gfcauchyaffine_raid[11][1])); + asm volatile ("vpbroadcastq %0,%%zmm28" : : "m" (raid_gfcauchyaffine_raid[12][0])); + asm volatile ("vpbroadcastq %0,%%zmm29" : : "m" (raid_gfcauchyaffine_raid[12][1])); + asm volatile ("vpbroadcastq %0,%%zmm30" : : "m" (raid_gfcauchyaffine_raid[13][0])); + asm volatile ("vpbroadcastq %0,%%zmm31" : : "m" (raid_gfcauchyaffine_raid[13][1])); for (i = 0; i < size; i += 64) { asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); @@ -599,11 +599,11 @@ static __always_inline void raid_gen3_avx512gfni_raid_gen(int nd, size_t size, v asm volatile ("vmovdqa64 %0,%%zmm3" : : "m" (v[d][i])); asm volatile ("vpxorq %zmm3,%zmm0,%zmm0"); - asm volatile ("vpbroadcastq %0,%%zmm4" : : "m" (raid_gfcauchyaffine_raid[d][0][0])); + asm volatile ("vpbroadcastq %0,%%zmm4" : : "m" (raid_gfcauchyaffine_raid[d][0])); asm volatile ("vgf2p8affineqb $0,%zmm4,%zmm3,%zmm4"); asm volatile ("vpxorq %zmm4,%zmm1,%zmm1"); - asm volatile ("vpbroadcastq %0,%%zmm5" : : "m" (raid_gfcauchyaffine_raid[d][1][0])); + asm volatile ("vpbroadcastq %0,%%zmm5" : : "m" (raid_gfcauchyaffine_raid[d][1])); asm volatile ("vgf2p8affineqb $0,%zmm5,%zmm3,%zmm5"); asm volatile ("vpxorq %zmm5,%zmm2,%zmm2"); } @@ -659,15 +659,15 @@ static __always_inline void raid_genX_avx512gfni_raid(int nd, size_t size, void for (d = 1; d < nd; ++d) { asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[d][i])); - asm volatile ("vpbroadcastq %0,%%zmm7" : : "m" (raid_gfcauchyaffine_raid[d][0][0])); + asm volatile ("vpbroadcastq %0,%%zmm7" : : "m" (raid_gfcauchyaffine_raid[d][0])); if (np >= 3) - asm volatile ("vpbroadcastq %0,%%zmm8" : : "m" (raid_gfcauchyaffine_raid[d][1][0])); + asm volatile ("vpbroadcastq %0,%%zmm8" : : "m" (raid_gfcauchyaffine_raid[d][1])); if (np >= 4) - asm volatile ("vpbroadcastq %0,%%zmm9" : : "m" (raid_gfcauchyaffine_raid[d][2][0])); + asm volatile ("vpbroadcastq %0,%%zmm9" : : "m" (raid_gfcauchyaffine_raid[d][2])); if (np >= 5) - asm volatile ("vpbroadcastq %0,%%zmm10" : : "m" (raid_gfcauchyaffine_raid[d][3][0])); + asm volatile ("vpbroadcastq %0,%%zmm10" : : "m" (raid_gfcauchyaffine_raid[d][3])); if (np >= 6) - asm volatile ("vpbroadcastq %0,%%zmm11" : : "m" (raid_gfcauchyaffine_raid[d][4][0])); + asm volatile ("vpbroadcastq %0,%%zmm11" : : "m" (raid_gfcauchyaffine_raid[d][4])); asm volatile ("vgf2p8affineqb $0,%zmm7,%zmm6,%zmm12"); if (np >= 3) diff --git a/raid/neon.c b/raid/neon.c index 0ad46b33..34fd9bf9 100644 --- a/raid/neon.c +++ b/raid/neon.c @@ -80,7 +80,7 @@ static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, i asm volatile ( "ldr q29, %0\n" : - : "m" (gfconst16.poly[0]) + : "m" (gfconst16.poly) ); for (i = 0; i < size; i += 32) { @@ -167,7 +167,7 @@ static __always_inline void raid_genz_neon_gen(int nd, size_t size, void **vv) "ldr q29, %0\n" "ldr q30, %1\n" : - : "m" (gfconst16.poly[0]), "m" (gfconst16.half[0]) + : "m" (gfconst16.poly), "m" (gfconst16.half) ); for (i = 0; i < size; i += 32) { @@ -264,7 +264,7 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n "ldr q29, %0\n" "movi v28.16b, #0x0f\n" : - : "m" (gfconst16.poly[0]) + : "m" (gfconst16.poly) ); for (i = 0; i < size; i += 32) { diff --git a/raid/neon32.c b/raid/neon32.c index 4b37c83a..4ec4bfb0 100644 --- a/raid/neon32.c +++ b/raid/neon32.c @@ -85,7 +85,7 @@ static __always_inline void raid_gen2_neon32_gen(int nd, size_t size, asm volatile ( "vld1.8 {q14}, %0\n" : - : "Q" (gfconst16.poly[0]) + : "Q" (gfconst16.poly) ); for (i = 0; i < size; i += 32) { @@ -167,7 +167,7 @@ static __always_inline void raid_genz_neon32_gen(int nd, size_t size, void **vv) "vld1.8 {q14}, %0\n" "vld1.8 {q15}, %1\n" : - : "Q" (gfconst16.poly[0]), "Q" (gfconst16.half[0]) + : "Q" (gfconst16.poly), "Q" (gfconst16.half) ); for (i = 0; i < size; i += 32) { @@ -258,7 +258,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size, "vld1.8 {q14}, %0\n" "vmov.i8 q15, #0x0f\n" : - : "Q" (gfconst16.poly[0]) + : "Q" (gfconst16.poly) ); for (i = 0; i < size; i += 16) { @@ -288,8 +288,8 @@ static __always_inline void raid_genX_neon32(int nd, size_t size, "veor q2, q11, q12\n" : - : "Q" (raid_gfcauchypshufb[l][1][0][0]), - "Q" (raid_gfcauchypshufb[l][1][1][0]) + : "Q" (raid_gfcauchypshufb[l][1][0]), + "Q" (raid_gfcauchypshufb[l][1][1]) ); } @@ -305,8 +305,8 @@ static __always_inline void raid_genX_neon32(int nd, size_t size, "veor q3, q11, q12\n" : - : "Q" (raid_gfcauchypshufb[l][2][0][0]), - "Q" (raid_gfcauchypshufb[l][2][1][0]) + : "Q" (raid_gfcauchypshufb[l][2][0]), + "Q" (raid_gfcauchypshufb[l][2][1]) ); } @@ -322,8 +322,8 @@ static __always_inline void raid_genX_neon32(int nd, size_t size, "veor q4, q11, q12\n" : - : "Q" (raid_gfcauchypshufb[l][3][0][0]), - "Q" (raid_gfcauchypshufb[l][3][1][0]) + : "Q" (raid_gfcauchypshufb[l][3][0]), + "Q" (raid_gfcauchypshufb[l][3][1]) ); } @@ -339,8 +339,8 @@ static __always_inline void raid_genX_neon32(int nd, size_t size, "veor q5, q11, q12\n" : - : "Q" (raid_gfcauchypshufb[l][4][0][0]), - "Q" (raid_gfcauchypshufb[l][4][1][0]) + : "Q" (raid_gfcauchypshufb[l][4][0]), + "Q" (raid_gfcauchypshufb[l][4][1]) ); } @@ -386,8 +386,8 @@ static __always_inline void raid_genX_neon32(int nd, size_t size, "veor q11, q11, q12\n" "veor q2, q2, q11\n" : - : "Q" (raid_gfcauchypshufb[d][1][0][0]), - "Q" (raid_gfcauchypshufb[d][1][1][0]) + : "Q" (raid_gfcauchypshufb[d][1][0]), + "Q" (raid_gfcauchypshufb[d][1][1]) ); } @@ -402,8 +402,8 @@ static __always_inline void raid_genX_neon32(int nd, size_t size, "veor q11, q11, q12\n" "veor q3, q3, q11\n" : - : "Q" (raid_gfcauchypshufb[d][2][0][0]), - "Q" (raid_gfcauchypshufb[d][2][1][0]) + : "Q" (raid_gfcauchypshufb[d][2][0]), + "Q" (raid_gfcauchypshufb[d][2][1]) ); } @@ -418,8 +418,8 @@ static __always_inline void raid_genX_neon32(int nd, size_t size, "veor q11, q11, q12\n" "veor q4, q4, q11\n" : - : "Q" (raid_gfcauchypshufb[d][3][0][0]), - "Q" (raid_gfcauchypshufb[d][3][1][0]) + : "Q" (raid_gfcauchypshufb[d][3][0]), + "Q" (raid_gfcauchypshufb[d][3][1]) ); } @@ -434,8 +434,8 @@ static __always_inline void raid_genX_neon32(int nd, size_t size, "veor q11, q11, q12\n" "veor q5, q5, q11\n" : - : "Q" (raid_gfcauchypshufb[d][4][0][0]), - "Q" (raid_gfcauchypshufb[d][4][1][0]) + : "Q" (raid_gfcauchypshufb[d][4][0]), + "Q" (raid_gfcauchypshufb[d][4][1]) ); } } diff --git a/raid/sse2.c b/raid/sse2.c index 15e4c78c..478e4044 100644 --- a/raid/sse2.c +++ b/raid/sse2.c @@ -162,7 +162,7 @@ static __always_inline void raid_gen2_sse2_gen(int nd, size_t size, void **vv, i raid_sse_begin(); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly)); for (i = 0; i < size; i += 32) { asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i])); @@ -247,7 +247,7 @@ static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv raid_sse_begin(); - asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.poly)); for (i = 0; i < size; i += 64) { asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i])); @@ -486,9 +486,9 @@ static __always_inline void raid_genz_sse2_gen(int nd, size_t size, void **vv, i raid_sse_begin(); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); - asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.half[0])); - asm volatile ("movdqa %0,%%xmm6" : : "m" (gfconst16.low7[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly)); + asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.half)); + asm volatile ("movdqa %0,%%xmm6" : : "m" (gfconst16.low7)); for (i = 0; i < size; i += 16) { asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i])); @@ -551,9 +551,9 @@ static __always_inline void raid_genz_sse2ext_gen(int nd, size_t size, void **vv raid_sse_begin(); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); - asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.half[0])); - asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.low7[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly)); + asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.half)); + asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.low7)); for (i = 0; i < size; i += 32) { asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i])); diff --git a/raid/ssse3.c b/raid/ssse3.c index 3cc84150..3f2c0686 100644 --- a/raid/ssse3.c +++ b/raid/ssse3.c @@ -34,8 +34,8 @@ static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv, raid_sse_begin(); /* generic case with at least two data disks */ - asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.poly[0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.poly)); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4)); for (i = 0; i < size; i += 16) { /* last disk without the generator multiplication */ @@ -49,8 +49,8 @@ static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv, asm volatile ("pand %xmm7,%xmm4"); asm volatile ("pand %xmm7,%xmm5"); - asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0][0])); - asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[l][1][1][0])); + asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[l][1][1])); asm volatile ("pshufb %xmm4,%xmm2"); asm volatile ("pshufb %xmm5,%xmm6"); asm volatile ("pxor %xmm6,%xmm2"); @@ -77,10 +77,10 @@ static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv, asm volatile ("pand %xmm7,%xmm4"); asm volatile ("pand %xmm7,%xmm5"); - asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0])); asm volatile ("pshufb %xmm4,%xmm6"); asm volatile ("pxor %xmm6,%xmm2"); - asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][1])); asm volatile ("pshufb %xmm5,%xmm6"); asm volatile ("pxor %xmm6,%xmm2"); } @@ -148,7 +148,7 @@ static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv, /* generic case with at least two data disks */ for (i = 0; i < size; i += 16) { /* last disk without the generator multiplication */ - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4)); asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i])); asm volatile ("movdqa %xmm4,%xmm0"); @@ -159,21 +159,21 @@ static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv, asm volatile ("pand %xmm7,%xmm4"); asm volatile ("pand %xmm7,%xmm5"); - asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1][0])); + asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1])); asm volatile ("pshufb %xmm4,%xmm2"); asm volatile ("pshufb %xmm5,%xmm7"); asm volatile ("pxor %xmm7,%xmm2"); - asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][2][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1][0])); + asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][2][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1])); asm volatile ("pshufb %xmm4,%xmm3"); asm volatile ("pshufb %xmm5,%xmm7"); asm volatile ("pxor %xmm7,%xmm3"); /* intermediate disks */ for (d = l - 1; d > 0; --d) { - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly)); asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); if (generator == 3) @@ -186,7 +186,7 @@ static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv, if (generator == 3) asm volatile ("pxor %xmm6,%xmm1"); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4)); asm volatile ("pxor %xmm4,%xmm0"); asm volatile ("pxor %xmm4,%xmm1"); @@ -196,15 +196,15 @@ static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv, asm volatile ("pand %xmm7,%xmm4"); asm volatile ("pand %xmm7,%xmm5"); - asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1])); asm volatile ("pshufb %xmm4,%xmm6"); asm volatile ("pshufb %xmm5,%xmm7"); asm volatile ("pxor %xmm6,%xmm2"); asm volatile ("pxor %xmm7,%xmm2"); - asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1])); asm volatile ("pshufb %xmm4,%xmm6"); asm volatile ("pshufb %xmm5,%xmm7"); asm volatile ("pxor %xmm6,%xmm3"); @@ -212,7 +212,7 @@ static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv, } /* first disk with all coefficients at 1 */ - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly)); asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i])); if (generator == 3) @@ -289,26 +289,26 @@ static __always_inline void raid_gen5_ssse3_gen(int nd, size_t size, void **vv, asm volatile ("movdqa %xmm4,%xmm0"); asm volatile ("movdqa %%xmm4,%0" : "=m" (pd[0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4)); asm volatile ("movdqa %xmm4,%xmm5"); asm volatile ("psrlw $4,%xmm5"); asm volatile ("pand %xmm7,%xmm4"); asm volatile ("pand %xmm7,%xmm5"); - asm volatile ("movdqa %0,%%xmm1" : : "m" (raid_gfcauchypshufb[l][1][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1][0])); + asm volatile ("movdqa %0,%%xmm1" : : "m" (raid_gfcauchypshufb[l][1][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1])); asm volatile ("pshufb %xmm4,%xmm1"); asm volatile ("pshufb %xmm5,%xmm7"); asm volatile ("pxor %xmm7,%xmm1"); - asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][2][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1][0])); + asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][2][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1])); asm volatile ("pshufb %xmm4,%xmm2"); asm volatile ("pshufb %xmm5,%xmm7"); asm volatile ("pxor %xmm7,%xmm2"); - asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][3][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][3][1][0])); + asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][3][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][3][1])); asm volatile ("pshufb %xmm4,%xmm3"); asm volatile ("pshufb %xmm5,%xmm7"); asm volatile ("pxor %xmm7,%xmm3"); @@ -317,7 +317,7 @@ static __always_inline void raid_gen5_ssse3_gen(int nd, size_t size, void **vv, for (d = l - 1; d > 0; --d) { asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly)); asm volatile ("pxor %xmm5,%xmm5"); asm volatile ("pcmpgtb %xmm0,%xmm5"); @@ -329,28 +329,28 @@ static __always_inline void raid_gen5_ssse3_gen(int nd, size_t size, void **vv, asm volatile ("pxor %xmm4,%xmm6"); asm volatile ("movdqa %%xmm6,%0" : "=m" (pd[0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4)); asm volatile ("movdqa %xmm4,%xmm5"); asm volatile ("psrlw $4,%xmm5"); asm volatile ("pand %xmm7,%xmm4"); asm volatile ("pand %xmm7,%xmm5"); - asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1])); asm volatile ("pshufb %xmm4,%xmm6"); asm volatile ("pshufb %xmm5,%xmm7"); asm volatile ("pxor %xmm6,%xmm1"); asm volatile ("pxor %xmm7,%xmm1"); - asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1])); asm volatile ("pshufb %xmm4,%xmm6"); asm volatile ("pshufb %xmm5,%xmm7"); asm volatile ("pxor %xmm6,%xmm2"); asm volatile ("pxor %xmm7,%xmm2"); - asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][3][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][3][1][0])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][3][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][3][1])); asm volatile ("pshufb %xmm4,%xmm6"); asm volatile ("pshufb %xmm5,%xmm7"); asm volatile ("pxor %xmm6,%xmm3"); @@ -360,7 +360,7 @@ static __always_inline void raid_gen5_ssse3_gen(int nd, size_t size, void **vv, /* first disk with all coefficients at 1 */ asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i])); asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly)); asm volatile ("pxor %xmm5,%xmm5"); asm volatile ("pcmpgtb %xmm0,%xmm5"); @@ -437,32 +437,32 @@ static __always_inline void raid_gen6_ssse3_gen(int nd, size_t size, void **vv, asm volatile ("movdqa %%xmm4,%0" : "=m" (pd[0])); asm volatile ("movdqa %%xmm4,%0" : "=m" (pd[16])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4)); asm volatile ("movdqa %xmm4,%xmm5"); asm volatile ("psrlw $4,%xmm5"); asm volatile ("pand %xmm7,%xmm4"); asm volatile ("pand %xmm7,%xmm5"); - asm volatile ("movdqa %0,%%xmm0" : : "m" (raid_gfcauchypshufb[l][1][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1][0])); + asm volatile ("movdqa %0,%%xmm0" : : "m" (raid_gfcauchypshufb[l][1][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1])); asm volatile ("pshufb %xmm4,%xmm0"); asm volatile ("pshufb %xmm5,%xmm7"); asm volatile ("pxor %xmm7,%xmm0"); - asm volatile ("movdqa %0,%%xmm1" : : "m" (raid_gfcauchypshufb[l][2][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1][0])); + asm volatile ("movdqa %0,%%xmm1" : : "m" (raid_gfcauchypshufb[l][2][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1])); asm volatile ("pshufb %xmm4,%xmm1"); asm volatile ("pshufb %xmm5,%xmm7"); asm volatile ("pxor %xmm7,%xmm1"); - asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][3][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][3][1][0])); + asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][3][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][3][1])); asm volatile ("pshufb %xmm4,%xmm2"); asm volatile ("pshufb %xmm5,%xmm7"); asm volatile ("pxor %xmm7,%xmm2"); - asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][4][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][4][1][0])); + asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][4][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][4][1])); asm volatile ("pshufb %xmm4,%xmm3"); asm volatile ("pshufb %xmm5,%xmm7"); asm volatile ("pxor %xmm7,%xmm3"); @@ -471,7 +471,7 @@ static __always_inline void raid_gen6_ssse3_gen(int nd, size_t size, void **vv, for (d = l - 1; d > 0; --d) { asm volatile ("movdqa %0,%%xmm5" : : "m" (pd[0])); asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[16])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly)); asm volatile ("pxor %xmm4,%xmm4"); asm volatile ("pcmpgtb %xmm6,%xmm4"); @@ -486,35 +486,35 @@ static __always_inline void raid_gen6_ssse3_gen(int nd, size_t size, void **vv, asm volatile ("movdqa %%xmm5,%0" : "=m" (pd[0])); asm volatile ("movdqa %%xmm6,%0" : "=m" (pd[16])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4)); asm volatile ("movdqa %xmm4,%xmm5"); asm volatile ("psrlw $4,%xmm5"); asm volatile ("pand %xmm7,%xmm4"); asm volatile ("pand %xmm7,%xmm5"); - asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1])); asm volatile ("pshufb %xmm4,%xmm6"); asm volatile ("pshufb %xmm5,%xmm7"); asm volatile ("pxor %xmm6,%xmm0"); asm volatile ("pxor %xmm7,%xmm0"); - asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1])); asm volatile ("pshufb %xmm4,%xmm6"); asm volatile ("pshufb %xmm5,%xmm7"); asm volatile ("pxor %xmm6,%xmm1"); asm volatile ("pxor %xmm7,%xmm1"); - asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][3][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][3][1][0])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][3][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][3][1])); asm volatile ("pshufb %xmm4,%xmm6"); asm volatile ("pshufb %xmm5,%xmm7"); asm volatile ("pxor %xmm6,%xmm2"); asm volatile ("pxor %xmm7,%xmm2"); - asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][4][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][4][1][0])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][4][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][4][1])); asm volatile ("pshufb %xmm4,%xmm6"); asm volatile ("pshufb %xmm5,%xmm7"); asm volatile ("pxor %xmm6,%xmm3"); @@ -524,7 +524,7 @@ static __always_inline void raid_gen6_ssse3_gen(int nd, size_t size, void **vv, /* first disk with all coefficients at 1 */ asm volatile ("movdqa %0,%%xmm5" : : "m" (pd[0])); asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[16])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly)); asm volatile ("pxor %xmm4,%xmm4"); asm volatile ("pcmpgtb %xmm6,%xmm4"); @@ -594,8 +594,8 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v raid_sse_begin(); /* generic case with at least two data disks */ - asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.poly[0])); - asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.poly)); + asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.low4)); for (i = 0; i < size; i += 32) { /* last disk without the generator multiplication */ @@ -616,8 +616,8 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v asm volatile ("pand %xmm11,%xmm5"); asm volatile ("pand %xmm11,%xmm13"); - asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1][0])); + asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1])); asm volatile ("movdqa %xmm2,%xmm10"); asm volatile ("movdqa %xmm7,%xmm15"); asm volatile ("pshufb %xmm4,%xmm2"); @@ -632,8 +632,8 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][0][0])); - asm volatile ("movdqa %0,%%xmm15" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][0])); + asm volatile ("movdqa %0,%%xmm15" : : "m" (raid_gfcauchypshufb[d][1][1])); if (generator == 3) { asm volatile ("movdqa %xmm1,%xmm6"); @@ -682,8 +682,8 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d - 1][i])); asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d - 1][i + 16])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d - 1][1][0][0])); - asm volatile ("movdqa %0,%%xmm15" : : "m" (raid_gfcauchypshufb[d - 1][1][1][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d - 1][1][0])); + asm volatile ("movdqa %0,%%xmm15" : : "m" (raid_gfcauchypshufb[d - 1][1][1])); if (generator == 3) { asm volatile ("movdqa %xmm1,%xmm6"); @@ -768,8 +768,8 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v asm volatile ("pand %xmm11,%xmm5"); asm volatile ("pand %xmm11,%xmm13"); - asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[1][1][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[1][1][1][0])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[1][1][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[1][1][1])); asm volatile ("movdqa %xmm6,%xmm14"); asm volatile ("movdqa %xmm7,%xmm15"); asm volatile ("pshufb %xmm4,%xmm6"); @@ -869,7 +869,7 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v /* generic case with at least two data disks */ for (i = 0; i < size; i += 32) { /* last disk without the generator multiplication */ - asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4)); asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i])); asm volatile ("movdqa %0,%%xmm12" : : "m" (v[l][i + 16])); @@ -887,8 +887,8 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v asm volatile ("pand %xmm15,%xmm5"); asm volatile ("pand %xmm15,%xmm13"); - asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1][0])); + asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1])); asm volatile ("movdqa %xmm2,%xmm10"); asm volatile ("movdqa %xmm7,%xmm15"); asm volatile ("pshufb %xmm4,%xmm2"); @@ -898,8 +898,8 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v asm volatile ("pxor %xmm7,%xmm2"); asm volatile ("pxor %xmm15,%xmm10"); - asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][2][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1][0])); + asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][2][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1])); asm volatile ("movdqa %xmm3,%xmm11"); asm volatile ("movdqa %xmm7,%xmm15"); asm volatile ("pshufb %xmm4,%xmm3"); @@ -911,8 +911,8 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v /* intermediate disks */ for (d = l - 1; d > 0; --d) { - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); - asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly)); + asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4)); asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16])); @@ -949,8 +949,8 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v asm volatile ("pand %xmm15,%xmm5"); asm volatile ("pand %xmm15,%xmm13"); - asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1])); asm volatile ("movdqa %xmm6,%xmm14"); asm volatile ("movdqa %xmm7,%xmm15"); asm volatile ("pshufb %xmm4,%xmm6"); @@ -962,8 +962,8 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v asm volatile ("pxor %xmm7,%xmm2"); asm volatile ("pxor %xmm15,%xmm10"); - asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1])); asm volatile ("movdqa %xmm6,%xmm14"); asm volatile ("movdqa %xmm7,%xmm15"); asm volatile ("pshufb %xmm4,%xmm6"); @@ -977,8 +977,8 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v } /* first disk with all coefficients at 1 */ - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); - asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly)); + asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4)); asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i])); asm volatile ("movdqa %0,%%xmm12" : : "m" (v[0][i + 16])); @@ -1069,8 +1069,8 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i raid_sse_begin(); /* generic case with at least two data disks */ - asm volatile ("movdqa %0,%%xmm14" : : "m" (gfconst16.poly[0])); - asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm14" : : "m" (gfconst16.poly)); + asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4)); for (i = 0; i < size; i += 16) { /* last disk without the generator multiplication */ @@ -1085,32 +1085,32 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i asm volatile ("pand %xmm15,%xmm11"); if (np >= 3) { - asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0][0])); - asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[l][1][1][0])); + asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[l][1][1])); asm volatile ("pshufb %xmm10,%xmm2"); asm volatile ("pshufb %xmm11,%xmm13"); asm volatile ("pxor %xmm13,%xmm2"); } if (np >= 4) { - asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][2][0][0])); - asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[l][2][1][0])); + asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][2][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[l][2][1])); asm volatile ("pshufb %xmm10,%xmm3"); asm volatile ("pshufb %xmm11,%xmm13"); asm volatile ("pxor %xmm13,%xmm3"); } if (np >= 5) { - asm volatile ("movdqa %0,%%xmm4" : : "m" (raid_gfcauchypshufb[l][3][0][0])); - asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[l][3][1][0])); + asm volatile ("movdqa %0,%%xmm4" : : "m" (raid_gfcauchypshufb[l][3][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[l][3][1])); asm volatile ("pshufb %xmm10,%xmm4"); asm volatile ("pshufb %xmm11,%xmm13"); asm volatile ("pxor %xmm13,%xmm4"); } if (np >= 6) { - asm volatile ("movdqa %0,%%xmm5" : : "m" (raid_gfcauchypshufb[l][4][0][0])); - asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[l][4][1][0])); + asm volatile ("movdqa %0,%%xmm5" : : "m" (raid_gfcauchypshufb[l][4][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[l][4][1])); asm volatile ("pshufb %xmm10,%xmm5"); asm volatile ("pshufb %xmm11,%xmm13"); asm volatile ("pxor %xmm13,%xmm5"); @@ -1139,8 +1139,8 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i asm volatile ("pand %xmm15,%xmm11"); if (np >= 3) { - asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][1][0][0])); - asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][1][1][0])); + asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][1][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][1][1])); asm volatile ("pshufb %xmm10,%xmm12"); asm volatile ("pshufb %xmm11,%xmm13"); asm volatile ("pxor %xmm12,%xmm2"); @@ -1148,8 +1148,8 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i } if (np >= 4) { - asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][2][0][0])); - asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][2][1][0])); + asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][2][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][2][1])); asm volatile ("pshufb %xmm10,%xmm12"); asm volatile ("pshufb %xmm11,%xmm13"); asm volatile ("pxor %xmm12,%xmm3"); @@ -1157,8 +1157,8 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i } if (np >= 5) { - asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][3][0][0])); - asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][3][1][0])); + asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][3][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][3][1])); asm volatile ("pshufb %xmm10,%xmm12"); asm volatile ("pshufb %xmm11,%xmm13"); asm volatile ("pxor %xmm12,%xmm4"); @@ -1166,8 +1166,8 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i } if (np >= 6) { - asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][4][0][0])); - asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][4][1][0])); + asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][4][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][4][1])); asm volatile ("pshufb %xmm10,%xmm12"); asm volatile ("pshufb %xmm11,%xmm13"); asm volatile ("pxor %xmm12,%xmm5"); @@ -1268,9 +1268,9 @@ static __always_inline void raid_rec1_ssse3_1(int *id, int *ip, int nd, size_t s raid_sse_begin(); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); - asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfmulpshufb[V][0][0])); - asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfmulpshufb[V][1][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4)); + asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfmulpshufb[V][0])); + asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfmulpshufb[V][1])); for (i = 0; i < size; i += 16) { /* start from the selected parity block */ @@ -1350,11 +1350,11 @@ static __always_inline void raid_rec1_ssse3_q(int *id, int *ip, int nd, size_t s raid_sse_begin(); /* keep the inverse coefficient resident */ - asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfmulpshufb[V][0][0])); - asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfmulpshufb[V][1][0])); + asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfmulpshufb[V][0])); + asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfmulpshufb[V][1])); /* keep the active reduction polynomial resident */ - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly)); for (i = 0; i < size; i += 32) { /* last disk starts Horner without generator multiplication */ @@ -1413,7 +1413,7 @@ static __always_inline void raid_rec1_ssse3_q(int *id, int *ip, int nd, size_t s asm volatile ("pxor %0,%%xmm1" : : "m" (q[i + 16])); /* low-nibble mask */ - asm volatile ("movdqa %0,%%xmm6" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (gfconst16.low4)); /* split both Qdelta lanes into low/high nibbles */ asm volatile ("movdqa %xmm0,%xmm4"); @@ -1444,7 +1444,7 @@ static __always_inline void raid_rec1_ssse3_q(int *id, int *ip, int nd, size_t s asm volatile ("movdqa %%xmm7,%0" : "=m" (pa[i + 16])); /* restore the active reduction polynomial */ - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly)); } raid_sse_end(0); @@ -1518,7 +1518,7 @@ static __always_inline void raid_rec2_ssse3_2(int has_p, int *id, int *ip, int n raid_sse_begin(); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4)); for (i = 0; i < size; i += 16) { /* start from the two selected parity blocks */ @@ -1692,7 +1692,7 @@ static __always_inline void raid_rec2of2_ssse3(int *id, int *ip, int nd, size_t * xmm4-xmm6 temporaries * xmm7 active reduction polynomial */ - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly)); for (i = 0; i < size; i += 32) { /* Pdelta starts directly from stored P */ @@ -1771,7 +1771,7 @@ static __always_inline void raid_rec2of2_ssse3(int *id, int *ip, int nd, size_t * The survivor scan is complete, so xmm7 can now hold * the low-nibble mask instead of the polynomial. */ - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4)); /* * Lane 0. @@ -1862,7 +1862,7 @@ static __always_inline void raid_rec2of2_ssse3(int *id, int *ip, int nd, size_t /* * Restore the polynomial for the next 32-byte iteration. */ - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly)); } raid_sse_end(0); @@ -1935,7 +1935,7 @@ static __always_inline void raid_recX_ssse3_1234(int nr, int has_p, int *id, int raid_sse_begin(); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4)); for (i = 0; i < size; i += 16) { asm volatile ("movdqa %0,%%xmm0" : : "m" (p[0][i])); @@ -2157,7 +2157,7 @@ static __always_inline void raid_recX_ssse3(int nr, int has_p, int *id, int *ip, raid_sse_begin(); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4)); for (i = 0; i < size; i += 16) { /* pd_low[] temporarily contains the complete raw syndrome */ @@ -2294,11 +2294,11 @@ static __always_inline void raid_rec1_ssse3ext_1(int *id, int *ip, int nd, size_ raid_sse_begin(); - asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4)); /* keep the inverse coefficient resident */ - asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfmulpshufb[V][0][0])); - asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfmulpshufb[V][1][0])); + asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfmulpshufb[V][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfmulpshufb[V][1])); for (i = 0; i < size; i += 32) { /* start both lanes from the selected parity block */ @@ -2414,10 +2414,10 @@ static __always_inline void raid_rec1_ssse3ext_q(int *id, int *ip, int nd, size_ * xmm14 active reduction polynomial * xmm15 low-nibble mask */ - asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfmulpshufb[V][0][0])); - asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfmulpshufb[V][1][0])); - asm volatile ("movdqa %0,%%xmm14" : : "m" (gfconst16.poly[0])); - asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfmulpshufb[V][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfmulpshufb[V][1])); + asm volatile ("movdqa %0,%%xmm14" : : "m" (gfconst16.poly)); + asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4)); for (i = 0; i < size; i += 64) { /* last disk starts Horner without generator multiplication */ @@ -2648,14 +2648,14 @@ static __always_inline void raid_rec2_ssse3ext_2(int has_p, int *id, int *ip, in */ /* keep the first inverse-matrix row resident */ - asm volatile ("movdqa %0,%%xmm8" : : "m" (raid_gfmulpshufb[V[0]][0][0])); - asm volatile ("movdqa %0,%%xmm9" : : "m" (raid_gfmulpshufb[V[0]][1][0])); - asm volatile ("movdqa %0,%%xmm10" : : "m" (raid_gfmulpshufb[V[1]][0][0])); - asm volatile ("movdqa %0,%%xmm11" : : "m" (raid_gfmulpshufb[V[1]][1][0])); + asm volatile ("movdqa %0,%%xmm8" : : "m" (raid_gfmulpshufb[V[0]][0])); + asm volatile ("movdqa %0,%%xmm9" : : "m" (raid_gfmulpshufb[V[0]][1])); + asm volatile ("movdqa %0,%%xmm10" : : "m" (raid_gfmulpshufb[V[1]][0])); + asm volatile ("movdqa %0,%%xmm11" : : "m" (raid_gfmulpshufb[V[1]][1])); for (i = 0; i < size; i += 32) { /* xmm14 is reused during reconstruction, therefore restore the mask for each iteration */ - asm volatile ("movdqa %0,%%xmm14" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm14" : : "m" (gfconst16.low4)); /* start both syndrome lanes from the selected stored parity blocks */ asm volatile ("movdqa %0,%%xmm0" : : "m" (p[0][i])); @@ -2848,7 +2848,7 @@ static __always_inline void raid_rec2_ssse3ext_2(int has_p, int *id, int *ip, in */ /* V[2] * syndrome0 low */ - asm volatile ("movdqa %0,%%xmm14" : : "m" (raid_gfmulpshufb[V[2]][0][0])); + asm volatile ("movdqa %0,%%xmm14" : : "m" (raid_gfmulpshufb[V[2]][0])); asm volatile ("movdqa %xmm14,%xmm15"); asm volatile ("pshufb %xmm0,%xmm14"); asm volatile ("pshufb %xmm1,%xmm15"); @@ -2856,7 +2856,7 @@ static __always_inline void raid_rec2_ssse3ext_2(int has_p, int *id, int *ip, in asm volatile ("movdqa %xmm15,%xmm7"); /* V[2] * syndrome0 high */ - asm volatile ("movdqa %0,%%xmm14" : : "m" (raid_gfmulpshufb[V[2]][1][0])); + asm volatile ("movdqa %0,%%xmm14" : : "m" (raid_gfmulpshufb[V[2]][1])); asm volatile ("movdqa %xmm14,%xmm15"); asm volatile ("pshufb %xmm4,%xmm14"); asm volatile ("pshufb %xmm5,%xmm15"); @@ -2864,7 +2864,7 @@ static __always_inline void raid_rec2_ssse3ext_2(int has_p, int *id, int *ip, in asm volatile ("pxor %xmm15,%xmm7"); /* V[3] * syndrome1 low */ - asm volatile ("movdqa %0,%%xmm14" : : "m" (raid_gfmulpshufb[V[3]][0][0])); + asm volatile ("movdqa %0,%%xmm14" : : "m" (raid_gfmulpshufb[V[3]][0])); asm volatile ("movdqa %xmm14,%xmm15"); asm volatile ("pshufb %xmm2,%xmm14"); asm volatile ("pshufb %xmm3,%xmm15"); @@ -2872,7 +2872,7 @@ static __always_inline void raid_rec2_ssse3ext_2(int has_p, int *id, int *ip, in asm volatile ("pxor %xmm15,%xmm7"); /* V[3] * syndrome1 high */ - asm volatile ("movdqa %0,%%xmm14" : : "m" (raid_gfmulpshufb[V[3]][1][0])); + asm volatile ("movdqa %0,%%xmm14" : : "m" (raid_gfmulpshufb[V[3]][1])); asm volatile ("movdqa %xmm14,%xmm15"); asm volatile ("pshufb %xmm12,%xmm14"); asm volatile ("pshufb %xmm13,%xmm15"); @@ -2941,7 +2941,7 @@ static __always_inline void raid_rec2of2_ssse3ext(int *id, int *ip, int nd, size * xmm11 active reduction polynomial * xmm12-xmm15 analytical multiplication tables */ - asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.poly)); /* keep the analytical multiplication tables resident */ asm volatile ("movdqa %0,%%xmm12" : : "m" (R[0][0][0])); @@ -3077,7 +3077,7 @@ static __always_inline void raid_rec2of2_ssse3ext(int *id, int *ip, int nd, size asm volatile ("pxor %0,%%xmm7" : : "m" (q[i + 48])); /* the polynomial is no longer needed during reconstruction */ - asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.low4)); /* lane 0: split Pdelta */ asm volatile ("movdqa %xmm0,%xmm8"); @@ -3224,7 +3224,7 @@ static __always_inline void raid_rec2of2_ssse3ext(int *id, int *ip, int nd, size asm volatile ("movdqa %%xmm10,%0" : "=m" (qa[i + 48])); /* restore the polynomial for the next 64-byte iteration */ - asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.poly)); } raid_sse_end(0); @@ -3295,7 +3295,7 @@ static __always_inline void raid_recX_ssse3ext_123(int nr, int has_p, int *id, i * Xmm15 is later reused by reconstruction, therefore reload the * nibble mask on every iteration. */ - asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4)); asm volatile ("movdqa %0,%%xmm0" : : "m" (p[0][i])); asm volatile ("movdqa %0,%%xmm1" : : "m" (p[0][i + 16])); @@ -3555,7 +3555,7 @@ static __always_inline void raid_recX_ssse3ext_12345(int nr, int has_p, int *id, * xmm15 is reused by reconstruction, therefore reload the * nibble mask on every iteration. */ - asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4)); /* * Raw syndrome accumulators: @@ -3848,7 +3848,7 @@ static __always_inline void raid_recX_ssse3ext(int nr, int has_p, int *id, int * * Xmm15 is later reused by reconstruction, therefore reload the * nibble mask on every iteration. */ - asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4)); asm volatile ("movdqa %0,%%xmm0" : : "m" (p[0][i]));