rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-25 18:40:19 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix inline asm memory operand sizes for array-backed constants
Some checks are pending
Build / build-mac (push) Waiting to run
Details
Build / build-linux (push) Waiting to run
Details
Build / build-arm32 (push) Waiting to run
Details
Build / build-win (push) Waiting to run
Details
Build / build-freebsd (push) Waiting to run
Details
Build / build-musl (push) Waiting to run
Details
CodeQL Advanced / Analyze (c-cpp) (push) Waiting to run
Details
CodeQL Advanced / Analyze (actions) (push) Waiting to run
Details

Browse source
This commit is contained in:
Andrea Mazzoleni 2026-09-25 13:33:26 +02:00
parent 608e78f3a6
commit b070cbb894
7 changed files with 330 additions and 330 deletions
Download patch file Download diff file Expand all files Collapse all files

72
raid/avx2.c
View file

@ -380,8 +380,8 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv
asm volatile ("vpand %ymm11,%ymm5,%ymm5");
asm volatile ("vpand %ymm11,%ymm13,%ymm13");
asm volatile ("vbroadcasti128 %0,%%ymm10" : : "m" (raid_gfcauchypshufb[l][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[l][1][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm10" : : "m" (raid_gfcauchypshufb[l][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[l][1][1]));
asm volatile ("vpshufb %ymm4,%ymm10,%ymm2");
asm volatile ("vpshufb %ymm12,%ymm10,%ymm10");
asm volatile ("vpshufb %ymm5,%ymm15,%ymm7");
@ -394,8 +394,8 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i]));
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i + 32]));
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[d][1][1]));
if (generator == 3)
asm volatile ("vmovdqa %ymm1,%ymm6");
@ -444,8 +444,8 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d - 1][i]));
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d - 1][i + 32]));
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d - 1][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[d - 1][1][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d - 1][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[d - 1][1][1]));
if (generator == 3)
asm volatile ("vmovdqa %ymm1,%ymm6");
@ -514,8 +514,8 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv
asm volatile ("vpand %ymm11,%ymm4,%ymm4");
asm volatile ("vpand %ymm11,%ymm5,%ymm5");
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[1][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[1][1][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[1][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[1][1][1]));
asm volatile ("vpshufb %ymm4,%ymm14,%ymm6");
asm volatile ("vpshufb %ymm5,%ymm15,%ymm7");
@ -651,8 +651,8 @@ static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv
asm volatile ("vpand %ymm7,%ymm12,%ymm12");
asm volatile ("vpand %ymm7,%ymm13,%ymm13");
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[l][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[l][1][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[l][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[l][1][1]));
asm volatile ("vpshufb %ymm4,%ymm14,%ymm2");
asm volatile ("vpshufb %ymm5,%ymm15,%ymm3");
asm volatile ("vpxor %ymm3,%ymm2,%ymm2");
@ -660,8 +660,8 @@ static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv
asm volatile ("vpshufb %ymm13,%ymm15,%ymm11");
asm volatile ("vpxor %ymm11,%ymm10,%ymm10");
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[l][2][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[l][2][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[l][2][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[l][2][1]));
asm volatile ("vpshufb %ymm4,%ymm14,%ymm3");
asm volatile ("vpshufb %ymm5,%ymm15,%ymm5");
asm volatile ("vpxor %ymm5,%ymm3,%ymm3");
@ -704,23 +704,23 @@ static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv
asm volatile ("vpand %ymm7,%ymm12,%ymm12");
asm volatile ("vpand %ymm7,%ymm13,%ymm13");
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][0]));
asm volatile ("vpshufb %ymm4,%ymm14,%ymm15");
asm volatile ("vpshufb %ymm12,%ymm14,%ymm14");
asm volatile ("vpxor %ymm15,%ymm2,%ymm2");
asm volatile ("vpxor %ymm14,%ymm10,%ymm10");
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][1]));
asm volatile ("vpshufb %ymm5,%ymm14,%ymm15");
asm volatile ("vpshufb %ymm13,%ymm14,%ymm14");
asm volatile ("vpxor %ymm15,%ymm2,%ymm2");
asm volatile ("vpxor %ymm14,%ymm10,%ymm10");
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][2][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][2][0]));
asm volatile ("vpshufb %ymm4,%ymm14,%ymm15");
asm volatile ("vpshufb %ymm12,%ymm14,%ymm14");
asm volatile ("vpxor %ymm15,%ymm3,%ymm3");
asm volatile ("vpxor %ymm14,%ymm11,%ymm11");
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][2][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][2][1]));
asm volatile ("vpshufb %ymm5,%ymm14,%ymm15");
asm volatile ("vpshufb %ymm13,%ymm14,%ymm14");
asm volatile ("vpxor %ymm15,%ymm3,%ymm3");
@ -819,29 +819,29 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
asm volatile ("vpand %ymm15,%ymm10,%ymm10");
asm volatile ("vpand %ymm15,%ymm11,%ymm11");
if (np >= 3) {
asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (raid_gfcauchypshufb[l][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[l][1][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (raid_gfcauchypshufb[l][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[l][1][1]));
asm volatile ("vpshufb %ymm10,%ymm2,%ymm2");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm13,%ymm2,%ymm2");
}
if (np >= 4) {
asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (raid_gfcauchypshufb[l][2][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[l][2][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (raid_gfcauchypshufb[l][2][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[l][2][1]));
asm volatile ("vpshufb %ymm10,%ymm3,%ymm3");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm13,%ymm3,%ymm3");
}
if (np >= 5) {
asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (raid_gfcauchypshufb[l][3][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[l][3][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (raid_gfcauchypshufb[l][3][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[l][3][1]));
asm volatile ("vpshufb %ymm10,%ymm4,%ymm4");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm13,%ymm4,%ymm4");
}
if (np >= 6) {
asm volatile ("vbroadcasti128 %0,%%ymm5" : : "m" (raid_gfcauchypshufb[l][4][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[l][4][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm5" : : "m" (raid_gfcauchypshufb[l][4][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[l][4][1]));
asm volatile ("vpshufb %ymm10,%ymm5,%ymm5");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm13,%ymm5,%ymm5");
@ -867,32 +867,32 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
asm volatile ("vpand %ymm15,%ymm10,%ymm10");
asm volatile ("vpand %ymm15,%ymm11,%ymm11");
if (np >= 3) {
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfcauchypshufb[d][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[d][1][1]));
asm volatile ("vpshufb %ymm10,%ymm12,%ymm12");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm12,%ymm2,%ymm2");
asm volatile ("vpxor %ymm13,%ymm2,%ymm2");
}
if (np >= 4) {
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfcauchypshufb[d][2][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[d][2][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfcauchypshufb[d][2][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[d][2][1]));
asm volatile ("vpshufb %ymm10,%ymm12,%ymm12");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm12,%ymm3,%ymm3");
asm volatile ("vpxor %ymm13,%ymm3,%ymm3");
}
if (np >= 5) {
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfcauchypshufb[d][3][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[d][3][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfcauchypshufb[d][3][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[d][3][1]));
asm volatile ("vpshufb %ymm10,%ymm12,%ymm12");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm12,%ymm4,%ymm4");
asm volatile ("vpxor %ymm13,%ymm4,%ymm4");
}
if (np >= 6) {
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfcauchypshufb[d][4][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[d][4][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfcauchypshufb[d][4][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[d][4][1]));
asm volatile ("vpshufb %ymm10,%ymm12,%ymm12");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm12,%ymm5,%ymm5");
@ -1090,8 +1090,8 @@ static __always_inline void raid_rec1_avx2_q(int *id, int *ip, int nd, size_t si
raid_avx_begin();
/* keep the inverse coefficient resident */
asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (raid_gfmulpshufb[V][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (raid_gfmulpshufb[V][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (raid_gfmulpshufb[V][0]));
asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (raid_gfmulpshufb[V][1]));
/* keep zero and the active reduction polynomial resident */
asm volatile ("vpxor %ymm6,%ymm6,%ymm6");
@ -1637,8 +1637,8 @@ static __always_inline void raid_rec1_avx2ext_q(int *id, int *ip, int nd, size_t
raid_avx_begin();
/* keep the inverse coefficient, zero, and active reduction polynomial resident */
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfmulpshufb[V][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfmulpshufb[V][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfmulpshufb[V][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfmulpshufb[V][1]));
asm volatile ("vpxor %ymm14,%ymm14,%ymm14");
asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.poly[0]));

132
raid/avx512.c
View file

@ -77,32 +77,32 @@ static __always_inline void raid_gen2_avx512bw_gen(int nd, size_t size, void **v
asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gfconst16.low4[0]));
/* preload as many Q coefficient tables as possible */
asm volatile ("vbroadcasti32x4 %0,%%zmm4" : : "m" (raid_gfcauchypshufb[1][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm5" : : "m" (raid_gfcauchypshufb[1][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm6" : : "m" (raid_gfcauchypshufb[2][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm7" : : "m" (raid_gfcauchypshufb[2][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm8" : : "m" (raid_gfcauchypshufb[3][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm9" : : "m" (raid_gfcauchypshufb[3][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm10" : : "m" (raid_gfcauchypshufb[4][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm11" : : "m" (raid_gfcauchypshufb[4][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (raid_gfcauchypshufb[5][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (raid_gfcauchypshufb[5][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (raid_gfcauchypshufb[6][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (raid_gfcauchypshufb[6][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm16" : : "m" (raid_gfcauchypshufb[7][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm17" : : "m" (raid_gfcauchypshufb[7][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm18" : : "m" (raid_gfcauchypshufb[8][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm19" : : "m" (raid_gfcauchypshufb[8][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm20" : : "m" (raid_gfcauchypshufb[9][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm21" : : "m" (raid_gfcauchypshufb[9][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm22" : : "m" (raid_gfcauchypshufb[10][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm23" : : "m" (raid_gfcauchypshufb[10][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm24" : : "m" (raid_gfcauchypshufb[11][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm25" : : "m" (raid_gfcauchypshufb[11][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm26" : : "m" (raid_gfcauchypshufb[12][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm27" : : "m" (raid_gfcauchypshufb[12][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm28" : : "m" (raid_gfcauchypshufb[13][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm29" : : "m" (raid_gfcauchypshufb[13][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm4" : : "m" (raid_gfcauchypshufb[1][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm5" : : "m" (raid_gfcauchypshufb[1][0][1]));
asm volatile ("vbroadcasti32x4 %0,%%zmm6" : : "m" (raid_gfcauchypshufb[2][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm7" : : "m" (raid_gfcauchypshufb[2][0][1]));
asm volatile ("vbroadcasti32x4 %0,%%zmm8" : : "m" (raid_gfcauchypshufb[3][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm9" : : "m" (raid_gfcauchypshufb[3][0][1]));
asm volatile ("vbroadcasti32x4 %0,%%zmm10" : : "m" (raid_gfcauchypshufb[4][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm11" : : "m" (raid_gfcauchypshufb[4][0][1]));
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (raid_gfcauchypshufb[5][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (raid_gfcauchypshufb[5][0][1]));
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (raid_gfcauchypshufb[6][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (raid_gfcauchypshufb[6][0][1]));
asm volatile ("vbroadcasti32x4 %0,%%zmm16" : : "m" (raid_gfcauchypshufb[7][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm17" : : "m" (raid_gfcauchypshufb[7][0][1]));
asm volatile ("vbroadcasti32x4 %0,%%zmm18" : : "m" (raid_gfcauchypshufb[8][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm19" : : "m" (raid_gfcauchypshufb[8][0][1]));
asm volatile ("vbroadcasti32x4 %0,%%zmm20" : : "m" (raid_gfcauchypshufb[9][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm21" : : "m" (raid_gfcauchypshufb[9][0][1]));
asm volatile ("vbroadcasti32x4 %0,%%zmm22" : : "m" (raid_gfcauchypshufb[10][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm23" : : "m" (raid_gfcauchypshufb[10][0][1]));
asm volatile ("vbroadcasti32x4 %0,%%zmm24" : : "m" (raid_gfcauchypshufb[11][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm25" : : "m" (raid_gfcauchypshufb[11][0][1]));
asm volatile ("vbroadcasti32x4 %0,%%zmm26" : : "m" (raid_gfcauchypshufb[12][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm27" : : "m" (raid_gfcauchypshufb[12][0][1]));
asm volatile ("vbroadcasti32x4 %0,%%zmm28" : : "m" (raid_gfcauchypshufb[13][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm29" : : "m" (raid_gfcauchypshufb[13][0][1]));
for (i = 0; i < size; i += 64) {
asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i]));
@ -264,10 +264,10 @@ static __always_inline void raid_gen2_avx512bw_gen(int nd, size_t size, void **v
asm volatile ("vpandq %zmm31,%zmm2,%zmm2");
asm volatile ("vpandq %zmm31,%zmm3,%zmm3");
asm volatile ("vbroadcasti32x4 %0,%%zmm30" : : "m" (raid_gfcauchypshufb[d][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm30" : : "m" (raid_gfcauchypshufb[d][0][0]));
asm volatile ("vpshufb %zmm2,%zmm30,%zmm2");
asm volatile ("vbroadcasti32x4 %0,%%zmm30" : : "m" (raid_gfcauchypshufb[d][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm30" : : "m" (raid_gfcauchypshufb[d][0][1]));
asm volatile ("vpshufb %zmm3,%zmm30,%zmm3");
asm volatile ("vpternlogq $0x96,%zmm2,%zmm3,%zmm1");
@ -313,30 +313,30 @@ static __always_inline void raid_gen3_avx512bw_gen(int nd, size_t size, void **v
asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gfconst16.low4[0]));
/* preload as many Q and R coefficient tables as possible */
asm volatile ("vbroadcasti32x4 %0,%%zmm7" : : "m" (raid_gfcauchypshufb[1][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm8" : : "m" (raid_gfcauchypshufb[1][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm9" : : "m" (raid_gfcauchypshufb[1][1][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm10" : : "m" (raid_gfcauchypshufb[1][1][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm11" : : "m" (raid_gfcauchypshufb[2][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (raid_gfcauchypshufb[2][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (raid_gfcauchypshufb[2][1][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (raid_gfcauchypshufb[2][1][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (raid_gfcauchypshufb[3][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm16" : : "m" (raid_gfcauchypshufb[3][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm17" : : "m" (raid_gfcauchypshufb[3][1][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm18" : : "m" (raid_gfcauchypshufb[3][1][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm19" : : "m" (raid_gfcauchypshufb[4][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm20" : : "m" (raid_gfcauchypshufb[4][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm21" : : "m" (raid_gfcauchypshufb[4][1][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm22" : : "m" (raid_gfcauchypshufb[4][1][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm23" : : "m" (raid_gfcauchypshufb[5][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm24" : : "m" (raid_gfcauchypshufb[5][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm25" : : "m" (raid_gfcauchypshufb[5][1][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm26" : : "m" (raid_gfcauchypshufb[5][1][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm27" : : "m" (raid_gfcauchypshufb[6][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm28" : : "m" (raid_gfcauchypshufb[6][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm29" : : "m" (raid_gfcauchypshufb[6][1][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm30" : : "m" (raid_gfcauchypshufb[6][1][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm7" : : "m" (raid_gfcauchypshufb[1][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm8" : : "m" (raid_gfcauchypshufb[1][0][1]));
asm volatile ("vbroadcasti32x4 %0,%%zmm9" : : "m" (raid_gfcauchypshufb[1][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm10" : : "m" (raid_gfcauchypshufb[1][1][1]));
asm volatile ("vbroadcasti32x4 %0,%%zmm11" : : "m" (raid_gfcauchypshufb[2][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (raid_gfcauchypshufb[2][0][1]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (raid_gfcauchypshufb[2][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (raid_gfcauchypshufb[2][1][1]));
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (raid_gfcauchypshufb[3][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm16" : : "m" (raid_gfcauchypshufb[3][0][1]));
asm volatile ("vbroadcasti32x4 %0,%%zmm17" : : "m" (raid_gfcauchypshufb[3][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm18" : : "m" (raid_gfcauchypshufb[3][1][1]));
asm volatile ("vbroadcasti32x4 %0,%%zmm19" : : "m" (raid_gfcauchypshufb[4][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm20" : : "m" (raid_gfcauchypshufb[4][0][1]));
asm volatile ("vbroadcasti32x4 %0,%%zmm21" : : "m" (raid_gfcauchypshufb[4][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm22" : : "m" (raid_gfcauchypshufb[4][1][1]));
asm volatile ("vbroadcasti32x4 %0,%%zmm23" : : "m" (raid_gfcauchypshufb[5][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm24" : : "m" (raid_gfcauchypshufb[5][0][1]));
asm volatile ("vbroadcasti32x4 %0,%%zmm25" : : "m" (raid_gfcauchypshufb[5][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm26" : : "m" (raid_gfcauchypshufb[5][1][1]));
asm volatile ("vbroadcasti32x4 %0,%%zmm27" : : "m" (raid_gfcauchypshufb[6][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm28" : : "m" (raid_gfcauchypshufb[6][0][1]));
asm volatile ("vbroadcasti32x4 %0,%%zmm29" : : "m" (raid_gfcauchypshufb[6][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm30" : : "m" (raid_gfcauchypshufb[6][1][1]));
for (i = 0; i < size; i += 64) {
asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i]));
@ -438,15 +438,15 @@ static __always_inline void raid_gen3_avx512bw_gen(int nd, size_t size, void **v
asm volatile ("vpandq %zmm31,%zmm3,%zmm4");
asm volatile ("vpandq %zmm31,%zmm5,%zmm5");
asm volatile ("vbroadcasti32x4 %0,%%zmm6" : : "m" (raid_gfcauchypshufb[d][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm6" : : "m" (raid_gfcauchypshufb[d][0][0]));
asm volatile ("vpshufb %zmm4,%zmm6,%zmm6");
asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (raid_gfcauchypshufb[d][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (raid_gfcauchypshufb[d][0][1]));
asm volatile ("vpshufb %zmm5,%zmm3,%zmm3");
asm volatile ("vpternlogq $0x96,%zmm3,%zmm6,%zmm1");
asm volatile ("vbroadcasti32x4 %0,%%zmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm6" : : "m" (raid_gfcauchypshufb[d][1][0]));
asm volatile ("vpshufb %zmm4,%zmm6,%zmm6");
asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (raid_gfcauchypshufb[d][1][1]));
asm volatile ("vpshufb %zmm5,%zmm3,%zmm3");
asm volatile ("vpternlogq $0x96,%zmm3,%zmm6,%zmm2");
}
@ -507,23 +507,23 @@ static __always_inline void raid_genX_avx512bw(int nd, size_t size, void **vv, i
asm volatile ("vpandq %zmm31,%zmm10,%zmm10");
asm volatile ("vpandq %zmm31,%zmm11,%zmm11");
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (raid_gfcauchypshufb[d][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (raid_gfcauchypshufb[d][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (raid_gfcauchypshufb[d][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (raid_gfcauchypshufb[d][0][1]));
if (np >= 3) {
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (raid_gfcauchypshufb[d][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (raid_gfcauchypshufb[d][1][1]));
}
if (np >= 4) {
asm volatile ("vbroadcasti32x4 %0,%%zmm16" : : "m" (raid_gfcauchypshufb[d][2][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm17" : : "m" (raid_gfcauchypshufb[d][2][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm16" : : "m" (raid_gfcauchypshufb[d][2][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm17" : : "m" (raid_gfcauchypshufb[d][2][1]));
}
if (np >= 5) {
asm volatile ("vbroadcasti32x4 %0,%%zmm18" : : "m" (raid_gfcauchypshufb[d][3][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm19" : : "m" (raid_gfcauchypshufb[d][3][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm18" : : "m" (raid_gfcauchypshufb[d][3][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm19" : : "m" (raid_gfcauchypshufb[d][3][1]));
}
if (np >= 6) {
asm volatile ("vbroadcasti32x4 %0,%%zmm20" : : "m" (raid_gfcauchypshufb[d][4][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm21" : : "m" (raid_gfcauchypshufb[d][4][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm20" : : "m" (raid_gfcauchypshufb[d][4][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm21" : : "m" (raid_gfcauchypshufb[d][4][1]));
}
asm volatile ("vpshufb %zmm10,%zmm12,%zmm12");

134
raid/gfni.c
View file

@ -56,35 +56,35 @@ static __always_inline void raid_genX_avx2gfni_raid(int nd, size_t size, void **
asm volatile ("vpxor %ymm12,%ymm0,%ymm0");
asm volatile ("vpxor %ymm13,%ymm1,%ymm1");
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfcauchyaffine_raid[d][0][0]));
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfcauchyaffine_raid[d][0]));
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15");
asm volatile ("vpxor %ymm15,%ymm2,%ymm2");
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15");
asm volatile ("vpxor %ymm15,%ymm3,%ymm3");
if (np >= 3) {
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfcauchyaffine_raid[d][1][0]));
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfcauchyaffine_raid[d][1]));
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15");
asm volatile ("vpxor %ymm15,%ymm4,%ymm4");
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15");
asm volatile ("vpxor %ymm15,%ymm5,%ymm5");
}
if (np >= 4) {
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfcauchyaffine_raid[d][2][0]));
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfcauchyaffine_raid[d][2]));
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15");
asm volatile ("vpxor %ymm15,%ymm6,%ymm6");
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15");
asm volatile ("vpxor %ymm15,%ymm7,%ymm7");
}
if (np >= 5) {
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfcauchyaffine_raid[d][3][0]));
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfcauchyaffine_raid[d][3]));
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15");
asm volatile ("vpxor %ymm15,%ymm8,%ymm8");
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15");
asm volatile ("vpxor %ymm15,%ymm9,%ymm9");
}
if (np >= 6) {
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfcauchyaffine_raid[d][4][0]));
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfcauchyaffine_raid[d][4]));
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15");
asm volatile ("vpxor %ymm15,%ymm10,%ymm10");
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15");
@ -163,34 +163,34 @@ static __always_inline void raid_gen2_avx512gfni_raid_gen(int nd, size_t size, v
raid_avx_begin();
/* preload as many Q coefficient matrices as possible */
asm volatile ("vpbroadcastq %0,%%zmm4" : : "m" (raid_gfcauchyaffine_raid[1][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm5" : : "m" (raid_gfcauchyaffine_raid[2][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm6" : : "m" (raid_gfcauchyaffine_raid[3][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm7" : : "m" (raid_gfcauchyaffine_raid[4][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm8" : : "m" (raid_gfcauchyaffine_raid[5][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm9" : : "m" (raid_gfcauchyaffine_raid[6][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm10" : : "m" (raid_gfcauchyaffine_raid[7][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm11" : : "m" (raid_gfcauchyaffine_raid[8][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm12" : : "m" (raid_gfcauchyaffine_raid[9][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm13" : : "m" (raid_gfcauchyaffine_raid[10][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm14" : : "m" (raid_gfcauchyaffine_raid[11][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm15" : : "m" (raid_gfcauchyaffine_raid[12][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm16" : : "m" (raid_gfcauchyaffine_raid[13][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm17" : : "m" (raid_gfcauchyaffine_raid[14][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm18" : : "m" (raid_gfcauchyaffine_raid[15][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm19" : : "m" (raid_gfcauchyaffine_raid[16][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm20" : : "m" (raid_gfcauchyaffine_raid[17][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm21" : : "m" (raid_gfcauchyaffine_raid[18][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm22" : : "m" (raid_gfcauchyaffine_raid[19][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm23" : : "m" (raid_gfcauchyaffine_raid[20][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm24" : : "m" (raid_gfcauchyaffine_raid[21][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm25" : : "m" (raid_gfcauchyaffine_raid[22][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm26" : : "m" (raid_gfcauchyaffine_raid[23][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm27" : : "m" (raid_gfcauchyaffine_raid[24][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm28" : : "m" (raid_gfcauchyaffine_raid[25][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm29" : : "m" (raid_gfcauchyaffine_raid[26][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm30" : : "m" (raid_gfcauchyaffine_raid[27][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm31" : : "m" (raid_gfcauchyaffine_raid[28][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm4" : : "m" (raid_gfcauchyaffine_raid[1][0]));
asm volatile ("vpbroadcastq %0,%%zmm5" : : "m" (raid_gfcauchyaffine_raid[2][0]));
asm volatile ("vpbroadcastq %0,%%zmm6" : : "m" (raid_gfcauchyaffine_raid[3][0]));
asm volatile ("vpbroadcastq %0,%%zmm7" : : "m" (raid_gfcauchyaffine_raid[4][0]));
asm volatile ("vpbroadcastq %0,%%zmm8" : : "m" (raid_gfcauchyaffine_raid[5][0]));
asm volatile ("vpbroadcastq %0,%%zmm9" : : "m" (raid_gfcauchyaffine_raid[6][0]));
asm volatile ("vpbroadcastq %0,%%zmm10" : : "m" (raid_gfcauchyaffine_raid[7][0]));
asm volatile ("vpbroadcastq %0,%%zmm11" : : "m" (raid_gfcauchyaffine_raid[8][0]));
asm volatile ("vpbroadcastq %0,%%zmm12" : : "m" (raid_gfcauchyaffine_raid[9][0]));
asm volatile ("vpbroadcastq %0,%%zmm13" : : "m" (raid_gfcauchyaffine_raid[10][0]));
asm volatile ("vpbroadcastq %0,%%zmm14" : : "m" (raid_gfcauchyaffine_raid[11][0]));
asm volatile ("vpbroadcastq %0,%%zmm15" : : "m" (raid_gfcauchyaffine_raid[12][0]));
asm volatile ("vpbroadcastq %0,%%zmm16" : : "m" (raid_gfcauchyaffine_raid[13][0]));
asm volatile ("vpbroadcastq %0,%%zmm17" : : "m" (raid_gfcauchyaffine_raid[14][0]));
asm volatile ("vpbroadcastq %0,%%zmm18" : : "m" (raid_gfcauchyaffine_raid[15][0]));
asm volatile ("vpbroadcastq %0,%%zmm19" : : "m" (raid_gfcauchyaffine_raid[16][0]));
asm volatile ("vpbroadcastq %0,%%zmm20" : : "m" (raid_gfcauchyaffine_raid[17][0]));
asm volatile ("vpbroadcastq %0,%%zmm21" : : "m" (raid_gfcauchyaffine_raid[18][0]));
asm volatile ("vpbroadcastq %0,%%zmm22" : : "m" (raid_gfcauchyaffine_raid[19][0]));
asm volatile ("vpbroadcastq %0,%%zmm23" : : "m" (raid_gfcauchyaffine_raid[20][0]));
asm volatile ("vpbroadcastq %0,%%zmm24" : : "m" (raid_gfcauchyaffine_raid[21][0]));
asm volatile ("vpbroadcastq %0,%%zmm25" : : "m" (raid_gfcauchyaffine_raid[22][0]));
asm volatile ("vpbroadcastq %0,%%zmm26" : : "m" (raid_gfcauchyaffine_raid[23][0]));
asm volatile ("vpbroadcastq %0,%%zmm27" : : "m" (raid_gfcauchyaffine_raid[24][0]));
asm volatile ("vpbroadcastq %0,%%zmm28" : : "m" (raid_gfcauchyaffine_raid[25][0]));
asm volatile ("vpbroadcastq %0,%%zmm29" : : "m" (raid_gfcauchyaffine_raid[26][0]));
asm volatile ("vpbroadcastq %0,%%zmm30" : : "m" (raid_gfcauchyaffine_raid[27][0]));
asm volatile ("vpbroadcastq %0,%%zmm31" : : "m" (raid_gfcauchyaffine_raid[28][0]));
for (i = 0; i < size; i += 64) {
asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i]));
@ -399,7 +399,7 @@ static __always_inline void raid_gen2_avx512gfni_raid_gen(int nd, size_t size, v
for (d = 29; d < nd; ++d) {
asm volatile ("vmovdqa64 %0,%%zmm2" : : "m" (v[d][i]));
asm volatile ("vpxorq %zmm2,%zmm0,%zmm0");
asm volatile ("vpbroadcastq %0,%%zmm3" : : "m" (raid_gfcauchyaffine_raid[d][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm3" : : "m" (raid_gfcauchyaffine_raid[d][0]));
asm volatile ("vgf2p8affineqb $0,%zmm3,%zmm2,%zmm3");
asm volatile ("vpxorq %zmm3,%zmm1,%zmm1");
}
@ -442,32 +442,32 @@ static __always_inline void raid_gen3_avx512gfni_raid_gen(int nd, size_t size, v
raid_avx_begin();
/* preload as many Q and R coefficient matrices as possible */
asm volatile ("vpbroadcastq %0,%%zmm6" : : "m" (raid_gfcauchyaffine_raid[1][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm7" : : "m" (raid_gfcauchyaffine_raid[1][1][0]));
asm volatile ("vpbroadcastq %0,%%zmm8" : : "m" (raid_gfcauchyaffine_raid[2][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm9" : : "m" (raid_gfcauchyaffine_raid[2][1][0]));
asm volatile ("vpbroadcastq %0,%%zmm10" : : "m" (raid_gfcauchyaffine_raid[3][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm11" : : "m" (raid_gfcauchyaffine_raid[3][1][0]));
asm volatile ("vpbroadcastq %0,%%zmm12" : : "m" (raid_gfcauchyaffine_raid[4][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm13" : : "m" (raid_gfcauchyaffine_raid[4][1][0]));
asm volatile ("vpbroadcastq %0,%%zmm14" : : "m" (raid_gfcauchyaffine_raid[5][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm15" : : "m" (raid_gfcauchyaffine_raid[5][1][0]));
asm volatile ("vpbroadcastq %0,%%zmm16" : : "m" (raid_gfcauchyaffine_raid[6][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm17" : : "m" (raid_gfcauchyaffine_raid[6][1][0]));
asm volatile ("vpbroadcastq %0,%%zmm18" : : "m" (raid_gfcauchyaffine_raid[7][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm19" : : "m" (raid_gfcauchyaffine_raid[7][1][0]));
asm volatile ("vpbroadcastq %0,%%zmm20" : : "m" (raid_gfcauchyaffine_raid[8][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm21" : : "m" (raid_gfcauchyaffine_raid[8][1][0]));
asm volatile ("vpbroadcastq %0,%%zmm22" : : "m" (raid_gfcauchyaffine_raid[9][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm23" : : "m" (raid_gfcauchyaffine_raid[9][1][0]));
asm volatile ("vpbroadcastq %0,%%zmm24" : : "m" (raid_gfcauchyaffine_raid[10][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm25" : : "m" (raid_gfcauchyaffine_raid[10][1][0]));
asm volatile ("vpbroadcastq %0,%%zmm26" : : "m" (raid_gfcauchyaffine_raid[11][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm27" : : "m" (raid_gfcauchyaffine_raid[11][1][0]));
asm volatile ("vpbroadcastq %0,%%zmm28" : : "m" (raid_gfcauchyaffine_raid[12][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm29" : : "m" (raid_gfcauchyaffine_raid[12][1][0]));
asm volatile ("vpbroadcastq %0,%%zmm30" : : "m" (raid_gfcauchyaffine_raid[13][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm31" : : "m" (raid_gfcauchyaffine_raid[13][1][0]));
asm volatile ("vpbroadcastq %0,%%zmm6" : : "m" (raid_gfcauchyaffine_raid[1][0]));
asm volatile ("vpbroadcastq %0,%%zmm7" : : "m" (raid_gfcauchyaffine_raid[1][1]));
asm volatile ("vpbroadcastq %0,%%zmm8" : : "m" (raid_gfcauchyaffine_raid[2][0]));
asm volatile ("vpbroadcastq %0,%%zmm9" : : "m" (raid_gfcauchyaffine_raid[2][1]));
asm volatile ("vpbroadcastq %0,%%zmm10" : : "m" (raid_gfcauchyaffine_raid[3][0]));
asm volatile ("vpbroadcastq %0,%%zmm11" : : "m" (raid_gfcauchyaffine_raid[3][1]));
asm volatile ("vpbroadcastq %0,%%zmm12" : : "m" (raid_gfcauchyaffine_raid[4][0]));
asm volatile ("vpbroadcastq %0,%%zmm13" : : "m" (raid_gfcauchyaffine_raid[4][1]));
asm volatile ("vpbroadcastq %0,%%zmm14" : : "m" (raid_gfcauchyaffine_raid[5][0]));
asm volatile ("vpbroadcastq %0,%%zmm15" : : "m" (raid_gfcauchyaffine_raid[5][1]));
asm volatile ("vpbroadcastq %0,%%zmm16" : : "m" (raid_gfcauchyaffine_raid[6][0]));
asm volatile ("vpbroadcastq %0,%%zmm17" : : "m" (raid_gfcauchyaffine_raid[6][1]));
asm volatile ("vpbroadcastq %0,%%zmm18" : : "m" (raid_gfcauchyaffine_raid[7][0]));
asm volatile ("vpbroadcastq %0,%%zmm19" : : "m" (raid_gfcauchyaffine_raid[7][1]));
asm volatile ("vpbroadcastq %0,%%zmm20" : : "m" (raid_gfcauchyaffine_raid[8][0]));
asm volatile ("vpbroadcastq %0,%%zmm21" : : "m" (raid_gfcauchyaffine_raid[8][1]));
asm volatile ("vpbroadcastq %0,%%zmm22" : : "m" (raid_gfcauchyaffine_raid[9][0]));
asm volatile ("vpbroadcastq %0,%%zmm23" : : "m" (raid_gfcauchyaffine_raid[9][1]));
asm volatile ("vpbroadcastq %0,%%zmm24" : : "m" (raid_gfcauchyaffine_raid[10][0]));
asm volatile ("vpbroadcastq %0,%%zmm25" : : "m" (raid_gfcauchyaffine_raid[10][1]));
asm volatile ("vpbroadcastq %0,%%zmm26" : : "m" (raid_gfcauchyaffine_raid[11][0]));
asm volatile ("vpbroadcastq %0,%%zmm27" : : "m" (raid_gfcauchyaffine_raid[11][1]));
asm volatile ("vpbroadcastq %0,%%zmm28" : : "m" (raid_gfcauchyaffine_raid[12][0]));
asm volatile ("vpbroadcastq %0,%%zmm29" : : "m" (raid_gfcauchyaffine_raid[12][1]));
asm volatile ("vpbroadcastq %0,%%zmm30" : : "m" (raid_gfcauchyaffine_raid[13][0]));
asm volatile ("vpbroadcastq %0,%%zmm31" : : "m" (raid_gfcauchyaffine_raid[13][1]));
for (i = 0; i < size; i += 64) {
asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i]));
@ -599,11 +599,11 @@ static __always_inline void raid_gen3_avx512gfni_raid_gen(int nd, size_t size, v
asm volatile ("vmovdqa64 %0,%%zmm3" : : "m" (v[d][i]));
asm volatile ("vpxorq %zmm3,%zmm0,%zmm0");
asm volatile ("vpbroadcastq %0,%%zmm4" : : "m" (raid_gfcauchyaffine_raid[d][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm4" : : "m" (raid_gfcauchyaffine_raid[d][0]));
asm volatile ("vgf2p8affineqb $0,%zmm4,%zmm3,%zmm4");
asm volatile ("vpxorq %zmm4,%zmm1,%zmm1");
asm volatile ("vpbroadcastq %0,%%zmm5" : : "m" (raid_gfcauchyaffine_raid[d][1][0]));
asm volatile ("vpbroadcastq %0,%%zmm5" : : "m" (raid_gfcauchyaffine_raid[d][1]));
asm volatile ("vgf2p8affineqb $0,%zmm5,%zmm3,%zmm5");
asm volatile ("vpxorq %zmm5,%zmm2,%zmm2");
}
@ -659,15 +659,15 @@ static __always_inline void raid_genX_avx512gfni_raid(int nd, size_t size, void
for (d = 1; d < nd; ++d) {
asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[d][i]));
asm volatile ("vpbroadcastq %0,%%zmm7" : : "m" (raid_gfcauchyaffine_raid[d][0][0]));
asm volatile ("vpbroadcastq %0,%%zmm7" : : "m" (raid_gfcauchyaffine_raid[d][0]));
if (np >= 3)
asm volatile ("vpbroadcastq %0,%%zmm8" : : "m" (raid_gfcauchyaffine_raid[d][1][0]));
asm volatile ("vpbroadcastq %0,%%zmm8" : : "m" (raid_gfcauchyaffine_raid[d][1]));
if (np >= 4)
asm volatile ("vpbroadcastq %0,%%zmm9" : : "m" (raid_gfcauchyaffine_raid[d][2][0]));
asm volatile ("vpbroadcastq %0,%%zmm9" : : "m" (raid_gfcauchyaffine_raid[d][2]));
if (np >= 5)
asm volatile ("vpbroadcastq %0,%%zmm10" : : "m" (raid_gfcauchyaffine_raid[d][3][0]));
asm volatile ("vpbroadcastq %0,%%zmm10" : : "m" (raid_gfcauchyaffine_raid[d][3]));
if (np >= 6)
asm volatile ("vpbroadcastq %0,%%zmm11" : : "m" (raid_gfcauchyaffine_raid[d][4][0]));
asm volatile ("vpbroadcastq %0,%%zmm11" : : "m" (raid_gfcauchyaffine_raid[d][4]));
asm volatile ("vgf2p8affineqb $0,%zmm7,%zmm6,%zmm12");
if (np >= 3)

6
raid/neon.c
View file

@ -80,7 +80,7 @@ static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, i
asm volatile (
"ldr q29, %0\n"
:
: "m" (gfconst16.poly[0])
: "m" (gfconst16.poly)
);
for (i = 0; i < size; i += 32) {
@ -167,7 +167,7 @@ static __always_inline void raid_genz_neon_gen(int nd, size_t size, void **vv)
"ldr q29, %0\n"
"ldr q30, %1\n"
:
: "m" (gfconst16.poly[0]), "m" (gfconst16.half[0])
: "m" (gfconst16.poly), "m" (gfconst16.half)
);
for (i = 0; i < size; i += 32) {
@ -264,7 +264,7 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
"ldr q29, %0\n"
"movi v28.16b, #0x0f\n"
:
: "m" (gfconst16.poly[0])
: "m" (gfconst16.poly)
);
for (i = 0; i < size; i += 32) {

38
raid/neon32.c
View file

@ -85,7 +85,7 @@ static __always_inline void raid_gen2_neon32_gen(int nd, size_t size,
asm volatile (
"vld1.8 {q14}, %0\n"
:
: "Q" (gfconst16.poly[0])
: "Q" (gfconst16.poly)
);
for (i = 0; i < size; i += 32) {
@ -167,7 +167,7 @@ static __always_inline void raid_genz_neon32_gen(int nd, size_t size, void **vv)
"vld1.8 {q14}, %0\n"
"vld1.8 {q15}, %1\n"
:
: "Q" (gfconst16.poly[0]), "Q" (gfconst16.half[0])
: "Q" (gfconst16.poly), "Q" (gfconst16.half)
);
for (i = 0; i < size; i += 32) {
@ -258,7 +258,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
"vld1.8 {q14}, %0\n"
"vmov.i8 q15, #0x0f\n"
:
: "Q" (gfconst16.poly[0])
: "Q" (gfconst16.poly)
);
for (i = 0; i < size; i += 16) {
@ -288,8 +288,8 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
"veor q2, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][1][0][0]),
"Q" (raid_gfcauchypshufb[l][1][1][0])
: "Q" (raid_gfcauchypshufb[l][1][0]),
"Q" (raid_gfcauchypshufb[l][1][1])
);
}
@ -305,8 +305,8 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
"veor q3, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][2][0][0]),
"Q" (raid_gfcauchypshufb[l][2][1][0])
: "Q" (raid_gfcauchypshufb[l][2][0]),
"Q" (raid_gfcauchypshufb[l][2][1])
);
}
@ -322,8 +322,8 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
"veor q4, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][3][0][0]),
"Q" (raid_gfcauchypshufb[l][3][1][0])
: "Q" (raid_gfcauchypshufb[l][3][0]),
"Q" (raid_gfcauchypshufb[l][3][1])
);
}
@ -339,8 +339,8 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
"veor q5, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][4][0][0]),
"Q" (raid_gfcauchypshufb[l][4][1][0])
: "Q" (raid_gfcauchypshufb[l][4][0]),
"Q" (raid_gfcauchypshufb[l][4][1])
);
}
@ -386,8 +386,8 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
"veor q11, q11, q12\n"
"veor q2, q2, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][1][0][0]),
"Q" (raid_gfcauchypshufb[d][1][1][0])
: "Q" (raid_gfcauchypshufb[d][1][0]),
"Q" (raid_gfcauchypshufb[d][1][1])
);
}
@ -402,8 +402,8 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
"veor q11, q11, q12\n"
"veor q3, q3, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][2][0][0]),
"Q" (raid_gfcauchypshufb[d][2][1][0])
: "Q" (raid_gfcauchypshufb[d][2][0]),
"Q" (raid_gfcauchypshufb[d][2][1])
);
}
@ -418,8 +418,8 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
"veor q11, q11, q12\n"
"veor q4, q4, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][3][0][0]),
"Q" (raid_gfcauchypshufb[d][3][1][0])
: "Q" (raid_gfcauchypshufb[d][3][0]),
"Q" (raid_gfcauchypshufb[d][3][1])
);
}
@ -434,8 +434,8 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
"veor q11, q11, q12\n"
"veor q5, q5, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][4][0][0]),
"Q" (raid_gfcauchypshufb[d][4][1][0])
: "Q" (raid_gfcauchypshufb[d][4][0]),
"Q" (raid_gfcauchypshufb[d][4][1])
);
}
}

16
raid/sse2.c
View file

@ -162,7 +162,7 @@ static __always_inline void raid_gen2_sse2_gen(int nd, size_t size, void **vv, i
raid_sse_begin();
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly));
for (i = 0; i < size; i += 32) {
asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i]));
@ -247,7 +247,7 @@ static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv
raid_sse_begin();
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.poly[0]));
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.poly));
for (i = 0; i < size; i += 64) {
asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i]));
@ -486,9 +486,9 @@ static __always_inline void raid_genz_sse2_gen(int nd, size_t size, void **vv, i
raid_sse_begin();
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.half[0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfconst16.low7[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly));
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.half));
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfconst16.low7));
for (i = 0; i < size; i += 16) {
asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i]));
@ -551,9 +551,9 @@ static __always_inline void raid_genz_sse2ext_gen(int nd, size_t size, void **vv
raid_sse_begin();
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.half[0]));
asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.low7[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly));
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.half));
asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.low7));
for (i = 0; i < size; i += 32) {
asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i]));

262
raid/ssse3.c
View file

@ -34,8 +34,8 @@ static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv,
raid_sse_begin();
/* generic case with at least two data disks */
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.poly[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.poly));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4));
for (i = 0; i < size; i += 16) {
/* last disk without the generator multiplication */
@ -49,8 +49,8 @@ static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv,
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[l][1][1][0]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[l][1][1]));
asm volatile ("pshufb %xmm4,%xmm2");
asm volatile ("pshufb %xmm5,%xmm6");
asm volatile ("pxor %xmm6,%xmm2");
@ -77,10 +77,10 @@ static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv,
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pxor %xmm6,%xmm2");
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][1]));
asm volatile ("pshufb %xmm5,%xmm6");
asm volatile ("pxor %xmm6,%xmm2");
}
@ -148,7 +148,7 @@ static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv,
/* generic case with at least two data disks */
for (i = 0; i < size; i += 16) {
/* last disk without the generator multiplication */
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4));
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));
asm volatile ("movdqa %xmm4,%xmm0");
@ -159,21 +159,21 @@ static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv,
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1][0]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1]));
asm volatile ("pshufb %xmm4,%xmm2");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm7,%xmm2");
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][2][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1]));
asm volatile ("pshufb %xmm4,%xmm3");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm7,%xmm3");
/* intermediate disks */
for (d = l - 1; d > 0; --d) {
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly));
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
if (generator == 3)
@ -186,7 +186,7 @@ static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv,
if (generator == 3)
asm volatile ("pxor %xmm6,%xmm1");
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4));
asm volatile ("pxor %xmm4,%xmm0");
asm volatile ("pxor %xmm4,%xmm1");
@ -196,15 +196,15 @@ static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv,
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm6,%xmm2");
asm volatile ("pxor %xmm7,%xmm2");
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm6,%xmm3");
@ -212,7 +212,7 @@ static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv,
}
/* first disk with all coefficients at 1 */
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly));
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));
if (generator == 3)
@ -289,26 +289,26 @@ static __always_inline void raid_gen5_ssse3_gen(int nd, size_t size, void **vv,
asm volatile ("movdqa %xmm4,%xmm0");
asm volatile ("movdqa %%xmm4,%0" : "=m" (pd[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4));
asm volatile ("movdqa %xmm4,%xmm5");
asm volatile ("psrlw $4,%xmm5");
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("movdqa %0,%%xmm1" : : "m" (raid_gfcauchypshufb[l][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1][0]));
asm volatile ("movdqa %0,%%xmm1" : : "m" (raid_gfcauchypshufb[l][1][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1]));
asm volatile ("pshufb %xmm4,%xmm1");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm7,%xmm1");
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1][0]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][2][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1]));
asm volatile ("pshufb %xmm4,%xmm2");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm7,%xmm2");
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][3][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][3][1][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][3][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][3][1]));
asm volatile ("pshufb %xmm4,%xmm3");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm7,%xmm3");
@ -317,7 +317,7 @@ static __always_inline void raid_gen5_ssse3_gen(int nd, size_t size, void **vv,
for (d = l - 1; d > 0; --d) {
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly));
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pcmpgtb %xmm0,%xmm5");
@ -329,28 +329,28 @@ static __always_inline void raid_gen5_ssse3_gen(int nd, size_t size, void **vv,
asm volatile ("pxor %xmm4,%xmm6");
asm volatile ("movdqa %%xmm6,%0" : "=m" (pd[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4));
asm volatile ("movdqa %xmm4,%xmm5");
asm volatile ("psrlw $4,%xmm5");
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm6,%xmm1");
asm volatile ("pxor %xmm7,%xmm1");
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm6,%xmm2");
asm volatile ("pxor %xmm7,%xmm2");
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][3][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][3][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][3][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][3][1]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm6,%xmm3");
@ -360,7 +360,7 @@ static __always_inline void raid_gen5_ssse3_gen(int nd, size_t size, void **vv,
/* first disk with all coefficients at 1 */
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly));
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pcmpgtb %xmm0,%xmm5");
@ -437,32 +437,32 @@ static __always_inline void raid_gen6_ssse3_gen(int nd, size_t size, void **vv,
asm volatile ("movdqa %%xmm4,%0" : "=m" (pd[0]));
asm volatile ("movdqa %%xmm4,%0" : "=m" (pd[16]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4));
asm volatile ("movdqa %xmm4,%xmm5");
asm volatile ("psrlw $4,%xmm5");
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("movdqa %0,%%xmm0" : : "m" (raid_gfcauchypshufb[l][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1][0]));
asm volatile ("movdqa %0,%%xmm0" : : "m" (raid_gfcauchypshufb[l][1][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1]));
asm volatile ("pshufb %xmm4,%xmm0");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm7,%xmm0");
asm volatile ("movdqa %0,%%xmm1" : : "m" (raid_gfcauchypshufb[l][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1][0]));
asm volatile ("movdqa %0,%%xmm1" : : "m" (raid_gfcauchypshufb[l][2][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1]));
asm volatile ("pshufb %xmm4,%xmm1");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm7,%xmm1");
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][3][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][3][1][0]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][3][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][3][1]));
asm volatile ("pshufb %xmm4,%xmm2");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm7,%xmm2");
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][4][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][4][1][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][4][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][4][1]));
asm volatile ("pshufb %xmm4,%xmm3");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm7,%xmm3");
@ -471,7 +471,7 @@ static __always_inline void raid_gen6_ssse3_gen(int nd, size_t size, void **vv,
for (d = l - 1; d > 0; --d) {
asm volatile ("movdqa %0,%%xmm5" : : "m" (pd[0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[16]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly));
asm volatile ("pxor %xmm4,%xmm4");
asm volatile ("pcmpgtb %xmm6,%xmm4");
@ -486,35 +486,35 @@ static __always_inline void raid_gen6_ssse3_gen(int nd, size_t size, void **vv,
asm volatile ("movdqa %%xmm5,%0" : "=m" (pd[0]));
asm volatile ("movdqa %%xmm6,%0" : "=m" (pd[16]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4));
asm volatile ("movdqa %xmm4,%xmm5");
asm volatile ("psrlw $4,%xmm5");
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm6,%xmm0");
asm volatile ("pxor %xmm7,%xmm0");
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm6,%xmm1");
asm volatile ("pxor %xmm7,%xmm1");
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][3][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][3][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][3][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][3][1]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm6,%xmm2");
asm volatile ("pxor %xmm7,%xmm2");
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][4][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][4][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][4][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][4][1]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm6,%xmm3");
@ -524,7 +524,7 @@ static __always_inline void raid_gen6_ssse3_gen(int nd, size_t size, void **vv,
/* first disk with all coefficients at 1 */
asm volatile ("movdqa %0,%%xmm5" : : "m" (pd[0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[16]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly));
asm volatile ("pxor %xmm4,%xmm4");
asm volatile ("pcmpgtb %xmm6,%xmm4");
@ -594,8 +594,8 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v
raid_sse_begin();
/* generic case with at least two data disks */
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.poly[0]));
asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.low4[0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.poly));
asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.low4));
for (i = 0; i < size; i += 32) {
/* last disk without the generator multiplication */
@ -616,8 +616,8 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v
asm volatile ("pand %xmm11,%xmm5");
asm volatile ("pand %xmm11,%xmm13");
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1][0]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1]));
asm volatile ("movdqa %xmm2,%xmm10");
asm volatile ("movdqa %xmm7,%xmm15");
asm volatile ("pshufb %xmm4,%xmm2");
@ -632,8 +632,8 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
asm volatile ("movdqa %0,%%xmm15" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][0]));
asm volatile ("movdqa %0,%%xmm15" : : "m" (raid_gfcauchypshufb[d][1][1]));
if (generator == 3) {
asm volatile ("movdqa %xmm1,%xmm6");
@ -682,8 +682,8 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d - 1][i]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d - 1][i + 16]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d - 1][1][0][0]));
asm volatile ("movdqa %0,%%xmm15" : : "m" (raid_gfcauchypshufb[d - 1][1][1][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d - 1][1][0]));
asm volatile ("movdqa %0,%%xmm15" : : "m" (raid_gfcauchypshufb[d - 1][1][1]));
if (generator == 3) {
asm volatile ("movdqa %xmm1,%xmm6");
@ -768,8 +768,8 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v
asm volatile ("pand %xmm11,%xmm5");
asm volatile ("pand %xmm11,%xmm13");
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[1][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[1][1][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[1][1][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[1][1][1]));
asm volatile ("movdqa %xmm6,%xmm14");
asm volatile ("movdqa %xmm7,%xmm15");
asm volatile ("pshufb %xmm4,%xmm6");
@ -869,7 +869,7 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v
/* generic case with at least two data disks */
for (i = 0; i < size; i += 32) {
/* last disk without the generator multiplication */
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0]));
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4));
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[l][i + 16]));
@ -887,8 +887,8 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v
asm volatile ("pand %xmm15,%xmm5");
asm volatile ("pand %xmm15,%xmm13");
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1][0]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1]));
asm volatile ("movdqa %xmm2,%xmm10");
asm volatile ("movdqa %xmm7,%xmm15");
asm volatile ("pshufb %xmm4,%xmm2");
@ -898,8 +898,8 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v
asm volatile ("pxor %xmm7,%xmm2");
asm volatile ("pxor %xmm15,%xmm10");
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][2][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1]));
asm volatile ("movdqa %xmm3,%xmm11");
asm volatile ("movdqa %xmm7,%xmm15");
asm volatile ("pshufb %xmm4,%xmm3");
@ -911,8 +911,8 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v
/* intermediate disks */
for (d = l - 1; d > 0; --d) {
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly));
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4));
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16]));
@ -949,8 +949,8 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v
asm volatile ("pand %xmm15,%xmm5");
asm volatile ("pand %xmm15,%xmm13");
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1]));
asm volatile ("movdqa %xmm6,%xmm14");
asm volatile ("movdqa %xmm7,%xmm15");
asm volatile ("pshufb %xmm4,%xmm6");
@ -962,8 +962,8 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v
asm volatile ("pxor %xmm7,%xmm2");
asm volatile ("pxor %xmm15,%xmm10");
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1]));
asm volatile ("movdqa %xmm6,%xmm14");
asm volatile ("movdqa %xmm7,%xmm15");
asm volatile ("pshufb %xmm4,%xmm6");
@ -977,8 +977,8 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v
}
/* first disk with all coefficients at 1 */
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly));
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4));
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[0][i + 16]));
@ -1069,8 +1069,8 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i
raid_sse_begin();
/* generic case with at least two data disks */
asm volatile ("movdqa %0,%%xmm14" : : "m" (gfconst16.poly[0]));
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0]));
asm volatile ("movdqa %0,%%xmm14" : : "m" (gfconst16.poly));
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4));
for (i = 0; i < size; i += 16) {
/* last disk without the generator multiplication */
@ -1085,32 +1085,32 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i
asm volatile ("pand %xmm15,%xmm11");
if (np >= 3) {
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[l][1][1][0]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[l][1][1]));
asm volatile ("pshufb %xmm10,%xmm2");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm13,%xmm2");
}
if (np >= 4) {
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][2][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[l][2][1][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][2][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[l][2][1]));
asm volatile ("pshufb %xmm10,%xmm3");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm13,%xmm3");
}
if (np >= 5) {
asm volatile ("movdqa %0,%%xmm4" : : "m" (raid_gfcauchypshufb[l][3][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[l][3][1][0]));
asm volatile ("movdqa %0,%%xmm4" : : "m" (raid_gfcauchypshufb[l][3][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[l][3][1]));
asm volatile ("pshufb %xmm10,%xmm4");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm13,%xmm4");
}
if (np >= 6) {
asm volatile ("movdqa %0,%%xmm5" : : "m" (raid_gfcauchypshufb[l][4][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[l][4][1][0]));
asm volatile ("movdqa %0,%%xmm5" : : "m" (raid_gfcauchypshufb[l][4][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[l][4][1]));
asm volatile ("pshufb %xmm10,%xmm5");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm13,%xmm5");
@ -1139,8 +1139,8 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i
asm volatile ("pand %xmm15,%xmm11");
if (np >= 3) {
asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][1][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][1][1]));
asm volatile ("pshufb %xmm10,%xmm12");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm12,%xmm2");
@ -1148,8 +1148,8 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i
}
if (np >= 4) {
asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][2][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][2][1][0]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][2][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][2][1]));
asm volatile ("pshufb %xmm10,%xmm12");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm12,%xmm3");
@ -1157,8 +1157,8 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i
}
if (np >= 5) {
asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][3][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][3][1][0]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][3][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][3][1]));
asm volatile ("pshufb %xmm10,%xmm12");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm12,%xmm4");
@ -1166,8 +1166,8 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i
}
if (np >= 6) {
asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][4][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][4][1][0]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][4][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][4][1]));
asm volatile ("pshufb %xmm10,%xmm12");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm12,%xmm5");
@ -1268,9 +1268,9 @@ static __always_inline void raid_rec1_ssse3_1(int *id, int *ip, int nd, size_t s
raid_sse_begin();
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfmulpshufb[V][0][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfmulpshufb[V][1][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4));
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfmulpshufb[V][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfmulpshufb[V][1]));
for (i = 0; i < size; i += 16) {
/* start from the selected parity block */
@ -1350,11 +1350,11 @@ static __always_inline void raid_rec1_ssse3_q(int *id, int *ip, int nd, size_t s
raid_sse_begin();
/* keep the inverse coefficient resident */
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfmulpshufb[V][0][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfmulpshufb[V][1][0]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfmulpshufb[V][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfmulpshufb[V][1]));
/* keep the active reduction polynomial resident */
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly));
for (i = 0; i < size; i += 32) {
/* last disk starts Horner without generator multiplication */
@ -1413,7 +1413,7 @@ static __always_inline void raid_rec1_ssse3_q(int *id, int *ip, int nd, size_t s
asm volatile ("pxor %0,%%xmm1" : : "m" (q[i + 16]));
/* low-nibble mask */
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfconst16.low4[0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfconst16.low4));
/* split both Qdelta lanes into low/high nibbles */
asm volatile ("movdqa %xmm0,%xmm4");
@ -1444,7 +1444,7 @@ static __always_inline void raid_rec1_ssse3_q(int *id, int *ip, int nd, size_t s
asm volatile ("movdqa %%xmm7,%0" : "=m" (pa[i + 16]));
/* restore the active reduction polynomial */
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly));
}
raid_sse_end(0);
@ -1518,7 +1518,7 @@ static __always_inline void raid_rec2_ssse3_2(int has_p, int *id, int *ip, int n
raid_sse_begin();
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4));
for (i = 0; i < size; i += 16) {
/* start from the two selected parity blocks */
@ -1692,7 +1692,7 @@ static __always_inline void raid_rec2of2_ssse3(int *id, int *ip, int nd, size_t
* xmm4-xmm6 temporaries
* xmm7 active reduction polynomial
*/
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly));
for (i = 0; i < size; i += 32) {
/* Pdelta starts directly from stored P */
@ -1771,7 +1771,7 @@ static __always_inline void raid_rec2of2_ssse3(int *id, int *ip, int nd, size_t
* The survivor scan is complete, so xmm7 can now hold
* the low-nibble mask instead of the polynomial.
*/
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4));
/*
* Lane 0.
@ -1862,7 +1862,7 @@ static __always_inline void raid_rec2of2_ssse3(int *id, int *ip, int nd, size_t
/*
* Restore the polynomial for the next 32-byte iteration.
*/
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly));
}
raid_sse_end(0);
@ -1935,7 +1935,7 @@ static __always_inline void raid_recX_ssse3_1234(int nr, int has_p, int *id, int
raid_sse_begin();
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4));
for (i = 0; i < size; i += 16) {
asm volatile ("movdqa %0,%%xmm0" : : "m" (p[0][i]));
@ -2157,7 +2157,7 @@ static __always_inline void raid_recX_ssse3(int nr, int has_p, int *id, int *ip,
raid_sse_begin();
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4));
for (i = 0; i < size; i += 16) {
/* pd_low[] temporarily contains the complete raw syndrome */
@ -2294,11 +2294,11 @@ static __always_inline void raid_rec1_ssse3ext_1(int *id, int *ip, int nd, size_
raid_sse_begin();
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0]));
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4));
/* keep the inverse coefficient resident */
asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfmulpshufb[V][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfmulpshufb[V][1][0]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfmulpshufb[V][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfmulpshufb[V][1]));
for (i = 0; i < size; i += 32) {
/* start both lanes from the selected parity block */
@ -2414,10 +2414,10 @@ static __always_inline void raid_rec1_ssse3ext_q(int *id, int *ip, int nd, size_
* xmm14 active reduction polynomial
* xmm15 low-nibble mask
*/
asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfmulpshufb[V][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfmulpshufb[V][1][0]));
asm volatile ("movdqa %0,%%xmm14" : : "m" (gfconst16.poly[0]));
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfmulpshufb[V][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfmulpshufb[V][1]));
asm volatile ("movdqa %0,%%xmm14" : : "m" (gfconst16.poly));
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4));
for (i = 0; i < size; i += 64) {
/* last disk starts Horner without generator multiplication */
@ -2648,14 +2648,14 @@ static __always_inline void raid_rec2_ssse3ext_2(int has_p, int *id, int *ip, in
*/
/* keep the first inverse-matrix row resident */
asm volatile ("movdqa %0,%%xmm8" : : "m" (raid_gfmulpshufb[V[0]][0][0]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (raid_gfmulpshufb[V[0]][1][0]));
asm volatile ("movdqa %0,%%xmm10" : : "m" (raid_gfmulpshufb[V[1]][0][0]));
asm volatile ("movdqa %0,%%xmm11" : : "m" (raid_gfmulpshufb[V[1]][1][0]));
asm volatile ("movdqa %0,%%xmm8" : : "m" (raid_gfmulpshufb[V[0]][0]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (raid_gfmulpshufb[V[0]][1]));
asm volatile ("movdqa %0,%%xmm10" : : "m" (raid_gfmulpshufb[V[1]][0]));
asm volatile ("movdqa %0,%%xmm11" : : "m" (raid_gfmulpshufb[V[1]][1]));
for (i = 0; i < size; i += 32) {
/* xmm14 is reused during reconstruction, therefore restore the mask for each iteration */
asm volatile ("movdqa %0,%%xmm14" : : "m" (gfconst16.low4[0]));
asm volatile ("movdqa %0,%%xmm14" : : "m" (gfconst16.low4));
/* start both syndrome lanes from the selected stored parity blocks */
asm volatile ("movdqa %0,%%xmm0" : : "m" (p[0][i]));
@ -2848,7 +2848,7 @@ static __always_inline void raid_rec2_ssse3ext_2(int has_p, int *id, int *ip, in
*/
/* V[2] * syndrome0 low */
asm volatile ("movdqa %0,%%xmm14" : : "m" (raid_gfmulpshufb[V[2]][0][0]));
asm volatile ("movdqa %0,%%xmm14" : : "m" (raid_gfmulpshufb[V[2]][0]));
asm volatile ("movdqa %xmm14,%xmm15");
asm volatile ("pshufb %xmm0,%xmm14");
asm volatile ("pshufb %xmm1,%xmm15");
@ -2856,7 +2856,7 @@ static __always_inline void raid_rec2_ssse3ext_2(int has_p, int *id, int *ip, in
asm volatile ("movdqa %xmm15,%xmm7");
/* V[2] * syndrome0 high */
asm volatile ("movdqa %0,%%xmm14" : : "m" (raid_gfmulpshufb[V[2]][1][0]));
asm volatile ("movdqa %0,%%xmm14" : : "m" (raid_gfmulpshufb[V[2]][1]));
asm volatile ("movdqa %xmm14,%xmm15");
asm volatile ("pshufb %xmm4,%xmm14");
asm volatile ("pshufb %xmm5,%xmm15");
@ -2864,7 +2864,7 @@ static __always_inline void raid_rec2_ssse3ext_2(int has_p, int *id, int *ip, in
asm volatile ("pxor %xmm15,%xmm7");
/* V[3] * syndrome1 low */
asm volatile ("movdqa %0,%%xmm14" : : "m" (raid_gfmulpshufb[V[3]][0][0]));
asm volatile ("movdqa %0,%%xmm14" : : "m" (raid_gfmulpshufb[V[3]][0]));
asm volatile ("movdqa %xmm14,%xmm15");
asm volatile ("pshufb %xmm2,%xmm14");
asm volatile ("pshufb %xmm3,%xmm15");
@ -2872,7 +2872,7 @@ static __always_inline void raid_rec2_ssse3ext_2(int has_p, int *id, int *ip, in
asm volatile ("pxor %xmm15,%xmm7");
/* V[3] * syndrome1 high */
asm volatile ("movdqa %0,%%xmm14" : : "m" (raid_gfmulpshufb[V[3]][1][0]));
asm volatile ("movdqa %0,%%xmm14" : : "m" (raid_gfmulpshufb[V[3]][1]));
asm volatile ("movdqa %xmm14,%xmm15");
asm volatile ("pshufb %xmm12,%xmm14");
asm volatile ("pshufb %xmm13,%xmm15");
@ -2941,7 +2941,7 @@ static __always_inline void raid_rec2of2_ssse3ext(int *id, int *ip, int nd, size
* xmm11 active reduction polynomial
* xmm12-xmm15 analytical multiplication tables
*/
asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.poly[0]));
asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.poly));
/* keep the analytical multiplication tables resident */
asm volatile ("movdqa %0,%%xmm12" : : "m" (R[0][0][0]));
@ -3077,7 +3077,7 @@ static __always_inline void raid_rec2of2_ssse3ext(int *id, int *ip, int nd, size
asm volatile ("pxor %0,%%xmm7" : : "m" (q[i + 48]));
/* the polynomial is no longer needed during reconstruction */
asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.low4[0]));
asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.low4));
/* lane 0: split Pdelta */
asm volatile ("movdqa %xmm0,%xmm8");
@ -3224,7 +3224,7 @@ static __always_inline void raid_rec2of2_ssse3ext(int *id, int *ip, int nd, size
asm volatile ("movdqa %%xmm10,%0" : "=m" (qa[i + 48]));
/* restore the polynomial for the next 64-byte iteration */
asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.poly[0]));
asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.poly));
}
raid_sse_end(0);
@ -3295,7 +3295,7 @@ static __always_inline void raid_recX_ssse3ext_123(int nr, int has_p, int *id, i
* Xmm15 is later reused by reconstruction, therefore reload the
* nibble mask on every iteration.
*/
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0]));
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4));
asm volatile ("movdqa %0,%%xmm0" : : "m" (p[0][i]));
asm volatile ("movdqa %0,%%xmm1" : : "m" (p[0][i + 16]));
@ -3555,7 +3555,7 @@ static __always_inline void raid_recX_ssse3ext_12345(int nr, int has_p, int *id,
* xmm15 is reused by reconstruction, therefore reload the
* nibble mask on every iteration.
*/
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0]));
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4));
/*
* Raw syndrome accumulators:
@ -3848,7 +3848,7 @@ static __always_inline void raid_recX_ssse3ext(int nr, int has_p, int *id, int *
* Xmm15 is later reused by reconstruction, therefore reload the
* nibble mask on every iteration.
*/
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0]));
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4));
asm volatile ("movdqa %0,%%xmm0" : : "m" (p[0][i]));