Extend the pshufb table to include the Q parity coefficients

This commit is contained in:
Andrea Mazzoleni 2026-05-28 18:25:17 +02:00
parent 04df1f7b26
commit dd8e3bfcb0
4 changed files with 4181 additions and 181 deletions

View file

@ -514,8 +514,8 @@ void raid_gen3_ssse3(int nd, size_t size, void **vv)
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][0][0][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[l][0][1][0]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("pshufb %xmm4,%xmm2");
asm volatile ("pshufb %xmm5,%xmm6");
asm volatile ("pxor %xmm6,%xmm2");
@ -538,10 +538,10 @@ void raid_gen3_ssse3(int nd, size_t size, void **vv)
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][0][0][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pxor %xmm6,%xmm2");
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][0][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("pshufb %xmm5,%xmm6");
asm volatile ("pxor %xmm6,%xmm2");
}
@ -619,8 +619,8 @@ void raid_gen3_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("pand %xmm11,%xmm5");
asm volatile ("pand %xmm11,%xmm13");
asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][0][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][0][1][0]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("movdqa %xmm2,%xmm10");
asm volatile ("movdqa %xmm7,%xmm15");
asm volatile ("pshufb %xmm4,%xmm2");
@ -660,8 +660,8 @@ void raid_gen3_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("pand %xmm11,%xmm5");
asm volatile ("pand %xmm11,%xmm13");
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][0][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][0][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("movdqa %xmm6,%xmm14");
asm volatile ("movdqa %xmm7,%xmm15");
asm volatile ("pshufb %xmm4,%xmm6");
@ -758,8 +758,8 @@ void raid_gen3_avx2ext(int nd, size_t size, void **vv)
asm volatile ("vpand %ymm11,%ymm5,%ymm5");
asm volatile ("vpand %ymm11,%ymm13,%ymm13");
asm volatile ("vbroadcasti128 %0,%%ymm10" : : "m" (gfgenpshufb[l][0][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[l][0][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm10" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("vpshufb %ymm4,%ymm10,%ymm2");
asm volatile ("vpshufb %ymm12,%ymm10,%ymm10");
asm volatile ("vpshufb %ymm5,%ymm15,%ymm7");
@ -795,8 +795,8 @@ void raid_gen3_avx2ext(int nd, size_t size, void **vv)
asm volatile ("vpand %ymm11,%ymm5,%ymm5");
asm volatile ("vpand %ymm11,%ymm13,%ymm13");
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (gfgenpshufb[d][0][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[d][0][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("vpshufb %ymm4,%ymm14,%ymm6");
asm volatile ("vpshufb %ymm12,%ymm14,%ymm14");
asm volatile ("vpshufb %ymm5,%ymm15,%ymm7");
@ -890,8 +890,8 @@ void raid_gen3_avx512bw(int nd, size_t size, void **vv)
asm volatile ("vpandq %zmm15,%zmm10,%zmm10");
asm volatile ("vpandq %zmm15,%zmm11,%zmm11");
asm volatile ("vbroadcasti32x4 %0,%%zmm2" : : "m" (gfgenpshufb[l][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm2" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("vpshufb %zmm10,%zmm2,%zmm2");
asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");
asm volatile ("vpxorq %zmm13,%zmm2,%zmm2");
@ -913,8 +913,8 @@ void raid_gen3_avx512bw(int nd, size_t size, void **vv)
asm volatile ("vpandq %zmm15,%zmm10,%zmm10");
asm volatile ("vpandq %zmm15,%zmm11,%zmm11");
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("vpshufb %zmm10,%zmm12,%zmm12");
asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");
asm volatile ("vpxorq %zmm12,%zmm2,%zmm2");
@ -979,14 +979,14 @@ void raid_gen4_ssse3(int nd, size_t size, void **vv)
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][0][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][0][1][0]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("pshufb %xmm4,%xmm2");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm7,%xmm2");
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][1][1][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][3][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][3][1][0]));
asm volatile ("pshufb %xmm4,%xmm3");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm7,%xmm3");
@ -1012,15 +1012,15 @@ void raid_gen4_ssse3(int nd, size_t size, void **vv)
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][0][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][0][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm6,%xmm2");
asm volatile ("pxor %xmm7,%xmm2");
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][1][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][3][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][3][1][0]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm6,%xmm3");
@ -1103,8 +1103,8 @@ void raid_gen4_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("pand %xmm15,%xmm5");
asm volatile ("pand %xmm15,%xmm13");
asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][0][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][0][1][0]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("movdqa %xmm2,%xmm10");
asm volatile ("movdqa %xmm7,%xmm15");
asm volatile ("pshufb %xmm4,%xmm2");
@ -1114,8 +1114,8 @@ void raid_gen4_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("pxor %xmm7,%xmm2");
asm volatile ("pxor %xmm15,%xmm10");
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][1][1][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][3][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][3][1][0]));
asm volatile ("movdqa %xmm3,%xmm11");
asm volatile ("movdqa %xmm7,%xmm15");
asm volatile ("pshufb %xmm4,%xmm3");
@ -1157,8 +1157,8 @@ void raid_gen4_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("pand %xmm15,%xmm5");
asm volatile ("pand %xmm15,%xmm13");
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][0][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][0][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("movdqa %xmm6,%xmm14");
asm volatile ("movdqa %xmm7,%xmm15");
asm volatile ("pshufb %xmm4,%xmm6");
@ -1170,8 +1170,8 @@ void raid_gen4_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("pxor %xmm7,%xmm2");
asm volatile ("pxor %xmm15,%xmm10");
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][1][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][3][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][3][1][0]));
asm volatile ("movdqa %xmm6,%xmm14");
asm volatile ("movdqa %xmm7,%xmm15");
asm volatile ("pshufb %xmm4,%xmm6");
@ -1274,8 +1274,8 @@ void raid_gen4_avx2ext(int nd, size_t size, void **vv)
asm volatile ("vpand %ymm15,%ymm5,%ymm5");
asm volatile ("vpand %ymm15,%ymm13,%ymm13");
asm volatile ("vbroadcasti128 %0,%%ymm10" : : "m" (gfgenpshufb[l][0][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[l][0][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm10" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("vpshufb %ymm4,%ymm10,%ymm2");
asm volatile ("vpshufb %ymm5,%ymm15,%ymm7");
asm volatile ("vpshufb %ymm12,%ymm10,%ymm10");
@ -1283,8 +1283,8 @@ void raid_gen4_avx2ext(int nd, size_t size, void **vv)
asm volatile ("vpxor %ymm7,%ymm2,%ymm2");
asm volatile ("vpxor %ymm15,%ymm10,%ymm10");
asm volatile ("vbroadcasti128 %0,%%ymm11" : : "m" (gfgenpshufb[l][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[l][1][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm11" : : "m" (gfgenpshufb[l][3][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[l][3][1][0]));
asm volatile ("vpshufb %ymm4,%ymm11,%ymm3");
asm volatile ("vpshufb %ymm5,%ymm15,%ymm7");
asm volatile ("vpshufb %ymm12,%ymm11,%ymm11");
@ -1322,8 +1322,8 @@ void raid_gen4_avx2ext(int nd, size_t size, void **vv)
asm volatile ("vpand %ymm15,%ymm5,%ymm5");
asm volatile ("vpand %ymm15,%ymm13,%ymm13");
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (gfgenpshufb[d][0][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[d][0][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("vpshufb %ymm4,%ymm14,%ymm6");
asm volatile ("vpshufb %ymm5,%ymm15,%ymm7");
asm volatile ("vpshufb %ymm12,%ymm14,%ymm14");
@ -1333,8 +1333,8 @@ void raid_gen4_avx2ext(int nd, size_t size, void **vv)
asm volatile ("vpxor %ymm7,%ymm2,%ymm2");
asm volatile ("vpxor %ymm15,%ymm10,%ymm10");
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (gfgenpshufb[d][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[d][1][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (gfgenpshufb[d][3][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[d][3][1][0]));
asm volatile ("vpshufb %ymm4,%ymm14,%ymm6");
asm volatile ("vpshufb %ymm5,%ymm15,%ymm7");
asm volatile ("vpshufb %ymm12,%ymm14,%ymm14");
@ -1436,14 +1436,14 @@ void raid_gen4_avx512bw(int nd, size_t size, void **vv)
asm volatile ("vpandq %zmm15,%zmm10,%zmm10");
asm volatile ("vpandq %zmm15,%zmm11,%zmm11");
asm volatile ("vbroadcasti32x4 %0,%%zmm2" : : "m" (gfgenpshufb[l][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm2" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("vpshufb %zmm10,%zmm2,%zmm2");
asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");
asm volatile ("vpxorq %zmm13,%zmm2,%zmm2");
asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (gfgenpshufb[l][1][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][1][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (gfgenpshufb[l][3][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][3][1][0]));
asm volatile ("vpshufb %zmm10,%zmm3,%zmm3");
asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");
asm volatile ("vpxorq %zmm13,%zmm3,%zmm3");
@ -1465,15 +1465,15 @@ void raid_gen4_avx512bw(int nd, size_t size, void **vv)
asm volatile ("vpandq %zmm15,%zmm10,%zmm10");
asm volatile ("vpandq %zmm15,%zmm11,%zmm11");
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("vpshufb %zmm10,%zmm12,%zmm12");
asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");
asm volatile ("vpxorq %zmm12,%zmm2,%zmm2");
asm volatile ("vpxorq %zmm13,%zmm2,%zmm2");
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][1][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][1][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][3][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][3][1][0]));
asm volatile ("vpshufb %zmm10,%zmm12,%zmm12");
asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");
asm volatile ("vpxorq %zmm12,%zmm3,%zmm3");
@ -1544,20 +1544,20 @@ void raid_gen5_ssse3(int nd, size_t size, void **vv)
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("movdqa %0,%%xmm1" : : "m" (gfgenpshufb[l][0][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][0][1][0]));
asm volatile ("movdqa %0,%%xmm1" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("pshufb %xmm4,%xmm1");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm7,%xmm1");
asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][1][1][0]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][3][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][3][1][0]));
asm volatile ("pshufb %xmm4,%xmm2");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm7,%xmm2");
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][4][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][4][1][0]));
asm volatile ("pshufb %xmm4,%xmm3");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm7,%xmm3");
@ -1584,22 +1584,22 @@ void raid_gen5_ssse3(int nd, size_t size, void **vv)
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][0][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][0][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm6,%xmm1");
asm volatile ("pxor %xmm7,%xmm1");
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][1][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][3][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][3][1][0]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm6,%xmm2");
asm volatile ("pxor %xmm7,%xmm2");
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][4][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][4][1][0]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm6,%xmm3");
@ -1682,20 +1682,20 @@ void raid_gen5_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("pand %xmm15,%xmm10");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][0][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][0][1][0]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("pshufb %xmm10,%xmm2");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm13,%xmm2");
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][1][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][1][1][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][3][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][3][1][0]));
asm volatile ("pshufb %xmm10,%xmm3");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm13,%xmm3");
asm volatile ("movdqa %0,%%xmm4" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("movdqa %0,%%xmm4" : : "m" (gfgenpshufb[l][4][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][4][1][0]));
asm volatile ("pshufb %xmm10,%xmm4");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm13,%xmm4");
@ -1718,22 +1718,22 @@ void raid_gen5_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("pand %xmm15,%xmm10");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][0][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][0][1][0]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("pshufb %xmm10,%xmm12");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm12,%xmm2");
asm volatile ("pxor %xmm13,%xmm2");
asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][1][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][1][1][0]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][3][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][3][1][0]));
asm volatile ("pshufb %xmm10,%xmm12");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm12,%xmm3");
asm volatile ("pxor %xmm13,%xmm3");
asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][4][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][4][1][0]));
asm volatile ("pshufb %xmm10,%xmm12");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm12,%xmm4");
@ -1815,20 +1815,20 @@ void raid_gen5_avx2ext(int nd, size_t size, void **vv)
asm volatile ("vpand %ymm15,%ymm10,%ymm10");
asm volatile ("vpand %ymm15,%ymm11,%ymm11");
asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfgenpshufb[l][0][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][0][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("vpshufb %ymm10,%ymm2,%ymm2");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm13,%ymm2,%ymm2");
asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfgenpshufb[l][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][1][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfgenpshufb[l][3][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][3][1][0]));
asm volatile ("vpshufb %ymm10,%ymm3,%ymm3");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm13,%ymm3,%ymm3");
asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (gfgenpshufb[l][4][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][4][1][0]));
asm volatile ("vpshufb %ymm10,%ymm4,%ymm4");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm13,%ymm4,%ymm4");
@ -1849,22 +1849,22 @@ void raid_gen5_avx2ext(int nd, size_t size, void **vv)
asm volatile ("vpand %ymm15,%ymm10,%ymm10");
asm volatile ("vpand %ymm15,%ymm11,%ymm11");
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][0][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][0][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("vpshufb %ymm10,%ymm12,%ymm12");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm12,%ymm2,%ymm2");
asm volatile ("vpxor %ymm13,%ymm2,%ymm2");
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][1][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][3][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][3][1][0]));
asm volatile ("vpshufb %ymm10,%ymm12,%ymm12");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm12,%ymm3,%ymm3");
asm volatile ("vpxor %ymm13,%ymm3,%ymm3");
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][4][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][4][1][0]));
asm volatile ("vpshufb %ymm10,%ymm12,%ymm12");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm12,%ymm4,%ymm4");
@ -1949,20 +1949,20 @@ void raid_gen5_avx512bw(int nd, size_t size, void **vv)
asm volatile ("vpandq %zmm15,%zmm10,%zmm10");
asm volatile ("vpandq %zmm15,%zmm11,%zmm11");
asm volatile ("vbroadcasti32x4 %0,%%zmm2" : : "m" (gfgenpshufb[l][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm2" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("vpshufb %zmm10,%zmm2,%zmm2");
asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");
asm volatile ("vpxorq %zmm13,%zmm2,%zmm2");
asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (gfgenpshufb[l][1][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][1][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (gfgenpshufb[l][3][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][3][1][0]));
asm volatile ("vpshufb %zmm10,%zmm3,%zmm3");
asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");
asm volatile ("vpxorq %zmm13,%zmm3,%zmm3");
asm volatile ("vbroadcasti32x4 %0,%%zmm4" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm4" : : "m" (gfgenpshufb[l][4][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][4][1][0]));
asm volatile ("vpshufb %zmm10,%zmm4,%zmm4");
asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");
asm volatile ("vpxorq %zmm13,%zmm4,%zmm4");
@ -1984,22 +1984,22 @@ void raid_gen5_avx512bw(int nd, size_t size, void **vv)
asm volatile ("vpandq %zmm15,%zmm10,%zmm10");
asm volatile ("vpandq %zmm15,%zmm11,%zmm11");
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("vpshufb %zmm10,%zmm12,%zmm12");
asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");
asm volatile ("vpxorq %zmm12,%zmm2,%zmm2");
asm volatile ("vpxorq %zmm13,%zmm2,%zmm2");
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][1][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][1][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][3][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][3][1][0]));
asm volatile ("vpshufb %zmm10,%zmm12,%zmm12");
asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");
asm volatile ("vpxorq %zmm12,%zmm3,%zmm3");
asm volatile ("vpxorq %zmm13,%zmm3,%zmm3");
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][4][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][4][1][0]));
asm volatile ("vpshufb %zmm10,%zmm12,%zmm12");
asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");
asm volatile ("vpxorq %zmm12,%zmm4,%zmm4");
@ -2074,26 +2074,26 @@ void raid_gen6_ssse3(int nd, size_t size, void **vv)
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("movdqa %0,%%xmm0" : : "m" (gfgenpshufb[l][0][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][0][1][0]));
asm volatile ("movdqa %0,%%xmm0" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("pshufb %xmm4,%xmm0");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm7,%xmm0");
asm volatile ("movdqa %0,%%xmm1" : : "m" (gfgenpshufb[l][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][1][1][0]));
asm volatile ("movdqa %0,%%xmm1" : : "m" (gfgenpshufb[l][3][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][3][1][0]));
asm volatile ("pshufb %xmm4,%xmm1");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm7,%xmm1");
asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][4][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][4][1][0]));
asm volatile ("pshufb %xmm4,%xmm2");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm7,%xmm2");
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][3][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][3][1][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][5][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][5][1][0]));
asm volatile ("pshufb %xmm4,%xmm3");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm7,%xmm3");
@ -2123,29 +2123,29 @@ void raid_gen6_ssse3(int nd, size_t size, void **vv)
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][0][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][0][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm6,%xmm0");
asm volatile ("pxor %xmm7,%xmm0");
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][1][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][3][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][3][1][0]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm6,%xmm1");
asm volatile ("pxor %xmm7,%xmm1");
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][4][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][4][1][0]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm6,%xmm2");
asm volatile ("pxor %xmm7,%xmm2");
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][3][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][3][1][0]));
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][5][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][5][1][0]));
asm volatile ("pshufb %xmm4,%xmm6");
asm volatile ("pshufb %xmm5,%xmm7");
asm volatile ("pxor %xmm6,%xmm3");
@ -2233,26 +2233,26 @@ void raid_gen6_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("pand %xmm15,%xmm10");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][0][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][0][1][0]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("pshufb %xmm10,%xmm2");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm13,%xmm2");
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][1][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][1][1][0]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][3][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][3][1][0]));
asm volatile ("pshufb %xmm10,%xmm3");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm13,%xmm3");
asm volatile ("movdqa %0,%%xmm4" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("movdqa %0,%%xmm4" : : "m" (gfgenpshufb[l][4][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][4][1][0]));
asm volatile ("pshufb %xmm10,%xmm4");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm13,%xmm4");
asm volatile ("movdqa %0,%%xmm5" : : "m" (gfgenpshufb[l][3][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][3][1][0]));
asm volatile ("movdqa %0,%%xmm5" : : "m" (gfgenpshufb[l][5][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][5][1][0]));
asm volatile ("pshufb %xmm10,%xmm5");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm13,%xmm5");
@ -2275,29 +2275,29 @@ void raid_gen6_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("pand %xmm15,%xmm10");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][0][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][0][1][0]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("pshufb %xmm10,%xmm12");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm12,%xmm2");
asm volatile ("pxor %xmm13,%xmm2");
asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][1][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][1][1][0]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][3][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][3][1][0]));
asm volatile ("pshufb %xmm10,%xmm12");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm12,%xmm3");
asm volatile ("pxor %xmm13,%xmm3");
asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][4][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][4][1][0]));
asm volatile ("pshufb %xmm10,%xmm12");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm12,%xmm4");
asm volatile ("pxor %xmm13,%xmm4");
asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][3][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][3][1][0]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][5][0][0]));
asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][5][1][0]));
asm volatile ("pshufb %xmm10,%xmm12");
asm volatile ("pshufb %xmm11,%xmm13");
asm volatile ("pxor %xmm12,%xmm5");
@ -2394,26 +2394,26 @@ void raid_gen6_avx2ext(int nd, size_t size, void **vv)
asm volatile ("vpand %ymm15,%ymm10,%ymm10");
asm volatile ("vpand %ymm15,%ymm11,%ymm11");
asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfgenpshufb[l][0][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][0][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("vpshufb %ymm10,%ymm2,%ymm2");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm13,%ymm2,%ymm2");
asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfgenpshufb[l][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][1][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfgenpshufb[l][3][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][3][1][0]));
asm volatile ("vpshufb %ymm10,%ymm3,%ymm3");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm13,%ymm3,%ymm3");
asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (gfgenpshufb[l][4][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][4][1][0]));
asm volatile ("vpshufb %ymm10,%ymm4,%ymm4");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm13,%ymm4,%ymm4");
asm volatile ("vbroadcasti128 %0,%%ymm5" : : "m" (gfgenpshufb[l][3][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][3][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm5" : : "m" (gfgenpshufb[l][5][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][5][1][0]));
asm volatile ("vpshufb %ymm10,%ymm5,%ymm5");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm13,%ymm5,%ymm5");
@ -2434,29 +2434,29 @@ void raid_gen6_avx2ext(int nd, size_t size, void **vv)
asm volatile ("vpand %ymm15,%ymm10,%ymm10");
asm volatile ("vpand %ymm15,%ymm11,%ymm11");
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][0][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][0][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("vpshufb %ymm10,%ymm12,%ymm12");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm12,%ymm2,%ymm2");
asm volatile ("vpxor %ymm13,%ymm2,%ymm2");
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][1][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][3][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][3][1][0]));
asm volatile ("vpshufb %ymm10,%ymm12,%ymm12");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm12,%ymm3,%ymm3");
asm volatile ("vpxor %ymm13,%ymm3,%ymm3");
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][4][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][4][1][0]));
asm volatile ("vpshufb %ymm10,%ymm12,%ymm12");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm12,%ymm4,%ymm4");
asm volatile ("vpxor %ymm13,%ymm4,%ymm4");
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][3][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][3][1][0]));
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][5][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][5][1][0]));
asm volatile ("vpshufb %ymm10,%ymm12,%ymm12");
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm12,%ymm5,%ymm5");
@ -2556,26 +2556,26 @@ void raid_gen6_avx512bw(int nd, size_t size, void **vv)
asm volatile ("vpandq %zmm15,%zmm10,%zmm10");
asm volatile ("vpandq %zmm15,%zmm11,%zmm11");
asm volatile ("vbroadcasti32x4 %0,%%zmm2" : : "m" (gfgenpshufb[l][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm2" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("vpshufb %zmm10,%zmm2,%zmm2");
asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");
asm volatile ("vpxorq %zmm13,%zmm2,%zmm2");
asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (gfgenpshufb[l][1][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][1][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (gfgenpshufb[l][3][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][3][1][0]));
asm volatile ("vpshufb %zmm10,%zmm3,%zmm3");
asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");
asm volatile ("vpxorq %zmm13,%zmm3,%zmm3");
asm volatile ("vbroadcasti32x4 %0,%%zmm4" : : "m" (gfgenpshufb[l][2][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][2][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm4" : : "m" (gfgenpshufb[l][4][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][4][1][0]));
asm volatile ("vpshufb %zmm10,%zmm4,%zmm4");
asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");
asm volatile ("vpxorq %zmm13,%zmm4,%zmm4");
asm volatile ("vbroadcasti32x4 %0,%%zmm5" : : "m" (gfgenpshufb[l][3][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][3][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm5" : : "m" (gfgenpshufb[l][5][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][5][1][0]));
asm volatile ("vpshufb %zmm10,%zmm5,%zmm5");
asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");
asm volatile ("vpxorq %zmm13,%zmm5,%zmm5");
@ -2598,29 +2598,29 @@ void raid_gen6_avx512bw(int nd, size_t size, void **vv)
asm volatile ("vpandq %zmm15,%zmm10,%zmm10");
asm volatile ("vpandq %zmm15,%zmm11,%zmm11");
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][0][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][0][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("vpshufb %zmm10,%zmm12,%zmm12");
asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");
asm volatile ("vpxorq %zmm12,%zmm2,%zmm2");
asm volatile ("vpxorq %zmm13,%zmm2,%zmm2");
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][1][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][1][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][3][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][3][1][0]));
asm volatile ("vpshufb %zmm10,%zmm12,%zmm12");
asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");
asm volatile ("vpxorq %zmm12,%zmm3,%zmm3");
asm volatile ("vpxorq %zmm13,%zmm3,%zmm3");
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][2][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][2][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][4][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][4][1][0]));
asm volatile ("vpshufb %zmm10,%zmm12,%zmm12");
asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");
asm volatile ("vpxorq %zmm12,%zmm4,%zmm4");
asm volatile ("vpxorq %zmm13,%zmm4,%zmm4");
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][3][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][3][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][5][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][5][1][0]));
asm volatile ("vpshufb %zmm10,%zmm12,%zmm12");
asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");
asm volatile ("vpxorq %zmm12,%zmm5,%zmm5");