From cdda0bdd566d7f0b565dfc859561ac97be5b7878 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Fri, 5 Jun 2026 11:01:03 +0200 Subject: [PATCH] Refactor and reorganize parity generation functions using inline templates --- Makefile.am | 7 +- raid/avx2.c | 891 ++++++++++++ raid/avx512.c | 419 ++++++ raid/gfni.c | 749 ++++------- raid/int.c | 102 ++ raid/internal.h | 15 +- raid/intz.c | 108 -- raid/memory.h | 3 +- raid/module.c | 5 +- raid/neon.c | 1045 ++++----------- raid/sse2.c | 430 ++++++ raid/ssse3.c | 1278 ++++++++++++++++++ raid/tables.c | 20 + raid/test/Makefile | 2 +- raid/x86.c | 3195 -------------------------------------------- raid/x86z.c | 244 ---- 16 files changed, 3629 insertions(+), 4884 deletions(-) create mode 100644 raid/avx2.c create mode 100644 raid/avx512.c delete mode 100644 raid/intz.c create mode 100644 raid/sse2.c create mode 100644 raid/ssse3.c delete mode 100644 raid/x86.c delete mode 100644 raid/x86z.c diff --git a/Makefile.am b/Makefile.am index c99e3872..72a10a5d 100644 --- a/Makefile.am +++ b/Makefile.am @@ -25,11 +25,12 @@ snapraid_SOURCES = \ raid/module.c \ raid/tables.c \ raid/int.c \ - raid/x86.c \ + raid/sse2.c \ + raid/ssse3.c \ + raid/avx2.c \ + raid/avx512.c \ raid/gfni.c \ raid/neon.c \ - raid/intz.c \ - raid/x86z.c \ raid/helper.c \ raid/memory.c \ raid/test.c \ diff --git a/raid/avx2.c b/raid/avx2.c new file mode 100644 index 00000000..97fbd33e --- /dev/null +++ b/raid/avx2.c @@ -0,0 +1,891 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +// Copyright (C) 2026 Andrea Mazzoleni + +#include "internal.h" +#include "gf.h" +#include "cpu.h" + +#ifdef CONFIG_X86 + +/* + * GEN1 (RAID5 with xor) AVX2 implementation + */ +void raid_gen1_avx2(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + + raid_avx_begin(); + + for (i = 0; i < size; i += 64) { + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[0][i])); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[0][i + 32])); + + for (d = 1; d <= l; ++d) { + asm volatile ("vpxor %0,%%ymm0,%%ymm0" : : "m" (v[d][i])); + asm volatile ("vpxor %0,%%ymm1,%%ymm1" : : "m" (v[d][i + 32])); + } + + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); + asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); + } + + raid_avx_end(); +} + +/* + * GEN2 (RAID6 with powers of 2) AVX2 implementation + */ +void raid_gen2_avx2(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + + raid_avx_begin(); + + asm volatile ("vpbroadcastb %0, %%ymm7" : : "m" (gfconst16.poly[0])); + asm volatile ("vpxor %ymm6,%ymm6,%ymm6"); + + for (i = 0; i < size; i += 64) { + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[l][i])); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[l][i + 32])); + asm volatile ("vmovdqa %ymm0,%ymm2"); + asm volatile ("vmovdqa %ymm1,%ymm3"); + + for (d = l - 1; d >= 0; --d) { + /* scale by 2 before adding the current disk so the first (last processed) disk with factor 1 avoids doubling */ + asm volatile ("vpcmpgtb %ymm2,%ymm6,%ymm4"); + asm volatile ("vpcmpgtb %ymm3,%ymm6,%ymm5"); + asm volatile ("vpaddb %ymm2,%ymm2,%ymm2"); + asm volatile ("vpaddb %ymm3,%ymm3,%ymm3"); + asm volatile ("vpand %ymm7,%ymm4,%ymm4"); + asm volatile ("vpand %ymm7,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm4,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm5,%ymm3,%ymm3"); + + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i])); + asm volatile ("vmovdqa %0,%%ymm5" : : "m" (v[d][i + 32])); + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm4,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm5,%ymm3,%ymm3"); + } + + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); + asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); + asm volatile ("vmovntdq %%ymm2,%0" : "=m" (q[i])); + asm volatile ("vmovntdq %%ymm3,%0" : "=m" (q[i + 32])); + } + + raid_avx_end(); +} + +#ifdef CONFIG_X86_64 +/* + * GENz (triple parity with powers of 2^-1) AVX2 implementation + * + * Note that it uses 16 registers, meaning that x64 is required. + */ +void raid_genz_avx2ext(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t**)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + + raid_avx_begin(); + + asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.poly[0])); + asm volatile ("vpbroadcastb %0,%%ymm3" : : "m" (gfconst16.half[0])); + asm volatile ("vpbroadcastb %0,%%ymm11" : : "m" (gfconst16.low7[0])); + asm volatile ("vpxor %ymm15,%ymm15,%ymm15"); + + for (i = 0; i < size; i += 64) { + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[l][i])); + asm volatile ("vmovdqa %0,%%ymm8" : : "m" (v[l][i + 32])); + asm volatile ("vmovdqa %ymm0,%ymm1"); + asm volatile ("vmovdqa %ymm8,%ymm9"); + asm volatile ("vmovdqa %ymm0,%ymm2"); + asm volatile ("vmovdqa %ymm8,%ymm10"); + for (d = l - 1; d >= 0; --d) { + asm volatile ("vpsllw $7,%ymm2,%ymm6"); + asm volatile ("vpsllw $7,%ymm10,%ymm14"); + asm volatile ("vpsrlw $1,%ymm2,%ymm2"); + asm volatile ("vpsrlw $1,%ymm10,%ymm10"); + asm volatile ("vpcmpgtb %ymm1,%ymm15,%ymm4"); + asm volatile ("vpcmpgtb %ymm9,%ymm15,%ymm12"); + asm volatile ("vpcmpgtb %ymm6,%ymm15,%ymm5"); + asm volatile ("vpcmpgtb %ymm14,%ymm15,%ymm13"); + asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); + asm volatile ("vpaddb %ymm9,%ymm9,%ymm9"); + asm volatile ("vpand %ymm11,%ymm2,%ymm2"); + asm volatile ("vpand %ymm11,%ymm10,%ymm10"); + asm volatile ("vpand %ymm7,%ymm4,%ymm4"); + asm volatile ("vpand %ymm7,%ymm12,%ymm12"); + asm volatile ("vpand %ymm3,%ymm5,%ymm5"); + asm volatile ("vpand %ymm3,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); + asm volatile ("vpxor %ymm5,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm13,%ymm10,%ymm10"); + + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i])); + asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i + 32])); + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm4,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); + asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); + asm volatile ("vpxor %ymm12,%ymm10,%ymm10"); + } + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); + asm volatile ("vmovntdq %%ymm8,%0" : "=m" (p[i + 32])); + asm volatile ("vmovntdq %%ymm1,%0" : "=m" (q[i])); + asm volatile ("vmovntdq %%ymm9,%0" : "=m" (q[i + 32])); + asm volatile ("vmovntdq %%ymm2,%0" : "=m" (r[i])); + asm volatile ("vmovntdq %%ymm10,%0" : "=m" (r[i + 32])); + } + + raid_avx_end(); +} +#endif + +#ifdef CONFIG_X86_64 +/* + * GEN3 (triple parity with Cauchy matrix) AVX2 implementation + * + * Note that it uses 16 registers, meaning that x64 is required. + */ +void raid_gen3_avx2ext(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + + /* special case with only one data disk */ + if (l == 0) { + for (i = 0; i < 3; ++i) + memcpy(v[1 + i], v[0], size); + return; + } + + raid_avx_begin(); + + /* generic case with at least two data disks */ + asm volatile ("vpbroadcastb %0, %%ymm3" : : "m" (gfconst16.poly[0])); + asm volatile ("vpbroadcastb %0, %%ymm11" : : "m" (gfconst16.low4[0])); + + for (i = 0; i < size; i += 64) { + /* last disk without the by two multiplication */ + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[l][i])); + asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[l][i + 32])); + + asm volatile ("vmovdqa %ymm4,%ymm0"); + asm volatile ("vmovdqa %ymm4,%ymm1"); + asm volatile ("vmovdqa %ymm12,%ymm8"); + asm volatile ("vmovdqa %ymm12,%ymm9"); + + asm volatile ("vpsrlw $4,%ymm4,%ymm5"); + asm volatile ("vpsrlw $4,%ymm12,%ymm13"); + asm volatile ("vpand %ymm11,%ymm4,%ymm4"); + asm volatile ("vpand %ymm11,%ymm12,%ymm12"); + asm volatile ("vpand %ymm11,%ymm5,%ymm5"); + asm volatile ("vpand %ymm11,%ymm13,%ymm13"); + + asm volatile ("vbroadcasti128 %0,%%ymm10" : : "m" (gfgenpshufb[l][1][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[l][1][1][0])); + asm volatile ("vpshufb %ymm4,%ymm10,%ymm2"); + asm volatile ("vpshufb %ymm12,%ymm10,%ymm10"); + asm volatile ("vpshufb %ymm5,%ymm15,%ymm7"); + asm volatile ("vpshufb %ymm13,%ymm15,%ymm15"); + asm volatile ("vpxor %ymm7,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm15,%ymm10,%ymm10"); + + /* intermediate disks */ + for (d = l - 1; d > 0; --d) { + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i])); + asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i + 32])); + + asm volatile ("vpxor %ymm5,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm13,%ymm13,%ymm13"); + asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5"); + asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13"); + asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); + asm volatile ("vpaddb %ymm9,%ymm9,%ymm9"); + asm volatile ("vpand %ymm3,%ymm5,%ymm5"); + asm volatile ("vpand %ymm3,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm13,%ymm9,%ymm9"); + + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); + asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); + + asm volatile ("vpsrlw $4,%ymm4,%ymm5"); + asm volatile ("vpsrlw $4,%ymm12,%ymm13"); + asm volatile ("vpand %ymm11,%ymm4,%ymm4"); + asm volatile ("vpand %ymm11,%ymm12,%ymm12"); + asm volatile ("vpand %ymm11,%ymm5,%ymm5"); + asm volatile ("vpand %ymm11,%ymm13,%ymm13"); + + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (gfgenpshufb[d][1][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[d][1][1][0])); + asm volatile ("vpshufb %ymm4,%ymm14,%ymm6"); + asm volatile ("vpshufb %ymm12,%ymm14,%ymm14"); + asm volatile ("vpshufb %ymm5,%ymm15,%ymm7"); + asm volatile ("vpshufb %ymm13,%ymm15,%ymm15"); + asm volatile ("vpxor %ymm6,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm14,%ymm10,%ymm10"); + asm volatile ("vpxor %ymm7,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm15,%ymm10,%ymm10"); + } + + /* first disk with all coefficients at 1 */ + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[0][i])); + asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[0][i + 32])); + + asm volatile ("vpxor %ymm5,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm13,%ymm13,%ymm13"); + asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5"); + asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13"); + asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); + asm volatile ("vpaddb %ymm9,%ymm9,%ymm9"); + asm volatile ("vpand %ymm3,%ymm5,%ymm5"); + asm volatile ("vpand %ymm3,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm13,%ymm9,%ymm9"); + + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm4,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); + asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); + asm volatile ("vpxor %ymm12,%ymm10,%ymm10"); + + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); + asm volatile ("vmovntdq %%ymm8,%0" : "=m" (p[i + 32])); + asm volatile ("vmovntdq %%ymm1,%0" : "=m" (q[i])); + asm volatile ("vmovntdq %%ymm9,%0" : "=m" (q[i + 32])); + asm volatile ("vmovntdq %%ymm2,%0" : "=m" (r[i])); + asm volatile ("vmovntdq %%ymm10,%0" : "=m" (r[i + 32])); + } + + raid_avx_end(); +} +#endif + +#ifdef CONFIG_X86_64 +/* + * GEN4 (quad parity with Cauchy matrix) AVX2 implementation + * + * Note that it uses 16 registers, meaning that x64 is required. + */ +void raid_gen4_avx2ext(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + uint8_t *s; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + s = v[nd + 3]; + + /* special case with only one data disk */ + if (l == 0) { + for (i = 0; i < 4; ++i) + memcpy(v[1 + i], v[0], size); + return; + } + + raid_avx_begin(); + + /* generic case with at least two data disks */ + for (i = 0; i < size; i += 64) { + /* last disk without the by two multiplication */ + asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0])); + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[l][i])); + asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[l][i + 32])); + + asm volatile ("vmovdqa %ymm4,%ymm0"); + asm volatile ("vmovdqa %ymm4,%ymm1"); + asm volatile ("vmovdqa %ymm12,%ymm8"); + asm volatile ("vmovdqa %ymm12,%ymm9"); + + asm volatile ("vpsrlw $4,%ymm4,%ymm5"); + asm volatile ("vpsrlw $4,%ymm12,%ymm13"); + asm volatile ("vpand %ymm15,%ymm4,%ymm4"); + asm volatile ("vpand %ymm15,%ymm12,%ymm12"); + asm volatile ("vpand %ymm15,%ymm5,%ymm5"); + asm volatile ("vpand %ymm15,%ymm13,%ymm13"); + + asm volatile ("vbroadcasti128 %0,%%ymm10" : : "m" (gfgenpshufb[l][1][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[l][1][1][0])); + asm volatile ("vpshufb %ymm4,%ymm10,%ymm2"); + asm volatile ("vpshufb %ymm5,%ymm15,%ymm7"); + asm volatile ("vpshufb %ymm12,%ymm10,%ymm10"); + asm volatile ("vpshufb %ymm13,%ymm15,%ymm15"); + asm volatile ("vpxor %ymm7,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm15,%ymm10,%ymm10"); + + asm volatile ("vbroadcasti128 %0,%%ymm11" : : "m" (gfgenpshufb[l][2][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[l][2][1][0])); + asm volatile ("vpshufb %ymm4,%ymm11,%ymm3"); + asm volatile ("vpshufb %ymm5,%ymm15,%ymm7"); + asm volatile ("vpshufb %ymm12,%ymm11,%ymm11"); + asm volatile ("vpshufb %ymm13,%ymm15,%ymm15"); + asm volatile ("vpxor %ymm7,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm15,%ymm11,%ymm11"); + + /* intermediate disks */ + for (d = l - 1; d > 0; --d) { + asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.poly[0])); + asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0])); + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i])); + asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i + 32])); + + asm volatile ("vpxor %ymm5,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm13,%ymm13,%ymm13"); + asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5"); + asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13"); + asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); + asm volatile ("vpaddb %ymm9,%ymm9,%ymm9"); + asm volatile ("vpand %ymm7,%ymm5,%ymm5"); + asm volatile ("vpand %ymm7,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm13,%ymm9,%ymm9"); + + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); + asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); + + asm volatile ("vpsrlw $4,%ymm4,%ymm5"); + asm volatile ("vpsrlw $4,%ymm12,%ymm13"); + asm volatile ("vpand %ymm15,%ymm4,%ymm4"); + asm volatile ("vpand %ymm15,%ymm12,%ymm12"); + asm volatile ("vpand %ymm15,%ymm5,%ymm5"); + asm volatile ("vpand %ymm15,%ymm13,%ymm13"); + + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (gfgenpshufb[d][1][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[d][1][1][0])); + asm volatile ("vpshufb %ymm4,%ymm14,%ymm6"); + asm volatile ("vpshufb %ymm5,%ymm15,%ymm7"); + asm volatile ("vpshufb %ymm12,%ymm14,%ymm14"); + asm volatile ("vpshufb %ymm13,%ymm15,%ymm15"); + asm volatile ("vpxor %ymm6,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm14,%ymm10,%ymm10"); + asm volatile ("vpxor %ymm7,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm15,%ymm10,%ymm10"); + + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (gfgenpshufb[d][2][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[d][2][1][0])); + asm volatile ("vpshufb %ymm4,%ymm14,%ymm6"); + asm volatile ("vpshufb %ymm5,%ymm15,%ymm7"); + asm volatile ("vpshufb %ymm12,%ymm14,%ymm14"); + asm volatile ("vpshufb %ymm13,%ymm15,%ymm15"); + asm volatile ("vpxor %ymm6,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm14,%ymm11,%ymm11"); + asm volatile ("vpxor %ymm7,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm15,%ymm11,%ymm11"); + } + + /* first disk with all coefficients at 1 */ + asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.poly[0])); + asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0])); + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[0][i])); + asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[0][i + 32])); + + asm volatile ("vpxor %ymm5,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm13,%ymm13,%ymm13"); + asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5"); + asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13"); + asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); + asm volatile ("vpaddb %ymm9,%ymm9,%ymm9"); + asm volatile ("vpand %ymm7,%ymm5,%ymm5"); + asm volatile ("vpand %ymm7,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm13,%ymm9,%ymm9"); + + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm4,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm4,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); + asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); + asm volatile ("vpxor %ymm12,%ymm10,%ymm10"); + asm volatile ("vpxor %ymm12,%ymm11,%ymm11"); + + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); + asm volatile ("vmovntdq %%ymm8,%0" : "=m" (p[i + 32])); + asm volatile ("vmovntdq %%ymm1,%0" : "=m" (q[i])); + asm volatile ("vmovntdq %%ymm9,%0" : "=m" (q[i + 32])); + asm volatile ("vmovntdq %%ymm2,%0" : "=m" (r[i])); + asm volatile ("vmovntdq %%ymm10,%0" : "=m" (r[i + 32])); + asm volatile ("vmovntdq %%ymm3,%0" : "=m" (s[i])); + asm volatile ("vmovntdq %%ymm11,%0" : "=m" (s[i + 32])); + } + + raid_avx_end(); +} +#endif + +#ifdef CONFIG_X86_64 +/* + * GENX AVX2EXT implementation + */ +static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, int np) +{ + uint8_t **v = (uint8_t **)vv; + size_t i; + int d, l; + + l = nd - 1; + + /* special case with only one data disk */ + if (l == 0) { + for (d = 0; d < np; ++d) + memcpy(v[1 + d], v[0], size); + return; + } + + raid_avx_begin(); + + /* generic case with at least two data disks */ + asm volatile ("vpxor %ymm6,%ymm6,%ymm6"); + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfconst16.poly[0])); + asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0])); + + for (i = 0; i < size; i += 32) { + /* last disk without the by two multiplication */ + asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[l][i])); + + asm volatile ("vmovdqa %ymm10,%ymm0"); + asm volatile ("vmovdqa %ymm10,%ymm1"); + + asm volatile ("vpsrlw $4,%ymm10,%ymm11"); + asm volatile ("vpand %ymm15,%ymm10,%ymm10"); + asm volatile ("vpand %ymm15,%ymm11,%ymm11"); + + if (np >= 3) { + asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfgenpshufb[l][1][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][1][1][0])); + asm volatile ("vpshufb %ymm10,%ymm2,%ymm2"); + asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm13,%ymm2,%ymm2"); + } + + if (np >= 4) { + asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfgenpshufb[l][2][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][2][1][0])); + asm volatile ("vpshufb %ymm10,%ymm3,%ymm3"); + asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm13,%ymm3,%ymm3"); + } + + if (np >= 5) { + asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (gfgenpshufb[l][3][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][3][1][0])); + asm volatile ("vpshufb %ymm10,%ymm4,%ymm4"); + asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm13,%ymm4,%ymm4"); + } + + if (np >= 6) { + asm volatile ("vbroadcasti128 %0,%%ymm5" : : "m" (gfgenpshufb[l][4][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][4][1][0])); + asm volatile ("vpshufb %ymm10,%ymm5,%ymm5"); + asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm13,%ymm5,%ymm5"); + } + + /* intermediate disks */ + for (d = l - 1; d > 0; --d) { + asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[d][i])); + + asm volatile ("vpcmpgtb %ymm1,%ymm6,%ymm11"); + asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); + asm volatile ("vpand %ymm14,%ymm11,%ymm11"); + asm volatile ("vpxor %ymm11,%ymm1,%ymm1"); + + asm volatile ("vpxor %ymm10,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm10,%ymm1,%ymm1"); + + asm volatile ("vpsrlw $4,%ymm10,%ymm11"); + asm volatile ("vpand %ymm15,%ymm10,%ymm10"); + asm volatile ("vpand %ymm15,%ymm11,%ymm11"); + + if (np >= 3) { + asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][1][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][1][1][0])); + asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); + asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm12,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm13,%ymm2,%ymm2"); + } + + if (np >= 4) { + asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][2][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][2][1][0])); + asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); + asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm12,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm13,%ymm3,%ymm3"); + } + + if (np >= 5) { + asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][3][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][3][1][0])); + asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); + asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm12,%ymm4,%ymm4"); + asm volatile ("vpxor %ymm13,%ymm4,%ymm4"); + } + + if (np >= 6) { + asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][4][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][4][1][0])); + asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); + asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm12,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm13,%ymm5,%ymm5"); + } + } + + /* first disk with all coefficients at 1 */ + asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[0][i])); + + asm volatile ("vpcmpgtb %ymm1,%ymm6,%ymm11"); + asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); + asm volatile ("vpand %ymm14,%ymm11,%ymm11"); + asm volatile ("vpxor %ymm11,%ymm1,%ymm1"); + + asm volatile ("vpxor %ymm10,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm10,%ymm1,%ymm1"); + if (np >= 3) + asm volatile ("vpxor %ymm10,%ymm2,%ymm2"); + if (np >= 4) + asm volatile ("vpxor %ymm10,%ymm3,%ymm3"); + if (np >= 5) + asm volatile ("vpxor %ymm10,%ymm4,%ymm4"); + if (np >= 6) + asm volatile ("vpxor %ymm10,%ymm5,%ymm5"); + + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovntdq %%ymm1,%0" : "=m" (v[nd + 1][i])); + if (np >= 3) + asm volatile ("vmovntdq %%ymm2,%0" : "=m" (v[nd + 2][i])); + if (np >= 4) + asm volatile ("vmovntdq %%ymm3,%0" : "=m" (v[nd + 3][i])); + if (np >= 5) + asm volatile ("vmovntdq %%ymm4,%0" : "=m" (v[nd + 4][i])); + if (np >= 6) + asm volatile ("vmovntdq %%ymm5,%0" : "=m" (v[nd + 5][i])); + } + + raid_avx_end(); +} +#endif + +#ifdef CONFIG_X86_64 +/* + * GEN5 (penta parity with Cauchy matrix) AVX2 implementation + * + * Note that it uses 16 registers, meaning that x64 is required. + */ +void raid_gen5_avx2ext(int nd, size_t size, void **vv) +{ + raid_genX_avx2ext(nd, size, vv, 5); +} +#endif + +#ifdef CONFIG_X86_64 +/* + * GEN6 (hexa parity with Cauchy matrix) AVX2 implementation + * + * Note that it uses 16 registers, meaning that x64 is required. + */ +void raid_gen6_avx2ext(int nd, size_t size, void **vv) +{ + raid_genX_avx2ext(nd, size, vv, 6); +} +#endif + +/* + * RAID recovering for one disk AVX2 implementation + */ +void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *pa; + uint8_t G; + uint8_t V; + size_t i; + + (void)nr; /* unused, it's always 1 */ + + /* if it's RAID5 uses the faster function */ + if (ip[0] == 0) { + raid_rec1of1(id, nd, size, vv); + return; + } + + /* setup the coefficients matrix */ + G = A(ip[0], id[0]); + + /* invert it to solve the system of linear equations */ + V = inv(G); + + /* compute delta parity */ + raid_delta_gen(1, id, ip, nd, size, vv); + + p = v[nd + ip[0]]; + pa = v[id[0]]; + + raid_avx_begin(); + + asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.low4[0])); + asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (gfmulpshufb[V][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm5" : : "m" (gfmulpshufb[V][1][0])); + + for (i = 0; i < size; i += 32) { + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[i])); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (pa[i])); + asm volatile ("vpxor %ymm1,%ymm0,%ymm0"); + asm volatile ("vpsrlw $4,%ymm0,%ymm1"); + asm volatile ("vpand %ymm7,%ymm0,%ymm0"); + asm volatile ("vpand %ymm7,%ymm1,%ymm1"); + asm volatile ("vpshufb %ymm0,%ymm4,%ymm2"); + asm volatile ("vpshufb %ymm1,%ymm5,%ymm3"); + asm volatile ("vpxor %ymm3,%ymm2,%ymm2"); + asm volatile ("vmovdqa %%ymm2,%0" : "=m" (pa[i])); + } + + raid_avx_end(); +} + +/* + * RAID recovering for two disks AVX2 implementation + */ +void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + const int N = 2; + uint8_t *p[N]; + uint8_t *pa[N]; + uint8_t G[N * N]; + uint8_t V[N * N]; + size_t i; + int j, k; + + (void)nr; /* unused, it's always 2 */ + + /* setup the coefficients matrix */ + for (j = 0; j < N; ++j) + for (k = 0; k < N; ++k) + G[j * N + k] = A(ip[j], id[k]); + + /* invert it to solve the system of linear equations */ + raid_invert(G, V, N); + + /* compute delta parity */ + raid_delta_gen(N, id, ip, nd, size, vv); + + for (j = 0; j < N; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + raid_avx_begin(); + + asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.low4[0])); + + for (i = 0; i < size; i += 32) { + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i])); + asm volatile ("vmovdqa %0,%%ymm2" : : "m" (pa[0][i])); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[1][i])); + asm volatile ("vmovdqa %0,%%ymm3" : : "m" (pa[1][i])); + asm volatile ("vpxor %ymm2,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm3,%ymm1,%ymm1"); + + asm volatile ("vpxor %ymm6,%ymm6,%ymm6"); + + asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfmulpshufb[V[0]][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfmulpshufb[V[0]][1][0])); + asm volatile ("vpsrlw $4,%ymm0,%ymm5"); + asm volatile ("vpand %ymm7,%ymm0,%ymm4"); + asm volatile ("vpand %ymm7,%ymm5,%ymm5"); + asm volatile ("vpshufb %ymm4,%ymm2,%ymm2"); + asm volatile ("vpshufb %ymm5,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm2,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm3,%ymm6,%ymm6"); + + asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfmulpshufb[V[1]][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfmulpshufb[V[1]][1][0])); + asm volatile ("vpsrlw $4,%ymm1,%ymm5"); + asm volatile ("vpand %ymm7,%ymm1,%ymm4"); + asm volatile ("vpand %ymm7,%ymm5,%ymm5"); + asm volatile ("vpshufb %ymm4,%ymm2,%ymm2"); + asm volatile ("vpshufb %ymm5,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm2,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm3,%ymm6,%ymm6"); + + asm volatile ("vmovdqa %%ymm6,%0" : "=m" (pa[0][i])); + + asm volatile ("vpxor %ymm6,%ymm6,%ymm6"); + + asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfmulpshufb[V[2]][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfmulpshufb[V[2]][1][0])); + asm volatile ("vpsrlw $4,%ymm0,%ymm5"); + asm volatile ("vpand %ymm7,%ymm0,%ymm4"); + asm volatile ("vpand %ymm7,%ymm5,%ymm5"); + asm volatile ("vpshufb %ymm4,%ymm2,%ymm2"); + asm volatile ("vpshufb %ymm5,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm2,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm3,%ymm6,%ymm6"); + + asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfmulpshufb[V[3]][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfmulpshufb[V[3]][1][0])); + asm volatile ("vpsrlw $4,%ymm1,%ymm5"); + asm volatile ("vpand %ymm7,%ymm1,%ymm4"); + asm volatile ("vpand %ymm7,%ymm5,%ymm5"); + asm volatile ("vpshufb %ymm4,%ymm2,%ymm2"); + asm volatile ("vpshufb %ymm5,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm2,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm3,%ymm6,%ymm6"); + + asm volatile ("vmovdqa %%ymm6,%0" : "=m" (pa[1][i])); + } + + raid_avx_end(); +} + +/* + * RAID recovering AVX2 implementation + */ +void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + int N = nr; + uint8_t *p[RAID_PARITY_MAX]; + uint8_t *pa[RAID_PARITY_MAX]; + uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t buffer[RAID_PARITY_MAX*32+32]; + uint8_t *pd = __align_ptr(buffer, 32); + size_t i; + int j, k; + + /* setup the coefficients matrix */ + for (j = 0; j < N; ++j) + for (k = 0; k < N; ++k) + G[j * N + k] = A(ip[j], id[k]); + + /* invert it to solve the system of linear equations */ + raid_invert(G, V, N); + + /* compute delta parity */ + raid_delta_gen(N, id, ip, nd, size, vv); + + for (j = 0; j < N; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + raid_avx_begin(); + + asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.low4[0])); + + for (i = 0; i < size; i += 32) { + /* delta */ + for (j = 0; j < N; ++j) { + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[j][i])); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (pa[j][i])); + asm volatile ("vpxor %ymm1,%ymm0,%ymm0"); + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pd[j*32])); + } + + /* reconstruct */ + for (j = 0; j < N; ++j) { + asm volatile ("vpxor %ymm0,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm1,%ymm1,%ymm1"); + + for (k = 0; k < N; ++k) { + uint8_t m = V[j * N + k]; + + asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfmulpshufb[m][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfmulpshufb[m][1][0])); + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pd[k*32])); + asm volatile ("vpsrlw $4,%ymm4,%ymm5"); + asm volatile ("vpand %ymm7,%ymm4,%ymm4"); + asm volatile ("vpand %ymm7,%ymm5,%ymm5"); + asm volatile ("vpshufb %ymm4,%ymm2,%ymm2"); + asm volatile ("vpshufb %ymm5,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm2,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm3,%ymm1,%ymm1"); + } + + asm volatile ("vpxor %ymm1,%ymm0,%ymm0"); + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[j][i])); + } + } + + raid_avx_end(); +} + +void raid_register_avx2(void) +{ + if (raid_cpu_has_avx2()) { + raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "avx2", raid_gen1_avx2); + raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "avx2", raid_gen2_avx2); +#ifdef CONFIG_X86_64 + raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "avx2e", raid_gen3_avx2ext); + raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "avx2e", raid_gen4_avx2ext); + raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "avx2e", raid_gen5_avx2ext); + raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "avx2e", raid_gen6_avx2ext); + raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "avx2e", raid_genz_avx2ext); +#endif + + raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "avx2", raid_rec1_avx2); + raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "avx2", raid_rec2_avx2); + raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "avx2", raid_recX_avx2); + raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "avx2", raid_recX_avx2); + raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "avx2", raid_recX_avx2); + raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "avx2", raid_recX_avx2); + } +} +#endif diff --git a/raid/avx512.c b/raid/avx512.c new file mode 100644 index 00000000..0895a74e --- /dev/null +++ b/raid/avx512.c @@ -0,0 +1,419 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +// Copyright (C) 2026 Andrea Mazzoleni + +#include "internal.h" +#include "gf.h" +#include "cpu.h" + +#ifdef CONFIG_X86_64 + +/* + * GEN1 (RAID5 with xor) AVX512BW implementation + * + * Note that in true AVX512F would suffice, but we don't want to add + * specific support for AVX512F because this would be the only function + * to benefit from that. + */ +void raid_gen1_avx512bw(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t**)vv; + uint8_t *p; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + + raid_avx_begin(); + + for (i = 0; i < size; i += 64) { + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); + + for (d = 1; d <= l - 1; d += 2) { + asm volatile ( + "vmovdqa64 %0,%%zmm1\n\t" + "vpternlogq $0x96,%1,%%zmm1,%%zmm0" + : + : "m" (v[d][i]), "m" (v[d+1][i]) + ); + } + + if (d == l) { + asm volatile ("vpxorq %0,%%zmm0,%%zmm0" : : "m" (v[l][i])); + } + + asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i])); + } + + raid_avx_end(); +} + +/* + * GENX AVX512BW implementation + */ +static __always_inline void raid_genX_avx512bw(int nd, size_t size, void **vv, int np) +{ + uint8_t **v = (uint8_t **)vv; + size_t i; + int d; + + if (nd == 1) { + for (d = 0; d < np; ++d) + memcpy(v[1 + d], v[0], size); + return; + } + + raid_avx_begin(); + + asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gfconst16.low4[0])); + + for (i = 0; i < size; i += 64) { + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); + asm volatile ("vmovdqa64 %zmm0,%zmm1"); + if (np >= 3) + asm volatile ("vmovdqa64 %zmm0,%zmm2"); + if (np >= 4) + asm volatile ("vmovdqa64 %zmm0,%zmm3"); + if (np >= 5) + asm volatile ("vmovdqa64 %zmm0,%zmm4"); + if (np >= 6) + asm volatile ("vmovdqa64 %zmm0,%zmm5"); + + for (d = 1; d < nd; ++d) { + asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (v[d][i])); + + asm volatile ("vpxorq %zmm10,%zmm0,%zmm0"); + + asm volatile ("vpsrlw $4,%zmm10,%zmm11"); + asm volatile ("vpandq %zmm31,%zmm10,%zmm10"); + asm volatile ("vpandq %zmm31,%zmm11,%zmm11"); + + asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][0][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][0][1][0])); + if (np >= 3) { + asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (gfgenpshufb[d][1][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (gfgenpshufb[d][1][1][0])); + } + if (np >= 4) { + asm volatile ("vbroadcasti32x4 %0,%%zmm16" : : "m" (gfgenpshufb[d][2][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm17" : : "m" (gfgenpshufb[d][2][1][0])); + } + if (np >= 5) { + asm volatile ("vbroadcasti32x4 %0,%%zmm18" : : "m" (gfgenpshufb[d][3][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm19" : : "m" (gfgenpshufb[d][3][1][0])); + } + if (np >= 6) { + asm volatile ("vbroadcasti32x4 %0,%%zmm20" : : "m" (gfgenpshufb[d][4][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm21" : : "m" (gfgenpshufb[d][4][1][0])); + } + + asm volatile ("vpshufb %zmm10,%zmm12,%zmm12"); + asm volatile ("vpshufb %zmm11,%zmm13,%zmm13"); + if (np >= 3) { + asm volatile ("vpshufb %zmm10,%zmm14,%zmm14"); + asm volatile ("vpshufb %zmm11,%zmm15,%zmm15"); + } + if (np >= 4) { + asm volatile ("vpshufb %zmm10,%zmm16,%zmm16"); + asm volatile ("vpshufb %zmm11,%zmm17,%zmm17"); + } + if (np >= 5) { + asm volatile ("vpshufb %zmm10,%zmm18,%zmm18"); + asm volatile ("vpshufb %zmm11,%zmm19,%zmm19"); + } + if (np >= 6) { + asm volatile ("vpshufb %zmm10,%zmm20,%zmm20"); + asm volatile ("vpshufb %zmm11,%zmm21,%zmm21"); + } + + asm volatile ("vpternlogq $0x96,%zmm12,%zmm13,%zmm1"); + if (np >= 3) + asm volatile ("vpternlogq $0x96,%zmm14,%zmm15,%zmm2"); + if (np >= 4) + asm volatile ("vpternlogq $0x96,%zmm16,%zmm17,%zmm3"); + if (np >= 5) + asm volatile ("vpternlogq $0x96,%zmm18,%zmm19,%zmm4"); + if (np >= 6) + asm volatile ("vpternlogq $0x96,%zmm20,%zmm21,%zmm5"); + } + + asm volatile ("vmovntdq %%zmm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovntdq %%zmm1,%0" : "=m" (v[nd + 1][i])); + if (np >= 3) + asm volatile ("vmovntdq %%zmm2,%0" : "=m" (v[nd + 2][i])); + if (np >= 4) + asm volatile ("vmovntdq %%zmm3,%0" : "=m" (v[nd + 3][i])); + if (np >= 5) + asm volatile ("vmovntdq %%zmm4,%0" : "=m" (v[nd + 4][i])); + if (np >= 6) + asm volatile ("vmovntdq %%zmm5,%0" : "=m" (v[nd + 5][i])); + } + + raid_avx_end(); +} + +/* + * GEN2 (RAID6 with powers of 2) AVX512BW implementation + */ +void raid_gen2_avx512bw(int nd, size_t size, void **vv) +{ + raid_genX_avx512bw(nd, size, vv, 2); +} + +/* + * GEN3 (triple parity with Cauchy matrix) AVX512BW implementation + */ +void raid_gen3_avx512bw(int nd, size_t size, void **vv) +{ + raid_genX_avx512bw(nd, size, vv, 3); +} + +/* + * GEN4 (quad parity with Cauchy matrix) AVX512BW implementation + */ +void raid_gen4_avx512bw(int nd, size_t size, void **vv) +{ + raid_genX_avx512bw(nd, size, vv, 4); +} + +/* + * GEN5 (penta parity with Cauchy matrix) AVX512BW implementation + */ +void raid_gen5_avx512bw(int nd, size_t size, void **vv) +{ + raid_genX_avx512bw(nd, size, vv, 5); +} + +/* + * GEN6 (hexa parity with Cauchy matrix) AVX512BW implementation + */ +void raid_gen6_avx512bw(int nd, size_t size, void **vv) +{ + raid_genX_avx512bw(nd, size, vv, 6); +} + +/* + * RAID recovering for one disk AVX512BW implementation + */ +void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p, *pa; + uint8_t G, V; + size_t i; + + (void)nr; + + if (ip[0] == 0) { + raid_rec1of1(id, nd, size, vv); + return; + } + + G = A(ip[0], id[0]); + V = inv(G); + + raid_delta_gen(1, id, ip, nd, size, vv); + + p = v[nd + ip[0]]; + pa = v[id[0]]; + + raid_avx_begin(); + + asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm4" : : "m" (gfmulpshufb[V][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm5" : : "m" (gfmulpshufb[V][1][0])); + + for (i = 0; i < size; i += 64) { + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i])); + asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (pa[i])); + asm volatile ("vpxord %zmm1,%zmm0,%zmm0"); + asm volatile ("vpsrlw $4,%zmm0,%zmm1"); + asm volatile ("vpandd %zmm7,%zmm0,%zmm0"); + asm volatile ("vpandd %zmm7,%zmm1,%zmm1"); + asm volatile ("vpshufb %zmm0,%zmm4,%zmm2"); + asm volatile ("vpshufb %zmm1,%zmm5,%zmm3"); + asm volatile ("vpxord %zmm3,%zmm2,%zmm2"); + asm volatile ("vmovdqa64 %%zmm2, %0" : "=m" (pa[i])); + } + + raid_avx_end(); +} + +/* + * RAID recovering for two disks AVX512BW implementation + */ +void raid_rec2_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + const int N = 2; + uint8_t *p[N], *pa[N]; + uint8_t G[N * N], V[N * N]; + size_t i; + int j, k; + + (void)nr; + + for (j = 0; j < N; ++j) + for (k = 0; k < N; ++k) + G[j * N + k] = A(ip[j], id[k]); + + raid_invert(G, V, N); + raid_delta_gen(N, id, ip, nd, size, vv); + + for (j = 0; j < N; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + raid_avx_begin(); + + asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfconst16.low4[0])); + + for (i = 0; i < size; i += 64) { + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[0][i])); + asm volatile ("vmovdqa64 %0,%%zmm2" : : "m" (pa[0][i])); + asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (p[1][i])); + asm volatile ("vmovdqa64 %0,%%zmm3" : : "m" (pa[1][i])); + asm volatile ("vpxord %zmm2,%zmm0,%zmm0"); + asm volatile ("vpxord %zmm3,%zmm1,%zmm1"); + + asm volatile ("vpxord %zmm6,%zmm6,%zmm6"); + + asm volatile ("vbroadcasti32x4 %0,%%zmm2" : : "m" (gfmulpshufb[V[0]][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (gfmulpshufb[V[0]][1][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm10" : : "m" (gfmulpshufb[V[1]][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm11" : : "m" (gfmulpshufb[V[1]][1][0])); + asm volatile ("vpsrlw $4,%zmm0,%zmm5"); + asm volatile ("vpsrlw $4,%zmm1,%zmm13"); + asm volatile ("vpandd %zmm7,%zmm0,%zmm4"); + asm volatile ("vpandd %zmm7,%zmm5,%zmm5"); + asm volatile ("vpandd %zmm7,%zmm1,%zmm12"); + asm volatile ("vpandd %zmm7,%zmm13,%zmm13"); + asm volatile ("vpshufb %zmm4,%zmm2,%zmm2"); + asm volatile ("vpshufb %zmm5,%zmm3,%zmm3"); + asm volatile ("vpshufb %zmm12,%zmm10,%zmm10"); + asm volatile ("vpshufb %zmm13,%zmm11,%zmm11"); + asm volatile ("vpxord %zmm2,%zmm6,%zmm6"); + asm volatile ("vpxord %zmm3,%zmm6,%zmm6"); + asm volatile ("vpxord %zmm10,%zmm6,%zmm6"); + asm volatile ("vpxord %zmm11,%zmm6,%zmm6"); + + asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (pa[0][i])); + + asm volatile ("vpxord %zmm6,%zmm6,%zmm6"); + + asm volatile ("vbroadcasti32x4 %0,%%zmm2" : : "m" (gfmulpshufb[V[2]][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (gfmulpshufb[V[2]][1][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm10" : : "m" (gfmulpshufb[V[3]][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm11" : : "m" (gfmulpshufb[V[3]][1][0])); + asm volatile ("vpsrlw $4,%zmm0,%zmm5"); + asm volatile ("vpsrlw $4,%zmm1,%zmm13"); + asm volatile ("vpandd %zmm7,%zmm0,%zmm4"); + asm volatile ("vpandd %zmm7,%zmm5,%zmm5"); + asm volatile ("vpandd %zmm7,%zmm1,%zmm12"); + asm volatile ("vpandd %zmm7,%zmm13,%zmm13"); + asm volatile ("vpshufb %zmm4,%zmm2,%zmm2"); + asm volatile ("vpshufb %zmm5,%zmm3,%zmm3"); + asm volatile ("vpshufb %zmm12,%zmm10,%zmm10"); + asm volatile ("vpshufb %zmm13,%zmm11,%zmm11"); + asm volatile ("vpxord %zmm2,%zmm6,%zmm6"); + asm volatile ("vpxord %zmm3,%zmm6,%zmm6"); + asm volatile ("vpxord %zmm10,%zmm6,%zmm6"); + asm volatile ("vpxord %zmm11,%zmm6,%zmm6"); + + asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (pa[1][i])); + } + + raid_avx_end(); +} + +/* + * RAID recovering AVX512BW implementation + */ +void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + int N = nr; + uint8_t *p[RAID_PARITY_MAX]; + uint8_t *pa[RAID_PARITY_MAX]; + uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t buffer[RAID_PARITY_MAX*64+64]; + uint8_t *pd = __align_ptr(buffer, 64); + size_t i; + int j, k; + + /* setup the coefficients matrix */ + for (j = 0; j < N; ++j) + for (k = 0; k < N; ++k) + G[j * N + k] = A(ip[j], id[k]); + + /* invert it to solve the system of linear equations */ + raid_invert(G, V, N); + + /* compute delta parity */ + raid_delta_gen(N, id, ip, nd, size, vv); + + for (j = 0; j < N; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + raid_avx_begin(); + + asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfconst16.low4[0])); + + for (i = 0; i < size; i += 64) { + /* delta */ + for (j = 0; j < N; ++j) { + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[j][i])); + asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (pa[j][i])); + asm volatile ("vpxorq %zmm1,%zmm0,%zmm0"); + asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pd[j*64])); + } + /* reconstruct */ + for (j = 0; j < N; ++j) { + asm volatile ("vpxorq %zmm0,%zmm0,%zmm0"); + asm volatile ("vpxorq %zmm1,%zmm1,%zmm1"); + for (k = 0; k < N; ++k) { + uint8_t m = V[j * N + k]; + asm volatile ("vbroadcasti32x4 %0,%%zmm2" : : "m" (gfmulpshufb[m][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (gfmulpshufb[m][1][0])); + asm volatile ("vmovdqa64 %0,%%zmm4" : : "m" (pd[k*64])); + asm volatile ("vpsrlw $4,%zmm4,%zmm5"); + asm volatile ("vpandq %zmm7,%zmm4,%zmm4"); + asm volatile ("vpandq %zmm7,%zmm5,%zmm5"); + asm volatile ("vpshufb %zmm4,%zmm2,%zmm2"); + asm volatile ("vpshufb %zmm5,%zmm3,%zmm3"); + asm volatile ("vpxorq %zmm2,%zmm0,%zmm0"); + asm volatile ("vpxorq %zmm3,%zmm1,%zmm1"); + } + asm volatile ("vpxorq %zmm1,%zmm0,%zmm0"); + asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[j][i])); + } + } + + raid_avx_end(); +} + +void raid_register_avx512(void) +{ + if (raid_cpu_has_avx512bw()) { + if (!raid_cpu_has_slow_avx512()) { + raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "avx512", raid_gen1_avx512bw); + raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "avx512", raid_gen2_avx512bw); + raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "avx512", raid_gen3_avx512bw); + raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "avx512", raid_gen4_avx512bw); + raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "avx512", raid_gen5_avx512bw); + raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "avx512", raid_gen6_avx512bw); + + raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "avx512", raid_rec1_avx512bw); + raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "avx512", raid_rec2_avx512bw); + raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "avx512", raid_recX_avx512bw); + raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "avx512", raid_recX_avx512bw); + raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "avx512", raid_recX_avx512bw); + raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "avx512", raid_recX_avx512bw); + } + } +} +#endif diff --git a/raid/gfni.c b/raid/gfni.c index 187ca8de..bc07ae13 100644 --- a/raid/gfni.c +++ b/raid/gfni.c @@ -8,450 +8,48 @@ #ifdef CONFIG_X86_64 /* - * GEN2 (RAID6 with powers of 2) AVX2 GFNI implementation + * GENX AVX2 GFNI implementation */ -void raid_gen2_avx2gfni(int nd, size_t size, void **vv) +static __always_inline void raid_genX_avx2gfni(int nd, size_t size, void **vv, int np) { uint8_t **v = (uint8_t **)vv; - uint8_t *p = v[nd]; - uint8_t *q = v[nd + 1]; - int d, l = nd - 1; size_t i; + int d; + + if (nd == 1) { + for (d = 0; d < np; ++d) + memcpy(v[1 + d], v[0], size); + return; + } raid_avx_begin(); for (i = 0; i < size; i += 64) { + /* first disk with all coefficients at 1 */ asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[0][i])); asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[0][i + 32])); + asm volatile ("vmovdqa %ymm0,%ymm2"); asm volatile ("vmovdqa %ymm1,%ymm3"); - - for (d = 1; d <= l; ++d) { - asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i])); - asm volatile ("vmovdqa %0,%%ymm5" : : "m" (v[d][i + 32])); - - asm volatile ("vpbroadcastb %0,%%ymm6" : : "m" (gfcauchy[1][d])); - - asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); - - asm volatile ("vgf2p8mulb %ymm4,%ymm6,%ymm8"); - asm volatile ("vgf2p8mulb %ymm5,%ymm6,%ymm9"); - asm volatile ("vpxor %ymm8,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm9,%ymm3,%ymm3"); + if (np >= 3) { + asm volatile ("vmovdqa %ymm0,%ymm4"); + asm volatile ("vmovdqa %ymm1,%ymm5"); + } + if (np >= 4) { + asm volatile ("vmovdqa %ymm0,%ymm6"); + asm volatile ("vmovdqa %ymm1,%ymm7"); + } + if (np >= 5) { + asm volatile ("vmovdqa %ymm0,%ymm8"); + asm volatile ("vmovdqa %ymm1,%ymm9"); + } + if (np >= 6) { + asm volatile ("vmovdqa %ymm0,%ymm10"); + asm volatile ("vmovdqa %ymm1,%ymm11"); } - asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); - asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); - asm volatile ("vmovntdq %%ymm2,%0" : "=m" (q[i])); - asm volatile ("vmovntdq %%ymm3,%0" : "=m" (q[i + 32])); - } - - raid_avx_end(); -} - -/* - * GEN2 (RAID6 with powers of 2) GFNI implementation - */ -void raid_gen2_avx512gfni(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p = v[nd]; - uint8_t *q = v[nd + 1]; - int d, l = nd - 1; - size_t i; - - raid_avx_begin(); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); - asm volatile ("vmovdqa64 %zmm0,%zmm1"); - - for (d = 1; d <= l; ++d) { - asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[d][i])); - - asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfcauchy[1][d])); - - asm volatile ("vgf2p8mulb %zmm6,%zmm7,%zmm12"); - - asm volatile ("vpxorq %zmm6,%zmm0,%zmm0"); - asm volatile ("vpxorq %zmm12,%zmm1,%zmm1"); - } - - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i])); - asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i])); - } - - raid_avx_end(); -} - -/* - * GEN3 (triple parity with Cauchy matrix) AVX2 GFNI implementation - */ -void raid_gen3_avx2gfni(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p = v[nd]; - uint8_t *q = v[nd + 1]; - uint8_t *r = v[nd + 2]; - int d, l = nd - 1; - size_t i; - - raid_avx_begin(); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[0][i])); - asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[0][i + 32])); - asm volatile ("vmovdqa %ymm0,%ymm2"); - asm volatile ("vmovdqa %ymm1,%ymm3"); - asm volatile ("vmovdqa %ymm0,%ymm4"); - asm volatile ("vmovdqa %ymm1,%ymm5"); - - for (d = 1; d <= l; ++d) { - asm volatile ("vmovdqa %0,%%ymm6" : : "m" (v[d][i])); - asm volatile ("vmovdqa %0,%%ymm7" : : "m" (v[d][i + 32])); - - asm volatile ("vpbroadcastb %0,%%ymm8" : : "m" (gfcauchy[1][d])); - asm volatile ("vpbroadcastb %0,%%ymm9" : : "m" (gfcauchy[2][d])); - - asm volatile ("vpxor %ymm6,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm7,%ymm1,%ymm1"); - - asm volatile ("vgf2p8mulb %ymm6,%ymm8,%ymm10"); - asm volatile ("vgf2p8mulb %ymm7,%ymm8,%ymm11"); - asm volatile ("vpxor %ymm10,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm11,%ymm3,%ymm3"); - - asm volatile ("vgf2p8mulb %ymm6,%ymm9,%ymm10"); - asm volatile ("vgf2p8mulb %ymm7,%ymm9,%ymm11"); - asm volatile ("vpxor %ymm10,%ymm4,%ymm4"); - asm volatile ("vpxor %ymm11,%ymm5,%ymm5"); - } - - asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); - asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); - asm volatile ("vmovntdq %%ymm2,%0" : "=m" (q[i])); - asm volatile ("vmovntdq %%ymm3,%0" : "=m" (q[i + 32])); - asm volatile ("vmovntdq %%ymm4,%0" : "=m" (r[i])); - asm volatile ("vmovntdq %%ymm5,%0" : "=m" (r[i + 32])); - } - - raid_avx_end(); -} - -/* - * GEN3 (triple parity with Cauchy matrix) GFNI implementation - */ -void raid_gen3_avx512gfni(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p = v[nd]; - uint8_t *q = v[nd + 1]; - uint8_t *r = v[nd + 2]; - int d, l = nd - 1; - size_t i; - - raid_avx_begin(); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); - asm volatile ("vmovdqa64 %zmm0,%zmm1"); - asm volatile ("vmovdqa64 %zmm0,%zmm2"); - - for (d = 1; d <= l; ++d) { - asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[d][i])); - - asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfcauchy[1][d])); - asm volatile ("vpbroadcastb %0,%%zmm8" : : "m" (gfcauchy[2][d])); - - asm volatile ("vgf2p8mulb %zmm6,%zmm7,%zmm12"); - asm volatile ("vgf2p8mulb %zmm6,%zmm8,%zmm13"); - - asm volatile ("vpxorq %zmm6,%zmm0,%zmm0"); - asm volatile ("vpxorq %zmm12,%zmm1,%zmm1"); - asm volatile ("vpxorq %zmm13,%zmm2,%zmm2"); - } - - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i])); - asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i])); - asm volatile ("vmovntdq %%zmm2,%0" : "=m" (r[i])); - } - - raid_avx_end(); -} - -/* - * GEN4 (quad parity with Cauchy matrix) AVX2 GFNI implementation - */ -void raid_gen4_avx2gfni(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p = v[nd]; - uint8_t *q = v[nd + 1]; - uint8_t *r = v[nd + 2]; - uint8_t *s = v[nd + 3]; - int d, l = nd - 1; - size_t i; - - raid_avx_begin(); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[0][i])); - asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[0][i + 32])); - asm volatile ("vmovdqa %ymm0,%ymm2"); - asm volatile ("vmovdqa %ymm1,%ymm3"); - asm volatile ("vmovdqa %ymm0,%ymm4"); - asm volatile ("vmovdqa %ymm1,%ymm5"); - asm volatile ("vmovdqa %ymm0,%ymm6"); - asm volatile ("vmovdqa %ymm1,%ymm7"); - - for (d = 1; d <= l; ++d) { - asm volatile ("vmovdqa %0,%%ymm8" : : "m" (v[d][i])); - asm volatile ("vmovdqa %0,%%ymm9" : : "m" (v[d][i + 32])); - - asm volatile ("vpbroadcastb %0,%%ymm10" : : "m" (gfcauchy[1][d])); - asm volatile ("vpbroadcastb %0,%%ymm11" : : "m" (gfcauchy[2][d])); - asm volatile ("vpbroadcastb %0,%%ymm12" : : "m" (gfcauchy[3][d])); - - asm volatile ("vpxor %ymm8,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm9,%ymm1,%ymm1"); - - asm volatile ("vgf2p8mulb %ymm8,%ymm10,%ymm13"); - asm volatile ("vgf2p8mulb %ymm9,%ymm10,%ymm14"); - asm volatile ("vpxor %ymm13,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm14,%ymm3,%ymm3"); - - asm volatile ("vgf2p8mulb %ymm8,%ymm11,%ymm13"); - asm volatile ("vgf2p8mulb %ymm9,%ymm11,%ymm14"); - asm volatile ("vpxor %ymm13,%ymm4,%ymm4"); - asm volatile ("vpxor %ymm14,%ymm5,%ymm5"); - - asm volatile ("vgf2p8mulb %ymm8,%ymm12,%ymm13"); - asm volatile ("vgf2p8mulb %ymm9,%ymm12,%ymm14"); - asm volatile ("vpxor %ymm13,%ymm6,%ymm6"); - asm volatile ("vpxor %ymm14,%ymm7,%ymm7"); - } - - asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); - asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); - asm volatile ("vmovntdq %%ymm2,%0" : "=m" (q[i])); - asm volatile ("vmovntdq %%ymm3,%0" : "=m" (q[i + 32])); - asm volatile ("vmovntdq %%ymm4,%0" : "=m" (r[i])); - asm volatile ("vmovntdq %%ymm5,%0" : "=m" (r[i + 32])); - asm volatile ("vmovntdq %%ymm6,%0" : "=m" (s[i])); - asm volatile ("vmovntdq %%ymm7,%0" : "=m" (s[i + 32])); - } - - raid_avx_end(); -} - -/* - * GEN4 (quad parity with Cauchy matrix) GFNI implementation - */ -void raid_gen4_avx512gfni(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p = v[nd]; - uint8_t *q = v[nd + 1]; - uint8_t *r = v[nd + 2]; - uint8_t *s = v[nd + 3]; - int d, l = nd - 1; - size_t i; - - raid_avx_begin(); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); - asm volatile ("vmovdqa64 %zmm0,%zmm1"); - asm volatile ("vmovdqa64 %zmm0,%zmm2"); - asm volatile ("vmovdqa64 %zmm0,%zmm3"); - - for (d = 1; d <= l; ++d) { - asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[d][i])); - - asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfcauchy[1][d])); - asm volatile ("vpbroadcastb %0,%%zmm8" : : "m" (gfcauchy[2][d])); - asm volatile ("vpbroadcastb %0,%%zmm9" : : "m" (gfcauchy[3][d])); - - asm volatile ("vgf2p8mulb %zmm6,%zmm7,%zmm12"); - asm volatile ("vgf2p8mulb %zmm6,%zmm8,%zmm13"); - asm volatile ("vgf2p8mulb %zmm6,%zmm9,%zmm14"); - - asm volatile ("vpxorq %zmm6,%zmm0,%zmm0"); - asm volatile ("vpxorq %zmm12,%zmm1,%zmm1"); - asm volatile ("vpxorq %zmm13,%zmm2,%zmm2"); - asm volatile ("vpxorq %zmm14,%zmm3,%zmm3"); - } - - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i])); - asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i])); - asm volatile ("vmovntdq %%zmm2,%0" : "=m" (r[i])); - asm volatile ("vmovntdq %%zmm3,%0" : "=m" (s[i])); - } - - raid_avx_end(); -} - -/* - * GEN5 (penta parity with Cauchy matrix) AVX2 GFNI implementation - */ -void raid_gen5_avx2gfni(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p = v[nd]; - uint8_t *q = v[nd + 1]; - uint8_t *r = v[nd + 2]; - uint8_t *s = v[nd + 3]; - uint8_t *t = v[nd + 4]; - int d, l = nd - 1; - size_t i; - - raid_avx_begin(); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[0][i])); - asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[0][i + 32])); - asm volatile ("vmovdqa %ymm0,%ymm2"); - asm volatile ("vmovdqa %ymm1,%ymm3"); - asm volatile ("vmovdqa %ymm0,%ymm4"); - asm volatile ("vmovdqa %ymm1,%ymm5"); - asm volatile ("vmovdqa %ymm0,%ymm6"); - asm volatile ("vmovdqa %ymm1,%ymm7"); - asm volatile ("vmovdqa %ymm0,%ymm8"); - asm volatile ("vmovdqa %ymm1,%ymm9"); - - for (d = 1; d <= l; ++d) { - asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[d][i])); - asm volatile ("vmovdqa %0,%%ymm11" : : "m" (v[d][i + 32])); - - asm volatile ("vpxor %ymm10,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm11,%ymm1,%ymm1"); - - asm volatile ("vpbroadcastb %0,%%ymm12" : : "m" (gfcauchy[1][d])); - asm volatile ("vgf2p8mulb %ymm10,%ymm12,%ymm13"); - asm volatile ("vgf2p8mulb %ymm11,%ymm12,%ymm14"); - asm volatile ("vpxor %ymm13,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm14,%ymm3,%ymm3"); - - asm volatile ("vpbroadcastb %0,%%ymm12" : : "m" (gfcauchy[2][d])); - asm volatile ("vgf2p8mulb %ymm10,%ymm12,%ymm13"); - asm volatile ("vgf2p8mulb %ymm11,%ymm12,%ymm14"); - asm volatile ("vpxor %ymm13,%ymm4,%ymm4"); - asm volatile ("vpxor %ymm14,%ymm5,%ymm5"); - - asm volatile ("vpbroadcastb %0,%%ymm12" : : "m" (gfcauchy[3][d])); - asm volatile ("vgf2p8mulb %ymm10,%ymm12,%ymm13"); - asm volatile ("vgf2p8mulb %ymm11,%ymm12,%ymm14"); - asm volatile ("vpxor %ymm13,%ymm6,%ymm6"); - asm volatile ("vpxor %ymm14,%ymm7,%ymm7"); - - asm volatile ("vpbroadcastb %0,%%ymm12" : : "m" (gfcauchy[4][d])); - asm volatile ("vgf2p8mulb %ymm10,%ymm12,%ymm13"); - asm volatile ("vgf2p8mulb %ymm11,%ymm12,%ymm14"); - asm volatile ("vpxor %ymm13,%ymm8,%ymm8"); - asm volatile ("vpxor %ymm14,%ymm9,%ymm9"); - } - - asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); - asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); - asm volatile ("vmovntdq %%ymm2,%0" : "=m" (q[i])); - asm volatile ("vmovntdq %%ymm3,%0" : "=m" (q[i + 32])); - asm volatile ("vmovntdq %%ymm4,%0" : "=m" (r[i])); - asm volatile ("vmovntdq %%ymm5,%0" : "=m" (r[i + 32])); - asm volatile ("vmovntdq %%ymm6,%0" : "=m" (s[i])); - asm volatile ("vmovntdq %%ymm7,%0" : "=m" (s[i + 32])); - asm volatile ("vmovntdq %%ymm8,%0" : "=m" (t[i])); - asm volatile ("vmovntdq %%ymm9,%0" : "=m" (t[i + 32])); - } - - raid_avx_end(); -} - -/* - * GEN5 (penta parity with Cauchy matrix) GFNI implementation - */ -void raid_gen5_avx512gfni(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p = v[nd]; - uint8_t *q = v[nd + 1]; - uint8_t *r = v[nd + 2]; - uint8_t *s = v[nd + 3]; - uint8_t *t = v[nd + 4]; - int d, l = nd - 1; - size_t i; - - raid_avx_begin(); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); - asm volatile ("vmovdqa64 %zmm0,%zmm1"); - asm volatile ("vmovdqa64 %zmm0,%zmm2"); - asm volatile ("vmovdqa64 %zmm0,%zmm3"); - asm volatile ("vmovdqa64 %zmm0,%zmm4"); - - for (d = 1; d <= l; ++d) { - asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[d][i])); - - asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfcauchy[1][d])); - asm volatile ("vpbroadcastb %0,%%zmm8" : : "m" (gfcauchy[2][d])); - asm volatile ("vpbroadcastb %0,%%zmm9" : : "m" (gfcauchy[3][d])); - asm volatile ("vpbroadcastb %0,%%zmm10" : : "m" (gfcauchy[4][d])); - - asm volatile ("vgf2p8mulb %zmm6,%zmm7,%zmm12"); - asm volatile ("vgf2p8mulb %zmm6,%zmm8,%zmm13"); - asm volatile ("vgf2p8mulb %zmm6,%zmm9,%zmm14"); - asm volatile ("vgf2p8mulb %zmm6,%zmm10,%zmm15"); - - asm volatile ("vpxorq %zmm6,%zmm0,%zmm0"); - asm volatile ("vpxorq %zmm12,%zmm1,%zmm1"); - asm volatile ("vpxorq %zmm13,%zmm2,%zmm2"); - asm volatile ("vpxorq %zmm14,%zmm3,%zmm3"); - asm volatile ("vpxorq %zmm15,%zmm4,%zmm4"); - } - - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i])); - asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i])); - asm volatile ("vmovntdq %%zmm2,%0" : "=m" (r[i])); - asm volatile ("vmovntdq %%zmm3,%0" : "=m" (s[i])); - asm volatile ("vmovntdq %%zmm4,%0" : "=m" (t[i])); - } - - raid_avx_end(); -} - -/* - * GEN6 (hexa parity with Cauchy matrix) AVX2 GFNI implementation - */ -void raid_gen6_avx2gfni(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p = v[nd]; - uint8_t *q = v[nd + 1]; - uint8_t *r = v[nd + 2]; - uint8_t *s = v[nd + 3]; - uint8_t *t = v[nd + 4]; - uint8_t *u = v[nd + 5]; - int d, l = nd - 1; - size_t i; - - raid_avx_begin(); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[0][i])); - asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[0][i + 32])); - asm volatile ("vmovdqa %ymm0,%ymm2"); - asm volatile ("vmovdqa %ymm1,%ymm3"); - asm volatile ("vmovdqa %ymm0,%ymm4"); - asm volatile ("vmovdqa %ymm1,%ymm5"); - asm volatile ("vmovdqa %ymm0,%ymm6"); - asm volatile ("vmovdqa %ymm1,%ymm7"); - asm volatile ("vmovdqa %ymm0,%ymm8"); - asm volatile ("vmovdqa %ymm1,%ymm9"); - asm volatile ("vmovdqa %ymm0,%ymm10"); - asm volatile ("vmovdqa %ymm1,%ymm11"); - - for (d = 1; d <= l; ++d) { + /* all other disks */ + for (d = 1; d < nd; ++d) { asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i])); asm volatile ("vmovdqa %0,%%ymm13" : : "m" (v[d][i + 32])); @@ -463,106 +61,221 @@ void raid_gen6_avx2gfni(int nd, size_t size, void **vv) asm volatile ("vpxor %ymm15,%ymm2,%ymm2"); asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm15"); asm volatile ("vpxor %ymm15,%ymm3,%ymm3"); - - asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[2][d])); - asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm15"); - asm volatile ("vpxor %ymm15,%ymm4,%ymm4"); - asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm15"); - asm volatile ("vpxor %ymm15,%ymm5,%ymm5"); - - asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[3][d])); - asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm15"); - asm volatile ("vpxor %ymm15,%ymm6,%ymm6"); - asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm15"); - asm volatile ("vpxor %ymm15,%ymm7,%ymm7"); - - asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[4][d])); - asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm15"); - asm volatile ("vpxor %ymm15,%ymm8,%ymm8"); - asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm15"); - asm volatile ("vpxor %ymm15,%ymm9,%ymm9"); - - asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[5][d])); - asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm15"); - asm volatile ("vpxor %ymm15,%ymm10,%ymm10"); - asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm15"); - asm volatile ("vpxor %ymm15,%ymm11,%ymm11"); + if (np >= 3) { + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[2][d])); + asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm4,%ymm4"); + asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm5,%ymm5"); + } + if (np >= 4) { + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[3][d])); + asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm6,%ymm6"); + asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm7,%ymm7"); + } + if (np >= 5) { + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[4][d])); + asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm8,%ymm8"); + asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm9,%ymm9"); + } + if (np >= 6) { + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[5][d])); + asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm10,%ymm10"); + asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm11,%ymm11"); + } } - asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); - asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); - asm volatile ("vmovntdq %%ymm2,%0" : "=m" (q[i])); - asm volatile ("vmovntdq %%ymm3,%0" : "=m" (q[i + 32])); - asm volatile ("vmovntdq %%ymm4,%0" : "=m" (r[i])); - asm volatile ("vmovntdq %%ymm5,%0" : "=m" (r[i + 32])); - asm volatile ("vmovntdq %%ymm6,%0" : "=m" (s[i])); - asm volatile ("vmovntdq %%ymm7,%0" : "=m" (s[i + 32])); - asm volatile ("vmovntdq %%ymm8,%0" : "=m" (t[i])); - asm volatile ("vmovntdq %%ymm9,%0" : "=m" (t[i + 32])); - asm volatile ("vmovntdq %%ymm10,%0" : "=m" (u[i])); - asm volatile ("vmovntdq %%ymm11,%0" : "=m" (u[i + 32])); + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovntdq %%ymm1,%0" : "=m" (v[nd][i + 32])); + asm volatile ("vmovntdq %%ymm2,%0" : "=m" (v[nd + 1][i])); + asm volatile ("vmovntdq %%ymm3,%0" : "=m" (v[nd + 1][i + 32])); + if (np >= 3) { + asm volatile ("vmovntdq %%ymm4,%0" : "=m" (v[nd + 2][i])); + asm volatile ("vmovntdq %%ymm5,%0" : "=m" (v[nd + 2][i + 32])); + } + if (np >= 4) { + asm volatile ("vmovntdq %%ymm6,%0" : "=m" (v[nd + 3][i])); + asm volatile ("vmovntdq %%ymm7,%0" : "=m" (v[nd + 3][i + 32])); + } + if (np >= 5) { + asm volatile ("vmovntdq %%ymm8,%0" : "=m" (v[nd + 4][i])); + asm volatile ("vmovntdq %%ymm9,%0" : "=m" (v[nd + 4][i + 32])); + } + if (np >= 6) { + asm volatile ("vmovntdq %%ymm10,%0" : "=m" (v[nd + 5][i])); + asm volatile ("vmovntdq %%ymm11,%0" : "=m" (v[nd + 5][i + 32])); + } } raid_avx_end(); } +/* + * GENX AVX512 GFNI implementation + */ +static __always_inline void raid_genX_avx512gfni(int nd, size_t size, void **vv, int np) +{ + uint8_t **v = (uint8_t **)vv; + size_t i; + int d; + + if (nd == 1) { + for (d = 0; d < np; ++d) + memcpy(v[1 + d], v[0], size); + return; + } + + raid_avx_begin(); + + for (i = 0; i < size; i += 64) { + /* first disk with all coefficients at 1 */ + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); + + asm volatile ("vmovdqa64 %zmm0,%zmm1"); + if (np >= 3) + asm volatile ("vmovdqa64 %zmm0,%zmm2"); + if (np >= 4) + asm volatile ("vmovdqa64 %zmm0,%zmm3"); + if (np >= 5) + asm volatile ("vmovdqa64 %zmm0,%zmm4"); + if (np >= 6) + asm volatile ("vmovdqa64 %zmm0,%zmm5"); + + /* all other disks */ + for (d = 1; d < nd; ++d) { + asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[d][i])); + + asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfcauchy[1][d])); + if (np >= 3) + asm volatile ("vpbroadcastb %0,%%zmm8" : : "m" (gfcauchy[2][d])); + if (np >= 4) + asm volatile ("vpbroadcastb %0,%%zmm9" : : "m" (gfcauchy[3][d])); + if (np >= 5) + asm volatile ("vpbroadcastb %0,%%zmm10" : : "m" (gfcauchy[4][d])); + if (np >= 6) + asm volatile ("vpbroadcastb %0,%%zmm11" : : "m" (gfcauchy[5][d])); + + asm volatile ("vgf2p8mulb %zmm6,%zmm7,%zmm12"); + if (np >= 3) + asm volatile ("vgf2p8mulb %zmm6,%zmm8,%zmm13"); + if (np >= 4) + asm volatile ("vgf2p8mulb %zmm6,%zmm9,%zmm14"); + if (np >= 5) + asm volatile ("vgf2p8mulb %zmm6,%zmm10,%zmm15"); + if (np >= 6) + asm volatile ("vgf2p8mulb %zmm6,%zmm11,%zmm16"); + + asm volatile ("vpxorq %zmm6,%zmm0,%zmm0"); + asm volatile ("vpxorq %zmm12,%zmm1,%zmm1"); + if (np >= 3) + asm volatile ("vpxorq %zmm13,%zmm2,%zmm2"); + if (np >= 4) + asm volatile ("vpxorq %zmm14,%zmm3,%zmm3"); + if (np >= 5) + asm volatile ("vpxorq %zmm15,%zmm4,%zmm4"); + if (np >= 6) + asm volatile ("vpxorq %zmm16,%zmm5,%zmm5"); + } + + asm volatile ("vmovntdq %%zmm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovntdq %%zmm1,%0" : "=m" (v[nd + 1][i])); + if (np >= 3) + asm volatile ("vmovntdq %%zmm2,%0" : "=m" (v[nd + 2][i])); + if (np >= 4) + asm volatile ("vmovntdq %%zmm3,%0" : "=m" (v[nd + 3][i])); + if (np >= 5) + asm volatile ("vmovntdq %%zmm4,%0" : "=m" (v[nd + 4][i])); + if (np >= 6) + asm volatile ("vmovntdq %%zmm5,%0" : "=m" (v[nd + 5][i])); + } + + raid_avx_end(); +} + +/* + * GEN2 (RAID6 with powers of 2) AVX2 GFNI implementation + */ +void raid_gen2_avx2gfni(int nd, size_t size, void **vv) +{ + raid_genX_avx2gfni(nd, size, vv, 2); +} + +/* + * GEN2 (RAID6 with powers of 2) GFNI implementation + */ +void raid_gen2_avx512gfni(int nd, size_t size, void **vv) +{ + raid_genX_avx512gfni(nd, size, vv, 2); +} + +/* + * GEN3 (triple parity with Cauchy matrix) AVX2 GFNI implementation + */ +void raid_gen3_avx2gfni(int nd, size_t size, void **vv) +{ + raid_genX_avx2gfni(nd, size, vv, 3); +} + +/* + * GEN3 (triple parity with Cauchy matrix) GFNI implementation + */ +void raid_gen3_avx512gfni(int nd, size_t size, void **vv) +{ + raid_genX_avx512gfni(nd, size, vv, 3); +} + +/* + * GEN4 (quad parity with Cauchy matrix) AVX2 GFNI implementation + */ +void raid_gen4_avx2gfni(int nd, size_t size, void **vv) +{ + raid_genX_avx2gfni(nd, size, vv, 4); +} + +/* + * GEN4 (quad parity with Cauchy matrix) GFNI implementation + */ +void raid_gen4_avx512gfni(int nd, size_t size, void **vv) +{ + raid_genX_avx512gfni(nd, size, vv, 4); +} + +/* + * GEN5 (penta parity with Cauchy matrix) AVX2 GFNI implementation + */ +void raid_gen5_avx2gfni(int nd, size_t size, void **vv) +{ + raid_genX_avx2gfni(nd, size, vv, 5); +} + +/* + * GEN5 (penta parity with Cauchy matrix) GFNI implementation + */ +void raid_gen5_avx512gfni(int nd, size_t size, void **vv) +{ + raid_genX_avx512gfni(nd, size, vv, 5); +} + +/* + * GEN6 (hexa parity with Cauchy matrix) AVX2 GFNI implementation + */ +void raid_gen6_avx2gfni(int nd, size_t size, void **vv) +{ + raid_genX_avx2gfni(nd, size, vv, 6); +} + /* * GEN6 (hexa parity with Cauchy matrix) GFNI implementation */ void raid_gen6_avx512gfni(int nd, size_t size, void **vv) { - uint8_t **v = (uint8_t **)vv; - uint8_t *p = v[nd]; - uint8_t *q = v[nd + 1]; - uint8_t *r = v[nd + 2]; - uint8_t *s = v[nd + 3]; - uint8_t *t = v[nd + 4]; - uint8_t *u = v[nd + 5]; - int d, l = nd - 1; - size_t i; - - raid_avx_begin(); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); - asm volatile ("vmovdqa64 %zmm0,%zmm1"); - asm volatile ("vmovdqa64 %zmm0,%zmm2"); - asm volatile ("vmovdqa64 %zmm0,%zmm3"); - asm volatile ("vmovdqa64 %zmm0,%zmm4"); - asm volatile ("vmovdqa64 %zmm0,%zmm5"); - - for (d = 1; d <= l; ++d) { - asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[d][i])); - - asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfcauchy[1][d])); - asm volatile ("vpbroadcastb %0,%%zmm8" : : "m" (gfcauchy[2][d])); - asm volatile ("vpbroadcastb %0,%%zmm9" : : "m" (gfcauchy[3][d])); - asm volatile ("vpbroadcastb %0,%%zmm10" : : "m" (gfcauchy[4][d])); - asm volatile ("vpbroadcastb %0,%%zmm11" : : "m" (gfcauchy[5][d])); - - asm volatile ("vgf2p8mulb %zmm6,%zmm7,%zmm12"); - asm volatile ("vgf2p8mulb %zmm6,%zmm8,%zmm13"); - asm volatile ("vgf2p8mulb %zmm6,%zmm9,%zmm14"); - asm volatile ("vgf2p8mulb %zmm6,%zmm10,%zmm15"); - asm volatile ("vgf2p8mulb %zmm6,%zmm11,%zmm16"); - - asm volatile ("vpxorq %zmm6,%zmm0,%zmm0"); - asm volatile ("vpxorq %zmm12,%zmm1,%zmm1"); - asm volatile ("vpxorq %zmm13,%zmm2,%zmm2"); - asm volatile ("vpxorq %zmm14,%zmm3,%zmm3"); - asm volatile ("vpxorq %zmm15,%zmm4,%zmm4"); - asm volatile ("vpxorq %zmm16,%zmm5,%zmm5"); - } - - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i])); - asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i])); - asm volatile ("vmovntdq %%zmm2,%0" : "=m" (r[i])); - asm volatile ("vmovntdq %%zmm3,%0" : "=m" (s[i])); - asm volatile ("vmovntdq %%zmm4,%0" : "=m" (t[i])); - asm volatile ("vmovntdq %%zmm5,%0" : "=m" (u[i])); - } - - raid_avx_end(); + raid_genX_avx512gfni(nd, size, vv, 6); } /* diff --git a/raid/int.c b/raid/int.c index d976b6db..f018745e 100644 --- a/raid/int.c +++ b/raid/int.c @@ -142,6 +142,108 @@ void raid_gen2_int64(int nd, size_t size, void **vv) } } +/* + * GENz (triple parity with powers of 2^-1) 32bit C implementation + */ +void raid_genz_int32(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t**)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + int d, l; + size_t i; + + uint32_t d0, r0, q0, p0; + uint32_t d1, r1, q1, p1; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + + for (i = 0; i < size; i += 8) { + r0 = q0 = p0 = v_32(v[l][i]); + r1 = q1 = p1 = v_32(v[l][i + 4]); + for (d = l - 1; d >= 0; --d) { + d0 = v_32(v[d][i]); + d1 = v_32(v[d][i + 4]); + + p0 ^= d0; + p1 ^= d1; + + q0 = x2_32(q0); + q1 = x2_32(q1); + + q0 ^= d0; + q1 ^= d1; + + r0 = d2_32(r0); + r1 = d2_32(r1); + + r0 ^= d0; + r1 ^= d1; + } + v_32(p[i]) = p0; + v_32(p[i + 4]) = p1; + v_32(q[i]) = q0; + v_32(q[i + 4]) = q1; + v_32(r[i]) = r0; + v_32(r[i + 4]) = r1; + } +} + +/* + * GENz (triple parity with powers of 2^-1) 64bit C implementation + */ +void raid_genz_int64(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t**)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + int d, l; + size_t i; + + uint64_t d0, r0, q0, p0; + uint64_t d1, r1, q1, p1; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + + for (i = 0; i < size; i += 16) { + r0 = q0 = p0 = v_64(v[l][i]); + r1 = q1 = p1 = v_64(v[l][i + 8]); + for (d = l - 1; d >= 0; --d) { + d0 = v_64(v[d][i]); + d1 = v_64(v[d][i + 8]); + + p0 ^= d0; + p1 ^= d1; + + q0 = x2_64(q0); + q1 = x2_64(q1); + + q0 ^= d0; + q1 ^= d1; + + r0 = d2_64(r0); + r1 = d2_64(r1); + + r0 ^= d0; + r1 ^= d1; + } + v_64(p[i]) = p0; + v_64(p[i + 8]) = p1; + v_64(q[i]) = q0; + v_64(q[i + 8]) = q1; + v_64(r[i]) = r0; + v_64(r[i + 8]) = r1; + } +} + /* * GEN3 (triple parity with Cauchy matrix) 8bit C implementation * diff --git a/raid/internal.h b/raid/internal.h index f383c470..f8cadb66 100644 --- a/raid/internal.h +++ b/raid/internal.h @@ -258,7 +258,10 @@ void raid_register_int(void); /** * Register all the functions based on x86 intructions. */ -void raid_register_x86(void); +void raid_register_sse2(void); +void raid_register_ssse3(void); +void raid_register_avx2(void); +void raid_register_avx512(void); /** * Register all the functions based on GFNI intructions. @@ -281,6 +284,16 @@ const char *raid_rec_tag(int na); */ int raid_selftest(void); +#if defined(CONFIG_X86) || defined(CONFIG_NEON) +struct gfconst16 { + uint8_t poly[16]; + uint8_t low4[16]; + uint8_t half[16]; + uint8_t low7[16]; +}; +extern const struct gfconst16 gfconst16; +#endif + /* * Tables. */ diff --git a/raid/intz.c b/raid/intz.c deleted file mode 100644 index e3c222d4..00000000 --- a/raid/intz.c +++ /dev/null @@ -1,108 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -// Copyright (C) 2013 Andrea Mazzoleni - -#include "internal.h" -#include "gf.h" - -/* - * GENz (triple parity with powers of 2^-1) 32bit C implementation - */ -void raid_genz_int32(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t**)vv; - uint8_t *p; - uint8_t *q; - uint8_t *r; - int d, l; - size_t i; - - uint32_t d0, r0, q0, p0; - uint32_t d1, r1, q1, p1; - - l = nd - 1; - p = v[nd]; - q = v[nd + 1]; - r = v[nd + 2]; - - for (i = 0; i < size; i += 8) { - r0 = q0 = p0 = v_32(v[l][i]); - r1 = q1 = p1 = v_32(v[l][i + 4]); - for (d = l - 1; d >= 0; --d) { - d0 = v_32(v[d][i]); - d1 = v_32(v[d][i + 4]); - - p0 ^= d0; - p1 ^= d1; - - q0 = x2_32(q0); - q1 = x2_32(q1); - - q0 ^= d0; - q1 ^= d1; - - r0 = d2_32(r0); - r1 = d2_32(r1); - - r0 ^= d0; - r1 ^= d1; - } - v_32(p[i]) = p0; - v_32(p[i + 4]) = p1; - v_32(q[i]) = q0; - v_32(q[i + 4]) = q1; - v_32(r[i]) = r0; - v_32(r[i + 4]) = r1; - } -} - -/* - * GENz (triple parity with powers of 2^-1) 64bit C implementation - */ -void raid_genz_int64(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t**)vv; - uint8_t *p; - uint8_t *q; - uint8_t *r; - int d, l; - size_t i; - - uint64_t d0, r0, q0, p0; - uint64_t d1, r1, q1, p1; - - l = nd - 1; - p = v[nd]; - q = v[nd + 1]; - r = v[nd + 2]; - - for (i = 0; i < size; i += 16) { - r0 = q0 = p0 = v_64(v[l][i]); - r1 = q1 = p1 = v_64(v[l][i + 8]); - for (d = l - 1; d >= 0; --d) { - d0 = v_64(v[d][i]); - d1 = v_64(v[d][i + 8]); - - p0 ^= d0; - p1 ^= d1; - - q0 = x2_64(q0); - q1 = x2_64(q1); - - q0 ^= d0; - q1 ^= d1; - - r0 = d2_64(r0); - r1 = d2_64(r1); - - r0 ^= d0; - r1 ^= d1; - } - v_64(p[i]) = p0; - v_64(p[i + 8]) = p1; - v_64(q[i]) = q0; - v_64(q[i + 8]) = q1; - v_64(r[i]) = r0; - v_64(r[i + 8]) = r1; - } -} - diff --git a/raid/memory.h b/raid/memory.h index ee78420c..ec3c9692 100644 --- a/raid/memory.h +++ b/raid/memory.h @@ -20,7 +20,8 @@ * map to the exact same sets in the L1 Data Cache. If these blocks are * accessed in parallel during SIMD parity generation, the memory streams * will constantly evict each other, resulting in severe cache thrashing. - * * Furthermore, if buffers are separated by exact multiples of 4096 bytes, + * + * Furthermore, if buffers are separated by exact multiples of 4096 bytes, * modern CPUs suffer a "4K Aliasing" penalty. The memory disambiguation unit * misidentifies the lower 12-bits of the addresses as a match, attempting * invalid Store-to-Load Forwarding which results in a ~20-cycle pipeline flush. diff --git a/raid/module.c b/raid/module.c index f02ba966..19e874aa 100644 --- a/raid/module.c +++ b/raid/module.c @@ -245,7 +245,10 @@ void raid_init(void) { raid_register_int(); #ifdef CONFIG_X86 - raid_register_x86(); + raid_register_sse2(); + raid_register_ssse3(); + raid_register_avx2(); + raid_register_avx512(); #endif #ifdef CONFIG_X86_64 raid_register_avx2gfni(); diff --git a/raid/neon.c b/raid/neon.c index 8c2002d1..02f60ce7 100644 --- a/raid/neon.c +++ b/raid/neon.c @@ -59,25 +59,6 @@ void raid_gen1_neon(int nd, size_t size, void **vv) raid_neon_end(); } -static const struct gfconst16 { - uint8_t poly[16]; - uint8_t low4[16]; - uint8_t half[16]; -} gfconst16 __aligned(16) = { - { - RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, - RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY - }, - { - 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, - 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f - }, - { - RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, - RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE - } -}; - /* * GEN2 (RAID6 with powers of 2) NEON implementation */ @@ -244,26 +225,20 @@ void raid_genz_neon(int nd, size_t size, void **vv) } /* - * GEN3 (triple parity with Cauchy matrix) NEON implementation + * GENX NEON implementation */ -void raid_gen3_neon(int nd, size_t size, void **vv) +static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int np) { uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *q; - uint8_t *r; - int d, l; size_t i; + int d, l; l = nd - 1; - p = v[nd]; - q = v[nd + 1]; - r = v[nd + 2]; /* special case with only one data disk */ if (l == 0) { - for (i = 0; i < 3; ++i) - memcpy(v[1 + i], v[0], size); + for (d = 0; d < np; ++d) + memcpy(v[1 + d], v[0], size); return; } @@ -288,207 +263,90 @@ void raid_gen3_neon(int nd, size_t size, void **vv) : "m" (v[l][i]), "m" (v[l][i + 16]) ); - asm volatile ( - "ldr q24, %0\n" - "ldr q25, %1\n" - /* chunk 0 */ - "ushr v17.16b, v0.16b, #4\n" - "and v16.16b, v0.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v8.16b, {v24.16b}, v16.16b\n" - "tbl v20.16b, {v25.16b}, v17.16b\n" - "eor v8.16b, v8.16b, v20.16b\n" - /* chunk 1 */ - "ushr v17.16b, v1.16b, #4\n" - "and v16.16b, v1.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v9.16b, {v24.16b}, v16.16b\n" - "tbl v20.16b, {v25.16b}, v17.16b\n" - "eor v9.16b, v9.16b, v20.16b\n" - : - : "m" (gfgenpshufb[l][1][0][0]), "m" (gfgenpshufb[l][1][1][0]) - ); - - /* intermediate disks */ - for (d = l - 1; d > 0; --d) { + if (np >= 3) { asm volatile ( - /* double Q0-Q1 */ - "sshr v16.16b, v2.16b, #7\n" - "shl v2.16b, v2.16b, #1\n" - "and v16.16b, v16.16b, v29.16b\n" - "eor v2.16b, v2.16b, v16.16b\n" - - "sshr v16.16b, v3.16b, #7\n" - "shl v3.16b, v3.16b, #1\n" - "and v16.16b, v16.16b, v29.16b\n" - "eor v3.16b, v3.16b, v16.16b\n" - - /* load disk data and XOR into P and Q */ - "ldr q12, %0\n" - "ldr q13, %1\n" - "eor v0.16b, v0.16b, v12.16b\n" - "eor v1.16b, v1.16b, v13.16b\n" - "eor v2.16b, v2.16b, v12.16b\n" - "eor v3.16b, v3.16b, v13.16b\n" - - /* load R tables */ - "ldr q24, %2\n" - "ldr q25, %3\n" - - /* cauchy multiply v12-v13 and XOR into R */ - "ushr v17.16b, v12.16b, #4\n" - "and v16.16b, v12.16b, v28.16b\n" + "ushr v17.16b, v0.16b, #4\n" + "and v16.16b, v0.16b, v28.16b\n" "and v17.16b, v17.16b, v28.16b\n" - "tbl v20.16b, {v24.16b}, v16.16b\n" - "tbl v21.16b, {v25.16b}, v17.16b\n" - "eor v20.16b, v20.16b, v21.16b\n" + + "ushr v19.16b, v1.16b, #4\n" + "and v18.16b, v1.16b, v28.16b\n" + "and v19.16b, v19.16b, v28.16b\n" + ); + + asm volatile ( + "ldr q22, %0\n" + "ldr q23, %1\n" + "tbl v4.16b, {v22.16b}, v16.16b\n" + "tbl v20.16b, {v23.16b}, v17.16b\n" + "eor v4.16b, v4.16b, v20.16b\n" + + "tbl v5.16b, {v22.16b}, v18.16b\n" + "tbl v20.16b, {v23.16b}, v19.16b\n" + "eor v5.16b, v5.16b, v20.16b\n" + : + : "m" (gfgenpshufb[l][1][0][0]), "m" (gfgenpshufb[l][1][1][0]) + ); + } + if (np >= 4) { + asm volatile ( + "ldr q22, %0\n" + "ldr q23, %1\n" + "tbl v6.16b, {v22.16b}, v16.16b\n" + "tbl v20.16b, {v23.16b}, v17.16b\n" + "eor v6.16b, v6.16b, v20.16b\n" + + "tbl v7.16b, {v22.16b}, v18.16b\n" + "tbl v20.16b, {v23.16b}, v19.16b\n" + "eor v7.16b, v7.16b, v20.16b\n" + : + : "m" (gfgenpshufb[l][2][0][0]), "m" (gfgenpshufb[l][2][1][0]) + ); + } + if (np >= 5) { + asm volatile ( + "ldr q22, %0\n" + "ldr q23, %1\n" + "tbl v8.16b, {v22.16b}, v16.16b\n" + "tbl v20.16b, {v23.16b}, v17.16b\n" "eor v8.16b, v8.16b, v20.16b\n" - "ushr v17.16b, v13.16b, #4\n" - "and v16.16b, v13.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v20.16b, {v24.16b}, v16.16b\n" - "tbl v21.16b, {v25.16b}, v17.16b\n" - "eor v20.16b, v20.16b, v21.16b\n" + "tbl v9.16b, {v22.16b}, v18.16b\n" + "tbl v20.16b, {v23.16b}, v19.16b\n" "eor v9.16b, v9.16b, v20.16b\n" : - : "m" (v[d][i]), "m" (v[d][i + 16]), - "m" (gfgenpshufb[d][1][0][0]), "m" (gfgenpshufb[d][1][1][0]) + : "m" (gfgenpshufb[l][3][0][0]), "m" (gfgenpshufb[l][3][1][0]) + ); + } + if (np >= 6) { + asm volatile ( + "ldr q22, %0\n" + "ldr q23, %1\n" + "tbl v10.16b, {v22.16b}, v16.16b\n" + "tbl v20.16b, {v23.16b}, v17.16b\n" + "eor v10.16b, v10.16b, v20.16b\n" + + "tbl v11.16b, {v22.16b}, v18.16b\n" + "tbl v20.16b, {v23.16b}, v19.16b\n" + "eor v11.16b, v11.16b, v20.16b\n" + : + : "m" (gfgenpshufb[l][4][0][0]), "m" (gfgenpshufb[l][4][1][0]) ); } - /* first disk with all coefficients at 1 */ - asm volatile ( - /* double Q0-Q1 */ - "sshr v16.16b, v2.16b, #7\n" - "shl v2.16b, v2.16b, #1\n" - "and v16.16b, v16.16b, v29.16b\n" - "eor v2.16b, v2.16b, v16.16b\n" - - "sshr v16.16b, v3.16b, #7\n" - "shl v3.16b, v3.16b, #1\n" - "and v16.16b, v16.16b, v29.16b\n" - "eor v3.16b, v3.16b, v16.16b\n" - - /* load disk 0 data and XOR into P, Q, R */ - "ldr q12, %0\n" - "ldr q13, %1\n" - "eor v0.16b, v0.16b, v12.16b\n" - "eor v1.16b, v1.16b, v13.16b\n" - "eor v2.16b, v2.16b, v12.16b\n" - "eor v3.16b, v3.16b, v13.16b\n" - "eor v8.16b, v8.16b, v12.16b\n" - "eor v9.16b, v9.16b, v13.16b\n" - : - : "m" (v[0][i]), "m" (v[0][i + 16]) - ); - - asm volatile ( - "str q0, %0\n" - "str q1, %1\n" - "str q2, %2\n" - "str q3, %3\n" - "str q8, %4\n" - "str q9, %5\n" - : "=m" (p[i]), "=m" (p[i + 16]), - "=m" (q[i]), "=m" (q[i + 16]), - "=m" (r[i]), "=m" (r[i + 16]) - ); - } - - raid_neon_end(); -} - -/* - * GEN4 (quad parity with Cauchy matrix) NEON implementation - */ -void raid_gen4_neon(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *q; - uint8_t *r; - uint8_t *s; - int d, l; - size_t i; - - l = nd - 1; - p = v[nd]; - q = v[nd + 1]; - r = v[nd + 2]; - s = v[nd + 3]; - - /* special case with only one data disk */ - if (l == 0) { - for (i = 0; i < 4; ++i) - memcpy(v[1 + i], v[0], size); - return; - } - - raid_neon_begin(); - - /* generic case with at least two data disks */ - asm volatile ( - "ldr q29, %0\n" - "ldr q28, %1\n" - : - : "m" (gfconst16.poly[0]), "m" (gfconst16.low4[0]) - ); - - for (i = 0; i < size; i += 32) { - /* last disk without the multiplication by two */ - asm volatile ( - "ldr q0, %0\n" - "ldr q1, %1\n" - "mov v2.16b, v0.16b\n" - "mov v3.16b, v1.16b\n" - : - : "m" (v[l][i]), "m" (v[l][i + 16]) - ); - - asm volatile ( - "ldr q24, %0\n" - "ldr q25, %1\n" - "ldr q26, %2\n" - "ldr q27, %3\n" - /* chunk 0 */ - "ushr v17.16b, v0.16b, #4\n" - "and v16.16b, v0.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v4.16b, {v24.16b}, v16.16b\n" - "tbl v20.16b, {v25.16b}, v17.16b\n" - "eor v4.16b, v4.16b, v20.16b\n" - "tbl v6.16b, {v26.16b}, v16.16b\n" - "tbl v20.16b, {v27.16b}, v17.16b\n" - "eor v6.16b, v6.16b, v20.16b\n" - /* chunk 1 */ - "ushr v17.16b, v1.16b, #4\n" - "and v16.16b, v1.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v5.16b, {v24.16b}, v16.16b\n" - "tbl v20.16b, {v25.16b}, v17.16b\n" - "eor v5.16b, v5.16b, v20.16b\n" - "tbl v7.16b, {v26.16b}, v16.16b\n" - "tbl v20.16b, {v27.16b}, v17.16b\n" - "eor v7.16b, v7.16b, v20.16b\n" - : - : "m" (gfgenpshufb[l][1][0][0]), "m" (gfgenpshufb[l][1][1][0]), - "m" (gfgenpshufb[l][2][0][0]), "m" (gfgenpshufb[l][2][1][0]) - ); - /* intermediate disks */ for (d = l - 1; d > 0; --d) { asm volatile ( /* double Q0-Q1 */ - "sshr v16.16b, v2.16b, #7\n" + "sshr v20.16b, v2.16b, #7\n" "shl v2.16b, v2.16b, #1\n" - "and v16.16b, v16.16b, v29.16b\n" - "eor v2.16b, v2.16b, v16.16b\n" + "and v20.16b, v20.16b, v29.16b\n" + "eor v2.16b, v2.16b, v20.16b\n" - "sshr v16.16b, v3.16b, #7\n" + "sshr v20.16b, v3.16b, #7\n" "shl v3.16b, v3.16b, #1\n" - "and v16.16b, v16.16b, v29.16b\n" - "eor v3.16b, v3.16b, v16.16b\n" + "and v20.16b, v20.16b, v29.16b\n" + "eor v3.16b, v3.16b, v20.16b\n" /* load data */ "ldr q12, %0\n" @@ -497,58 +355,102 @@ void raid_gen4_neon(int nd, size_t size, void **vv) "eor v1.16b, v1.16b, v13.16b\n" "eor v2.16b, v2.16b, v12.16b\n" "eor v3.16b, v3.16b, v13.16b\n" - - /* load tables for R and S */ - "ldr q24, %2\n" - "ldr q25, %3\n" - "ldr q26, %4\n" - "ldr q27, %5\n" - - /* multiply and XOR into R and S */ - /* chunk 0 */ - "ushr v17.16b, v12.16b, #4\n" - "and v16.16b, v12.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v20.16b, {v24.16b}, v16.16b\n" - "tbl v21.16b, {v25.16b}, v17.16b\n" - "eor v20.16b, v20.16b, v21.16b\n" - "eor v4.16b, v4.16b, v20.16b\n" - "tbl v20.16b, {v26.16b}, v16.16b\n" - "tbl v21.16b, {v27.16b}, v17.16b\n" - "eor v20.16b, v20.16b, v21.16b\n" - "eor v6.16b, v6.16b, v20.16b\n" - - /* chunk 1 */ - "ushr v17.16b, v13.16b, #4\n" - "and v16.16b, v13.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v20.16b, {v24.16b}, v16.16b\n" - "tbl v21.16b, {v25.16b}, v17.16b\n" - "eor v20.16b, v20.16b, v21.16b\n" - "eor v5.16b, v5.16b, v20.16b\n" - "tbl v20.16b, {v26.16b}, v16.16b\n" - "tbl v21.16b, {v27.16b}, v17.16b\n" - "eor v20.16b, v20.16b, v21.16b\n" - "eor v7.16b, v7.16b, v20.16b\n" : - : "m" (v[d][i]), "m" (v[d][i + 16]), - "m" (gfgenpshufb[d][1][0][0]), "m" (gfgenpshufb[d][1][1][0]), - "m" (gfgenpshufb[d][2][0][0]), "m" (gfgenpshufb[d][2][1][0]) + : "m" (v[d][i]), "m" (v[d][i + 16]) ); + + if (np >= 3) { + asm volatile ( + "ushr v17.16b, v12.16b, #4\n" + "and v16.16b, v12.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + + "ushr v19.16b, v13.16b, #4\n" + "and v18.16b, v13.16b, v28.16b\n" + "and v19.16b, v19.16b, v28.16b\n" + ); + + asm volatile ( + "ldr q22, %0\n" + "ldr q23, %1\n" + "tbl v20.16b, {v22.16b}, v16.16b\n" + "tbl v21.16b, {v23.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v4.16b, v4.16b, v20.16b\n" + + "tbl v20.16b, {v22.16b}, v18.16b\n" + "tbl v21.16b, {v23.16b}, v19.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v5.16b, v5.16b, v20.16b\n" + : + : "m" (gfgenpshufb[d][1][0][0]), "m" (gfgenpshufb[d][1][1][0]) + ); + } + if (np >= 4) { + asm volatile ( + "ldr q22, %0\n" + "ldr q23, %1\n" + "tbl v20.16b, {v22.16b}, v16.16b\n" + "tbl v21.16b, {v23.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v6.16b, v6.16b, v20.16b\n" + + "tbl v20.16b, {v22.16b}, v18.16b\n" + "tbl v21.16b, {v23.16b}, v19.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v7.16b, v7.16b, v20.16b\n" + : + : "m" (gfgenpshufb[d][2][0][0]), "m" (gfgenpshufb[d][2][1][0]) + ); + } + if (np >= 5) { + asm volatile ( + "ldr q22, %0\n" + "ldr q23, %1\n" + "tbl v20.16b, {v22.16b}, v16.16b\n" + "tbl v21.16b, {v23.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v8.16b, v8.16b, v20.16b\n" + + "tbl v20.16b, {v22.16b}, v18.16b\n" + "tbl v21.16b, {v23.16b}, v19.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v9.16b, v9.16b, v20.16b\n" + : + : "m" (gfgenpshufb[d][3][0][0]), "m" (gfgenpshufb[d][3][1][0]) + ); + } + if (np >= 6) { + asm volatile ( + "ldr q22, %0\n" + "ldr q23, %1\n" + "tbl v20.16b, {v22.16b}, v16.16b\n" + "tbl v21.16b, {v23.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v10.16b, v10.16b, v20.16b\n" + + "tbl v20.16b, {v22.16b}, v18.16b\n" + "tbl v21.16b, {v23.16b}, v19.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v11.16b, v11.16b, v20.16b\n" + : + : "m" (gfgenpshufb[d][4][0][0]), "m" (gfgenpshufb[d][4][1][0]) + ); + } } /* first disk with all coefficients at 1 */ asm volatile ( /* double Q0-Q1 */ - "sshr v16.16b, v2.16b, #7\n" + "sshr v20.16b, v2.16b, #7\n" "shl v2.16b, v2.16b, #1\n" - "and v16.16b, v16.16b, v29.16b\n" - "eor v2.16b, v2.16b, v16.16b\n" + "and v20.16b, v20.16b, v29.16b\n" + "eor v2.16b, v2.16b, v20.16b\n" - "sshr v16.16b, v3.16b, #7\n" + "sshr v20.16b, v3.16b, #7\n" "shl v3.16b, v3.16b, #1\n" - "and v16.16b, v16.16b, v29.16b\n" - "eor v3.16b, v3.16b, v16.16b\n" + "and v20.16b, v20.16b, v29.16b\n" + "eor v3.16b, v3.16b, v20.16b\n" /* load disk 0 data and XOR */ "ldr q12, %0\n" @@ -557,286 +459,99 @@ void raid_gen4_neon(int nd, size_t size, void **vv) "eor v1.16b, v1.16b, v13.16b\n" "eor v2.16b, v2.16b, v12.16b\n" "eor v3.16b, v3.16b, v13.16b\n" - "eor v4.16b, v4.16b, v12.16b\n" - "eor v5.16b, v5.16b, v13.16b\n" - "eor v6.16b, v6.16b, v12.16b\n" - "eor v7.16b, v7.16b, v13.16b\n" : : "m" (v[0][i]), "m" (v[0][i + 16]) ); + if (np >= 3) { + asm volatile ( + "eor v4.16b, v4.16b, v12.16b\n" + "eor v5.16b, v5.16b, v13.16b\n" + ); + } + if (np >= 4) { + asm volatile ( + "eor v6.16b, v6.16b, v12.16b\n" + "eor v7.16b, v7.16b, v13.16b\n" + ); + } + if (np >= 5) { + asm volatile ( + "eor v8.16b, v8.16b, v12.16b\n" + "eor v9.16b, v9.16b, v13.16b\n" + ); + } + if (np >= 6) { + asm volatile ( + "eor v10.16b, v10.16b, v12.16b\n" + "eor v11.16b, v11.16b, v13.16b\n" + ); + } + asm volatile ( "str q0, %0\n" "str q1, %1\n" "str q2, %2\n" "str q3, %3\n" - "str q4, %4\n" - "str q5, %5\n" - "str q6, %6\n" - "str q7, %7\n" - : "=m" (p[i]), "=m" (p[i + 16]), - "=m" (q[i]), "=m" (q[i + 16]), - "=m" (r[i]), "=m" (r[i + 16]), - "=m" (s[i]), "=m" (s[i + 16]) + : "=m" (v[nd][i]), "=m" (v[nd][i + 16]), + "=m" (v[nd + 1][i]), "=m" (v[nd + 1][i + 16]) ); + + if (np >= 3) { + asm volatile ( + "str q4, %0\n" + "str q5, %1\n" + : "=m" (v[nd + 2][i]), "=m" (v[nd + 2][i + 16]) + ); + } + if (np >= 4) { + asm volatile ( + "str q6, %0\n" + "str q7, %1\n" + : "=m" (v[nd + 3][i]), "=m" (v[nd + 3][i + 16]) + ); + } + if (np >= 5) { + asm volatile ( + "str q8, %0\n" + "str q9, %1\n" + : "=m" (v[nd + 4][i]), "=m" (v[nd + 4][i + 16]) + ); + } + if (np >= 6) { + asm volatile ( + "str q10, %0\n" + "str q11, %1\n" + : "=m" (v[nd + 5][i]), "=m" (v[nd + 5][i + 16]) + ); + } } raid_neon_end(); } +/* + * GEN3 (triple parity with Cauchy matrix) NEON implementation + */ +void raid_gen3_neon(int nd, size_t size, void **vv) +{ + raid_genX_neon(nd, size, vv, 3); +} + +/* + * GEN4 (quad parity with Cauchy matrix) NEON implementation + */ +void raid_gen4_neon(int nd, size_t size, void **vv) +{ + raid_genX_neon(nd, size, vv, 4); +} + /* * GEN5 (penta parity with Cauchy matrix) NEON implementation */ void raid_gen5_neon(int nd, size_t size, void **vv) { - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *q; - uint8_t *r; - uint8_t *s; - uint8_t *t; - int d, l; - size_t i; - - l = nd - 1; - p = v[nd]; - q = v[nd + 1]; - r = v[nd + 2]; - s = v[nd + 3]; - t = v[nd + 4]; - - /* special case with only one data disk */ - if (l == 0) { - for (i = 0; i < 5; ++i) - memcpy(v[1 + i], v[0], size); - return; - } - - raid_neon_begin(); - - /* generic case with at least two data disks */ - asm volatile ( - "ldr q29, %0\n" - "ldr q28, %1\n" - : - : "m" (gfconst16.poly[0]), "m" (gfconst16.low4[0]) - ); - - for (i = 0; i < size; i += 32) { - /* last disk without the multiplication by two */ - asm volatile ( - "ldr q0, %0\n" - "ldr q1, %1\n" - "mov v2.16b, v0.16b\n" - "mov v3.16b, v1.16b\n" - : - : "m" (v[l][i]), "m" (v[l][i + 16]) - ); - - asm volatile ( - /* initialize R, S, T */ - /* table load Q (actually index 1 in gfgenpshufb is R) */ - "ldr q22, %0\n" - "ldr q23, %1\n" - /* chunk 0 */ - "ushr v17.16b, v0.16b, #4\n" - "and v16.16b, v0.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v4.16b, {v22.16b}, v16.16b\n" - "tbl v20.16b, {v23.16b}, v17.16b\n" - "eor v4.16b, v4.16b, v20.16b\n" - /* chunk 1 */ - "ushr v17.16b, v1.16b, #4\n" - "and v16.16b, v1.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v5.16b, {v22.16b}, v16.16b\n" - "tbl v20.16b, {v23.16b}, v17.16b\n" - "eor v5.16b, v5.16b, v20.16b\n" - - /* table load S (index 2) */ - "ldr q22, %2\n" - "ldr q23, %3\n" - /* chunk 0 */ - "ushr v17.16b, v0.16b, #4\n" - "and v16.16b, v0.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v6.16b, {v22.16b}, v16.16b\n" - "tbl v20.16b, {v23.16b}, v17.16b\n" - "eor v6.16b, v6.16b, v20.16b\n" - /* chunk 1 */ - "ushr v17.16b, v1.16b, #4\n" - "and v16.16b, v1.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v7.16b, {v22.16b}, v16.16b\n" - "tbl v20.16b, {v23.16b}, v17.16b\n" - "eor v7.16b, v7.16b, v20.16b\n" - - /* table load T (index 3) */ - "ldr q22, %4\n" - "ldr q23, %5\n" - /* chunk 0 */ - "ushr v17.16b, v0.16b, #4\n" - "and v16.16b, v0.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v8.16b, {v22.16b}, v16.16b\n" - "tbl v20.16b, {v23.16b}, v17.16b\n" - "eor v8.16b, v8.16b, v20.16b\n" - /* chunk 1 */ - "ushr v17.16b, v1.16b, #4\n" - "and v16.16b, v1.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v9.16b, {v22.16b}, v16.16b\n" - "tbl v20.16b, {v23.16b}, v17.16b\n" - "eor v9.16b, v9.16b, v20.16b\n" - : - : "m" (gfgenpshufb[l][1][0][0]), "m" (gfgenpshufb[l][1][1][0]), - "m" (gfgenpshufb[l][2][0][0]), "m" (gfgenpshufb[l][2][1][0]), - "m" (gfgenpshufb[l][3][0][0]), "m" (gfgenpshufb[l][3][1][0]) - ); - - /* intermediate disks */ - for (d = l - 1; d > 0; --d) { - asm volatile ( - /* double Q0-Q1 */ - "sshr v16.16b, v2.16b, #7\n" - "shl v2.16b, v2.16b, #1\n" - "and v16.16b, v16.16b, v29.16b\n" - "eor v2.16b, v2.16b, v16.16b\n" - - "sshr v16.16b, v3.16b, #7\n" - "shl v3.16b, v3.16b, #1\n" - "and v16.16b, v16.16b, v29.16b\n" - "eor v3.16b, v3.16b, v16.16b\n" - - /* load data */ - "ldr q12, %0\n" - "ldr q13, %1\n" - "eor v0.16b, v0.16b, v12.16b\n" - "eor v1.16b, v1.16b, v13.16b\n" - "eor v2.16b, v2.16b, v12.16b\n" - "eor v3.16b, v3.16b, v13.16b\n" - : - : "m" (v[d][i]), "m" (v[d][i + 16]) - ); - - /* multiply and XOR into R, S, T */ - asm volatile ( - /* load tables for R */ - "ldr q24, %0\n" - "ldr q25, %1\n" - /* chunk 0 */ - "ushr v17.16b, v12.16b, #4\n" - "and v16.16b, v12.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v20.16b, {v24.16b}, v16.16b\n" - "tbl v21.16b, {v25.16b}, v17.16b\n" - "eor v20.16b, v20.16b, v21.16b\n" - "eor v4.16b, v4.16b, v20.16b\n" - /* chunk 1 */ - "ushr v17.16b, v13.16b, #4\n" - "and v16.16b, v13.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v20.16b, {v24.16b}, v16.16b\n" - "tbl v21.16b, {v25.16b}, v17.16b\n" - "eor v20.16b, v20.16b, v21.16b\n" - "eor v5.16b, v5.16b, v20.16b\n" - - /* load tables for S */ - "ldr q24, %2\n" - "ldr q25, %3\n" - /* chunk 0 */ - "ushr v17.16b, v12.16b, #4\n" - "and v16.16b, v12.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v20.16b, {v24.16b}, v16.16b\n" - "tbl v21.16b, {v25.16b}, v17.16b\n" - "eor v20.16b, v20.16b, v21.16b\n" - "eor v6.16b, v6.16b, v20.16b\n" - /* chunk 1 */ - "ushr v17.16b, v13.16b, #4\n" - "and v16.16b, v13.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v20.16b, {v24.16b}, v16.16b\n" - "tbl v21.16b, {v25.16b}, v17.16b\n" - "eor v20.16b, v20.16b, v21.16b\n" - "eor v7.16b, v7.16b, v20.16b\n" - - /* load tables for T */ - "ldr q24, %4\n" - "ldr q25, %5\n" - /* chunk 0 */ - "ushr v17.16b, v12.16b, #4\n" - "and v16.16b, v12.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v20.16b, {v24.16b}, v16.16b\n" - "tbl v21.16b, {v25.16b}, v17.16b\n" - "eor v20.16b, v20.16b, v21.16b\n" - "eor v8.16b, v8.16b, v20.16b\n" - /* chunk 1 */ - "ushr v17.16b, v13.16b, #4\n" - "and v16.16b, v13.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v20.16b, {v24.16b}, v16.16b\n" - "tbl v21.16b, {v25.16b}, v17.16b\n" - "eor v20.16b, v20.16b, v21.16b\n" - "eor v9.16b, v9.16b, v20.16b\n" - : - : "m" (gfgenpshufb[d][1][0][0]), "m" (gfgenpshufb[d][1][1][0]), - "m" (gfgenpshufb[d][2][0][0]), "m" (gfgenpshufb[d][2][1][0]), - "m" (gfgenpshufb[d][3][0][0]), "m" (gfgenpshufb[d][3][1][0]) - ); - } - - /* first disk with all coefficients at 1 */ - asm volatile ( - /* double Q0-Q1 */ - "sshr v16.16b, v2.16b, #7\n" - "shl v2.16b, v2.16b, #1\n" - "and v16.16b, v16.16b, v29.16b\n" - "eor v2.16b, v2.16b, v16.16b\n" - - "sshr v16.16b, v3.16b, #7\n" - "shl v3.16b, v3.16b, #1\n" - "and v16.16b, v16.16b, v29.16b\n" - "eor v3.16b, v3.16b, v16.16b\n" - - /* load disk 0 data and XOR into P, Q, R, S, T */ - "ldr q12, %0\n" - "ldr q13, %1\n" - "eor v0.16b, v0.16b, v12.16b\n" - "eor v1.16b, v1.16b, v13.16b\n" - "eor v2.16b, v2.16b, v12.16b\n" - "eor v3.16b, v3.16b, v13.16b\n" - "eor v4.16b, v4.16b, v12.16b\n" - "eor v5.16b, v5.16b, v13.16b\n" - "eor v6.16b, v6.16b, v12.16b\n" - "eor v7.16b, v7.16b, v13.16b\n" - "eor v8.16b, v8.16b, v12.16b\n" - "eor v9.16b, v9.16b, v13.16b\n" - : - : "m" (v[0][i]), "m" (v[0][i + 16]) - ); - - asm volatile ( - "str q0, %0\n" - "str q1, %1\n" - "str q2, %2\n" - "str q3, %3\n" - "str q4, %4\n" - "str q5, %5\n" - "str q6, %6\n" - "str q7, %7\n" - "str q8, %8\n" - "str q9, %9\n" - : "=m" (p[i]), "=m" (p[i + 16]), - "=m" (q[i]), "=m" (q[i + 16]), - "=m" (r[i]), "=m" (r[i + 16]), - "=m" (s[i]), "=m" (s[i + 16]), - "=m" (t[i]), "=m" (t[i + 16]) - ); - } - - raid_neon_end(); + raid_genX_neon(nd, size, vv, 5); } /* @@ -844,301 +559,7 @@ void raid_gen5_neon(int nd, size_t size, void **vv) */ void raid_gen6_neon(int nd, size_t size, void **vv) { - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *q; - uint8_t *r; - uint8_t *s; - uint8_t *t; - uint8_t *u; - int d, l; - size_t i; - - l = nd - 1; - p = v[nd]; - q = v[nd + 1]; - r = v[nd + 2]; - s = v[nd + 3]; - t = v[nd + 4]; - u = v[nd + 5]; - - /* special case with only one data disk */ - if (l == 0) { - for (i = 0; i < 6; ++i) - memcpy(v[1 + i], v[0], size); - return; - } - - raid_neon_begin(); - - /* generic case with at least two data disks */ - asm volatile ( - "ldr q29, %0\n" - "ldr q28, %1\n" - : - : "m" (gfconst16.poly[0]), "m" (gfconst16.low4[0]) - ); - - for (i = 0; i < size; i += 32) { - /* last disk without the multiplication by two */ - asm volatile ( - "ldr q0, %0\n" - "ldr q1, %1\n" - "mov v2.16b, v0.16b\n" - "mov v3.16b, v1.16b\n" - : - : "m" (v[l][i]), "m" (v[l][i + 16]) - ); - - asm volatile ( - /* initialize R, S, T, U */ - /* table load Q (actually index 1 in gfgenpshufb is R) */ - "ldr q22, %0\n" - "ldr q23, %1\n" - /* chunk 0 */ - "ushr v17.16b, v0.16b, #4\n" - "and v16.16b, v0.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v4.16b, {v22.16b}, v16.16b\n" - "tbl v20.16b, {v23.16b}, v17.16b\n" - "eor v4.16b, v4.16b, v20.16b\n" - /* chunk 1 */ - "ushr v17.16b, v1.16b, #4\n" - "and v16.16b, v1.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v5.16b, {v22.16b}, v16.16b\n" - "tbl v20.16b, {v23.16b}, v17.16b\n" - "eor v5.16b, v5.16b, v20.16b\n" - - /* table load S (index 2) */ - "ldr q22, %2\n" - "ldr q23, %3\n" - /* chunk 0 */ - "ushr v17.16b, v0.16b, #4\n" - "and v16.16b, v0.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v6.16b, {v22.16b}, v16.16b\n" - "tbl v20.16b, {v23.16b}, v17.16b\n" - "eor v6.16b, v6.16b, v20.16b\n" - /* chunk 1 */ - "ushr v17.16b, v1.16b, #4\n" - "and v16.16b, v1.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v7.16b, {v22.16b}, v16.16b\n" - "tbl v20.16b, {v23.16b}, v17.16b\n" - "eor v7.16b, v7.16b, v20.16b\n" - - /* table load T (index 3) */ - "ldr q22, %4\n" - "ldr q23, %5\n" - /* chunk 0 */ - "ushr v17.16b, v0.16b, #4\n" - "and v16.16b, v0.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v8.16b, {v22.16b}, v16.16b\n" - "tbl v20.16b, {v23.16b}, v17.16b\n" - "eor v8.16b, v8.16b, v20.16b\n" - /* chunk 1 */ - "ushr v17.16b, v1.16b, #4\n" - "and v16.16b, v1.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v9.16b, {v22.16b}, v16.16b\n" - "tbl v20.16b, {v23.16b}, v17.16b\n" - "eor v9.16b, v9.16b, v20.16b\n" - - /* table load U (index 4) */ - "ldr q22, %6\n" - "ldr q23, %7\n" - /* chunk 0 */ - "ushr v17.16b, v0.16b, #4\n" - "and v16.16b, v0.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v10.16b, {v22.16b}, v16.16b\n" - "tbl v20.16b, {v23.16b}, v17.16b\n" - "eor v10.16b, v10.16b, v20.16b\n" - /* chunk 1 */ - "ushr v17.16b, v1.16b, #4\n" - "and v16.16b, v1.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v11.16b, {v22.16b}, v16.16b\n" - "tbl v20.16b, {v23.16b}, v17.16b\n" - "eor v11.16b, v11.16b, v20.16b\n" - : - : "m" (gfgenpshufb[l][1][0][0]), "m" (gfgenpshufb[l][1][1][0]), - "m" (gfgenpshufb[l][2][0][0]), "m" (gfgenpshufb[l][2][1][0]), - "m" (gfgenpshufb[l][3][0][0]), "m" (gfgenpshufb[l][3][1][0]), - "m" (gfgenpshufb[l][4][0][0]), "m" (gfgenpshufb[l][4][1][0]) - ); - - /* intermediate disks */ - for (d = l - 1; d > 0; --d) { - asm volatile ( - /* double Q0-Q1 */ - "sshr v16.16b, v2.16b, #7\n" - "shl v2.16b, v2.16b, #1\n" - "and v16.16b, v16.16b, v29.16b\n" - "eor v2.16b, v2.16b, v16.16b\n" - - "sshr v16.16b, v3.16b, #7\n" - "shl v3.16b, v3.16b, #1\n" - "and v16.16b, v16.16b, v29.16b\n" - "eor v3.16b, v3.16b, v16.16b\n" - - /* load data */ - "ldr q12, %0\n" - "ldr q13, %1\n" - "eor v0.16b, v0.16b, v12.16b\n" - "eor v1.16b, v1.16b, v13.16b\n" - "eor v2.16b, v2.16b, v12.16b\n" - "eor v3.16b, v3.16b, v13.16b\n" - : - : "m" (v[d][i]), "m" (v[d][i + 16]) - ); - - /* multiply and XOR into R, S, T, U */ - asm volatile ( - /* load tables for R */ - "ldr q24, %0\n" - "ldr q25, %1\n" - /* chunk 0 */ - "ushr v17.16b, v12.16b, #4\n" - "and v16.16b, v12.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v20.16b, {v24.16b}, v16.16b\n" - "tbl v21.16b, {v25.16b}, v17.16b\n" - "eor v20.16b, v20.16b, v21.16b\n" - "eor v4.16b, v4.16b, v20.16b\n" - /* chunk 1 */ - "ushr v17.16b, v13.16b, #4\n" - "and v16.16b, v13.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v20.16b, {v24.16b}, v16.16b\n" - "tbl v21.16b, {v25.16b}, v17.16b\n" - "eor v20.16b, v20.16b, v21.16b\n" - "eor v5.16b, v5.16b, v20.16b\n" - - /* load tables for S */ - "ldr q24, %2\n" - "ldr q25, %3\n" - /* chunk 0 */ - "ushr v17.16b, v12.16b, #4\n" - "and v16.16b, v12.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v20.16b, {v24.16b}, v16.16b\n" - "tbl v21.16b, {v25.16b}, v17.16b\n" - "eor v20.16b, v20.16b, v21.16b\n" - "eor v6.16b, v6.16b, v20.16b\n" - /* chunk 1 */ - "ushr v17.16b, v13.16b, #4\n" - "and v16.16b, v13.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v20.16b, {v24.16b}, v16.16b\n" - "tbl v21.16b, {v25.16b}, v17.16b\n" - "eor v20.16b, v20.16b, v21.16b\n" - "eor v7.16b, v7.16b, v20.16b\n" - - /* load tables for T */ - "ldr q24, %4\n" - "ldr q25, %5\n" - /* chunk 0 */ - "ushr v17.16b, v12.16b, #4\n" - "and v16.16b, v12.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v20.16b, {v24.16b}, v16.16b\n" - "tbl v21.16b, {v25.16b}, v17.16b\n" - "eor v20.16b, v20.16b, v21.16b\n" - "eor v8.16b, v8.16b, v20.16b\n" - /* chunk 1 */ - "ushr v17.16b, v13.16b, #4\n" - "and v16.16b, v13.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v20.16b, {v24.16b}, v16.16b\n" - "tbl v21.16b, {v25.16b}, v17.16b\n" - "eor v20.16b, v20.16b, v21.16b\n" - "eor v9.16b, v9.16b, v20.16b\n" - - /* load tables for U */ - "ldr q24, %6\n" - "ldr q25, %7\n" - /* chunk 0 */ - "ushr v17.16b, v12.16b, #4\n" - "and v16.16b, v12.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v20.16b, {v24.16b}, v16.16b\n" - "tbl v21.16b, {v25.16b}, v17.16b\n" - "eor v20.16b, v20.16b, v21.16b\n" - "eor v10.16b, v10.16b, v20.16b\n" - /* chunk 1 */ - "ushr v17.16b, v13.16b, #4\n" - "and v16.16b, v13.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v20.16b, {v24.16b}, v16.16b\n" - "tbl v21.16b, {v25.16b}, v17.16b\n" - "eor v20.16b, v20.16b, v21.16b\n" - "eor v11.16b, v11.16b, v20.16b\n" - : - : "m" (gfgenpshufb[d][1][0][0]), "m" (gfgenpshufb[d][1][1][0]), - "m" (gfgenpshufb[d][2][0][0]), "m" (gfgenpshufb[d][2][1][0]), - "m" (gfgenpshufb[d][3][0][0]), "m" (gfgenpshufb[d][3][1][0]), - "m" (gfgenpshufb[d][4][0][0]), "m" (gfgenpshufb[d][4][1][0]) - ); - } - - /* first disk with all coefficients at 1 */ - asm volatile ( - /* double Q0-Q1 */ - "sshr v16.16b, v2.16b, #7\n" - "shl v2.16b, v2.16b, #1\n" - "and v16.16b, v16.16b, v29.16b\n" - "eor v2.16b, v2.16b, v16.16b\n" - - "sshr v16.16b, v3.16b, #7\n" - "shl v3.16b, v3.16b, #1\n" - "and v16.16b, v16.16b, v29.16b\n" - "eor v3.16b, v3.16b, v16.16b\n" - - /* load disk 0 data and XOR into P, Q, R, S, T, U */ - "ldr q12, %0\n" - "ldr q13, %1\n" - "eor v0.16b, v0.16b, v12.16b\n" - "eor v1.16b, v1.16b, v13.16b\n" - "eor v2.16b, v2.16b, v12.16b\n" - "eor v3.16b, v3.16b, v13.16b\n" - "eor v4.16b, v4.16b, v12.16b\n" - "eor v5.16b, v5.16b, v13.16b\n" - "eor v6.16b, v6.16b, v12.16b\n" - "eor v7.16b, v7.16b, v13.16b\n" - "eor v8.16b, v8.16b, v12.16b\n" - "eor v9.16b, v9.16b, v13.16b\n" - "eor v10.16b, v10.16b, v12.16b\n" - "eor v11.16b, v11.16b, v13.16b\n" - : - : "m" (v[0][i]), "m" (v[0][i + 16]) - ); - - asm volatile ( - "str q0, %0\n" - "str q1, %1\n" - "str q2, %2\n" - "str q3, %3\n" - "str q4, %4\n" - "str q5, %5\n" - "str q6, %6\n" - "str q7, %7\n" - "str q8, %8\n" - "str q9, %9\n" - "str q10, %10\n" - "str q11, %11\n" - : "=m" (p[i]), "=m" (p[i + 16]), - "=m" (q[i]), "=m" (q[i + 16]), - "=m" (r[i]), "=m" (r[i + 16]), - "=m" (s[i]), "=m" (s[i + 16]), - "=m" (t[i]), "=m" (t[i + 16]), - "=m" (u[i]), "=m" (u[i + 16]) - ); - } - - raid_neon_end(); + raid_genX_neon(nd, size, vv, 6); } /* diff --git a/raid/sse2.c b/raid/sse2.c new file mode 100644 index 00000000..e84c7b07 --- /dev/null +++ b/raid/sse2.c @@ -0,0 +1,430 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +// Copyright (C) 2013 Andrea Mazzoleni + +#include "internal.h" +#include "gf.h" +#include "cpu.h" + +#ifdef CONFIG_X86 + +/* ================================================================ + * Optimizations notes + * ================================================================ + * + * For x86 optimizations you can see: + * + * Software optimization resources + * http://www.agner.org/optimize/ + * + * x86, x64 Instruction Latency, Memory Latency and CPUID dumps + * http://users.atw.hu/instlatx64/ + * + * Optimization notes: + * - Cache Line Sizing: Intentionally don't process more than 64 bytes + * because 64 is the typical cache block. Processing 128 bytes or more + * doesn't increase performance, and in some cases it even decreases it. + * - Register Pressure: Processing exactly 64 bytes perfectly balances + * the SIMD register file. It allows 128-bit architectures (SSE/NEON) + * to hold 4 data vectors while keeping all necessary GF constants and + * masks in the remaining registers, avoiding slow stack spills. + * - Write-Combining Buffers: The final write of the P,Q,R,S,T parities + * always uses non-temporal stores (movntdq), even for writes smaller + * than the cache line, as the CPU has enough write-combining buffers + * to aggregate the sequential streams and avoid the RFO (Read-For-Ownership) + * memory penalty. + * - Parity Accumulator Locality: The outer loop iterates by block size, + * and the inner loop iterates by disk. This guarantees that the running + * parity accumulators never leave the CPU's SIMD registers during the + * computation. + * - D0 Cauchy Bypass: The final disk processed (D0) always has a + * coefficient of 1 for all parities. It is handled separately to bypass + * the expensive Cauchy matrix lookup and `pshufb` multiplications. + * - Pipeline Dependency Breaking: Galois Field doubling (Horner's method + * multiplication by 2) is strategically placed relative to the XOR + * operations to break serial dependency chains. This allows the CPU's + * out-of-order execution engine to hide the latency of the GF math + * behind memory loads. + */ + +/* + * ================================================================ + * RAID Parity Generation: Q-Parity Horner's Method Implementations + * ================================================================ + * + * This file implements different strategies for computing the Q parity. + * Q parity requires scaling each data disk by a descending power of 2 + * in the Galois Field: + * + * Q = (D_l * 2^l) + (D_{l-1} * 2^{l-1}) + ... + (D_1 * 2^1) + (D_0 * 1) + * + * To efficiently calculate this sequentially using SIMD instructions, we + * use Horner's method. Depending on hardware architecture and whether we + * need to optimize out the final Cauchy matrix multiplication for D0, the + * "multiply by 2" operation can be positioned in three distinct ways: + * + * ------------------------------------------------------------------------- + * CASE 1: Multiplication BEFORE XOR, handling all disks inside the loop + * ------------------------------------------------------------------------- + * Used in GEN2 (RAID6) where all disks can be processed uniformly. + * The accumulator is doubled BEFORE adding the current disk. Because the + * loop runs all the way down to D0 (d >= 0), the final disk is XORed into + * the accumulator and the loop terminates. This naturally leaves D0 with + * a coefficient of 1, as it escapes the doubling step. + * + * Simplified Code: + * Q = D_l; + * for (d = l - 1; d >= 0; --d) { + * Q = Q * 2; // Scale accumulator + * Q = Q ^ D_d; // Add current disk + * } + * + * ------------------------------------------------------------------------- + * CASE 2: Multiplication BEFORE XOR, handling D0 separately + * ------------------------------------------------------------------------- + * Used in GEN3/GEN4 when the loop contains expensive operations (like + * SSSE3 pshufb for Cauchy matrices) that we want to skip for the very + * last disk (which only requires coefficient 1). The Q-parity logic is + * identical to CASE 1, but the final iteration for D0 is unrolled. + * + * Simplified Code: + * Q = D_l; + * for (d = l - 1; d > 0; --d) { + * Q = Q * 2; // Scale accumulator + * Q = Q ^ D_d; // Add current disk + * } + * + * // Final disk (D0) processed outside the loop to skip Cauchy math + * Q = Q * 2; + * Q = Q ^ D_0; + */ + +/* + * GEN1 (RAID5 with xor) SSE2 implementation + */ +void raid_gen1_sse2(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + + raid_sse_begin(); + + for (i = 0; i < size; i += 64) { + asm volatile ("movdqa %0,%%xmm0" : : "m" (v[0][i])); + asm volatile ("movdqa %0,%%xmm1" : : "m" (v[0][i + 16])); + asm volatile ("movdqa %0,%%xmm2" : : "m" (v[0][i + 32])); + asm volatile ("movdqa %0,%%xmm3" : : "m" (v[0][i + 48])); + + for (d = 1; d <= l; ++d) { + asm volatile ("pxor %0,%%xmm0" : : "m" (v[d][i])); + asm volatile ("pxor %0,%%xmm1" : : "m" (v[d][i + 16])); + asm volatile ("pxor %0,%%xmm2" : : "m" (v[d][i + 32])); + asm volatile ("pxor %0,%%xmm3" : : "m" (v[d][i + 48])); + } + + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (p[i + 16])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (p[i + 32])); + asm volatile ("movntdq %%xmm3,%0" : "=m" (p[i + 48])); + } + + raid_sse_end(); +} + +/* + * GEN2 (RAID6 with powers of 2) SSE2 implementation + */ +void raid_gen2_sse2(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + + raid_sse_begin(); + + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + + for (i = 0; i < size; i += 32) { + asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i])); + asm volatile ("movdqa %0,%%xmm1" : : "m" (v[l][i + 16])); + asm volatile ("movdqa %xmm0,%xmm2"); + asm volatile ("movdqa %xmm1,%xmm3"); + + for (d = l - 1; d >= 0; --d) { + /* scale by 2 before adding the current disk so the first (last processed) disk with factor 1 avoids doubling */ + asm volatile ("pxor %xmm4,%xmm4"); + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pcmpgtb %xmm2,%xmm4"); + asm volatile ("pcmpgtb %xmm3,%xmm5"); + asm volatile ("paddb %xmm2,%xmm2"); + asm volatile ("paddb %xmm3,%xmm3"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + asm volatile ("pxor %xmm4,%xmm2"); + asm volatile ("pxor %xmm5,%xmm3"); + + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); + asm volatile ("movdqa %0,%%xmm5" : : "m" (v[d][i + 16])); + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm5,%xmm1"); + asm volatile ("pxor %xmm4,%xmm2"); + asm volatile ("pxor %xmm5,%xmm3"); + } + + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (p[i + 16])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (q[i])); + asm volatile ("movntdq %%xmm3,%0" : "=m" (q[i + 16])); + } + + raid_sse_end(); +} + +#ifdef CONFIG_X86_64 +/* + * GEN2 (RAID6 with powers of 2) SSE2 implementation + * + * Note that it uses 16 registers, meaning that x64 is required. + */ +void raid_gen2_sse2ext(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + + raid_sse_begin(); + + asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.poly[0])); + + for (i = 0; i < size; i += 64) { + asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i])); + asm volatile ("movdqa %0,%%xmm1" : : "m" (v[l][i + 16])); + asm volatile ("movdqa %0,%%xmm2" : : "m" (v[l][i + 32])); + asm volatile ("movdqa %0,%%xmm3" : : "m" (v[l][i + 48])); + asm volatile ("movdqa %xmm0,%xmm4"); + asm volatile ("movdqa %xmm1,%xmm5"); + asm volatile ("movdqa %xmm2,%xmm6"); + asm volatile ("movdqa %xmm3,%xmm7"); + + for (d = l - 1; d >= 0; --d) { + /* scale by 2 before adding the current disk so the first (last processed) disk with factor 1 avoids doubling */ + asm volatile ("pxor %xmm8,%xmm8"); + asm volatile ("pxor %xmm9,%xmm9"); + asm volatile ("pxor %xmm10,%xmm10"); + asm volatile ("pxor %xmm11,%xmm11"); + asm volatile ("pcmpgtb %xmm4,%xmm8"); + asm volatile ("pcmpgtb %xmm5,%xmm9"); + asm volatile ("pcmpgtb %xmm6,%xmm10"); + asm volatile ("pcmpgtb %xmm7,%xmm11"); + asm volatile ("paddb %xmm4,%xmm4"); + asm volatile ("paddb %xmm5,%xmm5"); + asm volatile ("paddb %xmm6,%xmm6"); + asm volatile ("paddb %xmm7,%xmm7"); + asm volatile ("pand %xmm15,%xmm8"); + asm volatile ("pand %xmm15,%xmm9"); + asm volatile ("pand %xmm15,%xmm10"); + asm volatile ("pand %xmm15,%xmm11"); + asm volatile ("pxor %xmm8,%xmm4"); + asm volatile ("pxor %xmm9,%xmm5"); + asm volatile ("pxor %xmm10,%xmm6"); + asm volatile ("pxor %xmm11,%xmm7"); + + asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i])); + asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d][i + 16])); + asm volatile ("movdqa %0,%%xmm10" : : "m" (v[d][i + 32])); + asm volatile ("movdqa %0,%%xmm11" : : "m" (v[d][i + 48])); + asm volatile ("pxor %xmm8,%xmm0"); + asm volatile ("pxor %xmm9,%xmm1"); + asm volatile ("pxor %xmm10,%xmm2"); + asm volatile ("pxor %xmm11,%xmm3"); + asm volatile ("pxor %xmm8,%xmm4"); + asm volatile ("pxor %xmm9,%xmm5"); + asm volatile ("pxor %xmm10,%xmm6"); + asm volatile ("pxor %xmm11,%xmm7"); + } + + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (p[i + 16])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (p[i + 32])); + asm volatile ("movntdq %%xmm3,%0" : "=m" (p[i + 48])); + asm volatile ("movntdq %%xmm4,%0" : "=m" (q[i])); + asm volatile ("movntdq %%xmm5,%0" : "=m" (q[i + 16])); + asm volatile ("movntdq %%xmm6,%0" : "=m" (q[i + 32])); + asm volatile ("movntdq %%xmm7,%0" : "=m" (q[i + 48])); + } + + raid_sse_end(); +} +#endif + +/* + * GENz (triple parity with powers of 2^-1) SSE2 implementation + */ +void raid_genz_sse2(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + + raid_sse_begin(); + + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.half[0])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (gfconst16.low7[0])); + + for (i = 0; i < size; i += 16) { + asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i])); + asm volatile ("movdqa %xmm0,%xmm1"); + asm volatile ("movdqa %xmm0,%xmm2"); + for (d = l - 1; d >= 0; --d) { + asm volatile ("pxor %xmm4,%xmm4"); + asm volatile ("pcmpgtb %xmm1,%xmm4"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pxor %xmm4,%xmm1"); + + asm volatile ("movdqa %xmm2,%xmm4"); + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("psllw $7,%xmm4"); + asm volatile ("psrlw $1,%xmm2"); + asm volatile ("pcmpgtb %xmm4,%xmm5"); + asm volatile ("pand %xmm6,%xmm2"); + asm volatile ("pand %xmm3,%xmm5"); + asm volatile ("pxor %xmm5,%xmm2"); + + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + asm volatile ("pxor %xmm4,%xmm2"); + } + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); + } + + raid_sse_end(); +} + +#ifdef CONFIG_X86_64 +/* + * GENz (triple parity with powers of 2^-1) SSE2 implementation + * + * Note that it uses 16 registers, meaning that x64 is required. + */ +void raid_genz_sse2ext(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + + raid_sse_begin(); + + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.half[0])); + asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.low7[0])); + + for (i = 0; i < size; i += 32) { + asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i])); + asm volatile ("movdqa %0,%%xmm8" : : "m" (v[l][i + 16])); + asm volatile ("movdqa %xmm0,%xmm1"); + asm volatile ("movdqa %xmm8,%xmm9"); + asm volatile ("movdqa %xmm0,%xmm2"); + asm volatile ("movdqa %xmm8,%xmm10"); + for (d = l - 1; d >= 0; --d) { + asm volatile ("movdqa %xmm2,%xmm6"); + asm volatile ("movdqa %xmm10,%xmm14"); + asm volatile ("pxor %xmm4,%xmm4"); + asm volatile ("pxor %xmm12,%xmm12"); + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pxor %xmm13,%xmm13"); + asm volatile ("psllw $7,%xmm6"); + asm volatile ("psllw $7,%xmm14"); + asm volatile ("psrlw $1,%xmm2"); + asm volatile ("psrlw $1,%xmm10"); + asm volatile ("pcmpgtb %xmm1,%xmm4"); + asm volatile ("pcmpgtb %xmm9,%xmm12"); + asm volatile ("pcmpgtb %xmm6,%xmm5"); + asm volatile ("pcmpgtb %xmm14,%xmm13"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("paddb %xmm9,%xmm9"); + asm volatile ("pand %xmm11,%xmm2"); + asm volatile ("pand %xmm11,%xmm10"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm12"); + asm volatile ("pand %xmm3,%xmm5"); + asm volatile ("pand %xmm3,%xmm13"); + asm volatile ("pxor %xmm4,%xmm1"); + asm volatile ("pxor %xmm12,%xmm9"); + asm volatile ("pxor %xmm5,%xmm2"); + asm volatile ("pxor %xmm13,%xmm10"); + + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); + asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16])); + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + asm volatile ("pxor %xmm4,%xmm2"); + asm volatile ("pxor %xmm12,%xmm8"); + asm volatile ("pxor %xmm12,%xmm9"); + asm volatile ("pxor %xmm12,%xmm10"); + } + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm8,%0" : "=m" (p[i + 16])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); + asm volatile ("movntdq %%xmm9,%0" : "=m" (q[i + 16])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); + asm volatile ("movntdq %%xmm10,%0" : "=m" (r[i + 16])); + } + + raid_sse_end(); +} +#endif + +void raid_register_sse2(void) +{ + if (raid_cpu_has_sse2()) { + raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "sse2", raid_gen1_sse2); + raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "sse2", raid_gen2_sse2); +#ifdef CONFIG_X86_64 + if (!raid_cpu_has_slow_extendedreg()) + raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "sse2e", raid_gen2_sse2ext); + /* note that raid_cpu_has_slow_extendedreg() doesn't affect vandermonde */ + raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "sse2e", raid_genz_sse2ext); +#else + raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "sse2", raid_genz_sse2); +#endif + } +} + +#endif /* CONFIG_X86 */ diff --git a/raid/ssse3.c b/raid/ssse3.c new file mode 100644 index 00000000..a3da76d4 --- /dev/null +++ b/raid/ssse3.c @@ -0,0 +1,1278 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +// Copyright (C) 2013 Andrea Mazzoleni + +#include "internal.h" +#include "gf.h" +#include "cpu.h" + +#ifdef CONFIG_X86 +/* + * GEN3 (triple parity with Cauchy matrix) SSSE3 implementation + */ +void raid_gen3_ssse3(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + + /* special case with only one data disk */ + if (l == 0) { + for (i = 0; i < 3; ++i) + memcpy(v[1 + i], v[0], size); + return; + } + + raid_sse_begin(); + + /* generic case with at least two data disks */ + asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + + for (i = 0; i < size; i += 16) { + /* last disk without the multiplication by two */ + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i])); + + asm volatile ("movdqa %xmm4,%xmm0"); + asm volatile ("movdqa %xmm4,%xmm1"); + + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + + asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][1][0][0])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[l][1][1][0])); + asm volatile ("pshufb %xmm4,%xmm2"); + asm volatile ("pshufb %xmm5,%xmm6"); + asm volatile ("pxor %xmm6,%xmm2"); + + /* intermediate disks */ + for (d = l - 1; d > 0; --d) { + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); + + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pcmpgtb %xmm1,%xmm5"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("pand %xmm3,%xmm5"); + asm volatile ("pxor %xmm5,%xmm1"); + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + + asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][0][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pxor %xmm6,%xmm2"); + asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][1][0])); + asm volatile ("pshufb %xmm5,%xmm6"); + asm volatile ("pxor %xmm6,%xmm2"); + } + + /* first disk with all coefficients at 1 */ + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i])); + + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pcmpgtb %xmm1,%xmm5"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("pand %xmm3,%xmm5"); + asm volatile ("pxor %xmm5,%xmm1"); + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + asm volatile ("pxor %xmm4,%xmm2"); + + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); + } + + raid_sse_end(); +} + +#ifdef CONFIG_X86_64 +/* + * GEN3 (triple parity with Cauchy matrix) SSSE3 implementation + * + * Note that it uses 16 registers, meaning that x64 is required. + */ +void raid_gen3_ssse3ext(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + + /* special case with only one data disk */ + if (l == 0) { + for (i = 0; i < 3; ++i) + memcpy(v[1 + i], v[0], size); + return; + } + + raid_sse_begin(); + + /* generic case with at least two data disks */ + asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.low4[0])); + + for (i = 0; i < size; i += 32) { + /* last disk without the by two multiplication */ + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i])); + asm volatile ("movdqa %0,%%xmm12" : : "m" (v[l][i + 16])); + + asm volatile ("movdqa %xmm4,%xmm0"); + asm volatile ("movdqa %xmm4,%xmm1"); + asm volatile ("movdqa %xmm12,%xmm8"); + asm volatile ("movdqa %xmm12,%xmm9"); + + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("movdqa %xmm12,%xmm13"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("psrlw $4,%xmm13"); + asm volatile ("pand %xmm11,%xmm4"); + asm volatile ("pand %xmm11,%xmm12"); + asm volatile ("pand %xmm11,%xmm5"); + asm volatile ("pand %xmm11,%xmm13"); + + asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][1][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][1][1][0])); + asm volatile ("movdqa %xmm2,%xmm10"); + asm volatile ("movdqa %xmm7,%xmm15"); + asm volatile ("pshufb %xmm4,%xmm2"); + asm volatile ("pshufb %xmm12,%xmm10"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pshufb %xmm13,%xmm15"); + asm volatile ("pxor %xmm7,%xmm2"); + asm volatile ("pxor %xmm15,%xmm10"); + + /* intermediate disks */ + for (d = l - 1; d > 0; --d) { + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); + asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16])); + + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pxor %xmm13,%xmm13"); + asm volatile ("pcmpgtb %xmm1,%xmm5"); + asm volatile ("pcmpgtb %xmm9,%xmm13"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("paddb %xmm9,%xmm9"); + asm volatile ("pand %xmm3,%xmm5"); + asm volatile ("pand %xmm3,%xmm13"); + asm volatile ("pxor %xmm5,%xmm1"); + asm volatile ("pxor %xmm13,%xmm9"); + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + asm volatile ("pxor %xmm12,%xmm8"); + asm volatile ("pxor %xmm12,%xmm9"); + + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("movdqa %xmm12,%xmm13"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("psrlw $4,%xmm13"); + asm volatile ("pand %xmm11,%xmm4"); + asm volatile ("pand %xmm11,%xmm12"); + asm volatile ("pand %xmm11,%xmm5"); + asm volatile ("pand %xmm11,%xmm13"); + + asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][1][1][0])); + asm volatile ("movdqa %xmm6,%xmm14"); + asm volatile ("movdqa %xmm7,%xmm15"); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm12,%xmm14"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pshufb %xmm13,%xmm15"); + asm volatile ("pxor %xmm6,%xmm2"); + asm volatile ("pxor %xmm14,%xmm10"); + asm volatile ("pxor %xmm7,%xmm2"); + asm volatile ("pxor %xmm15,%xmm10"); + } + + /* first disk with all coefficients at 1 */ + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i])); + asm volatile ("movdqa %0,%%xmm12" : : "m" (v[0][i + 16])); + + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pxor %xmm13,%xmm13"); + asm volatile ("pcmpgtb %xmm1,%xmm5"); + asm volatile ("pcmpgtb %xmm9,%xmm13"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("paddb %xmm9,%xmm9"); + asm volatile ("pand %xmm3,%xmm5"); + asm volatile ("pand %xmm3,%xmm13"); + asm volatile ("pxor %xmm5,%xmm1"); + asm volatile ("pxor %xmm13,%xmm9"); + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + asm volatile ("pxor %xmm4,%xmm2"); + asm volatile ("pxor %xmm12,%xmm8"); + asm volatile ("pxor %xmm12,%xmm9"); + asm volatile ("pxor %xmm12,%xmm10"); + + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm8,%0" : "=m" (p[i + 16])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); + asm volatile ("movntdq %%xmm9,%0" : "=m" (q[i + 16])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); + asm volatile ("movntdq %%xmm10,%0" : "=m" (r[i + 16])); + } + + raid_sse_end(); +} +#endif + +/* + * GEN4 (quad parity with Cauchy matrix) SSSE3 implementation + */ +void raid_gen4_ssse3(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + uint8_t *s; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + s = v[nd + 3]; + + /* special case with only one data disk */ + if (l == 0) { + for (i = 0; i < 4; ++i) + memcpy(v[1 + i], v[0], size); + return; + } + + raid_sse_begin(); + + /* generic case with at least two data disks */ + for (i = 0; i < size; i += 16) { + /* last disk without the by two multiplication */ + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i])); + + asm volatile ("movdqa %xmm4,%xmm0"); + asm volatile ("movdqa %xmm4,%xmm1"); + + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + + asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][1][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][1][1][0])); + asm volatile ("pshufb %xmm4,%xmm2"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm7,%xmm2"); + + asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][2][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][2][1][0])); + asm volatile ("pshufb %xmm4,%xmm3"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm7,%xmm3"); + + /* intermediate disks */ + for (d = l - 1; d > 0; --d) { + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); + + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pcmpgtb %xmm1,%xmm5"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("pand %xmm7,%xmm5"); + asm volatile ("pxor %xmm5,%xmm1"); + + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + + asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][1][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm2"); + asm volatile ("pxor %xmm7,%xmm2"); + + asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][2][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][2][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm3"); + asm volatile ("pxor %xmm7,%xmm3"); + } + + /* first disk with all coefficients at 1 */ + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i])); + + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pcmpgtb %xmm1,%xmm5"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("pand %xmm7,%xmm5"); + asm volatile ("pxor %xmm5,%xmm1"); + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + asm volatile ("pxor %xmm4,%xmm2"); + asm volatile ("pxor %xmm4,%xmm3"); + + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); + asm volatile ("movntdq %%xmm3,%0" : "=m" (s[i])); + } + + raid_sse_end(); +} + +#ifdef CONFIG_X86_64 +/* + * GEN4 (quad parity with Cauchy matrix) SSSE3 implementation + * + * Note that it uses 16 registers, meaning that x64 is required. + */ +void raid_gen4_ssse3ext(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + uint8_t *s; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + s = v[nd + 3]; + + /* special case with only one data disk */ + if (l == 0) { + for (i = 0; i < 4; ++i) + memcpy(v[1 + i], v[0], size); + return; + } + + raid_sse_begin(); + + /* generic case with at least two data disks */ + for (i = 0; i < size; i += 32) { + /* last disk without the by two multiplication */ + asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i])); + asm volatile ("movdqa %0,%%xmm12" : : "m" (v[l][i + 16])); + + asm volatile ("movdqa %xmm4,%xmm0"); + asm volatile ("movdqa %xmm4,%xmm1"); + asm volatile ("movdqa %xmm12,%xmm8"); + asm volatile ("movdqa %xmm12,%xmm9"); + + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("movdqa %xmm12,%xmm13"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("psrlw $4,%xmm13"); + asm volatile ("pand %xmm15,%xmm4"); + asm volatile ("pand %xmm15,%xmm12"); + asm volatile ("pand %xmm15,%xmm5"); + asm volatile ("pand %xmm15,%xmm13"); + + asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][1][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][1][1][0])); + asm volatile ("movdqa %xmm2,%xmm10"); + asm volatile ("movdqa %xmm7,%xmm15"); + asm volatile ("pshufb %xmm4,%xmm2"); + asm volatile ("pshufb %xmm12,%xmm10"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pshufb %xmm13,%xmm15"); + asm volatile ("pxor %xmm7,%xmm2"); + asm volatile ("pxor %xmm15,%xmm10"); + + asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][2][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][2][1][0])); + asm volatile ("movdqa %xmm3,%xmm11"); + asm volatile ("movdqa %xmm7,%xmm15"); + asm volatile ("pshufb %xmm4,%xmm3"); + asm volatile ("pshufb %xmm12,%xmm11"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pshufb %xmm13,%xmm15"); + asm volatile ("pxor %xmm7,%xmm3"); + asm volatile ("pxor %xmm15,%xmm11"); + + /* intermediate disks */ + for (d = l - 1; d > 0; --d) { + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); + asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16])); + + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pxor %xmm13,%xmm13"); + asm volatile ("pcmpgtb %xmm1,%xmm5"); + asm volatile ("pcmpgtb %xmm9,%xmm13"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("paddb %xmm9,%xmm9"); + asm volatile ("pand %xmm7,%xmm5"); + asm volatile ("pand %xmm7,%xmm13"); + asm volatile ("pxor %xmm5,%xmm1"); + asm volatile ("pxor %xmm13,%xmm9"); + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + asm volatile ("pxor %xmm12,%xmm8"); + asm volatile ("pxor %xmm12,%xmm9"); + + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("movdqa %xmm12,%xmm13"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("psrlw $4,%xmm13"); + asm volatile ("pand %xmm15,%xmm4"); + asm volatile ("pand %xmm15,%xmm12"); + asm volatile ("pand %xmm15,%xmm5"); + asm volatile ("pand %xmm15,%xmm13"); + + asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][1][1][0])); + asm volatile ("movdqa %xmm6,%xmm14"); + asm volatile ("movdqa %xmm7,%xmm15"); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm12,%xmm14"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pshufb %xmm13,%xmm15"); + asm volatile ("pxor %xmm6,%xmm2"); + asm volatile ("pxor %xmm14,%xmm10"); + asm volatile ("pxor %xmm7,%xmm2"); + asm volatile ("pxor %xmm15,%xmm10"); + + asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][2][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][2][1][0])); + asm volatile ("movdqa %xmm6,%xmm14"); + asm volatile ("movdqa %xmm7,%xmm15"); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm12,%xmm14"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pshufb %xmm13,%xmm15"); + asm volatile ("pxor %xmm6,%xmm3"); + asm volatile ("pxor %xmm14,%xmm11"); + asm volatile ("pxor %xmm7,%xmm3"); + asm volatile ("pxor %xmm15,%xmm11"); + } + + /* first disk with all coefficients at 1 */ + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i])); + asm volatile ("movdqa %0,%%xmm12" : : "m" (v[0][i + 16])); + + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pxor %xmm13,%xmm13"); + asm volatile ("pcmpgtb %xmm1,%xmm5"); + asm volatile ("pcmpgtb %xmm9,%xmm13"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("paddb %xmm9,%xmm9"); + asm volatile ("pand %xmm7,%xmm5"); + asm volatile ("pand %xmm7,%xmm13"); + asm volatile ("pxor %xmm5,%xmm1"); + asm volatile ("pxor %xmm13,%xmm9"); + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + asm volatile ("pxor %xmm4,%xmm2"); + asm volatile ("pxor %xmm4,%xmm3"); + asm volatile ("pxor %xmm12,%xmm8"); + asm volatile ("pxor %xmm12,%xmm9"); + asm volatile ("pxor %xmm12,%xmm10"); + asm volatile ("pxor %xmm12,%xmm11"); + + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm8,%0" : "=m" (p[i + 16])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); + asm volatile ("movntdq %%xmm9,%0" : "=m" (q[i + 16])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); + asm volatile ("movntdq %%xmm10,%0" : "=m" (r[i + 16])); + asm volatile ("movntdq %%xmm3,%0" : "=m" (s[i])); + asm volatile ("movntdq %%xmm11,%0" : "=m" (s[i + 16])); + } + + raid_sse_end(); +} +#endif + +/* + * GEN5 (penta parity with Cauchy matrix) SSSE3 implementation + */ +void raid_gen5_ssse3(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + uint8_t *s; + uint8_t *t; + int d, l; + size_t i; + uint8_t buffer[16+16]; + uint8_t *pd = __align_ptr(buffer, 16); + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + s = v[nd + 3]; + t = v[nd + 4]; + + /* special case with only one data disk */ + if (l == 0) { + for (i = 0; i < 5; ++i) + memcpy(v[1 + i], v[0], size); + return; + } + + raid_sse_begin(); + + /* generic case with at least two data disks */ + for (i = 0; i < size; i += 16) { + /* last disk without the by two multiplication */ + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i])); + + asm volatile ("movdqa %xmm4,%xmm0"); + asm volatile ("movdqa %%xmm4,%0" : "=m" (pd[0])); + + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + + asm volatile ("movdqa %0,%%xmm1" : : "m" (gfgenpshufb[l][1][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][1][1][0])); + asm volatile ("pshufb %xmm4,%xmm1"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm7,%xmm1"); + + asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][2][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][2][1][0])); + asm volatile ("pshufb %xmm4,%xmm2"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm7,%xmm2"); + + asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][3][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][3][1][0])); + asm volatile ("pshufb %xmm4,%xmm3"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm7,%xmm3"); + + /* intermediate disks */ + for (d = l - 1; d > 0; --d) { + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pcmpgtb %xmm0,%xmm5"); + asm volatile ("paddb %xmm0,%xmm0"); + asm volatile ("pand %xmm7,%xmm5"); + asm volatile ("pxor %xmm5,%xmm0"); + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm6"); + asm volatile ("movdqa %%xmm6,%0" : "=m" (pd[0])); + + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + + asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][1][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm1"); + asm volatile ("pxor %xmm7,%xmm1"); + + asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][2][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][2][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm2"); + asm volatile ("pxor %xmm7,%xmm2"); + + asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][3][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][3][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm3"); + asm volatile ("pxor %xmm7,%xmm3"); + } + + /* first disk with all coefficients at 1 */ + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + + asm volatile ("pxor %xmm5,%xmm5"); + asm volatile ("pcmpgtb %xmm0,%xmm5"); + asm volatile ("paddb %xmm0,%xmm0"); + asm volatile ("pand %xmm7,%xmm5"); + asm volatile ("pxor %xmm5,%xmm0"); + + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + asm volatile ("pxor %xmm4,%xmm2"); + asm volatile ("pxor %xmm4,%xmm3"); + asm volatile ("pxor %xmm4,%xmm6"); + + asm volatile ("movntdq %%xmm6,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm0,%0" : "=m" (q[i])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (r[i])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (s[i])); + asm volatile ("movntdq %%xmm3,%0" : "=m" (t[i])); + } + + raid_sse_end(); +} + +#ifdef CONFIG_X86_64 +/* + * GENX SSSE3ext implementation + */ +static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, int np) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + uint8_t *s; + uint8_t *t; + uint8_t *u; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + s = v[nd + 3]; + t = v[nd + 4]; + if (np >= 6) + u = v[nd + 5]; + + /* special case with only one data disk */ + if (l == 0) { + for (d = 0; d < np; ++d) + memcpy(v[1 + d], v[0], size); + return; + } + + raid_sse_begin(); + + /* generic case with at least two data disks */ + asm volatile ("movdqa %0,%%xmm14" : : "m" (gfconst16.poly[0])); + asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); + + for (i = 0; i < size; i += 16) { + /* last disk without the by two multiplication */ + asm volatile ("movdqa %0,%%xmm10" : : "m" (v[l][i])); + + asm volatile ("movdqa %xmm10,%xmm0"); + asm volatile ("movdqa %xmm10,%xmm1"); + + asm volatile ("movdqa %xmm10,%xmm11"); + asm volatile ("psrlw $4,%xmm11"); + asm volatile ("pand %xmm15,%xmm10"); + asm volatile ("pand %xmm15,%xmm11"); + + if (np >= 3) { + asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][1][0][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][1][1][0])); + asm volatile ("pshufb %xmm10,%xmm2"); + asm volatile ("pshufb %xmm11,%xmm13"); + asm volatile ("pxor %xmm13,%xmm2"); + } + + if (np >= 4) { + asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][2][0][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][2][1][0])); + asm volatile ("pshufb %xmm10,%xmm3"); + asm volatile ("pshufb %xmm11,%xmm13"); + asm volatile ("pxor %xmm13,%xmm3"); + } + + if (np >= 5) { + asm volatile ("movdqa %0,%%xmm4" : : "m" (gfgenpshufb[l][3][0][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][3][1][0])); + asm volatile ("pshufb %xmm10,%xmm4"); + asm volatile ("pshufb %xmm11,%xmm13"); + asm volatile ("pxor %xmm13,%xmm4"); + } + + if (np >= 6) { + asm volatile ("movdqa %0,%%xmm5" : : "m" (gfgenpshufb[l][4][0][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][4][1][0])); + asm volatile ("pshufb %xmm10,%xmm5"); + asm volatile ("pshufb %xmm11,%xmm13"); + asm volatile ("pxor %xmm13,%xmm5"); + } + + /* intermediate disks */ + for (d = l - 1; d > 0; --d) { + asm volatile ("movdqa %0,%%xmm10" : : "m" (v[d][i])); + + asm volatile ("pxor %xmm11,%xmm11"); + asm volatile ("pcmpgtb %xmm1,%xmm11"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("pand %xmm14,%xmm11"); + asm volatile ("pxor %xmm11,%xmm1"); + + asm volatile ("pxor %xmm10,%xmm0"); + asm volatile ("pxor %xmm10,%xmm1"); + + asm volatile ("movdqa %xmm10,%xmm11"); + asm volatile ("psrlw $4,%xmm11"); + asm volatile ("pand %xmm15,%xmm10"); + asm volatile ("pand %xmm15,%xmm11"); + + if (np >= 3) { + asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][1][0][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][1][1][0])); + asm volatile ("pshufb %xmm10,%xmm12"); + asm volatile ("pshufb %xmm11,%xmm13"); + asm volatile ("pxor %xmm12,%xmm2"); + asm volatile ("pxor %xmm13,%xmm2"); + } + + if (np >= 4) { + asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][2][0][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][2][1][0])); + asm volatile ("pshufb %xmm10,%xmm12"); + asm volatile ("pshufb %xmm11,%xmm13"); + asm volatile ("pxor %xmm12,%xmm3"); + asm volatile ("pxor %xmm13,%xmm3"); + } + + if (np >= 5) { + asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][3][0][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][3][1][0])); + asm volatile ("pshufb %xmm10,%xmm12"); + asm volatile ("pshufb %xmm11,%xmm13"); + asm volatile ("pxor %xmm12,%xmm4"); + asm volatile ("pxor %xmm13,%xmm4"); + } + + if (np >= 6) { + asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][4][0][0])); + asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][4][1][0])); + asm volatile ("pshufb %xmm10,%xmm12"); + asm volatile ("pshufb %xmm11,%xmm13"); + asm volatile ("pxor %xmm12,%xmm5"); + asm volatile ("pxor %xmm13,%xmm5"); + } + } + + /* first disk with all coefficients at 1 */ + asm volatile ("movdqa %0,%%xmm10" : : "m" (v[0][i])); + + asm volatile ("pxor %xmm11,%xmm11"); + asm volatile ("pcmpgtb %xmm1,%xmm11"); + asm volatile ("paddb %xmm1,%xmm1"); + asm volatile ("pand %xmm14,%xmm11"); + asm volatile ("pxor %xmm11,%xmm1"); + + asm volatile ("pxor %xmm10,%xmm0"); + asm volatile ("pxor %xmm10,%xmm1"); + if (np >= 3) + asm volatile ("pxor %xmm10,%xmm2"); + if (np >= 4) + asm volatile ("pxor %xmm10,%xmm3"); + if (np >= 5) + asm volatile ("pxor %xmm10,%xmm4"); + if (np >= 6) + asm volatile ("pxor %xmm10,%xmm5"); + + asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); + if (np >= 3) + asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); + if (np >= 4) + asm volatile ("movntdq %%xmm3,%0" : "=m" (s[i])); + if (np >= 5) + asm volatile ("movntdq %%xmm4,%0" : "=m" (t[i])); + if (np >= 6) + asm volatile ("movntdq %%xmm5,%0" : "=m" (u[i])); + } + + raid_sse_end(); +} + +/* + * GEN5 (penta parity with Cauchy matrix) SSSE3 implementation + * + * Note that it uses 16 registers, meaning that x64 is required. + */ +void raid_gen5_ssse3ext(int nd, size_t size, void **vv) +{ + raid_genX_ssse3ext(nd, size, vv, 5); +} +#endif + +/* + * GEN6 (hexa parity with Cauchy matrix) SSSE3 implementation + */ +void raid_gen6_ssse3(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + uint8_t *s; + uint8_t *t; + uint8_t *u; + int d, l; + size_t i; + uint8_t buffer[2*16+16]; + uint8_t *pd = __align_ptr(buffer, 16); + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + s = v[nd + 3]; + t = v[nd + 4]; + u = v[nd + 5]; + + /* special case with only one data disk */ + if (l == 0) { + for (i = 0; i < 6; ++i) + memcpy(v[1 + i], v[0], size); + return; + } + + raid_sse_begin(); + + /* generic case with at least two data disks */ + for (i = 0; i < size; i += 16) { + /* last disk without the by two multiplication */ + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i])); + + asm volatile ("movdqa %%xmm4,%0" : "=m" (pd[0])); + asm volatile ("movdqa %%xmm4,%0" : "=m" (pd[16])); + + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + + asm volatile ("movdqa %0,%%xmm0" : : "m" (gfgenpshufb[l][1][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][1][1][0])); + asm volatile ("pshufb %xmm4,%xmm0"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm7,%xmm0"); + + asm volatile ("movdqa %0,%%xmm1" : : "m" (gfgenpshufb[l][2][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][2][1][0])); + asm volatile ("pshufb %xmm4,%xmm1"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm7,%xmm1"); + + asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][3][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][3][1][0])); + asm volatile ("pshufb %xmm4,%xmm2"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm7,%xmm2"); + + asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][4][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][4][1][0])); + asm volatile ("pshufb %xmm4,%xmm3"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm7,%xmm3"); + + /* intermediate disks */ + for (d = l - 1; d > 0; --d) { + asm volatile ("movdqa %0,%%xmm5" : : "m" (pd[0])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[16])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + + asm volatile ("pxor %xmm4,%xmm4"); + asm volatile ("pcmpgtb %xmm6,%xmm4"); + asm volatile ("paddb %xmm6,%xmm6"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pxor %xmm4,%xmm6"); + + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); + + asm volatile ("pxor %xmm4,%xmm5"); + asm volatile ("pxor %xmm4,%xmm6"); + asm volatile ("movdqa %%xmm5,%0" : "=m" (pd[0])); + asm volatile ("movdqa %%xmm6,%0" : "=m" (pd[16])); + + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + + asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][1][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm0"); + asm volatile ("pxor %xmm7,%xmm0"); + + asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][2][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][2][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm1"); + asm volatile ("pxor %xmm7,%xmm1"); + + asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][3][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][3][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm2"); + asm volatile ("pxor %xmm7,%xmm2"); + + asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][4][0][0])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][4][1][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("pshufb %xmm5,%xmm7"); + asm volatile ("pxor %xmm6,%xmm3"); + asm volatile ("pxor %xmm7,%xmm3"); + } + + /* first disk with all coefficients at 1 */ + asm volatile ("movdqa %0,%%xmm5" : : "m" (pd[0])); + asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[16])); + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); + + asm volatile ("pxor %xmm4,%xmm4"); + asm volatile ("pcmpgtb %xmm6,%xmm4"); + asm volatile ("paddb %xmm6,%xmm6"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pxor %xmm4,%xmm6"); + + asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i])); + asm volatile ("pxor %xmm4,%xmm0"); + asm volatile ("pxor %xmm4,%xmm1"); + asm volatile ("pxor %xmm4,%xmm2"); + asm volatile ("pxor %xmm4,%xmm3"); + asm volatile ("pxor %xmm4,%xmm5"); + asm volatile ("pxor %xmm4,%xmm6"); + + asm volatile ("movntdq %%xmm5,%0" : "=m" (p[i])); + asm volatile ("movntdq %%xmm6,%0" : "=m" (q[i])); + asm volatile ("movntdq %%xmm0,%0" : "=m" (r[i])); + asm volatile ("movntdq %%xmm1,%0" : "=m" (s[i])); + asm volatile ("movntdq %%xmm2,%0" : "=m" (t[i])); + asm volatile ("movntdq %%xmm3,%0" : "=m" (u[i])); + } + + raid_sse_end(); +} + +#ifdef CONFIG_X86_64 +/* + * GEN6 (hexa parity with Cauchy matrix) SSSE3 implementation + * + * Note that it uses 16 registers, meaning that x64 is required. + */ +void raid_gen6_ssse3ext(int nd, size_t size, void **vv) +{ + raid_genX_ssse3ext(nd, size, vv, 6); +} +#endif + +/* + * RAID recovering for one disk SSSE3 implementation + */ +void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *pa; + uint8_t G; + uint8_t V; + size_t i; + + (void)nr; /* unused, it's always 1 */ + + /* if it's RAID5 uses the faster function */ + if (ip[0] == 0) { + raid_rec1of1(id, nd, size, vv); + return; + } + + /* setup the coefficients matrix */ + G = A(ip[0], id[0]); + + /* invert it to solve the system of linear equations */ + V = inv(G); + + /* compute delta parity */ + raid_delta_gen(1, id, ip, nd, size, vv); + + p = v[nd + ip[0]]; + pa = v[id[0]]; + + raid_sse_begin(); + + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm4" : : "m" (gfmulpshufb[V][0][0])); + asm volatile ("movdqa %0,%%xmm5" : : "m" (gfmulpshufb[V][1][0])); + + for (i = 0; i < size; i += 16) { + asm volatile ("movdqa %0,%%xmm0" : : "m" (p[i])); + asm volatile ("movdqa %0,%%xmm1" : : "m" (pa[i])); + asm volatile ("movdqa %xmm4,%xmm2"); + asm volatile ("movdqa %xmm5,%xmm3"); + asm volatile ("pxor %xmm0,%xmm1"); + asm volatile ("movdqa %xmm1,%xmm0"); + asm volatile ("psrlw $4,%xmm1"); + asm volatile ("pand %xmm7,%xmm0"); + asm volatile ("pand %xmm7,%xmm1"); + asm volatile ("pshufb %xmm0,%xmm2"); + asm volatile ("pshufb %xmm1,%xmm3"); + asm volatile ("pxor %xmm3,%xmm2"); + asm volatile ("movdqa %%xmm2,%0" : "=m" (pa[i])); + } + + raid_sse_end(); +} + +/* + * RAID recovering for two disks SSSE3 implementation + */ +void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + const int N = 2; + uint8_t *p[N]; + uint8_t *pa[N]; + uint8_t G[N * N]; + uint8_t V[N * N]; + size_t i; + int j, k; + + (void)nr; /* unused, it's always 2 */ + + /* setup the coefficients matrix */ + for (j = 0; j < N; ++j) + for (k = 0; k < N; ++k) + G[j * N + k] = A(ip[j], id[k]); + + /* invert it to solve the system of linear equations */ + raid_invert(G, V, N); + + /* compute delta parity */ + raid_delta_gen(N, id, ip, nd, size, vv); + + for (j = 0; j < N; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + raid_sse_begin(); + + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + + for (i = 0; i < size; i += 16) { + asm volatile ("movdqa %0,%%xmm0" : : "m" (p[0][i])); + asm volatile ("movdqa %0,%%xmm2" : : "m" (pa[0][i])); + asm volatile ("movdqa %0,%%xmm1" : : "m" (p[1][i])); + asm volatile ("movdqa %0,%%xmm3" : : "m" (pa[1][i])); + asm volatile ("pxor %xmm2,%xmm0"); + asm volatile ("pxor %xmm3,%xmm1"); + + asm volatile ("pxor %xmm6,%xmm6"); + + asm volatile ("movdqa %0,%%xmm2" : : "m" (gfmulpshufb[V[0]][0][0])); + asm volatile ("movdqa %0,%%xmm3" : : "m" (gfmulpshufb[V[0]][1][0])); + asm volatile ("movdqa %xmm0,%xmm4"); + asm volatile ("movdqa %xmm0,%xmm5"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + asm volatile ("pshufb %xmm4,%xmm2"); + asm volatile ("pshufb %xmm5,%xmm3"); + asm volatile ("pxor %xmm2,%xmm6"); + asm volatile ("pxor %xmm3,%xmm6"); + + asm volatile ("movdqa %0,%%xmm2" : : "m" (gfmulpshufb[V[1]][0][0])); + asm volatile ("movdqa %0,%%xmm3" : : "m" (gfmulpshufb[V[1]][1][0])); + asm volatile ("movdqa %xmm1,%xmm4"); + asm volatile ("movdqa %xmm1,%xmm5"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + asm volatile ("pshufb %xmm4,%xmm2"); + asm volatile ("pshufb %xmm5,%xmm3"); + asm volatile ("pxor %xmm2,%xmm6"); + asm volatile ("pxor %xmm3,%xmm6"); + + asm volatile ("movdqa %%xmm6,%0" : "=m" (pa[0][i])); + + asm volatile ("pxor %xmm6,%xmm6"); + + asm volatile ("movdqa %0,%%xmm2" : : "m" (gfmulpshufb[V[2]][0][0])); + asm volatile ("movdqa %0,%%xmm3" : : "m" (gfmulpshufb[V[2]][1][0])); + asm volatile ("movdqa %xmm0,%xmm4"); + asm volatile ("movdqa %xmm0,%xmm5"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + asm volatile ("pshufb %xmm4,%xmm2"); + asm volatile ("pshufb %xmm5,%xmm3"); + asm volatile ("pxor %xmm2,%xmm6"); + asm volatile ("pxor %xmm3,%xmm6"); + + asm volatile ("movdqa %0,%%xmm2" : : "m" (gfmulpshufb[V[3]][0][0])); + asm volatile ("movdqa %0,%%xmm3" : : "m" (gfmulpshufb[V[3]][1][0])); + asm volatile ("movdqa %xmm1,%xmm4"); + asm volatile ("movdqa %xmm1,%xmm5"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + asm volatile ("pshufb %xmm4,%xmm2"); + asm volatile ("pshufb %xmm5,%xmm3"); + asm volatile ("pxor %xmm2,%xmm6"); + asm volatile ("pxor %xmm3,%xmm6"); + + asm volatile ("movdqa %%xmm6,%0" : "=m" (pa[1][i])); + } + + raid_sse_end(); +} + +/* + * RAID recovering SSSE3 implementation + */ +void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + int N = nr; + uint8_t *p[RAID_PARITY_MAX]; + uint8_t *pa[RAID_PARITY_MAX]; + uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t buffer[RAID_PARITY_MAX*16+16]; + uint8_t *pd = __align_ptr(buffer, 16); + size_t i; + int j, k; + + /* setup the coefficients matrix */ + for (j = 0; j < N; ++j) + for (k = 0; k < N; ++k) + G[j * N + k] = A(ip[j], id[k]); + + /* invert it to solve the system of linear equations */ + raid_invert(G, V, N); + + /* compute delta parity */ + raid_delta_gen(N, id, ip, nd, size, vv); + + for (j = 0; j < N; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + raid_sse_begin(); + + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + + for (i = 0; i < size; i += 16) { + /* delta */ + for (j = 0; j < N; ++j) { + asm volatile ("movdqa %0,%%xmm0" : : "m" (p[j][i])); + asm volatile ("movdqa %0,%%xmm1" : : "m" (pa[j][i])); + asm volatile ("pxor %xmm1,%xmm0"); + asm volatile ("movdqa %%xmm0,%0" : "=m" (pd[j*16])); + } + + /* reconstruct */ + for (j = 0; j < N; ++j) { + asm volatile ("pxor %xmm0,%xmm0"); + asm volatile ("pxor %xmm1,%xmm1"); + + for (k = 0; k < N; ++k) { + uint8_t m = V[j * N + k]; + + asm volatile ("movdqa %0,%%xmm2" : : "m" (gfmulpshufb[m][0][0])); + asm volatile ("movdqa %0,%%xmm3" : : "m" (gfmulpshufb[m][1][0])); + asm volatile ("movdqa %0,%%xmm4" : : "m" (pd[k*16])); + asm volatile ("movdqa %xmm4,%xmm5"); + asm volatile ("psrlw $4,%xmm5"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + asm volatile ("pshufb %xmm4,%xmm2"); + asm volatile ("pshufb %xmm5,%xmm3"); + asm volatile ("pxor %xmm2,%xmm0"); + asm volatile ("pxor %xmm3,%xmm1"); + } + + asm volatile ("pxor %xmm1,%xmm0"); + asm volatile ("movdqa %%xmm0,%0" : "=m" (pa[j][i])); + } + } + + raid_sse_end(); +} + +void raid_register_ssse3(void) +{ + if (raid_cpu_has_ssse3()) { + raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3", raid_gen3_ssse3); + raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3", raid_gen4_ssse3); + raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3", raid_gen5_ssse3); + raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3", raid_gen6_ssse3); +#ifdef CONFIG_X86_64 + if (!raid_cpu_has_slow_extendedreg()) { + raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3e", raid_gen3_ssse3ext); + raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3e", raid_gen4_ssse3ext); + raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3e", raid_gen5_ssse3ext); + raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3e", raid_gen6_ssse3ext); + } +#endif + + raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "ssse3", raid_rec1_ssse3); + raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "ssse3", raid_rec2_ssse3); + raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "ssse3", raid_recX_ssse3); + raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "ssse3", raid_recX_ssse3); + raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "ssse3", raid_recX_ssse3); + raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "ssse3", raid_recX_ssse3); + } +} + +#endif /* CONFIG_X86 */ diff --git a/raid/tables.c b/raid/tables.c index 0b2f9684..4d534031 100644 --- a/raid/tables.c +++ b/raid/tables.c @@ -2,6 +2,26 @@ // Copyright (C) 2013 Andrea Mazzoleni #include "internal.h" +#include "gf.h" + +const struct gfconst16 gfconst16 __aligned(16) = { + { + RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, + RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY + }, + { + 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, + 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f + }, + { + RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, + RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE + }, + { + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f + } +}; #ifdef USE_RAID_AES /* Tables with the AES Polynomial 0x1b */ diff --git a/raid/test/Makefile b/raid/test/Makefile index 56dfe5dd..8fc0864b 100644 --- a/raid/test/Makefile +++ b/raid/test/Makefile @@ -36,7 +36,7 @@ CFLAGS += -O2 -DUSE_RAID_AES else CFLAGS += -O0 --coverage -DCOVERAGE=1 -DNDEBUG=1 endif -OBJS = raid.o check.o int.o intz.o x86.o x86z.o gfni.o neon.o tables.o memory.o test.o helper.o module.o +OBJS = raid.o check.o int.o sse2.o ssse3.o avx2.o avx512.o gfni.o neon.o tables.o memory.o test.o helper.o module.o %.o: ../%.c $(CC) $(CFLAGS) -c -o $@ $< diff --git a/raid/x86.c b/raid/x86.c deleted file mode 100644 index a6fa3ffd..00000000 --- a/raid/x86.c +++ /dev/null @@ -1,3195 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -// Copyright (C) 2013 Andrea Mazzoleni - -#include "internal.h" -#include "gf.h" -#include "cpu.h" - -#ifdef CONFIG_X86 - -/* ================================================================ - * Optimizations notes - * ================================================================ - * - * For x86 optimizations you can see: - * - * Software optimization resources - * http://www.agner.org/optimize/ - * - * x86, x64 Instruction Latency, Memory Latency and CPUID dumps - * http://users.atw.hu/instlatx64/ - * - * Optimization notes: - * - Cache Line Sizing: Intentionally don't process more than 64 bytes - * because 64 is the typical cache block. Processing 128 bytes or more - * doesn't increase performance, and in some cases it even decreases it. - * - Register Pressure: Processing exactly 64 bytes perfectly balances - * the SIMD register file. It allows 128-bit architectures (SSE/NEON) - * to hold 4 data vectors while keeping all necessary GF constants and - * masks in the remaining registers, avoiding slow stack spills. - * - Write-Combining Buffers: The final write of the P,Q,R,S,T parities - * always uses non-temporal stores (movntdq), even for writes smaller - * than the cache line, as the CPU has enough write-combining buffers - * to aggregate the sequential streams and avoid the RFO (Read-For-Ownership) - * memory penalty. - * - Parity Accumulator Locality: The outer loop iterates by block size, - * and the inner loop iterates by disk. This guarantees that the running - * parity accumulators never leave the CPU's SIMD registers during the - * computation. - * - D0 Cauchy Bypass: The final disk processed (D0) always has a - * coefficient of 1 for all parities. It is handled separately to bypass - * the expensive Cauchy matrix lookup and `pshufb` multiplications. - * - Pipeline Dependency Breaking: Galois Field doubling (Horner's method - * multiplication by 2) is strategically placed relative to the XOR - * operations to break serial dependency chains. This allows the CPU's - * out-of-order execution engine to hide the latency of the GF math - * behind memory loads. - */ - -/* - * ================================================================ - * RAID Parity Generation: Q-Parity Horner's Method Implementations - * ================================================================ - * - * This file implements different strategies for computing the Q parity. - * Q parity requires scaling each data disk by a descending power of 2 - * in the Galois Field: - * - * Q = (D_l * 2^l) + (D_{l-1} * 2^{l-1}) + ... + (D_1 * 2^1) + (D_0 * 1) - * - * To efficiently calculate this sequentially using SIMD instructions, we - * use Horner's method. Depending on hardware architecture and whether we - * need to optimize out the final Cauchy matrix multiplication for D0, the - * "multiply by 2" operation can be positioned in three distinct ways: - * - * ------------------------------------------------------------------------- - * CASE 1: Multiplication BEFORE XOR, handling all disks inside the loop - * ------------------------------------------------------------------------- - * Used in GEN2 (RAID6) where all disks can be processed uniformly. - * The accumulator is doubled BEFORE adding the current disk. Because the - * loop runs all the way down to D0 (d >= 0), the final disk is XORed into - * the accumulator and the loop terminates. This naturally leaves D0 with - * a coefficient of 1, as it escapes the doubling step. - * - * Simplified Code: - * Q = D_l; - * for (d = l - 1; d >= 0; --d) { - * Q = Q * 2; // Scale accumulator - * Q = Q ^ D_d; // Add current disk - * } - * - * ------------------------------------------------------------------------- - * CASE 2: Multiplication BEFORE XOR, handling D0 separately - * ------------------------------------------------------------------------- - * Used in GEN3/GEN4 when the loop contains expensive operations (like - * SSSE3 pshufb for Cauchy matrices) that we want to skip for the very - * last disk (which only requires coefficient 1). The Q-parity logic is - * identical to CASE 1, but the final iteration for D0 is unrolled. - * - * Simplified Code: - * Q = D_l; - * for (d = l - 1; d > 0; --d) { - * Q = Q * 2; // Scale accumulator - * Q = Q ^ D_d; // Add current disk - * } - * - * // Final disk (D0) processed outside the loop to skip Cauchy math - * Q = Q * 2; - * Q = Q ^ D_0; - * - * ------------------------------------------------------------------------- - * CASE 3: Multiplication AFTER XOR, pre-scaling the accumulator - * ------------------------------------------------------------------------- - * Used in highly pipelined architectures (like AVX-512) to improve - * instruction-level parallelism. Placing the multiplication AFTER the XOR - * in the loop allows the CPU to load and XOR the next disk while the - * complex doubling logic is still finishing. To ensure the math aligns - * correctly, the initial disk (D_l) must be pre-multiplied before entering - * the loop. - * - * Simplified Code: - * Q = D_l; - * Q = Q * 2; // Pre-scale the initial disk - * - * for (d = l - 1; d > 0; --d) { - * Q = Q ^ D_d; // Add current disk FIRST - * Q = Q * 2; // Scale accumulator AFTER - * } - * - * // Final disk (D0) simply XORed, keeping coefficient 1 - * Q = Q ^ D_0; - */ - -/* - * GEN1 (RAID5 with xor) SSE2 implementation - */ -void raid_gen1_sse2(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - int d, l; - size_t i; - - l = nd - 1; - p = v[nd]; - - raid_sse_begin(); - - for (i = 0; i < size; i += 64) { - asm volatile ("movdqa %0,%%xmm0" : : "m" (v[0][i])); - asm volatile ("movdqa %0,%%xmm1" : : "m" (v[0][i + 16])); - asm volatile ("movdqa %0,%%xmm2" : : "m" (v[0][i + 32])); - asm volatile ("movdqa %0,%%xmm3" : : "m" (v[0][i + 48])); - - for (d = 1; d <= l; ++d) { - asm volatile ("pxor %0,%%xmm0" : : "m" (v[d][i])); - asm volatile ("pxor %0,%%xmm1" : : "m" (v[d][i + 16])); - asm volatile ("pxor %0,%%xmm2" : : "m" (v[d][i + 32])); - asm volatile ("pxor %0,%%xmm3" : : "m" (v[d][i + 48])); - } - - asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); - asm volatile ("movntdq %%xmm1,%0" : "=m" (p[i + 16])); - asm volatile ("movntdq %%xmm2,%0" : "=m" (p[i + 32])); - asm volatile ("movntdq %%xmm3,%0" : "=m" (p[i + 48])); - } - - raid_sse_end(); -} - -/* - * GEN1 (RAID5 with xor) AVX2 implementation - */ -void raid_gen1_avx2(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - int d, l; - size_t i; - - l = nd - 1; - p = v[nd]; - - raid_avx_begin(); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[0][i])); - asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[0][i + 32])); - - for (d = 1; d <= l; ++d) { - asm volatile ("vpxor %0,%%ymm0,%%ymm0" : : "m" (v[d][i])); - asm volatile ("vpxor %0,%%ymm1,%%ymm1" : : "m" (v[d][i + 32])); - } - - asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); - asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); - } - - raid_avx_end(); -} - -#ifdef CONFIG_X86_64 -/* - * GEN1 (RAID5 with xor) AVX512BW implementation - * - * Note that in true AVX512F would suffice, but we don't want to add - * specific support for AVX512F because this would be the only function - * to benefit from that. - */ -void raid_gen1_avx512bw(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t**)vv; - uint8_t *p; - int d, l; - size_t i; - - l = nd - 1; - p = v[nd]; - - raid_avx_begin(); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); - - for (d = 1; d <= l - 1; d += 2) { - asm volatile ( - "vmovdqa64 %0,%%zmm1\n\t" - "vpternlogq $0x96,%1,%%zmm1,%%zmm0" - : - : "m" (v[d][i]), "m" (v[d+1][i]) - ); - } - - if (d == l) { - asm volatile ("vpxorq %0,%%zmm0,%%zmm0" : : "m" (v[l][i])); - } - - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i])); - } - - raid_avx_end(); -} -#endif - -static const struct gfconst16 { - uint8_t poly[16]; - uint8_t low4[16]; -} gfconst16 __aligned(16) = { - { - RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, - RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY - }, - { - 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, - 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f - }, -}; - -/* - * GEN2 (RAID6 with powers of 2) SSE2 implementation - */ -void raid_gen2_sse2(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *q; - int d, l; - size_t i; - - l = nd - 1; - p = v[nd]; - q = v[nd + 1]; - - raid_sse_begin(); - - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); - - for (i = 0; i < size; i += 32) { - asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i])); - asm volatile ("movdqa %0,%%xmm1" : : "m" (v[l][i + 16])); - asm volatile ("movdqa %xmm0,%xmm2"); - asm volatile ("movdqa %xmm1,%xmm3"); - - for (d = l - 1; d >= 0; --d) { - /* scale by 2 before adding the current disk so the first (last processed) disk with factor 1 avoids doubling */ - asm volatile ("pxor %xmm4,%xmm4"); - asm volatile ("pxor %xmm5,%xmm5"); - asm volatile ("pcmpgtb %xmm2,%xmm4"); - asm volatile ("pcmpgtb %xmm3,%xmm5"); - asm volatile ("paddb %xmm2,%xmm2"); - asm volatile ("paddb %xmm3,%xmm3"); - asm volatile ("pand %xmm7,%xmm4"); - asm volatile ("pand %xmm7,%xmm5"); - asm volatile ("pxor %xmm4,%xmm2"); - asm volatile ("pxor %xmm5,%xmm3"); - - asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); - asm volatile ("movdqa %0,%%xmm5" : : "m" (v[d][i + 16])); - asm volatile ("pxor %xmm4,%xmm0"); - asm volatile ("pxor %xmm5,%xmm1"); - asm volatile ("pxor %xmm4,%xmm2"); - asm volatile ("pxor %xmm5,%xmm3"); - } - - asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); - asm volatile ("movntdq %%xmm1,%0" : "=m" (p[i + 16])); - asm volatile ("movntdq %%xmm2,%0" : "=m" (q[i])); - asm volatile ("movntdq %%xmm3,%0" : "=m" (q[i + 16])); - } - - raid_sse_end(); -} - -/* - * GEN2 (RAID6 with powers of 2) AVX2 implementation - */ -void raid_gen2_avx2(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *q; - int d, l; - size_t i; - - l = nd - 1; - p = v[nd]; - q = v[nd + 1]; - - raid_avx_begin(); - - asm volatile ("vpbroadcastb %0, %%ymm7" : : "m" (gfconst16.poly[0])); - asm volatile ("vpxor %ymm6,%ymm6,%ymm6"); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[l][i])); - asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[l][i + 32])); - asm volatile ("vmovdqa %ymm0,%ymm2"); - asm volatile ("vmovdqa %ymm1,%ymm3"); - - for (d = l - 1; d >= 0; --d) { - /* scale by 2 before adding the current disk so the first (last processed) disk with factor 1 avoids doubling */ - asm volatile ("vpcmpgtb %ymm2,%ymm6,%ymm4"); - asm volatile ("vpcmpgtb %ymm3,%ymm6,%ymm5"); - asm volatile ("vpaddb %ymm2,%ymm2,%ymm2"); - asm volatile ("vpaddb %ymm3,%ymm3,%ymm3"); - asm volatile ("vpand %ymm7,%ymm4,%ymm4"); - asm volatile ("vpand %ymm7,%ymm5,%ymm5"); - asm volatile ("vpxor %ymm4,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm5,%ymm3,%ymm3"); - - asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i])); - asm volatile ("vmovdqa %0,%%ymm5" : : "m" (v[d][i + 32])); - asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); - asm volatile ("vpxor %ymm4,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm5,%ymm3,%ymm3"); - } - - asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); - asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); - asm volatile ("vmovntdq %%ymm2,%0" : "=m" (q[i])); - asm volatile ("vmovntdq %%ymm3,%0" : "=m" (q[i + 32])); - } - - raid_avx_end(); -} - -#ifdef CONFIG_X86_64 -/* - * GEN2 (RAID6 with powers of 2) AVX512BW implementation - */ -void raid_gen2_avx512bw(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *q; - int d, l; - size_t i; - - l = nd - 1; - p = v[nd]; - q = v[nd + 1]; - - raid_avx_begin(); - - asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gfconst16.low4[0])); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); - asm volatile ("vmovdqa64 %zmm0,%zmm1"); - - for (d = 1; d <= l; ++d) { - asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (v[d][i])); - - asm volatile ("vpxorq %zmm10,%zmm0,%zmm0"); - - asm volatile ("vpsrlw $4,%zmm10,%zmm11"); - asm volatile ("vpandq %zmm31,%zmm10,%zmm10"); - asm volatile ("vpandq %zmm31,%zmm11,%zmm11"); - - asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][0][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][0][1][0])); - asm volatile ("vpshufb %zmm10,%zmm12,%zmm12"); - asm volatile ("vpshufb %zmm11,%zmm13,%zmm13"); - asm volatile ("vpxorq %zmm12,%zmm1,%zmm1"); - asm volatile ("vpxorq %zmm13,%zmm1,%zmm1"); - } - - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i])); - asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i])); - } - - raid_avx_end(); -} -#endif - -#ifdef CONFIG_X86_64 -/* - * GEN2 (RAID6 with powers of 2) SSE2 implementation - * - * Note that it uses 16 registers, meaning that x64 is required. - */ -void raid_gen2_sse2ext(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *q; - int d, l; - size_t i; - - l = nd - 1; - p = v[nd]; - q = v[nd + 1]; - - raid_sse_begin(); - - asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.poly[0])); - - for (i = 0; i < size; i += 64) { - asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i])); - asm volatile ("movdqa %0,%%xmm1" : : "m" (v[l][i + 16])); - asm volatile ("movdqa %0,%%xmm2" : : "m" (v[l][i + 32])); - asm volatile ("movdqa %0,%%xmm3" : : "m" (v[l][i + 48])); - asm volatile ("movdqa %xmm0,%xmm4"); - asm volatile ("movdqa %xmm1,%xmm5"); - asm volatile ("movdqa %xmm2,%xmm6"); - asm volatile ("movdqa %xmm3,%xmm7"); - - for (d = l - 1; d >= 0; --d) { - /* scale by 2 before adding the current disk so the first (last processed) disk with factor 1 avoids doubling */ - asm volatile ("pxor %xmm8,%xmm8"); - asm volatile ("pxor %xmm9,%xmm9"); - asm volatile ("pxor %xmm10,%xmm10"); - asm volatile ("pxor %xmm11,%xmm11"); - asm volatile ("pcmpgtb %xmm4,%xmm8"); - asm volatile ("pcmpgtb %xmm5,%xmm9"); - asm volatile ("pcmpgtb %xmm6,%xmm10"); - asm volatile ("pcmpgtb %xmm7,%xmm11"); - asm volatile ("paddb %xmm4,%xmm4"); - asm volatile ("paddb %xmm5,%xmm5"); - asm volatile ("paddb %xmm6,%xmm6"); - asm volatile ("paddb %xmm7,%xmm7"); - asm volatile ("pand %xmm15,%xmm8"); - asm volatile ("pand %xmm15,%xmm9"); - asm volatile ("pand %xmm15,%xmm10"); - asm volatile ("pand %xmm15,%xmm11"); - asm volatile ("pxor %xmm8,%xmm4"); - asm volatile ("pxor %xmm9,%xmm5"); - asm volatile ("pxor %xmm10,%xmm6"); - asm volatile ("pxor %xmm11,%xmm7"); - - asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i])); - asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d][i + 16])); - asm volatile ("movdqa %0,%%xmm10" : : "m" (v[d][i + 32])); - asm volatile ("movdqa %0,%%xmm11" : : "m" (v[d][i + 48])); - asm volatile ("pxor %xmm8,%xmm0"); - asm volatile ("pxor %xmm9,%xmm1"); - asm volatile ("pxor %xmm10,%xmm2"); - asm volatile ("pxor %xmm11,%xmm3"); - asm volatile ("pxor %xmm8,%xmm4"); - asm volatile ("pxor %xmm9,%xmm5"); - asm volatile ("pxor %xmm10,%xmm6"); - asm volatile ("pxor %xmm11,%xmm7"); - } - - asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); - asm volatile ("movntdq %%xmm1,%0" : "=m" (p[i + 16])); - asm volatile ("movntdq %%xmm2,%0" : "=m" (p[i + 32])); - asm volatile ("movntdq %%xmm3,%0" : "=m" (p[i + 48])); - asm volatile ("movntdq %%xmm4,%0" : "=m" (q[i])); - asm volatile ("movntdq %%xmm5,%0" : "=m" (q[i + 16])); - asm volatile ("movntdq %%xmm6,%0" : "=m" (q[i + 32])); - asm volatile ("movntdq %%xmm7,%0" : "=m" (q[i + 48])); - } - - raid_sse_end(); -} -#endif - -/* - * GEN3 (triple parity with Cauchy matrix) SSSE3 implementation - */ -void raid_gen3_ssse3(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *q; - uint8_t *r; - int d, l; - size_t i; - - l = nd - 1; - p = v[nd]; - q = v[nd + 1]; - r = v[nd + 2]; - - /* special case with only one data disk */ - if (l == 0) { - for (i = 0; i < 3; ++i) - memcpy(v[1 + i], v[0], size); - return; - } - - raid_sse_begin(); - - /* generic case with at least two data disks */ - asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.poly[0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); - - for (i = 0; i < size; i += 16) { - /* last disk without the multiplication by two */ - asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i])); - - asm volatile ("movdqa %xmm4,%xmm0"); - asm volatile ("movdqa %xmm4,%xmm1"); - - asm volatile ("movdqa %xmm4,%xmm5"); - asm volatile ("psrlw $4,%xmm5"); - asm volatile ("pand %xmm7,%xmm4"); - asm volatile ("pand %xmm7,%xmm5"); - - asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][1][0][0])); - asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[l][1][1][0])); - asm volatile ("pshufb %xmm4,%xmm2"); - asm volatile ("pshufb %xmm5,%xmm6"); - asm volatile ("pxor %xmm6,%xmm2"); - - /* intermediate disks */ - for (d = l - 1; d > 0; --d) { - asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); - - asm volatile ("pxor %xmm5,%xmm5"); - asm volatile ("pcmpgtb %xmm1,%xmm5"); - asm volatile ("paddb %xmm1,%xmm1"); - asm volatile ("pand %xmm3,%xmm5"); - asm volatile ("pxor %xmm5,%xmm1"); - - asm volatile ("pxor %xmm4,%xmm0"); - asm volatile ("pxor %xmm4,%xmm1"); - - asm volatile ("movdqa %xmm4,%xmm5"); - asm volatile ("psrlw $4,%xmm5"); - asm volatile ("pand %xmm7,%xmm4"); - asm volatile ("pand %xmm7,%xmm5"); - - asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][0][0])); - asm volatile ("pshufb %xmm4,%xmm6"); - asm volatile ("pxor %xmm6,%xmm2"); - asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][1][0])); - asm volatile ("pshufb %xmm5,%xmm6"); - asm volatile ("pxor %xmm6,%xmm2"); - } - - /* first disk with all coefficients at 1 */ - asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i])); - - asm volatile ("pxor %xmm5,%xmm5"); - asm volatile ("pcmpgtb %xmm1,%xmm5"); - asm volatile ("paddb %xmm1,%xmm1"); - asm volatile ("pand %xmm3,%xmm5"); - asm volatile ("pxor %xmm5,%xmm1"); - - asm volatile ("pxor %xmm4,%xmm0"); - asm volatile ("pxor %xmm4,%xmm1"); - asm volatile ("pxor %xmm4,%xmm2"); - - asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); - asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); - asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); - } - - raid_sse_end(); -} - -#ifdef CONFIG_X86_64 -/* - * GEN3 (triple parity with Cauchy matrix) SSSE3 implementation - * - * Note that it uses 16 registers, meaning that x64 is required. - */ -void raid_gen3_ssse3ext(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *q; - uint8_t *r; - int d, l; - size_t i; - - l = nd - 1; - p = v[nd]; - q = v[nd + 1]; - r = v[nd + 2]; - - /* special case with only one data disk */ - if (l == 0) { - for (i = 0; i < 3; ++i) - memcpy(v[1 + i], v[0], size); - return; - } - - raid_sse_begin(); - - /* generic case with at least two data disks */ - asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.poly[0])); - asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.low4[0])); - - for (i = 0; i < size; i += 32) { - /* last disk without the by two multiplication */ - asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i])); - asm volatile ("movdqa %0,%%xmm12" : : "m" (v[l][i + 16])); - - asm volatile ("movdqa %xmm4,%xmm0"); - asm volatile ("movdqa %xmm4,%xmm1"); - asm volatile ("movdqa %xmm12,%xmm8"); - asm volatile ("movdqa %xmm12,%xmm9"); - - asm volatile ("movdqa %xmm4,%xmm5"); - asm volatile ("movdqa %xmm12,%xmm13"); - asm volatile ("psrlw $4,%xmm5"); - asm volatile ("psrlw $4,%xmm13"); - asm volatile ("pand %xmm11,%xmm4"); - asm volatile ("pand %xmm11,%xmm12"); - asm volatile ("pand %xmm11,%xmm5"); - asm volatile ("pand %xmm11,%xmm13"); - - asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][1][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][1][1][0])); - asm volatile ("movdqa %xmm2,%xmm10"); - asm volatile ("movdqa %xmm7,%xmm15"); - asm volatile ("pshufb %xmm4,%xmm2"); - asm volatile ("pshufb %xmm12,%xmm10"); - asm volatile ("pshufb %xmm5,%xmm7"); - asm volatile ("pshufb %xmm13,%xmm15"); - asm volatile ("pxor %xmm7,%xmm2"); - asm volatile ("pxor %xmm15,%xmm10"); - - /* intermediate disks */ - for (d = l - 1; d > 0; --d) { - asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); - asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16])); - - asm volatile ("pxor %xmm5,%xmm5"); - asm volatile ("pxor %xmm13,%xmm13"); - asm volatile ("pcmpgtb %xmm1,%xmm5"); - asm volatile ("pcmpgtb %xmm9,%xmm13"); - asm volatile ("paddb %xmm1,%xmm1"); - asm volatile ("paddb %xmm9,%xmm9"); - asm volatile ("pand %xmm3,%xmm5"); - asm volatile ("pand %xmm3,%xmm13"); - asm volatile ("pxor %xmm5,%xmm1"); - asm volatile ("pxor %xmm13,%xmm9"); - - asm volatile ("pxor %xmm4,%xmm0"); - asm volatile ("pxor %xmm4,%xmm1"); - asm volatile ("pxor %xmm12,%xmm8"); - asm volatile ("pxor %xmm12,%xmm9"); - - asm volatile ("movdqa %xmm4,%xmm5"); - asm volatile ("movdqa %xmm12,%xmm13"); - asm volatile ("psrlw $4,%xmm5"); - asm volatile ("psrlw $4,%xmm13"); - asm volatile ("pand %xmm11,%xmm4"); - asm volatile ("pand %xmm11,%xmm12"); - asm volatile ("pand %xmm11,%xmm5"); - asm volatile ("pand %xmm11,%xmm13"); - - asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][1][1][0])); - asm volatile ("movdqa %xmm6,%xmm14"); - asm volatile ("movdqa %xmm7,%xmm15"); - asm volatile ("pshufb %xmm4,%xmm6"); - asm volatile ("pshufb %xmm12,%xmm14"); - asm volatile ("pshufb %xmm5,%xmm7"); - asm volatile ("pshufb %xmm13,%xmm15"); - asm volatile ("pxor %xmm6,%xmm2"); - asm volatile ("pxor %xmm14,%xmm10"); - asm volatile ("pxor %xmm7,%xmm2"); - asm volatile ("pxor %xmm15,%xmm10"); - } - - /* first disk with all coefficients at 1 */ - asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i])); - asm volatile ("movdqa %0,%%xmm12" : : "m" (v[0][i + 16])); - - asm volatile ("pxor %xmm5,%xmm5"); - asm volatile ("pxor %xmm13,%xmm13"); - asm volatile ("pcmpgtb %xmm1,%xmm5"); - asm volatile ("pcmpgtb %xmm9,%xmm13"); - asm volatile ("paddb %xmm1,%xmm1"); - asm volatile ("paddb %xmm9,%xmm9"); - asm volatile ("pand %xmm3,%xmm5"); - asm volatile ("pand %xmm3,%xmm13"); - asm volatile ("pxor %xmm5,%xmm1"); - asm volatile ("pxor %xmm13,%xmm9"); - - asm volatile ("pxor %xmm4,%xmm0"); - asm volatile ("pxor %xmm4,%xmm1"); - asm volatile ("pxor %xmm4,%xmm2"); - asm volatile ("pxor %xmm12,%xmm8"); - asm volatile ("pxor %xmm12,%xmm9"); - asm volatile ("pxor %xmm12,%xmm10"); - - asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); - asm volatile ("movntdq %%xmm8,%0" : "=m" (p[i + 16])); - asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); - asm volatile ("movntdq %%xmm9,%0" : "=m" (q[i + 16])); - asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); - asm volatile ("movntdq %%xmm10,%0" : "=m" (r[i + 16])); - } - - raid_sse_end(); -} -#endif - -#ifdef CONFIG_X86_64 -/* - * GEN3 (triple parity with Cauchy matrix) AVX2 implementation - * - * Note that it uses 16 registers, meaning that x64 is required. - */ -void raid_gen3_avx2ext(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *q; - uint8_t *r; - int d, l; - size_t i; - - l = nd - 1; - p = v[nd]; - q = v[nd + 1]; - r = v[nd + 2]; - - /* special case with only one data disk */ - if (l == 0) { - for (i = 0; i < 3; ++i) - memcpy(v[1 + i], v[0], size); - return; - } - - raid_avx_begin(); - - /* generic case with at least two data disks */ - asm volatile ("vpbroadcastb %0, %%ymm3" : : "m" (gfconst16.poly[0])); - asm volatile ("vpbroadcastb %0, %%ymm11" : : "m" (gfconst16.low4[0])); - - for (i = 0; i < size; i += 64) { - /* last disk without the by two multiplication */ - asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[l][i])); - asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[l][i + 32])); - - asm volatile ("vmovdqa %ymm4,%ymm0"); - asm volatile ("vmovdqa %ymm4,%ymm1"); - asm volatile ("vmovdqa %ymm12,%ymm8"); - asm volatile ("vmovdqa %ymm12,%ymm9"); - - asm volatile ("vpsrlw $4,%ymm4,%ymm5"); - asm volatile ("vpsrlw $4,%ymm12,%ymm13"); - asm volatile ("vpand %ymm11,%ymm4,%ymm4"); - asm volatile ("vpand %ymm11,%ymm12,%ymm12"); - asm volatile ("vpand %ymm11,%ymm5,%ymm5"); - asm volatile ("vpand %ymm11,%ymm13,%ymm13"); - - asm volatile ("vbroadcasti128 %0,%%ymm10" : : "m" (gfgenpshufb[l][1][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[l][1][1][0])); - asm volatile ("vpshufb %ymm4,%ymm10,%ymm2"); - asm volatile ("vpshufb %ymm12,%ymm10,%ymm10"); - asm volatile ("vpshufb %ymm5,%ymm15,%ymm7"); - asm volatile ("vpshufb %ymm13,%ymm15,%ymm15"); - asm volatile ("vpxor %ymm7,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm15,%ymm10,%ymm10"); - - /* intermediate disks */ - for (d = l - 1; d > 0; --d) { - asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i])); - asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i + 32])); - - asm volatile ("vpxor %ymm5,%ymm5,%ymm5"); - asm volatile ("vpxor %ymm13,%ymm13,%ymm13"); - asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5"); - asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13"); - asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); - asm volatile ("vpaddb %ymm9,%ymm9,%ymm9"); - asm volatile ("vpand %ymm3,%ymm5,%ymm5"); - asm volatile ("vpand %ymm3,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); - asm volatile ("vpxor %ymm13,%ymm9,%ymm9"); - - asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); - asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); - asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); - - asm volatile ("vpsrlw $4,%ymm4,%ymm5"); - asm volatile ("vpsrlw $4,%ymm12,%ymm13"); - asm volatile ("vpand %ymm11,%ymm4,%ymm4"); - asm volatile ("vpand %ymm11,%ymm12,%ymm12"); - asm volatile ("vpand %ymm11,%ymm5,%ymm5"); - asm volatile ("vpand %ymm11,%ymm13,%ymm13"); - - asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (gfgenpshufb[d][1][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[d][1][1][0])); - asm volatile ("vpshufb %ymm4,%ymm14,%ymm6"); - asm volatile ("vpshufb %ymm12,%ymm14,%ymm14"); - asm volatile ("vpshufb %ymm5,%ymm15,%ymm7"); - asm volatile ("vpshufb %ymm13,%ymm15,%ymm15"); - asm volatile ("vpxor %ymm6,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm14,%ymm10,%ymm10"); - asm volatile ("vpxor %ymm7,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm15,%ymm10,%ymm10"); - } - - /* first disk with all coefficients at 1 */ - asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[0][i])); - asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[0][i + 32])); - - asm volatile ("vpxor %ymm5,%ymm5,%ymm5"); - asm volatile ("vpxor %ymm13,%ymm13,%ymm13"); - asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5"); - asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13"); - asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); - asm volatile ("vpaddb %ymm9,%ymm9,%ymm9"); - asm volatile ("vpand %ymm3,%ymm5,%ymm5"); - asm volatile ("vpand %ymm3,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); - asm volatile ("vpxor %ymm13,%ymm9,%ymm9"); - - asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); - asm volatile ("vpxor %ymm4,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); - asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); - asm volatile ("vpxor %ymm12,%ymm10,%ymm10"); - - asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); - asm volatile ("vmovntdq %%ymm8,%0" : "=m" (p[i + 32])); - asm volatile ("vmovntdq %%ymm1,%0" : "=m" (q[i])); - asm volatile ("vmovntdq %%ymm9,%0" : "=m" (q[i + 32])); - asm volatile ("vmovntdq %%ymm2,%0" : "=m" (r[i])); - asm volatile ("vmovntdq %%ymm10,%0" : "=m" (r[i + 32])); - } - - raid_avx_end(); -} -#endif - -#ifdef CONFIG_X86_64 -/* - * GEN3 (triple parity with Cauchy matrix) AVX512BW implementation - */ -void raid_gen3_avx512bw(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *q; - uint8_t *r; - int d, l; - size_t i; - - l = nd - 1; - p = v[nd]; - q = v[nd + 1]; - r = v[nd + 2]; - - raid_avx_begin(); - - asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gfconst16.low4[0])); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); - asm volatile ("vmovdqa64 %zmm0,%zmm1"); - asm volatile ("vmovdqa64 %zmm0,%zmm2"); - - for (d = 1; d <= l; ++d) { - asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (v[d][i])); - - asm volatile ("vpxorq %zmm10,%zmm0,%zmm0"); - - asm volatile ("vpsrlw $4,%zmm10,%zmm11"); - asm volatile ("vpandq %zmm31,%zmm10,%zmm10"); - asm volatile ("vpandq %zmm31,%zmm11,%zmm11"); - - asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][0][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][0][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (gfgenpshufb[d][1][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (gfgenpshufb[d][1][1][0])); - asm volatile ("vpshufb %zmm10,%zmm12,%zmm12"); - asm volatile ("vpshufb %zmm11,%zmm13,%zmm13"); - asm volatile ("vpshufb %zmm10,%zmm14,%zmm14"); - asm volatile ("vpshufb %zmm11,%zmm15,%zmm15"); - asm volatile ("vpternlogq $0x96,%zmm12,%zmm13,%zmm1"); - asm volatile ("vpternlogq $0x96,%zmm14,%zmm15,%zmm2"); - } - - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i])); - asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i])); - asm volatile ("vmovntdq %%zmm2,%0" : "=m" (r[i])); - } - - raid_avx_end(); -} -#endif - -/* - * GEN4 (quad parity with Cauchy matrix) SSSE3 implementation - */ -void raid_gen4_ssse3(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *q; - uint8_t *r; - uint8_t *s; - int d, l; - size_t i; - - l = nd - 1; - p = v[nd]; - q = v[nd + 1]; - r = v[nd + 2]; - s = v[nd + 3]; - - /* special case with only one data disk */ - if (l == 0) { - for (i = 0; i < 4; ++i) - memcpy(v[1 + i], v[0], size); - return; - } - - raid_sse_begin(); - - /* generic case with at least two data disks */ - for (i = 0; i < size; i += 16) { - /* last disk without the by two multiplication */ - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); - asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i])); - - asm volatile ("movdqa %xmm4,%xmm0"); - asm volatile ("movdqa %xmm4,%xmm1"); - - asm volatile ("movdqa %xmm4,%xmm5"); - asm volatile ("psrlw $4,%xmm5"); - asm volatile ("pand %xmm7,%xmm4"); - asm volatile ("pand %xmm7,%xmm5"); - - asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][1][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][1][1][0])); - asm volatile ("pshufb %xmm4,%xmm2"); - asm volatile ("pshufb %xmm5,%xmm7"); - asm volatile ("pxor %xmm7,%xmm2"); - - asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][2][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][2][1][0])); - asm volatile ("pshufb %xmm4,%xmm3"); - asm volatile ("pshufb %xmm5,%xmm7"); - asm volatile ("pxor %xmm7,%xmm3"); - - /* intermediate disks */ - for (d = l - 1; d > 0; --d) { - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); - asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); - - asm volatile ("pxor %xmm5,%xmm5"); - asm volatile ("pcmpgtb %xmm1,%xmm5"); - asm volatile ("paddb %xmm1,%xmm1"); - asm volatile ("pand %xmm7,%xmm5"); - asm volatile ("pxor %xmm5,%xmm1"); - - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); - - asm volatile ("pxor %xmm4,%xmm0"); - asm volatile ("pxor %xmm4,%xmm1"); - - asm volatile ("movdqa %xmm4,%xmm5"); - asm volatile ("psrlw $4,%xmm5"); - asm volatile ("pand %xmm7,%xmm4"); - asm volatile ("pand %xmm7,%xmm5"); - - asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][1][1][0])); - asm volatile ("pshufb %xmm4,%xmm6"); - asm volatile ("pshufb %xmm5,%xmm7"); - asm volatile ("pxor %xmm6,%xmm2"); - asm volatile ("pxor %xmm7,%xmm2"); - - asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][2][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][2][1][0])); - asm volatile ("pshufb %xmm4,%xmm6"); - asm volatile ("pshufb %xmm5,%xmm7"); - asm volatile ("pxor %xmm6,%xmm3"); - asm volatile ("pxor %xmm7,%xmm3"); - } - - /* first disk with all coefficients at 1 */ - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); - asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i])); - - asm volatile ("pxor %xmm5,%xmm5"); - asm volatile ("pcmpgtb %xmm1,%xmm5"); - asm volatile ("paddb %xmm1,%xmm1"); - asm volatile ("pand %xmm7,%xmm5"); - asm volatile ("pxor %xmm5,%xmm1"); - - asm volatile ("pxor %xmm4,%xmm0"); - asm volatile ("pxor %xmm4,%xmm1"); - asm volatile ("pxor %xmm4,%xmm2"); - asm volatile ("pxor %xmm4,%xmm3"); - - asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); - asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); - asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); - asm volatile ("movntdq %%xmm3,%0" : "=m" (s[i])); - } - - raid_sse_end(); -} - -#ifdef CONFIG_X86_64 -/* - * GEN4 (quad parity with Cauchy matrix) SSSE3 implementation - * - * Note that it uses 16 registers, meaning that x64 is required. - */ -void raid_gen4_ssse3ext(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *q; - uint8_t *r; - uint8_t *s; - int d, l; - size_t i; - - l = nd - 1; - p = v[nd]; - q = v[nd + 1]; - r = v[nd + 2]; - s = v[nd + 3]; - - /* special case with only one data disk */ - if (l == 0) { - for (i = 0; i < 4; ++i) - memcpy(v[1 + i], v[0], size); - return; - } - - raid_sse_begin(); - - /* generic case with at least two data disks */ - for (i = 0; i < size; i += 32) { - /* last disk without the by two multiplication */ - asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); - asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i])); - asm volatile ("movdqa %0,%%xmm12" : : "m" (v[l][i + 16])); - - asm volatile ("movdqa %xmm4,%xmm0"); - asm volatile ("movdqa %xmm4,%xmm1"); - asm volatile ("movdqa %xmm12,%xmm8"); - asm volatile ("movdqa %xmm12,%xmm9"); - - asm volatile ("movdqa %xmm4,%xmm5"); - asm volatile ("movdqa %xmm12,%xmm13"); - asm volatile ("psrlw $4,%xmm5"); - asm volatile ("psrlw $4,%xmm13"); - asm volatile ("pand %xmm15,%xmm4"); - asm volatile ("pand %xmm15,%xmm12"); - asm volatile ("pand %xmm15,%xmm5"); - asm volatile ("pand %xmm15,%xmm13"); - - asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][1][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][1][1][0])); - asm volatile ("movdqa %xmm2,%xmm10"); - asm volatile ("movdqa %xmm7,%xmm15"); - asm volatile ("pshufb %xmm4,%xmm2"); - asm volatile ("pshufb %xmm12,%xmm10"); - asm volatile ("pshufb %xmm5,%xmm7"); - asm volatile ("pshufb %xmm13,%xmm15"); - asm volatile ("pxor %xmm7,%xmm2"); - asm volatile ("pxor %xmm15,%xmm10"); - - asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][2][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][2][1][0])); - asm volatile ("movdqa %xmm3,%xmm11"); - asm volatile ("movdqa %xmm7,%xmm15"); - asm volatile ("pshufb %xmm4,%xmm3"); - asm volatile ("pshufb %xmm12,%xmm11"); - asm volatile ("pshufb %xmm5,%xmm7"); - asm volatile ("pshufb %xmm13,%xmm15"); - asm volatile ("pxor %xmm7,%xmm3"); - asm volatile ("pxor %xmm15,%xmm11"); - - /* intermediate disks */ - for (d = l - 1; d > 0; --d) { - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); - asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); - asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); - asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16])); - - asm volatile ("pxor %xmm5,%xmm5"); - asm volatile ("pxor %xmm13,%xmm13"); - asm volatile ("pcmpgtb %xmm1,%xmm5"); - asm volatile ("pcmpgtb %xmm9,%xmm13"); - asm volatile ("paddb %xmm1,%xmm1"); - asm volatile ("paddb %xmm9,%xmm9"); - asm volatile ("pand %xmm7,%xmm5"); - asm volatile ("pand %xmm7,%xmm13"); - asm volatile ("pxor %xmm5,%xmm1"); - asm volatile ("pxor %xmm13,%xmm9"); - - asm volatile ("pxor %xmm4,%xmm0"); - asm volatile ("pxor %xmm4,%xmm1"); - asm volatile ("pxor %xmm12,%xmm8"); - asm volatile ("pxor %xmm12,%xmm9"); - - asm volatile ("movdqa %xmm4,%xmm5"); - asm volatile ("movdqa %xmm12,%xmm13"); - asm volatile ("psrlw $4,%xmm5"); - asm volatile ("psrlw $4,%xmm13"); - asm volatile ("pand %xmm15,%xmm4"); - asm volatile ("pand %xmm15,%xmm12"); - asm volatile ("pand %xmm15,%xmm5"); - asm volatile ("pand %xmm15,%xmm13"); - - asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][1][1][0])); - asm volatile ("movdqa %xmm6,%xmm14"); - asm volatile ("movdqa %xmm7,%xmm15"); - asm volatile ("pshufb %xmm4,%xmm6"); - asm volatile ("pshufb %xmm12,%xmm14"); - asm volatile ("pshufb %xmm5,%xmm7"); - asm volatile ("pshufb %xmm13,%xmm15"); - asm volatile ("pxor %xmm6,%xmm2"); - asm volatile ("pxor %xmm14,%xmm10"); - asm volatile ("pxor %xmm7,%xmm2"); - asm volatile ("pxor %xmm15,%xmm10"); - - asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][2][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][2][1][0])); - asm volatile ("movdqa %xmm6,%xmm14"); - asm volatile ("movdqa %xmm7,%xmm15"); - asm volatile ("pshufb %xmm4,%xmm6"); - asm volatile ("pshufb %xmm12,%xmm14"); - asm volatile ("pshufb %xmm5,%xmm7"); - asm volatile ("pshufb %xmm13,%xmm15"); - asm volatile ("pxor %xmm6,%xmm3"); - asm volatile ("pxor %xmm14,%xmm11"); - asm volatile ("pxor %xmm7,%xmm3"); - asm volatile ("pxor %xmm15,%xmm11"); - } - - /* first disk with all coefficients at 1 */ - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); - asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); - asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i])); - asm volatile ("movdqa %0,%%xmm12" : : "m" (v[0][i + 16])); - - asm volatile ("pxor %xmm5,%xmm5"); - asm volatile ("pxor %xmm13,%xmm13"); - asm volatile ("pcmpgtb %xmm1,%xmm5"); - asm volatile ("pcmpgtb %xmm9,%xmm13"); - asm volatile ("paddb %xmm1,%xmm1"); - asm volatile ("paddb %xmm9,%xmm9"); - asm volatile ("pand %xmm7,%xmm5"); - asm volatile ("pand %xmm7,%xmm13"); - asm volatile ("pxor %xmm5,%xmm1"); - asm volatile ("pxor %xmm13,%xmm9"); - - asm volatile ("pxor %xmm4,%xmm0"); - asm volatile ("pxor %xmm4,%xmm1"); - asm volatile ("pxor %xmm4,%xmm2"); - asm volatile ("pxor %xmm4,%xmm3"); - asm volatile ("pxor %xmm12,%xmm8"); - asm volatile ("pxor %xmm12,%xmm9"); - asm volatile ("pxor %xmm12,%xmm10"); - asm volatile ("pxor %xmm12,%xmm11"); - - asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); - asm volatile ("movntdq %%xmm8,%0" : "=m" (p[i + 16])); - asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); - asm volatile ("movntdq %%xmm9,%0" : "=m" (q[i + 16])); - asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); - asm volatile ("movntdq %%xmm10,%0" : "=m" (r[i + 16])); - asm volatile ("movntdq %%xmm3,%0" : "=m" (s[i])); - asm volatile ("movntdq %%xmm11,%0" : "=m" (s[i + 16])); - } - - raid_sse_end(); -} -#endif - -#ifdef CONFIG_X86_64 -/* - * GEN4 (quad parity with Cauchy matrix) AVX2 implementation - * - * Note that it uses 16 registers, meaning that x64 is required. - */ -void raid_gen4_avx2ext(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *q; - uint8_t *r; - uint8_t *s; - int d, l; - size_t i; - - l = nd - 1; - p = v[nd]; - q = v[nd + 1]; - r = v[nd + 2]; - s = v[nd + 3]; - - /* special case with only one data disk */ - if (l == 0) { - for (i = 0; i < 4; ++i) - memcpy(v[1 + i], v[0], size); - return; - } - - raid_avx_begin(); - - /* generic case with at least two data disks */ - for (i = 0; i < size; i += 64) { - /* last disk without the by two multiplication */ - asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0])); - asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[l][i])); - asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[l][i + 32])); - - asm volatile ("vmovdqa %ymm4,%ymm0"); - asm volatile ("vmovdqa %ymm4,%ymm1"); - asm volatile ("vmovdqa %ymm12,%ymm8"); - asm volatile ("vmovdqa %ymm12,%ymm9"); - - asm volatile ("vpsrlw $4,%ymm4,%ymm5"); - asm volatile ("vpsrlw $4,%ymm12,%ymm13"); - asm volatile ("vpand %ymm15,%ymm4,%ymm4"); - asm volatile ("vpand %ymm15,%ymm12,%ymm12"); - asm volatile ("vpand %ymm15,%ymm5,%ymm5"); - asm volatile ("vpand %ymm15,%ymm13,%ymm13"); - - asm volatile ("vbroadcasti128 %0,%%ymm10" : : "m" (gfgenpshufb[l][1][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[l][1][1][0])); - asm volatile ("vpshufb %ymm4,%ymm10,%ymm2"); - asm volatile ("vpshufb %ymm5,%ymm15,%ymm7"); - asm volatile ("vpshufb %ymm12,%ymm10,%ymm10"); - asm volatile ("vpshufb %ymm13,%ymm15,%ymm15"); - asm volatile ("vpxor %ymm7,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm15,%ymm10,%ymm10"); - - asm volatile ("vbroadcasti128 %0,%%ymm11" : : "m" (gfgenpshufb[l][2][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[l][2][1][0])); - asm volatile ("vpshufb %ymm4,%ymm11,%ymm3"); - asm volatile ("vpshufb %ymm5,%ymm15,%ymm7"); - asm volatile ("vpshufb %ymm12,%ymm11,%ymm11"); - asm volatile ("vpshufb %ymm13,%ymm15,%ymm15"); - asm volatile ("vpxor %ymm7,%ymm3,%ymm3"); - asm volatile ("vpxor %ymm15,%ymm11,%ymm11"); - - /* intermediate disks */ - for (d = l - 1; d > 0; --d) { - asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.poly[0])); - asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0])); - asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i])); - asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i + 32])); - - asm volatile ("vpxor %ymm5,%ymm5,%ymm5"); - asm volatile ("vpxor %ymm13,%ymm13,%ymm13"); - asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5"); - asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13"); - asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); - asm volatile ("vpaddb %ymm9,%ymm9,%ymm9"); - asm volatile ("vpand %ymm7,%ymm5,%ymm5"); - asm volatile ("vpand %ymm7,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); - asm volatile ("vpxor %ymm13,%ymm9,%ymm9"); - - asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); - asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); - asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); - - asm volatile ("vpsrlw $4,%ymm4,%ymm5"); - asm volatile ("vpsrlw $4,%ymm12,%ymm13"); - asm volatile ("vpand %ymm15,%ymm4,%ymm4"); - asm volatile ("vpand %ymm15,%ymm12,%ymm12"); - asm volatile ("vpand %ymm15,%ymm5,%ymm5"); - asm volatile ("vpand %ymm15,%ymm13,%ymm13"); - - asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (gfgenpshufb[d][1][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[d][1][1][0])); - asm volatile ("vpshufb %ymm4,%ymm14,%ymm6"); - asm volatile ("vpshufb %ymm5,%ymm15,%ymm7"); - asm volatile ("vpshufb %ymm12,%ymm14,%ymm14"); - asm volatile ("vpshufb %ymm13,%ymm15,%ymm15"); - asm volatile ("vpxor %ymm6,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm14,%ymm10,%ymm10"); - asm volatile ("vpxor %ymm7,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm15,%ymm10,%ymm10"); - - asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (gfgenpshufb[d][2][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[d][2][1][0])); - asm volatile ("vpshufb %ymm4,%ymm14,%ymm6"); - asm volatile ("vpshufb %ymm5,%ymm15,%ymm7"); - asm volatile ("vpshufb %ymm12,%ymm14,%ymm14"); - asm volatile ("vpshufb %ymm13,%ymm15,%ymm15"); - asm volatile ("vpxor %ymm6,%ymm3,%ymm3"); - asm volatile ("vpxor %ymm14,%ymm11,%ymm11"); - asm volatile ("vpxor %ymm7,%ymm3,%ymm3"); - asm volatile ("vpxor %ymm15,%ymm11,%ymm11"); - } - - /* first disk with all coefficients at 1 */ - asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.poly[0])); - asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0])); - asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[0][i])); - asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[0][i + 32])); - - asm volatile ("vpxor %ymm5,%ymm5,%ymm5"); - asm volatile ("vpxor %ymm13,%ymm13,%ymm13"); - asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5"); - asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13"); - asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); - asm volatile ("vpaddb %ymm9,%ymm9,%ymm9"); - asm volatile ("vpand %ymm7,%ymm5,%ymm5"); - asm volatile ("vpand %ymm7,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); - asm volatile ("vpxor %ymm13,%ymm9,%ymm9"); - - asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); - asm volatile ("vpxor %ymm4,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm4,%ymm3,%ymm3"); - asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); - asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); - asm volatile ("vpxor %ymm12,%ymm10,%ymm10"); - asm volatile ("vpxor %ymm12,%ymm11,%ymm11"); - - asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); - asm volatile ("vmovntdq %%ymm8,%0" : "=m" (p[i + 32])); - asm volatile ("vmovntdq %%ymm1,%0" : "=m" (q[i])); - asm volatile ("vmovntdq %%ymm9,%0" : "=m" (q[i + 32])); - asm volatile ("vmovntdq %%ymm2,%0" : "=m" (r[i])); - asm volatile ("vmovntdq %%ymm10,%0" : "=m" (r[i + 32])); - asm volatile ("vmovntdq %%ymm3,%0" : "=m" (s[i])); - asm volatile ("vmovntdq %%ymm11,%0" : "=m" (s[i + 32])); - } - - raid_avx_end(); -} -#endif - -#ifdef CONFIG_X86_64 -/* - * GEN4 (quad parity with Cauchy matrix) AVX512BW implementation - */ -void raid_gen4_avx512bw(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *q; - uint8_t *r; - uint8_t *s; - int d, l; - size_t i; - - l = nd - 1; - p = v[nd]; - q = v[nd + 1]; - r = v[nd + 2]; - s = v[nd + 3]; - - raid_avx_begin(); - - asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gfconst16.low4[0])); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); - asm volatile ("vmovdqa64 %zmm0,%zmm1"); - asm volatile ("vmovdqa64 %zmm0,%zmm2"); - asm volatile ("vmovdqa64 %zmm0,%zmm3"); - - for (d = 1; d <= l; ++d) { - asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (v[d][i])); - - asm volatile ("vpxorq %zmm10,%zmm0,%zmm0"); - - asm volatile ("vpsrlw $4,%zmm10,%zmm11"); - asm volatile ("vpandq %zmm31,%zmm10,%zmm10"); - asm volatile ("vpandq %zmm31,%zmm11,%zmm11"); - - asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][0][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][0][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (gfgenpshufb[d][1][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (gfgenpshufb[d][1][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm16" : : "m" (gfgenpshufb[d][2][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm17" : : "m" (gfgenpshufb[d][2][1][0])); - asm volatile ("vpshufb %zmm10,%zmm12,%zmm12"); - asm volatile ("vpshufb %zmm11,%zmm13,%zmm13"); - asm volatile ("vpshufb %zmm10,%zmm14,%zmm14"); - asm volatile ("vpshufb %zmm11,%zmm15,%zmm15"); - asm volatile ("vpshufb %zmm10,%zmm16,%zmm16"); - asm volatile ("vpshufb %zmm11,%zmm17,%zmm17"); - asm volatile ("vpternlogq $0x96,%zmm12,%zmm13,%zmm1"); - asm volatile ("vpternlogq $0x96,%zmm14,%zmm15,%zmm2"); - asm volatile ("vpternlogq $0x96,%zmm16,%zmm17,%zmm3"); - } - - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i])); - asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i])); - asm volatile ("vmovntdq %%zmm2,%0" : "=m" (r[i])); - asm volatile ("vmovntdq %%zmm3,%0" : "=m" (s[i])); - } - - raid_avx_end(); -} -#endif - -/* - * GEN5 (penta parity with Cauchy matrix) SSSE3 implementation - */ -void raid_gen5_ssse3(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *q; - uint8_t *r; - uint8_t *s; - uint8_t *t; - int d, l; - size_t i; - uint8_t buffer[16+16]; - uint8_t *pd = __align_ptr(buffer, 16); - - l = nd - 1; - p = v[nd]; - q = v[nd + 1]; - r = v[nd + 2]; - s = v[nd + 3]; - t = v[nd + 4]; - - /* special case with only one data disk */ - if (l == 0) { - for (i = 0; i < 5; ++i) - memcpy(v[1 + i], v[0], size); - return; - } - - raid_sse_begin(); - - /* generic case with at least two data disks */ - for (i = 0; i < size; i += 16) { - /* last disk without the by two multiplication */ - asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i])); - - asm volatile ("movdqa %xmm4,%xmm0"); - asm volatile ("movdqa %%xmm4,%0" : "=m" (pd[0])); - - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); - asm volatile ("movdqa %xmm4,%xmm5"); - asm volatile ("psrlw $4,%xmm5"); - asm volatile ("pand %xmm7,%xmm4"); - asm volatile ("pand %xmm7,%xmm5"); - - asm volatile ("movdqa %0,%%xmm1" : : "m" (gfgenpshufb[l][1][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][1][1][0])); - asm volatile ("pshufb %xmm4,%xmm1"); - asm volatile ("pshufb %xmm5,%xmm7"); - asm volatile ("pxor %xmm7,%xmm1"); - - asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][2][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][2][1][0])); - asm volatile ("pshufb %xmm4,%xmm2"); - asm volatile ("pshufb %xmm5,%xmm7"); - asm volatile ("pxor %xmm7,%xmm2"); - - asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][3][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][3][1][0])); - asm volatile ("pshufb %xmm4,%xmm3"); - asm volatile ("pshufb %xmm5,%xmm7"); - asm volatile ("pxor %xmm7,%xmm3"); - - /* intermediate disks */ - for (d = l - 1; d > 0; --d) { - asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); - asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); - - asm volatile ("pxor %xmm5,%xmm5"); - asm volatile ("pcmpgtb %xmm0,%xmm5"); - asm volatile ("paddb %xmm0,%xmm0"); - asm volatile ("pand %xmm7,%xmm5"); - asm volatile ("pxor %xmm5,%xmm0"); - - asm volatile ("pxor %xmm4,%xmm0"); - asm volatile ("pxor %xmm4,%xmm6"); - asm volatile ("movdqa %%xmm6,%0" : "=m" (pd[0])); - - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); - asm volatile ("movdqa %xmm4,%xmm5"); - asm volatile ("psrlw $4,%xmm5"); - asm volatile ("pand %xmm7,%xmm4"); - asm volatile ("pand %xmm7,%xmm5"); - - asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][1][1][0])); - asm volatile ("pshufb %xmm4,%xmm6"); - asm volatile ("pshufb %xmm5,%xmm7"); - asm volatile ("pxor %xmm6,%xmm1"); - asm volatile ("pxor %xmm7,%xmm1"); - - asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][2][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][2][1][0])); - asm volatile ("pshufb %xmm4,%xmm6"); - asm volatile ("pshufb %xmm5,%xmm7"); - asm volatile ("pxor %xmm6,%xmm2"); - asm volatile ("pxor %xmm7,%xmm2"); - - asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][3][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][3][1][0])); - asm volatile ("pshufb %xmm4,%xmm6"); - asm volatile ("pshufb %xmm5,%xmm7"); - asm volatile ("pxor %xmm6,%xmm3"); - asm volatile ("pxor %xmm7,%xmm3"); - } - - /* first disk with all coefficients at 1 */ - asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i])); - asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); - - asm volatile ("pxor %xmm5,%xmm5"); - asm volatile ("pcmpgtb %xmm0,%xmm5"); - asm volatile ("paddb %xmm0,%xmm0"); - asm volatile ("pand %xmm7,%xmm5"); - asm volatile ("pxor %xmm5,%xmm0"); - - asm volatile ("pxor %xmm4,%xmm0"); - asm volatile ("pxor %xmm4,%xmm1"); - asm volatile ("pxor %xmm4,%xmm2"); - asm volatile ("pxor %xmm4,%xmm3"); - asm volatile ("pxor %xmm4,%xmm6"); - - asm volatile ("movntdq %%xmm6,%0" : "=m" (p[i])); - asm volatile ("movntdq %%xmm0,%0" : "=m" (q[i])); - asm volatile ("movntdq %%xmm1,%0" : "=m" (r[i])); - asm volatile ("movntdq %%xmm2,%0" : "=m" (s[i])); - asm volatile ("movntdq %%xmm3,%0" : "=m" (t[i])); - } - - raid_sse_end(); -} - -#ifdef CONFIG_X86_64 -/* - * GEN5 (penta parity with Cauchy matrix) SSSE3 implementation - * - * Note that it uses 16 registers, meaning that x64 is required. - */ -void raid_gen5_ssse3ext(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *q; - uint8_t *r; - uint8_t *s; - uint8_t *t; - int d, l; - size_t i; - - l = nd - 1; - p = v[nd]; - q = v[nd + 1]; - r = v[nd + 2]; - s = v[nd + 3]; - t = v[nd + 4]; - - /* special case with only one data disk */ - if (l == 0) { - for (i = 0; i < 5; ++i) - memcpy(v[1 + i], v[0], size); - return; - } - - raid_sse_begin(); - - /* generic case with at least two data disks */ - asm volatile ("movdqa %0,%%xmm14" : : "m" (gfconst16.poly[0])); - asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); - - for (i = 0; i < size; i += 16) { - /* last disk without the by two multiplication */ - asm volatile ("movdqa %0,%%xmm10" : : "m" (v[l][i])); - - asm volatile ("movdqa %xmm10,%xmm0"); - asm volatile ("movdqa %xmm10,%xmm1"); - - asm volatile ("movdqa %xmm10,%xmm11"); - asm volatile ("psrlw $4,%xmm11"); - asm volatile ("pand %xmm15,%xmm10"); - asm volatile ("pand %xmm15,%xmm11"); - - asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][1][0][0])); - asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][1][1][0])); - asm volatile ("pshufb %xmm10,%xmm2"); - asm volatile ("pshufb %xmm11,%xmm13"); - asm volatile ("pxor %xmm13,%xmm2"); - - asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][2][0][0])); - asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][2][1][0])); - asm volatile ("pshufb %xmm10,%xmm3"); - asm volatile ("pshufb %xmm11,%xmm13"); - asm volatile ("pxor %xmm13,%xmm3"); - - asm volatile ("movdqa %0,%%xmm4" : : "m" (gfgenpshufb[l][3][0][0])); - asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][3][1][0])); - asm volatile ("pshufb %xmm10,%xmm4"); - asm volatile ("pshufb %xmm11,%xmm13"); - asm volatile ("pxor %xmm13,%xmm4"); - - /* intermediate disks */ - for (d = l - 1; d > 0; --d) { - asm volatile ("movdqa %0,%%xmm10" : : "m" (v[d][i])); - - asm volatile ("pxor %xmm11,%xmm11"); - asm volatile ("pcmpgtb %xmm1,%xmm11"); - asm volatile ("paddb %xmm1,%xmm1"); - asm volatile ("pand %xmm14,%xmm11"); - asm volatile ("pxor %xmm11,%xmm1"); - - asm volatile ("pxor %xmm10,%xmm0"); - asm volatile ("pxor %xmm10,%xmm1"); - - asm volatile ("movdqa %xmm10,%xmm11"); - asm volatile ("psrlw $4,%xmm11"); - asm volatile ("pand %xmm15,%xmm10"); - asm volatile ("pand %xmm15,%xmm11"); - - asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][1][0][0])); - asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][1][1][0])); - asm volatile ("pshufb %xmm10,%xmm12"); - asm volatile ("pshufb %xmm11,%xmm13"); - asm volatile ("pxor %xmm12,%xmm2"); - asm volatile ("pxor %xmm13,%xmm2"); - - asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][2][0][0])); - asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][2][1][0])); - asm volatile ("pshufb %xmm10,%xmm12"); - asm volatile ("pshufb %xmm11,%xmm13"); - asm volatile ("pxor %xmm12,%xmm3"); - asm volatile ("pxor %xmm13,%xmm3"); - - asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][3][0][0])); - asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][3][1][0])); - asm volatile ("pshufb %xmm10,%xmm12"); - asm volatile ("pshufb %xmm11,%xmm13"); - asm volatile ("pxor %xmm12,%xmm4"); - asm volatile ("pxor %xmm13,%xmm4"); - } - - /* first disk with all coefficients at 1 */ - asm volatile ("movdqa %0,%%xmm10" : : "m" (v[0][i])); - - asm volatile ("pxor %xmm11,%xmm11"); - asm volatile ("pcmpgtb %xmm1,%xmm11"); - asm volatile ("paddb %xmm1,%xmm1"); - asm volatile ("pand %xmm14,%xmm11"); - asm volatile ("pxor %xmm11,%xmm1"); - - asm volatile ("pxor %xmm10,%xmm0"); - asm volatile ("pxor %xmm10,%xmm1"); - asm volatile ("pxor %xmm10,%xmm2"); - asm volatile ("pxor %xmm10,%xmm3"); - asm volatile ("pxor %xmm10,%xmm4"); - - asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); - asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); - asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); - asm volatile ("movntdq %%xmm3,%0" : "=m" (s[i])); - asm volatile ("movntdq %%xmm4,%0" : "=m" (t[i])); - } - - raid_sse_end(); -} -#endif - -#ifdef CONFIG_X86_64 -/* - * GEN5 (penta parity with Cauchy matrix) AVX2 implementation - * - * Note that it uses 16 registers, meaning that x64 is required. - */ -void raid_gen5_avx2ext(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *q; - uint8_t *r; - uint8_t *s; - uint8_t *t; - int d, l; - size_t i; - - l = nd - 1; - p = v[nd]; - q = v[nd + 1]; - r = v[nd + 2]; - s = v[nd + 3]; - t = v[nd + 4]; - - /* special case with only one data disk */ - if (l == 0) { - for (i = 0; i < 5; ++i) - memcpy(v[1 + i], v[0], size); - return; - } - - raid_avx_begin(); - - /* generic case with at least two data disks */ - asm volatile ("vpxor %ymm6,%ymm6,%ymm6"); - asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfconst16.poly[0])); - asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0])); - - for (i = 0; i < size; i += 32) { - /* last disk without the by two multiplication */ - asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[l][i])); - - asm volatile ("vmovdqa %ymm10,%ymm0"); - asm volatile ("vmovdqa %ymm10,%ymm1"); - - asm volatile ("vpsrlw $4,%ymm10,%ymm11"); - asm volatile ("vpand %ymm15,%ymm10,%ymm10"); - asm volatile ("vpand %ymm15,%ymm11,%ymm11"); - - asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfgenpshufb[l][1][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][1][1][0])); - asm volatile ("vpshufb %ymm10,%ymm2,%ymm2"); - asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm13,%ymm2,%ymm2"); - - asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfgenpshufb[l][2][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][2][1][0])); - asm volatile ("vpshufb %ymm10,%ymm3,%ymm3"); - asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm13,%ymm3,%ymm3"); - - asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (gfgenpshufb[l][3][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][3][1][0])); - asm volatile ("vpshufb %ymm10,%ymm4,%ymm4"); - asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm13,%ymm4,%ymm4"); - - /* intermediate disks */ - for (d = l - 1; d > 0; --d) { - asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[d][i])); - - asm volatile ("vpcmpgtb %ymm1,%ymm6,%ymm11"); - asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); - asm volatile ("vpand %ymm14,%ymm11,%ymm11"); - asm volatile ("vpxor %ymm11,%ymm1,%ymm1"); - - asm volatile ("vpxor %ymm10,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm10,%ymm1,%ymm1"); - - asm volatile ("vpsrlw $4,%ymm10,%ymm11"); - asm volatile ("vpand %ymm15,%ymm10,%ymm10"); - asm volatile ("vpand %ymm15,%ymm11,%ymm11"); - - asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][1][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][1][1][0])); - asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); - asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm12,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm13,%ymm2,%ymm2"); - - asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][2][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][2][1][0])); - asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); - asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm12,%ymm3,%ymm3"); - asm volatile ("vpxor %ymm13,%ymm3,%ymm3"); - - asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][3][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][3][1][0])); - asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); - asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm12,%ymm4,%ymm4"); - asm volatile ("vpxor %ymm13,%ymm4,%ymm4"); - } - - /* first disk with all coefficients at 1 */ - asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[0][i])); - - asm volatile ("vpcmpgtb %ymm1,%ymm6,%ymm11"); - asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); - asm volatile ("vpand %ymm14,%ymm11,%ymm11"); - asm volatile ("vpxor %ymm11,%ymm1,%ymm1"); - - asm volatile ("vpxor %ymm10,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm10,%ymm1,%ymm1"); - asm volatile ("vpxor %ymm10,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm10,%ymm3,%ymm3"); - asm volatile ("vpxor %ymm10,%ymm4,%ymm4"); - - asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); - asm volatile ("vmovntdq %%ymm1,%0" : "=m" (q[i])); - asm volatile ("vmovntdq %%ymm2,%0" : "=m" (r[i])); - asm volatile ("vmovntdq %%ymm3,%0" : "=m" (s[i])); - asm volatile ("vmovntdq %%ymm4,%0" : "=m" (t[i])); - } - - raid_avx_end(); -} -#endif - -#ifdef CONFIG_X86_64 -/* - * GEN5 (penta parity with Cauchy matrix) AVX512BW implementation - */ -void raid_gen5_avx512bw(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *q; - uint8_t *r; - uint8_t *s; - uint8_t *t; - int d, l; - size_t i; - - l = nd - 1; - p = v[nd]; - q = v[nd + 1]; - r = v[nd + 2]; - s = v[nd + 3]; - t = v[nd + 4]; - - raid_avx_begin(); - - asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gfconst16.low4[0])); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); - asm volatile ("vmovdqa64 %zmm0,%zmm1"); - asm volatile ("vmovdqa64 %zmm0,%zmm2"); - asm volatile ("vmovdqa64 %zmm0,%zmm3"); - asm volatile ("vmovdqa64 %zmm0,%zmm4"); - - for (d = 1; d <= l; ++d) { - asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (v[d][i])); - - asm volatile ("vpxorq %zmm10,%zmm0,%zmm0"); - - asm volatile ("vpsrlw $4,%zmm10,%zmm11"); - asm volatile ("vpandq %zmm31,%zmm10,%zmm10"); - asm volatile ("vpandq %zmm31,%zmm11,%zmm11"); - - asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][0][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][0][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (gfgenpshufb[d][1][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (gfgenpshufb[d][1][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm16" : : "m" (gfgenpshufb[d][2][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm17" : : "m" (gfgenpshufb[d][2][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm18" : : "m" (gfgenpshufb[d][3][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm19" : : "m" (gfgenpshufb[d][3][1][0])); - asm volatile ("vpshufb %zmm10,%zmm12,%zmm12"); - asm volatile ("vpshufb %zmm11,%zmm13,%zmm13"); - asm volatile ("vpshufb %zmm10,%zmm14,%zmm14"); - asm volatile ("vpshufb %zmm11,%zmm15,%zmm15"); - asm volatile ("vpshufb %zmm10,%zmm16,%zmm16"); - asm volatile ("vpshufb %zmm11,%zmm17,%zmm17"); - asm volatile ("vpshufb %zmm10,%zmm18,%zmm18"); - asm volatile ("vpshufb %zmm11,%zmm19,%zmm19"); - asm volatile ("vpternlogq $0x96,%zmm12,%zmm13,%zmm1"); - asm volatile ("vpternlogq $0x96,%zmm14,%zmm15,%zmm2"); - asm volatile ("vpternlogq $0x96,%zmm16,%zmm17,%zmm3"); - asm volatile ("vpternlogq $0x96,%zmm18,%zmm19,%zmm4"); - } - - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i])); - asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i])); - asm volatile ("vmovntdq %%zmm2,%0" : "=m" (r[i])); - asm volatile ("vmovntdq %%zmm3,%0" : "=m" (s[i])); - asm volatile ("vmovntdq %%zmm4,%0" : "=m" (t[i])); - } - - raid_avx_end(); -} -#endif - -/* - * GEN6 (hexa parity with Cauchy matrix) SSSE3 implementation - */ -void raid_gen6_ssse3(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *q; - uint8_t *r; - uint8_t *s; - uint8_t *t; - uint8_t *u; - int d, l; - size_t i; - uint8_t buffer[2*16+16]; - uint8_t *pd = __align_ptr(buffer, 16); - - l = nd - 1; - p = v[nd]; - q = v[nd + 1]; - r = v[nd + 2]; - s = v[nd + 3]; - t = v[nd + 4]; - u = v[nd + 5]; - - /* special case with only one data disk */ - if (l == 0) { - for (i = 0; i < 6; ++i) - memcpy(v[1 + i], v[0], size); - return; - } - - raid_sse_begin(); - - /* generic case with at least two data disks */ - for (i = 0; i < size; i += 16) { - /* last disk without the by two multiplication */ - asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i])); - - asm volatile ("movdqa %%xmm4,%0" : "=m" (pd[0])); - asm volatile ("movdqa %%xmm4,%0" : "=m" (pd[16])); - - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); - asm volatile ("movdqa %xmm4,%xmm5"); - asm volatile ("psrlw $4,%xmm5"); - asm volatile ("pand %xmm7,%xmm4"); - asm volatile ("pand %xmm7,%xmm5"); - - asm volatile ("movdqa %0,%%xmm0" : : "m" (gfgenpshufb[l][1][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][1][1][0])); - asm volatile ("pshufb %xmm4,%xmm0"); - asm volatile ("pshufb %xmm5,%xmm7"); - asm volatile ("pxor %xmm7,%xmm0"); - - asm volatile ("movdqa %0,%%xmm1" : : "m" (gfgenpshufb[l][2][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][2][1][0])); - asm volatile ("pshufb %xmm4,%xmm1"); - asm volatile ("pshufb %xmm5,%xmm7"); - asm volatile ("pxor %xmm7,%xmm1"); - - asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][3][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][3][1][0])); - asm volatile ("pshufb %xmm4,%xmm2"); - asm volatile ("pshufb %xmm5,%xmm7"); - asm volatile ("pxor %xmm7,%xmm2"); - - asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][4][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][4][1][0])); - asm volatile ("pshufb %xmm4,%xmm3"); - asm volatile ("pshufb %xmm5,%xmm7"); - asm volatile ("pxor %xmm7,%xmm3"); - - /* intermediate disks */ - for (d = l - 1; d > 0; --d) { - asm volatile ("movdqa %0,%%xmm5" : : "m" (pd[0])); - asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[16])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); - - asm volatile ("pxor %xmm4,%xmm4"); - asm volatile ("pcmpgtb %xmm6,%xmm4"); - asm volatile ("paddb %xmm6,%xmm6"); - asm volatile ("pand %xmm7,%xmm4"); - asm volatile ("pxor %xmm4,%xmm6"); - - asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); - - asm volatile ("pxor %xmm4,%xmm5"); - asm volatile ("pxor %xmm4,%xmm6"); - asm volatile ("movdqa %%xmm5,%0" : "=m" (pd[0])); - asm volatile ("movdqa %%xmm6,%0" : "=m" (pd[16])); - - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); - asm volatile ("movdqa %xmm4,%xmm5"); - asm volatile ("psrlw $4,%xmm5"); - asm volatile ("pand %xmm7,%xmm4"); - asm volatile ("pand %xmm7,%xmm5"); - - asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][1][1][0])); - asm volatile ("pshufb %xmm4,%xmm6"); - asm volatile ("pshufb %xmm5,%xmm7"); - asm volatile ("pxor %xmm6,%xmm0"); - asm volatile ("pxor %xmm7,%xmm0"); - - asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][2][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][2][1][0])); - asm volatile ("pshufb %xmm4,%xmm6"); - asm volatile ("pshufb %xmm5,%xmm7"); - asm volatile ("pxor %xmm6,%xmm1"); - asm volatile ("pxor %xmm7,%xmm1"); - - asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][3][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][3][1][0])); - asm volatile ("pshufb %xmm4,%xmm6"); - asm volatile ("pshufb %xmm5,%xmm7"); - asm volatile ("pxor %xmm6,%xmm2"); - asm volatile ("pxor %xmm7,%xmm2"); - - asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][4][0][0])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][4][1][0])); - asm volatile ("pshufb %xmm4,%xmm6"); - asm volatile ("pshufb %xmm5,%xmm7"); - asm volatile ("pxor %xmm6,%xmm3"); - asm volatile ("pxor %xmm7,%xmm3"); - } - - /* first disk with all coefficients at 1 */ - asm volatile ("movdqa %0,%%xmm5" : : "m" (pd[0])); - asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[16])); - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); - - asm volatile ("pxor %xmm4,%xmm4"); - asm volatile ("pcmpgtb %xmm6,%xmm4"); - asm volatile ("paddb %xmm6,%xmm6"); - asm volatile ("pand %xmm7,%xmm4"); - asm volatile ("pxor %xmm4,%xmm6"); - - asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i])); - asm volatile ("pxor %xmm4,%xmm0"); - asm volatile ("pxor %xmm4,%xmm1"); - asm volatile ("pxor %xmm4,%xmm2"); - asm volatile ("pxor %xmm4,%xmm3"); - asm volatile ("pxor %xmm4,%xmm5"); - asm volatile ("pxor %xmm4,%xmm6"); - - asm volatile ("movntdq %%xmm5,%0" : "=m" (p[i])); - asm volatile ("movntdq %%xmm6,%0" : "=m" (q[i])); - asm volatile ("movntdq %%xmm0,%0" : "=m" (r[i])); - asm volatile ("movntdq %%xmm1,%0" : "=m" (s[i])); - asm volatile ("movntdq %%xmm2,%0" : "=m" (t[i])); - asm volatile ("movntdq %%xmm3,%0" : "=m" (u[i])); - } - - raid_sse_end(); -} - -#ifdef CONFIG_X86_64 -/* - * GEN6 (hexa parity with Cauchy matrix) SSSE3 implementation - * - * Note that it uses 16 registers, meaning that x64 is required. - */ -void raid_gen6_ssse3ext(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *q; - uint8_t *r; - uint8_t *s; - uint8_t *t; - uint8_t *u; - int d, l; - size_t i; - - l = nd - 1; - p = v[nd]; - q = v[nd + 1]; - r = v[nd + 2]; - s = v[nd + 3]; - t = v[nd + 4]; - u = v[nd + 5]; - - /* special case with only one data disk */ - if (l == 0) { - for (i = 0; i < 6; ++i) - memcpy(v[1 + i], v[0], size); - return; - } - - raid_sse_begin(); - - /* generic case with at least two data disks */ - asm volatile ("movdqa %0,%%xmm14" : : "m" (gfconst16.poly[0])); - asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); - - for (i = 0; i < size; i += 16) { - /* last disk without the by two multiplication */ - asm volatile ("movdqa %0,%%xmm10" : : "m" (v[l][i])); - - asm volatile ("movdqa %xmm10,%xmm0"); - asm volatile ("movdqa %xmm10,%xmm1"); - - asm volatile ("movdqa %xmm10,%xmm11"); - asm volatile ("psrlw $4,%xmm11"); - asm volatile ("pand %xmm15,%xmm10"); - asm volatile ("pand %xmm15,%xmm11"); - - asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][1][0][0])); - asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][1][1][0])); - asm volatile ("pshufb %xmm10,%xmm2"); - asm volatile ("pshufb %xmm11,%xmm13"); - asm volatile ("pxor %xmm13,%xmm2"); - - asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][2][0][0])); - asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][2][1][0])); - asm volatile ("pshufb %xmm10,%xmm3"); - asm volatile ("pshufb %xmm11,%xmm13"); - asm volatile ("pxor %xmm13,%xmm3"); - - asm volatile ("movdqa %0,%%xmm4" : : "m" (gfgenpshufb[l][3][0][0])); - asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][3][1][0])); - asm volatile ("pshufb %xmm10,%xmm4"); - asm volatile ("pshufb %xmm11,%xmm13"); - asm volatile ("pxor %xmm13,%xmm4"); - - asm volatile ("movdqa %0,%%xmm5" : : "m" (gfgenpshufb[l][4][0][0])); - asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][4][1][0])); - asm volatile ("pshufb %xmm10,%xmm5"); - asm volatile ("pshufb %xmm11,%xmm13"); - asm volatile ("pxor %xmm13,%xmm5"); - - /* intermediate disks */ - for (d = l - 1; d > 0; --d) { - asm volatile ("movdqa %0,%%xmm10" : : "m" (v[d][i])); - - asm volatile ("pxor %xmm11,%xmm11"); - asm volatile ("pcmpgtb %xmm1,%xmm11"); - asm volatile ("paddb %xmm1,%xmm1"); - asm volatile ("pand %xmm14,%xmm11"); - asm volatile ("pxor %xmm11,%xmm1"); - - asm volatile ("pxor %xmm10,%xmm0"); - asm volatile ("pxor %xmm10,%xmm1"); - - asm volatile ("movdqa %xmm10,%xmm11"); - asm volatile ("psrlw $4,%xmm11"); - asm volatile ("pand %xmm15,%xmm10"); - asm volatile ("pand %xmm15,%xmm11"); - - asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][1][0][0])); - asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][1][1][0])); - asm volatile ("pshufb %xmm10,%xmm12"); - asm volatile ("pshufb %xmm11,%xmm13"); - asm volatile ("pxor %xmm12,%xmm2"); - asm volatile ("pxor %xmm13,%xmm2"); - - asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][2][0][0])); - asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][2][1][0])); - asm volatile ("pshufb %xmm10,%xmm12"); - asm volatile ("pshufb %xmm11,%xmm13"); - asm volatile ("pxor %xmm12,%xmm3"); - asm volatile ("pxor %xmm13,%xmm3"); - - asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][3][0][0])); - asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][3][1][0])); - asm volatile ("pshufb %xmm10,%xmm12"); - asm volatile ("pshufb %xmm11,%xmm13"); - asm volatile ("pxor %xmm12,%xmm4"); - asm volatile ("pxor %xmm13,%xmm4"); - - asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][4][0][0])); - asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][4][1][0])); - asm volatile ("pshufb %xmm10,%xmm12"); - asm volatile ("pshufb %xmm11,%xmm13"); - asm volatile ("pxor %xmm12,%xmm5"); - asm volatile ("pxor %xmm13,%xmm5"); - } - - /* first disk with all coefficients at 1 */ - asm volatile ("movdqa %0,%%xmm10" : : "m" (v[0][i])); - - asm volatile ("pxor %xmm11,%xmm11"); - asm volatile ("pcmpgtb %xmm1,%xmm11"); - asm volatile ("paddb %xmm1,%xmm1"); - asm volatile ("pand %xmm14,%xmm11"); - asm volatile ("pxor %xmm11,%xmm1"); - - asm volatile ("pxor %xmm10,%xmm0"); - asm volatile ("pxor %xmm10,%xmm1"); - asm volatile ("pxor %xmm10,%xmm2"); - asm volatile ("pxor %xmm10,%xmm3"); - asm volatile ("pxor %xmm10,%xmm4"); - asm volatile ("pxor %xmm10,%xmm5"); - - asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); - asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); - asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); - asm volatile ("movntdq %%xmm3,%0" : "=m" (s[i])); - asm volatile ("movntdq %%xmm4,%0" : "=m" (t[i])); - asm volatile ("movntdq %%xmm5,%0" : "=m" (u[i])); - } - - raid_sse_end(); -} -#endif - -#ifdef CONFIG_X86_64 -/* - * GEN6 (hexa parity with Cauchy matrix) AVX2 implementation - * - * Note that it uses 16 registers, meaning that x64 is required. - */ -void raid_gen6_avx2ext(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *q; - uint8_t *r; - uint8_t *s; - uint8_t *t; - uint8_t *u; - int d, l; - size_t i; - - l = nd - 1; - p = v[nd]; - q = v[nd + 1]; - r = v[nd + 2]; - s = v[nd + 3]; - t = v[nd + 4]; - u = v[nd + 5]; - - /* special case with only one data disk */ - if (l == 0) { - for (i = 0; i < 6; ++i) - memcpy(v[1 + i], v[0], size); - return; - } - - raid_avx_begin(); - - /* - * Register mapping: - * - * 0,1,2,3,4,5 PARITY - * 6 ZERO - * 10,11 MUL BY 2 - * 12,13 PSHUFB - * 14 POLY - * 15 LOW4 - */ - - /* generic case with at least two data disks */ - asm volatile ("vpxor %ymm6,%ymm6,%ymm6"); - asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfconst16.poly[0])); - asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0])); - - for (i = 0; i < size; i += 32) { - /* last disk without the by two multiplication */ - asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[l][i])); - - asm volatile ("vmovdqa %ymm10,%ymm0"); - asm volatile ("vmovdqa %ymm10,%ymm1"); - - asm volatile ("vpsrlw $4,%ymm10,%ymm11"); - asm volatile ("vpand %ymm15,%ymm10,%ymm10"); - asm volatile ("vpand %ymm15,%ymm11,%ymm11"); - - asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfgenpshufb[l][1][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][1][1][0])); - asm volatile ("vpshufb %ymm10,%ymm2,%ymm2"); - asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm13,%ymm2,%ymm2"); - - asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfgenpshufb[l][2][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][2][1][0])); - asm volatile ("vpshufb %ymm10,%ymm3,%ymm3"); - asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm13,%ymm3,%ymm3"); - - asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (gfgenpshufb[l][3][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][3][1][0])); - asm volatile ("vpshufb %ymm10,%ymm4,%ymm4"); - asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm13,%ymm4,%ymm4"); - - asm volatile ("vbroadcasti128 %0,%%ymm5" : : "m" (gfgenpshufb[l][4][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][4][1][0])); - asm volatile ("vpshufb %ymm10,%ymm5,%ymm5"); - asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm13,%ymm5,%ymm5"); - - /* intermediate disks */ - for (d = l - 1; d > 0; --d) { - asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[d][i])); - - asm volatile ("vpcmpgtb %ymm1,%ymm6,%ymm11"); - asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); - asm volatile ("vpand %ymm14,%ymm11,%ymm11"); - asm volatile ("vpxor %ymm11,%ymm1,%ymm1"); - - asm volatile ("vpxor %ymm10,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm10,%ymm1,%ymm1"); - - asm volatile ("vpsrlw $4,%ymm10,%ymm11"); - asm volatile ("vpand %ymm15,%ymm10,%ymm10"); - asm volatile ("vpand %ymm15,%ymm11,%ymm11"); - - asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][1][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][1][1][0])); - asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); - asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm12,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm13,%ymm2,%ymm2"); - - asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][2][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][2][1][0])); - asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); - asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm12,%ymm3,%ymm3"); - asm volatile ("vpxor %ymm13,%ymm3,%ymm3"); - - asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][3][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][3][1][0])); - asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); - asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm12,%ymm4,%ymm4"); - asm volatile ("vpxor %ymm13,%ymm4,%ymm4"); - - asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][4][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][4][1][0])); - asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); - asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm12,%ymm5,%ymm5"); - asm volatile ("vpxor %ymm13,%ymm5,%ymm5"); - } - - /* first disk with all coefficients at 1 */ - asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[0][i])); - - asm volatile ("vpcmpgtb %ymm1,%ymm6,%ymm11"); - asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); - asm volatile ("vpand %ymm14,%ymm11,%ymm11"); - asm volatile ("vpxor %ymm11,%ymm1,%ymm1"); - - asm volatile ("vpxor %ymm10,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm10,%ymm1,%ymm1"); - asm volatile ("vpxor %ymm10,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm10,%ymm3,%ymm3"); - asm volatile ("vpxor %ymm10,%ymm4,%ymm4"); - asm volatile ("vpxor %ymm10,%ymm5,%ymm5"); - - asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); - asm volatile ("vmovntdq %%ymm1,%0" : "=m" (q[i])); - asm volatile ("vmovntdq %%ymm2,%0" : "=m" (r[i])); - asm volatile ("vmovntdq %%ymm3,%0" : "=m" (s[i])); - asm volatile ("vmovntdq %%ymm4,%0" : "=m" (t[i])); - asm volatile ("vmovntdq %%ymm5,%0" : "=m" (u[i])); - } - - raid_avx_end(); -} -#endif - -#ifdef CONFIG_X86_64 -/* - * GEN6 (hexa parity with Cauchy matrix) AVX512BW implementation - */ -void raid_gen6_avx512bw(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *q; - uint8_t *r; - uint8_t *s; - uint8_t *t; - uint8_t *u; - int d, l; - size_t i; - - l = nd - 1; - p = v[nd]; - q = v[nd + 1]; - r = v[nd + 2]; - s = v[nd + 3]; - t = v[nd + 4]; - u = v[nd + 5]; - - raid_avx_begin(); - - asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gfconst16.low4[0])); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); - asm volatile ("vmovdqa64 %zmm0,%zmm1"); - asm volatile ("vmovdqa64 %zmm0,%zmm2"); - asm volatile ("vmovdqa64 %zmm0,%zmm3"); - asm volatile ("vmovdqa64 %zmm0,%zmm4"); - asm volatile ("vmovdqa64 %zmm0,%zmm5"); - - for (d = 1; d <= l; ++d) { - asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (v[d][i])); - - asm volatile ("vpxorq %zmm10,%zmm0,%zmm0"); - - asm volatile ("vpsrlw $4,%zmm10,%zmm11"); - asm volatile ("vpandq %zmm31,%zmm10,%zmm10"); - asm volatile ("vpandq %zmm31,%zmm11,%zmm11"); - - asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][0][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][0][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (gfgenpshufb[d][1][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (gfgenpshufb[d][1][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm16" : : "m" (gfgenpshufb[d][2][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm17" : : "m" (gfgenpshufb[d][2][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm18" : : "m" (gfgenpshufb[d][3][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm19" : : "m" (gfgenpshufb[d][3][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm20" : : "m" (gfgenpshufb[d][4][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm21" : : "m" (gfgenpshufb[d][4][1][0])); - asm volatile ("vpshufb %zmm10,%zmm12,%zmm12"); - asm volatile ("vpshufb %zmm11,%zmm13,%zmm13"); - asm volatile ("vpshufb %zmm10,%zmm14,%zmm14"); - asm volatile ("vpshufb %zmm11,%zmm15,%zmm15"); - asm volatile ("vpshufb %zmm10,%zmm16,%zmm16"); - asm volatile ("vpshufb %zmm11,%zmm17,%zmm17"); - asm volatile ("vpshufb %zmm10,%zmm18,%zmm18"); - asm volatile ("vpshufb %zmm11,%zmm19,%zmm19"); - asm volatile ("vpshufb %zmm10,%zmm20,%zmm20"); - asm volatile ("vpshufb %zmm11,%zmm21,%zmm21"); - asm volatile ("vpternlogq $0x96,%zmm12,%zmm13,%zmm1"); - asm volatile ("vpternlogq $0x96,%zmm14,%zmm15,%zmm2"); - asm volatile ("vpternlogq $0x96,%zmm16,%zmm17,%zmm3"); - asm volatile ("vpternlogq $0x96,%zmm18,%zmm19,%zmm4"); - asm volatile ("vpternlogq $0x96,%zmm20,%zmm21,%zmm5"); - } - - asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i])); - asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i])); - asm volatile ("vmovntdq %%zmm2,%0" : "=m" (r[i])); - asm volatile ("vmovntdq %%zmm3,%0" : "=m" (s[i])); - asm volatile ("vmovntdq %%zmm4,%0" : "=m" (t[i])); - asm volatile ("vmovntdq %%zmm5,%0" : "=m" (u[i])); - } - - raid_avx_end(); -} -#endif - -/* - * RAID recovering for one disk SSSE3 implementation - */ -void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *pa; - uint8_t G; - uint8_t V; - size_t i; - - (void)nr; /* unused, it's always 1 */ - - /* if it's RAID5 uses the faster function */ - if (ip[0] == 0) { - raid_rec1of1(id, nd, size, vv); - return; - } - - /* setup the coefficients matrix */ - G = A(ip[0], id[0]); - - /* invert it to solve the system of linear equations */ - V = inv(G); - - /* compute delta parity */ - raid_delta_gen(1, id, ip, nd, size, vv); - - p = v[nd + ip[0]]; - pa = v[id[0]]; - - raid_sse_begin(); - - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); - asm volatile ("movdqa %0,%%xmm4" : : "m" (gfmulpshufb[V][0][0])); - asm volatile ("movdqa %0,%%xmm5" : : "m" (gfmulpshufb[V][1][0])); - - for (i = 0; i < size; i += 16) { - asm volatile ("movdqa %0,%%xmm0" : : "m" (p[i])); - asm volatile ("movdqa %0,%%xmm1" : : "m" (pa[i])); - asm volatile ("movdqa %xmm4,%xmm2"); - asm volatile ("movdqa %xmm5,%xmm3"); - asm volatile ("pxor %xmm0,%xmm1"); - asm volatile ("movdqa %xmm1,%xmm0"); - asm volatile ("psrlw $4,%xmm1"); - asm volatile ("pand %xmm7,%xmm0"); - asm volatile ("pand %xmm7,%xmm1"); - asm volatile ("pshufb %xmm0,%xmm2"); - asm volatile ("pshufb %xmm1,%xmm3"); - asm volatile ("pxor %xmm3,%xmm2"); - asm volatile ("movdqa %%xmm2,%0" : "=m" (pa[i])); - } - - raid_sse_end(); -} - -/* - * RAID recovering for two disks SSSE3 implementation - */ -void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - const int N = 2; - uint8_t *p[N]; - uint8_t *pa[N]; - uint8_t G[N * N]; - uint8_t V[N * N]; - size_t i; - int j, k; - - (void)nr; /* unused, it's always 2 */ - - /* setup the coefficients matrix */ - for (j = 0; j < N; ++j) - for (k = 0; k < N; ++k) - G[j * N + k] = A(ip[j], id[k]); - - /* invert it to solve the system of linear equations */ - raid_invert(G, V, N); - - /* compute delta parity */ - raid_delta_gen(N, id, ip, nd, size, vv); - - for (j = 0; j < N; ++j) { - p[j] = v[nd + ip[j]]; - pa[j] = v[id[j]]; - } - - raid_sse_begin(); - - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); - - for (i = 0; i < size; i += 16) { - asm volatile ("movdqa %0,%%xmm0" : : "m" (p[0][i])); - asm volatile ("movdqa %0,%%xmm2" : : "m" (pa[0][i])); - asm volatile ("movdqa %0,%%xmm1" : : "m" (p[1][i])); - asm volatile ("movdqa %0,%%xmm3" : : "m" (pa[1][i])); - asm volatile ("pxor %xmm2,%xmm0"); - asm volatile ("pxor %xmm3,%xmm1"); - - asm volatile ("pxor %xmm6,%xmm6"); - - asm volatile ("movdqa %0,%%xmm2" : : "m" (gfmulpshufb[V[0]][0][0])); - asm volatile ("movdqa %0,%%xmm3" : : "m" (gfmulpshufb[V[0]][1][0])); - asm volatile ("movdqa %xmm0,%xmm4"); - asm volatile ("movdqa %xmm0,%xmm5"); - asm volatile ("psrlw $4,%xmm5"); - asm volatile ("pand %xmm7,%xmm4"); - asm volatile ("pand %xmm7,%xmm5"); - asm volatile ("pshufb %xmm4,%xmm2"); - asm volatile ("pshufb %xmm5,%xmm3"); - asm volatile ("pxor %xmm2,%xmm6"); - asm volatile ("pxor %xmm3,%xmm6"); - - asm volatile ("movdqa %0,%%xmm2" : : "m" (gfmulpshufb[V[1]][0][0])); - asm volatile ("movdqa %0,%%xmm3" : : "m" (gfmulpshufb[V[1]][1][0])); - asm volatile ("movdqa %xmm1,%xmm4"); - asm volatile ("movdqa %xmm1,%xmm5"); - asm volatile ("psrlw $4,%xmm5"); - asm volatile ("pand %xmm7,%xmm4"); - asm volatile ("pand %xmm7,%xmm5"); - asm volatile ("pshufb %xmm4,%xmm2"); - asm volatile ("pshufb %xmm5,%xmm3"); - asm volatile ("pxor %xmm2,%xmm6"); - asm volatile ("pxor %xmm3,%xmm6"); - - asm volatile ("movdqa %%xmm6,%0" : "=m" (pa[0][i])); - - asm volatile ("pxor %xmm6,%xmm6"); - - asm volatile ("movdqa %0,%%xmm2" : : "m" (gfmulpshufb[V[2]][0][0])); - asm volatile ("movdqa %0,%%xmm3" : : "m" (gfmulpshufb[V[2]][1][0])); - asm volatile ("movdqa %xmm0,%xmm4"); - asm volatile ("movdqa %xmm0,%xmm5"); - asm volatile ("psrlw $4,%xmm5"); - asm volatile ("pand %xmm7,%xmm4"); - asm volatile ("pand %xmm7,%xmm5"); - asm volatile ("pshufb %xmm4,%xmm2"); - asm volatile ("pshufb %xmm5,%xmm3"); - asm volatile ("pxor %xmm2,%xmm6"); - asm volatile ("pxor %xmm3,%xmm6"); - - asm volatile ("movdqa %0,%%xmm2" : : "m" (gfmulpshufb[V[3]][0][0])); - asm volatile ("movdqa %0,%%xmm3" : : "m" (gfmulpshufb[V[3]][1][0])); - asm volatile ("movdqa %xmm1,%xmm4"); - asm volatile ("movdqa %xmm1,%xmm5"); - asm volatile ("psrlw $4,%xmm5"); - asm volatile ("pand %xmm7,%xmm4"); - asm volatile ("pand %xmm7,%xmm5"); - asm volatile ("pshufb %xmm4,%xmm2"); - asm volatile ("pshufb %xmm5,%xmm3"); - asm volatile ("pxor %xmm2,%xmm6"); - asm volatile ("pxor %xmm3,%xmm6"); - - asm volatile ("movdqa %%xmm6,%0" : "=m" (pa[1][i])); - } - - raid_sse_end(); -} - -/* - * RAID recovering SSSE3 implementation - */ -void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - int N = nr; - uint8_t *p[RAID_PARITY_MAX]; - uint8_t *pa[RAID_PARITY_MAX]; - uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; - uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; - uint8_t buffer[RAID_PARITY_MAX*16+16]; - uint8_t *pd = __align_ptr(buffer, 16); - size_t i; - int j, k; - - /* setup the coefficients matrix */ - for (j = 0; j < N; ++j) - for (k = 0; k < N; ++k) - G[j * N + k] = A(ip[j], id[k]); - - /* invert it to solve the system of linear equations */ - raid_invert(G, V, N); - - /* compute delta parity */ - raid_delta_gen(N, id, ip, nd, size, vv); - - for (j = 0; j < N; ++j) { - p[j] = v[nd + ip[j]]; - pa[j] = v[id[j]]; - } - - raid_sse_begin(); - - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); - - for (i = 0; i < size; i += 16) { - /* delta */ - for (j = 0; j < N; ++j) { - asm volatile ("movdqa %0,%%xmm0" : : "m" (p[j][i])); - asm volatile ("movdqa %0,%%xmm1" : : "m" (pa[j][i])); - asm volatile ("pxor %xmm1,%xmm0"); - asm volatile ("movdqa %%xmm0,%0" : "=m" (pd[j*16])); - } - - /* reconstruct */ - for (j = 0; j < N; ++j) { - asm volatile ("pxor %xmm0,%xmm0"); - asm volatile ("pxor %xmm1,%xmm1"); - - for (k = 0; k < N; ++k) { - uint8_t m = V[j * N + k]; - - asm volatile ("movdqa %0,%%xmm2" : : "m" (gfmulpshufb[m][0][0])); - asm volatile ("movdqa %0,%%xmm3" : : "m" (gfmulpshufb[m][1][0])); - asm volatile ("movdqa %0,%%xmm4" : : "m" (pd[k*16])); - asm volatile ("movdqa %xmm4,%xmm5"); - asm volatile ("psrlw $4,%xmm5"); - asm volatile ("pand %xmm7,%xmm4"); - asm volatile ("pand %xmm7,%xmm5"); - asm volatile ("pshufb %xmm4,%xmm2"); - asm volatile ("pshufb %xmm5,%xmm3"); - asm volatile ("pxor %xmm2,%xmm0"); - asm volatile ("pxor %xmm3,%xmm1"); - } - - asm volatile ("pxor %xmm1,%xmm0"); - asm volatile ("movdqa %%xmm0,%0" : "=m" (pa[j][i])); - } - } - - raid_sse_end(); -} - -/* - * RAID recovering for one disk AVX2 implementation - */ -void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *pa; - uint8_t G; - uint8_t V; - size_t i; - - (void)nr; /* unused, it's always 1 */ - - /* if it's RAID5 uses the faster function */ - if (ip[0] == 0) { - raid_rec1of1(id, nd, size, vv); - return; - } - - /* setup the coefficients matrix */ - G = A(ip[0], id[0]); - - /* invert it to solve the system of linear equations */ - V = inv(G); - - /* compute delta parity */ - raid_delta_gen(1, id, ip, nd, size, vv); - - p = v[nd + ip[0]]; - pa = v[id[0]]; - - raid_avx_begin(); - - asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.low4[0])); - asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (gfmulpshufb[V][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm5" : : "m" (gfmulpshufb[V][1][0])); - - for (i = 0; i < size; i += 32) { - asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[i])); - asm volatile ("vmovdqa %0,%%ymm1" : : "m" (pa[i])); - asm volatile ("vpxor %ymm1,%ymm0,%ymm0"); - asm volatile ("vpsrlw $4,%ymm0,%ymm1"); - asm volatile ("vpand %ymm7,%ymm0,%ymm0"); - asm volatile ("vpand %ymm7,%ymm1,%ymm1"); - asm volatile ("vpshufb %ymm0,%ymm4,%ymm2"); - asm volatile ("vpshufb %ymm1,%ymm5,%ymm3"); - asm volatile ("vpxor %ymm3,%ymm2,%ymm2"); - asm volatile ("vmovdqa %%ymm2,%0" : "=m" (pa[i])); - } - - raid_avx_end(); -} - -/* - * RAID recovering for two disks AVX2 implementation - */ -void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - const int N = 2; - uint8_t *p[N]; - uint8_t *pa[N]; - uint8_t G[N * N]; - uint8_t V[N * N]; - size_t i; - int j, k; - - (void)nr; /* unused, it's always 2 */ - - /* setup the coefficients matrix */ - for (j = 0; j < N; ++j) - for (k = 0; k < N; ++k) - G[j * N + k] = A(ip[j], id[k]); - - /* invert it to solve the system of linear equations */ - raid_invert(G, V, N); - - /* compute delta parity */ - raid_delta_gen(N, id, ip, nd, size, vv); - - for (j = 0; j < N; ++j) { - p[j] = v[nd + ip[j]]; - pa[j] = v[id[j]]; - } - - raid_avx_begin(); - - asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.low4[0])); - - for (i = 0; i < size; i += 32) { - asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i])); - asm volatile ("vmovdqa %0,%%ymm2" : : "m" (pa[0][i])); - asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[1][i])); - asm volatile ("vmovdqa %0,%%ymm3" : : "m" (pa[1][i])); - asm volatile ("vpxor %ymm2,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm3,%ymm1,%ymm1"); - - asm volatile ("vpxor %ymm6,%ymm6,%ymm6"); - - asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfmulpshufb[V[0]][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfmulpshufb[V[0]][1][0])); - asm volatile ("vpsrlw $4,%ymm0,%ymm5"); - asm volatile ("vpand %ymm7,%ymm0,%ymm4"); - asm volatile ("vpand %ymm7,%ymm5,%ymm5"); - asm volatile ("vpshufb %ymm4,%ymm2,%ymm2"); - asm volatile ("vpshufb %ymm5,%ymm3,%ymm3"); - asm volatile ("vpxor %ymm2,%ymm6,%ymm6"); - asm volatile ("vpxor %ymm3,%ymm6,%ymm6"); - - asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfmulpshufb[V[1]][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfmulpshufb[V[1]][1][0])); - asm volatile ("vpsrlw $4,%ymm1,%ymm5"); - asm volatile ("vpand %ymm7,%ymm1,%ymm4"); - asm volatile ("vpand %ymm7,%ymm5,%ymm5"); - asm volatile ("vpshufb %ymm4,%ymm2,%ymm2"); - asm volatile ("vpshufb %ymm5,%ymm3,%ymm3"); - asm volatile ("vpxor %ymm2,%ymm6,%ymm6"); - asm volatile ("vpxor %ymm3,%ymm6,%ymm6"); - - asm volatile ("vmovdqa %%ymm6,%0" : "=m" (pa[0][i])); - - asm volatile ("vpxor %ymm6,%ymm6,%ymm6"); - - asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfmulpshufb[V[2]][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfmulpshufb[V[2]][1][0])); - asm volatile ("vpsrlw $4,%ymm0,%ymm5"); - asm volatile ("vpand %ymm7,%ymm0,%ymm4"); - asm volatile ("vpand %ymm7,%ymm5,%ymm5"); - asm volatile ("vpshufb %ymm4,%ymm2,%ymm2"); - asm volatile ("vpshufb %ymm5,%ymm3,%ymm3"); - asm volatile ("vpxor %ymm2,%ymm6,%ymm6"); - asm volatile ("vpxor %ymm3,%ymm6,%ymm6"); - - asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfmulpshufb[V[3]][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfmulpshufb[V[3]][1][0])); - asm volatile ("vpsrlw $4,%ymm1,%ymm5"); - asm volatile ("vpand %ymm7,%ymm1,%ymm4"); - asm volatile ("vpand %ymm7,%ymm5,%ymm5"); - asm volatile ("vpshufb %ymm4,%ymm2,%ymm2"); - asm volatile ("vpshufb %ymm5,%ymm3,%ymm3"); - asm volatile ("vpxor %ymm2,%ymm6,%ymm6"); - asm volatile ("vpxor %ymm3,%ymm6,%ymm6"); - - asm volatile ("vmovdqa %%ymm6,%0" : "=m" (pa[1][i])); - } - - raid_avx_end(); -} - -/* - * RAID recovering AVX2 implementation - */ -void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - int N = nr; - uint8_t *p[RAID_PARITY_MAX]; - uint8_t *pa[RAID_PARITY_MAX]; - uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; - uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; - uint8_t buffer[RAID_PARITY_MAX*32+32]; - uint8_t *pd = __align_ptr(buffer, 32); - size_t i; - int j, k; - - /* setup the coefficients matrix */ - for (j = 0; j < N; ++j) - for (k = 0; k < N; ++k) - G[j * N + k] = A(ip[j], id[k]); - - /* invert it to solve the system of linear equations */ - raid_invert(G, V, N); - - /* compute delta parity */ - raid_delta_gen(N, id, ip, nd, size, vv); - - for (j = 0; j < N; ++j) { - p[j] = v[nd + ip[j]]; - pa[j] = v[id[j]]; - } - - raid_avx_begin(); - - asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.low4[0])); - - for (i = 0; i < size; i += 32) { - /* delta */ - for (j = 0; j < N; ++j) { - asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[j][i])); - asm volatile ("vmovdqa %0,%%ymm1" : : "m" (pa[j][i])); - asm volatile ("vpxor %ymm1,%ymm0,%ymm0"); - asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pd[j*32])); - } - - /* reconstruct */ - for (j = 0; j < N; ++j) { - asm volatile ("vpxor %ymm0,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm1,%ymm1,%ymm1"); - - for (k = 0; k < N; ++k) { - uint8_t m = V[j * N + k]; - - asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfmulpshufb[m][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfmulpshufb[m][1][0])); - asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pd[k*32])); - asm volatile ("vpsrlw $4,%ymm4,%ymm5"); - asm volatile ("vpand %ymm7,%ymm4,%ymm4"); - asm volatile ("vpand %ymm7,%ymm5,%ymm5"); - asm volatile ("vpshufb %ymm4,%ymm2,%ymm2"); - asm volatile ("vpshufb %ymm5,%ymm3,%ymm3"); - asm volatile ("vpxor %ymm2,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm3,%ymm1,%ymm1"); - } - - asm volatile ("vpxor %ymm1,%ymm0,%ymm0"); - asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[j][i])); - } - } - - raid_avx_end(); -} - -#ifdef CONFIG_X86_64 -/* - * RAID recovering for one disk AVX512BW implementation - */ -void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p, *pa; - uint8_t G, V; - size_t i; - - (void)nr; - - if (ip[0] == 0) { - raid_rec1of1(id, nd, size, vv); - return; - } - - G = A(ip[0], id[0]); - V = inv(G); - - raid_delta_gen(1, id, ip, nd, size, vv); - - p = v[nd + ip[0]]; - pa = v[id[0]]; - - raid_avx_begin(); - - asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfconst16.low4[0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm4" : : "m" (gfmulpshufb[V][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm5" : : "m" (gfmulpshufb[V][1][0])); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i])); - asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (pa[i])); - asm volatile ("vpxord %zmm1,%zmm0,%zmm0"); - asm volatile ("vpsrlw $4,%zmm0,%zmm1"); - asm volatile ("vpandd %zmm7,%zmm0,%zmm0"); - asm volatile ("vpandd %zmm7,%zmm1,%zmm1"); - asm volatile ("vpshufb %zmm0,%zmm4,%zmm2"); - asm volatile ("vpshufb %zmm1,%zmm5,%zmm3"); - asm volatile ("vpxord %zmm3,%zmm2,%zmm2"); - asm volatile ("vmovdqa64 %%zmm2, %0" : "=m" (pa[i])); - } - - raid_avx_end(); -} -#endif - -#ifdef CONFIG_X86_64 -/* - * RAID recovering for two disks AVX512BW implementation - */ -void raid_rec2_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - const int N = 2; - uint8_t *p[N], *pa[N]; - uint8_t G[N * N], V[N * N]; - size_t i; - int j, k; - - (void)nr; - - for (j = 0; j < N; ++j) - for (k = 0; k < N; ++k) - G[j * N + k] = A(ip[j], id[k]); - - raid_invert(G, V, N); - raid_delta_gen(N, id, ip, nd, size, vv); - - for (j = 0; j < N; ++j) { - p[j] = v[nd + ip[j]]; - pa[j] = v[id[j]]; - } - - raid_avx_begin(); - - asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfconst16.low4[0])); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[0][i])); - asm volatile ("vmovdqa64 %0,%%zmm2" : : "m" (pa[0][i])); - asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (p[1][i])); - asm volatile ("vmovdqa64 %0,%%zmm3" : : "m" (pa[1][i])); - asm volatile ("vpxord %zmm2,%zmm0,%zmm0"); - asm volatile ("vpxord %zmm3,%zmm1,%zmm1"); - - asm volatile ("vpxord %zmm6,%zmm6,%zmm6"); - - asm volatile ("vbroadcasti32x4 %0,%%zmm2" : : "m" (gfmulpshufb[V[0]][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (gfmulpshufb[V[0]][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm10" : : "m" (gfmulpshufb[V[1]][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm11" : : "m" (gfmulpshufb[V[1]][1][0])); - asm volatile ("vpsrlw $4,%zmm0,%zmm5"); - asm volatile ("vpsrlw $4,%zmm1,%zmm13"); - asm volatile ("vpandd %zmm7,%zmm0,%zmm4"); - asm volatile ("vpandd %zmm7,%zmm5,%zmm5"); - asm volatile ("vpandd %zmm7,%zmm1,%zmm12"); - asm volatile ("vpandd %zmm7,%zmm13,%zmm13"); - asm volatile ("vpshufb %zmm4,%zmm2,%zmm2"); - asm volatile ("vpshufb %zmm5,%zmm3,%zmm3"); - asm volatile ("vpshufb %zmm12,%zmm10,%zmm10"); - asm volatile ("vpshufb %zmm13,%zmm11,%zmm11"); - asm volatile ("vpxord %zmm2,%zmm6,%zmm6"); - asm volatile ("vpxord %zmm3,%zmm6,%zmm6"); - asm volatile ("vpxord %zmm10,%zmm6,%zmm6"); - asm volatile ("vpxord %zmm11,%zmm6,%zmm6"); - - asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (pa[0][i])); - - asm volatile ("vpxord %zmm6,%zmm6,%zmm6"); - - asm volatile ("vbroadcasti32x4 %0,%%zmm2" : : "m" (gfmulpshufb[V[2]][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (gfmulpshufb[V[2]][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm10" : : "m" (gfmulpshufb[V[3]][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm11" : : "m" (gfmulpshufb[V[3]][1][0])); - asm volatile ("vpsrlw $4,%zmm0,%zmm5"); - asm volatile ("vpsrlw $4,%zmm1,%zmm13"); - asm volatile ("vpandd %zmm7,%zmm0,%zmm4"); - asm volatile ("vpandd %zmm7,%zmm5,%zmm5"); - asm volatile ("vpandd %zmm7,%zmm1,%zmm12"); - asm volatile ("vpandd %zmm7,%zmm13,%zmm13"); - asm volatile ("vpshufb %zmm4,%zmm2,%zmm2"); - asm volatile ("vpshufb %zmm5,%zmm3,%zmm3"); - asm volatile ("vpshufb %zmm12,%zmm10,%zmm10"); - asm volatile ("vpshufb %zmm13,%zmm11,%zmm11"); - asm volatile ("vpxord %zmm2,%zmm6,%zmm6"); - asm volatile ("vpxord %zmm3,%zmm6,%zmm6"); - asm volatile ("vpxord %zmm10,%zmm6,%zmm6"); - asm volatile ("vpxord %zmm11,%zmm6,%zmm6"); - - asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (pa[1][i])); - } - - raid_avx_end(); -} -#endif - -#ifdef CONFIG_X86_64 -/* - * RAID recovering AVX512BW implementation - */ -void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - int N = nr; - uint8_t *p[RAID_PARITY_MAX]; - uint8_t *pa[RAID_PARITY_MAX]; - uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; - uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; - uint8_t buffer[RAID_PARITY_MAX*64+64]; - uint8_t *pd = __align_ptr(buffer, 64); - size_t i; - int j, k; - - /* setup the coefficients matrix */ - for (j = 0; j < N; ++j) - for (k = 0; k < N; ++k) - G[j * N + k] = A(ip[j], id[k]); - - /* invert it to solve the system of linear equations */ - raid_invert(G, V, N); - - /* compute delta parity */ - raid_delta_gen(N, id, ip, nd, size, vv); - - for (j = 0; j < N; ++j) { - p[j] = v[nd + ip[j]]; - pa[j] = v[id[j]]; - } - - raid_avx_begin(); - - asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfconst16.low4[0])); - - for (i = 0; i < size; i += 64) { - /* delta */ - for (j = 0; j < N; ++j) { - asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[j][i])); - asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (pa[j][i])); - asm volatile ("vpxorq %zmm1,%zmm0,%zmm0"); - asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pd[j*64])); - } - /* reconstruct */ - for (j = 0; j < N; ++j) { - asm volatile ("vpxorq %zmm0,%zmm0,%zmm0"); - asm volatile ("vpxorq %zmm1,%zmm1,%zmm1"); - for (k = 0; k < N; ++k) { - uint8_t m = V[j * N + k]; - asm volatile ("vbroadcasti32x4 %0,%%zmm2" : : "m" (gfmulpshufb[m][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (gfmulpshufb[m][1][0])); - asm volatile ("vmovdqa64 %0,%%zmm4" : : "m" (pd[k*64])); - asm volatile ("vpsrlw $4,%zmm4,%zmm5"); - asm volatile ("vpandq %zmm7,%zmm4,%zmm4"); - asm volatile ("vpandq %zmm7,%zmm5,%zmm5"); - asm volatile ("vpshufb %zmm4,%zmm2,%zmm2"); - asm volatile ("vpshufb %zmm5,%zmm3,%zmm3"); - asm volatile ("vpxorq %zmm2,%zmm0,%zmm0"); - asm volatile ("vpxorq %zmm3,%zmm1,%zmm1"); - } - asm volatile ("vpxorq %zmm1,%zmm0,%zmm0"); - asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[j][i])); - } - } - - raid_avx_end(); -} -#endif - -void raid_register_x86(void) -{ - if (raid_cpu_has_sse2()) { - raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "sse2", raid_gen1_sse2); - raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "sse2", raid_gen2_sse2); -#ifdef CONFIG_X86_64 - if (!raid_cpu_has_slow_extendedreg()) - raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "sse2e", raid_gen2_sse2ext); - /* note that raid_cpu_has_slow_extendedreg() doesn't affect vandermonde */ - raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "sse2e", raid_genz_sse2ext); -#else - raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "sse2", raid_genz_sse2); -#endif - } - - if (raid_cpu_has_ssse3()) { - raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3", raid_gen3_ssse3); - raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3", raid_gen4_ssse3); - raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3", raid_gen5_ssse3); - raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3", raid_gen6_ssse3); -#ifdef CONFIG_X86_64 - if (!raid_cpu_has_slow_extendedreg()) { - raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3e", raid_gen3_ssse3ext); - raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3e", raid_gen4_ssse3ext); - raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3e", raid_gen5_ssse3ext); - raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3e", raid_gen6_ssse3ext); - } -#endif - - raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "ssse3", raid_rec1_ssse3); - raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "ssse3", raid_rec2_ssse3); - raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "ssse3", raid_recX_ssse3); - raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "ssse3", raid_recX_ssse3); - raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "ssse3", raid_recX_ssse3); - raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "ssse3", raid_recX_ssse3); - } - - if (raid_cpu_has_avx2()) { - raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "avx2", raid_gen1_avx2); - raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "avx2", raid_gen2_avx2); -#ifdef CONFIG_X86_64 - raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "avx2e", raid_gen3_avx2ext); - raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "avx2e", raid_gen4_avx2ext); - raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "avx2e", raid_gen5_avx2ext); - raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "avx2e", raid_gen6_avx2ext); - raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "avx2e", raid_genz_avx2ext); -#endif - - raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "avx2", raid_rec1_avx2); - raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "avx2", raid_rec2_avx2); - raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "avx2", raid_recX_avx2); - raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "avx2", raid_recX_avx2); - raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "avx2", raid_recX_avx2); - raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "avx2", raid_recX_avx2); - } - -#ifdef CONFIG_X86_64 - if (raid_cpu_has_avx512bw()) { - if (!raid_cpu_has_slow_avx512()) { - raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "avx512", raid_gen1_avx512bw); - raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "avx512", raid_gen2_avx512bw); - raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "avx512", raid_gen3_avx512bw); - raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "avx512", raid_gen4_avx512bw); - raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "avx512", raid_gen5_avx512bw); - raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "avx512", raid_gen6_avx512bw); - - raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "avx512", raid_rec1_avx512bw); - raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "avx512", raid_rec2_avx512bw); - raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "avx512", raid_recX_avx512bw); - raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "avx512", raid_recX_avx512bw); - raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "avx512", raid_recX_avx512bw); - raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "avx512", raid_recX_avx512bw); - } - } -#endif -} - -#endif /* CONFIG_X86 */ diff --git a/raid/x86z.c b/raid/x86z.c deleted file mode 100644 index 717c0757..00000000 --- a/raid/x86z.c +++ /dev/null @@ -1,244 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -// Copyright (C) 2013 Andrea Mazzoleni - -#include "internal.h" -#include "gf.h" - -#ifdef CONFIG_X86 - -static const struct gfzconst16 { - uint8_t poly[16]; - uint8_t half[16]; - uint8_t low7[16]; -} gfzconst16 __aligned(16) = -{ - { - RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, - RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY - }, - { - RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, - RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE - }, - { - 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, - 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f - } -}; - -/* - * GENz (triple parity with powers of 2^-1) SSE2 implementation - */ -void raid_genz_sse2(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t**)vv; - uint8_t *p; - uint8_t *q; - uint8_t *r; - int d, l; - size_t i; - - l = nd - 1; - p = v[nd]; - q = v[nd + 1]; - r = v[nd + 2]; - - raid_sse_begin(); - - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfzconst16.poly[0])); - asm volatile ("movdqa %0,%%xmm3" : : "m" (gfzconst16.half[0])); - asm volatile ("movdqa %0,%%xmm6" : : "m" (gfzconst16.low7[0])); - - for (i = 0; i < size; i += 16) { - asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i])); - asm volatile ("movdqa %xmm0,%xmm1"); - asm volatile ("movdqa %xmm0,%xmm2"); - for (d = l - 1; d >= 0; --d) { - asm volatile ("pxor %xmm4,%xmm4"); - asm volatile ("pcmpgtb %xmm1,%xmm4"); - asm volatile ("paddb %xmm1,%xmm1"); - asm volatile ("pand %xmm7,%xmm4"); - asm volatile ("pxor %xmm4,%xmm1"); - - asm volatile ("movdqa %xmm2,%xmm4"); - asm volatile ("pxor %xmm5,%xmm5"); - asm volatile ("psllw $7,%xmm4"); - asm volatile ("psrlw $1,%xmm2"); - asm volatile ("pcmpgtb %xmm4,%xmm5"); - asm volatile ("pand %xmm6,%xmm2"); - asm volatile ("pand %xmm3,%xmm5"); - asm volatile ("pxor %xmm5,%xmm2"); - - asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); - asm volatile ("pxor %xmm4,%xmm0"); - asm volatile ("pxor %xmm4,%xmm1"); - asm volatile ("pxor %xmm4,%xmm2"); - } - asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); - asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); - asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); - } - - raid_sse_end(); -} - -#ifdef CONFIG_X86_64 -/* - * GENz (triple parity with powers of 2^-1) SSE2 implementation - * - * Note that it uses 16 registers, meaning that x64 is required. - */ -void raid_genz_sse2ext(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t**)vv; - uint8_t *p; - uint8_t *q; - uint8_t *r; - int d, l; - size_t i; - - l = nd - 1; - p = v[nd]; - q = v[nd + 1]; - r = v[nd + 2]; - - raid_sse_begin(); - - asm volatile ("movdqa %0,%%xmm7" : : "m" (gfzconst16.poly[0])); - asm volatile ("movdqa %0,%%xmm3" : : "m" (gfzconst16.half[0])); - asm volatile ("movdqa %0,%%xmm11" : : "m" (gfzconst16.low7[0])); - - for (i = 0; i < size; i += 32) { - asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i])); - asm volatile ("movdqa %0,%%xmm8" : : "m" (v[l][i + 16])); - asm volatile ("movdqa %xmm0,%xmm1"); - asm volatile ("movdqa %xmm8,%xmm9"); - asm volatile ("movdqa %xmm0,%xmm2"); - asm volatile ("movdqa %xmm8,%xmm10"); - for (d = l - 1; d >= 0; --d) { - asm volatile ("movdqa %xmm2,%xmm6"); - asm volatile ("movdqa %xmm10,%xmm14"); - asm volatile ("pxor %xmm4,%xmm4"); - asm volatile ("pxor %xmm12,%xmm12"); - asm volatile ("pxor %xmm5,%xmm5"); - asm volatile ("pxor %xmm13,%xmm13"); - asm volatile ("psllw $7,%xmm6"); - asm volatile ("psllw $7,%xmm14"); - asm volatile ("psrlw $1,%xmm2"); - asm volatile ("psrlw $1,%xmm10"); - asm volatile ("pcmpgtb %xmm1,%xmm4"); - asm volatile ("pcmpgtb %xmm9,%xmm12"); - asm volatile ("pcmpgtb %xmm6,%xmm5"); - asm volatile ("pcmpgtb %xmm14,%xmm13"); - asm volatile ("paddb %xmm1,%xmm1"); - asm volatile ("paddb %xmm9,%xmm9"); - asm volatile ("pand %xmm11,%xmm2"); - asm volatile ("pand %xmm11,%xmm10"); - asm volatile ("pand %xmm7,%xmm4"); - asm volatile ("pand %xmm7,%xmm12"); - asm volatile ("pand %xmm3,%xmm5"); - asm volatile ("pand %xmm3,%xmm13"); - asm volatile ("pxor %xmm4,%xmm1"); - asm volatile ("pxor %xmm12,%xmm9"); - asm volatile ("pxor %xmm5,%xmm2"); - asm volatile ("pxor %xmm13,%xmm10"); - - asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i])); - asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16])); - asm volatile ("pxor %xmm4,%xmm0"); - asm volatile ("pxor %xmm4,%xmm1"); - asm volatile ("pxor %xmm4,%xmm2"); - asm volatile ("pxor %xmm12,%xmm8"); - asm volatile ("pxor %xmm12,%xmm9"); - asm volatile ("pxor %xmm12,%xmm10"); - } - asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i])); - asm volatile ("movntdq %%xmm8,%0" : "=m" (p[i + 16])); - asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i])); - asm volatile ("movntdq %%xmm9,%0" : "=m" (q[i + 16])); - asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); - asm volatile ("movntdq %%xmm10,%0" : "=m" (r[i + 16])); - } - - raid_sse_end(); -} -#endif - -#ifdef CONFIG_X86_64 -/* - * GENz (triple parity with powers of 2^-1) AVX2 implementation - * - * Note that it uses 16 registers, meaning that x64 is required. - */ -void raid_genz_avx2ext(int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t**)vv; - uint8_t *p; - uint8_t *q; - uint8_t *r; - int d, l; - size_t i; - - l = nd - 1; - p = v[nd]; - q = v[nd + 1]; - r = v[nd + 2]; - - raid_avx_begin(); - - asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfzconst16.poly[0])); - asm volatile ("vpbroadcastb %0,%%ymm3" : : "m" (gfzconst16.half[0])); - asm volatile ("vpbroadcastb %0,%%ymm11" : : "m" (gfzconst16.low7[0])); - asm volatile ("vpxor %ymm15,%ymm15,%ymm15"); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[l][i])); - asm volatile ("vmovdqa %0,%%ymm8" : : "m" (v[l][i + 32])); - asm volatile ("vmovdqa %ymm0,%ymm1"); - asm volatile ("vmovdqa %ymm8,%ymm9"); - asm volatile ("vmovdqa %ymm0,%ymm2"); - asm volatile ("vmovdqa %ymm8,%ymm10"); - for (d = l - 1; d >= 0; --d) { - asm volatile ("vpsllw $7,%ymm2,%ymm6"); - asm volatile ("vpsllw $7,%ymm10,%ymm14"); - asm volatile ("vpsrlw $1,%ymm2,%ymm2"); - asm volatile ("vpsrlw $1,%ymm10,%ymm10"); - asm volatile ("vpcmpgtb %ymm1,%ymm15,%ymm4"); - asm volatile ("vpcmpgtb %ymm9,%ymm15,%ymm12"); - asm volatile ("vpcmpgtb %ymm6,%ymm15,%ymm5"); - asm volatile ("vpcmpgtb %ymm14,%ymm15,%ymm13"); - asm volatile ("vpaddb %ymm1,%ymm1,%ymm1"); - asm volatile ("vpaddb %ymm9,%ymm9,%ymm9"); - asm volatile ("vpand %ymm11,%ymm2,%ymm2"); - asm volatile ("vpand %ymm11,%ymm10,%ymm10"); - asm volatile ("vpand %ymm7,%ymm4,%ymm4"); - asm volatile ("vpand %ymm7,%ymm12,%ymm12"); - asm volatile ("vpand %ymm3,%ymm5,%ymm5"); - asm volatile ("vpand %ymm3,%ymm13,%ymm13"); - asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); - asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); - asm volatile ("vpxor %ymm5,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm13,%ymm10,%ymm10"); - - asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i])); - asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i + 32])); - asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); - asm volatile ("vpxor %ymm4,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); - asm volatile ("vpxor %ymm12,%ymm9,%ymm9"); - asm volatile ("vpxor %ymm12,%ymm10,%ymm10"); - } - asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); - asm volatile ("vmovntdq %%ymm8,%0" : "=m" (p[i + 32])); - asm volatile ("vmovntdq %%ymm1,%0" : "=m" (q[i])); - asm volatile ("vmovntdq %%ymm9,%0" : "=m" (q[i + 32])); - asm volatile ("vmovntdq %%ymm2,%0" : "=m" (r[i])); - asm volatile ("vmovntdq %%ymm10,%0" : "=m" (r[i + 32])); - } - - raid_avx_end(); -} -#endif - -#endif /* CONFIG_X86 */