rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-25 18:40:19 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Use primitive generator 3 for AES Cauchy mode

Browse source
This commit is contained in:
Andrea Mazzoleni 2026-08-19 01:36:18 +02:00
parent f855bf4d82
commit b5d507fbe9
21 changed files with 7444 additions and 5791 deletions
Download patch file Download diff file Expand all files Collapse all files

12
cmdline/selftest.c
View file

@ -1163,6 +1163,12 @@ void selftest(void)
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (raid_test_poly(RAID_MODE_CAUCHY_RAID) != 0) {
/* LCOV_EXCL_START */
log_fatal(EINTERNAL, "Failed POLY Cauchy RAID test\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (raid_test_par(RAID_MODE_CAUCHY_RAID, 32, 256) != 0) {
/* LCOV_EXCL_START */
log_fatal(EINTERNAL, "Failed GEN Cauchy RAID test\n");
@ -1181,6 +1187,12 @@ void selftest(void)
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (raid_test_poly(RAID_MODE_CAUCHY_AES) != 0) {
/* LCOV_EXCL_START */
log_fatal(EINTERNAL, "Failed POLY Cauchy AES test\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (raid_test_par(RAID_MODE_CAUCHY_AES, 32, 256) != 0) {
/* LCOV_EXCL_START */
log_fatal(EINTERNAL, "Failed GEN Cauchy AES test\n");

692
cmdline/speed.c
View file

File diff suppressed because it is too large Load diff

17
raid/README
View file

@ -88,9 +88,9 @@ Operating Modes and Polynomials
RAID_MODE_CAUCHY_RAID (Default mode = 0)
Uses Cauchy matrix generator with standard RAID polynomial 0x1d
(x^8 + x^4 + x^3 + x^2 + 1, decimal 285). Supports parities 1 to 6.
Matches standard Linux RAID-6 (Anvin method) for 2-parity arrays and
ensures on-disk compatibility.
(x^8 + x^4 + x^3 + x^2 + 1, decimal 285, 0x11d) and primitive generator
g = 2. Supports parities 1 to 6. Matches standard Linux RAID-6 (Anvin
method) for 2-parity arrays and ensures on-disk compatibility.
RAID_MODE_VANDERMONDE_RAID (Mode = 1)
Legacy mode supporting up to 3 parities using standard polynomial 0x1d.
@ -98,10 +98,13 @@ Operating Modes and Polynomials
(Z parity). Optimized for low-end CPUs without vector instructions.
RAID_MODE_CAUCHY_AES (Mode = 2)
Uses Cauchy matrix with AES polynomial 0x1b (x^8 + x^4 + x^3 + x + 1).
Enables hardware acceleration via Intel GFNI instructions (vgf2p8mulb)
on supported CPUs (Ice Lake or newer). Parity data is incompatible with
standard 0x1d polynomial arrays.
Uses Cauchy matrix with AES polynomial 0x1b (x^8 + x^4 + x^3 + x + 1,
0x11b) and primitive generator g = 3. Enables hardware acceleration via
Intel GFNI instructions (vgf2p8mulb) on supported CPUs (Ice Lake or newer).
Parity data is incompatible with standard 0x1d polynomial arrays.
Both Cauchy modes use the same Extended Cauchy construction with coefficients
based on powers of their respective primitive generator.
Implementation Structure

573
raid/avx2.c
View file

@ -39,9 +39,9 @@ void raid_gen1_avx2(int nd, size_t size, void **vv)
}
/*
* GEN2 (RAID6 with powers of 2) AVX2 implementation
* GEN2 Cauchy AVX2 implementation using the active generator
*/
void raid_gen2_avx2(int nd, size_t size, void **vv)
static __always_inline void raid_gen2_avx2_gen(int nd, size_t size, void **vv, int generator)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -65,15 +65,29 @@ void raid_gen2_avx2(int nd, size_t size, void **vv)
asm volatile ("vmovdqa %ymm1,%ymm3");
for (d = l - 1; d >= 0; --d) {
/* scale by 2 before adding the current disk so the first (last processed) disk with factor 1 avoids doubling */
asm volatile ("vpcmpgtb %ymm2,%ymm6,%ymm4");
asm volatile ("vpcmpgtb %ymm3,%ymm6,%ymm5");
asm volatile ("vpaddb %ymm2,%ymm2,%ymm2");
asm volatile ("vpaddb %ymm3,%ymm3,%ymm3");
asm volatile ("vpand %ymm7,%ymm4,%ymm4");
asm volatile ("vpand %ymm7,%ymm5,%ymm5");
asm volatile ("vpxor %ymm4,%ymm2,%ymm2");
asm volatile ("vpxor %ymm5,%ymm3,%ymm3");
if (generator == 3) {
asm volatile ("vmovdqa %ymm2,%ymm4");
asm volatile ("vmovdqa %ymm3,%ymm5");
asm volatile ("vpaddb %ymm2,%ymm2,%ymm2");
asm volatile ("vpaddb %ymm3,%ymm3,%ymm3");
asm volatile ("vpxor %ymm4,%ymm2,%ymm2");
asm volatile ("vpxor %ymm5,%ymm3,%ymm3");
asm volatile ("vpcmpgtb %ymm4,%ymm6,%ymm4");
asm volatile ("vpcmpgtb %ymm5,%ymm6,%ymm5");
asm volatile ("vpand %ymm7,%ymm4,%ymm4");
asm volatile ("vpand %ymm7,%ymm5,%ymm5");
asm volatile ("vpxor %ymm4,%ymm2,%ymm2");
asm volatile ("vpxor %ymm5,%ymm3,%ymm3");
} else {
asm volatile ("vpcmpgtb %ymm2,%ymm6,%ymm4");
asm volatile ("vpcmpgtb %ymm3,%ymm6,%ymm5");
asm volatile ("vpaddb %ymm2,%ymm2,%ymm2");
asm volatile ("vpaddb %ymm3,%ymm3,%ymm3");
asm volatile ("vpand %ymm7,%ymm4,%ymm4");
asm volatile ("vpand %ymm7,%ymm5,%ymm5");
asm volatile ("vpxor %ymm4,%ymm2,%ymm2");
asm volatile ("vpxor %ymm5,%ymm3,%ymm3");
}
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i]));
asm volatile ("vmovdqa %0,%%ymm5" : : "m" (v[d][i + 32]));
@ -92,13 +106,148 @@ void raid_gen2_avx2(int nd, size_t size, void **vv)
raid_avx_end();
}
void raid_gen2_avx2_raid(int nd, size_t size, void **vv)
{
raid_gen2_avx2_gen(nd, size, vv, 2);
}
void raid_gen2_avx2_aes(int nd, size_t size, void **vv)
{
raid_gen2_avx2_gen(nd, size, vv, 3);
}
#ifdef CONFIG_X86_64
/*
* GEN2 Cauchy AVX2 implementation using the extended register set.
*
* Process two data disks at a time and process the two 32-byte lanes
* sequentially.
*
* Note that it uses 16 registers, meaning that x64 is required.
*/
static __always_inline void raid_gen2_avx2ext_gen(int nd, size_t size,
void **vv, int generator)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
uint8_t *q;
int d, l;
size_t i;
l = nd - 1;
p = v[nd];
q = v[nd + 1];
raid_avx_begin();
asm volatile ("vpxor %ymm14,%ymm14,%ymm14");
asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.poly[0]));
for (i = 0; i < size; i += 64) {
asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[l][i]));
asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[l][i + 32]));
asm volatile ("vmovdqa %ymm0,%ymm2");
asm volatile ("vmovdqa %ymm1,%ymm3");
/* process two disks per iteration */
for (d = l - 1; d >= 1; d -= 2) {
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i]));
asm volatile ("vmovdqa %0,%%ymm5" : : "m" (v[d][i + 32]));
asm volatile ("vmovdqa %0,%%ymm6" : : "m" (v[d - 1][i]));
asm volatile ("vmovdqa %0,%%ymm7" : : "m" (v[d - 1][i + 32]));
asm volatile ("vpcmpgtb %ymm2,%ymm14,%ymm10");
asm volatile ("vpaddb %ymm2,%ymm2,%ymm8");
asm volatile ("vpxor %ymm6,%ymm4,%ymm12");
asm volatile ("vpand %ymm15,%ymm10,%ymm10");
asm volatile ("vpxor %ymm10,%ymm8,%ymm8");
asm volatile ("vpxor %ymm4,%ymm8,%ymm8");
asm volatile ("vpcmpgtb %ymm8,%ymm14,%ymm10");
asm volatile ("vpaddb %ymm8,%ymm8,%ymm8");
asm volatile ("vpxor %ymm12,%ymm0,%ymm0");
if (generator == 3)
asm volatile ("vpxor %ymm2,%ymm12,%ymm12");
asm volatile ("vpand %ymm15,%ymm10,%ymm10");
asm volatile ("vpxor %ymm10,%ymm8,%ymm8");
if (generator == 3)
asm volatile ("vpxor %ymm12,%ymm8,%ymm2");
else
asm volatile ("vpxor %ymm6,%ymm8,%ymm2");
asm volatile ("vpcmpgtb %ymm3,%ymm14,%ymm11");
asm volatile ("vpaddb %ymm3,%ymm3,%ymm9");
asm volatile ("vpxor %ymm7,%ymm5,%ymm13");
asm volatile ("vpand %ymm15,%ymm11,%ymm11");
asm volatile ("vpxor %ymm11,%ymm9,%ymm9");
asm volatile ("vpxor %ymm5,%ymm9,%ymm9");
asm volatile ("vpcmpgtb %ymm9,%ymm14,%ymm11");
asm volatile ("vpaddb %ymm9,%ymm9,%ymm9");
asm volatile ("vpxor %ymm13,%ymm1,%ymm1");
if (generator == 3)
asm volatile ("vpxor %ymm3,%ymm13,%ymm13");
asm volatile ("vpand %ymm15,%ymm11,%ymm11");
asm volatile ("vpxor %ymm11,%ymm9,%ymm9");
if (generator == 3)
asm volatile ("vpxor %ymm13,%ymm9,%ymm3");
else
asm volatile ("vpxor %ymm7,%ymm9,%ymm3");
}
/* single remaining disk */
if (d == 0) {
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[0][i]));
asm volatile ("vmovdqa %0,%%ymm5" : : "m" (v[0][i + 32]));
asm volatile ("vpcmpgtb %ymm2,%ymm14,%ymm10");
asm volatile ("vpaddb %ymm2,%ymm2,%ymm8");
asm volatile ("vpxor %ymm4,%ymm0,%ymm0");
if (generator == 3)
asm volatile ("vpxor %ymm4,%ymm2,%ymm12");
asm volatile ("vpand %ymm15,%ymm10,%ymm10");
asm volatile ("vpxor %ymm10,%ymm8,%ymm8");
if (generator == 3)
asm volatile ("vpxor %ymm12,%ymm8,%ymm2");
else
asm volatile ("vpxor %ymm4,%ymm8,%ymm2");
asm volatile ("vpcmpgtb %ymm3,%ymm14,%ymm11");
asm volatile ("vpaddb %ymm3,%ymm3,%ymm9");
asm volatile ("vpxor %ymm5,%ymm1,%ymm1");
if (generator == 3)
asm volatile ("vpxor %ymm5,%ymm3,%ymm13");
asm volatile ("vpand %ymm15,%ymm11,%ymm11");
asm volatile ("vpxor %ymm11,%ymm9,%ymm9");
if (generator == 3)
asm volatile ("vpxor %ymm13,%ymm9,%ymm3");
else
asm volatile ("vpxor %ymm5,%ymm9,%ymm3");
}
asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i]));
asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32]));
asm volatile ("vmovntdq %%ymm2,%0" : "=m" (q[i]));
asm volatile ("vmovntdq %%ymm3,%0" : "=m" (q[i + 32]));
}
raid_avx_end();
}
void raid_gen2_avx2ext_raid(int nd, size_t size, void **vv)
{
raid_gen2_avx2ext_gen(nd, size, vv, 2);
}
void raid_gen2_avx2ext_aes(int nd, size_t size, void **vv)
{
raid_gen2_avx2ext_gen(nd, size, vv, 3);
}
/*
* GENz (triple parity with powers of 2^-1) AVX2 implementation
*
* Note that it uses 16 registers, meaning that x64 is required.
*/
void raid_genz_avx2ext(int nd, size_t size, void **vv)
void raid_genz_avx2ext_raid(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -175,7 +324,7 @@ void raid_genz_avx2ext(int nd, size_t size, void **vv)
*
* Note that it uses 16 registers, meaning that x64 is required.
*/
void raid_gen3_avx2ext(int nd, size_t size, void **vv)
static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv, int generator)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -203,7 +352,7 @@ void raid_gen3_avx2ext(int nd, size_t size, void **vv)
asm volatile ("vpbroadcastb %0, %%ymm11" : : "m" (gfconst16.low4[0]));
for (i = 0; i < size; i += 64) {
/* last disk without the by two multiplication */
/* last disk without the generator multiplication */
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[l][i]));
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[l][i + 32]));
@ -228,50 +377,170 @@ void raid_gen3_avx2ext(int nd, size_t size, void **vv)
asm volatile ("vpxor %ymm7,%ymm2,%ymm2");
asm volatile ("vpxor %ymm15,%ymm10,%ymm10");
/* intermediate disks */
for (d = l - 1; d > 0; --d) {
/* process two intermediate disks per iteration */
for (d = l - 1; d > 1; d -= 2) {
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i]));
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i + 32]));
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
if (generator == 3)
asm volatile ("vmovdqa %ymm1,%ymm6");
asm volatile ("vpxor %ymm5,%ymm5,%ymm5");
asm volatile ("vpxor %ymm13,%ymm13,%ymm13");
asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5");
asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13");
asm volatile ("vpaddb %ymm1,%ymm1,%ymm1");
asm volatile ("vpaddb %ymm9,%ymm9,%ymm9");
asm volatile ("vpand %ymm3,%ymm5,%ymm5");
asm volatile ("vpand %ymm3,%ymm13,%ymm13");
asm volatile ("vpxor %ymm5,%ymm1,%ymm1");
asm volatile ("vpxor %ymm13,%ymm9,%ymm9");
if (generator == 3)
asm volatile ("vpxor %ymm6,%ymm1,%ymm1");
asm volatile ("vpxor %ymm4,%ymm0,%ymm0");
asm volatile ("vpxor %ymm4,%ymm1,%ymm1");
asm volatile ("vpsrlw $4,%ymm4,%ymm5");
asm volatile ("vpand %ymm11,%ymm4,%ymm4");
asm volatile ("vpand %ymm11,%ymm5,%ymm5");
asm volatile ("vpshufb %ymm4,%ymm14,%ymm6");
asm volatile ("vpshufb %ymm5,%ymm15,%ymm7");
asm volatile ("vpxor %ymm7,%ymm6,%ymm6");
asm volatile ("vpxor %ymm6,%ymm2,%ymm2");
if (generator == 3)
asm volatile ("vmovdqa %ymm9,%ymm6");
asm volatile ("vpxor %ymm13,%ymm13,%ymm13");
asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13");
asm volatile ("vpaddb %ymm9,%ymm9,%ymm9");
asm volatile ("vpand %ymm3,%ymm13,%ymm13");
asm volatile ("vpxor %ymm13,%ymm9,%ymm9");
if (generator == 3)
asm volatile ("vpxor %ymm6,%ymm9,%ymm9");
asm volatile ("vpxor %ymm12,%ymm8,%ymm8");
asm volatile ("vpxor %ymm12,%ymm9,%ymm9");
asm volatile ("vpsrlw $4,%ymm4,%ymm5");
asm volatile ("vpsrlw $4,%ymm12,%ymm13");
asm volatile ("vpand %ymm11,%ymm4,%ymm4");
asm volatile ("vpand %ymm11,%ymm12,%ymm12");
asm volatile ("vpand %ymm11,%ymm5,%ymm5");
asm volatile ("vpand %ymm11,%ymm13,%ymm13");
asm volatile ("vpsrlw $4,%ymm12,%ymm13");
asm volatile ("vpand %ymm11,%ymm12,%ymm12");
asm volatile ("vpand %ymm11,%ymm13,%ymm13");
asm volatile ("vpshufb %ymm12,%ymm14,%ymm6");
asm volatile ("vpshufb %ymm13,%ymm15,%ymm7");
asm volatile ("vpxor %ymm7,%ymm6,%ymm6");
asm volatile ("vpxor %ymm6,%ymm10,%ymm10");
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d - 1][i]));
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d - 1][i + 32]));
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d - 1][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[d - 1][1][1][0]));
if (generator == 3)
asm volatile ("vmovdqa %ymm1,%ymm6");
asm volatile ("vpxor %ymm5,%ymm5,%ymm5");
asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5");
asm volatile ("vpaddb %ymm1,%ymm1,%ymm1");
asm volatile ("vpand %ymm3,%ymm5,%ymm5");
asm volatile ("vpxor %ymm5,%ymm1,%ymm1");
if (generator == 3)
asm volatile ("vpxor %ymm6,%ymm1,%ymm1");
asm volatile ("vpxor %ymm4,%ymm0,%ymm0");
asm volatile ("vpxor %ymm4,%ymm1,%ymm1");
asm volatile ("vpsrlw $4,%ymm4,%ymm5");
asm volatile ("vpand %ymm11,%ymm4,%ymm4");
asm volatile ("vpand %ymm11,%ymm5,%ymm5");
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
asm volatile ("vpshufb %ymm4,%ymm14,%ymm6");
asm volatile ("vpshufb %ymm12,%ymm14,%ymm14");
asm volatile ("vpshufb %ymm5,%ymm15,%ymm7");
asm volatile ("vpshufb %ymm13,%ymm15,%ymm15");
asm volatile ("vpxor %ymm6,%ymm2,%ymm2");
asm volatile ("vpxor %ymm14,%ymm10,%ymm10");
asm volatile ("vpxor %ymm7,%ymm2,%ymm2");
asm volatile ("vpxor %ymm15,%ymm10,%ymm10");
asm volatile ("vpxor %ymm7,%ymm6,%ymm6");
asm volatile ("vpxor %ymm6,%ymm2,%ymm2");
if (generator == 3)
asm volatile ("vmovdqa %ymm9,%ymm6");
asm volatile ("vpxor %ymm13,%ymm13,%ymm13");
asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13");
asm volatile ("vpaddb %ymm9,%ymm9,%ymm9");
asm volatile ("vpand %ymm3,%ymm13,%ymm13");
asm volatile ("vpxor %ymm13,%ymm9,%ymm9");
if (generator == 3)
asm volatile ("vpxor %ymm6,%ymm9,%ymm9");
asm volatile ("vpxor %ymm12,%ymm8,%ymm8");
asm volatile ("vpxor %ymm12,%ymm9,%ymm9");
asm volatile ("vpsrlw $4,%ymm12,%ymm13");
asm volatile ("vpand %ymm11,%ymm12,%ymm12");
asm volatile ("vpand %ymm11,%ymm13,%ymm13");
asm volatile ("vpshufb %ymm12,%ymm14,%ymm6");
asm volatile ("vpshufb %ymm13,%ymm15,%ymm7");
asm volatile ("vpxor %ymm7,%ymm6,%ymm6");
asm volatile ("vpxor %ymm6,%ymm10,%ymm10");
}
/* single remaining intermediate disk */
if (d == 1) {
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[1][i]));
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[1][i + 32]));
if (generator == 3)
asm volatile ("vmovdqa %ymm1,%ymm6");
asm volatile ("vpxor %ymm5,%ymm5,%ymm5");
asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5");
asm volatile ("vpaddb %ymm1,%ymm1,%ymm1");
asm volatile ("vpand %ymm3,%ymm5,%ymm5");
asm volatile ("vpxor %ymm5,%ymm1,%ymm1");
if (generator == 3)
asm volatile ("vpxor %ymm6,%ymm1,%ymm1");
asm volatile ("vpxor %ymm4,%ymm0,%ymm0");
asm volatile ("vpxor %ymm4,%ymm1,%ymm1");
asm volatile ("vpsrlw $4,%ymm4,%ymm5");
asm volatile ("vpand %ymm11,%ymm4,%ymm4");
asm volatile ("vpand %ymm11,%ymm5,%ymm5");
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[1][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[1][1][1][0]));
asm volatile ("vpshufb %ymm4,%ymm14,%ymm6");
asm volatile ("vpshufb %ymm5,%ymm15,%ymm7");
asm volatile ("vpxor %ymm7,%ymm6,%ymm6");
asm volatile ("vpxor %ymm6,%ymm2,%ymm2");
if (generator == 3)
asm volatile ("vmovdqa %ymm9,%ymm6");
asm volatile ("vpxor %ymm13,%ymm13,%ymm13");
asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13");
asm volatile ("vpaddb %ymm9,%ymm9,%ymm9");
asm volatile ("vpand %ymm3,%ymm13,%ymm13");
asm volatile ("vpxor %ymm13,%ymm9,%ymm9");
if (generator == 3)
asm volatile ("vpxor %ymm6,%ymm9,%ymm9");
asm volatile ("vpxor %ymm12,%ymm8,%ymm8");
asm volatile ("vpxor %ymm12,%ymm9,%ymm9");
asm volatile ("vpsrlw $4,%ymm12,%ymm13");
asm volatile ("vpand %ymm11,%ymm12,%ymm12");
asm volatile ("vpand %ymm11,%ymm13,%ymm13");
asm volatile ("vpshufb %ymm12,%ymm14,%ymm6");
asm volatile ("vpshufb %ymm13,%ymm15,%ymm7");
asm volatile ("vpxor %ymm7,%ymm6,%ymm6");
asm volatile ("vpxor %ymm6,%ymm10,%ymm10");
}
/* first disk with all coefficients at 1 */
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[0][i]));
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[0][i + 32]));
if (generator == 3) {
asm volatile ("vmovdqa %ymm1,%ymm6");
asm volatile ("vmovdqa %ymm9,%ymm14");
}
asm volatile ("vpxor %ymm5,%ymm5,%ymm5");
asm volatile ("vpxor %ymm13,%ymm13,%ymm13");
asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5");
@ -282,6 +551,10 @@ void raid_gen3_avx2ext(int nd, size_t size, void **vv)
asm volatile ("vpand %ymm3,%ymm13,%ymm13");
asm volatile ("vpxor %ymm5,%ymm1,%ymm1");
asm volatile ("vpxor %ymm13,%ymm9,%ymm9");
if (generator == 3) {
asm volatile ("vpxor %ymm6,%ymm1,%ymm1");
asm volatile ("vpxor %ymm14,%ymm9,%ymm9");
}
asm volatile ("vpxor %ymm4,%ymm0,%ymm0");
asm volatile ("vpxor %ymm4,%ymm1,%ymm1");
@ -300,6 +573,16 @@ void raid_gen3_avx2ext(int nd, size_t size, void **vv)
raid_avx_end();
}
void raid_gen3_avx2ext_raid(int nd, size_t size, void **vv)
{
raid_gen3_avx2ext_gen(nd, size, vv, 2);
}
void raid_gen3_avx2ext_aes(int nd, size_t size, void **vv)
{
raid_gen3_avx2ext_gen(nd, size, vv, 3);
}
#endif
#ifdef CONFIG_X86_64
@ -308,7 +591,7 @@ void raid_gen3_avx2ext(int nd, size_t size, void **vv)
*
* Note that it uses 16 registers, meaning that x64 is required.
*/
void raid_gen4_avx2ext(int nd, size_t size, void **vv)
static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv, int generator)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -333,10 +616,13 @@ void raid_gen4_avx2ext(int nd, size_t size, void **vv)
raid_avx_begin();
/* keep the constants resident for the whole function */
asm volatile ("vpbroadcastb %0,%%ymm6" : : "m" (gfconst16.poly[0]));
asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.low4[0]));
/* generic case with at least two data disks */
for (i = 0; i < size; i += 64) {
/* last disk without the by two multiplication */
asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0]));
/* last disk without the generator multiplication */
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[l][i]));
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[l][i + 32]));
@ -345,105 +631,115 @@ void raid_gen4_avx2ext(int nd, size_t size, void **vv)
asm volatile ("vmovdqa %ymm12,%ymm8");
asm volatile ("vmovdqa %ymm12,%ymm9");
asm volatile ("vpsrlw $4,%ymm4,%ymm5");
asm volatile ("vpsrlw $4,%ymm12,%ymm13");
asm volatile ("vpand %ymm15,%ymm4,%ymm4");
asm volatile ("vpand %ymm15,%ymm12,%ymm12");
asm volatile ("vpand %ymm15,%ymm5,%ymm5");
asm volatile ("vpand %ymm15,%ymm13,%ymm13");
asm volatile ("vpsrlw $4,%ymm4,%ymm5");
asm volatile ("vpand %ymm7,%ymm4,%ymm4");
asm volatile ("vpand %ymm7,%ymm5,%ymm5");
asm volatile ("vpsrlw $4,%ymm12,%ymm13");
asm volatile ("vpand %ymm7,%ymm12,%ymm12");
asm volatile ("vpand %ymm7,%ymm13,%ymm13");
asm volatile ("vbroadcasti128 %0,%%ymm10" : : "m" (raid_gfcauchypshufb[l][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[l][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[l][1][1][0]));
asm volatile ("vpshufb %ymm4,%ymm10,%ymm2");
asm volatile ("vpshufb %ymm5,%ymm15,%ymm7");
asm volatile ("vpshufb %ymm12,%ymm10,%ymm10");
asm volatile ("vpshufb %ymm13,%ymm15,%ymm15");
asm volatile ("vpxor %ymm7,%ymm2,%ymm2");
asm volatile ("vpxor %ymm15,%ymm10,%ymm10");
asm volatile ("vpshufb %ymm4,%ymm14,%ymm2");
asm volatile ("vpshufb %ymm5,%ymm15,%ymm3");
asm volatile ("vpxor %ymm3,%ymm2,%ymm2");
asm volatile ("vpshufb %ymm12,%ymm14,%ymm10");
asm volatile ("vpshufb %ymm13,%ymm15,%ymm11");
asm volatile ("vpxor %ymm11,%ymm10,%ymm10");
asm volatile ("vbroadcasti128 %0,%%ymm11" : : "m" (raid_gfcauchypshufb[l][2][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[l][2][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[l][2][1][0]));
asm volatile ("vpshufb %ymm4,%ymm11,%ymm3");
asm volatile ("vpshufb %ymm5,%ymm15,%ymm7");
asm volatile ("vpshufb %ymm12,%ymm11,%ymm11");
asm volatile ("vpshufb %ymm13,%ymm15,%ymm15");
asm volatile ("vpxor %ymm7,%ymm3,%ymm3");
asm volatile ("vpxor %ymm15,%ymm11,%ymm11");
asm volatile ("vpshufb %ymm4,%ymm14,%ymm3");
asm volatile ("vpshufb %ymm5,%ymm15,%ymm5");
asm volatile ("vpxor %ymm5,%ymm3,%ymm3");
asm volatile ("vpshufb %ymm12,%ymm14,%ymm11");
asm volatile ("vpshufb %ymm13,%ymm15,%ymm13");
asm volatile ("vpxor %ymm13,%ymm11,%ymm11");
/* intermediate disks */
for (d = l - 1; d > 0; --d) {
asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.poly[0]));
asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0]));
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i]));
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i + 32]));
if (generator == 3)
asm volatile ("vmovdqa %ymm1,%ymm4");
asm volatile ("vpxor %ymm5,%ymm5,%ymm5");
asm volatile ("vpxor %ymm13,%ymm13,%ymm13");
asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5");
asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13");
asm volatile ("vpaddb %ymm1,%ymm1,%ymm1");
asm volatile ("vpaddb %ymm9,%ymm9,%ymm9");
asm volatile ("vpand %ymm7,%ymm5,%ymm5");
asm volatile ("vpand %ymm7,%ymm13,%ymm13");
asm volatile ("vpand %ymm6,%ymm5,%ymm5");
asm volatile ("vpxor %ymm5,%ymm1,%ymm1");
asm volatile ("vpxor %ymm13,%ymm9,%ymm9");
if (generator == 3)
asm volatile ("vpxor %ymm4,%ymm1,%ymm1");
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i]));
asm volatile ("vpxor %ymm4,%ymm0,%ymm0");
asm volatile ("vpxor %ymm4,%ymm1,%ymm1");
asm volatile ("vpsrlw $4,%ymm4,%ymm5");
asm volatile ("vpand %ymm7,%ymm4,%ymm4");
asm volatile ("vpand %ymm7,%ymm5,%ymm5");
if (generator == 3)
asm volatile ("vmovdqa %ymm9,%ymm12");
asm volatile ("vpxor %ymm13,%ymm13,%ymm13");
asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13");
asm volatile ("vpaddb %ymm9,%ymm9,%ymm9");
asm volatile ("vpand %ymm6,%ymm13,%ymm13");
asm volatile ("vpxor %ymm13,%ymm9,%ymm9");
if (generator == 3)
asm volatile ("vpxor %ymm12,%ymm9,%ymm9");
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i + 32]));
asm volatile ("vpxor %ymm12,%ymm8,%ymm8");
asm volatile ("vpxor %ymm12,%ymm9,%ymm9");
asm volatile ("vpsrlw $4,%ymm4,%ymm5");
asm volatile ("vpsrlw $4,%ymm12,%ymm13");
asm volatile ("vpand %ymm15,%ymm4,%ymm4");
asm volatile ("vpand %ymm15,%ymm12,%ymm12");
asm volatile ("vpand %ymm15,%ymm5,%ymm5");
asm volatile ("vpand %ymm15,%ymm13,%ymm13");
asm volatile ("vpsrlw $4,%ymm12,%ymm13");
asm volatile ("vpand %ymm7,%ymm12,%ymm12");
asm volatile ("vpand %ymm7,%ymm13,%ymm13");
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
asm volatile ("vpshufb %ymm4,%ymm14,%ymm6");
asm volatile ("vpshufb %ymm5,%ymm15,%ymm7");
asm volatile ("vpshufb %ymm4,%ymm14,%ymm15");
asm volatile ("vpshufb %ymm12,%ymm14,%ymm14");
asm volatile ("vpshufb %ymm13,%ymm15,%ymm15");
asm volatile ("vpxor %ymm6,%ymm2,%ymm2");
asm volatile ("vpxor %ymm14,%ymm10,%ymm10");
asm volatile ("vpxor %ymm7,%ymm2,%ymm2");
asm volatile ("vpxor %ymm15,%ymm10,%ymm10");
asm volatile ("vpxor %ymm15,%ymm2,%ymm2");
asm volatile ("vpxor %ymm14,%ymm10,%ymm10");
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
asm volatile ("vpshufb %ymm5,%ymm14,%ymm15");
asm volatile ("vpshufb %ymm13,%ymm14,%ymm14");
asm volatile ("vpxor %ymm15,%ymm2,%ymm2");
asm volatile ("vpxor %ymm14,%ymm10,%ymm10");
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][2][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[d][2][1][0]));
asm volatile ("vpshufb %ymm4,%ymm14,%ymm6");
asm volatile ("vpshufb %ymm5,%ymm15,%ymm7");
asm volatile ("vpshufb %ymm4,%ymm14,%ymm15");
asm volatile ("vpshufb %ymm12,%ymm14,%ymm14");
asm volatile ("vpshufb %ymm13,%ymm15,%ymm15");
asm volatile ("vpxor %ymm6,%ymm3,%ymm3");
asm volatile ("vpxor %ymm14,%ymm11,%ymm11");
asm volatile ("vpxor %ymm7,%ymm3,%ymm3");
asm volatile ("vpxor %ymm15,%ymm11,%ymm11");
asm volatile ("vpxor %ymm15,%ymm3,%ymm3");
asm volatile ("vpxor %ymm14,%ymm11,%ymm11");
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][2][1][0]));
asm volatile ("vpshufb %ymm5,%ymm14,%ymm15");
asm volatile ("vpshufb %ymm13,%ymm14,%ymm14");
asm volatile ("vpxor %ymm15,%ymm3,%ymm3");
asm volatile ("vpxor %ymm14,%ymm11,%ymm11");
}
/* first disk with all coefficients at 1 */
asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.poly[0]));
asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0]));
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[0][i]));
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[0][i + 32]));
if (generator == 3)
asm volatile ("vmovdqa %ymm1,%ymm4");
asm volatile ("vpxor %ymm5,%ymm5,%ymm5");
asm volatile ("vpxor %ymm13,%ymm13,%ymm13");
asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5");
asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13");
asm volatile ("vpaddb %ymm1,%ymm1,%ymm1");
asm volatile ("vpaddb %ymm9,%ymm9,%ymm9");
asm volatile ("vpand %ymm7,%ymm5,%ymm5");
asm volatile ("vpand %ymm7,%ymm13,%ymm13");
asm volatile ("vpand %ymm6,%ymm5,%ymm5");
asm volatile ("vpxor %ymm5,%ymm1,%ymm1");
asm volatile ("vpxor %ymm13,%ymm9,%ymm9");
if (generator == 3)
asm volatile ("vpxor %ymm4,%ymm1,%ymm1");
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[0][i]));
asm volatile ("vpxor %ymm4,%ymm0,%ymm0");
asm volatile ("vpxor %ymm4,%ymm1,%ymm1");
asm volatile ("vpxor %ymm4,%ymm2,%ymm2");
asm volatile ("vpxor %ymm4,%ymm3,%ymm3");
if (generator == 3)
asm volatile ("vmovdqa %ymm9,%ymm12");
asm volatile ("vpxor %ymm13,%ymm13,%ymm13");
asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13");
asm volatile ("vpaddb %ymm9,%ymm9,%ymm9");
asm volatile ("vpand %ymm6,%ymm13,%ymm13");
asm volatile ("vpxor %ymm13,%ymm9,%ymm9");
if (generator == 3)
asm volatile ("vpxor %ymm12,%ymm9,%ymm9");
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[0][i + 32]));
asm volatile ("vpxor %ymm12,%ymm8,%ymm8");
asm volatile ("vpxor %ymm12,%ymm9,%ymm9");
asm volatile ("vpxor %ymm12,%ymm10,%ymm10");
@ -461,13 +757,23 @@ void raid_gen4_avx2ext(int nd, size_t size, void **vv)
raid_avx_end();
}
void raid_gen4_avx2ext_raid(int nd, size_t size, void **vv)
{
raid_gen4_avx2ext_gen(nd, size, vv, 2);
}
void raid_gen4_avx2ext_aes(int nd, size_t size, void **vv)
{
raid_gen4_avx2ext_gen(nd, size, vv, 3);
}
#endif
#ifdef CONFIG_X86_64
/*
* GENX AVX2EXT implementation
*/
static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, int np)
static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, int np, int generator)
{
uint8_t **v = (uint8_t **)vv;
size_t i;
@ -490,7 +796,7 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0]));
for (i = 0; i < size; i += 32) {
/* last disk without the by two multiplication */
/* last disk without the generator multiplication */
asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[l][i]));
asm volatile ("vmovdqa %ymm10,%ymm0");
@ -499,7 +805,6 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
asm volatile ("vpsrlw $4,%ymm10,%ymm11");
asm volatile ("vpand %ymm15,%ymm10,%ymm10");
asm volatile ("vpand %ymm15,%ymm11,%ymm11");
if (np >= 3) {
asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (raid_gfcauchypshufb[l][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[l][1][1][0]));
@ -507,7 +812,6 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm13,%ymm2,%ymm2");
}
if (np >= 4) {
asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (raid_gfcauchypshufb[l][2][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[l][2][1][0]));
@ -515,7 +819,6 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm13,%ymm3,%ymm3");
}
if (np >= 5) {
asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (raid_gfcauchypshufb[l][3][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[l][3][1][0]));
@ -523,7 +826,6 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
asm volatile ("vpxor %ymm13,%ymm4,%ymm4");
}
if (np >= 6) {
asm volatile ("vbroadcasti128 %0,%%ymm5" : : "m" (raid_gfcauchypshufb[l][4][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[l][4][1][0]));
@ -536,10 +838,14 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
for (d = l - 1; d > 0; --d) {
asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[d][i]));
if (generator == 3)
asm volatile ("vmovdqa %ymm1,%ymm12");
asm volatile ("vpcmpgtb %ymm1,%ymm6,%ymm11");
asm volatile ("vpaddb %ymm1,%ymm1,%ymm1");
asm volatile ("vpand %ymm14,%ymm11,%ymm11");
asm volatile ("vpxor %ymm11,%ymm1,%ymm1");
if (generator == 3)
asm volatile ("vpxor %ymm12,%ymm1,%ymm1");
asm volatile ("vpxor %ymm10,%ymm0,%ymm0");
asm volatile ("vpxor %ymm10,%ymm1,%ymm1");
@ -547,7 +853,6 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
asm volatile ("vpsrlw $4,%ymm10,%ymm11");
asm volatile ("vpand %ymm15,%ymm10,%ymm10");
asm volatile ("vpand %ymm15,%ymm11,%ymm11");
if (np >= 3) {
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
@ -556,7 +861,6 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
asm volatile ("vpxor %ymm12,%ymm2,%ymm2");
asm volatile ("vpxor %ymm13,%ymm2,%ymm2");
}
if (np >= 4) {
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfcauchypshufb[d][2][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[d][2][1][0]));
@ -565,7 +869,6 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
asm volatile ("vpxor %ymm12,%ymm3,%ymm3");
asm volatile ("vpxor %ymm13,%ymm3,%ymm3");
}
if (np >= 5) {
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfcauchypshufb[d][3][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[d][3][1][0]));
@ -574,7 +877,6 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
asm volatile ("vpxor %ymm12,%ymm4,%ymm4");
asm volatile ("vpxor %ymm13,%ymm4,%ymm4");
}
if (np >= 6) {
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfcauchypshufb[d][4][0][0]));
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfcauchypshufb[d][4][1][0]));
@ -588,10 +890,14 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
/* first disk with all coefficients at 1 */
asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[0][i]));
if (generator == 3)
asm volatile ("vmovdqa %ymm1,%ymm12");
asm volatile ("vpcmpgtb %ymm1,%ymm6,%ymm11");
asm volatile ("vpaddb %ymm1,%ymm1,%ymm1");
asm volatile ("vpand %ymm14,%ymm11,%ymm11");
asm volatile ("vpxor %ymm11,%ymm1,%ymm1");
if (generator == 3)
asm volatile ("vpxor %ymm12,%ymm1,%ymm1");
asm volatile ("vpxor %ymm10,%ymm0,%ymm0");
asm volatile ("vpxor %ymm10,%ymm1,%ymm1");
@ -626,9 +932,14 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
*
* Note that it uses 16 registers, meaning that x64 is required.
*/
void raid_gen5_avx2ext(int nd, size_t size, void **vv)
void raid_gen5_avx2ext_raid(int nd, size_t size, void **vv)
{
raid_genX_avx2ext(nd, size, vv, 5);
raid_genX_avx2ext(nd, size, vv, 5, 2);
}
void raid_gen5_avx2ext_aes(int nd, size_t size, void **vv)
{
raid_genX_avx2ext(nd, size, vv, 5, 3);
}
#endif
@ -638,9 +949,14 @@ void raid_gen5_avx2ext(int nd, size_t size, void **vv)
*
* Note that it uses 16 registers, meaning that x64 is required.
*/
void raid_gen6_avx2ext(int nd, size_t size, void **vv)
void raid_gen6_avx2ext_raid(int nd, size_t size, void **vv)
{
raid_genX_avx2ext(nd, size, vv, 6);
raid_genX_avx2ext(nd, size, vv, 6, 2);
}
void raid_gen6_avx2ext_aes(int nd, size_t size, void **vv)
{
raid_genX_avx2ext(nd, size, vv, 6, 3);
}
#endif
@ -871,13 +1187,20 @@ void raid_register_avx2(void)
{
if (raid_cpu_has_avx2()) {
raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "avx2", raid_gen1_avx2, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "avx2", raid_gen2_avx2, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "avx2", raid_gen2_avx2_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "avx2", raid_gen2_avx2_aes, RAID_POLY_AES);
#ifdef CONFIG_X86_64
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "avx2e", raid_gen3_avx2ext, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "avx2e", raid_gen4_avx2ext, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "avx2e", raid_gen5_avx2ext, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "avx2e", raid_gen6_avx2ext, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "avx2e", raid_genz_avx2ext, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "avx2e", raid_gen2_avx2ext_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "avx2e", raid_gen2_avx2ext_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "avx2e", raid_gen3_avx2ext_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "avx2e", raid_gen3_avx2ext_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "avx2e", raid_gen4_avx2ext_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "avx2e", raid_gen4_avx2ext_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "avx2e", raid_gen5_avx2ext_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "avx2e", raid_gen5_avx2ext_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "avx2e", raid_gen6_avx2ext_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "avx2e", raid_gen6_avx2ext_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "avx2e", raid_genz_avx2ext_raid, RAID_POLY_RAID);
#endif
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "avx2", raid_rec1_avx2, RAID_POLY_ANY);

2
raid/avx512.c
View file

@ -153,7 +153,7 @@ static __always_inline void raid_genX_avx512bw(int nd, size_t size, void **vv, i
}
/*
* GEN2 (RAID6 with powers of 2) AVX512BW implementation
* GEN2 Cauchy AVX512BW implementation
*/
void raid_gen2_avx512bw(int nd, size_t size, void **vv)
{

4
raid/gf.h
View file

@ -30,9 +30,9 @@ static __always_inline uint8_t inv(uint8_t v)
}
/*
* GF 2^a.
* GF generator^a for the active Cauchy matrix.
*/
static __always_inline uint8_t pow2(int v)
static __always_inline uint8_t powgen(int v)
{
BUG_ON(v < 0 || v > 254); /* invalid exponent */

68
raid/gfni.c
View file

@ -10,7 +10,7 @@
/*
* GENX AVX2 GFNI implementation
*/
static __always_inline void raid_genX_avx2gfni(int nd, size_t size, void **vv, int np)
static __always_inline void raid_genX_avx2gfni_aes(int nd, size_t size, void **vv, int np)
{
uint8_t **v = (uint8_t **)vv;
size_t i;
@ -119,7 +119,7 @@ static __always_inline void raid_genX_avx2gfni(int nd, size_t size, void **vv, i
/*
* GENX AVX512 GFNI implementation
*/
static __always_inline void raid_genX_avx512gfni(int nd, size_t size, void **vv, int np)
static __always_inline void raid_genX_avx512gfni_aes(int nd, size_t size, void **vv, int np)
{
uint8_t **v = (uint8_t **)vv;
size_t i;
@ -199,83 +199,83 @@ static __always_inline void raid_genX_avx512gfni(int nd, size_t size, void **vv,
}
/*
* GEN2 (RAID6 with powers of 2) AVX2 GFNI implementation
* GEN2 Cauchy AVX2 GFNI implementation
*/
void raid_gen2_avx2gfni(int nd, size_t size, void **vv)
void raid_gen2_avx2gfni_aes(int nd, size_t size, void **vv)
{
raid_genX_avx2gfni(nd, size, vv, 2);
raid_genX_avx2gfni_aes(nd, size, vv, 2);
}
/*
* GEN2 (RAID6 with powers of 2) GFNI implementation
* GEN2 Cauchy GFNI implementation
*/
void raid_gen2_avx512gfni(int nd, size_t size, void **vv)
void raid_gen2_avx512gfni_aes(int nd, size_t size, void **vv)
{
raid_genX_avx512gfni(nd, size, vv, 2);
raid_genX_avx512gfni_aes(nd, size, vv, 2);
}
/*
* GEN3 (triple parity with Cauchy matrix) AVX2 GFNI implementation
*/
void raid_gen3_avx2gfni(int nd, size_t size, void **vv)
void raid_gen3_avx2gfni_aes(int nd, size_t size, void **vv)
{
raid_genX_avx2gfni(nd, size, vv, 3);
raid_genX_avx2gfni_aes(nd, size, vv, 3);
}
/*
* GEN3 (triple parity with Cauchy matrix) GFNI implementation
*/
void raid_gen3_avx512gfni(int nd, size_t size, void **vv)
void raid_gen3_avx512gfni_aes(int nd, size_t size, void **vv)
{
raid_genX_avx512gfni(nd, size, vv, 3);
raid_genX_avx512gfni_aes(nd, size, vv, 3);
}
/*
* GEN4 (quad parity with Cauchy matrix) AVX2 GFNI implementation
*/
void raid_gen4_avx2gfni(int nd, size_t size, void **vv)
void raid_gen4_avx2gfni_aes(int nd, size_t size, void **vv)
{
raid_genX_avx2gfni(nd, size, vv, 4);
raid_genX_avx2gfni_aes(nd, size, vv, 4);
}
/*
* GEN4 (quad parity with Cauchy matrix) GFNI implementation
*/
void raid_gen4_avx512gfni(int nd, size_t size, void **vv)
void raid_gen4_avx512gfni_aes(int nd, size_t size, void **vv)
{
raid_genX_avx512gfni(nd, size, vv, 4);
raid_genX_avx512gfni_aes(nd, size, vv, 4);
}
/*
* GEN5 (penta parity with Cauchy matrix) AVX2 GFNI implementation
*/
void raid_gen5_avx2gfni(int nd, size_t size, void **vv)
void raid_gen5_avx2gfni_aes(int nd, size_t size, void **vv)
{
raid_genX_avx2gfni(nd, size, vv, 5);
raid_genX_avx2gfni_aes(nd, size, vv, 5);
}
/*
* GEN5 (penta parity with Cauchy matrix) GFNI implementation
*/
void raid_gen5_avx512gfni(int nd, size_t size, void **vv)
void raid_gen5_avx512gfni_aes(int nd, size_t size, void **vv)
{
raid_genX_avx512gfni(nd, size, vv, 5);
raid_genX_avx512gfni_aes(nd, size, vv, 5);
}
/*
* GEN6 (hexa parity with Cauchy matrix) AVX2 GFNI implementation
*/
void raid_gen6_avx2gfni(int nd, size_t size, void **vv)
void raid_gen6_avx2gfni_aes(int nd, size_t size, void **vv)
{
raid_genX_avx2gfni(nd, size, vv, 6);
raid_genX_avx2gfni_aes(nd, size, vv, 6);
}
/*
* GEN6 (hexa parity with Cauchy matrix) GFNI implementation
*/
void raid_gen6_avx512gfni(int nd, size_t size, void **vv)
void raid_gen6_avx512gfni_aes(int nd, size_t size, void **vv)
{
raid_genX_avx512gfni(nd, size, vv, 6);
raid_genX_avx512gfni_aes(nd, size, vv, 6);
}
/*
@ -636,11 +636,11 @@ void raid_recX_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **
void raid_register_avx2gfni(void)
{
if (raid_cpu_has_avx2gfni()) {
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_gen2_avx2gfni, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_gen3_avx2gfni, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_gen4_avx2gfni, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_gen5_avx2gfni, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_gen6_avx2gfni, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_gen2_avx2gfni_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_gen3_avx2gfni_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_gen4_avx2gfni_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_gen5_avx2gfni_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_gen6_avx2gfni_aes, RAID_POLY_AES);
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni", raid_rec1_avx2gfni, RAID_POLY_AES);
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_rec2_avx2gfni, RAID_POLY_AES);
@ -655,11 +655,11 @@ void raid_register_avx512gfni(void)
{
if (raid_cpu_has_avx512gfni()) {
if (!raid_cpu_has_slow_avx512()) {
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_gen2_avx512gfni, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_gen3_avx512gfni, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_gen4_avx512gfni, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_gen5_avx512gfni, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_gen6_avx512gfni, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_gen2_avx512gfni_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_gen3_avx512gfni_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_gen4_avx512gfni_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_gen5_avx512gfni_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_gen6_avx512gfni_aes, RAID_POLY_AES);
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni512", raid_rec1_avx512gfni, RAID_POLY_AES);
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_rec2_avx512gfni, RAID_POLY_AES);

75
raid/int.c
View file

@ -61,9 +61,9 @@ void raid_gen1_int64(int nd, size_t size, void **vv)
}
/*
* GEN2 (RAID6 with powers of 2) 32bit C implementation
* GEN2 (double parity) 32bit C implementation
*/
void raid_gen2_int32(int nd, size_t size, void **vv)
static __always_inline void raid_gen2_int32_gen(int nd, size_t size, void **vv, int generator)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -74,6 +74,7 @@ void raid_gen2_int32(int nd, size_t size, void **vv)
uint32_t d0, q0, p0;
uint32_t d1, q1, p1;
uint32_t qold;
l = nd - 1;
p = v[nd];
@ -89,8 +90,15 @@ void raid_gen2_int32(int nd, size_t size, void **vv)
p0 ^= d0;
p1 ^= d1;
qold = q0;
q0 = x2_32(q0, poly_32);
if (generator == 3)
q0 ^= qold;
qold = q1;
q1 = x2_32(q1, poly_32);
if (generator == 3)
q1 ^= qold;
q0 ^= d0;
q1 ^= d1;
@ -102,10 +110,20 @@ void raid_gen2_int32(int nd, size_t size, void **vv)
}
}
void raid_gen2_int32_raid(int nd, size_t size, void **vv)
{
raid_gen2_int32_gen(nd, size, vv, 2);
}
void raid_gen2_int32_aes(int nd, size_t size, void **vv)
{
raid_gen2_int32_gen(nd, size, vv, 3);
}
/*
* GEN2 (RAID6 with powers of 2) 64bit C implementation
* GEN2 (double parity) 64bit C implementation
*/
void raid_gen2_int64(int nd, size_t size, void **vv)
static __always_inline void raid_gen2_int64_gen(int nd, size_t size, void **vv, int generator)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -116,6 +134,7 @@ void raid_gen2_int64(int nd, size_t size, void **vv)
uint64_t d0, q0, p0;
uint64_t d1, q1, p1;
uint64_t qold;
l = nd - 1;
p = v[nd];
@ -131,8 +150,15 @@ void raid_gen2_int64(int nd, size_t size, void **vv)
p0 ^= d0;
p1 ^= d1;
qold = q0;
q0 = x2_64(q0, poly_64);
if (generator == 3)
q0 ^= qold;
qold = q1;
q1 = x2_64(q1, poly_64);
if (generator == 3)
q1 ^= qold;
q0 ^= d0;
q1 ^= d1;
@ -144,10 +170,25 @@ void raid_gen2_int64(int nd, size_t size, void **vv)
}
}
void raid_gen2_int64_raid(int nd, size_t size, void **vv)
{
raid_gen2_int64_gen(nd, size, vv, 2);
}
void raid_gen2_int64_aes(int nd, size_t size, void **vv)
{
raid_gen2_int64_gen(nd, size, vv, 3);
}
void raid_gen2_int8(int nd, size_t size, void **vv)
{
raid_gen_ref(nd, 2, size, vv);
}
/*
* GENz (triple parity with powers of 2^-1) 32bit C implementation
*/
void raid_genz_int32(int nd, size_t size, void **vv)
void raid_genz_int32_raid(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -200,7 +241,7 @@ void raid_genz_int32(int nd, size_t size, void **vv)
/*
* GENz (triple parity with powers of 2^-1) 64bit C implementation
*/
void raid_genz_int64(int nd, size_t size, void **vv)
void raid_genz_int64_raid(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -653,20 +694,24 @@ void raid_recX_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv)
void raid_register_int(void)
{
if (sizeof(void *) == 4) {
raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "int32", raid_gen1_int32, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "int32", raid_gen2_int32, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "int32", raid_genz_int32, RAID_POLY_ANY);
} else {
raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "int64", raid_gen1_int64, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "int64", raid_gen2_int64, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "int64", raid_genz_int64, RAID_POLY_ANY);
}
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "int8", raid_gen2_int8, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "int8", raid_gen3_int8, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "int8", raid_gen4_int8, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "int8", raid_gen5_int8, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "int8", raid_gen6_int8, RAID_POLY_ANY);
if (sizeof(void *) == 4) {
raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "int32", raid_gen1_int32, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "int32", raid_gen2_int32_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "int32", raid_gen2_int32_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "int32", raid_genz_int32_raid, RAID_POLY_RAID);
} else {
raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "int64", raid_gen1_int64, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "int64", raid_gen2_int64_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "int64", raid_gen2_int64_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "int64", raid_genz_int64_raid, RAID_POLY_RAID);
}
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "int8", raid_rec1_int8, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "int8", raid_rec2_int8, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "int8", raid_recX_int8, RAID_POLY_ANY);

99
raid/internal.h
View file

@ -90,47 +90,65 @@ void raid_gen1_int64(int nd, size_t size, void **vv);
void raid_gen1_sse2(int nd, size_t size, void **vv);
void raid_gen1_avx2(int nd, size_t size, void **vv);
void raid_gen1_avx512bw(int nd, size_t size, void **vv);
void raid_gen2_int32(int nd, size_t size, void **vv);
void raid_gen2_int64(int nd, size_t size, void **vv);
void raid_gen2_sse2(int nd, size_t size, void **vv);
void raid_gen2_avx2(int nd, size_t size, void **vv);
void raid_gen2_sse2ext(int nd, size_t size, void **vv);
void raid_gen2_int8(int nd, size_t size, void **vv);
void raid_gen2_int32_raid(int nd, size_t size, void **vv);
void raid_gen2_int32_aes(int nd, size_t size, void **vv);
void raid_gen2_int64_raid(int nd, size_t size, void **vv);
void raid_gen2_int64_aes(int nd, size_t size, void **vv);
void raid_gen2_sse2_raid(int nd, size_t size, void **vv);
void raid_gen2_sse2_aes(int nd, size_t size, void **vv);
void raid_gen2_avx2_raid(int nd, size_t size, void **vv);
void raid_gen2_avx2_aes(int nd, size_t size, void **vv);
void raid_gen2_sse2ext_raid(int nd, size_t size, void **vv);
void raid_gen2_sse2ext_aes(int nd, size_t size, void **vv);
void raid_gen2_avx2ext_raid(int nd, size_t size, void **vv);
void raid_gen2_avx2ext_aes(int nd, size_t size, void **vv);
void raid_gen2_avx512bw(int nd, size_t size, void **vv);
void raid_gen2_avx2gfni(int nd, size_t size, void **vv);
void raid_gen2_avx512gfni(int nd, size_t size, void **vv);
void raid_genz_int32(int nd, size_t size, void **vv);
void raid_genz_int64(int nd, size_t size, void **vv);
void raid_genz_sse2(int nd, size_t size, void **vv);
void raid_genz_sse2ext(int nd, size_t size, void **vv);
void raid_genz_avx2ext(int nd, size_t size, void **vv);
void raid_gen2_avx2gfni_aes(int nd, size_t size, void **vv);
void raid_gen2_avx512gfni_aes(int nd, size_t size, void **vv);
void raid_genz_int32_raid(int nd, size_t size, void **vv);
void raid_genz_int64_raid(int nd, size_t size, void **vv);
void raid_genz_sse2_raid(int nd, size_t size, void **vv);
void raid_genz_sse2ext_raid(int nd, size_t size, void **vv);
void raid_genz_avx2ext_raid(int nd, size_t size, void **vv);
void raid_gen3_int8(int nd, size_t size, void **vv);
void raid_gen3_ssse3(int nd, size_t size, void **vv);
void raid_gen3_ssse3ext(int nd, size_t size, void **vv);
void raid_gen3_avx2ext(int nd, size_t size, void **vv);
void raid_gen3_ssse3_raid(int nd, size_t size, void **vv);
void raid_gen3_ssse3_aes(int nd, size_t size, void **vv);
void raid_gen3_ssse3ext_raid(int nd, size_t size, void **vv);
void raid_gen3_ssse3ext_aes(int nd, size_t size, void **vv);
void raid_gen3_avx2ext_raid(int nd, size_t size, void **vv);
void raid_gen3_avx2ext_aes(int nd, size_t size, void **vv);
void raid_gen3_avx512bw(int nd, size_t size, void **vv);
void raid_gen3_avx2gfni(int nd, size_t size, void **vv);
void raid_gen3_avx512gfni(int nd, size_t size, void **vv);
void raid_gen3_avx2gfni_aes(int nd, size_t size, void **vv);
void raid_gen3_avx512gfni_aes(int nd, size_t size, void **vv);
void raid_gen4_int8(int nd, size_t size, void **vv);
void raid_gen4_ssse3(int nd, size_t size, void **vv);
void raid_gen4_ssse3ext(int nd, size_t size, void **vv);
void raid_gen4_avx2ext(int nd, size_t size, void **vv);
void raid_gen4_ssse3_raid(int nd, size_t size, void **vv);
void raid_gen4_ssse3_aes(int nd, size_t size, void **vv);
void raid_gen4_ssse3ext_raid(int nd, size_t size, void **vv);
void raid_gen4_ssse3ext_aes(int nd, size_t size, void **vv);
void raid_gen4_avx2ext_raid(int nd, size_t size, void **vv);
void raid_gen4_avx2ext_aes(int nd, size_t size, void **vv);
void raid_gen4_avx512bw(int nd, size_t size, void **vv);
void raid_gen4_avx2gfni(int nd, size_t size, void **vv);
void raid_gen4_avx512gfni(int nd, size_t size, void **vv);
void raid_gen4_avx2gfni_aes(int nd, size_t size, void **vv);
void raid_gen4_avx512gfni_aes(int nd, size_t size, void **vv);
void raid_gen5_int8(int nd, size_t size, void **vv);
void raid_gen5_ssse3(int nd, size_t size, void **vv);
void raid_gen5_ssse3ext(int nd, size_t size, void **vv);
void raid_gen5_avx2ext(int nd, size_t size, void **vv);
void raid_gen5_ssse3_raid(int nd, size_t size, void **vv);
void raid_gen5_ssse3ext_raid(int nd, size_t size, void **vv);
void raid_gen5_ssse3ext_aes(int nd, size_t size, void **vv);
void raid_gen5_avx2ext_raid(int nd, size_t size, void **vv);
void raid_gen5_avx2ext_aes(int nd, size_t size, void **vv);
void raid_gen5_avx512bw(int nd, size_t size, void **vv);
void raid_gen5_avx2gfni(int nd, size_t size, void **vv);
void raid_gen5_avx512gfni(int nd, size_t size, void **vv);
void raid_gen5_avx2gfni_aes(int nd, size_t size, void **vv);
void raid_gen5_avx512gfni_aes(int nd, size_t size, void **vv);
void raid_gen6_int8(int nd, size_t size, void **vv);
void raid_gen6_ssse3(int nd, size_t size, void **vv);
void raid_gen6_ssse3ext(int nd, size_t size, void **vv);
void raid_gen6_avx2ext(int nd, size_t size, void **vv);
void raid_gen6_ssse3_raid(int nd, size_t size, void **vv);
void raid_gen6_ssse3ext_raid(int nd, size_t size, void **vv);
void raid_gen6_ssse3ext_aes(int nd, size_t size, void **vv);
void raid_gen6_avx2ext_raid(int nd, size_t size, void **vv);
void raid_gen6_avx2ext_aes(int nd, size_t size, void **vv);
void raid_gen6_avx512bw(int nd, size_t size, void **vv);
void raid_gen6_avx2gfni(int nd, size_t size, void **vv);
void raid_gen6_avx512gfni(int nd, size_t size, void **vv);
void raid_gen6_avx2gfni_aes(int nd, size_t size, void **vv);
void raid_gen6_avx512gfni_aes(int nd, size_t size, void **vv);
void raid_rec1_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec2_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_recX_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
@ -150,12 +168,17 @@ void raid_rec1_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **
void raid_rec2_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_recX_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_gen1_neon(int nd, size_t size, void **vv);
void raid_gen2_neon(int nd, size_t size, void **vv);
void raid_genz_neon(int nd, size_t size, void **vv);
void raid_gen3_neon(int nd, size_t size, void **vv);
void raid_gen4_neon(int nd, size_t size, void **vv);
void raid_gen5_neon(int nd, size_t size, void **vv);
void raid_gen6_neon(int nd, size_t size, void **vv);
void raid_gen2_neon_raid(int nd, size_t size, void **vv);
void raid_gen2_neon_aes(int nd, size_t size, void **vv);
void raid_genz_neon_raid(int nd, size_t size, void **vv);
void raid_gen3_neon_raid(int nd, size_t size, void **vv);
void raid_gen3_neon_aes(int nd, size_t size, void **vv);
void raid_gen4_neon_raid(int nd, size_t size, void **vv);
void raid_gen4_neon_aes(int nd, size_t size, void **vv);
void raid_gen5_neon_raid(int nd, size_t size, void **vv);
void raid_gen5_neon_aes(int nd, size_t size, void **vv);
void raid_gen6_neon_raid(int nd, size_t size, void **vv);
void raid_gen6_neon_aes(int nd, size_t size, void **vv);
void raid_rec1_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec2_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);

42
raid/mktables.c
View file

@ -58,7 +58,7 @@ uint8_t raid_gfinv[256];
/**
* Setup the Cauchy matrix used to generate the parity.
*/
static void set_cauchy(uint8_t poly, uint8_t *matrix)
static void set_cauchy(uint8_t poly, uint8_t generator, uint8_t *matrix)
{
int i, j;
uint8_t inv_x, y;
@ -83,18 +83,21 @@ static void set_cauchy(uint8_t poly, uint8_t *matrix)
matrix[0 * DISK + i] = 1;
/*
* The second row is formed with powers 2^i, and it's the first
* The second row is formed with powers g^i, and it's the first
* row of the Cauchy matrix.
*
* Each element of the Cauchy matrix is in the form 1/(x_i + y_j)
* where all x_i and y_j must be different for any i and j.
*
* For the first row with j=0, we choose x_i = 2^-i and y_0 = 0
* For the first row with j=0, we choose x_i = g^-i and y_0 = 0
* and we obtain a first row formed as:
*
* 1/(x_i + y_0) = 1/(2^-i + 0) = 2^i
* 1/(x_i + y_0) = 1/(g^-i + 0) = g^i
*
* with 2^-i != 0 for any i
* with g^-i != 0 for any i
*
* The numerical examples below use RAID_MODE_CAUCHY_RAID,
* with polynomial 0x11d and generator g=2.
*
* In the example we get:
*
@ -116,17 +119,18 @@ static void set_cauchy(uint8_t poly, uint8_t *matrix)
inv_x = 1;
for (i = 0; i < DISK; ++i) {
matrix[1 * DISK + i] = inv_x;
inv_x = raid_gfmul(poly, 2, inv_x);
inv_x = raid_gfmul(poly, generator, inv_x);
}
/*
* The rest of the Cauchy matrix is formed by choosing for each row j
* a new y_j = 2^j and reusing the x_i already assigned in the first
* a new y_j = g^j and reusing the x_i already assigned in the first
* row obtaining :
*
* 1/(x_i + y_j) = 1/(2^-i + 2^j)
* 1/(x_i + y_j) = 1/(g^-i + g^j)
*
* with 2^-i + 2^j != 0 for any i,j with i>=0,j>=1,i+j<255
* with g^-i + g^j != 0, because g is primitive and has
* multiplicative order 255, with i + j < 255 for all supported entries.
*
* In the example we get:
*
@ -140,17 +144,17 @@ static void set_cauchy(uint8_t poly, uint8_t *matrix)
* 244 83 78 183 118 47
* 167 39 213 59 153 82
*/
y = 2;
y = generator;
for (j = 0; j < PARITY - 2; ++j) {
inv_x = 1;
for (i = 0; i < DISK; ++i) {
uint8_t x = raid_gfinv[inv_x];
matrix[(j + 2) * DISK + i] = raid_gfinv[y ^ x];
inv_x = raid_gfmul(poly, 2, inv_x);
inv_x = raid_gfmul(poly, generator, inv_x);
}
y = raid_gfmul(poly, 2, y);
y = raid_gfmul(poly, generator, y);
}
/*
@ -164,7 +168,7 @@ static void set_cauchy(uint8_t poly, uint8_t *matrix)
* 1 1 1 1 1 1
* 1 2 4 8 16 32
* 1 245 210 196 154 113
* 1 187 166 215 7 106
* 1 187 166 215 199 7
*/
for (j = 0; j < PARITY - 2; ++j) {
uint8_t f = raid_gfinv[matrix[(j + 2) * DISK]];
@ -200,7 +204,7 @@ static void set_power(uint8_t poly, uint8_t *matrix)
}
}
void tables(uint8_t poly, const char *tag)
void tables(uint8_t poly, uint8_t generator, const char *tag)
{
uint8_t v;
int i, j, k, p;
@ -227,7 +231,7 @@ void tables(uint8_t poly, const char *tag)
}
printf("};\n\n");
/* 2^a */
/* generator^a */
printf("const uint8_t __aligned(256) raid_gfexp_%s[256] =\n", tag);
printf("{\n");
v = 1;
@ -235,7 +239,7 @@ void tables(uint8_t poly, const char *tag)
if (i % 8 == 0)
printf("\t");
printf("0x%02x,", v);
v = raid_gfmul(poly, v, 2);
v = raid_gfmul(poly, v, generator);
if (i % 8 == 7)
printf("\n");
else
@ -263,7 +267,7 @@ void tables(uint8_t poly, const char *tag)
printf("};\n\n");
/* cauchy matrix */
set_cauchy(poly, matrix);
set_cauchy(poly, generator, matrix);
printf("/**\n");
printf(" * Cauchy matrix used to generate parity.\n");
@ -403,10 +407,10 @@ int main(void)
printf("\n");
printf("/* Tables with the RAID Polynomial 0x1d */\n");
tables(RAID_POLY_RAID, "raid");
tables(RAID_POLY_RAID, 2, "raid");
printf("/* Tables with the AES Polynomial 0x1b */\n");
tables(RAID_POLY_AES, "aes");
tables(RAID_POLY_AES, 3, "aes");
return 0;
}

110
raid/neon.c
View file

@ -60,9 +60,9 @@ void raid_gen1_neon(int nd, size_t size, void **vv)
}
/*
* GEN2 (RAID6 with powers of 2) NEON implementation
* GEN2 Cauchy NEON implementation using the active generator
*/
void raid_gen2_neon(int nd, size_t size, void **vv)
static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, int generator)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -93,6 +93,13 @@ void raid_gen2_neon(int nd, size_t size, void **vv)
);
for (d = l - 1; d >= 0; --d) {
if (generator == 3) {
/* v14-v15 are otherwise unused and preserve Q across the AES xtime. */
asm volatile (
"mov v14.16b, v2.16b\n"
"mov v15.16b, v3.16b\n"
);
}
asm volatile (
/* double Q0-Q1 */
"sshr v16.16b, v2.16b, #7\n"
@ -115,6 +122,12 @@ void raid_gen2_neon(int nd, size_t size, void **vv)
:
: "m" (v[d][i]), "m" (v[d][i + 16])
);
if (generator == 3) {
asm volatile (
"eor v2.16b, v2.16b, v14.16b\n"
"eor v3.16b, v3.16b, v15.16b\n"
);
}
}
asm volatile (
@ -130,10 +143,20 @@ void raid_gen2_neon(int nd, size_t size, void **vv)
raid_neon_end();
}
void raid_gen2_neon_raid(int nd, size_t size, void **vv)
{
raid_gen2_neon_gen(nd, size, vv, 2);
}
void raid_gen2_neon_aes(int nd, size_t size, void **vv)
{
raid_gen2_neon_gen(nd, size, vv, 3);
}
/*
* GENz (triple parity with powers of 2^-1) NEON implementation
*/
void raid_genz_neon(int nd, size_t size, void **vv)
void raid_genz_neon_raid(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -227,7 +250,7 @@ void raid_genz_neon(int nd, size_t size, void **vv)
/*
* GENX NEON implementation
*/
static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int np)
static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int np, int generator)
{
uint8_t **v = (uint8_t **)vv;
size_t i;
@ -253,7 +276,7 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
);
for (i = 0; i < size; i += 32) {
/* last disk without the multiplication by two */
/* last disk without the generator multiplication */
asm volatile (
"ldr q0, %0\n"
"ldr q1, %1\n"
@ -336,6 +359,12 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
/* intermediate disks */
for (d = l - 1; d > 0; --d) {
if (generator == 3) {
asm volatile (
"mov v14.16b, v2.16b\n"
"mov v15.16b, v3.16b\n"
);
}
asm volatile (
/* double Q0-Q1 */
"sshr v20.16b, v2.16b, #7\n"
@ -358,6 +387,12 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
:
: "m" (v[d][i]), "m" (v[d][i + 16])
);
if (generator == 3) {
asm volatile (
"eor v2.16b, v2.16b, v14.16b\n"
"eor v3.16b, v3.16b, v15.16b\n"
);
}
if (np >= 3) {
asm volatile (
@ -440,6 +475,12 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
}
/* first disk with all coefficients at 1 */
if (generator == 3) {
asm volatile (
"mov v14.16b, v2.16b\n"
"mov v15.16b, v3.16b\n"
);
}
asm volatile (
/* double Q0-Q1 */
"sshr v20.16b, v2.16b, #7\n"
@ -462,6 +503,12 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
:
: "m" (v[0][i]), "m" (v[0][i + 16])
);
if (generator == 3) {
asm volatile (
"eor v2.16b, v2.16b, v14.16b\n"
"eor v3.16b, v3.16b, v15.16b\n"
);
}
if (np >= 3) {
asm volatile (
@ -533,33 +580,53 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
/*
* GEN3 (triple parity with Cauchy matrix) NEON implementation
*/
void raid_gen3_neon(int nd, size_t size, void **vv)
void raid_gen3_neon_raid(int nd, size_t size, void **vv)
{
raid_genX_neon(nd, size, vv, 3);
raid_genX_neon(nd, size, vv, 3, 2);
}
void raid_gen3_neon_aes(int nd, size_t size, void **vv)
{
raid_genX_neon(nd, size, vv, 3, 3);
}
/*
* GEN4 (quad parity with Cauchy matrix) NEON implementation
*/
void raid_gen4_neon(int nd, size_t size, void **vv)
void raid_gen4_neon_raid(int nd, size_t size, void **vv)
{
raid_genX_neon(nd, size, vv, 4);
raid_genX_neon(nd, size, vv, 4, 2);
}
void raid_gen4_neon_aes(int nd, size_t size, void **vv)
{
raid_genX_neon(nd, size, vv, 4, 3);
}
/*
* GEN5 (penta parity with Cauchy matrix) NEON implementation
*/
void raid_gen5_neon(int nd, size_t size, void **vv)
void raid_gen5_neon_raid(int nd, size_t size, void **vv)
{
raid_genX_neon(nd, size, vv, 5);
raid_genX_neon(nd, size, vv, 5, 2);
}
void raid_gen5_neon_aes(int nd, size_t size, void **vv)
{
raid_genX_neon(nd, size, vv, 5, 3);
}
/*
* GEN6 (hexa parity with Cauchy matrix) NEON implementation
*/
void raid_gen6_neon(int nd, size_t size, void **vv)
void raid_gen6_neon_raid(int nd, size_t size, void **vv)
{
raid_genX_neon(nd, size, vv, 6);
raid_genX_neon(nd, size, vv, 6, 2);
}
void raid_gen6_neon_aes(int nd, size_t size, void **vv)
{
raid_genX_neon(nd, size, vv, 6, 3);
}
/*
@ -811,12 +878,17 @@ void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
void raid_register_neon(void)
{
raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "neon", raid_gen1_neon, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "neon", raid_gen2_neon, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "neon", raid_genz_neon, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "neon", raid_gen3_neon, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "neon", raid_gen4_neon, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "neon", raid_gen5_neon, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "neon", raid_gen6_neon, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "neon", raid_gen2_neon_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "neon", raid_gen2_neon_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "neon", raid_genz_neon_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "neon", raid_gen3_neon_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "neon", raid_gen3_neon_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "neon", raid_gen4_neon_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "neon", raid_gen4_neon_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "neon", raid_gen5_neon_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "neon", raid_gen5_neon_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "neon", raid_gen6_neon_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "neon", raid_gen6_neon_aes, RAID_POLY_AES);
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "neon", raid_rec1_neon, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "neon", raid_rec2_neon, RAID_POLY_ANY);

53
raid/raid.c
View file

@ -38,6 +38,19 @@
* nonsingular, so the linear system is always solvable for any combination
* of missing disks. This guarantees an MDS code.
*
* The Cauchy construction below uses a primitive generator g.
*
* RAID_MODE_CAUCHY_RAID uses:
* polynomial 0x11d
* generator g = 2
*
* RAID_MODE_CAUCHY_AES uses:
* polynomial 0x11b
* generator g = 3
*
* The numerical examples below refer to RAID_MODE_CAUCHY_RAID,
* using polynomial 0x11d and generator g = 2.
*
* How the matrix is obtained:
*
* The matrix is constructed to match Linux RAID coefficients for the
@ -47,17 +60,18 @@
* where all x_i and y_j are distinct elements (the textbook definition
* of a Cauchy matrix).
*
* 2. For the first row (j=0), set x_i = 2^-i and y_0 = 0, resulting in:
* 2. For the first row (j=0), set x_i = g^-i and y_0 = 0, resulting in:
*
* row j=0 -> 1/(x_i + y_0) = 1/(2^-i + 0) = 2^i
* row j=0 -> 1/(x_i + y_0) = 1/(g^-i + 0) = g^i
*
* which reproduces the RAID-6 coefficients.
*
* 3. For subsequent rows (j>0), set y_j = 2^j, yielding:
* 3. For subsequent rows (j>0), set y_j = g^j, yielding:
*
* rows j>0 -> 1/(x_i + y_j) = 1/(2^-i + 2^j)
* rows j>0 -> 1/(x_i + y_j) = 1/(g^-i + g^j)
*
* ensuring x_i != y_j for any i >= 0, j >= 1, and i + j < 255.
* Since g is primitive and has multiplicative order 255,
* g^-i != g^j for the supported indexes where i + j < 255.
*
* 4. Place a row filled with 1 at the top of the matrix, transforming it
* into an Extended Cauchy Matrix. This preserves the nonsingularity
@ -71,7 +85,8 @@
*
* Concrete example in GF(256) for k=6, m=4:
*
* First, create a 3x6 Cauchy matrix using x_i = 2^-i and y_0 = 0, y_j = 2^j for j>0:
* First, create a 3x6 Cauchy matrix using x_i = g^-i and y_0 = 0, y_j = g^j for j>0.
* For RAID_MODE_CAUCHY_RAID, where g=2, this gives:
*
* x = { 1, 142, 71, 173, 216, 108 }
* y = { 0, 2, 4 }
@ -124,17 +139,20 @@
* Parity computation is as follows:
*
* P = sum(Di)
* Q = sum(2^i * Di)
* Q = sum(g^i * Di)
* R = sum(A[2,i] * Di)
* S = sum(A[3,i] * Di)
* T = sum(A[4,i] * Di)
* U = sum(A[5,i] * Di) for 0 <= i < N
*
* where g=2 for RAID_MODE_CAUCHY_RAID and
* g=3 for RAID_MODE_CAUCHY_AES.
*
* Recovery from six disk failures at indices x, y, z, h, v, w (0 <= x < y < z < h < v < w < N)
* involves computing parity of the remaining N-6 disks:
*
* Pa = sum(Di)
* Qa = sum(2^i * Di)
* Qa = sum(g^i * Di)
* Ra = sum(A[2,i] * Di)
* Sa = sum(A[3,i] * Di)
* Ta = sum(A[4,i] * Di)
@ -152,12 +170,15 @@
* yields:
*
* Pd = Dx + Dy + Dz + Dh + Dv + Dw
* Qd = 2^x * Dx + 2^y * Dy + 2^z * Dz + 2^h * Dh + 2^v * Dv + 2^w * Dw
* Qd = g^x * Dx + g^y * Dy + g^z * Dz + g^h * Dh + g^v * Dv + g^w * Dw
* Rd = A[2,x] * Dx + A[2,y] * Dy + A[2,z] * Dz + A[2,h] * Dh + A[2,v] * Dv + A[2,w] * Dw
* Sd = A[3,x] * Dx + A[3,y] * Dy + A[3,z] * Dz + A[3,h] * Dh + A[3,v] * Dv + A[3,w] * Dw
* Td = A[4,x] * Dx + A[4,y] * Dy + A[4,z] * Dz + A[4,h] * Dh + A[4,v] * Dv + A[4,w] * Dw
* Ud = A[5,x] * Dx + A[5,y] * Dy + A[5,z] * Dz + A[5,h] * Dh + A[5,v] * Dv + A[5,w] * Dw
*
* with g=2 for RAID_MODE_CAUCHY_RAID and
* g=3 for RAID_MODE_CAUCHY_AES.
*
* This linear system is always solvable since the coefficient matrix is
* nonsingular due to the properties of A[].
*
@ -398,20 +419,20 @@ void raid_rec1of1(int *id, int nd, size_t size, void **v)
* Starting from the equations:
*
* Pd = Dx + Dy
* Qd = 2^id[0] * Dx + 2^id[1] * Dy
* Qd = g^id[0] * Dx + g^id[1] * Dy
*
* and solving we get:
*
* 1 2^(-id[0])
* 1 g^(-id[0])
* Dy = ------------------- * Pd + ------------------- * Qd
* 2^(id[1]-id[0]) + 1 2^(id[1]-id[0]) + 1
* g^(id[1]-id[0]) + 1 g^(id[1]-id[0]) + 1
*
* Dx = Dy + Pd
*
* with conditions:
*
* 2^id[0] != 0
* 2^(id[1]-id[0]) + 1 != 0
* g^id[0] != 0
* g^(id[1]-id[0]) + 1 != 0
*
* That are always satisfied for any 0<=id[0]<id[1]<255.
*/
@ -426,8 +447,8 @@ void raid_rec2of2_int8(int *id, int *ip, int nd, size_t size, void **vv)
const uint8_t *T[2];
/* get multiplication tables */
T[0] = table(inv(pow2(id[1] - id[0]) ^ 1));
T[1] = table(inv(pow2(id[0]) ^ pow2(id[1])));
T[0] = table(inv(powgen(id[1] - id[0]) ^ 1));
T[1] = table(inv(powgen(id[0]) ^ powgen(id[1])));
/* compute delta parity */
raid_delta_gen(2, id, ip, nd, size, vv);

33
raid/raid.h
View file

@ -9,23 +9,26 @@
*
* RAID parity computation operates in GF(2^8), a finite field where
* arithmetic is performed modulo an irreducible polynomial of degree 8.
* The choice of polynomial affects both correctness and performance:
* all tables, coefficients, and parity data are tied to a specific
* polynomial and cannot be mixed.
* The choice of operating mode affects both correctness and performance:
* all tables, coefficients, and parity data are tied to the selected
* polynomial and primitive generator and cannot be mixed between
* incompatible operating modes.
*
* Two polynomials are supported:
*
* 0x1d (x^8 + x^4 + x^3 + x^2 + 1) -- Standard RAID polynomial
*
* The polynomial used by the original RAID-6 specification and the
* Linux kernel RAID implementation. Choosing this polynomial ensures
* on-disk compatibility with existing RAID arrays and interchange with
* other RAID implementations. Use this when reading or writing data
* that may be accessed by other software or hardware RAID controllers.
* Linux kernel RAID implementation, used with primitive generator g=2.
* Choosing this polynomial ensures on-disk compatibility with existing
* RAID arrays and interchange with other RAID implementations. Use this
* when reading or writing data that may be accessed by other software or
* hardware RAID controllers.
*
* 0x1b (x^8 + x^4 + x^3 + x + 1) -- AES polynomial
*
* The polynomial used by the AES encryption standard. Choosing this
* The polynomial used by the AES encryption standard (0x11b including
* the x^8 term), used with primitive generator g=3. Choosing this
* polynomial enables the use of the Intel GFNI (Galois Field New
* Instructions) extension, specifically the vgf2p8mulb instruction,
* which performs GF(2^8) multiplication natively in hardware using
@ -36,8 +39,8 @@
* single vgf2p8mulb instruction. On CPUs supporting GFNI (Ice Lake and
* later), this yields a significant throughput improvement. The
* trade-off is that parity data is not compatible with arrays using
* the standard 0x1d polynomial, so this option requires that all
* data be generated and recovered using the same polynomial choice.
* the standard 0x1d polynomial (g=2), so this option requires that all
* data be generated and recovered using the same operating mode.
*/
/**
@ -46,7 +49,7 @@
#define RAID_MODE_GET -1
/**
* Default RAID mode supporting up to 6 parities using the standard polynomial 0x1d.
* Default RAID mode supporting up to 6 parities using standard polynomial 0x1d and generator g=2.
*
* Provides high performance on modern CPUs with SSSE3, AVX2, AVX-512, or NEON support.
* Ensures maximum compatibility with standard Linux RAID arrays.
@ -66,11 +69,11 @@
#define RAID_MODE_VANDERMONDE_RAID 1
/**
* RAID mode supporting up to 6 parities using AES polynomial 0x1b.
* RAID mode supporting up to 6 parities using AES polynomial 0x1b (0x11b) and primitive generator g=3.
*
* It has the same performance as RAID_MODE_CAUCHY_RAID on CPUs without GFNI,
* but achieves significantly higher throughput on CPUs supporting GFNI hardware
* acceleration.
* It has slightly lower performance than RAID_MODE_CAUCHY_RAID on CPUs without GFNI due
* to the additional cost of generator g=3, but achieves significantly higher throughput on
* CPUs supporting GFNI hardware acceleration.
*/
#define RAID_MODE_CAUCHY_AES 2

307
raid/sse2.c
View file

@ -52,29 +52,30 @@
* ================================================================
*
* This file implements different strategies for computing the Q parity.
* Q parity requires scaling each data disk by a descending power of 2
* in the Galois Field:
* Q parity requires scaling each data disk by a descending power of the
* primitive generator g (g=2 for RAID_MODE_CAUCHY_RAID, g=3 for
* RAID_MODE_CAUCHY_AES) in the Galois Field:
*
* Q = (D_l * 2^l) + (D_{l-1} * 2^{l-1}) + ... + (D_1 * 2^1) + (D_0 * 1)
* Q = (D_l * g^l) + (D_{l-1} * g^{l-1}) + ... + (D_1 * g^1) + (D_0 * 1)
*
* To efficiently calculate this sequentially using SIMD instructions, we
* use Horner's method. Depending on hardware architecture and whether we
* need to optimize out the final Cauchy matrix multiplication for D0, the
* "multiply by 2" operation can be positioned in three distinct ways:
* scaling operation can be positioned in distinct ways:
*
* -------------------------------------------------------------------------
* CASE 1: Multiplication BEFORE XOR, handling all disks inside the loop
* -------------------------------------------------------------------------
* Used in GEN2 (RAID6) where all disks can be processed uniformly.
* The accumulator is doubled BEFORE adding the current disk. Because the
* loop runs all the way down to D0 (d >= 0), the final disk is XORed into
* the accumulator and the loop terminates. This naturally leaves D0 with
* a coefficient of 1, as it escapes the doubling step.
* Used in GEN2 where all disks can be processed uniformly.
* The accumulator is scaled by the active generator BEFORE adding the current disk.
* Because the loop runs all the way down to D0 (d >= 0), the final disk is XORed
* into the accumulator and the loop terminates. This naturally leaves D0 with
* a coefficient of 1, as it escapes the scaling step.
*
* Simplified Code:
* Q = D_l;
* for (d = l - 1; d >= 0; --d) {
* Q = Q * 2; // Scale accumulator
* Q = Q * g; // Scale accumulator by active generator
* Q = Q ^ D_d; // Add current disk
* }
*
@ -89,12 +90,12 @@
* Simplified Code:
* Q = D_l;
* for (d = l - 1; d > 0; --d) {
* Q = Q * 2; // Scale accumulator
* Q = Q * g; // Scale accumulator by active generator
* Q = Q ^ D_d; // Add current disk
* }
*
* // Final disk (D0) processed outside the loop to skip Cauchy math
* Q = Q * 2;
* Q = Q * g;
* Q = Q ^ D_0;
*/
@ -136,9 +137,9 @@ void raid_gen1_sse2(int nd, size_t size, void **vv)
}
/*
* GEN2 (RAID6 with powers of 2) SSE2 implementation
* GEN2 Cauchy SSE2 implementation using the active generator
*/
void raid_gen2_sse2(int nd, size_t size, void **vv)
static __always_inline void raid_gen2_sse2_gen(int nd, size_t size, void **vv, int generator)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -161,17 +162,34 @@ void raid_gen2_sse2(int nd, size_t size, void **vv)
asm volatile ("movdqa %xmm1,%xmm3");
for (d = l - 1; d >= 0; --d) {
/* scale by 2 before adding the current disk so the first (last processed) disk with factor 1 avoids doubling */
asm volatile ("pxor %xmm4,%xmm4");
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pcmpgtb %xmm2,%xmm4");
asm volatile ("pcmpgtb %xmm3,%xmm5");
asm volatile ("paddb %xmm2,%xmm2");
asm volatile ("paddb %xmm3,%xmm3");
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("pxor %xmm4,%xmm2");
asm volatile ("pxor %xmm5,%xmm3");
/* scale Q by the active generator before adding the current disk */
if (generator == 3) {
asm volatile ("movdqa %xmm2,%xmm4");
asm volatile ("movdqa %xmm3,%xmm5");
asm volatile ("pxor %xmm6,%xmm6");
asm volatile ("pcmpgtb %xmm4,%xmm6");
asm volatile ("paddb %xmm2,%xmm2");
asm volatile ("paddb %xmm3,%xmm3");
asm volatile ("pxor %xmm4,%xmm2");
asm volatile ("pxor %xmm4,%xmm4");
asm volatile ("pcmpgtb %xmm5,%xmm4");
asm volatile ("pand %xmm7,%xmm6");
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pxor %xmm6,%xmm2");
asm volatile ("pxor %xmm4,%xmm3");
asm volatile ("pxor %xmm5,%xmm3");
} else {
asm volatile ("pxor %xmm4,%xmm4");
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pcmpgtb %xmm2,%xmm4");
asm volatile ("pcmpgtb %xmm3,%xmm5");
asm volatile ("paddb %xmm2,%xmm2");
asm volatile ("paddb %xmm3,%xmm3");
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("pxor %xmm4,%xmm2");
asm volatile ("pxor %xmm5,%xmm3");
}
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
asm volatile ("movdqa %0,%%xmm5" : : "m" (v[d][i + 16]));
@ -190,13 +208,23 @@ void raid_gen2_sse2(int nd, size_t size, void **vv)
raid_sse_end();
}
void raid_gen2_sse2_raid(int nd, size_t size, void **vv)
{
raid_gen2_sse2_gen(nd, size, vv, 2);
}
void raid_gen2_sse2_aes(int nd, size_t size, void **vv)
{
raid_gen2_sse2_gen(nd, size, vv, 3);
}
#ifdef CONFIG_X86_64
/*
* GEN2 (RAID6 with powers of 2) SSE2 implementation
* GEN2 Cauchy SSE2 implementation using the active generator
*
* Note that it uses 16 registers, meaning that x64 is required.
*/
void raid_gen2_sse2ext(int nd, size_t size, void **vv)
static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv, int generator)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -222,33 +250,179 @@ void raid_gen2_sse2ext(int nd, size_t size, void **vv)
asm volatile ("movdqa %xmm2,%xmm6");
asm volatile ("movdqa %xmm3,%xmm7");
for (d = l - 1; d >= 0; --d) {
/* scale by 2 before adding the current disk so the first (last processed) disk with factor 1 avoids doubling */
asm volatile ("pxor %xmm8,%xmm8");
asm volatile ("pxor %xmm9,%xmm9");
asm volatile ("pxor %xmm10,%xmm10");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm4,%xmm8");
asm volatile ("pcmpgtb %xmm5,%xmm9");
asm volatile ("pcmpgtb %xmm6,%xmm10");
asm volatile ("pcmpgtb %xmm7,%xmm11");
asm volatile ("paddb %xmm4,%xmm4");
asm volatile ("paddb %xmm5,%xmm5");
asm volatile ("paddb %xmm6,%xmm6");
asm volatile ("paddb %xmm7,%xmm7");
asm volatile ("pand %xmm15,%xmm8");
asm volatile ("pand %xmm15,%xmm9");
asm volatile ("pand %xmm15,%xmm10");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm8,%xmm4");
asm volatile ("pxor %xmm9,%xmm5");
asm volatile ("pxor %xmm10,%xmm6");
asm volatile ("pxor %xmm11,%xmm7");
/* process two disks per iteration */
for (d = l - 1; d >= 1; d -= 2) {
/* lane 0 */
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d][i + 16]));
asm volatile ("movdqa %0,%%xmm10" : : "m" (v[d][i + 32]));
asm volatile ("movdqa %0,%%xmm11" : : "m" (v[d][i + 48]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i]));
if (generator == 3)
asm volatile ("movdqa %xmm4,%xmm10");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm4,%xmm11");
asm volatile ("paddb %xmm4,%xmm4");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm4");
asm volatile ("pxor %xmm8,%xmm4");
asm volatile ("pxor %xmm9,%xmm8");
asm volatile ("pxor %xmm8,%xmm0");
if (generator == 3)
asm volatile ("pxor %xmm10,%xmm8");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm4,%xmm11");
asm volatile ("paddb %xmm4,%xmm4");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm4");
if (generator == 3)
asm volatile ("pxor %xmm8,%xmm4");
else
asm volatile ("pxor %xmm9,%xmm4");
/* lane 1 */
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i + 16]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i + 16]));
if (generator == 3)
asm volatile ("movdqa %xmm5,%xmm10");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm5,%xmm11");
asm volatile ("paddb %xmm5,%xmm5");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm5");
asm volatile ("pxor %xmm8,%xmm5");
asm volatile ("pxor %xmm9,%xmm8");
asm volatile ("pxor %xmm8,%xmm1");
if (generator == 3)
asm volatile ("pxor %xmm10,%xmm8");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm5,%xmm11");
asm volatile ("paddb %xmm5,%xmm5");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm5");
if (generator == 3)
asm volatile ("pxor %xmm8,%xmm5");
else
asm volatile ("pxor %xmm9,%xmm5");
/* lane 2 */
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i + 32]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i + 32]));
if (generator == 3)
asm volatile ("movdqa %xmm6,%xmm10");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm6,%xmm11");
asm volatile ("paddb %xmm6,%xmm6");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm6");
asm volatile ("pxor %xmm8,%xmm6");
asm volatile ("pxor %xmm9,%xmm8");
asm volatile ("pxor %xmm8,%xmm2");
if (generator == 3)
asm volatile ("pxor %xmm10,%xmm8");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm6,%xmm11");
asm volatile ("paddb %xmm6,%xmm6");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm6");
if (generator == 3)
asm volatile ("pxor %xmm8,%xmm6");
else
asm volatile ("pxor %xmm9,%xmm6");
/* lane 3 */
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i + 48]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i + 48]));
if (generator == 3)
asm volatile ("movdqa %xmm7,%xmm10");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm7,%xmm11");
asm volatile ("paddb %xmm7,%xmm7");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm7");
asm volatile ("pxor %xmm8,%xmm7");
asm volatile ("pxor %xmm9,%xmm8");
asm volatile ("pxor %xmm8,%xmm3");
if (generator == 3)
asm volatile ("pxor %xmm10,%xmm8");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm7,%xmm11");
asm volatile ("paddb %xmm7,%xmm7");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm7");
if (generator == 3)
asm volatile ("pxor %xmm8,%xmm7");
else
asm volatile ("pxor %xmm9,%xmm7");
}
/* single remaining disk */
if (d == 0) {
if (generator == 3) {
asm volatile ("movdqa %xmm4,%xmm8");
asm volatile ("movdqa %xmm5,%xmm9");
asm volatile ("movdqa %xmm6,%xmm10");
asm volatile ("movdqa %xmm7,%xmm11");
asm volatile ("pxor %xmm12,%xmm12");
asm volatile ("pxor %xmm13,%xmm13");
asm volatile ("pxor %xmm14,%xmm14");
asm volatile ("pcmpgtb %xmm8,%xmm12");
asm volatile ("pcmpgtb %xmm9,%xmm13");
asm volatile ("pcmpgtb %xmm10,%xmm14");
asm volatile ("paddb %xmm4,%xmm4");
asm volatile ("paddb %xmm5,%xmm5");
asm volatile ("paddb %xmm6,%xmm6");
asm volatile ("paddb %xmm7,%xmm7");
asm volatile ("pxor %xmm8,%xmm4");
asm volatile ("pxor %xmm8,%xmm8");
asm volatile ("pcmpgtb %xmm11,%xmm8");
asm volatile ("pand %xmm15,%xmm12");
asm volatile ("pand %xmm15,%xmm13");
asm volatile ("pand %xmm15,%xmm14");
asm volatile ("pand %xmm15,%xmm8");
asm volatile ("pxor %xmm12,%xmm4");
asm volatile ("pxor %xmm13,%xmm5");
asm volatile ("pxor %xmm14,%xmm6");
asm volatile ("pxor %xmm8,%xmm7");
asm volatile ("pxor %xmm9,%xmm5");
asm volatile ("pxor %xmm10,%xmm6");
asm volatile ("pxor %xmm11,%xmm7");
} else {
asm volatile ("pxor %xmm8,%xmm8");
asm volatile ("pxor %xmm9,%xmm9");
asm volatile ("pxor %xmm10,%xmm10");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm4,%xmm8");
asm volatile ("pcmpgtb %xmm5,%xmm9");
asm volatile ("pcmpgtb %xmm6,%xmm10");
asm volatile ("pcmpgtb %xmm7,%xmm11");
asm volatile ("paddb %xmm4,%xmm4");
asm volatile ("paddb %xmm5,%xmm5");
asm volatile ("paddb %xmm6,%xmm6");
asm volatile ("paddb %xmm7,%xmm7");
asm volatile ("pand %xmm15,%xmm8");
asm volatile ("pand %xmm15,%xmm9");
asm volatile ("pand %xmm15,%xmm10");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm8,%xmm4");
asm volatile ("pxor %xmm9,%xmm5");
asm volatile ("pxor %xmm10,%xmm6");
asm volatile ("pxor %xmm11,%xmm7");
}
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[0][i]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[0][i + 16]));
asm volatile ("movdqa %0,%%xmm10" : : "m" (v[0][i + 32]));
asm volatile ("movdqa %0,%%xmm11" : : "m" (v[0][i + 48]));
asm volatile ("pxor %xmm8,%xmm0");
asm volatile ("pxor %xmm9,%xmm1");
asm volatile ("pxor %xmm10,%xmm2");
@ -271,12 +445,22 @@ void raid_gen2_sse2ext(int nd, size_t size, void **vv)
raid_sse_end();
}
void raid_gen2_sse2ext_raid(int nd, size_t size, void **vv)
{
raid_gen2_sse2ext_gen(nd, size, vv, 2);
}
void raid_gen2_sse2ext_aes(int nd, size_t size, void **vv)
{
raid_gen2_sse2ext_gen(nd, size, vv, 3);
}
#endif
/*
* GENz (triple parity with powers of 2^-1) SSE2 implementation
*/
void raid_genz_sse2(int nd, size_t size, void **vv)
void raid_genz_sse2_raid(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -335,7 +519,7 @@ void raid_genz_sse2(int nd, size_t size, void **vv)
*
* Note that it uses 16 registers, meaning that x64 is required.
*/
void raid_genz_sse2ext(int nd, size_t size, void **vv)
void raid_genz_sse2ext_raid(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -415,14 +599,17 @@ void raid_register_sse2(void)
{
if (raid_cpu_has_sse2()) {
raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "sse2", raid_gen1_sse2, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "sse2", raid_gen2_sse2, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "sse2", raid_gen2_sse2_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "sse2", raid_gen2_sse2_aes, RAID_POLY_AES);
#ifdef CONFIG_X86_64
if (!raid_cpu_has_slow_extendedreg())
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "sse2e", raid_gen2_sse2ext, RAID_POLY_ANY);
if (!raid_cpu_has_slow_extendedreg()) {
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "sse2e", raid_gen2_sse2ext_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "sse2e", raid_gen2_sse2ext_aes, RAID_POLY_AES);
}
/* note that raid_cpu_has_slow_extendedreg() doesn't affect vandermonde */
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "sse2e", raid_genz_sse2ext, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "sse2e", raid_genz_sse2ext_raid, RAID_POLY_RAID);
#else
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "sse2", raid_genz_sse2, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "sse2", raid_genz_sse2_raid, RAID_POLY_RAID);
#endif
}
}

275
raid/ssse3.c
View file

@ -9,7 +9,7 @@
/*
* GEN3 (triple parity with Cauchy matrix) SSSE3 implementation
*/
void raid_gen3_ssse3(int nd, size_t size, void **vv)
static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv, int generator)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -37,7 +37,7 @@ void raid_gen3_ssse3(int nd, size_t size, void **vv)
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
for (i = 0; i < size; i += 16) {
/* last disk without the multiplication by two */
/* last disk without the generator multiplication */
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));
asm volatile ("movdqa %xmm4,%xmm0");
@ -58,11 +58,15 @@ void raid_gen3_ssse3(int nd, size_t size, void **vv)
for (d = l - 1; d > 0; --d) {
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
if (generator == 3)
asm volatile ("movdqa %xmm1,%xmm6");
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pcmpgtb %xmm1,%xmm5");
asm volatile ("paddb %xmm1,%xmm1");
asm volatile ("pand %xmm3,%xmm5");
asm volatile ("pxor %xmm5,%xmm1");
if (generator == 3)
asm volatile ("pxor %xmm6,%xmm1");
asm volatile ("pxor %xmm4,%xmm0");
asm volatile ("pxor %xmm4,%xmm1");
@ -83,11 +87,15 @@ void raid_gen3_ssse3(int nd, size_t size, void **vv)
/* first disk with all coefficients at 1 */
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));
if (generator == 3)
asm volatile ("movdqa %xmm1,%xmm6");
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pcmpgtb %xmm1,%xmm5");
asm volatile ("paddb %xmm1,%xmm1");
asm volatile ("pand %xmm3,%xmm5");
asm volatile ("pxor %xmm5,%xmm1");
if (generator == 3)
asm volatile ("pxor %xmm6,%xmm1");
asm volatile ("pxor %xmm4,%xmm0");
asm volatile ("pxor %xmm4,%xmm1");
@ -101,13 +109,23 @@ void raid_gen3_ssse3(int nd, size_t size, void **vv)
raid_sse_end();
}
void raid_gen3_ssse3_raid(int nd, size_t size, void **vv)
{
raid_gen3_ssse3_gen(nd, size, vv, 2);
}
void raid_gen3_ssse3_aes(int nd, size_t size, void **vv)
{
raid_gen3_ssse3_gen(nd, size, vv, 3);
}
#ifdef CONFIG_X86_64
/*
* GEN3 (triple parity with Cauchy matrix) SSSE3 implementation
*
* Note that it uses 16 registers, meaning that x64 is required.
*/
void raid_gen3_ssse3ext(int nd, size_t size, void **vv)
static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **vv, int generator)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -135,7 +153,7 @@ void raid_gen3_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.low4[0]));
for (i = 0; i < size; i += 32) {
/* last disk without the by two multiplication */
/* last disk without the generator multiplication */
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[l][i + 16]));
@ -164,11 +182,19 @@ void raid_gen3_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("pxor %xmm7,%xmm2");
asm volatile ("pxor %xmm15,%xmm10");
/* intermediate disks */
for (d = l - 1; d > 0; --d) {
/* process two intermediate disks per iteration */
for (d = l - 1; d > 1; d -= 2) {
/* disk d */
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
asm volatile ("movdqa %0,%%xmm15" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
if (generator == 3) {
asm volatile ("movdqa %xmm1,%xmm6");
asm volatile ("movdqa %xmm9,%xmm14");
}
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pxor %xmm13,%xmm13");
asm volatile ("pcmpgtb %xmm1,%xmm5");
@ -179,6 +205,10 @@ void raid_gen3_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("pand %xmm3,%xmm13");
asm volatile ("pxor %xmm5,%xmm1");
asm volatile ("pxor %xmm13,%xmm9");
if (generator == 3) {
asm volatile ("pxor %xmm6,%xmm1");
asm volatile ("pxor %xmm14,%xmm9");
}
asm volatile ("pxor %xmm4,%xmm0");
asm volatile ("pxor %xmm4,%xmm1");
@ -194,8 +224,107 @@ void raid_gen3_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("pand %xmm11,%xmm5");
asm volatile ("pand %xmm11,%xmm13");
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
asm volatile ("movdqa %xmm7,%xmm6");
asm volatile ("movdqa %xmm15,%xmm14");
asm volatile ("pshufb %xmm4,%xmm7");
asm volatile ("pshufb %xmm12,%xmm6");
asm volatile ("pshufb %xmm5,%xmm15");
asm volatile ("pshufb %xmm13,%xmm14");
asm volatile ("pxor %xmm7,%xmm2");
asm volatile ("pxor %xmm6,%xmm10");
asm volatile ("pxor %xmm15,%xmm2");
asm volatile ("pxor %xmm14,%xmm10");
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d - 1][i]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d - 1][i + 16]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d - 1][1][0][0]));
asm volatile ("movdqa %0,%%xmm15" : : "m" (raid_gfcauchypshufb[d - 1][1][1][0]));
if (generator == 3) {
asm volatile ("movdqa %xmm1,%xmm6");
asm volatile ("movdqa %xmm9,%xmm14");
}
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pxor %xmm13,%xmm13");
asm volatile ("pcmpgtb %xmm1,%xmm5");
asm volatile ("pcmpgtb %xmm9,%xmm13");
asm volatile ("paddb %xmm1,%xmm1");
asm volatile ("paddb %xmm9,%xmm9");
asm volatile ("pand %xmm3,%xmm5");
asm volatile ("pand %xmm3,%xmm13");
asm volatile ("pxor %xmm5,%xmm1");
asm volatile ("pxor %xmm13,%xmm9");
if (generator == 3) {
asm volatile ("pxor %xmm6,%xmm1");
asm volatile ("pxor %xmm14,%xmm9");
}
asm volatile ("pxor %xmm4,%xmm0");
asm volatile ("pxor %xmm4,%xmm1");
asm volatile ("pxor %xmm12,%xmm8");
asm volatile ("pxor %xmm12,%xmm9");
asm volatile ("movdqa %xmm4,%xmm5");
asm volatile ("movdqa %xmm12,%xmm13");
asm volatile ("psrlw $4,%xmm5");
asm volatile ("psrlw $4,%xmm13");
asm volatile ("pand %xmm11,%xmm4");
asm volatile ("pand %xmm11,%xmm12");
asm volatile ("pand %xmm11,%xmm5");
asm volatile ("pand %xmm11,%xmm13");
asm volatile ("movdqa %xmm7,%xmm6");
asm volatile ("movdqa %xmm15,%xmm14");
asm volatile ("pshufb %xmm4,%xmm7");
asm volatile ("pshufb %xmm12,%xmm6");
asm volatile ("pshufb %xmm5,%xmm15");
asm volatile ("pshufb %xmm13,%xmm14");
asm volatile ("pxor %xmm7,%xmm2");
asm volatile ("pxor %xmm6,%xmm10");
asm volatile ("pxor %xmm15,%xmm2");
asm volatile ("pxor %xmm14,%xmm10");
}
if (d == 1) {
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[1][i]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[1][i + 16]));
if (generator == 3) {
asm volatile ("movdqa %xmm1,%xmm6");
asm volatile ("movdqa %xmm9,%xmm14");
}
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pxor %xmm13,%xmm13");
asm volatile ("pcmpgtb %xmm1,%xmm5");
asm volatile ("pcmpgtb %xmm9,%xmm13");
asm volatile ("paddb %xmm1,%xmm1");
asm volatile ("paddb %xmm9,%xmm9");
asm volatile ("pand %xmm3,%xmm5");
asm volatile ("pand %xmm3,%xmm13");
asm volatile ("pxor %xmm5,%xmm1");
asm volatile ("pxor %xmm13,%xmm9");
if (generator == 3) {
asm volatile ("pxor %xmm6,%xmm1");
asm volatile ("pxor %xmm14,%xmm9");
}
asm volatile ("pxor %xmm4,%xmm0");
asm volatile ("pxor %xmm4,%xmm1");
asm volatile ("pxor %xmm12,%xmm8");
asm volatile ("pxor %xmm12,%xmm9");
asm volatile ("movdqa %xmm4,%xmm5");
asm volatile ("movdqa %xmm12,%xmm13");
asm volatile ("psrlw $4,%xmm5");
asm volatile ("psrlw $4,%xmm13");
asm volatile ("pand %xmm11,%xmm4");
asm volatile ("pand %xmm11,%xmm12");
asm volatile ("pand %xmm11,%xmm5");
asm volatile ("pand %xmm11,%xmm13");
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[1][1][0][0]));
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[1][1][1][0]));
asm volatile ("movdqa %xmm6,%xmm14");
asm volatile ("movdqa %xmm7,%xmm15");
asm volatile ("pshufb %xmm4,%xmm6");
@ -212,6 +341,10 @@ void raid_gen3_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[0][i + 16]));
if (generator == 3) {
asm volatile ("movdqa %xmm1,%xmm6");
asm volatile ("movdqa %xmm9,%xmm14");
}
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pxor %xmm13,%xmm13");
asm volatile ("pcmpgtb %xmm1,%xmm5");
@ -222,6 +355,10 @@ void raid_gen3_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("pand %xmm3,%xmm13");
asm volatile ("pxor %xmm5,%xmm1");
asm volatile ("pxor %xmm13,%xmm9");
if (generator == 3) {
asm volatile ("pxor %xmm6,%xmm1");
asm volatile ("pxor %xmm14,%xmm9");
}
asm volatile ("pxor %xmm4,%xmm0");
asm volatile ("pxor %xmm4,%xmm1");
@ -240,12 +377,22 @@ void raid_gen3_ssse3ext(int nd, size_t size, void **vv)
raid_sse_end();
}
void raid_gen3_ssse3ext_raid(int nd, size_t size, void **vv)
{
raid_gen3_ssse3ext_gen(nd, size, vv, 2);
}
void raid_gen3_ssse3ext_aes(int nd, size_t size, void **vv)
{
raid_gen3_ssse3ext_gen(nd, size, vv, 3);
}
#endif
/*
* GEN4 (quad parity with Cauchy matrix) SSSE3 implementation
*/
void raid_gen4_ssse3(int nd, size_t size, void **vv)
static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv, int generator)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -272,7 +419,7 @@ void raid_gen4_ssse3(int nd, size_t size, void **vv)
/* generic case with at least two data disks */
for (i = 0; i < size; i += 16) {
/* last disk without the by two multiplication */
/* last disk without the generator multiplication */
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));
@ -301,11 +448,15 @@ void raid_gen4_ssse3(int nd, size_t size, void **vv)
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
if (generator == 3)
asm volatile ("movdqa %xmm1,%xmm6");
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pcmpgtb %xmm1,%xmm5");
asm volatile ("paddb %xmm1,%xmm1");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("pxor %xmm5,%xmm1");
if (generator == 3)
asm volatile ("pxor %xmm6,%xmm1");
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
@ -336,11 +487,15 @@ void raid_gen4_ssse3(int nd, size_t size, void **vv)
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));
if (generator == 3)
asm volatile ("movdqa %xmm1,%xmm6");
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pcmpgtb %xmm1,%xmm5");
asm volatile ("paddb %xmm1,%xmm1");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("pxor %xmm5,%xmm1");
if (generator == 3)
asm volatile ("pxor %xmm6,%xmm1");
asm volatile ("pxor %xmm4,%xmm0");
asm volatile ("pxor %xmm4,%xmm1");
@ -356,13 +511,23 @@ void raid_gen4_ssse3(int nd, size_t size, void **vv)
raid_sse_end();
}
void raid_gen4_ssse3_raid(int nd, size_t size, void **vv)
{
raid_gen4_ssse3_gen(nd, size, vv, 2);
}
void raid_gen4_ssse3_aes(int nd, size_t size, void **vv)
{
raid_gen4_ssse3_gen(nd, size, vv, 3);
}
#ifdef CONFIG_X86_64
/*
* GEN4 (quad parity with Cauchy matrix) SSSE3 implementation
*
* Note that it uses 16 registers, meaning that x64 is required.
*/
void raid_gen4_ssse3ext(int nd, size_t size, void **vv)
static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **vv, int generator)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -389,7 +554,7 @@ void raid_gen4_ssse3ext(int nd, size_t size, void **vv)
/* generic case with at least two data disks */
for (i = 0; i < size; i += 32) {
/* last disk without the by two multiplication */
/* last disk without the generator multiplication */
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0]));
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[l][i + 16]));
@ -437,6 +602,10 @@ void raid_gen4_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16]));
if (generator == 3) {
asm volatile ("movdqa %xmm1,%xmm6");
asm volatile ("movdqa %xmm9,%xmm14");
}
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pxor %xmm13,%xmm13");
asm volatile ("pcmpgtb %xmm1,%xmm5");
@ -447,6 +616,10 @@ void raid_gen4_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("pand %xmm7,%xmm13");
asm volatile ("pxor %xmm5,%xmm1");
asm volatile ("pxor %xmm13,%xmm9");
if (generator == 3) {
asm volatile ("pxor %xmm6,%xmm1");
asm volatile ("pxor %xmm14,%xmm9");
}
asm volatile ("pxor %xmm4,%xmm0");
asm volatile ("pxor %xmm4,%xmm1");
@ -495,6 +668,10 @@ void raid_gen4_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[0][i + 16]));
if (generator == 3) {
asm volatile ("movdqa %xmm1,%xmm6");
asm volatile ("movdqa %xmm9,%xmm14");
}
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pxor %xmm13,%xmm13");
asm volatile ("pcmpgtb %xmm1,%xmm5");
@ -505,6 +682,10 @@ void raid_gen4_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("pand %xmm7,%xmm13");
asm volatile ("pxor %xmm5,%xmm1");
asm volatile ("pxor %xmm13,%xmm9");
if (generator == 3) {
asm volatile ("pxor %xmm6,%xmm1");
asm volatile ("pxor %xmm14,%xmm9");
}
asm volatile ("pxor %xmm4,%xmm0");
asm volatile ("pxor %xmm4,%xmm1");
@ -527,12 +708,22 @@ void raid_gen4_ssse3ext(int nd, size_t size, void **vv)
raid_sse_end();
}
void raid_gen4_ssse3ext_raid(int nd, size_t size, void **vv)
{
raid_gen4_ssse3ext_gen(nd, size, vv, 2);
}
void raid_gen4_ssse3ext_aes(int nd, size_t size, void **vv)
{
raid_gen4_ssse3ext_gen(nd, size, vv, 3);
}
#endif
/*
* GEN5 (penta parity with Cauchy matrix) SSSE3 implementation
*/
void raid_gen5_ssse3(int nd, size_t size, void **vv)
void raid_gen5_ssse3_raid(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -563,7 +754,7 @@ void raid_gen5_ssse3(int nd, size_t size, void **vv)
/* generic case with at least two data disks */
for (i = 0; i < size; i += 16) {
/* last disk without the by two multiplication */
/* last disk without the generator multiplication */
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));
asm volatile ("movdqa %xmm4,%xmm0");
@ -668,7 +859,7 @@ void raid_gen5_ssse3(int nd, size_t size, void **vv)
/*
* GENX SSSE3ext implementation
*/
static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, int np)
static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, int np, int generator)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -703,7 +894,7 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0]));
for (i = 0; i < size; i += 16) {
/* last disk without the by two multiplication */
/* last disk without the generator multiplication */
asm volatile ("movdqa %0,%%xmm10" : : "m" (v[l][i]));
asm volatile ("movdqa %xmm10,%xmm0");
@ -750,11 +941,15 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i
for (d = l - 1; d > 0; --d) {
asm volatile ("movdqa %0,%%xmm10" : : "m" (v[d][i]));
if (generator == 3)
asm volatile ("movdqa %xmm1,%xmm12");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm1,%xmm11");
asm volatile ("paddb %xmm1,%xmm1");
asm volatile ("pand %xmm14,%xmm11");
asm volatile ("pxor %xmm11,%xmm1");
if (generator == 3)
asm volatile ("pxor %xmm12,%xmm1");
asm volatile ("pxor %xmm10,%xmm0");
asm volatile ("pxor %xmm10,%xmm1");
@ -804,11 +999,15 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i
/* first disk with all coefficients at 1 */
asm volatile ("movdqa %0,%%xmm10" : : "m" (v[0][i]));
if (generator == 3)
asm volatile ("movdqa %xmm1,%xmm12");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm1,%xmm11");
asm volatile ("paddb %xmm1,%xmm1");
asm volatile ("pand %xmm14,%xmm11");
asm volatile ("pxor %xmm11,%xmm1");
if (generator == 3)
asm volatile ("pxor %xmm12,%xmm1");
asm volatile ("pxor %xmm10,%xmm0");
asm volatile ("pxor %xmm10,%xmm1");
@ -841,16 +1040,21 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i
*
* Note that it uses 16 registers, meaning that x64 is required.
*/
void raid_gen5_ssse3ext(int nd, size_t size, void **vv)
void raid_gen5_ssse3ext_raid(int nd, size_t size, void **vv)
{
raid_genX_ssse3ext(nd, size, vv, 5);
raid_genX_ssse3ext(nd, size, vv, 5, 2);
}
void raid_gen5_ssse3ext_aes(int nd, size_t size, void **vv)
{
raid_genX_ssse3ext(nd, size, vv, 5, 3);
}
#endif
/*
* GEN6 (hexa parity with Cauchy matrix) SSSE3 implementation
*/
void raid_gen6_ssse3(int nd, size_t size, void **vv)
void raid_gen6_ssse3_raid(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -883,7 +1087,7 @@ void raid_gen6_ssse3(int nd, size_t size, void **vv)
/* generic case with at least two data disks */
for (i = 0; i < size; i += 16) {
/* last disk without the by two multiplication */
/* last disk without the generator multiplication */
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));
asm volatile ("movdqa %%xmm4,%0" : "=m" (pd[0]));
@ -1009,9 +1213,14 @@ void raid_gen6_ssse3(int nd, size_t size, void **vv)
*
* Note that it uses 16 registers, meaning that x64 is required.
*/
void raid_gen6_ssse3ext(int nd, size_t size, void **vv)
void raid_gen6_ssse3ext_raid(int nd, size_t size, void **vv)
{
raid_genX_ssse3ext(nd, size, vv, 6);
raid_genX_ssse3ext(nd, size, vv, 6, 2);
}
void raid_gen6_ssse3ext_aes(int nd, size_t size, void **vv)
{
raid_genX_ssse3ext(nd, size, vv, 6, 3);
}
#endif
@ -1253,16 +1462,22 @@ void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
void raid_register_ssse3(void)
{
if (raid_cpu_has_ssse3()) {
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3", raid_gen3_ssse3, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3", raid_gen4_ssse3, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3", raid_gen5_ssse3, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3", raid_gen6_ssse3, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3", raid_gen3_ssse3_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3", raid_gen3_ssse3_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3", raid_gen4_ssse3_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3", raid_gen4_ssse3_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3", raid_gen5_ssse3_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3", raid_gen6_ssse3_raid, RAID_POLY_RAID);
#ifdef CONFIG_X86_64
if (!raid_cpu_has_slow_extendedreg()) {
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3e", raid_gen3_ssse3ext, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3e", raid_gen4_ssse3ext, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3e", raid_gen5_ssse3ext, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3e", raid_gen6_ssse3ext, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3e", raid_gen3_ssse3ext_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3e", raid_gen3_ssse3ext_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3e", raid_gen4_ssse3ext_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3e", raid_gen4_ssse3ext_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3e", raid_gen5_ssse3ext_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3e", raid_gen5_ssse3ext_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3e", raid_gen6_ssse3ext_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3e", raid_gen6_ssse3ext_aes, RAID_POLY_AES);
}
#endif

9806
raid/tables.c
View file

File diff suppressed because it is too large Load diff

285
raid/test.c
View file

@ -141,6 +141,165 @@ int raid_test_sort(void)
return 0;
}
/*
* Tests the mathematical properties required by a Cauchy RAID mode.
*
* This is intentionally only a fast sanity check. It does not prove that
* all square submatrices are nonsingular. The exhaustive verification of
* the MDS property is performed by raid/test/invtest.
*
* SnapRAID uses two Extended Cauchy constructions:
*
* RAID_MODE_CAUCHY_RAID:
* polynomial 0x11d
* primitive generator g=2
*
* RAID_MODE_CAUCHY_AES:
* polynomial 0x11b
* primitive generator g=3
*
* The parity generation tests compare optimized implementations against
* raid_gen_ref(), but both use the same active generator matrix. Therefore
* they can agree even if the matrix itself is mathematically unsuitable.
*
* This test instead checks a few fundamental properties of the field
* generator and of the generated Cauchy matrix.
*/
int raid_test_poly(unsigned mode)
{
int i, j;
int p, d;
/*
* This test is defined only for the two Cauchy modes.
*
* Vandermonde parity intentionally uses a different construction and
* is not part of this test.
*/
if (mode != RAID_MODE_CAUCHY_RAID
&& mode != RAID_MODE_CAUCHY_AES)
return -1;
/*
* Select the mode first so that:
*
* raid_gfexp
* raid_gfcauchy
* raid_gfmul
* raid_gfinv
*
* all refer to the tables of the field being tested.
*
* This also verifies the normal raid_mode() table selection path
* instead of accessing the *_raid or *_aes tables directly.
*/
raid_mode(mode);
/*
* Primitive order alone is not enough to protect parity format
* compatibility. Each Cauchy mode has a fixed reducing polynomial
* and primitive generator, which determine the actual matrix coefficients.
*/
if (mode == RAID_MODE_CAUCHY_RAID) {
if (raid_poly_byte != RAID_POLY_RAID)
return -1;
if (raid_gfexp[1] != 2)
return -1;
} else if (mode == RAID_MODE_CAUCHY_AES) {
if (raid_poly_byte != RAID_POLY_AES)
return -1;
if (raid_gfexp[1] != 3)
return -1;
}
/*
* raid_gfexp[a] contains g^a for the primitive generator associated
* with the active Cauchy mode.
*
* The multiplicative group GF(2^8)* contains 255 elements:
*
* 255 = 3 * 5 * 17
*
* The order of every nonzero field element divides 255.
*
* To prove that g has order exactly 255, it is therefore sufficient
* to verify:
*
* g^255 == 1
*
* and to exclude the three maximal proper divisors:
*
* 255 / 3 = 85
* 255 / 5 = 51
* 255 / 17 = 15
*
* Hence a primitive generator must satisfy:
*
* g^85 != 1
* g^51 != 1
* g^15 != 1
*/
if (raid_gfexp[0] != 1)
return -1;
if (raid_gfexp[255] != 1)
return -1;
if (raid_gfexp[85] == 1)
return -1;
if (raid_gfexp[51] == 1)
return -1;
if (raid_gfexp[15] == 1)
return -1;
/*
* The Q row of the Extended Cauchy matrix contains:
*
* 1, g, g^2, g^3, ...
*
* SnapRAID supports RAID_DATA_MAX data disks, so all generator powers
* corresponding to those columns must be distinct.
*
* A primitive generator of order 255 guarantees this, but checking it
* explicitly is inexpensive and also verifies the generated
* raid_gfexp[] table itself.
*/
for (i = 0; i < RAID_DATA_MAX; ++i)
for (j = i + 1; j < RAID_DATA_MAX; ++j)
if (raid_gfexp[i] == raid_gfexp[j])
return -1;
/*
* The second row of the Extended Cauchy matrix is exactly g^d.
*
* Verify that the exponent table and the actual matrix used for
* parity generation agree for every supported data column.
*
* This makes sure that the primitive generator checked above is the
* same generator actually used by the Q row.
*/
for (d = 0; d < RAID_DATA_MAX; ++d)
if (raid_gfcauchy[1][d] != raid_gfexp[d])
return -1;
/*
* Every coefficient of the active 6x251 Cauchy generator matrix must
* be nonzero.
*
* A single coefficient is itself a 1x1 submatrix, so a zero
* coefficient would make that submatrix singular and immediately
* disprove the MDS property.
*
* Absence of zero coefficients is only a necessary condition. It does
* not prove that every larger square submatrix is nonsingular. That
* exhaustive verification remains the responsibility of invtest.
*/
for (p = 0; p < RAID_PARITY_MAX; ++p)
for (d = 0; d < RAID_DATA_MAX; ++d)
if (raid_gfcauchy[p][d] == 0)
return -1;
return 0;
}
#define test_setup(i) f[i - 1][nf[i - 1]++]
int raid_test_rec(int mode, int nd, size_t size)
@ -374,25 +533,49 @@ int raid_test_par(int mode, int nd, size_t size)
/* setup all the available functions */
test_setup(1) = raid_gen1_int32;
test_setup(1) = raid_gen1_int64;
test_setup(2) = raid_gen2_int32;
test_setup(2) = raid_gen2_int64;
test_setup(2) = raid_gen2_int8;
if (mode == RAID_MODE_CAUCHY_AES) {
test_setup(2) = raid_gen2_int32_aes;
test_setup(2) = raid_gen2_int64_aes;
} else {
test_setup(2) = raid_gen2_int32_raid;
test_setup(2) = raid_gen2_int64_raid;
}
#ifdef CONFIG_NEON
test_setup(1) = raid_gen1_neon;
test_setup(2) = raid_gen2_neon;
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(2) = raid_gen2_neon_aes;
else
test_setup(2) = raid_gen2_neon_raid;
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_sse2()) {
test_setup(1) = raid_gen1_sse2;
test_setup(2) = raid_gen2_sse2;
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(2) = raid_gen2_sse2_aes;
else
test_setup(2) = raid_gen2_sse2_raid;
#ifdef CONFIG_X86_64
test_setup(2) = raid_gen2_sse2ext;
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(2) = raid_gen2_sse2ext_aes;
else
test_setup(2) = raid_gen2_sse2ext_raid;
#endif
}
if (raid_cpu_has_avx2()) {
test_setup(1) = raid_gen1_avx2;
test_setup(2) = raid_gen2_avx2;
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(2) = raid_gen2_avx2_aes;
else
test_setup(2) = raid_gen2_avx2_raid;
#ifdef CONFIG_X86_64
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(2) = raid_gen2_avx2ext_aes;
else
test_setup(2) = raid_gen2_avx2ext_raid;
#endif
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx512bw()) {
@ -401,10 +584,10 @@ int raid_test_par(int mode, int nd, size_t size)
}
if (mode == RAID_MODE_CAUCHY_AES) {
if (raid_cpu_has_avx2gfni()) {
test_setup(2) = raid_gen2_avx2gfni;
test_setup(2) = raid_gen2_avx2gfni_aes;
}
if (raid_cpu_has_avx512gfni()) {
test_setup(2) = raid_gen2_avx512gfni;
test_setup(2) = raid_gen2_avx512gfni_aes;
}
}
#endif
@ -417,31 +600,57 @@ int raid_test_par(int mode, int nd, size_t size)
test_setup(6) = raid_gen6_int8;
#ifdef CONFIG_NEON
test_setup(3) = raid_gen3_neon;
test_setup(4) = raid_gen4_neon;
test_setup(5) = raid_gen5_neon;
test_setup(6) = raid_gen6_neon;
if (mode == RAID_MODE_CAUCHY_AES) {
test_setup(3) = raid_gen3_neon_aes;
test_setup(4) = raid_gen4_neon_aes;
test_setup(5) = raid_gen5_neon_aes;
test_setup(6) = raid_gen6_neon_aes;
} else {
test_setup(3) = raid_gen3_neon_raid;
test_setup(4) = raid_gen4_neon_raid;
test_setup(5) = raid_gen5_neon_raid;
test_setup(6) = raid_gen6_neon_raid;
}
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
test_setup(3) = raid_gen3_ssse3;
test_setup(4) = raid_gen4_ssse3;
test_setup(5) = raid_gen5_ssse3;
test_setup(6) = raid_gen6_ssse3;
if (mode == RAID_MODE_CAUCHY_AES) {
test_setup(3) = raid_gen3_ssse3_aes;
test_setup(4) = raid_gen4_ssse3_aes;
} else {
test_setup(3) = raid_gen3_ssse3_raid;
test_setup(4) = raid_gen4_ssse3_raid;
test_setup(5) = raid_gen5_ssse3_raid;
test_setup(6) = raid_gen6_ssse3_raid;
}
#ifdef CONFIG_X86_64
test_setup(3) = raid_gen3_ssse3ext;
test_setup(4) = raid_gen4_ssse3ext;
test_setup(5) = raid_gen5_ssse3ext;
test_setup(6) = raid_gen6_ssse3ext;
if (mode == RAID_MODE_CAUCHY_AES) {
test_setup(3) = raid_gen3_ssse3ext_aes;
test_setup(4) = raid_gen4_ssse3ext_aes;
test_setup(5) = raid_gen5_ssse3ext_aes;
test_setup(6) = raid_gen6_ssse3ext_aes;
} else {
test_setup(3) = raid_gen3_ssse3ext_raid;
test_setup(4) = raid_gen4_ssse3ext_raid;
test_setup(5) = raid_gen5_ssse3ext_raid;
test_setup(6) = raid_gen6_ssse3ext_raid;
}
#endif
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
test_setup(3) = raid_gen3_avx2ext;
test_setup(4) = raid_gen4_avx2ext;
test_setup(5) = raid_gen5_avx2ext;
test_setup(6) = raid_gen6_avx2ext;
if (mode == RAID_MODE_CAUCHY_AES) {
test_setup(3) = raid_gen3_avx2ext_aes;
test_setup(4) = raid_gen4_avx2ext_aes;
test_setup(5) = raid_gen5_avx2ext_aes;
test_setup(6) = raid_gen6_avx2ext_aes;
} else {
test_setup(3) = raid_gen3_avx2ext_raid;
test_setup(4) = raid_gen4_avx2ext_raid;
test_setup(5) = raid_gen5_avx2ext_raid;
test_setup(6) = raid_gen6_avx2ext_raid;
}
}
if (raid_cpu_has_avx512bw()) {
test_setup(3) = raid_gen3_avx512bw;
@ -451,38 +660,38 @@ int raid_test_par(int mode, int nd, size_t size)
}
if (mode == RAID_MODE_CAUCHY_AES) {
if (raid_cpu_has_avx2gfni()) {
test_setup(3) = raid_gen3_avx2gfni;
test_setup(4) = raid_gen4_avx2gfni;
test_setup(5) = raid_gen5_avx2gfni;
test_setup(6) = raid_gen6_avx2gfni;
test_setup(3) = raid_gen3_avx2gfni_aes;
test_setup(4) = raid_gen4_avx2gfni_aes;
test_setup(5) = raid_gen5_avx2gfni_aes;
test_setup(6) = raid_gen6_avx2gfni_aes;
}
if (raid_cpu_has_avx512gfni()) {
test_setup(3) = raid_gen3_avx512gfni;
test_setup(4) = raid_gen4_avx512gfni;
test_setup(5) = raid_gen5_avx512gfni;
test_setup(6) = raid_gen6_avx512gfni;
test_setup(3) = raid_gen3_avx512gfni_aes;
test_setup(4) = raid_gen4_avx512gfni_aes;
test_setup(5) = raid_gen5_avx512gfni_aes;
test_setup(6) = raid_gen6_avx512gfni_aes;
}
}
#endif
#endif
} else {
test_setup(3) = raid_genz_int32;
test_setup(3) = raid_genz_int64;
test_setup(3) = raid_genz_int32_raid;
test_setup(3) = raid_genz_int64_raid;
#ifdef CONFIG_NEON
test_setup(3) = raid_genz_neon;
test_setup(3) = raid_genz_neon_raid;
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_sse2()) {
test_setup(3) = raid_genz_sse2;
test_setup(3) = raid_genz_sse2_raid;
#ifdef CONFIG_X86_64
test_setup(3) = raid_genz_sse2ext;
test_setup(3) = raid_genz_sse2ext_raid;
#endif
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2())
test_setup(3) = raid_genz_avx2ext;
test_setup(3) = raid_genz_avx2ext_raid;
#endif
#endif
}

10
raid/test.h
View file

@ -31,6 +31,16 @@ int raid_test_sort(void);
*/
int raid_test_combo(void);
/**
* Tests the Galois field and Cauchy matrix properties.
*
* Verifies the expected polynomial and primitive generator and basic
* properties of the Extended Cauchy matrix for the specified RAID mode.
*
* Returns 0 on success.
*/
int raid_test_poly(unsigned mode);
/**
* Tests recovering functions.
*

70
raid/test/invtest.c
View file

@ -1,7 +1,15 @@
// SPDX-License-Identifier: GPL-2.0-or-later
// Copyright (C) 2013 Andrea Mazzoleni
/* Matrix inversion test for the RAID library */
/*
* Matrix inversion test for the RAID library.
*
* Verifies that all square submatrices of the Extended Cauchy matrix are
* nonsingular (invertible) for both operating modes:
*
* RAID: GF polynomial 0x11d, primitive generator g=2
* AES: GF polynomial 0x11b, primitive generator g=3
*/
#include "internal.h"
@ -72,7 +80,7 @@ long long EXPECTED[RAID_PARITY_MAX] = {
327012476050LL
};
static __always_inline int test_sub_matrix(int nr, int64_t *total)
static __always_inline int test_sub_matrix(int nr, int mode, const char *name, int64_t *total)
{
uint8_t M[RAID_PARITY_MAX * RAID_PARITY_MAX];
int np = RAID_PARITY_MAX;
@ -82,7 +90,10 @@ static __always_inline int test_sub_matrix(int nr, int64_t *total)
int64_t count;
int64_t expected;
printf("\n%ux%u\n", nr, nr);
/* select the operating mode before reading raid_gfgen or using inv()/mul() */
raid_mode(mode);
printf("\n%ux%u %s\n", nr, nr, name);
count = 0;
expected = EXPECTED[nr - 1];
@ -101,8 +112,18 @@ static __always_inline int test_sub_matrix(int nr, int64_t *total)
M[i * nr + j] = raid_gfgen[ip[i]][id[j]];
/* invert */
if (raid_invert_fast(M, nr) != 0)
if (raid_invert_fast(M, nr) != 0) {
int k;
printf("\nFAILED %s %ux%u submatrix\n", name, nr, nr);
printf("parity:");
for (k = 0; k < nr; ++k)
printf(" %d", ip[k]);
printf("\ndata:");
for (k = 0; k < nr; ++k)
printf(" %d", id[k]);
printf("\n");
return -1;
}
if (++count % TEST_REFRESH == 0) {
printf("\r%.3f %%", count * (double)100 / expected);
@ -111,8 +132,10 @@ static __always_inline int test_sub_matrix(int nr, int64_t *total)
} while (combination_next(nr, np, ip));
} while (combination_next(nr, nd, id));
if (count != expected)
if (count != expected) {
printf("\nFAILED %s %ux%u count mismatch: %" PRIi64 " != %" PRIi64 "\n", name, nr, nr, count, expected);
return -1;
}
printf("\rTested %" PRIi64 " matrix\n", count);
@ -123,30 +146,45 @@ static __always_inline int test_sub_matrix(int nr, int64_t *total)
int test_all_sub_matrix(void)
{
int64_t total;
int64_t total_raid;
int64_t total_aes;
printf("Invert all square submatrices of the %dx%d Cauchy matrix\n",
printf("Invert all square submatrices of the %dx%d Cauchy matrices in RAID and AES modes\n",
RAID_PARITY_MAX, RAID_DATA_MAX);
printf("\nPlease wait about 2 days...\n");
printf("\nThis exhaustive test may take days...\n");
total = 0;
total_raid = 0;
total_aes = 0;
/* force inlining of everything */
if (test_sub_matrix(1, &total) != 0)
if (test_sub_matrix(1, RAID_MODE_CAUCHY_RAID, "RAID", &total_raid) != 0)
return -1;
if (test_sub_matrix(2, &total) != 0)
if (test_sub_matrix(1, RAID_MODE_CAUCHY_AES, "AES", &total_aes) != 0)
return -1;
if (test_sub_matrix(3, &total) != 0)
if (test_sub_matrix(2, RAID_MODE_CAUCHY_RAID, "RAID", &total_raid) != 0)
return -1;
if (test_sub_matrix(4, &total) != 0)
if (test_sub_matrix(2, RAID_MODE_CAUCHY_AES, "AES", &total_aes) != 0)
return -1;
if (test_sub_matrix(5, &total) != 0)
if (test_sub_matrix(3, RAID_MODE_CAUCHY_RAID, "RAID", &total_raid) != 0)
return -1;
if (test_sub_matrix(6, &total) != 0)
if (test_sub_matrix(3, RAID_MODE_CAUCHY_AES, "AES", &total_aes) != 0)
return -1;
if (test_sub_matrix(4, RAID_MODE_CAUCHY_RAID, "RAID", &total_raid) != 0)
return -1;
if (test_sub_matrix(4, RAID_MODE_CAUCHY_AES, "AES", &total_aes) != 0)
return -1;
if (test_sub_matrix(5, RAID_MODE_CAUCHY_RAID, "RAID", &total_raid) != 0)
return -1;
if (test_sub_matrix(5, RAID_MODE_CAUCHY_AES, "AES", &total_aes) != 0)
return -1;
if (test_sub_matrix(6, RAID_MODE_CAUCHY_RAID, "RAID", &total_raid) != 0)
return -1;
if (test_sub_matrix(6, RAID_MODE_CAUCHY_AES, "AES", &total_aes) != 0)
return -1;
printf("\nTested in total %" PRIi64 " matrix\n", total);
printf("\nTested in total %" PRIi64 " RAID matrix\n", total_raid);
printf("Tested in total %" PRIi64 " AES matrix\n", total_aes);
return 0;
}

13
raid/test/selftest.c
View file

@ -4,6 +4,7 @@
/* Self-test for the RAID library */
#include "internal.h"
#include "test.h"
#include "cpu.h"
#include <stdio.h>
@ -16,15 +17,23 @@ int main(void)
raid_init();
printf("Test Cauchy RAID\n");
if (raid_test_poly(RAID_MODE_CAUCHY_RAID) != 0) {
/* LCOV_EXCL_START */
goto bail;
/* LCOV_EXCL_STOP */
}
if (raid_selftest() != 0) {
/* LCOV_EXCL_START */
goto bail;
/* LCOV_EXCL_STOP */
}
raid_mode(RAID_MODE_CAUCHY_AES);
printf("Test Cauchy AES\n");
if (raid_test_poly(RAID_MODE_CAUCHY_AES) != 0) {
/* LCOV_EXCL_START */
goto bail;
/* LCOV_EXCL_STOP */
}
if (raid_selftest() != 0) {
/* LCOV_EXCL_START */
goto bail;

689
raid/test/speedtest.c
View file

File diff suppressed because it is too large Load diff