mirror of
https://github.com/amadvance/snapraid.git
synced 2026-10-26 19:01:57 -04:00
b070cbb894
Some checks are pending
Build / build-mac (push) Waiting to run
Build / build-linux (push) Waiting to run
Build / build-arm32 (push) Waiting to run
Build / build-win (push) Waiting to run
Build / build-freebsd (push) Waiting to run
Build / build-musl (push) Waiting to run
CodeQL Advanced / Analyze (c-cpp) (push) Waiting to run
CodeQL Advanced / Analyze (actions) (push) Waiting to run
4349 lines
131 KiB
C
4349 lines
131 KiB
C
// SPDX-License-Identifier: GPL-2.0-or-later
|
|
// Copyright (C) 2026 Andrea Mazzoleni
|
|
|
|
#include "internal.h"
|
|
#include "gf.h"
|
|
#include "cpu.h"
|
|
|
|
#ifdef CONFIG_X86
|
|
/*
|
|
* Generate three parity blocks with Cauchy matrix using SSSE3 implementation.
|
|
*/
|
|
static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv, int generator, int streaming)
|
|
{
|
|
uint8_t **v = (uint8_t **)vv;
|
|
uint8_t *p;
|
|
uint8_t *q;
|
|
uint8_t *r;
|
|
int d, l;
|
|
size_t i;
|
|
|
|
l = nd - 1;
|
|
p = v[nd];
|
|
q = v[nd + 1];
|
|
r = v[nd + 2];
|
|
|
|
/* special case with only one data disk */
|
|
if (l == 0) {
|
|
for (i = 0; i < 3; ++i)
|
|
if (v[1 + i] != v[0])
|
|
memcpy(v[1 + i], v[0], size);
|
|
return;
|
|
}
|
|
|
|
raid_sse_begin();
|
|
|
|
/* generic case with at least two data disks */
|
|
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.poly));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4));
|
|
|
|
for (i = 0; i < size; i += 16) {
|
|
/* last disk without the generator multiplication */
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));
|
|
|
|
asm volatile ("movdqa %xmm4,%xmm0");
|
|
asm volatile ("movdqa %xmm4,%xmm1");
|
|
|
|
asm volatile ("movdqa %xmm4,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("pand %xmm7,%xmm4");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
|
|
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0]));
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[l][1][1]));
|
|
asm volatile ("pshufb %xmm4,%xmm2");
|
|
asm volatile ("pshufb %xmm5,%xmm6");
|
|
asm volatile ("pxor %xmm6,%xmm2");
|
|
|
|
/* intermediate disks */
|
|
for (d = l - 1; d > 0; --d) {
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
|
|
|
|
if (generator == 3)
|
|
asm volatile ("movdqa %xmm1,%xmm6");
|
|
asm volatile ("pxor %xmm5,%xmm5");
|
|
asm volatile ("pcmpgtb %xmm1,%xmm5");
|
|
asm volatile ("paddb %xmm1,%xmm1");
|
|
asm volatile ("pand %xmm3,%xmm5");
|
|
asm volatile ("pxor %xmm5,%xmm1");
|
|
if (generator == 3)
|
|
asm volatile ("pxor %xmm6,%xmm1");
|
|
|
|
asm volatile ("pxor %xmm4,%xmm0");
|
|
asm volatile ("pxor %xmm4,%xmm1");
|
|
|
|
asm volatile ("movdqa %xmm4,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("pand %xmm7,%xmm4");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0]));
|
|
asm volatile ("pshufb %xmm4,%xmm6");
|
|
asm volatile ("pxor %xmm6,%xmm2");
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][1]));
|
|
asm volatile ("pshufb %xmm5,%xmm6");
|
|
asm volatile ("pxor %xmm6,%xmm2");
|
|
}
|
|
|
|
/* first disk with all coefficients at 1 */
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));
|
|
|
|
if (generator == 3)
|
|
asm volatile ("movdqa %xmm1,%xmm6");
|
|
asm volatile ("pxor %xmm5,%xmm5");
|
|
asm volatile ("pcmpgtb %xmm1,%xmm5");
|
|
asm volatile ("paddb %xmm1,%xmm1");
|
|
asm volatile ("pand %xmm3,%xmm5");
|
|
asm volatile ("pxor %xmm5,%xmm1");
|
|
if (generator == 3)
|
|
asm volatile ("pxor %xmm6,%xmm1");
|
|
|
|
asm volatile ("pxor %xmm4,%xmm0");
|
|
asm volatile ("pxor %xmm4,%xmm1");
|
|
asm volatile ("pxor %xmm4,%xmm2");
|
|
|
|
if (streaming) {
|
|
asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i]));
|
|
asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i]));
|
|
asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i]));
|
|
} else {
|
|
asm volatile ("movdqa %%xmm0,%0" : "=m" (p[i]));
|
|
asm volatile ("movdqa %%xmm1,%0" : "=m" (q[i]));
|
|
asm volatile ("movdqa %%xmm2,%0" : "=m" (r[i]));
|
|
}
|
|
}
|
|
|
|
raid_sse_end(streaming);
|
|
}
|
|
|
|
/*
|
|
* Generate four parity blocks with Cauchy matrix using SSSE3 implementation.
|
|
*/
|
|
static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv, int generator, int streaming)
|
|
{
|
|
uint8_t **v = (uint8_t **)vv;
|
|
uint8_t *p;
|
|
uint8_t *q;
|
|
uint8_t *r;
|
|
uint8_t *s;
|
|
int d, l;
|
|
size_t i;
|
|
|
|
l = nd - 1;
|
|
p = v[nd];
|
|
q = v[nd + 1];
|
|
r = v[nd + 2];
|
|
s = v[nd + 3];
|
|
|
|
/* special case with only one data disk */
|
|
if (l == 0) {
|
|
for (i = 0; i < 4; ++i)
|
|
if (v[1 + i] != v[0])
|
|
memcpy(v[1 + i], v[0], size);
|
|
return;
|
|
}
|
|
|
|
raid_sse_begin();
|
|
|
|
/* generic case with at least two data disks */
|
|
for (i = 0; i < size; i += 16) {
|
|
/* last disk without the generator multiplication */
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4));
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));
|
|
|
|
asm volatile ("movdqa %xmm4,%xmm0");
|
|
asm volatile ("movdqa %xmm4,%xmm1");
|
|
|
|
asm volatile ("movdqa %xmm4,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("pand %xmm7,%xmm4");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
|
|
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0]));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1]));
|
|
asm volatile ("pshufb %xmm4,%xmm2");
|
|
asm volatile ("pshufb %xmm5,%xmm7");
|
|
asm volatile ("pxor %xmm7,%xmm2");
|
|
|
|
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][2][0]));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1]));
|
|
asm volatile ("pshufb %xmm4,%xmm3");
|
|
asm volatile ("pshufb %xmm5,%xmm7");
|
|
asm volatile ("pxor %xmm7,%xmm3");
|
|
|
|
/* intermediate disks */
|
|
for (d = l - 1; d > 0; --d) {
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly));
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
|
|
|
|
if (generator == 3)
|
|
asm volatile ("movdqa %xmm1,%xmm6");
|
|
asm volatile ("pxor %xmm5,%xmm5");
|
|
asm volatile ("pcmpgtb %xmm1,%xmm5");
|
|
asm volatile ("paddb %xmm1,%xmm1");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
asm volatile ("pxor %xmm5,%xmm1");
|
|
if (generator == 3)
|
|
asm volatile ("pxor %xmm6,%xmm1");
|
|
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4));
|
|
|
|
asm volatile ("pxor %xmm4,%xmm0");
|
|
asm volatile ("pxor %xmm4,%xmm1");
|
|
|
|
asm volatile ("movdqa %xmm4,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("pand %xmm7,%xmm4");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0]));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1]));
|
|
asm volatile ("pshufb %xmm4,%xmm6");
|
|
asm volatile ("pshufb %xmm5,%xmm7");
|
|
asm volatile ("pxor %xmm6,%xmm2");
|
|
asm volatile ("pxor %xmm7,%xmm2");
|
|
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0]));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1]));
|
|
asm volatile ("pshufb %xmm4,%xmm6");
|
|
asm volatile ("pshufb %xmm5,%xmm7");
|
|
asm volatile ("pxor %xmm6,%xmm3");
|
|
asm volatile ("pxor %xmm7,%xmm3");
|
|
}
|
|
|
|
/* first disk with all coefficients at 1 */
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly));
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));
|
|
|
|
if (generator == 3)
|
|
asm volatile ("movdqa %xmm1,%xmm6");
|
|
asm volatile ("pxor %xmm5,%xmm5");
|
|
asm volatile ("pcmpgtb %xmm1,%xmm5");
|
|
asm volatile ("paddb %xmm1,%xmm1");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
asm volatile ("pxor %xmm5,%xmm1");
|
|
if (generator == 3)
|
|
asm volatile ("pxor %xmm6,%xmm1");
|
|
|
|
asm volatile ("pxor %xmm4,%xmm0");
|
|
asm volatile ("pxor %xmm4,%xmm1");
|
|
asm volatile ("pxor %xmm4,%xmm2");
|
|
asm volatile ("pxor %xmm4,%xmm3");
|
|
|
|
if (streaming) {
|
|
asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i]));
|
|
asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i]));
|
|
asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i]));
|
|
asm volatile ("movntdq %%xmm3,%0" : "=m" (s[i]));
|
|
} else {
|
|
asm volatile ("movdqa %%xmm0,%0" : "=m" (p[i]));
|
|
asm volatile ("movdqa %%xmm1,%0" : "=m" (q[i]));
|
|
asm volatile ("movdqa %%xmm2,%0" : "=m" (r[i]));
|
|
asm volatile ("movdqa %%xmm3,%0" : "=m" (s[i]));
|
|
}
|
|
}
|
|
|
|
raid_sse_end(streaming);
|
|
}
|
|
|
|
/*
|
|
* Generate five parity blocks with Cauchy matrix using SSSE3 implementation.
|
|
*
|
|
* Uses an aligned stack buffer for parity accumulators under 8-register pressure.
|
|
*/
|
|
static __always_inline void raid_gen5_ssse3_gen(int nd, size_t size, void **vv, int streaming)
|
|
{
|
|
uint8_t **v = (uint8_t **)vv;
|
|
uint8_t *p;
|
|
uint8_t *q;
|
|
uint8_t *r;
|
|
uint8_t *s;
|
|
uint8_t *t;
|
|
int d, l;
|
|
size_t i;
|
|
uint8_t buffer[16 + 16];
|
|
uint8_t *pd = __align_ptr(buffer, 16);
|
|
|
|
l = nd - 1;
|
|
p = v[nd];
|
|
q = v[nd + 1];
|
|
r = v[nd + 2];
|
|
s = v[nd + 3];
|
|
t = v[nd + 4];
|
|
|
|
/* special case with only one data disk */
|
|
if (l == 0) {
|
|
for (i = 0; i < 5; ++i)
|
|
if (v[1 + i] != v[0])
|
|
memcpy(v[1 + i], v[0], size);
|
|
return;
|
|
}
|
|
|
|
raid_sse_begin();
|
|
|
|
/* generic case with at least two data disks */
|
|
for (i = 0; i < size; i += 16) {
|
|
/* last disk without the generator multiplication */
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));
|
|
|
|
asm volatile ("movdqa %xmm4,%xmm0");
|
|
asm volatile ("movdqa %%xmm4,%0" : "=m" (pd[0]));
|
|
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4));
|
|
asm volatile ("movdqa %xmm4,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("pand %xmm7,%xmm4");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
|
|
asm volatile ("movdqa %0,%%xmm1" : : "m" (raid_gfcauchypshufb[l][1][0]));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1]));
|
|
asm volatile ("pshufb %xmm4,%xmm1");
|
|
asm volatile ("pshufb %xmm5,%xmm7");
|
|
asm volatile ("pxor %xmm7,%xmm1");
|
|
|
|
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][2][0]));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1]));
|
|
asm volatile ("pshufb %xmm4,%xmm2");
|
|
asm volatile ("pshufb %xmm5,%xmm7");
|
|
asm volatile ("pxor %xmm7,%xmm2");
|
|
|
|
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][3][0]));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][3][1]));
|
|
asm volatile ("pshufb %xmm4,%xmm3");
|
|
asm volatile ("pshufb %xmm5,%xmm7");
|
|
asm volatile ("pxor %xmm7,%xmm3");
|
|
|
|
/* intermediate disks */
|
|
for (d = l - 1; d > 0; --d) {
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[0]));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly));
|
|
|
|
asm volatile ("pxor %xmm5,%xmm5");
|
|
asm volatile ("pcmpgtb %xmm0,%xmm5");
|
|
asm volatile ("paddb %xmm0,%xmm0");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
asm volatile ("pxor %xmm5,%xmm0");
|
|
|
|
asm volatile ("pxor %xmm4,%xmm0");
|
|
asm volatile ("pxor %xmm4,%xmm6");
|
|
asm volatile ("movdqa %%xmm6,%0" : "=m" (pd[0]));
|
|
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4));
|
|
asm volatile ("movdqa %xmm4,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("pand %xmm7,%xmm4");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0]));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1]));
|
|
asm volatile ("pshufb %xmm4,%xmm6");
|
|
asm volatile ("pshufb %xmm5,%xmm7");
|
|
asm volatile ("pxor %xmm6,%xmm1");
|
|
asm volatile ("pxor %xmm7,%xmm1");
|
|
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0]));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1]));
|
|
asm volatile ("pshufb %xmm4,%xmm6");
|
|
asm volatile ("pshufb %xmm5,%xmm7");
|
|
asm volatile ("pxor %xmm6,%xmm2");
|
|
asm volatile ("pxor %xmm7,%xmm2");
|
|
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][3][0]));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][3][1]));
|
|
asm volatile ("pshufb %xmm4,%xmm6");
|
|
asm volatile ("pshufb %xmm5,%xmm7");
|
|
asm volatile ("pxor %xmm6,%xmm3");
|
|
asm volatile ("pxor %xmm7,%xmm3");
|
|
}
|
|
|
|
/* first disk with all coefficients at 1 */
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[0]));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly));
|
|
|
|
asm volatile ("pxor %xmm5,%xmm5");
|
|
asm volatile ("pcmpgtb %xmm0,%xmm5");
|
|
asm volatile ("paddb %xmm0,%xmm0");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
asm volatile ("pxor %xmm5,%xmm0");
|
|
|
|
asm volatile ("pxor %xmm4,%xmm0");
|
|
asm volatile ("pxor %xmm4,%xmm1");
|
|
asm volatile ("pxor %xmm4,%xmm2");
|
|
asm volatile ("pxor %xmm4,%xmm3");
|
|
asm volatile ("pxor %xmm4,%xmm6");
|
|
|
|
if (streaming) {
|
|
asm volatile ("movntdq %%xmm6,%0" : "=m" (p[i]));
|
|
asm volatile ("movntdq %%xmm0,%0" : "=m" (q[i]));
|
|
asm volatile ("movntdq %%xmm1,%0" : "=m" (r[i]));
|
|
asm volatile ("movntdq %%xmm2,%0" : "=m" (s[i]));
|
|
asm volatile ("movntdq %%xmm3,%0" : "=m" (t[i]));
|
|
} else {
|
|
asm volatile ("movdqa %%xmm6,%0" : "=m" (p[i]));
|
|
asm volatile ("movdqa %%xmm0,%0" : "=m" (q[i]));
|
|
asm volatile ("movdqa %%xmm1,%0" : "=m" (r[i]));
|
|
asm volatile ("movdqa %%xmm2,%0" : "=m" (s[i]));
|
|
asm volatile ("movdqa %%xmm3,%0" : "=m" (t[i]));
|
|
}
|
|
}
|
|
|
|
raid_sse_end(streaming);
|
|
}
|
|
|
|
/*
|
|
* Generate six parity blocks with Cauchy matrix using SSSE3 implementation.
|
|
*
|
|
* Uses an aligned stack buffer for parity accumulators under 8-register pressure.
|
|
*/
|
|
static __always_inline void raid_gen6_ssse3_gen(int nd, size_t size, void **vv, int streaming)
|
|
{
|
|
uint8_t **v = (uint8_t **)vv;
|
|
uint8_t *p;
|
|
uint8_t *q;
|
|
uint8_t *r;
|
|
uint8_t *s;
|
|
uint8_t *t;
|
|
uint8_t *u;
|
|
int d, l;
|
|
size_t i;
|
|
uint8_t buffer[2 * 16 + 16];
|
|
uint8_t *pd = __align_ptr(buffer, 16);
|
|
|
|
l = nd - 1;
|
|
p = v[nd];
|
|
q = v[nd + 1];
|
|
r = v[nd + 2];
|
|
s = v[nd + 3];
|
|
t = v[nd + 4];
|
|
u = v[nd + 5];
|
|
|
|
/* special case with only one data disk */
|
|
if (l == 0) {
|
|
for (i = 0; i < 6; ++i)
|
|
if (v[1 + i] != v[0])
|
|
memcpy(v[1 + i], v[0], size);
|
|
return;
|
|
}
|
|
|
|
raid_sse_begin();
|
|
|
|
/* generic case with at least two data disks */
|
|
for (i = 0; i < size; i += 16) {
|
|
/* last disk without the generator multiplication */
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));
|
|
|
|
asm volatile ("movdqa %%xmm4,%0" : "=m" (pd[0]));
|
|
asm volatile ("movdqa %%xmm4,%0" : "=m" (pd[16]));
|
|
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4));
|
|
asm volatile ("movdqa %xmm4,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("pand %xmm7,%xmm4");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
|
|
asm volatile ("movdqa %0,%%xmm0" : : "m" (raid_gfcauchypshufb[l][1][0]));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1]));
|
|
asm volatile ("pshufb %xmm4,%xmm0");
|
|
asm volatile ("pshufb %xmm5,%xmm7");
|
|
asm volatile ("pxor %xmm7,%xmm0");
|
|
|
|
asm volatile ("movdqa %0,%%xmm1" : : "m" (raid_gfcauchypshufb[l][2][0]));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1]));
|
|
asm volatile ("pshufb %xmm4,%xmm1");
|
|
asm volatile ("pshufb %xmm5,%xmm7");
|
|
asm volatile ("pxor %xmm7,%xmm1");
|
|
|
|
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][3][0]));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][3][1]));
|
|
asm volatile ("pshufb %xmm4,%xmm2");
|
|
asm volatile ("pshufb %xmm5,%xmm7");
|
|
asm volatile ("pxor %xmm7,%xmm2");
|
|
|
|
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][4][0]));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][4][1]));
|
|
asm volatile ("pshufb %xmm4,%xmm3");
|
|
asm volatile ("pshufb %xmm5,%xmm7");
|
|
asm volatile ("pxor %xmm7,%xmm3");
|
|
|
|
/* intermediate disks */
|
|
for (d = l - 1; d > 0; --d) {
|
|
asm volatile ("movdqa %0,%%xmm5" : : "m" (pd[0]));
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[16]));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly));
|
|
|
|
asm volatile ("pxor %xmm4,%xmm4");
|
|
asm volatile ("pcmpgtb %xmm6,%xmm4");
|
|
asm volatile ("paddb %xmm6,%xmm6");
|
|
asm volatile ("pand %xmm7,%xmm4");
|
|
asm volatile ("pxor %xmm4,%xmm6");
|
|
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
|
|
|
|
asm volatile ("pxor %xmm4,%xmm5");
|
|
asm volatile ("pxor %xmm4,%xmm6");
|
|
asm volatile ("movdqa %%xmm5,%0" : "=m" (pd[0]));
|
|
asm volatile ("movdqa %%xmm6,%0" : "=m" (pd[16]));
|
|
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4));
|
|
asm volatile ("movdqa %xmm4,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("pand %xmm7,%xmm4");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0]));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1]));
|
|
asm volatile ("pshufb %xmm4,%xmm6");
|
|
asm volatile ("pshufb %xmm5,%xmm7");
|
|
asm volatile ("pxor %xmm6,%xmm0");
|
|
asm volatile ("pxor %xmm7,%xmm0");
|
|
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0]));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1]));
|
|
asm volatile ("pshufb %xmm4,%xmm6");
|
|
asm volatile ("pshufb %xmm5,%xmm7");
|
|
asm volatile ("pxor %xmm6,%xmm1");
|
|
asm volatile ("pxor %xmm7,%xmm1");
|
|
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][3][0]));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][3][1]));
|
|
asm volatile ("pshufb %xmm4,%xmm6");
|
|
asm volatile ("pshufb %xmm5,%xmm7");
|
|
asm volatile ("pxor %xmm6,%xmm2");
|
|
asm volatile ("pxor %xmm7,%xmm2");
|
|
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][4][0]));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][4][1]));
|
|
asm volatile ("pshufb %xmm4,%xmm6");
|
|
asm volatile ("pshufb %xmm5,%xmm7");
|
|
asm volatile ("pxor %xmm6,%xmm3");
|
|
asm volatile ("pxor %xmm7,%xmm3");
|
|
}
|
|
|
|
/* first disk with all coefficients at 1 */
|
|
asm volatile ("movdqa %0,%%xmm5" : : "m" (pd[0]));
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[16]));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly));
|
|
|
|
asm volatile ("pxor %xmm4,%xmm4");
|
|
asm volatile ("pcmpgtb %xmm6,%xmm4");
|
|
asm volatile ("paddb %xmm6,%xmm6");
|
|
asm volatile ("pand %xmm7,%xmm4");
|
|
asm volatile ("pxor %xmm4,%xmm6");
|
|
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));
|
|
asm volatile ("pxor %xmm4,%xmm0");
|
|
asm volatile ("pxor %xmm4,%xmm1");
|
|
asm volatile ("pxor %xmm4,%xmm2");
|
|
asm volatile ("pxor %xmm4,%xmm3");
|
|
asm volatile ("pxor %xmm4,%xmm5");
|
|
asm volatile ("pxor %xmm4,%xmm6");
|
|
|
|
if (streaming) {
|
|
asm volatile ("movntdq %%xmm5,%0" : "=m" (p[i]));
|
|
asm volatile ("movntdq %%xmm6,%0" : "=m" (q[i]));
|
|
asm volatile ("movntdq %%xmm0,%0" : "=m" (r[i]));
|
|
asm volatile ("movntdq %%xmm1,%0" : "=m" (s[i]));
|
|
asm volatile ("movntdq %%xmm2,%0" : "=m" (t[i]));
|
|
asm volatile ("movntdq %%xmm3,%0" : "=m" (u[i]));
|
|
} else {
|
|
asm volatile ("movdqa %%xmm5,%0" : "=m" (p[i]));
|
|
asm volatile ("movdqa %%xmm6,%0" : "=m" (q[i]));
|
|
asm volatile ("movdqa %%xmm0,%0" : "=m" (r[i]));
|
|
asm volatile ("movdqa %%xmm1,%0" : "=m" (s[i]));
|
|
asm volatile ("movdqa %%xmm2,%0" : "=m" (t[i]));
|
|
asm volatile ("movdqa %%xmm3,%0" : "=m" (u[i]));
|
|
}
|
|
}
|
|
|
|
raid_sse_end(streaming);
|
|
}
|
|
|
|
#ifdef CONFIG_X86_64
|
|
/*
|
|
* Generate three parity blocks with Cauchy matrix using SSSE3 extended implementation.
|
|
*
|
|
* Uses the extended register set (16 XMM registers) and processes two disks
|
|
* per iteration across two 16-byte XMM lanes (32 bytes/step).
|
|
*
|
|
* Note that it uses 16 registers, meaning that x64 is required.
|
|
*/
|
|
static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **vv, int generator, int streaming)
|
|
{
|
|
uint8_t **v = (uint8_t **)vv;
|
|
uint8_t *p;
|
|
uint8_t *q;
|
|
uint8_t *r;
|
|
int d, l;
|
|
size_t i;
|
|
|
|
l = nd - 1;
|
|
p = v[nd];
|
|
q = v[nd + 1];
|
|
r = v[nd + 2];
|
|
|
|
/* special case with only one data disk */
|
|
if (l == 0) {
|
|
for (i = 0; i < 3; ++i)
|
|
if (v[1 + i] != v[0])
|
|
memcpy(v[1 + i], v[0], size);
|
|
return;
|
|
}
|
|
|
|
raid_sse_begin();
|
|
|
|
/* generic case with at least two data disks */
|
|
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.poly));
|
|
asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.low4));
|
|
|
|
for (i = 0; i < size; i += 32) {
|
|
/* last disk without the generator multiplication */
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));
|
|
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[l][i + 16]));
|
|
|
|
asm volatile ("movdqa %xmm4,%xmm0");
|
|
asm volatile ("movdqa %xmm4,%xmm1");
|
|
asm volatile ("movdqa %xmm12,%xmm8");
|
|
asm volatile ("movdqa %xmm12,%xmm9");
|
|
|
|
asm volatile ("movdqa %xmm4,%xmm5");
|
|
asm volatile ("movdqa %xmm12,%xmm13");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm13");
|
|
asm volatile ("pand %xmm11,%xmm4");
|
|
asm volatile ("pand %xmm11,%xmm12");
|
|
asm volatile ("pand %xmm11,%xmm5");
|
|
asm volatile ("pand %xmm11,%xmm13");
|
|
|
|
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0]));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1]));
|
|
asm volatile ("movdqa %xmm2,%xmm10");
|
|
asm volatile ("movdqa %xmm7,%xmm15");
|
|
asm volatile ("pshufb %xmm4,%xmm2");
|
|
asm volatile ("pshufb %xmm12,%xmm10");
|
|
asm volatile ("pshufb %xmm5,%xmm7");
|
|
asm volatile ("pshufb %xmm13,%xmm15");
|
|
asm volatile ("pxor %xmm7,%xmm2");
|
|
asm volatile ("pxor %xmm15,%xmm10");
|
|
|
|
/* process two intermediate disks per iteration */
|
|
for (d = l - 1; d > 1; d -= 2) {
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
|
|
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16]));
|
|
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][0]));
|
|
asm volatile ("movdqa %0,%%xmm15" : : "m" (raid_gfcauchypshufb[d][1][1]));
|
|
|
|
if (generator == 3) {
|
|
asm volatile ("movdqa %xmm1,%xmm6");
|
|
asm volatile ("movdqa %xmm9,%xmm14");
|
|
}
|
|
asm volatile ("pxor %xmm5,%xmm5");
|
|
asm volatile ("pxor %xmm13,%xmm13");
|
|
asm volatile ("pcmpgtb %xmm1,%xmm5");
|
|
asm volatile ("pcmpgtb %xmm9,%xmm13");
|
|
asm volatile ("paddb %xmm1,%xmm1");
|
|
asm volatile ("paddb %xmm9,%xmm9");
|
|
asm volatile ("pand %xmm3,%xmm5");
|
|
asm volatile ("pand %xmm3,%xmm13");
|
|
asm volatile ("pxor %xmm5,%xmm1");
|
|
asm volatile ("pxor %xmm13,%xmm9");
|
|
if (generator == 3) {
|
|
asm volatile ("pxor %xmm6,%xmm1");
|
|
asm volatile ("pxor %xmm14,%xmm9");
|
|
}
|
|
|
|
asm volatile ("pxor %xmm4,%xmm0");
|
|
asm volatile ("pxor %xmm4,%xmm1");
|
|
asm volatile ("pxor %xmm12,%xmm8");
|
|
asm volatile ("pxor %xmm12,%xmm9");
|
|
|
|
asm volatile ("movdqa %xmm4,%xmm5");
|
|
asm volatile ("movdqa %xmm12,%xmm13");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm13");
|
|
asm volatile ("pand %xmm11,%xmm4");
|
|
asm volatile ("pand %xmm11,%xmm12");
|
|
asm volatile ("pand %xmm11,%xmm5");
|
|
asm volatile ("pand %xmm11,%xmm13");
|
|
|
|
asm volatile ("movdqa %xmm7,%xmm6");
|
|
asm volatile ("movdqa %xmm15,%xmm14");
|
|
asm volatile ("pshufb %xmm4,%xmm7");
|
|
asm volatile ("pshufb %xmm12,%xmm6");
|
|
asm volatile ("pshufb %xmm5,%xmm15");
|
|
asm volatile ("pshufb %xmm13,%xmm14");
|
|
asm volatile ("pxor %xmm7,%xmm2");
|
|
asm volatile ("pxor %xmm6,%xmm10");
|
|
asm volatile ("pxor %xmm15,%xmm2");
|
|
asm volatile ("pxor %xmm14,%xmm10");
|
|
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d - 1][i]));
|
|
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d - 1][i + 16]));
|
|
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d - 1][1][0]));
|
|
asm volatile ("movdqa %0,%%xmm15" : : "m" (raid_gfcauchypshufb[d - 1][1][1]));
|
|
|
|
if (generator == 3) {
|
|
asm volatile ("movdqa %xmm1,%xmm6");
|
|
asm volatile ("movdqa %xmm9,%xmm14");
|
|
}
|
|
asm volatile ("pxor %xmm5,%xmm5");
|
|
asm volatile ("pxor %xmm13,%xmm13");
|
|
asm volatile ("pcmpgtb %xmm1,%xmm5");
|
|
asm volatile ("pcmpgtb %xmm9,%xmm13");
|
|
asm volatile ("paddb %xmm1,%xmm1");
|
|
asm volatile ("paddb %xmm9,%xmm9");
|
|
asm volatile ("pand %xmm3,%xmm5");
|
|
asm volatile ("pand %xmm3,%xmm13");
|
|
asm volatile ("pxor %xmm5,%xmm1");
|
|
asm volatile ("pxor %xmm13,%xmm9");
|
|
if (generator == 3) {
|
|
asm volatile ("pxor %xmm6,%xmm1");
|
|
asm volatile ("pxor %xmm14,%xmm9");
|
|
}
|
|
|
|
asm volatile ("pxor %xmm4,%xmm0");
|
|
asm volatile ("pxor %xmm4,%xmm1");
|
|
asm volatile ("pxor %xmm12,%xmm8");
|
|
asm volatile ("pxor %xmm12,%xmm9");
|
|
|
|
asm volatile ("movdqa %xmm4,%xmm5");
|
|
asm volatile ("movdqa %xmm12,%xmm13");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm13");
|
|
asm volatile ("pand %xmm11,%xmm4");
|
|
asm volatile ("pand %xmm11,%xmm12");
|
|
asm volatile ("pand %xmm11,%xmm5");
|
|
asm volatile ("pand %xmm11,%xmm13");
|
|
|
|
asm volatile ("movdqa %xmm7,%xmm6");
|
|
asm volatile ("movdqa %xmm15,%xmm14");
|
|
asm volatile ("pshufb %xmm4,%xmm7");
|
|
asm volatile ("pshufb %xmm12,%xmm6");
|
|
asm volatile ("pshufb %xmm5,%xmm15");
|
|
asm volatile ("pshufb %xmm13,%xmm14");
|
|
asm volatile ("pxor %xmm7,%xmm2");
|
|
asm volatile ("pxor %xmm6,%xmm10");
|
|
asm volatile ("pxor %xmm15,%xmm2");
|
|
asm volatile ("pxor %xmm14,%xmm10");
|
|
}
|
|
|
|
/* single remaining intermediate disk */
|
|
if (d == 1) {
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[1][i]));
|
|
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[1][i + 16]));
|
|
|
|
if (generator == 3) {
|
|
asm volatile ("movdqa %xmm1,%xmm6");
|
|
asm volatile ("movdqa %xmm9,%xmm14");
|
|
}
|
|
asm volatile ("pxor %xmm5,%xmm5");
|
|
asm volatile ("pxor %xmm13,%xmm13");
|
|
asm volatile ("pcmpgtb %xmm1,%xmm5");
|
|
asm volatile ("pcmpgtb %xmm9,%xmm13");
|
|
asm volatile ("paddb %xmm1,%xmm1");
|
|
asm volatile ("paddb %xmm9,%xmm9");
|
|
asm volatile ("pand %xmm3,%xmm5");
|
|
asm volatile ("pand %xmm3,%xmm13");
|
|
asm volatile ("pxor %xmm5,%xmm1");
|
|
asm volatile ("pxor %xmm13,%xmm9");
|
|
if (generator == 3) {
|
|
asm volatile ("pxor %xmm6,%xmm1");
|
|
asm volatile ("pxor %xmm14,%xmm9");
|
|
}
|
|
|
|
asm volatile ("pxor %xmm4,%xmm0");
|
|
asm volatile ("pxor %xmm4,%xmm1");
|
|
asm volatile ("pxor %xmm12,%xmm8");
|
|
asm volatile ("pxor %xmm12,%xmm9");
|
|
|
|
asm volatile ("movdqa %xmm4,%xmm5");
|
|
asm volatile ("movdqa %xmm12,%xmm13");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm13");
|
|
asm volatile ("pand %xmm11,%xmm4");
|
|
asm volatile ("pand %xmm11,%xmm12");
|
|
asm volatile ("pand %xmm11,%xmm5");
|
|
asm volatile ("pand %xmm11,%xmm13");
|
|
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[1][1][0]));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[1][1][1]));
|
|
asm volatile ("movdqa %xmm6,%xmm14");
|
|
asm volatile ("movdqa %xmm7,%xmm15");
|
|
asm volatile ("pshufb %xmm4,%xmm6");
|
|
asm volatile ("pshufb %xmm12,%xmm14");
|
|
asm volatile ("pshufb %xmm5,%xmm7");
|
|
asm volatile ("pshufb %xmm13,%xmm15");
|
|
asm volatile ("pxor %xmm6,%xmm2");
|
|
asm volatile ("pxor %xmm14,%xmm10");
|
|
asm volatile ("pxor %xmm7,%xmm2");
|
|
asm volatile ("pxor %xmm15,%xmm10");
|
|
}
|
|
|
|
/* first disk with all coefficients at 1 */
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));
|
|
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[0][i + 16]));
|
|
|
|
if (generator == 3) {
|
|
asm volatile ("movdqa %xmm1,%xmm6");
|
|
asm volatile ("movdqa %xmm9,%xmm14");
|
|
}
|
|
asm volatile ("pxor %xmm5,%xmm5");
|
|
asm volatile ("pxor %xmm13,%xmm13");
|
|
asm volatile ("pcmpgtb %xmm1,%xmm5");
|
|
asm volatile ("pcmpgtb %xmm9,%xmm13");
|
|
asm volatile ("paddb %xmm1,%xmm1");
|
|
asm volatile ("paddb %xmm9,%xmm9");
|
|
asm volatile ("pand %xmm3,%xmm5");
|
|
asm volatile ("pand %xmm3,%xmm13");
|
|
asm volatile ("pxor %xmm5,%xmm1");
|
|
asm volatile ("pxor %xmm13,%xmm9");
|
|
if (generator == 3) {
|
|
asm volatile ("pxor %xmm6,%xmm1");
|
|
asm volatile ("pxor %xmm14,%xmm9");
|
|
}
|
|
|
|
asm volatile ("pxor %xmm4,%xmm0");
|
|
asm volatile ("pxor %xmm4,%xmm1");
|
|
asm volatile ("pxor %xmm4,%xmm2");
|
|
asm volatile ("pxor %xmm12,%xmm8");
|
|
asm volatile ("pxor %xmm12,%xmm9");
|
|
asm volatile ("pxor %xmm12,%xmm10");
|
|
|
|
if (streaming) {
|
|
asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i]));
|
|
asm volatile ("movntdq %%xmm8,%0" : "=m" (p[i + 16]));
|
|
asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i]));
|
|
asm volatile ("movntdq %%xmm9,%0" : "=m" (q[i + 16]));
|
|
asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i]));
|
|
asm volatile ("movntdq %%xmm10,%0" : "=m" (r[i + 16]));
|
|
} else {
|
|
asm volatile ("movdqa %%xmm0,%0" : "=m" (p[i]));
|
|
asm volatile ("movdqa %%xmm8,%0" : "=m" (p[i + 16]));
|
|
asm volatile ("movdqa %%xmm1,%0" : "=m" (q[i]));
|
|
asm volatile ("movdqa %%xmm9,%0" : "=m" (q[i + 16]));
|
|
asm volatile ("movdqa %%xmm2,%0" : "=m" (r[i]));
|
|
asm volatile ("movdqa %%xmm10,%0" : "=m" (r[i + 16]));
|
|
}
|
|
}
|
|
|
|
raid_sse_end(streaming);
|
|
}
|
|
|
|
/*
|
|
* Generate four parity blocks with Cauchy matrix using SSSE3 extended implementation.
|
|
*
|
|
* Uses the extended register set (16 XMM registers) and processes two disks
|
|
* per iteration across two 16-byte XMM lanes (32 bytes/step).
|
|
*
|
|
* Note that it uses 16 registers, meaning that x64 is required.
|
|
*/
|
|
static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **vv, int generator, int streaming)
|
|
{
|
|
uint8_t **v = (uint8_t **)vv;
|
|
uint8_t *p;
|
|
uint8_t *q;
|
|
uint8_t *r;
|
|
uint8_t *s;
|
|
int d, l;
|
|
size_t i;
|
|
|
|
l = nd - 1;
|
|
p = v[nd];
|
|
q = v[nd + 1];
|
|
r = v[nd + 2];
|
|
s = v[nd + 3];
|
|
|
|
/* special case with only one data disk */
|
|
if (l == 0) {
|
|
for (i = 0; i < 4; ++i)
|
|
if (v[1 + i] != v[0])
|
|
memcpy(v[1 + i], v[0], size);
|
|
return;
|
|
}
|
|
|
|
raid_sse_begin();
|
|
|
|
/* generic case with at least two data disks */
|
|
for (i = 0; i < size; i += 32) {
|
|
/* last disk without the generator multiplication */
|
|
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4));
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));
|
|
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[l][i + 16]));
|
|
|
|
asm volatile ("movdqa %xmm4,%xmm0");
|
|
asm volatile ("movdqa %xmm4,%xmm1");
|
|
asm volatile ("movdqa %xmm12,%xmm8");
|
|
asm volatile ("movdqa %xmm12,%xmm9");
|
|
|
|
asm volatile ("movdqa %xmm4,%xmm5");
|
|
asm volatile ("movdqa %xmm12,%xmm13");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm13");
|
|
asm volatile ("pand %xmm15,%xmm4");
|
|
asm volatile ("pand %xmm15,%xmm12");
|
|
asm volatile ("pand %xmm15,%xmm5");
|
|
asm volatile ("pand %xmm15,%xmm13");
|
|
|
|
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0]));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1]));
|
|
asm volatile ("movdqa %xmm2,%xmm10");
|
|
asm volatile ("movdqa %xmm7,%xmm15");
|
|
asm volatile ("pshufb %xmm4,%xmm2");
|
|
asm volatile ("pshufb %xmm12,%xmm10");
|
|
asm volatile ("pshufb %xmm5,%xmm7");
|
|
asm volatile ("pshufb %xmm13,%xmm15");
|
|
asm volatile ("pxor %xmm7,%xmm2");
|
|
asm volatile ("pxor %xmm15,%xmm10");
|
|
|
|
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][2][0]));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1]));
|
|
asm volatile ("movdqa %xmm3,%xmm11");
|
|
asm volatile ("movdqa %xmm7,%xmm15");
|
|
asm volatile ("pshufb %xmm4,%xmm3");
|
|
asm volatile ("pshufb %xmm12,%xmm11");
|
|
asm volatile ("pshufb %xmm5,%xmm7");
|
|
asm volatile ("pshufb %xmm13,%xmm15");
|
|
asm volatile ("pxor %xmm7,%xmm3");
|
|
asm volatile ("pxor %xmm15,%xmm11");
|
|
|
|
/* intermediate disks */
|
|
for (d = l - 1; d > 0; --d) {
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly));
|
|
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4));
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
|
|
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16]));
|
|
|
|
if (generator == 3) {
|
|
asm volatile ("movdqa %xmm1,%xmm6");
|
|
asm volatile ("movdqa %xmm9,%xmm14");
|
|
}
|
|
asm volatile ("pxor %xmm5,%xmm5");
|
|
asm volatile ("pxor %xmm13,%xmm13");
|
|
asm volatile ("pcmpgtb %xmm1,%xmm5");
|
|
asm volatile ("pcmpgtb %xmm9,%xmm13");
|
|
asm volatile ("paddb %xmm1,%xmm1");
|
|
asm volatile ("paddb %xmm9,%xmm9");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
asm volatile ("pand %xmm7,%xmm13");
|
|
asm volatile ("pxor %xmm5,%xmm1");
|
|
asm volatile ("pxor %xmm13,%xmm9");
|
|
if (generator == 3) {
|
|
asm volatile ("pxor %xmm6,%xmm1");
|
|
asm volatile ("pxor %xmm14,%xmm9");
|
|
}
|
|
|
|
asm volatile ("pxor %xmm4,%xmm0");
|
|
asm volatile ("pxor %xmm4,%xmm1");
|
|
asm volatile ("pxor %xmm12,%xmm8");
|
|
asm volatile ("pxor %xmm12,%xmm9");
|
|
|
|
asm volatile ("movdqa %xmm4,%xmm5");
|
|
asm volatile ("movdqa %xmm12,%xmm13");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm13");
|
|
asm volatile ("pand %xmm15,%xmm4");
|
|
asm volatile ("pand %xmm15,%xmm12");
|
|
asm volatile ("pand %xmm15,%xmm5");
|
|
asm volatile ("pand %xmm15,%xmm13");
|
|
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0]));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1]));
|
|
asm volatile ("movdqa %xmm6,%xmm14");
|
|
asm volatile ("movdqa %xmm7,%xmm15");
|
|
asm volatile ("pshufb %xmm4,%xmm6");
|
|
asm volatile ("pshufb %xmm12,%xmm14");
|
|
asm volatile ("pshufb %xmm5,%xmm7");
|
|
asm volatile ("pshufb %xmm13,%xmm15");
|
|
asm volatile ("pxor %xmm6,%xmm2");
|
|
asm volatile ("pxor %xmm14,%xmm10");
|
|
asm volatile ("pxor %xmm7,%xmm2");
|
|
asm volatile ("pxor %xmm15,%xmm10");
|
|
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0]));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1]));
|
|
asm volatile ("movdqa %xmm6,%xmm14");
|
|
asm volatile ("movdqa %xmm7,%xmm15");
|
|
asm volatile ("pshufb %xmm4,%xmm6");
|
|
asm volatile ("pshufb %xmm12,%xmm14");
|
|
asm volatile ("pshufb %xmm5,%xmm7");
|
|
asm volatile ("pshufb %xmm13,%xmm15");
|
|
asm volatile ("pxor %xmm6,%xmm3");
|
|
asm volatile ("pxor %xmm14,%xmm11");
|
|
asm volatile ("pxor %xmm7,%xmm3");
|
|
asm volatile ("pxor %xmm15,%xmm11");
|
|
}
|
|
|
|
/* first disk with all coefficients at 1 */
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly));
|
|
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4));
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));
|
|
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[0][i + 16]));
|
|
|
|
if (generator == 3) {
|
|
asm volatile ("movdqa %xmm1,%xmm6");
|
|
asm volatile ("movdqa %xmm9,%xmm14");
|
|
}
|
|
asm volatile ("pxor %xmm5,%xmm5");
|
|
asm volatile ("pxor %xmm13,%xmm13");
|
|
asm volatile ("pcmpgtb %xmm1,%xmm5");
|
|
asm volatile ("pcmpgtb %xmm9,%xmm13");
|
|
asm volatile ("paddb %xmm1,%xmm1");
|
|
asm volatile ("paddb %xmm9,%xmm9");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
asm volatile ("pand %xmm7,%xmm13");
|
|
asm volatile ("pxor %xmm5,%xmm1");
|
|
asm volatile ("pxor %xmm13,%xmm9");
|
|
if (generator == 3) {
|
|
asm volatile ("pxor %xmm6,%xmm1");
|
|
asm volatile ("pxor %xmm14,%xmm9");
|
|
}
|
|
|
|
asm volatile ("pxor %xmm4,%xmm0");
|
|
asm volatile ("pxor %xmm4,%xmm1");
|
|
asm volatile ("pxor %xmm4,%xmm2");
|
|
asm volatile ("pxor %xmm4,%xmm3");
|
|
asm volatile ("pxor %xmm12,%xmm8");
|
|
asm volatile ("pxor %xmm12,%xmm9");
|
|
asm volatile ("pxor %xmm12,%xmm10");
|
|
asm volatile ("pxor %xmm12,%xmm11");
|
|
|
|
if (streaming) {
|
|
asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i]));
|
|
asm volatile ("movntdq %%xmm8,%0" : "=m" (p[i + 16]));
|
|
asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i]));
|
|
asm volatile ("movntdq %%xmm9,%0" : "=m" (q[i + 16]));
|
|
asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i]));
|
|
asm volatile ("movntdq %%xmm10,%0" : "=m" (r[i + 16]));
|
|
asm volatile ("movntdq %%xmm3,%0" : "=m" (s[i]));
|
|
asm volatile ("movntdq %%xmm11,%0" : "=m" (s[i + 16]));
|
|
} else {
|
|
asm volatile ("movdqa %%xmm0,%0" : "=m" (p[i]));
|
|
asm volatile ("movdqa %%xmm8,%0" : "=m" (p[i + 16]));
|
|
asm volatile ("movdqa %%xmm1,%0" : "=m" (q[i]));
|
|
asm volatile ("movdqa %%xmm9,%0" : "=m" (q[i + 16]));
|
|
asm volatile ("movdqa %%xmm2,%0" : "=m" (r[i]));
|
|
asm volatile ("movdqa %%xmm10,%0" : "=m" (r[i + 16]));
|
|
asm volatile ("movdqa %%xmm3,%0" : "=m" (s[i]));
|
|
asm volatile ("movdqa %%xmm11,%0" : "=m" (s[i + 16]));
|
|
}
|
|
}
|
|
|
|
raid_sse_end(streaming);
|
|
}
|
|
|
|
/*
|
|
* Generate N parity blocks with Cauchy matrix using SSSE3 extended implementation.
|
|
*/
|
|
static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, int np, int generator, int streaming)
|
|
{
|
|
uint8_t **v = (uint8_t **)vv;
|
|
uint8_t *p;
|
|
uint8_t *q;
|
|
uint8_t *r;
|
|
uint8_t *s;
|
|
uint8_t *t;
|
|
uint8_t *u;
|
|
int d, l;
|
|
size_t i;
|
|
|
|
l = nd - 1;
|
|
p = v[nd];
|
|
q = v[nd + 1];
|
|
r = v[nd + 2];
|
|
s = v[nd + 3];
|
|
t = v[nd + 4];
|
|
if (np >= 6)
|
|
u = v[nd + 5];
|
|
|
|
/* special case with only one data disk */
|
|
if (l == 0) {
|
|
for (d = 0; d < np; ++d)
|
|
if (v[1 + d] != v[0])
|
|
memcpy(v[1 + d], v[0], size);
|
|
return;
|
|
}
|
|
|
|
raid_sse_begin();
|
|
|
|
/* generic case with at least two data disks */
|
|
asm volatile ("movdqa %0,%%xmm14" : : "m" (gfconst16.poly));
|
|
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4));
|
|
|
|
for (i = 0; i < size; i += 16) {
|
|
/* last disk without the generator multiplication */
|
|
asm volatile ("movdqa %0,%%xmm10" : : "m" (v[l][i]));
|
|
|
|
asm volatile ("movdqa %xmm10,%xmm0");
|
|
asm volatile ("movdqa %xmm10,%xmm1");
|
|
|
|
asm volatile ("movdqa %xmm10,%xmm11");
|
|
asm volatile ("psrlw $4,%xmm11");
|
|
asm volatile ("pand %xmm15,%xmm10");
|
|
asm volatile ("pand %xmm15,%xmm11");
|
|
|
|
if (np >= 3) {
|
|
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0]));
|
|
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[l][1][1]));
|
|
asm volatile ("pshufb %xmm10,%xmm2");
|
|
asm volatile ("pshufb %xmm11,%xmm13");
|
|
asm volatile ("pxor %xmm13,%xmm2");
|
|
}
|
|
|
|
if (np >= 4) {
|
|
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][2][0]));
|
|
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[l][2][1]));
|
|
asm volatile ("pshufb %xmm10,%xmm3");
|
|
asm volatile ("pshufb %xmm11,%xmm13");
|
|
asm volatile ("pxor %xmm13,%xmm3");
|
|
}
|
|
|
|
if (np >= 5) {
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (raid_gfcauchypshufb[l][3][0]));
|
|
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[l][3][1]));
|
|
asm volatile ("pshufb %xmm10,%xmm4");
|
|
asm volatile ("pshufb %xmm11,%xmm13");
|
|
asm volatile ("pxor %xmm13,%xmm4");
|
|
}
|
|
|
|
if (np >= 6) {
|
|
asm volatile ("movdqa %0,%%xmm5" : : "m" (raid_gfcauchypshufb[l][4][0]));
|
|
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[l][4][1]));
|
|
asm volatile ("pshufb %xmm10,%xmm5");
|
|
asm volatile ("pshufb %xmm11,%xmm13");
|
|
asm volatile ("pxor %xmm13,%xmm5");
|
|
}
|
|
|
|
/* intermediate disks */
|
|
for (d = l - 1; d > 0; --d) {
|
|
asm volatile ("movdqa %0,%%xmm10" : : "m" (v[d][i]));
|
|
|
|
if (generator == 3)
|
|
asm volatile ("movdqa %xmm1,%xmm12");
|
|
asm volatile ("pxor %xmm11,%xmm11");
|
|
asm volatile ("pcmpgtb %xmm1,%xmm11");
|
|
asm volatile ("paddb %xmm1,%xmm1");
|
|
asm volatile ("pand %xmm14,%xmm11");
|
|
asm volatile ("pxor %xmm11,%xmm1");
|
|
if (generator == 3)
|
|
asm volatile ("pxor %xmm12,%xmm1");
|
|
|
|
asm volatile ("pxor %xmm10,%xmm0");
|
|
asm volatile ("pxor %xmm10,%xmm1");
|
|
|
|
asm volatile ("movdqa %xmm10,%xmm11");
|
|
asm volatile ("psrlw $4,%xmm11");
|
|
asm volatile ("pand %xmm15,%xmm10");
|
|
asm volatile ("pand %xmm15,%xmm11");
|
|
|
|
if (np >= 3) {
|
|
asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][1][0]));
|
|
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][1][1]));
|
|
asm volatile ("pshufb %xmm10,%xmm12");
|
|
asm volatile ("pshufb %xmm11,%xmm13");
|
|
asm volatile ("pxor %xmm12,%xmm2");
|
|
asm volatile ("pxor %xmm13,%xmm2");
|
|
}
|
|
|
|
if (np >= 4) {
|
|
asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][2][0]));
|
|
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][2][1]));
|
|
asm volatile ("pshufb %xmm10,%xmm12");
|
|
asm volatile ("pshufb %xmm11,%xmm13");
|
|
asm volatile ("pxor %xmm12,%xmm3");
|
|
asm volatile ("pxor %xmm13,%xmm3");
|
|
}
|
|
|
|
if (np >= 5) {
|
|
asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][3][0]));
|
|
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][3][1]));
|
|
asm volatile ("pshufb %xmm10,%xmm12");
|
|
asm volatile ("pshufb %xmm11,%xmm13");
|
|
asm volatile ("pxor %xmm12,%xmm4");
|
|
asm volatile ("pxor %xmm13,%xmm4");
|
|
}
|
|
|
|
if (np >= 6) {
|
|
asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfcauchypshufb[d][4][0]));
|
|
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfcauchypshufb[d][4][1]));
|
|
asm volatile ("pshufb %xmm10,%xmm12");
|
|
asm volatile ("pshufb %xmm11,%xmm13");
|
|
asm volatile ("pxor %xmm12,%xmm5");
|
|
asm volatile ("pxor %xmm13,%xmm5");
|
|
}
|
|
}
|
|
|
|
/* first disk with all coefficients at 1 */
|
|
asm volatile ("movdqa %0,%%xmm10" : : "m" (v[0][i]));
|
|
|
|
if (generator == 3)
|
|
asm volatile ("movdqa %xmm1,%xmm12");
|
|
asm volatile ("pxor %xmm11,%xmm11");
|
|
asm volatile ("pcmpgtb %xmm1,%xmm11");
|
|
asm volatile ("paddb %xmm1,%xmm1");
|
|
asm volatile ("pand %xmm14,%xmm11");
|
|
asm volatile ("pxor %xmm11,%xmm1");
|
|
if (generator == 3)
|
|
asm volatile ("pxor %xmm12,%xmm1");
|
|
|
|
asm volatile ("pxor %xmm10,%xmm0");
|
|
asm volatile ("pxor %xmm10,%xmm1");
|
|
if (np >= 3)
|
|
asm volatile ("pxor %xmm10,%xmm2");
|
|
if (np >= 4)
|
|
asm volatile ("pxor %xmm10,%xmm3");
|
|
if (np >= 5)
|
|
asm volatile ("pxor %xmm10,%xmm4");
|
|
if (np >= 6)
|
|
asm volatile ("pxor %xmm10,%xmm5");
|
|
|
|
if (streaming) {
|
|
asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i]));
|
|
asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i]));
|
|
if (np >= 3)
|
|
asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i]));
|
|
if (np >= 4)
|
|
asm volatile ("movntdq %%xmm3,%0" : "=m" (s[i]));
|
|
if (np >= 5)
|
|
asm volatile ("movntdq %%xmm4,%0" : "=m" (t[i]));
|
|
if (np >= 6)
|
|
asm volatile ("movntdq %%xmm5,%0" : "=m" (u[i]));
|
|
} else {
|
|
asm volatile ("movdqa %%xmm0,%0" : "=m" (p[i]));
|
|
asm volatile ("movdqa %%xmm1,%0" : "=m" (q[i]));
|
|
if (np >= 3)
|
|
asm volatile ("movdqa %%xmm2,%0" : "=m" (r[i]));
|
|
if (np >= 4)
|
|
asm volatile ("movdqa %%xmm3,%0" : "=m" (s[i]));
|
|
if (np >= 5)
|
|
asm volatile ("movdqa %%xmm4,%0" : "=m" (t[i]));
|
|
if (np >= 6)
|
|
asm volatile ("movdqa %%xmm5,%0" : "=m" (u[i]));
|
|
}
|
|
}
|
|
|
|
raid_sse_end(streaming);
|
|
}
|
|
|
|
#endif
|
|
|
|
/*
|
|
* Recover failure of one data block using selected parity with SSSE3, optimized for one failure.
|
|
*
|
|
* Computes the selected syndrome in a single survivor scan and reconstructs
|
|
* the missing block directly, avoiding extra memory passes.
|
|
*/
|
|
static __always_inline void raid_rec1_ssse3_1(int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
uint8_t **v = (uint8_t **)vv;
|
|
uint8_t *p;
|
|
uint8_t *pa;
|
|
uint8_t *src[RAID_DATA_MAX];
|
|
const uint8_t *S[RAID_DATA_MAX];
|
|
uint8_t G;
|
|
uint8_t V;
|
|
size_t i;
|
|
int d, s;
|
|
int ns;
|
|
|
|
G = A(ip[0], id[0]);
|
|
V = inv(G);
|
|
|
|
p = v[nd + ip[0]];
|
|
pa = v[id[0]];
|
|
|
|
ns = 0;
|
|
for (d = 0; d < nd; ++d) {
|
|
if (d == id[0])
|
|
continue;
|
|
|
|
src[ns] = v[d];
|
|
S[ns] = &raid_gfmulpshufb[A(ip[0], d)][0][0];
|
|
++ns;
|
|
}
|
|
|
|
BUG_ON(ns != nd - 1);
|
|
|
|
raid_sse_begin();
|
|
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4));
|
|
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfmulpshufb[V][0]));
|
|
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfmulpshufb[V][1]));
|
|
|
|
for (i = 0; i < size; i += 16) {
|
|
/* start from the selected parity block */
|
|
asm volatile ("movdqa %0,%%xmm0" : : "m" (p[i]));
|
|
|
|
/* compute only the selected syndrome */
|
|
for (s = 0; s < ns; ++s) {
|
|
const uint8_t *t = S[s];
|
|
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (src[s][i]));
|
|
asm volatile ("movdqa %xmm4,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("pand %xmm7,%xmm4");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (t[0]));
|
|
asm volatile ("pshufb %xmm4,%xmm6");
|
|
asm volatile ("pxor %xmm6,%xmm0");
|
|
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (t[16]));
|
|
asm volatile ("pshufb %xmm5,%xmm6");
|
|
asm volatile ("pxor %xmm6,%xmm0");
|
|
}
|
|
|
|
/* multiply the syndrome by the inverse coefficient */
|
|
asm volatile ("movdqa %xmm0,%xmm4");
|
|
asm volatile ("movdqa %xmm0,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("pand %xmm7,%xmm4");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
|
|
asm volatile ("movdqa %xmm2,%xmm6");
|
|
asm volatile ("movdqa %xmm3,%xmm1");
|
|
asm volatile ("pshufb %xmm4,%xmm6");
|
|
asm volatile ("pshufb %xmm5,%xmm1");
|
|
asm volatile ("pxor %xmm1,%xmm6");
|
|
|
|
/* recovery data must remain cacheable */
|
|
asm volatile ("movdqa %%xmm6,%0" : "=m" (pa[i]));
|
|
}
|
|
|
|
raid_sse_end(0);
|
|
}
|
|
|
|
/*
|
|
* Recover failure of one data block using Q with SSSE3.
|
|
*
|
|
* Computes Q of all surviving data directly with Horner's method and
|
|
* reconstructs the missing block from Qdelta.
|
|
*
|
|
* Processes two 16-byte lanes per iteration, avoiding temporary parity
|
|
* buffers and the extra pass over the data.
|
|
*/
|
|
static __always_inline void raid_rec1_ssse3_q(int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
uint8_t **v = (uint8_t **)vv;
|
|
uint8_t *q;
|
|
uint8_t *pa;
|
|
uint8_t V;
|
|
int generator;
|
|
int l;
|
|
int d;
|
|
size_t i;
|
|
|
|
BUG_ON(ip[0] != 1);
|
|
|
|
V = inv(A(1, id[0]));
|
|
|
|
generator = powgen(1);
|
|
BUG_ON(generator != 2 && generator != 3);
|
|
|
|
l = nd - 1;
|
|
|
|
q = v[nd + 1];
|
|
pa = v[id[0]];
|
|
|
|
raid_sse_begin();
|
|
|
|
/* keep the inverse coefficient resident */
|
|
asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfmulpshufb[V][0]));
|
|
asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfmulpshufb[V][1]));
|
|
|
|
/* keep the active reduction polynomial resident */
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly));
|
|
|
|
for (i = 0; i < size; i += 32) {
|
|
/* last disk starts Horner without generator multiplication */
|
|
if (l == id[0]) {
|
|
asm volatile ("pxor %xmm0,%xmm0");
|
|
asm volatile ("pxor %xmm1,%xmm1");
|
|
} else {
|
|
asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i]));
|
|
asm volatile ("movdqa %0,%%xmm1" : : "m" (v[l][i + 16]));
|
|
}
|
|
|
|
for (d = l - 1; d >= 0; --d) {
|
|
/* multiply both Q lanes by the active generator */
|
|
if (generator == 3) {
|
|
/* lane 0 */
|
|
asm volatile ("movdqa %xmm0,%xmm4");
|
|
asm volatile ("pxor %xmm5,%xmm5");
|
|
asm volatile ("pcmpgtb %xmm0,%xmm5");
|
|
asm volatile ("paddb %xmm0,%xmm0");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
asm volatile ("pxor %xmm5,%xmm0");
|
|
asm volatile ("pxor %xmm4,%xmm0");
|
|
|
|
/* lane 1 */
|
|
asm volatile ("movdqa %xmm1,%xmm4");
|
|
asm volatile ("pxor %xmm5,%xmm5");
|
|
asm volatile ("pcmpgtb %xmm1,%xmm5");
|
|
asm volatile ("paddb %xmm1,%xmm1");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
asm volatile ("pxor %xmm5,%xmm1");
|
|
asm volatile ("pxor %xmm4,%xmm1");
|
|
} else {
|
|
/* multiply both lanes by 2 in parallel */
|
|
asm volatile ("pxor %xmm4,%xmm4");
|
|
asm volatile ("pxor %xmm5,%xmm5");
|
|
asm volatile ("pcmpgtb %xmm0,%xmm4");
|
|
asm volatile ("pcmpgtb %xmm1,%xmm5");
|
|
asm volatile ("paddb %xmm0,%xmm0");
|
|
asm volatile ("paddb %xmm1,%xmm1");
|
|
asm volatile ("pand %xmm7,%xmm4");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
asm volatile ("pxor %xmm4,%xmm0");
|
|
asm volatile ("pxor %xmm5,%xmm1");
|
|
}
|
|
|
|
/* missing disk contributes zero */
|
|
if (d == id[0])
|
|
continue;
|
|
|
|
asm volatile ("pxor %0,%%xmm0" : : "m" (v[d][i]));
|
|
asm volatile ("pxor %0,%%xmm1" : : "m" (v[d][i + 16]));
|
|
}
|
|
|
|
/* Qdelta = stored Q ^ Q of all surviving data */
|
|
asm volatile ("pxor %0,%%xmm0" : : "m" (q[i]));
|
|
asm volatile ("pxor %0,%%xmm1" : : "m" (q[i + 16]));
|
|
|
|
/* low-nibble mask */
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (gfconst16.low4));
|
|
|
|
/* split both Qdelta lanes into low/high nibbles */
|
|
asm volatile ("movdqa %xmm0,%xmm4");
|
|
asm volatile ("movdqa %xmm1,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm0");
|
|
asm volatile ("psrlw $4,%xmm1");
|
|
asm volatile ("pand %xmm6,%xmm4");
|
|
asm volatile ("pand %xmm6,%xmm5");
|
|
asm volatile ("pand %xmm6,%xmm0");
|
|
asm volatile ("pand %xmm6,%xmm1");
|
|
|
|
/* low-nibble products */
|
|
asm volatile ("movdqa %xmm2,%xmm6");
|
|
asm volatile ("movdqa %xmm2,%xmm7");
|
|
asm volatile ("pshufb %xmm4,%xmm6");
|
|
asm volatile ("pshufb %xmm5,%xmm7");
|
|
|
|
/* high-nibble products */
|
|
asm volatile ("movdqa %xmm3,%xmm4");
|
|
asm volatile ("movdqa %xmm3,%xmm5");
|
|
asm volatile ("pshufb %xmm0,%xmm4");
|
|
asm volatile ("pshufb %xmm1,%xmm5");
|
|
asm volatile ("pxor %xmm4,%xmm6");
|
|
asm volatile ("pxor %xmm5,%xmm7");
|
|
|
|
/* recovery data must remain cacheable */
|
|
asm volatile ("movdqa %%xmm6,%0" : "=m" (pa[i]));
|
|
asm volatile ("movdqa %%xmm7,%0" : "=m" (pa[i + 16]));
|
|
|
|
/* restore the active reduction polynomial */
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly));
|
|
}
|
|
|
|
raid_sse_end(0);
|
|
}
|
|
|
|
/*
|
|
* Recover failure of two data blocks using selected parity with SSSE3,
|
|
* optimized specifically for two failures.
|
|
*
|
|
* Computes only the two selected syndromes in a single survivor scan and
|
|
* reconstructs the missing blocks directly, avoiding temporary syndrome
|
|
* buffers and the generation of unused parity rows.
|
|
*
|
|
* has_p is expected to be a compile-time constant after inlining.
|
|
*
|
|
* If P is available, reconstructs only the first missing block through the
|
|
* inverse matrix and obtains the second one by XORing it out of Pdelta.
|
|
*/
|
|
static __always_inline void raid_rec2_ssse3_2(int has_p, int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
uint8_t **v = (uint8_t **)vv;
|
|
uint8_t *p[2];
|
|
uint8_t *pa[2];
|
|
uint8_t *src[RAID_DATA_MAX];
|
|
const uint8_t *S0[RAID_DATA_MAX];
|
|
const uint8_t *S1[RAID_DATA_MAX];
|
|
const uint8_t *R[4];
|
|
uint8_t G[2 * 2];
|
|
uint8_t V[2 * 2];
|
|
size_t i;
|
|
int d, s;
|
|
int ns;
|
|
|
|
/* setup and invert the 2x2 coefficients matrix */
|
|
G[0] = A(ip[0], id[0]);
|
|
G[1] = A(ip[0], id[1]);
|
|
G[2] = A(ip[1], id[0]);
|
|
G[3] = A(ip[1], id[1]);
|
|
raid_invert(G, V, 2);
|
|
|
|
p[0] = v[nd + ip[0]];
|
|
p[1] = v[nd + ip[1]];
|
|
pa[0] = v[id[0]];
|
|
pa[1] = v[id[1]];
|
|
|
|
/* collect surviving data blocks and their selected coefficients */
|
|
ns = 0;
|
|
for (d = 0; d < nd; ++d) {
|
|
if (d == id[0] || d == id[1])
|
|
continue;
|
|
|
|
src[ns] = v[d];
|
|
|
|
if (!has_p)
|
|
S0[ns] = &raid_gfmulpshufb[A(ip[0], d)][0][0];
|
|
|
|
S1[ns] = &raid_gfmulpshufb[A(ip[1], d)][0][0];
|
|
++ns;
|
|
}
|
|
|
|
BUG_ON(ns != nd - 2);
|
|
|
|
/* inverse-matrix multiplication tables */
|
|
R[0] = &raid_gfmulpshufb[V[0]][0][0];
|
|
R[1] = &raid_gfmulpshufb[V[1]][0][0];
|
|
|
|
if (!has_p) {
|
|
R[2] = &raid_gfmulpshufb[V[2]][0][0];
|
|
R[3] = &raid_gfmulpshufb[V[3]][0][0];
|
|
}
|
|
|
|
raid_sse_begin();
|
|
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4));
|
|
|
|
for (i = 0; i < size; i += 16) {
|
|
/* start from the two selected parity blocks */
|
|
asm volatile ("movdqa %0,%%xmm0" : : "m" (p[0][i]));
|
|
asm volatile ("movdqa %0,%%xmm1" : : "m" (p[1][i]));
|
|
|
|
/* compute only the two selected syndromes */
|
|
for (s = 0; s < ns; ++s) {
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (src[s][i]));
|
|
|
|
/* P has coefficient 1 */
|
|
if (has_p)
|
|
asm volatile ("pxor %xmm4,%xmm0");
|
|
|
|
/* split source into low/high nibbles once */
|
|
asm volatile ("movdqa %xmm4,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("pand %xmm7,%xmm4");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
|
|
if (!has_p) {
|
|
const uint8_t *t = S0[s];
|
|
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (t[0]));
|
|
asm volatile ("pshufb %xmm4,%xmm6");
|
|
asm volatile ("pxor %xmm6,%xmm0");
|
|
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (t[16]));
|
|
asm volatile ("pshufb %xmm5,%xmm6");
|
|
asm volatile ("pxor %xmm6,%xmm0");
|
|
}
|
|
|
|
{
|
|
const uint8_t *t = S1[s];
|
|
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (t[0]));
|
|
asm volatile ("pshufb %xmm4,%xmm6");
|
|
asm volatile ("pxor %xmm6,%xmm1");
|
|
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (t[16]));
|
|
asm volatile ("pshufb %xmm5,%xmm6");
|
|
asm volatile ("pxor %xmm6,%xmm1");
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Keep the complete P delta before splitting the syndromes.
|
|
* xmm6 is no longer needed by the survivor scan.
|
|
*/
|
|
if (has_p)
|
|
asm volatile ("movdqa %xmm0,%xmm6");
|
|
|
|
/*
|
|
* xmm0 = syndrome 0 low
|
|
* xmm2 = syndrome 0 high
|
|
* xmm1 = syndrome 1 low
|
|
* xmm3 = syndrome 1 high
|
|
*/
|
|
asm volatile ("movdqa %xmm0,%xmm2");
|
|
asm volatile ("movdqa %xmm1,%xmm3");
|
|
asm volatile ("psrlw $4,%xmm2");
|
|
asm volatile ("psrlw $4,%xmm3");
|
|
asm volatile ("pand %xmm7,%xmm0");
|
|
asm volatile ("pand %xmm7,%xmm2");
|
|
asm volatile ("pand %xmm7,%xmm1");
|
|
asm volatile ("pand %xmm7,%xmm3");
|
|
|
|
/* pa[0] = V[0] * syndrome0 ^ V[1] * syndrome1 */
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (R[0][0]));
|
|
asm volatile ("movdqa %0,%%xmm5" : : "m" (R[0][16]));
|
|
asm volatile ("pshufb %xmm0,%xmm4");
|
|
asm volatile ("pshufb %xmm2,%xmm5");
|
|
asm volatile ("pxor %xmm5,%xmm4");
|
|
|
|
asm volatile ("movdqa %0,%%xmm5" : : "m" (R[1][0]));
|
|
asm volatile ("pshufb %xmm1,%xmm5");
|
|
asm volatile ("pxor %xmm5,%xmm4");
|
|
|
|
asm volatile ("movdqa %0,%%xmm5" : : "m" (R[1][16]));
|
|
asm volatile ("pshufb %xmm3,%xmm5");
|
|
asm volatile ("pxor %xmm5,%xmm4");
|
|
|
|
if (has_p) {
|
|
/*
|
|
* Pdelta = pa[0] ^ pa[1].
|
|
* xmm4 is pa[0], so obtain pa[1] directly.
|
|
*/
|
|
asm volatile ("pxor %xmm4,%xmm6");
|
|
|
|
/* recovery data must remain cacheable */
|
|
asm volatile ("movdqa %%xmm4,%0" : "=m" (pa[0][i]));
|
|
asm volatile ("movdqa %%xmm6,%0" : "=m" (pa[1][i]));
|
|
} else {
|
|
/* recovery data must remain cacheable */
|
|
asm volatile ("movdqa %%xmm4,%0" : "=m" (pa[0][i]));
|
|
|
|
/* pa[1] = V[2] * syndrome0 ^ V[3] * syndrome1 */
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (R[2][0]));
|
|
asm volatile ("movdqa %0,%%xmm5" : : "m" (R[2][16]));
|
|
asm volatile ("pshufb %xmm0,%xmm4");
|
|
asm volatile ("pshufb %xmm2,%xmm5");
|
|
asm volatile ("pxor %xmm5,%xmm4");
|
|
|
|
asm volatile ("movdqa %0,%%xmm5" : : "m" (R[3][0]));
|
|
asm volatile ("pshufb %xmm1,%xmm5");
|
|
asm volatile ("pxor %xmm5,%xmm4");
|
|
|
|
asm volatile ("movdqa %0,%%xmm5" : : "m" (R[3][16]));
|
|
asm volatile ("pshufb %xmm3,%xmm5");
|
|
asm volatile ("pxor %xmm5,%xmm4");
|
|
|
|
/* recovery data must remain cacheable */
|
|
asm volatile ("movdqa %%xmm4,%0" : "=m" (pa[1][i]));
|
|
}
|
|
}
|
|
|
|
raid_sse_end(0);
|
|
}
|
|
|
|
/*
|
|
* Recover failure of two data blocks using P and Q with SSSE3.
|
|
*
|
|
* Computes Pdelta and Qdelta directly in a single survivor scan.
|
|
* Q is computed with Horner's method using the active field generator.
|
|
*
|
|
* Processes two 16-byte lanes per iteration, avoiding temporary parity
|
|
* buffers and the second pass over the data.
|
|
*/
|
|
static __always_inline void raid_rec2of2_ssse3(int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
uint8_t **v = (uint8_t **)vv;
|
|
uint8_t *p;
|
|
uint8_t *q;
|
|
uint8_t *pa;
|
|
uint8_t *qa;
|
|
const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
|
|
uint8_t C[2];
|
|
int generator;
|
|
int l;
|
|
int d;
|
|
size_t i;
|
|
|
|
BUG_ON(ip[0] != 0 || ip[1] != 1);
|
|
|
|
/* get multiplication coefficients */
|
|
C[0] = inv(powgen(id[1] - id[0]) ^ 1);
|
|
C[1] = inv(powgen(id[0]) ^ powgen(id[1]));
|
|
|
|
R[0][0] = &raid_gfmulpshufb[C[0]][0][0];
|
|
R[0][1] = &raid_gfmulpshufb[C[1]][0][0];
|
|
|
|
generator = powgen(1);
|
|
BUG_ON(generator != 2 && generator != 3);
|
|
|
|
l = nd - 1;
|
|
|
|
p = v[nd];
|
|
q = v[nd + 1];
|
|
pa = v[id[0]];
|
|
qa = v[id[1]];
|
|
|
|
raid_sse_begin();
|
|
|
|
/*
|
|
* During the survivor scan:
|
|
*
|
|
* xmm0 Pdelta lane 0
|
|
* xmm1 Pdelta lane 1
|
|
* xmm2 Qa lane 0
|
|
* xmm3 Qa lane 1
|
|
* xmm4-xmm6 temporaries
|
|
* xmm7 active reduction polynomial
|
|
*/
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly));
|
|
|
|
for (i = 0; i < size; i += 32) {
|
|
/* Pdelta starts directly from stored P */
|
|
asm volatile ("movdqa %0,%%xmm0" : : "m" (p[i]));
|
|
asm volatile ("movdqa %0,%%xmm1" : : "m" (p[i + 16]));
|
|
|
|
/* Qa starts at zero for both lanes */
|
|
asm volatile ("pxor %xmm2,%xmm2");
|
|
asm volatile ("pxor %xmm3,%xmm3");
|
|
|
|
/* last disk starts Horner without generator multiplication */
|
|
if (l != id[0] && l != id[1]) {
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));
|
|
asm volatile ("movdqa %0,%%xmm5" : : "m" (v[l][i + 16]));
|
|
asm volatile ("pxor %xmm4,%xmm0");
|
|
asm volatile ("pxor %xmm5,%xmm1");
|
|
asm volatile ("pxor %xmm4,%xmm2");
|
|
asm volatile ("pxor %xmm5,%xmm3");
|
|
}
|
|
|
|
for (d = l - 1; d >= 0; --d) {
|
|
/* multiply both Qa lanes by the active generator */
|
|
if (generator == 3) {
|
|
/* lane 0: Qa = 3 * Qa */
|
|
asm volatile ("movdqa %xmm2,%xmm4");
|
|
asm volatile ("pxor %xmm5,%xmm5");
|
|
asm volatile ("pcmpgtb %xmm2,%xmm5");
|
|
asm volatile ("paddb %xmm2,%xmm2");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
asm volatile ("pxor %xmm5,%xmm2");
|
|
asm volatile ("pxor %xmm4,%xmm2");
|
|
|
|
/* lane 1: Qa = 3 * Qa */
|
|
asm volatile ("movdqa %xmm3,%xmm4");
|
|
asm volatile ("pxor %xmm5,%xmm5");
|
|
asm volatile ("pcmpgtb %xmm3,%xmm5");
|
|
asm volatile ("paddb %xmm3,%xmm3");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
asm volatile ("pxor %xmm5,%xmm3");
|
|
asm volatile ("pxor %xmm4,%xmm3");
|
|
} else {
|
|
/* multiply both lanes by 2 in parallel */
|
|
asm volatile ("pxor %xmm4,%xmm4");
|
|
asm volatile ("pxor %xmm5,%xmm5");
|
|
asm volatile ("pcmpgtb %xmm2,%xmm4");
|
|
asm volatile ("pcmpgtb %xmm3,%xmm5");
|
|
asm volatile ("paddb %xmm2,%xmm2");
|
|
asm volatile ("paddb %xmm3,%xmm3");
|
|
asm volatile ("pand %xmm7,%xmm4");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
asm volatile ("pxor %xmm4,%xmm2");
|
|
asm volatile ("pxor %xmm5,%xmm3");
|
|
}
|
|
|
|
/* missing disks contribute zero */
|
|
if (d == id[0] || d == id[1])
|
|
continue;
|
|
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
|
|
asm volatile ("movdqa %0,%%xmm5" : : "m" (v[d][i + 16]));
|
|
|
|
/* Pdelta */
|
|
asm volatile ("pxor %xmm4,%xmm0");
|
|
asm volatile ("pxor %xmm5,%xmm1");
|
|
|
|
/* Horner contribution to Qa */
|
|
asm volatile ("pxor %xmm4,%xmm2");
|
|
asm volatile ("pxor %xmm5,%xmm3");
|
|
}
|
|
|
|
/* Qdelta = Q ^ Qa */
|
|
asm volatile ("pxor %0,%%xmm2" : : "m" (q[i]));
|
|
asm volatile ("pxor %0,%%xmm3" : : "m" (q[i + 16]));
|
|
|
|
/*
|
|
* The survivor scan is complete, so xmm7 can now hold
|
|
* the low-nibble mask instead of the polynomial.
|
|
*/
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4));
|
|
|
|
/*
|
|
* Lane 0.
|
|
*
|
|
* xmm0 = Pdelta and must remain intact until Dx is computed.
|
|
* xmm2 = Qdelta and may be destroyed during reconstruction.
|
|
*/
|
|
|
|
/* split Pdelta */
|
|
asm volatile ("movdqa %xmm0,%xmm4");
|
|
asm volatile ("movdqa %xmm0,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("pand %xmm7,%xmm4");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
|
|
/* Dy = C0 * Pdelta */
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (R[0][0][0]));
|
|
asm volatile ("pshufb %xmm4,%xmm6");
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (R[0][0][16]));
|
|
asm volatile ("pshufb %xmm5,%xmm4");
|
|
asm volatile ("pxor %xmm4,%xmm6");
|
|
|
|
/* split Qdelta */
|
|
asm volatile ("movdqa %xmm2,%xmm4");
|
|
asm volatile ("movdqa %xmm2,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("pand %xmm7,%xmm4");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
|
|
/* Dy ^= C1 * Qdelta */
|
|
asm volatile ("movdqa %0,%%xmm2" : : "m" (R[0][1][0]));
|
|
asm volatile ("pshufb %xmm4,%xmm2");
|
|
asm volatile ("pxor %xmm2,%xmm6");
|
|
asm volatile ("movdqa %0,%%xmm2" : : "m" (R[0][1][16]));
|
|
asm volatile ("pshufb %xmm5,%xmm2");
|
|
asm volatile ("pxor %xmm2,%xmm6");
|
|
|
|
/* Dx = Pdelta ^ Dy */
|
|
asm volatile ("pxor %xmm6,%xmm0");
|
|
|
|
/* recovery data must remain cacheable */
|
|
asm volatile ("movdqa %%xmm0,%0" : "=m" (pa[i]));
|
|
asm volatile ("movdqa %%xmm6,%0" : "=m" (qa[i]));
|
|
|
|
/*
|
|
* Lane 1.
|
|
*
|
|
* xmm1 = Pdelta and must remain intact until Dx is computed.
|
|
* xmm3 = Qdelta and may be destroyed during reconstruction.
|
|
*/
|
|
|
|
/* split Pdelta */
|
|
asm volatile ("movdqa %xmm1,%xmm4");
|
|
asm volatile ("movdqa %xmm1,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("pand %xmm7,%xmm4");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
|
|
/* Dy = C0 * Pdelta */
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (R[0][0][0]));
|
|
asm volatile ("pshufb %xmm4,%xmm6");
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (R[0][0][16]));
|
|
asm volatile ("pshufb %xmm5,%xmm4");
|
|
asm volatile ("pxor %xmm4,%xmm6");
|
|
|
|
/* split Qdelta */
|
|
asm volatile ("movdqa %xmm3,%xmm4");
|
|
asm volatile ("movdqa %xmm3,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("pand %xmm7,%xmm4");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
|
|
/* Dy ^= C1 * Qdelta */
|
|
asm volatile ("movdqa %0,%%xmm3" : : "m" (R[0][1][0]));
|
|
asm volatile ("pshufb %xmm4,%xmm3");
|
|
asm volatile ("pxor %xmm3,%xmm6");
|
|
asm volatile ("movdqa %0,%%xmm3" : : "m" (R[0][1][16]));
|
|
asm volatile ("pshufb %xmm5,%xmm3");
|
|
asm volatile ("pxor %xmm3,%xmm6");
|
|
|
|
/* Dx = Pdelta ^ Dy */
|
|
asm volatile ("pxor %xmm6,%xmm1");
|
|
|
|
/* recovery data must remain cacheable */
|
|
asm volatile ("movdqa %%xmm1,%0" : "=m" (pa[i + 16]));
|
|
asm volatile ("movdqa %%xmm6,%0" : "=m" (qa[i + 16]));
|
|
|
|
/*
|
|
* Restore the polynomial for the next 32-byte iteration.
|
|
*/
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly));
|
|
}
|
|
|
|
raid_sse_end(0);
|
|
}
|
|
|
|
/*
|
|
* Recover multiple data failures using selected parity blocks with SSSE3 optimized for up to four failures.
|
|
*
|
|
* If P is available, keep the complete P delta syndrome in xmm6.
|
|
* Reconstruct only nr - 1 missing blocks through the inverse matrix and
|
|
* obtain the last block by XORing the reconstructed blocks out of Pdelta.
|
|
*
|
|
* After the survivor scan xmm6 is no longer needed, leaving it available
|
|
* for the P delta accumulator.
|
|
*/
|
|
static __always_inline void raid_recX_ssse3_1234(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
uint8_t **v = (uint8_t **)vv;
|
|
uint8_t *p[RAID_PARITY_MAX];
|
|
uint8_t *pa[RAID_PARITY_MAX];
|
|
uint8_t *src[RAID_DATA_MAX];
|
|
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
|
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
|
const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX];
|
|
const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
|
|
uint8_t buffer_low[RAID_PARITY_MAX * 16 + 16];
|
|
uint8_t buffer_high[RAID_PARITY_MAX * 16 + 16];
|
|
uint8_t *pd_low = __align_ptr(buffer_low, 16);
|
|
uint8_t *pd_high = __align_ptr(buffer_high, 16);
|
|
size_t i;
|
|
int d, j, k, s;
|
|
int ns;
|
|
|
|
BUG_ON(nr < 1 || nr > 4);
|
|
|
|
for (j = 0; j < nr; ++j)
|
|
for (k = 0; k < nr; ++k)
|
|
G[j * nr + k] = A(ip[j], id[k]);
|
|
|
|
raid_invert(G, V, nr);
|
|
|
|
for (j = 0; j < nr; ++j) {
|
|
p[j] = v[nd + ip[j]];
|
|
pa[j] = v[id[j]];
|
|
}
|
|
|
|
ns = 0;
|
|
k = 0;
|
|
|
|
for (d = 0; d < nd; ++d) {
|
|
if (k < nr && d == id[k]) {
|
|
++k;
|
|
continue;
|
|
}
|
|
|
|
src[ns] = v[d];
|
|
|
|
for (j = 0; j < nr; ++j)
|
|
S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0];
|
|
|
|
++ns;
|
|
}
|
|
|
|
BUG_ON(k != nr);
|
|
BUG_ON(ns != nd - nr);
|
|
|
|
for (j = 0; j < nr - has_p; ++j)
|
|
for (k = 0; k < nr; ++k)
|
|
R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
|
|
|
|
raid_sse_begin();
|
|
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4));
|
|
|
|
for (i = 0; i < size; i += 16) {
|
|
asm volatile ("movdqa %0,%%xmm0" : : "m" (p[0][i]));
|
|
|
|
if (nr >= 2)
|
|
asm volatile ("movdqa %0,%%xmm1" : : "m" (p[1][i]));
|
|
|
|
if (nr >= 3)
|
|
asm volatile ("movdqa %0,%%xmm2" : : "m" (p[2][i]));
|
|
|
|
if (nr >= 4)
|
|
asm volatile ("movdqa %0,%%xmm3" : : "m" (p[3][i]));
|
|
|
|
for (s = 0; s < ns; ++s) {
|
|
const uint8_t **t = S[s];
|
|
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (src[s][i]));
|
|
|
|
if (has_p) {
|
|
asm volatile ("pxor %xmm4,%xmm0");
|
|
|
|
asm volatile ("movdqa %xmm4,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("pand %xmm7,%xmm4");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
} else {
|
|
asm volatile ("movdqa %xmm4,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("pand %xmm7,%xmm4");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (t[0][0]));
|
|
asm volatile ("pshufb %xmm4,%xmm6");
|
|
asm volatile ("pxor %xmm6,%xmm0");
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (t[0][16]));
|
|
asm volatile ("pshufb %xmm5,%xmm6");
|
|
asm volatile ("pxor %xmm6,%xmm0");
|
|
}
|
|
|
|
if (nr >= 2) {
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (t[1][0]));
|
|
asm volatile ("pshufb %xmm4,%xmm6");
|
|
asm volatile ("pxor %xmm6,%xmm1");
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (t[1][16]));
|
|
asm volatile ("pshufb %xmm5,%xmm6");
|
|
asm volatile ("pxor %xmm6,%xmm1");
|
|
}
|
|
|
|
if (nr >= 3) {
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (t[2][0]));
|
|
asm volatile ("pshufb %xmm4,%xmm6");
|
|
asm volatile ("pxor %xmm6,%xmm2");
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (t[2][16]));
|
|
asm volatile ("pshufb %xmm5,%xmm6");
|
|
asm volatile ("pxor %xmm6,%xmm2");
|
|
}
|
|
|
|
if (nr >= 4) {
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (t[3][0]));
|
|
asm volatile ("pshufb %xmm4,%xmm6");
|
|
asm volatile ("pxor %xmm6,%xmm3");
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (t[3][16]));
|
|
asm volatile ("pshufb %xmm5,%xmm6");
|
|
asm volatile ("pxor %xmm6,%xmm3");
|
|
}
|
|
}
|
|
|
|
/* preserve the complete P delta before splitting syndrome 0 */
|
|
if (has_p)
|
|
asm volatile ("movdqa %xmm0,%xmm6");
|
|
|
|
/*
|
|
* Split completed syndromes once and store the low/high nibbles.
|
|
* The survivor scan is already complete.
|
|
*/
|
|
|
|
asm volatile ("movdqa %xmm0,%xmm4");
|
|
asm volatile ("movdqa %xmm0,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("pand %xmm7,%xmm4");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
asm volatile ("movdqa %%xmm4,%0" : "=m" (pd_low[0]));
|
|
asm volatile ("movdqa %%xmm5,%0" : "=m" (pd_high[0]));
|
|
|
|
if (nr >= 2) {
|
|
asm volatile ("movdqa %xmm1,%xmm4");
|
|
asm volatile ("movdqa %xmm1,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("pand %xmm7,%xmm4");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
asm volatile ("movdqa %%xmm4,%0" : "=m" (pd_low[16]));
|
|
asm volatile ("movdqa %%xmm5,%0" : "=m" (pd_high[16]));
|
|
}
|
|
|
|
if (nr >= 3) {
|
|
asm volatile ("movdqa %xmm2,%xmm4");
|
|
asm volatile ("movdqa %xmm2,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("pand %xmm7,%xmm4");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
asm volatile ("movdqa %%xmm4,%0" : "=m" (pd_low[32]));
|
|
asm volatile ("movdqa %%xmm5,%0" : "=m" (pd_high[32]));
|
|
}
|
|
|
|
if (nr >= 4) {
|
|
asm volatile ("movdqa %xmm3,%xmm4");
|
|
asm volatile ("movdqa %xmm3,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("pand %xmm7,%xmm4");
|
|
asm volatile ("pand %xmm7,%xmm5");
|
|
asm volatile ("movdqa %%xmm4,%0" : "=m" (pd_low[48]));
|
|
asm volatile ("movdqa %%xmm5,%0" : "=m" (pd_high[48]));
|
|
}
|
|
|
|
/*
|
|
* Reconstruct all but the last missing block when P is
|
|
* available. xmm6 keeps the remaining P delta.
|
|
*/
|
|
for (j = 0; j < nr - has_p; ++j) {
|
|
const uint8_t **t = R[j];
|
|
|
|
asm volatile ("movdqa %0,%%xmm0" : : "m" (t[0][0]));
|
|
asm volatile ("movdqa %0,%%xmm1" : : "m" (t[0][16]));
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (pd_low[0]));
|
|
asm volatile ("movdqa %0,%%xmm5" : : "m" (pd_high[0]));
|
|
asm volatile ("pshufb %xmm4,%xmm0");
|
|
asm volatile ("pshufb %xmm5,%xmm1");
|
|
|
|
for (k = 1; k < nr; ++k) {
|
|
asm volatile ("movdqa %0,%%xmm2" : : "m" (t[k][0]));
|
|
asm volatile ("movdqa %0,%%xmm3" : : "m" (t[k][16]));
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (pd_low[k * 16]));
|
|
asm volatile ("movdqa %0,%%xmm5" : : "m" (pd_high[k * 16]));
|
|
asm volatile ("pshufb %xmm4,%xmm2");
|
|
asm volatile ("pshufb %xmm5,%xmm3");
|
|
asm volatile ("pxor %xmm2,%xmm0");
|
|
asm volatile ("pxor %xmm3,%xmm1");
|
|
}
|
|
|
|
asm volatile ("pxor %xmm1,%xmm0");
|
|
|
|
if (has_p)
|
|
asm volatile ("pxor %xmm0,%xmm6");
|
|
|
|
asm volatile ("movdqa %%xmm0,%0" : "=m" (pa[j][i]));
|
|
}
|
|
|
|
if (has_p)
|
|
asm volatile ("movdqa %%xmm6,%0" : "=m" (pa[nr - 1][i]));
|
|
}
|
|
|
|
raid_sse_end(0);
|
|
}
|
|
|
|
/*
|
|
* Recover multiple data failures using selected parity blocks with SSSE3.
|
|
*
|
|
* If P is available, keep the complete P delta syndrome in xmm6.
|
|
* Reconstruct only nr - 1 missing blocks through the inverse matrix and
|
|
* obtain the last block by XORing the reconstructed blocks out of Pdelta.
|
|
*
|
|
* The completed syndromes are kept in temporary memory, leaving xmm6
|
|
* available for the P delta accumulator during reconstruction.
|
|
*/
|
|
static __always_inline void raid_recX_ssse3(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
uint8_t **v = (uint8_t **)vv;
|
|
uint8_t *p[RAID_PARITY_MAX];
|
|
uint8_t *pa[RAID_PARITY_MAX];
|
|
uint8_t *src[RAID_DATA_MAX];
|
|
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
|
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
|
const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX];
|
|
const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
|
|
uint8_t buffer_low[RAID_PARITY_MAX * 16 + 16];
|
|
uint8_t buffer_high[RAID_PARITY_MAX * 16 + 16];
|
|
uint8_t *pd_low = __align_ptr(buffer_low, 16);
|
|
uint8_t *pd_high = __align_ptr(buffer_high, 16);
|
|
size_t i;
|
|
int d, j, k, s;
|
|
int ns;
|
|
|
|
BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
|
|
|
|
for (j = 0; j < nr; ++j)
|
|
for (k = 0; k < nr; ++k)
|
|
G[j * nr + k] = A(ip[j], id[k]);
|
|
|
|
raid_invert(G, V, nr);
|
|
|
|
for (j = 0; j < nr; ++j) {
|
|
p[j] = v[nd + ip[j]];
|
|
pa[j] = v[id[j]];
|
|
}
|
|
|
|
ns = 0;
|
|
k = 0;
|
|
|
|
for (d = 0; d < nd; ++d) {
|
|
if (k < nr && d == id[k]) {
|
|
++k;
|
|
continue;
|
|
}
|
|
|
|
src[ns] = v[d];
|
|
|
|
for (j = 0; j < nr; ++j)
|
|
S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0];
|
|
|
|
++ns;
|
|
}
|
|
|
|
BUG_ON(k != nr);
|
|
BUG_ON(ns != nd - nr);
|
|
|
|
for (j = 0; j < nr - has_p; ++j)
|
|
for (k = 0; k < nr; ++k)
|
|
R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
|
|
|
|
raid_sse_begin();
|
|
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4));
|
|
|
|
for (i = 0; i < size; i += 16) {
|
|
/* pd_low[] temporarily contains the complete raw syndrome */
|
|
for (j = 0; j < nr; ++j) {
|
|
asm volatile ("movdqa %0,%%xmm0" : : "m" (p[j][i]));
|
|
asm volatile ("movdqa %%xmm0,%0" : "=m" (pd_low[j * 16]));
|
|
}
|
|
|
|
/* single survivor scan */
|
|
for (s = 0; s < ns; ++s) {
|
|
const uint8_t **t = S[s];
|
|
|
|
asm volatile ("movdqa %0,%%xmm0" : : "m" (src[s][i]));
|
|
|
|
/* p must use the original source before nibble splitting */
|
|
if (has_p) {
|
|
asm volatile ("movdqa %xmm0,%xmm4");
|
|
asm volatile ("pxor %0,%%xmm4" : : "m" (pd_low[0]));
|
|
asm volatile ("movdqa %%xmm4,%0" : "=m" (pd_low[0]));
|
|
}
|
|
|
|
asm volatile ("movdqa %xmm0,%xmm1");
|
|
asm volatile ("psrlw $4,%xmm1");
|
|
asm volatile ("pand %xmm7,%xmm0");
|
|
asm volatile ("pand %xmm7,%xmm1");
|
|
|
|
for (j = has_p; j < nr; ++j) {
|
|
asm volatile ("movdqa %0,%%xmm2" : : "m" (t[j][0]));
|
|
asm volatile ("movdqa %0,%%xmm3" : : "m" (t[j][16]));
|
|
asm volatile ("pshufb %xmm0,%xmm2");
|
|
asm volatile ("pshufb %xmm1,%xmm3");
|
|
asm volatile ("pxor %xmm3,%xmm2");
|
|
asm volatile ("pxor %0,%%xmm2" : : "m" (pd_low[j * 16]));
|
|
asm volatile ("movdqa %%xmm2,%0" : "=m" (pd_low[j * 16]));
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Preserve the complete P delta before pd_low[0] is converted
|
|
* to its low-nibble representation.
|
|
*/
|
|
if (has_p)
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (pd_low[0]));
|
|
|
|
/* convert all completed raw syndromes to low/high nibble form */
|
|
for (k = 0; k < nr; ++k) {
|
|
asm volatile ("movdqa %0,%%xmm0" : : "m" (pd_low[k * 16]));
|
|
asm volatile ("movdqa %xmm0,%xmm1");
|
|
asm volatile ("psrlw $4,%xmm1");
|
|
asm volatile ("pand %xmm7,%xmm0");
|
|
asm volatile ("pand %xmm7,%xmm1");
|
|
asm volatile ("movdqa %%xmm0,%0" : "=m" (pd_low[k * 16]));
|
|
asm volatile ("movdqa %%xmm1,%0" : "=m" (pd_high[k * 16]));
|
|
}
|
|
|
|
/*
|
|
* Reconstruct all but the last missing block when P is
|
|
* available. xmm6 keeps the remaining P delta.
|
|
*/
|
|
for (j = 0; j < nr - has_p; ++j) {
|
|
const uint8_t **t = R[j];
|
|
|
|
asm volatile ("movdqa %0,%%xmm0" : : "m" (t[0][0]));
|
|
asm volatile ("movdqa %0,%%xmm1" : : "m" (t[0][16]));
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (pd_low[0]));
|
|
asm volatile ("movdqa %0,%%xmm5" : : "m" (pd_high[0]));
|
|
asm volatile ("pshufb %xmm4,%xmm0");
|
|
asm volatile ("pshufb %xmm5,%xmm1");
|
|
|
|
for (k = 1; k < nr; ++k) {
|
|
asm volatile ("movdqa %0,%%xmm2" : : "m" (t[k][0]));
|
|
asm volatile ("movdqa %0,%%xmm3" : : "m" (t[k][16]));
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (pd_low[k * 16]));
|
|
asm volatile ("movdqa %0,%%xmm5" : : "m" (pd_high[k * 16]));
|
|
asm volatile ("pshufb %xmm4,%xmm2");
|
|
asm volatile ("pshufb %xmm5,%xmm3");
|
|
asm volatile ("pxor %xmm2,%xmm0");
|
|
asm volatile ("pxor %xmm3,%xmm1");
|
|
}
|
|
|
|
asm volatile ("pxor %xmm1,%xmm0");
|
|
|
|
if (has_p)
|
|
asm volatile ("pxor %xmm0,%xmm6");
|
|
|
|
asm volatile ("movdqa %%xmm0,%0" : "=m" (pa[j][i]));
|
|
}
|
|
|
|
if (has_p)
|
|
asm volatile ("movdqa %%xmm6,%0" : "=m" (pa[nr - 1][i]));
|
|
}
|
|
|
|
raid_sse_end(0);
|
|
}
|
|
|
|
#ifdef CONFIG_X86_64
|
|
/*
|
|
* Recover failure of one data block using selected parity with SSSE3 extended,
|
|
* optimized specifically for one failure.
|
|
*
|
|
* Computes only the selected syndrome in a single survivor scan and processes
|
|
* two 16-byte lanes per iteration to expose instruction-level parallelism.
|
|
*/
|
|
static __always_inline void raid_rec1_ssse3ext_1(int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
uint8_t **v = (uint8_t **)vv;
|
|
uint8_t *p;
|
|
uint8_t *pa;
|
|
uint8_t *src[RAID_DATA_MAX];
|
|
const uint8_t *S[RAID_DATA_MAX];
|
|
uint8_t G;
|
|
uint8_t V;
|
|
size_t i;
|
|
int d, s;
|
|
int ns;
|
|
|
|
G = A(ip[0], id[0]);
|
|
V = inv(G);
|
|
|
|
p = v[nd + ip[0]];
|
|
pa = v[id[0]];
|
|
|
|
ns = 0;
|
|
for (d = 0; d < nd; ++d) {
|
|
if (d == id[0])
|
|
continue;
|
|
|
|
src[ns] = v[d];
|
|
S[ns] = &raid_gfmulpshufb[A(ip[0], d)][0][0];
|
|
++ns;
|
|
}
|
|
|
|
BUG_ON(ns != nd - 1);
|
|
|
|
raid_sse_begin();
|
|
|
|
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4));
|
|
|
|
/* keep the inverse coefficient resident */
|
|
asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfmulpshufb[V][0]));
|
|
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfmulpshufb[V][1]));
|
|
|
|
for (i = 0; i < size; i += 32) {
|
|
/* start both lanes from the selected parity block */
|
|
asm volatile ("movdqa %0,%%xmm0" : : "m" (p[i]));
|
|
asm volatile ("movdqa %0,%%xmm1" : : "m" (p[i + 16]));
|
|
|
|
/* compute only the selected syndrome */
|
|
for (s = 0; s < ns; ++s) {
|
|
const uint8_t *t = S[s];
|
|
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (src[s][i]));
|
|
asm volatile ("movdqa %0,%%xmm5" : : "m" (src[s][i + 16]));
|
|
|
|
/* split both source lanes into low/high nibbles */
|
|
asm volatile ("movdqa %xmm4,%xmm6");
|
|
asm volatile ("movdqa %xmm5,%xmm7");
|
|
asm volatile ("psrlw $4,%xmm6");
|
|
asm volatile ("psrlw $4,%xmm7");
|
|
asm volatile ("pand %xmm15,%xmm4");
|
|
asm volatile ("pand %xmm15,%xmm5");
|
|
asm volatile ("pand %xmm15,%xmm6");
|
|
asm volatile ("pand %xmm15,%xmm7");
|
|
|
|
/* low-nibble products, both lanes */
|
|
asm volatile ("movdqa %0,%%xmm8" : : "m" (t[0]));
|
|
asm volatile ("movdqa %xmm8,%xmm9");
|
|
asm volatile ("pshufb %xmm4,%xmm8");
|
|
asm volatile ("pshufb %xmm5,%xmm9");
|
|
asm volatile ("pxor %xmm8,%xmm0");
|
|
asm volatile ("pxor %xmm9,%xmm1");
|
|
|
|
/* high-nibble products, both lanes */
|
|
asm volatile ("movdqa %0,%%xmm10" : : "m" (t[16]));
|
|
asm volatile ("movdqa %xmm10,%xmm11");
|
|
asm volatile ("pshufb %xmm6,%xmm10");
|
|
asm volatile ("pshufb %xmm7,%xmm11");
|
|
asm volatile ("pxor %xmm10,%xmm0");
|
|
asm volatile ("pxor %xmm11,%xmm1");
|
|
}
|
|
|
|
/* split both completed syndromes */
|
|
asm volatile ("movdqa %xmm0,%xmm4");
|
|
asm volatile ("movdqa %xmm1,%xmm5");
|
|
asm volatile ("movdqa %xmm0,%xmm6");
|
|
asm volatile ("movdqa %xmm1,%xmm7");
|
|
asm volatile ("psrlw $4,%xmm6");
|
|
asm volatile ("psrlw $4,%xmm7");
|
|
asm volatile ("pand %xmm15,%xmm4");
|
|
asm volatile ("pand %xmm15,%xmm5");
|
|
asm volatile ("pand %xmm15,%xmm6");
|
|
asm volatile ("pand %xmm15,%xmm7");
|
|
|
|
/* multiply both lanes by the inverse coefficient */
|
|
asm volatile ("movdqa %xmm12,%xmm8");
|
|
asm volatile ("movdqa %xmm12,%xmm9");
|
|
asm volatile ("pshufb %xmm4,%xmm8");
|
|
asm volatile ("pshufb %xmm5,%xmm9");
|
|
|
|
asm volatile ("movdqa %xmm13,%xmm10");
|
|
asm volatile ("movdqa %xmm13,%xmm11");
|
|
asm volatile ("pshufb %xmm6,%xmm10");
|
|
asm volatile ("pshufb %xmm7,%xmm11");
|
|
asm volatile ("pxor %xmm10,%xmm8");
|
|
asm volatile ("pxor %xmm11,%xmm9");
|
|
|
|
/* recovery data must remain cacheable */
|
|
asm volatile ("movdqa %%xmm8,%0" : "=m" (pa[i]));
|
|
asm volatile ("movdqa %%xmm9,%0" : "=m" (pa[i + 16]));
|
|
}
|
|
|
|
raid_sse_end(0);
|
|
}
|
|
|
|
/*
|
|
* Recover failure of one data block using Q with SSSE3 extended.
|
|
*
|
|
* Computes Q of all surviving data directly with Horner's method and
|
|
* reconstructs the missing block from Qdelta.
|
|
*
|
|
* Processes four 16-byte lanes per iteration, avoiding temporary parity
|
|
* buffers and the extra pass over the data.
|
|
*/
|
|
static __always_inline void raid_rec1_ssse3ext_q(int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
uint8_t **v = (uint8_t **)vv;
|
|
uint8_t *q;
|
|
uint8_t *pa;
|
|
uint8_t V;
|
|
int generator;
|
|
int l;
|
|
int d;
|
|
size_t i;
|
|
|
|
BUG_ON(ip[0] != 1);
|
|
|
|
V = inv(A(1, id[0]));
|
|
|
|
generator = powgen(1);
|
|
BUG_ON(generator != 2 && generator != 3);
|
|
|
|
l = nd - 1;
|
|
|
|
q = v[nd + 1];
|
|
pa = v[id[0]];
|
|
|
|
raid_sse_begin();
|
|
|
|
/*
|
|
* xmm0-xmm3 Q accumulators
|
|
* xmm4-xmm11 temporaries
|
|
* xmm12 inverse coefficient low table
|
|
* xmm13 inverse coefficient high table
|
|
* xmm14 active reduction polynomial
|
|
* xmm15 low-nibble mask
|
|
*/
|
|
asm volatile ("movdqa %0,%%xmm12" : : "m" (raid_gfmulpshufb[V][0]));
|
|
asm volatile ("movdqa %0,%%xmm13" : : "m" (raid_gfmulpshufb[V][1]));
|
|
asm volatile ("movdqa %0,%%xmm14" : : "m" (gfconst16.poly));
|
|
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4));
|
|
|
|
for (i = 0; i < size; i += 64) {
|
|
/* last disk starts Horner without generator multiplication */
|
|
if (l == id[0]) {
|
|
asm volatile ("pxor %xmm0,%xmm0");
|
|
asm volatile ("pxor %xmm1,%xmm1");
|
|
asm volatile ("pxor %xmm2,%xmm2");
|
|
asm volatile ("pxor %xmm3,%xmm3");
|
|
} else {
|
|
asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i]));
|
|
asm volatile ("movdqa %0,%%xmm1" : : "m" (v[l][i + 16]));
|
|
asm volatile ("movdqa %0,%%xmm2" : : "m" (v[l][i + 32]));
|
|
asm volatile ("movdqa %0,%%xmm3" : : "m" (v[l][i + 48]));
|
|
}
|
|
|
|
for (d = l - 1; d >= 0; --d) {
|
|
/* multiply all four Q lanes by the active generator */
|
|
if (generator == 3) {
|
|
asm volatile ("movdqa %xmm0,%xmm8");
|
|
asm volatile ("movdqa %xmm1,%xmm9");
|
|
asm volatile ("movdqa %xmm2,%xmm10");
|
|
asm volatile ("movdqa %xmm3,%xmm11");
|
|
|
|
asm volatile ("pxor %xmm4,%xmm4");
|
|
asm volatile ("pxor %xmm5,%xmm5");
|
|
asm volatile ("pxor %xmm6,%xmm6");
|
|
asm volatile ("pxor %xmm7,%xmm7");
|
|
|
|
asm volatile ("pcmpgtb %xmm0,%xmm4");
|
|
asm volatile ("pcmpgtb %xmm1,%xmm5");
|
|
asm volatile ("pcmpgtb %xmm2,%xmm6");
|
|
asm volatile ("pcmpgtb %xmm3,%xmm7");
|
|
|
|
asm volatile ("paddb %xmm0,%xmm0");
|
|
asm volatile ("paddb %xmm1,%xmm1");
|
|
asm volatile ("paddb %xmm2,%xmm2");
|
|
asm volatile ("paddb %xmm3,%xmm3");
|
|
|
|
asm volatile ("pand %xmm14,%xmm4");
|
|
asm volatile ("pand %xmm14,%xmm5");
|
|
asm volatile ("pand %xmm14,%xmm6");
|
|
asm volatile ("pand %xmm14,%xmm7");
|
|
|
|
asm volatile ("pxor %xmm4,%xmm0");
|
|
asm volatile ("pxor %xmm5,%xmm1");
|
|
asm volatile ("pxor %xmm6,%xmm2");
|
|
asm volatile ("pxor %xmm7,%xmm3");
|
|
|
|
asm volatile ("pxor %xmm8,%xmm0");
|
|
asm volatile ("pxor %xmm9,%xmm1");
|
|
asm volatile ("pxor %xmm10,%xmm2");
|
|
asm volatile ("pxor %xmm11,%xmm3");
|
|
} else {
|
|
asm volatile ("pxor %xmm4,%xmm4");
|
|
asm volatile ("pxor %xmm5,%xmm5");
|
|
asm volatile ("pxor %xmm6,%xmm6");
|
|
asm volatile ("pxor %xmm7,%xmm7");
|
|
|
|
asm volatile ("pcmpgtb %xmm0,%xmm4");
|
|
asm volatile ("pcmpgtb %xmm1,%xmm5");
|
|
asm volatile ("pcmpgtb %xmm2,%xmm6");
|
|
asm volatile ("pcmpgtb %xmm3,%xmm7");
|
|
|
|
asm volatile ("paddb %xmm0,%xmm0");
|
|
asm volatile ("paddb %xmm1,%xmm1");
|
|
asm volatile ("paddb %xmm2,%xmm2");
|
|
asm volatile ("paddb %xmm3,%xmm3");
|
|
|
|
asm volatile ("pand %xmm14,%xmm4");
|
|
asm volatile ("pand %xmm14,%xmm5");
|
|
asm volatile ("pand %xmm14,%xmm6");
|
|
asm volatile ("pand %xmm14,%xmm7");
|
|
|
|
asm volatile ("pxor %xmm4,%xmm0");
|
|
asm volatile ("pxor %xmm5,%xmm1");
|
|
asm volatile ("pxor %xmm6,%xmm2");
|
|
asm volatile ("pxor %xmm7,%xmm3");
|
|
}
|
|
|
|
/* missing disk contributes zero */
|
|
if (d == id[0])
|
|
continue;
|
|
|
|
asm volatile ("pxor %0,%%xmm0" : : "m" (v[d][i]));
|
|
asm volatile ("pxor %0,%%xmm1" : : "m" (v[d][i + 16]));
|
|
asm volatile ("pxor %0,%%xmm2" : : "m" (v[d][i + 32]));
|
|
asm volatile ("pxor %0,%%xmm3" : : "m" (v[d][i + 48]));
|
|
}
|
|
|
|
/* Qdelta = stored Q ^ Q of all surviving data */
|
|
asm volatile ("pxor %0,%%xmm0" : : "m" (q[i]));
|
|
asm volatile ("pxor %0,%%xmm1" : : "m" (q[i + 16]));
|
|
asm volatile ("pxor %0,%%xmm2" : : "m" (q[i + 32]));
|
|
asm volatile ("pxor %0,%%xmm3" : : "m" (q[i + 48]));
|
|
|
|
/* keep low nibbles in xmm4-xmm7 and high nibbles in xmm0-xmm3 */
|
|
asm volatile ("movdqa %xmm0,%xmm4");
|
|
asm volatile ("movdqa %xmm1,%xmm5");
|
|
asm volatile ("movdqa %xmm2,%xmm6");
|
|
asm volatile ("movdqa %xmm3,%xmm7");
|
|
|
|
asm volatile ("psrlw $4,%xmm0");
|
|
asm volatile ("psrlw $4,%xmm1");
|
|
asm volatile ("psrlw $4,%xmm2");
|
|
asm volatile ("psrlw $4,%xmm3");
|
|
|
|
asm volatile ("pand %xmm15,%xmm4");
|
|
asm volatile ("pand %xmm15,%xmm5");
|
|
asm volatile ("pand %xmm15,%xmm6");
|
|
asm volatile ("pand %xmm15,%xmm7");
|
|
|
|
asm volatile ("pand %xmm15,%xmm0");
|
|
asm volatile ("pand %xmm15,%xmm1");
|
|
asm volatile ("pand %xmm15,%xmm2");
|
|
asm volatile ("pand %xmm15,%xmm3");
|
|
|
|
/* low-nibble products */
|
|
asm volatile ("movdqa %xmm12,%xmm8");
|
|
asm volatile ("movdqa %xmm12,%xmm9");
|
|
asm volatile ("movdqa %xmm12,%xmm10");
|
|
asm volatile ("movdqa %xmm12,%xmm11");
|
|
|
|
asm volatile ("pshufb %xmm4,%xmm8");
|
|
asm volatile ("pshufb %xmm5,%xmm9");
|
|
asm volatile ("pshufb %xmm6,%xmm10");
|
|
asm volatile ("pshufb %xmm7,%xmm11");
|
|
|
|
/* high-nibble products */
|
|
asm volatile ("movdqa %xmm13,%xmm4");
|
|
asm volatile ("movdqa %xmm13,%xmm5");
|
|
asm volatile ("movdqa %xmm13,%xmm6");
|
|
asm volatile ("movdqa %xmm13,%xmm7");
|
|
|
|
asm volatile ("pshufb %xmm0,%xmm4");
|
|
asm volatile ("pshufb %xmm1,%xmm5");
|
|
asm volatile ("pshufb %xmm2,%xmm6");
|
|
asm volatile ("pshufb %xmm3,%xmm7");
|
|
|
|
asm volatile ("pxor %xmm4,%xmm8");
|
|
asm volatile ("pxor %xmm5,%xmm9");
|
|
asm volatile ("pxor %xmm6,%xmm10");
|
|
asm volatile ("pxor %xmm7,%xmm11");
|
|
|
|
/* recovery data must remain cacheable */
|
|
asm volatile ("movdqa %%xmm8,%0" : "=m" (pa[i]));
|
|
asm volatile ("movdqa %%xmm9,%0" : "=m" (pa[i + 16]));
|
|
asm volatile ("movdqa %%xmm10,%0" : "=m" (pa[i + 32]));
|
|
asm volatile ("movdqa %%xmm11,%0" : "=m" (pa[i + 48]));
|
|
}
|
|
|
|
raid_sse_end(0);
|
|
}
|
|
|
|
/*
|
|
* Recover failure of two data blocks using selected parity with SSSE3 extended,
|
|
* optimized specifically for two failures.
|
|
*
|
|
* Computes only the two selected syndromes in a single survivor scan and
|
|
* processes two 16-byte lanes per iteration.
|
|
*
|
|
* Keeps the first inverse-matrix row resident for the complete recovery loop.
|
|
* When P is available this is the only inverse row required, because the
|
|
* second missing block is obtained directly from Pdelta. Without P, the
|
|
* second inverse row is loaded only during reconstruction and each table
|
|
* load is shared between both lanes.
|
|
*
|
|
* has_p is expected to be a compile-time constant after inlining.
|
|
*
|
|
* Note that it uses 16 registers, meaning that x64 is required.
|
|
*/
|
|
static __always_inline void raid_rec2_ssse3ext_2(int has_p, int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
uint8_t **v = (uint8_t **)vv;
|
|
uint8_t *p[2];
|
|
uint8_t *pa[2];
|
|
uint8_t *src[RAID_DATA_MAX];
|
|
const uint8_t *S0[RAID_DATA_MAX];
|
|
const uint8_t *S1[RAID_DATA_MAX];
|
|
uint8_t G[2 * 2];
|
|
uint8_t V[2 * 2];
|
|
size_t i;
|
|
int d, s;
|
|
int ns;
|
|
|
|
/* setup and invert the 2x2 coefficients matrix */
|
|
G[0] = A(ip[0], id[0]);
|
|
G[1] = A(ip[0], id[1]);
|
|
G[2] = A(ip[1], id[0]);
|
|
G[3] = A(ip[1], id[1]);
|
|
raid_invert(G, V, 2);
|
|
|
|
p[0] = v[nd + ip[0]];
|
|
p[1] = v[nd + ip[1]];
|
|
pa[0] = v[id[0]];
|
|
pa[1] = v[id[1]];
|
|
|
|
/* collect surviving data blocks and selected coefficient tables */
|
|
ns = 0;
|
|
for (d = 0; d < nd; ++d) {
|
|
if (d == id[0] || d == id[1])
|
|
continue;
|
|
|
|
src[ns] = v[d];
|
|
|
|
if (!has_p)
|
|
S0[ns] = &raid_gfmulpshufb[A(ip[0], d)][0][0];
|
|
|
|
S1[ns] = &raid_gfmulpshufb[A(ip[1], d)][0][0];
|
|
|
|
++ns;
|
|
}
|
|
|
|
BUG_ON(ns != nd - 2);
|
|
|
|
raid_sse_begin();
|
|
|
|
/*
|
|
* Register allocation during the survivor scan:
|
|
*
|
|
* xmm0/xmm1 syndrome 0, lanes 0/1
|
|
* xmm2/xmm3 syndrome 1, lanes 0/1
|
|
* xmm4/xmm5 source low nibbles
|
|
* xmm6/xmm7 source high nibbles
|
|
* xmm8-xmm11 resident first inverse-matrix row
|
|
* xmm12/xmm13 multiplication table/results for both lanes
|
|
* xmm14 low-nibble mask
|
|
* xmm15 temporary
|
|
*/
|
|
|
|
/* keep the first inverse-matrix row resident */
|
|
asm volatile ("movdqa %0,%%xmm8" : : "m" (raid_gfmulpshufb[V[0]][0]));
|
|
asm volatile ("movdqa %0,%%xmm9" : : "m" (raid_gfmulpshufb[V[0]][1]));
|
|
asm volatile ("movdqa %0,%%xmm10" : : "m" (raid_gfmulpshufb[V[1]][0]));
|
|
asm volatile ("movdqa %0,%%xmm11" : : "m" (raid_gfmulpshufb[V[1]][1]));
|
|
|
|
for (i = 0; i < size; i += 32) {
|
|
/* xmm14 is reused during reconstruction, therefore restore the mask for each iteration */
|
|
asm volatile ("movdqa %0,%%xmm14" : : "m" (gfconst16.low4));
|
|
|
|
/* start both syndrome lanes from the selected stored parity blocks */
|
|
asm volatile ("movdqa %0,%%xmm0" : : "m" (p[0][i]));
|
|
asm volatile ("movdqa %0,%%xmm1" : : "m" (p[0][i + 16]));
|
|
asm volatile ("movdqa %0,%%xmm2" : : "m" (p[1][i]));
|
|
asm volatile ("movdqa %0,%%xmm3" : : "m" (p[1][i + 16]));
|
|
|
|
/* compute both selected syndromes in a single survivor scan */
|
|
for (s = 0; s < ns; ++s) {
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (src[s][i]));
|
|
asm volatile ("movdqa %0,%%xmm5" : : "m" (src[s][i + 16]));
|
|
|
|
/* P has coefficient 1 and must use the original source before nibble splitting */
|
|
if (has_p) {
|
|
asm volatile ("pxor %xmm4,%xmm0");
|
|
asm volatile ("pxor %xmm5,%xmm1");
|
|
}
|
|
|
|
/* split both source lanes into low/high nibbles */
|
|
asm volatile ("movdqa %xmm4,%xmm6");
|
|
asm volatile ("movdqa %xmm5,%xmm7");
|
|
asm volatile ("psrlw $4,%xmm6");
|
|
asm volatile ("psrlw $4,%xmm7");
|
|
asm volatile ("pand %xmm14,%xmm4");
|
|
asm volatile ("pand %xmm14,%xmm5");
|
|
asm volatile ("pand %xmm14,%xmm6");
|
|
asm volatile ("pand %xmm14,%xmm7");
|
|
|
|
/* syndrome 0 only needs multiplication when it is not P */
|
|
if (!has_p) {
|
|
const uint8_t *t = S0[s];
|
|
|
|
asm volatile ("movdqa %0,%%xmm12" : : "m" (t[0]));
|
|
asm volatile ("movdqa %xmm12,%xmm13");
|
|
asm volatile ("pshufb %xmm4,%xmm12");
|
|
asm volatile ("pshufb %xmm5,%xmm13");
|
|
asm volatile ("pxor %xmm12,%xmm0");
|
|
asm volatile ("pxor %xmm13,%xmm1");
|
|
|
|
asm volatile ("movdqa %0,%%xmm12" : : "m" (t[16]));
|
|
asm volatile ("movdqa %xmm12,%xmm13");
|
|
asm volatile ("pshufb %xmm6,%xmm12");
|
|
asm volatile ("pshufb %xmm7,%xmm13");
|
|
asm volatile ("pxor %xmm12,%xmm0");
|
|
asm volatile ("pxor %xmm13,%xmm1");
|
|
}
|
|
|
|
/* syndrome 1 */
|
|
{
|
|
const uint8_t *t = S1[s];
|
|
|
|
asm volatile ("movdqa %0,%%xmm12" : : "m" (t[0]));
|
|
asm volatile ("movdqa %xmm12,%xmm13");
|
|
asm volatile ("pshufb %xmm4,%xmm12");
|
|
asm volatile ("pshufb %xmm5,%xmm13");
|
|
asm volatile ("pxor %xmm12,%xmm2");
|
|
asm volatile ("pxor %xmm13,%xmm3");
|
|
|
|
asm volatile ("movdqa %0,%%xmm12" : : "m" (t[16]));
|
|
asm volatile ("movdqa %xmm12,%xmm13");
|
|
asm volatile ("pshufb %xmm6,%xmm12");
|
|
asm volatile ("pshufb %xmm7,%xmm13");
|
|
asm volatile ("pxor %xmm12,%xmm2");
|
|
asm volatile ("pxor %xmm13,%xmm3");
|
|
}
|
|
}
|
|
|
|
/* preserve the complete Pdelta for both lanes */
|
|
if (has_p) {
|
|
asm volatile ("movdqa %xmm0,%xmm6");
|
|
asm volatile ("movdqa %xmm1,%xmm7");
|
|
}
|
|
|
|
/*
|
|
* Split the completed syndromes:
|
|
*
|
|
* xmm0/xmm1 syndrome 0 low, lanes 0/1
|
|
* xmm4/xmm5 syndrome 0 high, lanes 0/1
|
|
* xmm2/xmm3 syndrome 1 low, lanes 0/1
|
|
* xmm12/xmm13 syndrome 1 high, lanes 0/1
|
|
*/
|
|
asm volatile ("movdqa %xmm0,%xmm4");
|
|
asm volatile ("movdqa %xmm1,%xmm5");
|
|
asm volatile ("movdqa %xmm2,%xmm12");
|
|
asm volatile ("movdqa %xmm3,%xmm13");
|
|
asm volatile ("psrlw $4,%xmm4");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm12");
|
|
asm volatile ("psrlw $4,%xmm13");
|
|
asm volatile ("pand %xmm14,%xmm0");
|
|
asm volatile ("pand %xmm14,%xmm1");
|
|
asm volatile ("pand %xmm14,%xmm2");
|
|
asm volatile ("pand %xmm14,%xmm3");
|
|
asm volatile ("pand %xmm14,%xmm4");
|
|
asm volatile ("pand %xmm14,%xmm5");
|
|
asm volatile ("pand %xmm14,%xmm12");
|
|
asm volatile ("pand %xmm14,%xmm13");
|
|
|
|
if (has_p) {
|
|
/*
|
|
* Reconstruct pa[0] for both lanes.
|
|
*
|
|
* The syndrome nibbles can be destroyed because the second
|
|
* missing block is obtained directly from Pdelta.
|
|
*/
|
|
|
|
/* V[0] * syndrome0 low */
|
|
asm volatile ("movdqa %xmm8,%xmm14");
|
|
asm volatile ("movdqa %xmm8,%xmm15");
|
|
asm volatile ("pshufb %xmm0,%xmm14");
|
|
asm volatile ("pshufb %xmm1,%xmm15");
|
|
asm volatile ("movdqa %xmm14,%xmm0");
|
|
asm volatile ("movdqa %xmm15,%xmm1");
|
|
|
|
/* V[0] * syndrome0 high */
|
|
asm volatile ("movdqa %xmm9,%xmm14");
|
|
asm volatile ("movdqa %xmm9,%xmm15");
|
|
asm volatile ("pshufb %xmm4,%xmm14");
|
|
asm volatile ("pshufb %xmm5,%xmm15");
|
|
asm volatile ("pxor %xmm14,%xmm0");
|
|
asm volatile ("pxor %xmm15,%xmm1");
|
|
|
|
/* V[1] * syndrome1 low */
|
|
asm volatile ("movdqa %xmm10,%xmm14");
|
|
asm volatile ("movdqa %xmm10,%xmm15");
|
|
asm volatile ("pshufb %xmm2,%xmm14");
|
|
asm volatile ("pshufb %xmm3,%xmm15");
|
|
asm volatile ("pxor %xmm14,%xmm0");
|
|
asm volatile ("pxor %xmm15,%xmm1");
|
|
|
|
/* V[1] * syndrome1 high */
|
|
asm volatile ("movdqa %xmm11,%xmm14");
|
|
asm volatile ("movdqa %xmm11,%xmm15");
|
|
asm volatile ("pshufb %xmm12,%xmm14");
|
|
asm volatile ("pshufb %xmm13,%xmm15");
|
|
asm volatile ("pxor %xmm14,%xmm0");
|
|
asm volatile ("pxor %xmm15,%xmm1");
|
|
|
|
/* pa[1] = Pdelta ^ pa[0] */
|
|
asm volatile ("pxor %xmm0,%xmm6");
|
|
asm volatile ("pxor %xmm1,%xmm7");
|
|
|
|
/* recovery data must remain cacheable */
|
|
asm volatile ("movdqa %%xmm0,%0" : "=m" (pa[0][i]));
|
|
asm volatile ("movdqa %%xmm1,%0" : "=m" (pa[0][i + 16]));
|
|
asm volatile ("movdqa %%xmm6,%0" : "=m" (pa[1][i]));
|
|
asm volatile ("movdqa %%xmm7,%0" : "=m" (pa[1][i + 16]));
|
|
} else {
|
|
/*
|
|
* Reconstruct pa[0] using the resident first inverse row.
|
|
* Keep all syndrome nibbles intact because they are needed
|
|
* again for pa[1].
|
|
*/
|
|
|
|
asm volatile ("movdqa %xmm8,%xmm6");
|
|
asm volatile ("movdqa %xmm8,%xmm7");
|
|
asm volatile ("pshufb %xmm0,%xmm6");
|
|
asm volatile ("pshufb %xmm1,%xmm7");
|
|
|
|
asm volatile ("movdqa %xmm9,%xmm14");
|
|
asm volatile ("movdqa %xmm9,%xmm15");
|
|
asm volatile ("pshufb %xmm4,%xmm14");
|
|
asm volatile ("pshufb %xmm5,%xmm15");
|
|
asm volatile ("pxor %xmm14,%xmm6");
|
|
asm volatile ("pxor %xmm15,%xmm7");
|
|
|
|
asm volatile ("movdqa %xmm10,%xmm14");
|
|
asm volatile ("movdqa %xmm10,%xmm15");
|
|
asm volatile ("pshufb %xmm2,%xmm14");
|
|
asm volatile ("pshufb %xmm3,%xmm15");
|
|
asm volatile ("pxor %xmm14,%xmm6");
|
|
asm volatile ("pxor %xmm15,%xmm7");
|
|
|
|
asm volatile ("movdqa %xmm11,%xmm14");
|
|
asm volatile ("movdqa %xmm11,%xmm15");
|
|
asm volatile ("pshufb %xmm12,%xmm14");
|
|
asm volatile ("pshufb %xmm13,%xmm15");
|
|
asm volatile ("pxor %xmm14,%xmm6");
|
|
asm volatile ("pxor %xmm15,%xmm7");
|
|
|
|
/* recovery data must remain cacheable */
|
|
asm volatile ("movdqa %%xmm6,%0" : "=m" (pa[0][i]));
|
|
asm volatile ("movdqa %%xmm7,%0" : "=m" (pa[0][i + 16]));
|
|
|
|
/*
|
|
* Reconstruct pa[1].
|
|
*
|
|
* The second inverse row is not kept resident. Each table is
|
|
* loaded once for the 32-byte iteration and shared by both lanes.
|
|
*/
|
|
|
|
/* V[2] * syndrome0 low */
|
|
asm volatile ("movdqa %0,%%xmm14" : : "m" (raid_gfmulpshufb[V[2]][0]));
|
|
asm volatile ("movdqa %xmm14,%xmm15");
|
|
asm volatile ("pshufb %xmm0,%xmm14");
|
|
asm volatile ("pshufb %xmm1,%xmm15");
|
|
asm volatile ("movdqa %xmm14,%xmm6");
|
|
asm volatile ("movdqa %xmm15,%xmm7");
|
|
|
|
/* V[2] * syndrome0 high */
|
|
asm volatile ("movdqa %0,%%xmm14" : : "m" (raid_gfmulpshufb[V[2]][1]));
|
|
asm volatile ("movdqa %xmm14,%xmm15");
|
|
asm volatile ("pshufb %xmm4,%xmm14");
|
|
asm volatile ("pshufb %xmm5,%xmm15");
|
|
asm volatile ("pxor %xmm14,%xmm6");
|
|
asm volatile ("pxor %xmm15,%xmm7");
|
|
|
|
/* V[3] * syndrome1 low */
|
|
asm volatile ("movdqa %0,%%xmm14" : : "m" (raid_gfmulpshufb[V[3]][0]));
|
|
asm volatile ("movdqa %xmm14,%xmm15");
|
|
asm volatile ("pshufb %xmm2,%xmm14");
|
|
asm volatile ("pshufb %xmm3,%xmm15");
|
|
asm volatile ("pxor %xmm14,%xmm6");
|
|
asm volatile ("pxor %xmm15,%xmm7");
|
|
|
|
/* V[3] * syndrome1 high */
|
|
asm volatile ("movdqa %0,%%xmm14" : : "m" (raid_gfmulpshufb[V[3]][1]));
|
|
asm volatile ("movdqa %xmm14,%xmm15");
|
|
asm volatile ("pshufb %xmm12,%xmm14");
|
|
asm volatile ("pshufb %xmm13,%xmm15");
|
|
asm volatile ("pxor %xmm14,%xmm6");
|
|
asm volatile ("pxor %xmm15,%xmm7");
|
|
|
|
/* recovery data must remain cacheable */
|
|
asm volatile ("movdqa %%xmm6,%0" : "=m" (pa[1][i]));
|
|
asm volatile ("movdqa %%xmm7,%0" : "=m" (pa[1][i + 16]));
|
|
}
|
|
}
|
|
|
|
raid_sse_end(0);
|
|
}
|
|
|
|
/*
|
|
* Recover failure of two data blocks using P and Q with SSSE3 extended.
|
|
*
|
|
* Computes Pdelta and Qdelta directly in a single survivor scan.
|
|
* Q is computed with Horner's method using the active field generator.
|
|
*
|
|
* Processes four 16-byte lanes per iteration and keeps all four
|
|
* multiplication tables used by the analytical solution resident.
|
|
*/
|
|
static __always_inline void raid_rec2of2_ssse3ext(int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
uint8_t **v = (uint8_t **)vv;
|
|
uint8_t *p;
|
|
uint8_t *q;
|
|
uint8_t *pa;
|
|
uint8_t *qa;
|
|
const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
|
|
uint8_t C[2];
|
|
int generator;
|
|
int l;
|
|
int d;
|
|
size_t i;
|
|
|
|
BUG_ON(ip[0] != 0 || ip[1] != 1);
|
|
|
|
/* get multiplication coefficients */
|
|
C[0] = inv(powgen(id[1] - id[0]) ^ 1);
|
|
C[1] = inv(powgen(id[0]) ^ powgen(id[1]));
|
|
|
|
R[0][0] = &raid_gfmulpshufb[C[0]][0][0];
|
|
R[0][1] = &raid_gfmulpshufb[C[1]][0][0];
|
|
|
|
generator = powgen(1);
|
|
BUG_ON(generator != 2 && generator != 3);
|
|
|
|
l = nd - 1;
|
|
|
|
p = v[nd];
|
|
q = v[nd + 1];
|
|
pa = v[id[0]];
|
|
qa = v[id[1]];
|
|
|
|
raid_sse_begin();
|
|
|
|
/*
|
|
* During the survivor scan:
|
|
*
|
|
* xmm0-xmm3 Pdelta lanes
|
|
* xmm4-xmm7 Qa lanes
|
|
* xmm8-xmm10 temporaries
|
|
* xmm11 active reduction polynomial
|
|
* xmm12-xmm15 analytical multiplication tables
|
|
*/
|
|
asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.poly));
|
|
|
|
/* keep the analytical multiplication tables resident */
|
|
asm volatile ("movdqa %0,%%xmm12" : : "m" (R[0][0][0]));
|
|
asm volatile ("movdqa %0,%%xmm13" : : "m" (R[0][0][16]));
|
|
asm volatile ("movdqa %0,%%xmm14" : : "m" (R[0][1][0]));
|
|
asm volatile ("movdqa %0,%%xmm15" : : "m" (R[0][1][16]));
|
|
|
|
for (i = 0; i < size; i += 64) {
|
|
/* Pdelta starts directly from stored P */
|
|
asm volatile ("movdqa %0,%%xmm0" : : "m" (p[i]));
|
|
asm volatile ("movdqa %0,%%xmm1" : : "m" (p[i + 16]));
|
|
asm volatile ("movdqa %0,%%xmm2" : : "m" (p[i + 32]));
|
|
asm volatile ("movdqa %0,%%xmm3" : : "m" (p[i + 48]));
|
|
|
|
/* Qa starts at zero for all four lanes */
|
|
asm volatile ("pxor %xmm4,%xmm4");
|
|
asm volatile ("pxor %xmm5,%xmm5");
|
|
asm volatile ("pxor %xmm6,%xmm6");
|
|
asm volatile ("pxor %xmm7,%xmm7");
|
|
|
|
/* last disk starts Horner without generator multiplication */
|
|
if (l != id[0] && l != id[1]) {
|
|
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[l][i]));
|
|
asm volatile ("pxor %xmm8,%xmm0");
|
|
asm volatile ("pxor %xmm8,%xmm4");
|
|
|
|
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[l][i + 16]));
|
|
asm volatile ("pxor %xmm8,%xmm1");
|
|
asm volatile ("pxor %xmm8,%xmm5");
|
|
|
|
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[l][i + 32]));
|
|
asm volatile ("pxor %xmm8,%xmm2");
|
|
asm volatile ("pxor %xmm8,%xmm6");
|
|
|
|
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[l][i + 48]));
|
|
asm volatile ("pxor %xmm8,%xmm3");
|
|
asm volatile ("pxor %xmm8,%xmm7");
|
|
}
|
|
|
|
for (d = l - 1; d >= 0; --d) {
|
|
/* multiply all four Qa lanes by the active generator */
|
|
if (generator == 3) {
|
|
/* lane 0 */
|
|
asm volatile ("movdqa %xmm4,%xmm8");
|
|
asm volatile ("pxor %xmm9,%xmm9");
|
|
asm volatile ("pcmpgtb %xmm4,%xmm9");
|
|
asm volatile ("paddb %xmm4,%xmm4");
|
|
asm volatile ("pand %xmm11,%xmm9");
|
|
asm volatile ("pxor %xmm9,%xmm4");
|
|
asm volatile ("pxor %xmm8,%xmm4");
|
|
|
|
/* lane 1 */
|
|
asm volatile ("movdqa %xmm5,%xmm8");
|
|
asm volatile ("pxor %xmm9,%xmm9");
|
|
asm volatile ("pcmpgtb %xmm5,%xmm9");
|
|
asm volatile ("paddb %xmm5,%xmm5");
|
|
asm volatile ("pand %xmm11,%xmm9");
|
|
asm volatile ("pxor %xmm9,%xmm5");
|
|
asm volatile ("pxor %xmm8,%xmm5");
|
|
|
|
/* lane 2 */
|
|
asm volatile ("movdqa %xmm6,%xmm8");
|
|
asm volatile ("pxor %xmm9,%xmm9");
|
|
asm volatile ("pcmpgtb %xmm6,%xmm9");
|
|
asm volatile ("paddb %xmm6,%xmm6");
|
|
asm volatile ("pand %xmm11,%xmm9");
|
|
asm volatile ("pxor %xmm9,%xmm6");
|
|
asm volatile ("pxor %xmm8,%xmm6");
|
|
|
|
/* lane 3 */
|
|
asm volatile ("movdqa %xmm7,%xmm8");
|
|
asm volatile ("pxor %xmm9,%xmm9");
|
|
asm volatile ("pcmpgtb %xmm7,%xmm9");
|
|
asm volatile ("paddb %xmm7,%xmm7");
|
|
asm volatile ("pand %xmm11,%xmm9");
|
|
asm volatile ("pxor %xmm9,%xmm7");
|
|
asm volatile ("pxor %xmm8,%xmm7");
|
|
} else {
|
|
/* lanes 0 and 1 */
|
|
asm volatile ("pxor %xmm8,%xmm8");
|
|
asm volatile ("pxor %xmm9,%xmm9");
|
|
asm volatile ("pcmpgtb %xmm4,%xmm8");
|
|
asm volatile ("pcmpgtb %xmm5,%xmm9");
|
|
asm volatile ("paddb %xmm4,%xmm4");
|
|
asm volatile ("paddb %xmm5,%xmm5");
|
|
asm volatile ("pand %xmm11,%xmm8");
|
|
asm volatile ("pand %xmm11,%xmm9");
|
|
asm volatile ("pxor %xmm8,%xmm4");
|
|
asm volatile ("pxor %xmm9,%xmm5");
|
|
|
|
/* lanes 2 and 3 */
|
|
asm volatile ("pxor %xmm8,%xmm8");
|
|
asm volatile ("pxor %xmm9,%xmm9");
|
|
asm volatile ("pcmpgtb %xmm6,%xmm8");
|
|
asm volatile ("pcmpgtb %xmm7,%xmm9");
|
|
asm volatile ("paddb %xmm6,%xmm6");
|
|
asm volatile ("paddb %xmm7,%xmm7");
|
|
asm volatile ("pand %xmm11,%xmm8");
|
|
asm volatile ("pand %xmm11,%xmm9");
|
|
asm volatile ("pxor %xmm8,%xmm6");
|
|
asm volatile ("pxor %xmm9,%xmm7");
|
|
}
|
|
|
|
/* missing disks contribute zero */
|
|
if (d == id[0] || d == id[1])
|
|
continue;
|
|
|
|
/* lane 0 */
|
|
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i]));
|
|
asm volatile ("pxor %xmm8,%xmm0");
|
|
asm volatile ("pxor %xmm8,%xmm4");
|
|
|
|
/* lane 1 */
|
|
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i + 16]));
|
|
asm volatile ("pxor %xmm8,%xmm1");
|
|
asm volatile ("pxor %xmm8,%xmm5");
|
|
|
|
/* lane 2 */
|
|
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i + 32]));
|
|
asm volatile ("pxor %xmm8,%xmm2");
|
|
asm volatile ("pxor %xmm8,%xmm6");
|
|
|
|
/* lane 3 */
|
|
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i + 48]));
|
|
asm volatile ("pxor %xmm8,%xmm3");
|
|
asm volatile ("pxor %xmm8,%xmm7");
|
|
}
|
|
|
|
/* Qdelta = Q ^ Qa */
|
|
asm volatile ("pxor %0,%%xmm4" : : "m" (q[i]));
|
|
asm volatile ("pxor %0,%%xmm5" : : "m" (q[i + 16]));
|
|
asm volatile ("pxor %0,%%xmm6" : : "m" (q[i + 32]));
|
|
asm volatile ("pxor %0,%%xmm7" : : "m" (q[i + 48]));
|
|
|
|
/* the polynomial is no longer needed during reconstruction */
|
|
asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.low4));
|
|
|
|
/* lane 0: split Pdelta */
|
|
asm volatile ("movdqa %xmm0,%xmm8");
|
|
asm volatile ("movdqa %xmm0,%xmm9");
|
|
asm volatile ("psrlw $4,%xmm9");
|
|
asm volatile ("pand %xmm11,%xmm8");
|
|
asm volatile ("pand %xmm11,%xmm9");
|
|
|
|
/* lane 0: Dy = C0 * Pdelta */
|
|
asm volatile ("movdqa %xmm12,%xmm10");
|
|
asm volatile ("pshufb %xmm8,%xmm10");
|
|
asm volatile ("movdqa %xmm13,%xmm8");
|
|
asm volatile ("pshufb %xmm9,%xmm8");
|
|
asm volatile ("pxor %xmm8,%xmm10");
|
|
|
|
/* lane 0: split Qdelta */
|
|
asm volatile ("movdqa %xmm4,%xmm8");
|
|
asm volatile ("movdqa %xmm4,%xmm9");
|
|
asm volatile ("psrlw $4,%xmm9");
|
|
asm volatile ("pand %xmm11,%xmm8");
|
|
asm volatile ("pand %xmm11,%xmm9");
|
|
|
|
/* lane 0: Dy ^= C1 * Qdelta */
|
|
asm volatile ("movdqa %xmm14,%xmm4");
|
|
asm volatile ("pshufb %xmm8,%xmm4");
|
|
asm volatile ("pxor %xmm4,%xmm10");
|
|
asm volatile ("movdqa %xmm15,%xmm4");
|
|
asm volatile ("pshufb %xmm9,%xmm4");
|
|
asm volatile ("pxor %xmm4,%xmm10");
|
|
|
|
/* lane 0: Dx = Pdelta ^ Dy */
|
|
asm volatile ("pxor %xmm10,%xmm0");
|
|
|
|
/* recovery data must remain cacheable */
|
|
asm volatile ("movdqa %%xmm0,%0" : "=m" (pa[i]));
|
|
asm volatile ("movdqa %%xmm10,%0" : "=m" (qa[i]));
|
|
|
|
/* lane 1: split Pdelta */
|
|
asm volatile ("movdqa %xmm1,%xmm8");
|
|
asm volatile ("movdqa %xmm1,%xmm9");
|
|
asm volatile ("psrlw $4,%xmm9");
|
|
asm volatile ("pand %xmm11,%xmm8");
|
|
asm volatile ("pand %xmm11,%xmm9");
|
|
|
|
/* lane 1: Dy = C0 * Pdelta */
|
|
asm volatile ("movdqa %xmm12,%xmm10");
|
|
asm volatile ("pshufb %xmm8,%xmm10");
|
|
asm volatile ("movdqa %xmm13,%xmm8");
|
|
asm volatile ("pshufb %xmm9,%xmm8");
|
|
asm volatile ("pxor %xmm8,%xmm10");
|
|
|
|
/* lane 1: split Qdelta */
|
|
asm volatile ("movdqa %xmm5,%xmm8");
|
|
asm volatile ("movdqa %xmm5,%xmm9");
|
|
asm volatile ("psrlw $4,%xmm9");
|
|
asm volatile ("pand %xmm11,%xmm8");
|
|
asm volatile ("pand %xmm11,%xmm9");
|
|
|
|
/* lane 1: Dy ^= C1 * Qdelta */
|
|
asm volatile ("movdqa %xmm14,%xmm5");
|
|
asm volatile ("pshufb %xmm8,%xmm5");
|
|
asm volatile ("pxor %xmm5,%xmm10");
|
|
asm volatile ("movdqa %xmm15,%xmm5");
|
|
asm volatile ("pshufb %xmm9,%xmm5");
|
|
asm volatile ("pxor %xmm5,%xmm10");
|
|
|
|
/* lane 1: Dx = Pdelta ^ Dy */
|
|
asm volatile ("pxor %xmm10,%xmm1");
|
|
|
|
/* recovery data must remain cacheable */
|
|
asm volatile ("movdqa %%xmm1,%0" : "=m" (pa[i + 16]));
|
|
asm volatile ("movdqa %%xmm10,%0" : "=m" (qa[i + 16]));
|
|
|
|
/* lane 2: split Pdelta */
|
|
asm volatile ("movdqa %xmm2,%xmm8");
|
|
asm volatile ("movdqa %xmm2,%xmm9");
|
|
asm volatile ("psrlw $4,%xmm9");
|
|
asm volatile ("pand %xmm11,%xmm8");
|
|
asm volatile ("pand %xmm11,%xmm9");
|
|
|
|
/* lane 2: Dy = C0 * Pdelta */
|
|
asm volatile ("movdqa %xmm12,%xmm10");
|
|
asm volatile ("pshufb %xmm8,%xmm10");
|
|
asm volatile ("movdqa %xmm13,%xmm8");
|
|
asm volatile ("pshufb %xmm9,%xmm8");
|
|
asm volatile ("pxor %xmm8,%xmm10");
|
|
|
|
/* lane 2: split Qdelta */
|
|
asm volatile ("movdqa %xmm6,%xmm8");
|
|
asm volatile ("movdqa %xmm6,%xmm9");
|
|
asm volatile ("psrlw $4,%xmm9");
|
|
asm volatile ("pand %xmm11,%xmm8");
|
|
asm volatile ("pand %xmm11,%xmm9");
|
|
|
|
/* lane 2: Dy ^= C1 * Qdelta */
|
|
asm volatile ("movdqa %xmm14,%xmm6");
|
|
asm volatile ("pshufb %xmm8,%xmm6");
|
|
asm volatile ("pxor %xmm6,%xmm10");
|
|
asm volatile ("movdqa %xmm15,%xmm6");
|
|
asm volatile ("pshufb %xmm9,%xmm6");
|
|
asm volatile ("pxor %xmm6,%xmm10");
|
|
|
|
/* lane 2: Dx = Pdelta ^ Dy */
|
|
asm volatile ("pxor %xmm10,%xmm2");
|
|
|
|
/* recovery data must remain cacheable */
|
|
asm volatile ("movdqa %%xmm2,%0" : "=m" (pa[i + 32]));
|
|
asm volatile ("movdqa %%xmm10,%0" : "=m" (qa[i + 32]));
|
|
|
|
/* lane 3: split Pdelta */
|
|
asm volatile ("movdqa %xmm3,%xmm8");
|
|
asm volatile ("movdqa %xmm3,%xmm9");
|
|
asm volatile ("psrlw $4,%xmm9");
|
|
asm volatile ("pand %xmm11,%xmm8");
|
|
asm volatile ("pand %xmm11,%xmm9");
|
|
|
|
/* lane 3: Dy = C0 * Pdelta */
|
|
asm volatile ("movdqa %xmm12,%xmm10");
|
|
asm volatile ("pshufb %xmm8,%xmm10");
|
|
asm volatile ("movdqa %xmm13,%xmm8");
|
|
asm volatile ("pshufb %xmm9,%xmm8");
|
|
asm volatile ("pxor %xmm8,%xmm10");
|
|
|
|
/* lane 3: split Qdelta */
|
|
asm volatile ("movdqa %xmm7,%xmm8");
|
|
asm volatile ("movdqa %xmm7,%xmm9");
|
|
asm volatile ("psrlw $4,%xmm9");
|
|
asm volatile ("pand %xmm11,%xmm8");
|
|
asm volatile ("pand %xmm11,%xmm9");
|
|
|
|
/* lane 3: Dy ^= C1 * Qdelta */
|
|
asm volatile ("movdqa %xmm14,%xmm7");
|
|
asm volatile ("pshufb %xmm8,%xmm7");
|
|
asm volatile ("pxor %xmm7,%xmm10");
|
|
asm volatile ("movdqa %xmm15,%xmm7");
|
|
asm volatile ("pshufb %xmm9,%xmm7");
|
|
asm volatile ("pxor %xmm7,%xmm10");
|
|
|
|
/* lane 3: Dx = Pdelta ^ Dy */
|
|
asm volatile ("pxor %xmm10,%xmm3");
|
|
|
|
/* recovery data must remain cacheable */
|
|
asm volatile ("movdqa %%xmm3,%0" : "=m" (pa[i + 48]));
|
|
asm volatile ("movdqa %%xmm10,%0" : "=m" (qa[i + 48]));
|
|
|
|
/* restore the polynomial for the next 64-byte iteration */
|
|
asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.poly));
|
|
}
|
|
|
|
raid_sse_end(0);
|
|
}
|
|
|
|
/*
|
|
* Recover multiple data failures using selected parity blocks with SSSE3 extended optimized for up to three failures.
|
|
*
|
|
* If P is available, keep the complete P delta syndrome in xmm6.
|
|
* Reconstruct only nr - 1 missing blocks through the inverse matrix and
|
|
* obtain the last block by XORing the reconstructed blocks out of Pdelta.
|
|
*/
|
|
static __always_inline void raid_recX_ssse3ext_123(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
uint8_t **v = (uint8_t **)vv;
|
|
uint8_t *p[RAID_PARITY_MAX];
|
|
uint8_t *pa[RAID_PARITY_MAX];
|
|
uint8_t *src[RAID_DATA_MAX];
|
|
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
|
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
|
const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX];
|
|
const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
|
|
size_t i;
|
|
int d, j, k, s;
|
|
int ns;
|
|
|
|
BUG_ON(nr < 1 || nr > 3);
|
|
|
|
for (j = 0; j < nr; ++j)
|
|
for (k = 0; k < nr; ++k)
|
|
G[j * nr + k] = A(ip[j], id[k]);
|
|
|
|
raid_invert(G, V, nr);
|
|
|
|
for (j = 0; j < nr; ++j) {
|
|
p[j] = v[nd + ip[j]];
|
|
pa[j] = v[id[j]];
|
|
}
|
|
|
|
ns = 0;
|
|
k = 0;
|
|
|
|
for (d = 0; d < nd; ++d) {
|
|
if (k < nr && d == id[k]) {
|
|
++k;
|
|
continue;
|
|
}
|
|
|
|
src[ns] = v[d];
|
|
|
|
for (j = 0; j < nr; ++j)
|
|
S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0];
|
|
|
|
++ns;
|
|
}
|
|
|
|
BUG_ON(k != nr);
|
|
BUG_ON(ns != nd - nr);
|
|
|
|
for (j = 0; j < nr; ++j)
|
|
for (k = 0; k < nr; ++k)
|
|
R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
|
|
|
|
raid_sse_begin();
|
|
|
|
for (i = 0; i < size; i += 32) {
|
|
/*
|
|
* Xmm15 is later reused by reconstruction, therefore reload the
|
|
* nibble mask on every iteration.
|
|
*/
|
|
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4));
|
|
|
|
asm volatile ("movdqa %0,%%xmm0" : : "m" (p[0][i]));
|
|
asm volatile ("movdqa %0,%%xmm1" : : "m" (p[0][i + 16]));
|
|
|
|
if (nr >= 2) {
|
|
asm volatile ("movdqa %0,%%xmm2" : : "m" (p[1][i]));
|
|
asm volatile ("movdqa %0,%%xmm3" : : "m" (p[1][i + 16]));
|
|
}
|
|
|
|
if (nr >= 3) {
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (p[2][i]));
|
|
asm volatile ("movdqa %0,%%xmm5" : : "m" (p[2][i + 16]));
|
|
}
|
|
|
|
for (s = 0; s < ns; ++s) {
|
|
const uint8_t **t = S[s];
|
|
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (src[s][i]));
|
|
asm volatile ("movdqa %0,%%xmm7" : : "m" (src[s][i + 16]));
|
|
|
|
if (has_p) {
|
|
asm volatile ("pxor %xmm6,%xmm0");
|
|
asm volatile ("pxor %xmm7,%xmm1");
|
|
|
|
asm volatile ("movdqa %xmm6,%xmm8");
|
|
asm volatile ("movdqa %xmm7,%xmm9");
|
|
asm volatile ("psrlw $4,%xmm8");
|
|
asm volatile ("psrlw $4,%xmm9");
|
|
asm volatile ("pand %xmm15,%xmm6");
|
|
asm volatile ("pand %xmm15,%xmm7");
|
|
asm volatile ("pand %xmm15,%xmm8");
|
|
asm volatile ("pand %xmm15,%xmm9");
|
|
} else {
|
|
asm volatile ("movdqa %xmm6,%xmm8");
|
|
asm volatile ("movdqa %xmm7,%xmm9");
|
|
asm volatile ("psrlw $4,%xmm8");
|
|
asm volatile ("psrlw $4,%xmm9");
|
|
asm volatile ("pand %xmm15,%xmm6");
|
|
asm volatile ("pand %xmm15,%xmm7");
|
|
asm volatile ("pand %xmm15,%xmm8");
|
|
asm volatile ("pand %xmm15,%xmm9");
|
|
|
|
asm volatile ("movdqa %0,%%xmm10" : : "m" (t[0][0]));
|
|
asm volatile ("movdqa %0,%%xmm11" : : "m" (t[0][16]));
|
|
asm volatile ("movdqa %xmm10,%xmm12");
|
|
asm volatile ("movdqa %xmm11,%xmm13");
|
|
asm volatile ("pshufb %xmm6,%xmm10");
|
|
asm volatile ("pshufb %xmm8,%xmm11");
|
|
asm volatile ("pxor %xmm11,%xmm10");
|
|
asm volatile ("pxor %xmm10,%xmm0");
|
|
asm volatile ("pshufb %xmm7,%xmm12");
|
|
asm volatile ("pshufb %xmm9,%xmm13");
|
|
asm volatile ("pxor %xmm13,%xmm12");
|
|
asm volatile ("pxor %xmm12,%xmm1");
|
|
}
|
|
|
|
if (nr >= 2) {
|
|
asm volatile ("movdqa %0,%%xmm10" : : "m" (t[1][0]));
|
|
asm volatile ("movdqa %0,%%xmm11" : : "m" (t[1][16]));
|
|
asm volatile ("movdqa %xmm10,%xmm12");
|
|
asm volatile ("movdqa %xmm11,%xmm13");
|
|
asm volatile ("pshufb %xmm6,%xmm10");
|
|
asm volatile ("pshufb %xmm8,%xmm11");
|
|
asm volatile ("pxor %xmm11,%xmm10");
|
|
asm volatile ("pxor %xmm10,%xmm2");
|
|
asm volatile ("pshufb %xmm7,%xmm12");
|
|
asm volatile ("pshufb %xmm9,%xmm13");
|
|
asm volatile ("pxor %xmm13,%xmm12");
|
|
asm volatile ("pxor %xmm12,%xmm3");
|
|
}
|
|
|
|
if (nr >= 3) {
|
|
asm volatile ("movdqa %0,%%xmm10" : : "m" (t[2][0]));
|
|
asm volatile ("movdqa %0,%%xmm11" : : "m" (t[2][16]));
|
|
asm volatile ("movdqa %xmm10,%xmm12");
|
|
asm volatile ("movdqa %xmm11,%xmm13");
|
|
asm volatile ("pshufb %xmm6,%xmm10");
|
|
asm volatile ("pshufb %xmm8,%xmm11");
|
|
asm volatile ("pxor %xmm11,%xmm10");
|
|
asm volatile ("pxor %xmm10,%xmm4");
|
|
asm volatile ("pshufb %xmm7,%xmm12");
|
|
asm volatile ("pshufb %xmm9,%xmm13");
|
|
asm volatile ("pxor %xmm13,%xmm12");
|
|
asm volatile ("pxor %xmm12,%xmm5");
|
|
}
|
|
}
|
|
|
|
if (nr >= 3) {
|
|
asm volatile ("movdqa %xmm4,%xmm8");
|
|
asm volatile ("movdqa %xmm4,%xmm9");
|
|
asm volatile ("psrlw $4,%xmm9");
|
|
asm volatile ("pand %xmm15,%xmm8");
|
|
asm volatile ("pand %xmm15,%xmm9");
|
|
|
|
asm volatile ("movdqa %xmm5,%xmm10");
|
|
asm volatile ("movdqa %xmm5,%xmm11");
|
|
asm volatile ("psrlw $4,%xmm11");
|
|
asm volatile ("pand %xmm15,%xmm10");
|
|
asm volatile ("pand %xmm15,%xmm11");
|
|
}
|
|
|
|
if (nr >= 2) {
|
|
asm volatile ("movdqa %xmm2,%xmm4");
|
|
asm volatile ("movdqa %xmm2,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("pand %xmm15,%xmm4");
|
|
asm volatile ("pand %xmm15,%xmm5");
|
|
|
|
asm volatile ("movdqa %xmm3,%xmm6");
|
|
asm volatile ("movdqa %xmm3,%xmm7");
|
|
asm volatile ("psrlw $4,%xmm7");
|
|
asm volatile ("pand %xmm15,%xmm6");
|
|
asm volatile ("pand %xmm15,%xmm7");
|
|
}
|
|
|
|
asm volatile ("movdqa %xmm1,%xmm2");
|
|
asm volatile ("movdqa %xmm1,%xmm3");
|
|
asm volatile ("psrlw $4,%xmm3");
|
|
asm volatile ("pand %xmm15,%xmm2");
|
|
asm volatile ("pand %xmm15,%xmm3");
|
|
|
|
asm volatile ("movdqa %xmm0,%xmm1");
|
|
asm volatile ("psrlw $4,%xmm1");
|
|
asm volatile ("pand %xmm15,%xmm0");
|
|
asm volatile ("pand %xmm15,%xmm1");
|
|
|
|
for (j = 0; j < nr; ++j) {
|
|
const uint8_t **t = R[j];
|
|
|
|
asm volatile ("pxor %xmm12,%xmm12");
|
|
asm volatile ("pxor %xmm13,%xmm13");
|
|
|
|
/* coefficient 0 */
|
|
asm volatile ("movdqa %0,%%xmm14" : : "m" (t[0][0]));
|
|
asm volatile ("movdqa %xmm14,%xmm15");
|
|
asm volatile ("pshufb %xmm0,%xmm14");
|
|
asm volatile ("pshufb %xmm2,%xmm15");
|
|
asm volatile ("pxor %xmm14,%xmm12");
|
|
asm volatile ("pxor %xmm15,%xmm13");
|
|
|
|
asm volatile ("movdqa %0,%%xmm14" : : "m" (t[0][16]));
|
|
asm volatile ("movdqa %xmm14,%xmm15");
|
|
asm volatile ("pshufb %xmm1,%xmm14");
|
|
asm volatile ("pshufb %xmm3,%xmm15");
|
|
asm volatile ("pxor %xmm14,%xmm12");
|
|
asm volatile ("pxor %xmm15,%xmm13");
|
|
|
|
if (nr >= 2) {
|
|
asm volatile ("movdqa %0,%%xmm14" : : "m" (t[1][0]));
|
|
asm volatile ("movdqa %xmm14,%xmm15");
|
|
asm volatile ("pshufb %xmm4,%xmm14");
|
|
asm volatile ("pshufb %xmm6,%xmm15");
|
|
asm volatile ("pxor %xmm14,%xmm12");
|
|
asm volatile ("pxor %xmm15,%xmm13");
|
|
|
|
asm volatile ("movdqa %0,%%xmm14" : : "m" (t[1][16]));
|
|
asm volatile ("movdqa %xmm14,%xmm15");
|
|
asm volatile ("pshufb %xmm5,%xmm14");
|
|
asm volatile ("pshufb %xmm7,%xmm15");
|
|
asm volatile ("pxor %xmm14,%xmm12");
|
|
asm volatile ("pxor %xmm15,%xmm13");
|
|
}
|
|
|
|
if (nr >= 3) {
|
|
asm volatile ("movdqa %0,%%xmm14" : : "m" (t[2][0]));
|
|
asm volatile ("movdqa %xmm14,%xmm15");
|
|
asm volatile ("pshufb %xmm8,%xmm14");
|
|
asm volatile ("pshufb %xmm10,%xmm15");
|
|
asm volatile ("pxor %xmm14,%xmm12");
|
|
asm volatile ("pxor %xmm15,%xmm13");
|
|
|
|
asm volatile ("movdqa %0,%%xmm14" : : "m" (t[2][16]));
|
|
asm volatile ("movdqa %xmm14,%xmm15");
|
|
asm volatile ("pshufb %xmm9,%xmm14");
|
|
asm volatile ("pshufb %xmm11,%xmm15");
|
|
asm volatile ("pxor %xmm14,%xmm12");
|
|
asm volatile ("pxor %xmm15,%xmm13");
|
|
}
|
|
|
|
asm volatile ("movdqa %%xmm12,%0" : "=m" (pa[j][i]));
|
|
asm volatile ("movdqa %%xmm13,%0" : "=m" (pa[j][i + 16]));
|
|
}
|
|
}
|
|
|
|
raid_sse_end(0);
|
|
}
|
|
|
|
/*
|
|
* Recover multiple data failures using selected parity blocks with SSSE3 extended optimized for up to five failures.
|
|
*
|
|
* If P is available, keep the complete P delta syndrome in xmm10.
|
|
* Reconstruct only nr - 1 missing blocks through the inverse matrix and
|
|
* obtain the last block by XORing the reconstructed blocks out of Pdelta.
|
|
*
|
|
* With at most five failures the low/high syndrome pairs use xmm0..xmm9,
|
|
* leaving xmm10 available for the P delta accumulator.
|
|
*/
|
|
static __always_inline void raid_recX_ssse3ext_12345(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
uint8_t **v = (uint8_t **)vv;
|
|
uint8_t *p[RAID_PARITY_MAX];
|
|
uint8_t *pa[RAID_PARITY_MAX];
|
|
uint8_t *src[RAID_DATA_MAX];
|
|
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
|
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
|
const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX];
|
|
const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
|
|
size_t i;
|
|
int d, j, k, s;
|
|
int ns;
|
|
|
|
BUG_ON(nr < 1 || nr > 5);
|
|
|
|
for (j = 0; j < nr; ++j)
|
|
for (k = 0; k < nr; ++k)
|
|
G[j * nr + k] = A(ip[j], id[k]);
|
|
|
|
raid_invert(G, V, nr);
|
|
|
|
for (j = 0; j < nr; ++j) {
|
|
p[j] = v[nd + ip[j]];
|
|
pa[j] = v[id[j]];
|
|
}
|
|
|
|
ns = 0;
|
|
k = 0;
|
|
|
|
for (d = 0; d < nd; ++d) {
|
|
if (k < nr && d == id[k]) {
|
|
++k;
|
|
continue;
|
|
}
|
|
|
|
src[ns] = v[d];
|
|
|
|
for (j = 0; j < nr; ++j)
|
|
S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0];
|
|
|
|
++ns;
|
|
}
|
|
|
|
BUG_ON(k != nr);
|
|
BUG_ON(ns != nd - nr);
|
|
|
|
/*
|
|
* If P is available the last inverse-matrix row isn't needed.
|
|
* The corresponding missing block is obtained by XOR.
|
|
*/
|
|
for (j = 0; j < nr - has_p; ++j)
|
|
for (k = 0; k < nr; ++k)
|
|
R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
|
|
|
|
raid_sse_begin();
|
|
|
|
for (i = 0; i < size; i += 16) {
|
|
/*
|
|
* xmm15 is reused by reconstruction, therefore reload the
|
|
* nibble mask on every iteration.
|
|
*/
|
|
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4));
|
|
|
|
/*
|
|
* Raw syndrome accumulators:
|
|
*
|
|
* xmm0 syndrome 0
|
|
* xmm1 syndrome 1
|
|
* xmm2 syndrome 2
|
|
* xmm3 syndrome 3
|
|
* xmm4 syndrome 4
|
|
*
|
|
* xmm10 remains free for Pdelta.
|
|
*/
|
|
asm volatile ("movdqa %0,%%xmm0" : : "m" (p[0][i]));
|
|
|
|
if (nr >= 2)
|
|
asm volatile ("movdqa %0,%%xmm1" : : "m" (p[1][i]));
|
|
|
|
if (nr >= 3)
|
|
asm volatile ("movdqa %0,%%xmm2" : : "m" (p[2][i]));
|
|
|
|
if (nr >= 4)
|
|
asm volatile ("movdqa %0,%%xmm3" : : "m" (p[3][i]));
|
|
|
|
if (nr >= 5)
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (p[4][i]));
|
|
|
|
/* add all surviving data contributions */
|
|
for (s = 0; s < ns; ++s) {
|
|
const uint8_t **t = S[s];
|
|
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (src[s][i]));
|
|
|
|
/*
|
|
* P has coefficient 1 for every data disk.
|
|
* XOR the original source before splitting it.
|
|
*/
|
|
if (has_p) {
|
|
asm volatile ("pxor %xmm6,%xmm0");
|
|
|
|
asm volatile ("movdqa %xmm6,%xmm7");
|
|
asm volatile ("psrlw $4,%xmm7");
|
|
asm volatile ("pand %xmm15,%xmm6");
|
|
asm volatile ("pand %xmm15,%xmm7");
|
|
} else {
|
|
asm volatile ("movdqa %xmm6,%xmm7");
|
|
asm volatile ("psrlw $4,%xmm7");
|
|
asm volatile ("pand %xmm15,%xmm6");
|
|
asm volatile ("pand %xmm15,%xmm7");
|
|
|
|
asm volatile ("movdqa %0,%%xmm8" : : "m" (t[0][0]));
|
|
asm volatile ("movdqa %0,%%xmm9" : : "m" (t[0][16]));
|
|
asm volatile ("pshufb %xmm6,%xmm8");
|
|
asm volatile ("pshufb %xmm7,%xmm9");
|
|
asm volatile ("pxor %xmm8,%xmm0");
|
|
asm volatile ("pxor %xmm9,%xmm0");
|
|
}
|
|
|
|
if (nr >= 2) {
|
|
asm volatile ("movdqa %0,%%xmm8" : : "m" (t[1][0]));
|
|
asm volatile ("movdqa %0,%%xmm9" : : "m" (t[1][16]));
|
|
asm volatile ("pshufb %xmm6,%xmm8");
|
|
asm volatile ("pshufb %xmm7,%xmm9");
|
|
asm volatile ("pxor %xmm8,%xmm1");
|
|
asm volatile ("pxor %xmm9,%xmm1");
|
|
}
|
|
|
|
if (nr >= 3) {
|
|
asm volatile ("movdqa %0,%%xmm8" : : "m" (t[2][0]));
|
|
asm volatile ("movdqa %0,%%xmm9" : : "m" (t[2][16]));
|
|
asm volatile ("pshufb %xmm6,%xmm8");
|
|
asm volatile ("pshufb %xmm7,%xmm9");
|
|
asm volatile ("pxor %xmm8,%xmm2");
|
|
asm volatile ("pxor %xmm9,%xmm2");
|
|
}
|
|
|
|
if (nr >= 4) {
|
|
asm volatile ("movdqa %0,%%xmm8" : : "m" (t[3][0]));
|
|
asm volatile ("movdqa %0,%%xmm9" : : "m" (t[3][16]));
|
|
asm volatile ("pshufb %xmm6,%xmm8");
|
|
asm volatile ("pshufb %xmm7,%xmm9");
|
|
asm volatile ("pxor %xmm8,%xmm3");
|
|
asm volatile ("pxor %xmm9,%xmm3");
|
|
}
|
|
|
|
if (nr >= 5) {
|
|
asm volatile ("movdqa %0,%%xmm8" : : "m" (t[4][0]));
|
|
asm volatile ("movdqa %0,%%xmm9" : : "m" (t[4][16]));
|
|
asm volatile ("pshufb %xmm6,%xmm8");
|
|
asm volatile ("pshufb %xmm7,%xmm9");
|
|
asm volatile ("pxor %xmm8,%xmm4");
|
|
asm volatile ("pxor %xmm9,%xmm4");
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Preserve the complete P delta before splitting syndrome 0.
|
|
*
|
|
* xmm10 = M0 ^ M1 ^ ... ^ M(nr-1)
|
|
*/
|
|
if (has_p)
|
|
asm volatile ("movdqa %xmm0,%xmm10");
|
|
|
|
/*
|
|
* Expand backwards to low/high syndrome pairs:
|
|
*
|
|
* xmm0/xmm1 syndrome 0 low/high
|
|
* xmm2/xmm3 syndrome 1 low/high
|
|
* xmm4/xmm5 syndrome 2 low/high
|
|
* xmm6/xmm7 syndrome 3 low/high
|
|
* xmm8/xmm9 syndrome 4 low/high
|
|
*
|
|
* xmm10 remains the complete remaining P delta.
|
|
*/
|
|
if (nr >= 5) {
|
|
asm volatile ("movdqa %xmm4,%xmm8");
|
|
asm volatile ("movdqa %xmm4,%xmm9");
|
|
asm volatile ("psrlw $4,%xmm9");
|
|
asm volatile ("pand %xmm15,%xmm8");
|
|
asm volatile ("pand %xmm15,%xmm9");
|
|
}
|
|
|
|
if (nr >= 4) {
|
|
asm volatile ("movdqa %xmm3,%xmm6");
|
|
asm volatile ("movdqa %xmm3,%xmm7");
|
|
asm volatile ("psrlw $4,%xmm7");
|
|
asm volatile ("pand %xmm15,%xmm6");
|
|
asm volatile ("pand %xmm15,%xmm7");
|
|
}
|
|
|
|
if (nr >= 3) {
|
|
asm volatile ("movdqa %xmm2,%xmm4");
|
|
asm volatile ("movdqa %xmm2,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("pand %xmm15,%xmm4");
|
|
asm volatile ("pand %xmm15,%xmm5");
|
|
}
|
|
|
|
if (nr >= 2) {
|
|
asm volatile ("movdqa %xmm1,%xmm2");
|
|
asm volatile ("movdqa %xmm1,%xmm3");
|
|
asm volatile ("psrlw $4,%xmm3");
|
|
asm volatile ("pand %xmm15,%xmm2");
|
|
asm volatile ("pand %xmm15,%xmm3");
|
|
}
|
|
|
|
asm volatile ("movdqa %xmm0,%xmm1");
|
|
asm volatile ("psrlw $4,%xmm1");
|
|
asm volatile ("pand %xmm15,%xmm0");
|
|
asm volatile ("pand %xmm15,%xmm1");
|
|
|
|
/*
|
|
* Reconstruct all but the last missing block when P is
|
|
* available.
|
|
*
|
|
* xmm12/xmm13 = low/high output accumulators
|
|
* xmm14/xmm15 = multiplication temporaries
|
|
* xmm10 = remaining P delta
|
|
*/
|
|
for (j = 0; j < nr - has_p; ++j) {
|
|
const uint8_t **t = R[j];
|
|
|
|
/* coefficient 0 initializes both accumulators */
|
|
asm volatile ("movdqa %0,%%xmm12" : : "m" (t[0][0]));
|
|
asm volatile ("movdqa %0,%%xmm13" : : "m" (t[0][16]));
|
|
asm volatile ("pshufb %xmm0,%xmm12");
|
|
asm volatile ("pshufb %xmm1,%xmm13");
|
|
|
|
if (nr >= 2) {
|
|
asm volatile ("movdqa %0,%%xmm14" : : "m" (t[1][0]));
|
|
asm volatile ("movdqa %0,%%xmm15" : : "m" (t[1][16]));
|
|
asm volatile ("pshufb %xmm2,%xmm14");
|
|
asm volatile ("pshufb %xmm3,%xmm15");
|
|
asm volatile ("pxor %xmm14,%xmm12");
|
|
asm volatile ("pxor %xmm15,%xmm13");
|
|
}
|
|
|
|
if (nr >= 3) {
|
|
asm volatile ("movdqa %0,%%xmm14" : : "m" (t[2][0]));
|
|
asm volatile ("movdqa %0,%%xmm15" : : "m" (t[2][16]));
|
|
asm volatile ("pshufb %xmm4,%xmm14");
|
|
asm volatile ("pshufb %xmm5,%xmm15");
|
|
asm volatile ("pxor %xmm14,%xmm12");
|
|
asm volatile ("pxor %xmm15,%xmm13");
|
|
}
|
|
|
|
if (nr >= 4) {
|
|
asm volatile ("movdqa %0,%%xmm14" : : "m" (t[3][0]));
|
|
asm volatile ("movdqa %0,%%xmm15" : : "m" (t[3][16]));
|
|
asm volatile ("pshufb %xmm6,%xmm14");
|
|
asm volatile ("pshufb %xmm7,%xmm15");
|
|
asm volatile ("pxor %xmm14,%xmm12");
|
|
asm volatile ("pxor %xmm15,%xmm13");
|
|
}
|
|
|
|
if (nr >= 5) {
|
|
asm volatile ("movdqa %0,%%xmm14" : : "m" (t[4][0]));
|
|
asm volatile ("movdqa %0,%%xmm15" : : "m" (t[4][16]));
|
|
asm volatile ("pshufb %xmm8,%xmm14");
|
|
asm volatile ("pshufb %xmm9,%xmm15");
|
|
asm volatile ("pxor %xmm14,%xmm12");
|
|
asm volatile ("pxor %xmm15,%xmm13");
|
|
}
|
|
|
|
asm volatile ("pxor %xmm13,%xmm12");
|
|
|
|
/*
|
|
* Remove the reconstructed block from Pdelta.
|
|
* After nr - 1 iterations xmm10 contains the final
|
|
* missing block.
|
|
*/
|
|
if (has_p)
|
|
asm volatile ("pxor %xmm12,%xmm10");
|
|
|
|
asm volatile ("movdqa %%xmm12,%0" : "=m" (pa[j][i]));
|
|
}
|
|
|
|
if (has_p)
|
|
asm volatile ("movdqa %%xmm10,%0" : "=m" (pa[nr - 1][i]));
|
|
}
|
|
|
|
raid_sse_end(0);
|
|
}
|
|
|
|
/*
|
|
* Recover multiple data failures using selected parity blocks with SSSE3 extended.
|
|
*
|
|
* This avoids temporary syndrome buffers and the
|
|
* generation of unused parity rows.
|
|
*
|
|
* If P is available, preserve the complete P delta syndrome and
|
|
* reconstruct only nr - 1 missing blocks through the inverse matrix.
|
|
* The last missing block is obtained by XORing the reconstructed blocks
|
|
* out of Pdelta.
|
|
*/
|
|
static __always_inline void raid_recX_ssse3ext(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
uint8_t **v = (uint8_t **)vv;
|
|
uint8_t *p[RAID_PARITY_MAX];
|
|
uint8_t *pa[RAID_PARITY_MAX];
|
|
uint8_t *src[RAID_DATA_MAX];
|
|
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
|
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
|
const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX];
|
|
const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
|
|
size_t i;
|
|
int d, j, k, s;
|
|
int ns;
|
|
|
|
BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
|
|
|
|
for (j = 0; j < nr; ++j)
|
|
for (k = 0; k < nr; ++k)
|
|
G[j * nr + k] = A(ip[j], id[k]);
|
|
|
|
raid_invert(G, V, nr);
|
|
|
|
for (j = 0; j < nr; ++j) {
|
|
p[j] = v[nd + ip[j]];
|
|
pa[j] = v[id[j]];
|
|
}
|
|
|
|
ns = 0;
|
|
k = 0;
|
|
|
|
for (d = 0; d < nd; ++d) {
|
|
if (k < nr && d == id[k]) {
|
|
++k;
|
|
continue;
|
|
}
|
|
|
|
src[ns] = v[d];
|
|
|
|
for (j = 0; j < nr; ++j)
|
|
S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0];
|
|
|
|
++ns;
|
|
}
|
|
|
|
BUG_ON(k != nr);
|
|
BUG_ON(ns != nd - nr);
|
|
|
|
for (j = 0; j < nr; ++j)
|
|
for (k = 0; k < nr; ++k)
|
|
R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
|
|
|
|
raid_sse_begin();
|
|
|
|
for (i = 0; i < size; i += 16) {
|
|
/*
|
|
* Xmm15 is later reused by reconstruction, therefore reload the
|
|
* nibble mask on every iteration.
|
|
*/
|
|
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4));
|
|
|
|
asm volatile ("movdqa %0,%%xmm0" : : "m" (p[0][i]));
|
|
|
|
if (nr >= 2)
|
|
asm volatile ("movdqa %0,%%xmm1" : : "m" (p[1][i]));
|
|
|
|
if (nr >= 3)
|
|
asm volatile ("movdqa %0,%%xmm2" : : "m" (p[2][i]));
|
|
|
|
if (nr >= 4)
|
|
asm volatile ("movdqa %0,%%xmm3" : : "m" (p[3][i]));
|
|
|
|
if (nr >= 5)
|
|
asm volatile ("movdqa %0,%%xmm4" : : "m" (p[4][i]));
|
|
|
|
if (nr >= 6)
|
|
asm volatile ("movdqa %0,%%xmm5" : : "m" (p[5][i]));
|
|
|
|
for (s = 0; s < ns; ++s) {
|
|
const uint8_t **t = S[s];
|
|
|
|
asm volatile ("movdqa %0,%%xmm6" : : "m" (src[s][i]));
|
|
|
|
if (has_p) {
|
|
asm volatile ("pxor %xmm6,%xmm0");
|
|
|
|
asm volatile ("movdqa %xmm6,%xmm7");
|
|
asm volatile ("psrlw $4,%xmm7");
|
|
asm volatile ("pand %xmm15,%xmm6");
|
|
asm volatile ("pand %xmm15,%xmm7");
|
|
} else {
|
|
asm volatile ("movdqa %xmm6,%xmm7");
|
|
asm volatile ("psrlw $4,%xmm7");
|
|
asm volatile ("pand %xmm15,%xmm6");
|
|
asm volatile ("pand %xmm15,%xmm7");
|
|
|
|
asm volatile ("movdqa %0,%%xmm8" : : "m" (t[0][0]));
|
|
asm volatile ("movdqa %0,%%xmm9" : : "m" (t[0][16]));
|
|
asm volatile ("pshufb %xmm6,%xmm8");
|
|
asm volatile ("pshufb %xmm7,%xmm9");
|
|
asm volatile ("pxor %xmm8,%xmm0");
|
|
asm volatile ("pxor %xmm9,%xmm0");
|
|
}
|
|
|
|
if (nr >= 2) {
|
|
asm volatile ("movdqa %0,%%xmm8" : : "m" (t[1][0]));
|
|
asm volatile ("movdqa %0,%%xmm9" : : "m" (t[1][16]));
|
|
asm volatile ("pshufb %xmm6,%xmm8");
|
|
asm volatile ("pshufb %xmm7,%xmm9");
|
|
asm volatile ("pxor %xmm8,%xmm1");
|
|
asm volatile ("pxor %xmm9,%xmm1");
|
|
}
|
|
|
|
if (nr >= 3) {
|
|
asm volatile ("movdqa %0,%%xmm8" : : "m" (t[2][0]));
|
|
asm volatile ("movdqa %0,%%xmm9" : : "m" (t[2][16]));
|
|
asm volatile ("pshufb %xmm6,%xmm8");
|
|
asm volatile ("pshufb %xmm7,%xmm9");
|
|
asm volatile ("pxor %xmm8,%xmm2");
|
|
asm volatile ("pxor %xmm9,%xmm2");
|
|
}
|
|
|
|
if (nr >= 4) {
|
|
asm volatile ("movdqa %0,%%xmm8" : : "m" (t[3][0]));
|
|
asm volatile ("movdqa %0,%%xmm9" : : "m" (t[3][16]));
|
|
asm volatile ("pshufb %xmm6,%xmm8");
|
|
asm volatile ("pshufb %xmm7,%xmm9");
|
|
asm volatile ("pxor %xmm8,%xmm3");
|
|
asm volatile ("pxor %xmm9,%xmm3");
|
|
}
|
|
|
|
if (nr >= 5) {
|
|
asm volatile ("movdqa %0,%%xmm8" : : "m" (t[4][0]));
|
|
asm volatile ("movdqa %0,%%xmm9" : : "m" (t[4][16]));
|
|
asm volatile ("pshufb %xmm6,%xmm8");
|
|
asm volatile ("pshufb %xmm7,%xmm9");
|
|
asm volatile ("pxor %xmm8,%xmm4");
|
|
asm volatile ("pxor %xmm9,%xmm4");
|
|
}
|
|
|
|
if (nr >= 6) {
|
|
asm volatile ("movdqa %0,%%xmm8" : : "m" (t[5][0]));
|
|
asm volatile ("movdqa %0,%%xmm9" : : "m" (t[5][16]));
|
|
asm volatile ("pshufb %xmm6,%xmm8");
|
|
asm volatile ("pshufb %xmm7,%xmm9");
|
|
asm volatile ("pxor %xmm8,%xmm5");
|
|
asm volatile ("pxor %xmm9,%xmm5");
|
|
}
|
|
}
|
|
|
|
/* expand backwards to low/high pairs */
|
|
if (nr >= 6) {
|
|
asm volatile ("movdqa %xmm5,%xmm10");
|
|
asm volatile ("movdqa %xmm5,%xmm11");
|
|
asm volatile ("psrlw $4,%xmm11");
|
|
asm volatile ("pand %xmm15,%xmm10");
|
|
asm volatile ("pand %xmm15,%xmm11");
|
|
}
|
|
|
|
if (nr >= 5) {
|
|
asm volatile ("movdqa %xmm4,%xmm8");
|
|
asm volatile ("movdqa %xmm4,%xmm9");
|
|
asm volatile ("psrlw $4,%xmm9");
|
|
asm volatile ("pand %xmm15,%xmm8");
|
|
asm volatile ("pand %xmm15,%xmm9");
|
|
}
|
|
|
|
if (nr >= 4) {
|
|
asm volatile ("movdqa %xmm3,%xmm6");
|
|
asm volatile ("movdqa %xmm3,%xmm7");
|
|
asm volatile ("psrlw $4,%xmm7");
|
|
asm volatile ("pand %xmm15,%xmm6");
|
|
asm volatile ("pand %xmm15,%xmm7");
|
|
}
|
|
|
|
if (nr >= 3) {
|
|
asm volatile ("movdqa %xmm2,%xmm4");
|
|
asm volatile ("movdqa %xmm2,%xmm5");
|
|
asm volatile ("psrlw $4,%xmm5");
|
|
asm volatile ("pand %xmm15,%xmm4");
|
|
asm volatile ("pand %xmm15,%xmm5");
|
|
}
|
|
|
|
if (nr >= 2) {
|
|
asm volatile ("movdqa %xmm1,%xmm2");
|
|
asm volatile ("movdqa %xmm1,%xmm3");
|
|
asm volatile ("psrlw $4,%xmm3");
|
|
asm volatile ("pand %xmm15,%xmm2");
|
|
asm volatile ("pand %xmm15,%xmm3");
|
|
}
|
|
|
|
asm volatile ("movdqa %xmm0,%xmm1");
|
|
asm volatile ("psrlw $4,%xmm1");
|
|
asm volatile ("pand %xmm15,%xmm0");
|
|
asm volatile ("pand %xmm15,%xmm1");
|
|
|
|
for (j = 0; j < nr; ++j) {
|
|
const uint8_t **t = R[j];
|
|
|
|
asm volatile ("movdqa %0,%%xmm12" : : "m" (t[0][0]));
|
|
asm volatile ("movdqa %0,%%xmm13" : : "m" (t[0][16]));
|
|
asm volatile ("pshufb %xmm0,%xmm12");
|
|
asm volatile ("pshufb %xmm1,%xmm13");
|
|
|
|
if (nr >= 2) {
|
|
asm volatile ("movdqa %0,%%xmm14" : : "m" (t[1][0]));
|
|
asm volatile ("movdqa %0,%%xmm15" : : "m" (t[1][16]));
|
|
asm volatile ("pshufb %xmm2,%xmm14");
|
|
asm volatile ("pshufb %xmm3,%xmm15");
|
|
asm volatile ("pxor %xmm14,%xmm12");
|
|
asm volatile ("pxor %xmm15,%xmm13");
|
|
}
|
|
|
|
if (nr >= 3) {
|
|
asm volatile ("movdqa %0,%%xmm14" : : "m" (t[2][0]));
|
|
asm volatile ("movdqa %0,%%xmm15" : : "m" (t[2][16]));
|
|
asm volatile ("pshufb %xmm4,%xmm14");
|
|
asm volatile ("pshufb %xmm5,%xmm15");
|
|
asm volatile ("pxor %xmm14,%xmm12");
|
|
asm volatile ("pxor %xmm15,%xmm13");
|
|
}
|
|
|
|
if (nr >= 4) {
|
|
asm volatile ("movdqa %0,%%xmm14" : : "m" (t[3][0]));
|
|
asm volatile ("movdqa %0,%%xmm15" : : "m" (t[3][16]));
|
|
asm volatile ("pshufb %xmm6,%xmm14");
|
|
asm volatile ("pshufb %xmm7,%xmm15");
|
|
asm volatile ("pxor %xmm14,%xmm12");
|
|
asm volatile ("pxor %xmm15,%xmm13");
|
|
}
|
|
|
|
if (nr >= 5) {
|
|
asm volatile ("movdqa %0,%%xmm14" : : "m" (t[4][0]));
|
|
asm volatile ("movdqa %0,%%xmm15" : : "m" (t[4][16]));
|
|
asm volatile ("pshufb %xmm8,%xmm14");
|
|
asm volatile ("pshufb %xmm9,%xmm15");
|
|
asm volatile ("pxor %xmm14,%xmm12");
|
|
asm volatile ("pxor %xmm15,%xmm13");
|
|
}
|
|
|
|
if (nr >= 6) {
|
|
asm volatile ("movdqa %0,%%xmm14" : : "m" (t[5][0]));
|
|
asm volatile ("movdqa %0,%%xmm15" : : "m" (t[5][16]));
|
|
asm volatile ("pshufb %xmm10,%xmm14");
|
|
asm volatile ("pshufb %xmm11,%xmm15");
|
|
asm volatile ("pxor %xmm14,%xmm12");
|
|
asm volatile ("pxor %xmm15,%xmm13");
|
|
}
|
|
|
|
asm volatile ("pxor %xmm13,%xmm12");
|
|
asm volatile ("movdqa %%xmm12,%0" : "=m" (pa[j][i]));
|
|
}
|
|
}
|
|
|
|
raid_sse_end(0);
|
|
}
|
|
|
|
#endif
|
|
|
|
void raid_gen3_ssse3_raid(int nd, size_t size, void **vv, int streaming)
|
|
{
|
|
if (streaming)
|
|
raid_gen3_ssse3_gen(nd, size, vv, 2, 1);
|
|
else
|
|
raid_gen3_ssse3_gen(nd, size, vv, 2, 0);
|
|
}
|
|
|
|
void raid_gen3_ssse3_aes(int nd, size_t size, void **vv, int streaming)
|
|
{
|
|
if (streaming)
|
|
raid_gen3_ssse3_gen(nd, size, vv, 3, 1);
|
|
else
|
|
raid_gen3_ssse3_gen(nd, size, vv, 3, 0);
|
|
}
|
|
|
|
void raid_gen4_ssse3_raid(int nd, size_t size, void **vv, int streaming)
|
|
{
|
|
if (streaming)
|
|
raid_gen4_ssse3_gen(nd, size, vv, 2, 1);
|
|
else
|
|
raid_gen4_ssse3_gen(nd, size, vv, 2, 0);
|
|
}
|
|
|
|
void raid_gen4_ssse3_aes(int nd, size_t size, void **vv, int streaming)
|
|
{
|
|
if (streaming)
|
|
raid_gen4_ssse3_gen(nd, size, vv, 3, 1);
|
|
else
|
|
raid_gen4_ssse3_gen(nd, size, vv, 3, 0);
|
|
}
|
|
|
|
void raid_gen5_ssse3_raid(int nd, size_t size, void **vv, int streaming)
|
|
{
|
|
if (streaming)
|
|
raid_gen5_ssse3_gen(nd, size, vv, 1);
|
|
else
|
|
raid_gen5_ssse3_gen(nd, size, vv, 0);
|
|
}
|
|
|
|
void raid_gen6_ssse3_raid(int nd, size_t size, void **vv, int streaming)
|
|
{
|
|
if (streaming)
|
|
raid_gen6_ssse3_gen(nd, size, vv, 1);
|
|
else
|
|
raid_gen6_ssse3_gen(nd, size, vv, 0);
|
|
}
|
|
|
|
#ifdef CONFIG_X86_64
|
|
void raid_gen3_ssse3ext_raid(int nd, size_t size, void **vv, int streaming)
|
|
{
|
|
if (streaming)
|
|
raid_gen3_ssse3ext_gen(nd, size, vv, 2, 1);
|
|
else
|
|
raid_gen3_ssse3ext_gen(nd, size, vv, 2, 0);
|
|
}
|
|
|
|
void raid_gen3_ssse3ext_aes(int nd, size_t size, void **vv, int streaming)
|
|
{
|
|
if (streaming)
|
|
raid_gen3_ssse3ext_gen(nd, size, vv, 3, 1);
|
|
else
|
|
raid_gen3_ssse3ext_gen(nd, size, vv, 3, 0);
|
|
}
|
|
|
|
void raid_gen4_ssse3ext_raid(int nd, size_t size, void **vv, int streaming)
|
|
{
|
|
if (streaming)
|
|
raid_gen4_ssse3ext_gen(nd, size, vv, 2, 1);
|
|
else
|
|
raid_gen4_ssse3ext_gen(nd, size, vv, 2, 0);
|
|
}
|
|
|
|
void raid_gen4_ssse3ext_aes(int nd, size_t size, void **vv, int streaming)
|
|
{
|
|
if (streaming)
|
|
raid_gen4_ssse3ext_gen(nd, size, vv, 3, 1);
|
|
else
|
|
raid_gen4_ssse3ext_gen(nd, size, vv, 3, 0);
|
|
}
|
|
|
|
void raid_gen5_ssse3ext_raid(int nd, size_t size, void **vv, int streaming)
|
|
{
|
|
if (streaming)
|
|
raid_genX_ssse3ext(nd, size, vv, 5, 2, 1);
|
|
else
|
|
raid_genX_ssse3ext(nd, size, vv, 5, 2, 0);
|
|
}
|
|
|
|
void raid_gen5_ssse3ext_aes(int nd, size_t size, void **vv, int streaming)
|
|
{
|
|
if (streaming)
|
|
raid_genX_ssse3ext(nd, size, vv, 5, 3, 1);
|
|
else
|
|
raid_genX_ssse3ext(nd, size, vv, 5, 3, 0);
|
|
}
|
|
|
|
void raid_gen6_ssse3ext_raid(int nd, size_t size, void **vv, int streaming)
|
|
{
|
|
if (streaming)
|
|
raid_genX_ssse3ext(nd, size, vv, 6, 2, 1);
|
|
else
|
|
raid_genX_ssse3ext(nd, size, vv, 6, 2, 0);
|
|
}
|
|
|
|
void raid_gen6_ssse3ext_aes(int nd, size_t size, void **vv, int streaming)
|
|
{
|
|
if (streaming)
|
|
raid_genX_ssse3ext(nd, size, vv, 6, 3, 1);
|
|
else
|
|
raid_genX_ssse3ext(nd, size, vv, 6, 3, 0);
|
|
}
|
|
|
|
#endif
|
|
|
|
void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
BUG_ON(nr != 1);
|
|
|
|
/* if recovering with P, use a custom XOR-only path with temporal stores */
|
|
if (ip[0] == 0) {
|
|
raid_rec1of1(id, nd, size, vv);
|
|
return;
|
|
}
|
|
|
|
/* if recovering with Q use the specialized function */
|
|
if (ip[0] == 1) {
|
|
raid_rec1_ssse3_q(id, ip, nd, size, vv);
|
|
return;
|
|
}
|
|
|
|
raid_rec1_ssse3_1(id, ip, nd, size, vv);
|
|
}
|
|
|
|
void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
BUG_ON(nr != 2);
|
|
|
|
/* if recovering with P,Q use the specialized function */
|
|
if (ip[0] == 0 && ip[1] == 1) {
|
|
raid_rec2of2_ssse3(id, ip, nd, size, vv);
|
|
return;
|
|
}
|
|
|
|
if (ip[0] == 0)
|
|
raid_rec2_ssse3_2(1, id, ip, nd, size, vv);
|
|
else
|
|
raid_rec2_ssse3_2(0, id, ip, nd, size, vv);
|
|
}
|
|
|
|
void raid_rec3_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
BUG_ON(nr != 3);
|
|
if (ip[0] == 0)
|
|
raid_recX_ssse3_1234(3, 1, id, ip, nd, size, vv);
|
|
else
|
|
raid_recX_ssse3_1234(3, 0, id, ip, nd, size, vv);
|
|
}
|
|
|
|
void raid_rec4_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
BUG_ON(nr != 4);
|
|
if (ip[0] == 0)
|
|
raid_recX_ssse3_1234(4, 1, id, ip, nd, size, vv);
|
|
else
|
|
raid_recX_ssse3_1234(4, 0, id, ip, nd, size, vv);
|
|
}
|
|
|
|
void raid_rec5_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
BUG_ON(nr != 5);
|
|
if (ip[0] == 0)
|
|
raid_recX_ssse3(5, 1, id, ip, nd, size, vv);
|
|
else
|
|
raid_recX_ssse3(5, 0, id, ip, nd, size, vv);
|
|
}
|
|
|
|
void raid_rec6_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
BUG_ON(nr != 6);
|
|
if (ip[0] == 0)
|
|
raid_recX_ssse3(6, 1, id, ip, nd, size, vv);
|
|
else
|
|
raid_recX_ssse3(6, 0, id, ip, nd, size, vv);
|
|
}
|
|
|
|
#ifdef CONFIG_X86_64
|
|
void raid_rec1_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
BUG_ON(nr != 1);
|
|
|
|
/* if recovering with P, use a custom XOR-only path with temporal stores */
|
|
if (ip[0] == 0) {
|
|
raid_rec1of1(id, nd, size, vv);
|
|
return;
|
|
}
|
|
|
|
/* if recovering with Q use the specialized function */
|
|
if (ip[0] == 1) {
|
|
raid_rec1_ssse3ext_q(id, ip, nd, size, vv);
|
|
return;
|
|
}
|
|
|
|
raid_rec1_ssse3ext_1(id, ip, nd, size, vv);
|
|
}
|
|
|
|
void raid_rec2_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
BUG_ON(nr != 2);
|
|
|
|
/* if recovering with P,Q use the specialized function */
|
|
if (ip[0] == 0 && ip[1] == 1) {
|
|
raid_rec2of2_ssse3ext(id, ip, nd, size, vv);
|
|
return;
|
|
}
|
|
|
|
if (ip[0] == 0)
|
|
raid_rec2_ssse3ext_2(1, id, ip, nd, size, vv);
|
|
else
|
|
raid_rec2_ssse3ext_2(0, id, ip, nd, size, vv);
|
|
}
|
|
|
|
void raid_rec3_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
BUG_ON(nr != 3);
|
|
if (ip[0] == 0)
|
|
raid_recX_ssse3ext_123(3, 1, id, ip, nd, size, vv);
|
|
else
|
|
raid_recX_ssse3ext_123(3, 0, id, ip, nd, size, vv);
|
|
}
|
|
|
|
void raid_rec4_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
BUG_ON(nr != 4);
|
|
if (ip[0] == 0)
|
|
raid_recX_ssse3ext_12345(4, 1, id, ip, nd, size, vv);
|
|
else
|
|
raid_recX_ssse3ext_12345(4, 0, id, ip, nd, size, vv);
|
|
}
|
|
|
|
void raid_rec5_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
BUG_ON(nr != 5);
|
|
if (ip[0] == 0)
|
|
raid_recX_ssse3ext_12345(5, 1, id, ip, nd, size, vv);
|
|
else
|
|
raid_recX_ssse3ext_12345(5, 0, id, ip, nd, size, vv);
|
|
}
|
|
|
|
void raid_rec6_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
BUG_ON(nr != 6);
|
|
if (ip[0] == 0)
|
|
raid_recX_ssse3ext(6, 1, id, ip, nd, size, vv);
|
|
else
|
|
raid_recX_ssse3ext(6, 0, id, ip, nd, size, vv);
|
|
}
|
|
#endif
|
|
|
|
void raid_register_ssse3(void)
|
|
{
|
|
if (raid_cpu_has_ssse3()) {
|
|
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3", raid_gen3_ssse3_raid, RAID_POLY_RAID);
|
|
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3", raid_gen3_ssse3_aes, RAID_POLY_AES);
|
|
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3", raid_gen4_ssse3_raid, RAID_POLY_RAID);
|
|
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3", raid_gen4_ssse3_aes, RAID_POLY_AES);
|
|
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3", raid_gen5_ssse3_raid, RAID_POLY_RAID);
|
|
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3", raid_gen6_ssse3_raid, RAID_POLY_RAID);
|
|
|
|
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "ssse3", raid_rec1_ssse3, RAID_POLY_ANY);
|
|
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "ssse3", raid_rec2_ssse3, RAID_POLY_ANY);
|
|
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "ssse3", raid_rec3_ssse3, RAID_POLY_ANY);
|
|
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "ssse3", raid_rec4_ssse3, RAID_POLY_ANY);
|
|
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "ssse3", raid_rec5_ssse3, RAID_POLY_ANY);
|
|
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "ssse3", raid_rec6_ssse3, RAID_POLY_ANY);
|
|
|
|
#ifdef CONFIG_X86_64
|
|
if (!raid_cpu_has_slow_extendedreg()) {
|
|
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3e", raid_gen3_ssse3ext_raid, RAID_POLY_RAID);
|
|
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3e", raid_gen3_ssse3ext_aes, RAID_POLY_AES);
|
|
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3e", raid_gen4_ssse3ext_raid, RAID_POLY_RAID);
|
|
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3e", raid_gen4_ssse3ext_aes, RAID_POLY_AES);
|
|
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3e", raid_gen5_ssse3ext_raid, RAID_POLY_RAID);
|
|
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3e", raid_gen5_ssse3ext_aes, RAID_POLY_AES);
|
|
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3e", raid_gen6_ssse3ext_raid, RAID_POLY_RAID);
|
|
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3e", raid_gen6_ssse3ext_aes, RAID_POLY_AES);
|
|
|
|
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "ssse3e", raid_rec1_ssse3ext, RAID_POLY_ANY);
|
|
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "ssse3e", raid_rec2_ssse3ext, RAID_POLY_ANY);
|
|
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "ssse3e", raid_rec3_ssse3ext, RAID_POLY_ANY);
|
|
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "ssse3e", raid_rec4_ssse3ext, RAID_POLY_ANY);
|
|
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "ssse3e", raid_rec5_ssse3ext, RAID_POLY_ANY);
|
|
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "ssse3e", raid_rec6_ssse3ext, RAID_POLY_ANY);
|
|
}
|
|
#endif
|
|
}
|
|
}
|
|
|
|
#endif
|