rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 19:01:57 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Split GEN x2 and G23 paths to eliminate AES hot-path overhead
Some checks are pending
Build / build-mac (push) Waiting to run
Details
Build / build-linux (push) Waiting to run
Details
Build / build-arm32 (push) Waiting to run
Details
Build / build-win (push) Waiting to run
Details
CodeQL Advanced / Analyze (c-cpp) (push) Waiting to run
Details
CodeQL Advanced / Analyze (actions) (push) Waiting to run
Details

Browse source
This commit is contained in:
Andrea Mazzoleni 2026-08-22 00:34:50 +02:00
parent 6a786a42b6
commit 2a7b8e19d4
10 changed files with 5344 additions and 1001 deletions
Download patch file Download diff file Expand all files Collapse all files

1612
raid/avx2.c
View file

File diff suppressed because it is too large Load diff

262
raid/int.c
View file

@ -61,9 +61,53 @@ void raid_gen1_int64(int nd, size_t size, void **vv)
}
/*
* GEN2 (double parity) 32bit C implementation
* GEN2 (double parity) 32bit C implementation using only Q *= 2.
*/
static __always_inline void raid_gen2_int32_gen(int nd, size_t size, void **vv, int g23)
static __always_inline void raid_gen2_int32_x2(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
uint8_t *q;
int d, l;
size_t i;
const uint32_t poly_32 = raid_poly_32;
uint32_t d0, q0, p0;
uint32_t d1, q1, p1;
l = nd - 1;
p = v[nd];
q = v[nd + 1];
for (i = 0; i < size; i += 8) {
q0 = p0 = v_read32(&v[l][i]);
q1 = p1 = v_read32(&v[l][i + 4]);
for (d = l - 1; d >= 0; --d) {
d0 = v_read32(&v[d][i]);
d1 = v_read32(&v[d][i + 4]);
p0 ^= d0;
p1 ^= d1;
q0 = x2_32(q0, poly_32);
q1 = x2_32(q1, poly_32);
q0 ^= d0;
q1 ^= d1;
}
v_write32(&p[i], p0);
v_write32(&p[i + 4], p1);
v_write32(&q[i], q0);
v_write32(&q[i + 4], q1);
}
}
/*
* GEN2 (double parity) 32bit C implementation using the AES G23 Q transition.
*/
static __always_inline void raid_gen2_int32_g23(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -79,43 +123,76 @@ static __always_inline void raid_gen2_int32_gen(int nd, size_t size, void **vv,
l = nd - 1;
p = v[nd];
q = v[nd + 1];
int g23_start = raid_g23_count(l - 1);
int g23_boundary = raid_g23_boundary(l - 1);
for (i = 0; i < size; i += 8) {
int g23_count = g23_start;
int g23_x3;
int boundary = g23_boundary;
q0 = p0 = v_read32(&v[l][i]);
q1 = p1 = v_read32(&v[l][i + 4]);
for (d = l - 1; d >= 0; --d) {
g23_x3 = 0;
if (g23) {
g23_x3 = --g23_count == 0;
if (g23_x3)
g23_count = 51;
d = l - 1;
for (;;) {
/*
* All transitions above 'boundary' are x2.
*/
for (; d > boundary; --d) {
d0 = v_read32(&v[d][i]);
d1 = v_read32(&v[d][i + 4]);
p0 ^= d0;
p1 ^= d1;
q0 = x2_32(q0, poly_32);
q1 = x2_32(q1, poly_32);
q0 ^= d0;
q1 ^= d1;
}
/*
* boundary == -1 identifies the final x2-only segment.
* At this point all remaining disks have been consumed.
*/
if (boundary < 0)
break;
/*
* d == boundary.
*
* Perform the fixed G23 x3 transition and then add
* the current disk to P and Q.
*/
d0 = v_read32(&v[d][i]);
d1 = v_read32(&v[d][i + 4]);
p0 ^= d0;
p1 ^= d1;
if (g23 && g23_x3) {
qold = q0;
q0 = x2_32(q0, poly_32);
q0 ^= qold;
qold = q1;
q1 = x2_32(q1, poly_32);
q1 ^= qold;
} else {
q0 = x2_32(q0, poly_32);
q1 = x2_32(q1, poly_32);
}
qold = q0;
q0 = x2_32(q0, poly_32);
q0 ^= qold;
qold = q1;
q1 = x2_32(q1, poly_32);
q1 ^= qold;
q0 ^= d0;
q1 ^= d1;
--d;
/*
* G23 boundaries are exactly 51 transitions apart.
* After d = 50 there are no more x3 transitions.
*/
boundary -= 51;
if (boundary < 50)
boundary = -1;
}
v_write32(&p[i], p0);
v_write32(&p[i + 4], p1);
v_write32(&q[i], q0);
@ -125,18 +202,65 @@ static __always_inline void raid_gen2_int32_gen(int nd, size_t size, void **vv,
void raid_gen2_int32_raid(int nd, size_t size, void **vv)
{
raid_gen2_int32_gen(nd, size, vv, 0);
raid_gen2_int32_x2(nd, size, vv);
}
void raid_gen2_int32_aes(int nd, size_t size, void **vv)
{
raid_gen2_int32_gen(nd, size, vv, 1);
if (raid_g23_x2_only(nd))
raid_gen2_int32_x2(nd, size, vv);
else
raid_gen2_int32_g23(nd, size, vv);
}
/*
* GEN2 (double parity) 64bit C implementation
* GEN2 (double parity) 64bit C implementation using only Q *= 2.
*/
static __always_inline void raid_gen2_int64_gen(int nd, size_t size, void **vv, int g23)
static __always_inline void raid_gen2_int64_x2(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
uint8_t *q;
int d, l;
size_t i;
const uint64_t poly_64 = raid_poly_64;
uint64_t d0, q0, p0;
uint64_t d1, q1, p1;
l = nd - 1;
p = v[nd];
q = v[nd + 1];
for (i = 0; i < size; i += 16) {
q0 = p0 = v_read64(&v[l][i]);
q1 = p1 = v_read64(&v[l][i + 8]);
for (d = l - 1; d >= 0; --d) {
d0 = v_read64(&v[d][i]);
d1 = v_read64(&v[d][i + 8]);
p0 ^= d0;
p1 ^= d1;
q0 = x2_64(q0, poly_64);
q1 = x2_64(q1, poly_64);
q0 ^= d0;
q1 ^= d1;
}
v_write64(&p[i], p0);
v_write64(&p[i + 8], p1);
v_write64(&q[i], q0);
v_write64(&q[i + 8], q1);
}
}
/*
* GEN2 (double parity) 64bit C implementation using the AES G23 Q transition.
*/
static __always_inline void raid_gen2_int64_g23(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -152,43 +276,76 @@ static __always_inline void raid_gen2_int64_gen(int nd, size_t size, void **vv,
l = nd - 1;
p = v[nd];
q = v[nd + 1];
int g23_start = raid_g23_count(l - 1);
int g23_boundary = raid_g23_boundary(l - 1);
for (i = 0; i < size; i += 16) {
int g23_count = g23_start;
int g23_x3;
int boundary = g23_boundary;
q0 = p0 = v_read64(&v[l][i]);
q1 = p1 = v_read64(&v[l][i + 8]);
for (d = l - 1; d >= 0; --d) {
g23_x3 = 0;
if (g23) {
g23_x3 = --g23_count == 0;
if (g23_x3)
g23_count = 51;
d = l - 1;
for (;;) {
/*
* All transitions above 'boundary' are x2.
*/
for (; d > boundary; --d) {
d0 = v_read64(&v[d][i]);
d1 = v_read64(&v[d][i + 8]);
p0 ^= d0;
p1 ^= d1;
q0 = x2_64(q0, poly_64);
q1 = x2_64(q1, poly_64);
q0 ^= d0;
q1 ^= d1;
}
/*
* boundary == -1 identifies the final x2-only segment.
* At this point all remaining disks have been consumed.
*/
if (boundary < 0)
break;
/*
* d == boundary.
*
* Perform the fixed G23 x3 transition and then add
* the current disk to P and Q.
*/
d0 = v_read64(&v[d][i]);
d1 = v_read64(&v[d][i + 8]);
p0 ^= d0;
p1 ^= d1;
if (g23 && g23_x3) {
qold = q0;
q0 = x2_64(q0, poly_64);
q0 ^= qold;
qold = q1;
q1 = x2_64(q1, poly_64);
q1 ^= qold;
} else {
q0 = x2_64(q0, poly_64);
q1 = x2_64(q1, poly_64);
}
qold = q0;
q0 = x2_64(q0, poly_64);
q0 ^= qold;
qold = q1;
q1 = x2_64(q1, poly_64);
q1 ^= qold;
q0 ^= d0;
q1 ^= d1;
--d;
/*
* G23 boundaries are exactly 51 transitions apart.
* After d = 50 there are no more x3 transitions.
*/
boundary -= 51;
if (boundary < 50)
boundary = -1;
}
v_write64(&p[i], p0);
v_write64(&p[i + 8], p1);
v_write64(&q[i], q0);
@ -198,17 +355,15 @@ static __always_inline void raid_gen2_int64_gen(int nd, size_t size, void **vv,
void raid_gen2_int64_raid(int nd, size_t size, void **vv)
{
raid_gen2_int64_gen(nd, size, vv, 0);
raid_gen2_int64_x2(nd, size, vv);
}
void raid_gen2_int64_aes(int nd, size_t size, void **vv)
{
raid_gen2_int64_gen(nd, size, vv, 1);
}
void raid_gen2_int8(int nd, size_t size, void **vv)
{
raid_gen_ref(nd, 2, size, vv);
if (raid_g23_x2_only(nd))
raid_gen2_int64_x2(nd, size, vv);
else
raid_gen2_int64_g23(nd, size, vv);
}
/*
@ -720,7 +875,6 @@ void raid_recX_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv)
void raid_register_int(void)
{
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "int8", raid_gen2_int8, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "int8", raid_gen3_int8, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "int8", raid_gen4_int8, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "int8", raid_gen5_int8, RAID_POLY_ANY);

75
raid/internal.h
View file

@ -75,7 +75,15 @@ static __always_inline void * __align_ptr(void *ptr, uintptr_t size)
}
/*
* Initialize the countdown used by the AES G23 Horner recurrence.
* Check if the number of data disks requires only x2 multipliers in AES G23.
*/
static __always_inline int raid_g23_x2_only(int nd)
{
return nd <= 51;
}
/*
* Find the next AES G23 x3 boundary in descending Horner order.
*
* The G23 Q coefficients are generated by multiplying by 2 at every step,
* except at the four boundaries between the five cosets, where the next
@ -92,43 +100,53 @@ static __always_inline void * __align_ptr(void *ptr, uintptr_t size)
*
* i.e. every 51 descending steps.
*
* Rather than comparing d against those four values in the hot loop, we keep
* a countdown. Before processing each Horner step the generator does:
* The generators can process all the x2 transitions between two consecutive
* boundaries in a branch-free loop, and handle the x3 transition separately.
* This function returns the first x3 boundary encountered by a descending
* traversal whose first processed step has index d.
*
* if (--g23_count == 0) {
* Q = 3 * Q;
* g23_count = 51;
* } else {
* Q = 2 * Q;
* }
* For d >= 50, (d - 50) % 51 gives the distance from the previous x3
* boundary in ascending index order. Subtracting this distance from d gives
* that boundary directly:
*
* This function returns the initial countdown for a traversal whose first
* processed step has index d. For d >= 50, (d - 50) % 51 gives the distance
* from the previous x3 boundary, so adding 1 gives the number of decrements
* required before the next descending x3 transition is reached.
* boundary = d - (d - 50) % 51
*
* For d < 50 there is no remaining x3 transition. Returning 51 is safe
* because fewer than 51 Horner steps remain, so the countdown cannot reach
* zero before the traversal ends.
* Since the Horner traversal proceeds toward decreasing indexes, this is also
* the next x3 transition that will be encountered.
*
* For d < 50 there is no remaining x3 transition, so -1 is returned. This
* value can also be used as the lower bound of the final x2-only segment,
* allowing all remaining steps down to d = 0 to be processed without a
* special G23 test.
*
* Examples:
*
* d = 203 -> 1 -> x3 immediately
* d = 204 -> 2 -> x2 at 204, x3 at 203
* d = 250 -> 48 -> x3 at 203
* d = 152 -> 1 -> x3 immediately
* d = 50 -> 1 -> x3 immediately
* d = 49 -> 51 -> no further x3 transition
* d = 249 -> 203 -> x2 down to 204, then x3 at 203
* d = 203 -> 203 -> x3 immediately
* d = 202 -> 152 -> x2 down to 153, then x3 at 152
* d = 152 -> 152 -> x3 immediately
* d = 151 -> 101 -> x2 down to 102, then x3 at 101
* d = 101 -> 101 -> x3 immediately
* d = 100 -> 50 -> x2 down to 51, then x3 at 50
* d = 50 -> 50 -> x3 immediately
* d = 49 -> -1 -> no further x3 transition
*
* The returned value is normally computed once as g23_start and copied into
* g23_count at the beginning of each independent Horner traversal.
* After processing a boundary, the following one in descending order is
* exactly 51 positions lower. Thus generators can continue with:
*
* boundary -= 51;
* if (boundary < 50)
* boundary = -1;
*
* The returned value is normally computed once before each independent
* Horner traversal and then advanced between the fixed G23 segments.
*/
static __always_inline int raid_g23_count(int d)
static __always_inline int raid_g23_boundary(int d)
{
if (d < 50)
return 51;
return -1;
return (d - 50) % 51 + 1;
return d - (d - 50) % 51;
}
/*
@ -152,7 +170,6 @@ void raid_gen1_int64(int nd, size_t size, void **vv);
void raid_gen1_sse2(int nd, size_t size, void **vv);
void raid_gen1_avx2(int nd, size_t size, void **vv);
void raid_gen1_avx512bw(int nd, size_t size, void **vv);
void raid_gen2_int8(int nd, size_t size, void **vv);
void raid_gen2_int32_raid(int nd, size_t size, void **vv);
void raid_gen2_int32_aes(int nd, size_t size, void **vv);
void raid_gen2_int64_raid(int nd, size_t size, void **vv);
@ -201,6 +218,7 @@ void raid_gen4_avx2gfni_aes(int nd, size_t size, void **vv);
void raid_gen4_avx512gfni_aes(int nd, size_t size, void **vv);
void raid_gen5_int8(int nd, size_t size, void **vv);
void raid_gen5_ssse3_raid(int nd, size_t size, void **vv);
void raid_gen5_ssse3_aes(int nd, size_t size, void **vv);
void raid_gen5_ssse3ext_raid(int nd, size_t size, void **vv);
void raid_gen5_ssse3ext_aes(int nd, size_t size, void **vv);
void raid_gen5_avx2ext_raid(int nd, size_t size, void **vv);
@ -212,6 +230,7 @@ void raid_gen5_avx2gfni_aes(int nd, size_t size, void **vv);
void raid_gen5_avx512gfni_aes(int nd, size_t size, void **vv);
void raid_gen6_int8(int nd, size_t size, void **vv);
void raid_gen6_ssse3_raid(int nd, size_t size, void **vv);
void raid_gen6_ssse3_aes(int nd, size_t size, void **vv);
void raid_gen6_ssse3ext_raid(int nd, size_t size, void **vv);
void raid_gen6_ssse3ext_aes(int nd, size_t size, void **vv);
void raid_gen6_avx2ext_raid(int nd, size_t size, void **vv);

819
raid/neon.c
View file

File diff suppressed because it is too large Load diff

727
raid/neon32.c
View file

@ -63,10 +63,9 @@ void raid_gen1_neon32(int nd, size_t size, void **vv)
}
/*
* GEN2 Cauchy AArch32 NEON implementation using the active Q transition.
* GEN2 Cauchy AArch32 NEON implementation using only Q *= 2.
*/
static __always_inline void raid_gen2_neon32_gen(int nd, size_t size,
void **vv, int g23)
static __always_inline void raid_gen2_neon32_x2(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -86,12 +85,8 @@ static __always_inline void raid_gen2_neon32_gen(int nd, size_t size,
:
: "Q" (gfconst16.poly[0])
);
int g23_start = raid_g23_count(l - 1);
for (i = 0; i < size; i += 32) {
int g23_count = g23_start;
int g23_x3;
asm volatile (
"vld1.8 {q0}, %0\n"
"vld1.8 {q1}, %1\n"
@ -102,28 +97,19 @@ static __always_inline void raid_gen2_neon32_gen(int nd, size_t size,
);
for (d = l - 1; d >= 0; --d) {
g23_x3 = 0;
if (g23) {
g23_x3 = --g23_count == 0;
if (g23_x3)
g23_count = 51;
}
if (g23 && g23_x3) {
asm volatile (
"vmov q10, q2\n"
"vmov q11, q3\n"
);
}
asm volatile (
/* double Q0-Q1 */
"vshr.s8 q8, q2, #7\n"
"vshl.i8 q2, q2, #1\n"
"vand q8, q8, q14\n"
"veor q2, q2, q8\n"
"vshr.s8 q8, q3, #7\n"
"vshl.i8 q3, q3, #1\n"
"vand q8, q8, q14\n"
"veor q3, q3, q8\n"
/* load and XOR */
"vld1.8 {q12}, %0\n"
"vld1.8 {q13}, %1\n"
"veor q0, q0, q12\n"
@ -133,12 +119,143 @@ static __always_inline void raid_gen2_neon32_gen(int nd, size_t size,
:
: "Q" (v[d][i]), "Q" (v[d][i + 16])
);
if (g23 && g23_x3) {
}
asm volatile (
"vst1.8 {q0}, %0\n"
"vst1.8 {q1}, %1\n"
"vst1.8 {q2}, %2\n"
"vst1.8 {q3}, %3\n"
: "=Q" (p[i]), "=Q" (p[i + 16]),
"=Q" (q[i]), "=Q" (q[i + 16])
);
}
raid_neon32_end();
}
/*
* GEN2 Cauchy AArch32 NEON implementation using the AES G23 Q transition.
*/
static __always_inline void raid_gen2_neon32_g23(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
uint8_t *q;
int d, l;
size_t i;
l = nd - 1;
p = v[nd];
q = v[nd + 1];
int g23_boundary = raid_g23_boundary(l - 1);
raid_neon32_begin();
/* q14 contains the active reduction polynomial. */
asm volatile (
"vld1.8 {q14}, %0\n"
:
: "Q" (gfconst16.poly[0])
);
for (i = 0; i < size; i += 32) {
int boundary = g23_boundary;
asm volatile (
"vld1.8 {q0}, %0\n"
"vld1.8 {q1}, %1\n"
"vmov q2, q0\n"
"vmov q3, q1\n"
:
: "Q" (v[l][i]), "Q" (v[l][i + 16])
);
d = l - 1;
for (;;) {
/*
* All transitions above 'boundary' are x2.
*/
for (; d > boundary; --d) {
asm volatile (
"veor q2, q2, q10\n"
"veor q3, q3, q11\n"
/* double Q0-Q1 */
"vshr.s8 q8, q2, #7\n"
"vshl.i8 q2, q2, #1\n"
"vand q8, q8, q14\n"
"veor q2, q2, q8\n"
"vshr.s8 q8, q3, #7\n"
"vshl.i8 q3, q3, #1\n"
"vand q8, q8, q14\n"
"veor q3, q3, q8\n"
/* load and XOR */
"vld1.8 {q12}, %0\n"
"vld1.8 {q13}, %1\n"
"veor q0, q0, q12\n"
"veor q1, q1, q13\n"
"veor q2, q2, q12\n"
"veor q3, q3, q13\n"
:
: "Q" (v[d][i]), "Q" (v[d][i + 16])
);
}
/*
* boundary == -1 identifies the final x2-only segment.
* At this point all remaining disks have been consumed.
*/
if (boundary < 0)
break;
/*
* d == boundary.
*
* Perform the fixed G23 x3 transition and then add
* the current disk to P and Q.
*/
asm volatile (
/* preserve Q for Q *= 3 */
"vmov q10, q2\n"
"vmov q11, q3\n"
/* double Q0-Q1 */
"vshr.s8 q8, q2, #7\n"
"vshl.i8 q2, q2, #1\n"
"vand q8, q8, q14\n"
"veor q2, q2, q8\n"
"vshr.s8 q8, q3, #7\n"
"vshl.i8 q3, q3, #1\n"
"vand q8, q8, q14\n"
"veor q3, q3, q8\n"
/* load and XOR */
"vld1.8 {q12}, %0\n"
"vld1.8 {q13}, %1\n"
"veor q0, q0, q12\n"
"veor q1, q1, q13\n"
"veor q2, q2, q12\n"
"veor q3, q3, q13\n"
/* complete Q *= 3 */
"veor q2, q2, q10\n"
"veor q3, q3, q11\n"
:
: "Q" (v[d][i]), "Q" (v[d][i + 16])
);
--d;
/*
* G23 boundaries are exactly 51 transitions apart.
* After d = 50 there are no more x3 transitions.
*/
boundary -= 51;
if (boundary < 50)
boundary = -1;
}
asm volatile (
@ -156,12 +273,15 @@ static __always_inline void raid_gen2_neon32_gen(int nd, size_t size,
void raid_gen2_neon32_raid(int nd, size_t size, void **vv)
{
raid_gen2_neon32_gen(nd, size, vv, 0);
raid_gen2_neon32_x2(nd, size, vv);
}
void raid_gen2_neon32_aes(int nd, size_t size, void **vv)
{
raid_gen2_neon32_gen(nd, size, vv, 1);
if (raid_g23_x2_only(nd))
raid_gen2_neon32_x2(nd, size, vv);
else
raid_gen2_neon32_g23(nd, size, vv);
}
/*
@ -252,10 +372,10 @@ void raid_genz_neon32_raid(int nd, size_t size, void **vv)
}
/*
* GENX AArch32 NEON implementation.
* GENX AArch32 NEON implementation using only Q *= 2.
*/
static __always_inline void raid_genX_neon32(int nd, size_t size,
void **vv, int np, int g23)
static __always_inline void raid_genX_neon32_x2(int nd, size_t size,
void **vv, int np)
{
uint8_t **v = (uint8_t **)vv;
size_t i;
@ -269,7 +389,6 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
memcpy(v[1 + d], v[0], size);
return;
}
int g23_start = raid_g23_count(l - 1);
raid_neon32_begin();
@ -281,9 +400,6 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
);
for (i = 0; i < size; i += 16) {
int g23_count = g23_start;
int g23_x3;
/* last disk without the generator multiplication */
asm volatile (
"vld1.8 {q0}, %0\n"
@ -311,7 +427,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
"veor q2, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][1][0][0]),
"Q" (raid_gfcauchypshufb[l][1][1][0])
"Q" (raid_gfcauchypshufb[l][1][1][0])
);
}
@ -328,7 +444,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
"veor q3, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][2][0][0]),
"Q" (raid_gfcauchypshufb[l][2][1][0])
"Q" (raid_gfcauchypshufb[l][2][1][0])
);
}
@ -345,7 +461,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
"veor q4, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][3][0][0]),
"Q" (raid_gfcauchypshufb[l][3][1][0])
"Q" (raid_gfcauchypshufb[l][3][1][0])
);
}
@ -362,40 +478,26 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
"veor q5, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][4][0][0]),
"Q" (raid_gfcauchypshufb[l][4][1][0])
"Q" (raid_gfcauchypshufb[l][4][1][0])
);
}
/* intermediate disks */
for (d = l - 1; d > 0; --d) {
g23_x3 = 0;
if (g23) {
g23_x3 = --g23_count == 0;
if (g23_x3)
g23_count = 51;
}
if (g23 && g23_x3) {
asm volatile (
"vmov q13, q1\n"
);
}
asm volatile (
/* Q *= 2 */
"vshr.s8 q11, q1, #7\n"
"vshl.i8 q1, q1, #1\n"
"vand q11, q11, q14\n"
"veor q1, q1, q11\n"
/* load disk and update P/Q */
"vld1.8 {q6}, %0\n"
"veor q0, q0, q6\n"
"veor q1, q1, q6\n"
:
: "Q" (v[d][i])
);
if (g23 && g23_x3) {
asm volatile (
"veor q1, q1, q13\n"
);
}
if (np >= 3) {
asm volatile (
@ -403,6 +505,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
"vand q7, q6, q15\n"
"vand q8, q8, q15\n"
);
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
@ -416,7 +519,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
"veor q2, q2, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][1][0][0]),
"Q" (raid_gfcauchypshufb[d][1][1][0])
"Q" (raid_gfcauchypshufb[d][1][1][0])
);
}
@ -432,7 +535,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
"veor q3, q3, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][2][0][0]),
"Q" (raid_gfcauchypshufb[d][2][1][0])
"Q" (raid_gfcauchypshufb[d][2][1][0])
);
}
@ -448,7 +551,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
"veor q4, q4, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][3][0][0]),
"Q" (raid_gfcauchypshufb[d][3][1][0])
"Q" (raid_gfcauchypshufb[d][3][1][0])
);
}
@ -464,94 +567,484 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
"veor q5, q5, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][4][0][0]),
"Q" (raid_gfcauchypshufb[d][4][1][0])
"Q" (raid_gfcauchypshufb[d][4][1][0])
);
}
}
/* first disk with all coefficients at 1 */
g23_x3 = 0;
if (g23) {
g23_x3 = --g23_count == 0;
if (g23_x3)
g23_count = 51;
}
if (g23 && g23_x3) {
asm volatile (
"vmov q13, q1\n"
);
}
asm volatile (
"vshr.s8 q11, q1, #7\n"
"vshl.i8 q1, q1, #1\n"
"vand q11, q11, q14\n"
"veor q1, q1, q11\n"
"vld1.8 {q6}, %0\n"
"veor q0, q0, q6\n"
"veor q1, q1, q6\n"
:
: "Q" (v[0][i])
);
if (g23 && g23_x3) {
asm volatile (
"veor q1, q1, q13\n"
);
}
if (np >= 3) {
asm volatile (
"veor q2, q2, q6\n"
);
}
if (np >= 4) {
asm volatile (
"veor q3, q3, q6\n"
);
}
if (np >= 5) {
asm volatile (
"veor q4, q4, q6\n"
);
}
if (np >= 6) {
asm volatile (
"veor q5, q5, q6\n"
);
}
if (np >= 3)
asm volatile ("veor q2, q2, q6\n");
if (np >= 4)
asm volatile ("veor q3, q3, q6\n");
if (np >= 5)
asm volatile ("veor q4, q4, q6\n");
if (np >= 6)
asm volatile ("veor q5, q5, q6\n");
/* write parity in increasing order */
asm volatile (
"vst1.8 {q0}, %0\n"
"vst1.8 {q1}, %1\n"
: "=Q" (v[nd][i]),
"=Q" (v[nd + 1][i])
"=Q" (v[nd + 1][i])
);
if (np >= 3)
asm volatile ("vst1.8 {q2}, %0\n" : "=Q" (v[nd + 2][i]));
if (np >= 4)
asm volatile ("vst1.8 {q3}, %0\n" : "=Q" (v[nd + 3][i]));
if (np >= 5)
asm volatile ("vst1.8 {q4}, %0\n" : "=Q" (v[nd + 4][i]));
if (np >= 6)
asm volatile ("vst1.8 {q5}, %0\n" : "=Q" (v[nd + 5][i]));
}
raid_neon32_end();
}
/*
* GENX AArch32 NEON implementation using the AES G23 Q transition.
*/
static __always_inline void raid_genX_neon32_g23(int nd, size_t size,
void **vv, int np)
{
uint8_t **v = (uint8_t **)vv;
size_t i;
int d, l;
l = nd - 1;
/* special case with only one data disk */
if (l == 0) {
for (d = 0; d < np; ++d)
memcpy(v[1 + d], v[0], size);
return;
}
int g23_boundary = raid_g23_boundary(l - 1);
raid_neon32_begin();
asm volatile (
"vld1.8 {q14}, %0\n"
"vld1.8 {q15}, %1\n"
:
: "Q" (gfconst16.poly[0]), "Q" (gfconst16.low4[0])
);
for (i = 0; i < size; i += 16) {
int boundary = g23_boundary;
/* last disk without the generator multiplication */
asm volatile (
"vld1.8 {q0}, %0\n"
"vld1.8 {q1}, %0\n"
:
: "Q" (v[l][i])
);
if (np >= 3) {
asm volatile (
"vst1.8 {q2}, %0\n"
: "=Q" (v[nd + 2][i])
"vshr.u8 q8, q0, #4\n"
"vand q7, q0, q15\n"
"vand q8, q8, q15\n"
);
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q2, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][1][0][0]),
"Q" (raid_gfcauchypshufb[l][1][1][0])
);
}
if (np >= 4) {
asm volatile (
"vst1.8 {q3}, %0\n"
: "=Q" (v[nd + 3][i])
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q3, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][2][0][0]),
"Q" (raid_gfcauchypshufb[l][2][1][0])
);
}
if (np >= 5) {
asm volatile (
"vst1.8 {q4}, %0\n"
: "=Q" (v[nd + 4][i])
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q4, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][3][0][0]),
"Q" (raid_gfcauchypshufb[l][3][1][0])
);
}
if (np >= 6) {
asm volatile (
"vst1.8 {q5}, %0\n"
: "=Q" (v[nd + 5][i])
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q5, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][4][0][0]),
"Q" (raid_gfcauchypshufb[l][4][1][0])
);
}
d = l - 1;
while (boundary >= 0) {
/*
* Everything above the next G23 boundary is Q *= 2.
*/
for (; d > boundary; --d) {
asm volatile (
"vshr.s8 q11, q1, #7\n"
"vshl.i8 q1, q1, #1\n"
"vand q11, q11, q14\n"
"veor q1, q1, q11\n"
"vld1.8 {q6}, %0\n"
"veor q0, q0, q6\n"
"veor q1, q1, q6\n"
:
: "Q" (v[d][i])
);
if (np >= 3) {
asm volatile (
"vshr.u8 q8, q6, #4\n"
"vand q7, q6, q15\n"
"vand q8, q8, q15\n"
);
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q2, q2, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][1][0][0]),
"Q" (raid_gfcauchypshufb[d][1][1][0])
);
}
if (np >= 4) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q3, q3, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][2][0][0]),
"Q" (raid_gfcauchypshufb[d][2][1][0])
);
}
if (np >= 5) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q4, q4, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][3][0][0]),
"Q" (raid_gfcauchypshufb[d][3][1][0])
);
}
if (np >= 6) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q5, q5, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][4][0][0]),
"Q" (raid_gfcauchypshufb[d][4][1][0])
);
}
}
/*
* d == boundary: Q *= 3.
*
* q13 is otherwise unused here, so preserve the old Q
* there and XOR it back after the normal x2 transition.
*/
asm volatile (
"vmov q13, q1\n"
"vshr.s8 q11, q1, #7\n"
"vshl.i8 q1, q1, #1\n"
"vand q11, q11, q14\n"
"veor q1, q1, q11\n"
"vld1.8 {q6}, %0\n"
"veor q0, q0, q6\n"
"veor q1, q1, q6\n"
/* complete Q *= 3 */
"veor q1, q1, q13\n"
:
: "Q" (v[d][i])
);
if (np >= 3) {
asm volatile (
"vshr.u8 q8, q6, #4\n"
"vand q7, q6, q15\n"
"vand q8, q8, q15\n"
);
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q2, q2, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][1][0][0]),
"Q" (raid_gfcauchypshufb[d][1][1][0])
);
}
if (np >= 4) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q3, q3, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][2][0][0]),
"Q" (raid_gfcauchypshufb[d][2][1][0])
);
}
if (np >= 5) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q4, q4, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][3][0][0]),
"Q" (raid_gfcauchypshufb[d][3][1][0])
);
}
if (np >= 6) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q5, q5, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][4][0][0]),
"Q" (raid_gfcauchypshufb[d][4][1][0])
);
}
--d;
boundary -= 51;
if (boundary < 50)
boundary = -1;
}
/*
* Remaining intermediate disks contain no more G23
* boundaries, so this loop is pure x2.
*/
for (; d > 0; --d) {
asm volatile (
"vshr.s8 q11, q1, #7\n"
"vshl.i8 q1, q1, #1\n"
"vand q11, q11, q14\n"
"veor q1, q1, q11\n"
"vld1.8 {q6}, %0\n"
"veor q0, q0, q6\n"
"veor q1, q1, q6\n"
:
: "Q" (v[d][i])
);
if (np >= 3) {
asm volatile (
"vshr.u8 q8, q6, #4\n"
"vand q7, q6, q15\n"
"vand q8, q8, q15\n"
);
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q2, q2, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][1][0][0]),
"Q" (raid_gfcauchypshufb[d][1][1][0])
);
}
if (np >= 4) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q3, q3, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][2][0][0]),
"Q" (raid_gfcauchypshufb[d][2][1][0])
);
}
if (np >= 5) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q4, q4, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][3][0][0]),
"Q" (raid_gfcauchypshufb[d][3][1][0])
);
}
if (np >= 6) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q5, q5, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][4][0][0]),
"Q" (raid_gfcauchypshufb[d][4][1][0])
);
}
}
/*
* D0 always follows Q *= 2. The lowest G23 boundary is 50.
*/
asm volatile (
"vshr.s8 q11, q1, #7\n"
"vshl.i8 q1, q1, #1\n"
"vand q11, q11, q14\n"
"veor q1, q1, q11\n"
"vld1.8 {q6}, %0\n"
"veor q0, q0, q6\n"
"veor q1, q1, q6\n"
:
: "Q" (v[0][i])
);
if (np >= 3)
asm volatile ("veor q2, q2, q6\n");
if (np >= 4)
asm volatile ("veor q3, q3, q6\n");
if (np >= 5)
asm volatile ("veor q4, q4, q6\n");
if (np >= 6)
asm volatile ("veor q5, q5, q6\n");
/* write parity in increasing order */
asm volatile (
"vst1.8 {q0}, %0\n"
"vst1.8 {q1}, %1\n"
: "=Q" (v[nd][i]),
"=Q" (v[nd + 1][i])
);
if (np >= 3)
asm volatile ("vst1.8 {q2}, %0\n" : "=Q" (v[nd + 2][i]));
if (np >= 4)
asm volatile ("vst1.8 {q3}, %0\n" : "=Q" (v[nd + 3][i]));
if (np >= 5)
asm volatile ("vst1.8 {q4}, %0\n" : "=Q" (v[nd + 4][i]));
if (np >= 6)
asm volatile ("vst1.8 {q5}, %0\n" : "=Q" (v[nd + 5][i]));
}
raid_neon32_end();
@ -562,12 +1055,15 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
*/
void raid_gen3_neon32_raid(int nd, size_t size, void **vv)
{
raid_genX_neon32(nd, size, vv, 3, 0);
raid_genX_neon32_x2(nd, size, vv, 3);
}
void raid_gen3_neon32_aes(int nd, size_t size, void **vv)
{
raid_genX_neon32(nd, size, vv, 3, 1);
if (raid_g23_x2_only(nd))
raid_genX_neon32_x2(nd, size, vv, 3);
else
raid_genX_neon32_g23(nd, size, vv, 3);
}
/*
@ -575,12 +1071,15 @@ void raid_gen3_neon32_aes(int nd, size_t size, void **vv)
*/
void raid_gen4_neon32_raid(int nd, size_t size, void **vv)
{
raid_genX_neon32(nd, size, vv, 4, 0);
raid_genX_neon32_x2(nd, size, vv, 4);
}
void raid_gen4_neon32_aes(int nd, size_t size, void **vv)
{
raid_genX_neon32(nd, size, vv, 4, 1);
if (raid_g23_x2_only(nd))
raid_genX_neon32_x2(nd, size, vv, 4);
else
raid_genX_neon32_g23(nd, size, vv, 4);
}
/*
@ -588,12 +1087,15 @@ void raid_gen4_neon32_aes(int nd, size_t size, void **vv)
*/
void raid_gen5_neon32_raid(int nd, size_t size, void **vv)
{
raid_genX_neon32(nd, size, vv, 5, 0);
raid_genX_neon32_x2(nd, size, vv, 5);
}
void raid_gen5_neon32_aes(int nd, size_t size, void **vv)
{
raid_genX_neon32(nd, size, vv, 5, 1);
if (raid_g23_x2_only(nd))
raid_genX_neon32_x2(nd, size, vv, 5);
else
raid_genX_neon32_g23(nd, size, vv, 5);
}
/*
@ -601,12 +1103,15 @@ void raid_gen5_neon32_aes(int nd, size_t size, void **vv)
*/
void raid_gen6_neon32_raid(int nd, size_t size, void **vv)
{
raid_genX_neon32(nd, size, vv, 6, 0);
raid_genX_neon32_x2(nd, size, vv, 6);
}
void raid_gen6_neon32_aes(int nd, size_t size, void **vv)
{
raid_genX_neon32(nd, size, vv, 6, 1);
if (raid_g23_x2_only(nd))
raid_genX_neon32_x2(nd, size, vv, 6);
else
raid_genX_neon32_g23(nd, size, vv, 6);
}
/*

558
raid/sse2.c
View file

@ -137,9 +137,9 @@ void raid_gen1_sse2(int nd, size_t size, void **vv)
}
/*
* GEN2 Cauchy SSE2 implementation using the active Q transition
* GEN2 Cauchy SSE2 implementation using only Q *= 2.
*/
static __always_inline void raid_gen2_sse2_gen(int nd, size_t size, void **vv, int g23)
static __always_inline void raid_gen2_sse2_x2(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -154,53 +154,25 @@ static __always_inline void raid_gen2_sse2_gen(int nd, size_t size, void **vv, i
raid_sse_begin();
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
int g23_start = raid_g23_count(l - 1);
for (i = 0; i < size; i += 32) {
int g23_count = g23_start;
int g23_x3;
asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i]));
asm volatile ("movdqa %0,%%xmm1" : : "m" (v[l][i + 16]));
asm volatile ("movdqa %xmm0,%xmm2");
asm volatile ("movdqa %xmm1,%xmm3");
for (d = l - 1; d >= 0; --d) {
g23_x3 = 0;
if (g23) {
g23_x3 = --g23_count == 0;
if (g23_x3)
g23_count = 51;
}
/* scale Q before adding the current disk */
if (g23 && g23_x3) {
asm volatile ("movdqa %xmm2,%xmm4");
asm volatile ("movdqa %xmm3,%xmm5");
asm volatile ("pxor %xmm6,%xmm6");
asm volatile ("pcmpgtb %xmm4,%xmm6");
asm volatile ("paddb %xmm2,%xmm2");
asm volatile ("paddb %xmm3,%xmm3");
asm volatile ("pxor %xmm4,%xmm2");
asm volatile ("pxor %xmm4,%xmm4");
asm volatile ("pcmpgtb %xmm5,%xmm4");
asm volatile ("pand %xmm7,%xmm6");
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pxor %xmm6,%xmm2");
asm volatile ("pxor %xmm4,%xmm3");
asm volatile ("pxor %xmm5,%xmm3");
} else {
asm volatile ("pxor %xmm4,%xmm4");
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pcmpgtb %xmm2,%xmm4");
asm volatile ("pcmpgtb %xmm3,%xmm5");
asm volatile ("paddb %xmm2,%xmm2");
asm volatile ("paddb %xmm3,%xmm3");
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("pxor %xmm4,%xmm2");
asm volatile ("pxor %xmm5,%xmm3");
}
asm volatile ("pxor %xmm4,%xmm4");
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pcmpgtb %xmm2,%xmm4");
asm volatile ("pcmpgtb %xmm3,%xmm5");
asm volatile ("paddb %xmm2,%xmm2");
asm volatile ("paddb %xmm3,%xmm3");
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("pxor %xmm4,%xmm2");
asm volatile ("pxor %xmm5,%xmm3");
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
asm volatile ("movdqa %0,%%xmm5" : : "m" (v[d][i + 16]));
@ -219,23 +191,137 @@ static __always_inline void raid_gen2_sse2_gen(int nd, size_t size, void **vv, i
raid_sse_end();
}
/*
* GEN2 Cauchy SSE2 implementation using the AES G23 Q transition.
*/
static __always_inline void raid_gen2_sse2_g23(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
uint8_t *q;
int d, l;
size_t i;
l = nd - 1;
p = v[nd];
q = v[nd + 1];
int g23_boundary = raid_g23_boundary(l - 1);
raid_sse_begin();
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
for (i = 0; i < size; i += 32) {
int boundary = g23_boundary;
asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i]));
asm volatile ("movdqa %0,%%xmm1" : : "m" (v[l][i + 16]));
asm volatile ("movdqa %xmm0,%xmm2");
asm volatile ("movdqa %xmm1,%xmm3");
d = l - 1;
for (;;) {
/*
* All transitions above 'boundary' are x2.
*/
for (; d > boundary; --d) {
/* scale Q before adding the current disk */
asm volatile ("pxor %xmm4,%xmm4");
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pcmpgtb %xmm2,%xmm4");
asm volatile ("pcmpgtb %xmm3,%xmm5");
asm volatile ("paddb %xmm2,%xmm2");
asm volatile ("paddb %xmm3,%xmm3");
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("pxor %xmm4,%xmm2");
asm volatile ("pxor %xmm5,%xmm3");
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
asm volatile ("movdqa %0,%%xmm5" : : "m" (v[d][i + 16]));
asm volatile ("pxor %xmm4,%xmm0");
asm volatile ("pxor %xmm5,%xmm1");
asm volatile ("pxor %xmm4,%xmm2");
asm volatile ("pxor %xmm5,%xmm3");
}
/*
* boundary == -1 identifies the final x2-only segment.
* At this point all remaining disks have been consumed.
*/
if (boundary < 0)
break;
/*
* d == boundary.
*
* Perform the fixed G23 x3 transition and then add
* the current disk to P and Q.
*/
asm volatile ("movdqa %xmm2,%xmm4");
asm volatile ("movdqa %xmm3,%xmm5");
asm volatile ("pxor %xmm6,%xmm6");
asm volatile ("pcmpgtb %xmm4,%xmm6");
asm volatile ("paddb %xmm2,%xmm2");
asm volatile ("paddb %xmm3,%xmm3");
asm volatile ("pxor %xmm4,%xmm2");
asm volatile ("pxor %xmm4,%xmm4");
asm volatile ("pcmpgtb %xmm5,%xmm4");
asm volatile ("pand %xmm7,%xmm6");
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pxor %xmm6,%xmm2");
asm volatile ("pxor %xmm4,%xmm3");
asm volatile ("pxor %xmm5,%xmm3");
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
asm volatile ("movdqa %0,%%xmm5" : : "m" (v[d][i + 16]));
asm volatile ("pxor %xmm4,%xmm0");
asm volatile ("pxor %xmm5,%xmm1");
asm volatile ("pxor %xmm4,%xmm2");
asm volatile ("pxor %xmm5,%xmm3");
--d;
/*
* G23 boundaries are exactly 51 transitions apart.
* After d = 50 there are no more x3 transitions.
*/
boundary -= 51;
if (boundary < 50)
boundary = -1;
}
asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i]));
asm volatile ("movntdq %%xmm1,%0" : "=m" (p[i + 16]));
asm volatile ("movntdq %%xmm2,%0" : "=m" (q[i]));
asm volatile ("movntdq %%xmm3,%0" : "=m" (q[i + 16]));
}
raid_sse_end();
}
void raid_gen2_sse2_raid(int nd, size_t size, void **vv)
{
raid_gen2_sse2_gen(nd, size, vv, 0);
raid_gen2_sse2_x2(nd, size, vv);
}
void raid_gen2_sse2_aes(int nd, size_t size, void **vv)
{
raid_gen2_sse2_gen(nd, size, vv, 1);
if (raid_g23_x2_only(nd))
raid_gen2_sse2_x2(nd, size, vv);
else
raid_gen2_sse2_g23(nd, size, vv);
}
#ifdef CONFIG_X86_64
/*
* GEN2 Cauchy SSE2 implementation using the active Q transition
* GEN2 Cauchy SSE2 implementation using only Q *= 2.
*
* Note that it uses 16 registers, meaning that x64 is required.
*/
static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv, int g23)
static __always_inline void raid_gen2_sse2ext_x2(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -250,13 +336,8 @@ static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv
raid_sse_begin();
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.poly[0]));
int g23_start = raid_g23_count(l - 1);
for (i = 0; i < size; i += 64) {
int g23_count = g23_start;
int g23_x3_d;
int g23_x3_d1;
asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i]));
asm volatile ("movdqa %0,%%xmm1" : : "m" (v[l][i + 16]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (v[l][i + 32]));
@ -268,41 +349,22 @@ static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv
/* process two disks per iteration */
for (d = l - 1; d >= 1; d -= 2) {
g23_x3_d = 0;
g23_x3_d1 = 0;
if (g23) {
g23_x3_d = --g23_count == 0;
if (g23_x3_d)
g23_count = 51;
g23_x3_d1 = --g23_count == 0;
if (g23_x3_d1)
g23_count = 51;
}
/* lane 0 */
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i]));
if (g23 && g23_x3_d)
asm volatile ("movdqa %xmm4,%xmm10");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm4,%xmm11");
asm volatile ("paddb %xmm4,%xmm4");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm4");
if (g23 && g23_x3_d)
asm volatile ("pxor %xmm10,%xmm4");
asm volatile ("pxor %xmm8,%xmm4");
if (g23 && g23_x3_d1)
asm volatile ("movdqa %xmm4,%xmm10");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm4,%xmm11");
asm volatile ("paddb %xmm4,%xmm4");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm4");
if (g23 && g23_x3_d1)
asm volatile ("pxor %xmm10,%xmm4");
asm volatile ("pxor %xmm9,%xmm4");
asm volatile ("pxor %xmm8,%xmm0");
@ -312,26 +374,18 @@ static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i + 16]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i + 16]));
if (g23 && g23_x3_d)
asm volatile ("movdqa %xmm5,%xmm10");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm5,%xmm11");
asm volatile ("paddb %xmm5,%xmm5");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm5");
if (g23 && g23_x3_d)
asm volatile ("pxor %xmm10,%xmm5");
asm volatile ("pxor %xmm8,%xmm5");
if (g23 && g23_x3_d1)
asm volatile ("movdqa %xmm5,%xmm10");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm5,%xmm11");
asm volatile ("paddb %xmm5,%xmm5");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm5");
if (g23 && g23_x3_d1)
asm volatile ("pxor %xmm10,%xmm5");
asm volatile ("pxor %xmm9,%xmm5");
asm volatile ("pxor %xmm8,%xmm1");
@ -341,26 +395,18 @@ static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i + 32]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i + 32]));
if (g23 && g23_x3_d)
asm volatile ("movdqa %xmm6,%xmm10");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm6,%xmm11");
asm volatile ("paddb %xmm6,%xmm6");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm6");
if (g23 && g23_x3_d)
asm volatile ("pxor %xmm10,%xmm6");
asm volatile ("pxor %xmm8,%xmm6");
if (g23 && g23_x3_d1)
asm volatile ("movdqa %xmm6,%xmm10");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm6,%xmm11");
asm volatile ("paddb %xmm6,%xmm6");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm6");
if (g23 && g23_x3_d1)
asm volatile ("pxor %xmm10,%xmm6");
asm volatile ("pxor %xmm9,%xmm6");
asm volatile ("pxor %xmm8,%xmm2");
@ -370,26 +416,18 @@ static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i + 48]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i + 48]));
if (g23 && g23_x3_d)
asm volatile ("movdqa %xmm7,%xmm10");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm7,%xmm11");
asm volatile ("paddb %xmm7,%xmm7");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm7");
if (g23 && g23_x3_d)
asm volatile ("pxor %xmm10,%xmm7");
asm volatile ("pxor %xmm8,%xmm7");
if (g23 && g23_x3_d1)
asm volatile ("movdqa %xmm7,%xmm10");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm7,%xmm11");
asm volatile ("paddb %xmm7,%xmm7");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm7");
if (g23 && g23_x3_d1)
asm volatile ("pxor %xmm10,%xmm7");
asm volatile ("pxor %xmm9,%xmm7");
asm volatile ("pxor %xmm8,%xmm3");
@ -398,66 +436,26 @@ static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv
/* single remaining disk */
if (d == 0) {
int g23_x3;
g23_x3 = 0;
if (g23) {
g23_x3 = --g23_count == 0;
if (g23_x3)
g23_count = 51;
}
if (g23 && g23_x3) {
asm volatile ("movdqa %xmm4,%xmm8");
asm volatile ("movdqa %xmm5,%xmm9");
asm volatile ("movdqa %xmm6,%xmm10");
asm volatile ("movdqa %xmm7,%xmm11");
asm volatile ("pxor %xmm12,%xmm12");
asm volatile ("pxor %xmm13,%xmm13");
asm volatile ("pxor %xmm14,%xmm14");
asm volatile ("pcmpgtb %xmm8,%xmm12");
asm volatile ("pcmpgtb %xmm9,%xmm13");
asm volatile ("pcmpgtb %xmm10,%xmm14");
asm volatile ("paddb %xmm4,%xmm4");
asm volatile ("paddb %xmm5,%xmm5");
asm volatile ("paddb %xmm6,%xmm6");
asm volatile ("paddb %xmm7,%xmm7");
asm volatile ("pxor %xmm8,%xmm4");
asm volatile ("pxor %xmm8,%xmm8");
asm volatile ("pcmpgtb %xmm11,%xmm8");
asm volatile ("pand %xmm15,%xmm12");
asm volatile ("pand %xmm15,%xmm13");
asm volatile ("pand %xmm15,%xmm14");
asm volatile ("pand %xmm15,%xmm8");
asm volatile ("pxor %xmm12,%xmm4");
asm volatile ("pxor %xmm13,%xmm5");
asm volatile ("pxor %xmm14,%xmm6");
asm volatile ("pxor %xmm8,%xmm7");
asm volatile ("pxor %xmm9,%xmm5");
asm volatile ("pxor %xmm10,%xmm6");
asm volatile ("pxor %xmm11,%xmm7");
} else {
asm volatile ("pxor %xmm8,%xmm8");
asm volatile ("pxor %xmm9,%xmm9");
asm volatile ("pxor %xmm10,%xmm10");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm4,%xmm8");
asm volatile ("pcmpgtb %xmm5,%xmm9");
asm volatile ("pcmpgtb %xmm6,%xmm10");
asm volatile ("pcmpgtb %xmm7,%xmm11");
asm volatile ("paddb %xmm4,%xmm4");
asm volatile ("paddb %xmm5,%xmm5");
asm volatile ("paddb %xmm6,%xmm6");
asm volatile ("paddb %xmm7,%xmm7");
asm volatile ("pand %xmm15,%xmm8");
asm volatile ("pand %xmm15,%xmm9");
asm volatile ("pand %xmm15,%xmm10");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm8,%xmm4");
asm volatile ("pxor %xmm9,%xmm5");
asm volatile ("pxor %xmm10,%xmm6");
asm volatile ("pxor %xmm11,%xmm7");
}
asm volatile ("pxor %xmm8,%xmm8");
asm volatile ("pxor %xmm9,%xmm9");
asm volatile ("pxor %xmm10,%xmm10");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm4,%xmm8");
asm volatile ("pcmpgtb %xmm5,%xmm9");
asm volatile ("pcmpgtb %xmm6,%xmm10");
asm volatile ("pcmpgtb %xmm7,%xmm11");
asm volatile ("paddb %xmm4,%xmm4");
asm volatile ("paddb %xmm5,%xmm5");
asm volatile ("paddb %xmm6,%xmm6");
asm volatile ("paddb %xmm7,%xmm7");
asm volatile ("pand %xmm15,%xmm8");
asm volatile ("pand %xmm15,%xmm9");
asm volatile ("pand %xmm15,%xmm10");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm8,%xmm4");
asm volatile ("pxor %xmm9,%xmm5");
asm volatile ("pxor %xmm10,%xmm6");
asm volatile ("pxor %xmm11,%xmm7");
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[0][i]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[0][i + 16]));
@ -486,14 +484,266 @@ static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv
raid_sse_end();
}
/*
* GEN2 Cauchy SSE2 implementation using the AES G23 Q transition.
*
* Note that it uses 16 registers, meaning that x64 is required.
*/
static __always_inline void raid_gen2_sse2ext_g23(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
uint8_t *q;
int d, l;
size_t i;
l = nd - 1;
p = v[nd];
q = v[nd + 1];
int g23_boundary = raid_g23_boundary(l - 1);
raid_sse_begin();
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.poly[0]));
for (i = 0; i < size; i += 64) {
int boundary = g23_boundary;
asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i]));
asm volatile ("movdqa %0,%%xmm1" : : "m" (v[l][i + 16]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (v[l][i + 32]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (v[l][i + 48]));
asm volatile ("movdqa %xmm0,%xmm4");
asm volatile ("movdqa %xmm1,%xmm5");
asm volatile ("movdqa %xmm2,%xmm6");
asm volatile ("movdqa %xmm3,%xmm7");
d = l - 1;
for (;;) {
/*
* All transitions above 'boundary' are x2.
* Do not let a two-disk iteration cross the boundary.
*/
for (; d >= boundary + 2; d -= 2) {
/* lane 0 */
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i]));
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm4,%xmm11");
asm volatile ("paddb %xmm4,%xmm4");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm4");
asm volatile ("pxor %xmm8,%xmm4");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm4,%xmm11");
asm volatile ("paddb %xmm4,%xmm4");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm4");
asm volatile ("pxor %xmm9,%xmm4");
asm volatile ("pxor %xmm8,%xmm0");
asm volatile ("pxor %xmm9,%xmm0");
/* lane 1 */
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i + 16]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i + 16]));
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm5,%xmm11");
asm volatile ("paddb %xmm5,%xmm5");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm5");
asm volatile ("pxor %xmm8,%xmm5");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm5,%xmm11");
asm volatile ("paddb %xmm5,%xmm5");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm5");
asm volatile ("pxor %xmm9,%xmm5");
asm volatile ("pxor %xmm8,%xmm1");
asm volatile ("pxor %xmm9,%xmm1");
/* lane 2 */
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i + 32]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i + 32]));
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm6,%xmm11");
asm volatile ("paddb %xmm6,%xmm6");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm6");
asm volatile ("pxor %xmm8,%xmm6");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm6,%xmm11");
asm volatile ("paddb %xmm6,%xmm6");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm6");
asm volatile ("pxor %xmm9,%xmm6");
asm volatile ("pxor %xmm8,%xmm2");
asm volatile ("pxor %xmm9,%xmm2");
/* lane 3 */
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i + 48]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i + 48]));
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm7,%xmm11");
asm volatile ("paddb %xmm7,%xmm7");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm7");
asm volatile ("pxor %xmm8,%xmm7");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm7,%xmm11");
asm volatile ("paddb %xmm7,%xmm7");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm7");
asm volatile ("pxor %xmm9,%xmm7");
asm volatile ("pxor %xmm8,%xmm3");
asm volatile ("pxor %xmm9,%xmm3");
}
/*
* The first x2 segment can have odd length.
* Consume its last x2 transition separately.
*/
if (d > boundary) {
asm volatile ("pxor %xmm8,%xmm8");
asm volatile ("pxor %xmm9,%xmm9");
asm volatile ("pxor %xmm10,%xmm10");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm4,%xmm8");
asm volatile ("pcmpgtb %xmm5,%xmm9");
asm volatile ("pcmpgtb %xmm6,%xmm10");
asm volatile ("pcmpgtb %xmm7,%xmm11");
asm volatile ("paddb %xmm4,%xmm4");
asm volatile ("paddb %xmm5,%xmm5");
asm volatile ("paddb %xmm6,%xmm6");
asm volatile ("paddb %xmm7,%xmm7");
asm volatile ("pand %xmm15,%xmm8");
asm volatile ("pand %xmm15,%xmm9");
asm volatile ("pand %xmm15,%xmm10");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm8,%xmm4");
asm volatile ("pxor %xmm9,%xmm5");
asm volatile ("pxor %xmm10,%xmm6");
asm volatile ("pxor %xmm11,%xmm7");
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d][i + 16]));
asm volatile ("movdqa %0,%%xmm10" : : "m" (v[d][i + 32]));
asm volatile ("movdqa %0,%%xmm11" : : "m" (v[d][i + 48]));
asm volatile ("pxor %xmm8,%xmm0");
asm volatile ("pxor %xmm9,%xmm1");
asm volatile ("pxor %xmm10,%xmm2");
asm volatile ("pxor %xmm11,%xmm3");
asm volatile ("pxor %xmm8,%xmm4");
asm volatile ("pxor %xmm9,%xmm5");
asm volatile ("pxor %xmm10,%xmm6");
asm volatile ("pxor %xmm11,%xmm7");
--d;
}
/*
* boundary == -1 identifies the final x2-only segment.
* At this point all remaining disks have been consumed.
*/
if (boundary < 0)
break;
/*
* d == boundary.
*
* Perform the fixed G23 x3 transition and then add
* the current disk to P and Q.
*/
asm volatile ("movdqa %xmm4,%xmm8");
asm volatile ("movdqa %xmm5,%xmm9");
asm volatile ("movdqa %xmm6,%xmm10");
asm volatile ("movdqa %xmm7,%xmm11");
asm volatile ("pxor %xmm12,%xmm12");
asm volatile ("pxor %xmm13,%xmm13");
asm volatile ("pxor %xmm14,%xmm14");
asm volatile ("pcmpgtb %xmm8,%xmm12");
asm volatile ("pcmpgtb %xmm9,%xmm13");
asm volatile ("pcmpgtb %xmm10,%xmm14");
asm volatile ("paddb %xmm4,%xmm4");
asm volatile ("paddb %xmm5,%xmm5");
asm volatile ("paddb %xmm6,%xmm6");
asm volatile ("paddb %xmm7,%xmm7");
asm volatile ("pxor %xmm8,%xmm4");
asm volatile ("pxor %xmm8,%xmm8");
asm volatile ("pcmpgtb %xmm11,%xmm8");
asm volatile ("pand %xmm15,%xmm12");
asm volatile ("pand %xmm15,%xmm13");
asm volatile ("pand %xmm15,%xmm14");
asm volatile ("pand %xmm15,%xmm8");
asm volatile ("pxor %xmm12,%xmm4");
asm volatile ("pxor %xmm13,%xmm5");
asm volatile ("pxor %xmm14,%xmm6");
asm volatile ("pxor %xmm8,%xmm7");
asm volatile ("pxor %xmm9,%xmm5");
asm volatile ("pxor %xmm10,%xmm6");
asm volatile ("pxor %xmm11,%xmm7");
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d][i + 16]));
asm volatile ("movdqa %0,%%xmm10" : : "m" (v[d][i + 32]));
asm volatile ("movdqa %0,%%xmm11" : : "m" (v[d][i + 48]));
asm volatile ("pxor %xmm8,%xmm0");
asm volatile ("pxor %xmm9,%xmm1");
asm volatile ("pxor %xmm10,%xmm2");
asm volatile ("pxor %xmm11,%xmm3");
asm volatile ("pxor %xmm8,%xmm4");
asm volatile ("pxor %xmm9,%xmm5");
asm volatile ("pxor %xmm10,%xmm6");
asm volatile ("pxor %xmm11,%xmm7");
--d;
/*
* G23 boundaries are exactly 51 transitions apart.
* After d = 50 there are no more x3 transitions.
*/
boundary -= 51;
if (boundary < 50)
boundary = -1;
}
asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i]));
asm volatile ("movntdq %%xmm1,%0" : "=m" (p[i + 16]));
asm volatile ("movntdq %%xmm2,%0" : "=m" (p[i + 32]));
asm volatile ("movntdq %%xmm3,%0" : "=m" (p[i + 48]));
asm volatile ("movntdq %%xmm4,%0" : "=m" (q[i]));
asm volatile ("movntdq %%xmm5,%0" : "=m" (q[i + 16]));
asm volatile ("movntdq %%xmm6,%0" : "=m" (q[i + 32]));
asm volatile ("movntdq %%xmm7,%0" : "=m" (q[i + 48]));
}
raid_sse_end();
}
void raid_gen2_sse2ext_raid(int nd, size_t size, void **vv)
{
raid_gen2_sse2ext_gen(nd, size, vv, 0);
raid_gen2_sse2ext_x2(nd, size, vv);
}
void raid_gen2_sse2ext_aes(int nd, size_t size, void **vv)
{
raid_gen2_sse2ext_gen(nd, size, vv, 1);
if (raid_g23_x2_only(nd))
raid_gen2_sse2ext_x2(nd, size, vv);
else
raid_gen2_sse2ext_g23(nd, size, vv);
}
#endif

2226
raid/ssse3.c
View file

File diff suppressed because it is too large Load diff

3
raid/test.c
View file

@ -887,7 +887,6 @@ int raid_test_par(int mode, int nd, size_t size)
/* setup all the available functions */
test_setup(1) = raid_gen1_int32;
test_setup(1) = raid_gen1_int64;
test_setup(2) = raid_gen2_int8;
if (mode == RAID_MODE_CAUCHY_AES) {
test_setup(2) = raid_gen2_int32_aes;
test_setup(2) = raid_gen2_int64_aes;
@ -996,6 +995,8 @@ int raid_test_par(int mode, int nd, size_t size)
if (mode == RAID_MODE_CAUCHY_AES) {
test_setup(3) = raid_gen3_ssse3_aes;
test_setup(4) = raid_gen4_ssse3_aes;
test_setup(5) = raid_gen5_ssse3_aes;
test_setup(6) = raid_gen6_ssse3_aes;
} else {
test_setup(3) = raid_gen3_ssse3_raid;
test_setup(4) = raid_gen4_ssse3_raid;

31
raid/test/speedtest.c
View file

@ -232,12 +232,7 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
printf("%8s", raid_gen_tag(RAID_ALGO_CAUCHY_PAR2));
fflush(stdout);
SPEED_START {
raid_gen2_int8(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
printf("%8s", "");
if (mode == RAID_MODE_CAUCHY_AES) {
SPEED_START {
@ -742,15 +737,17 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
#endif
}
if (raid_cpu_has_ssse3()) {
if (mode == RAID_MODE_CAUCHY_RAID) {
if (mode == RAID_MODE_CAUCHY_AES) {
SPEED_START {
raid_gen5_ssse3_aes(nd, size, v);
} SPEED_STOP
} else {
SPEED_START {
raid_gen5_ssse3_raid(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
} else {
printf("%8s", "");
}
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
if (mode == RAID_MODE_CAUCHY_AES) {
@ -881,15 +878,17 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
#endif
}
if (raid_cpu_has_ssse3()) {
if (mode == RAID_MODE_CAUCHY_RAID) {
if (mode == RAID_MODE_CAUCHY_AES) {
SPEED_START {
raid_gen6_ssse3_aes(nd, size, v);
} SPEED_STOP
} else {
SPEED_START {
raid_gen6_ssse3_raid(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
} else {
printf("%8s", "");
}
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
if (mode == RAID_MODE_CAUCHY_AES) {