rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 19:01:57 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Implement G23 Extended Cauchy construction for AES RAID mode

Browse source
This commit is contained in:
Andrea Mazzoleni 2026-08-21 19:40:48 +02:00
parent b5d7474c3b
commit 1b5c926936
22 changed files with 10924 additions and 6047 deletions
Download patch file Download diff file Expand all files Collapse all files

69
cmdline/selftest.c
View file

@ -1237,6 +1237,12 @@ void selftest(void)
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (raid_test_rec2_g23(256) != 0) {
/* LCOV_EXCL_START */
log_fatal(EINTERNAL, "Failed G23 P/Q recovery test\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (raid_test_tail(RAID_MODE_CAUCHY_AES, RAID_DATA_MAX, 256) != 0) {
/* LCOV_EXCL_START */
log_fatal(EINTERNAL, "Failed TAIL Cauchy AES test\n");
@ -1244,23 +1250,52 @@ void selftest(void)
/* LCOV_EXCL_STOP */
}
/* verify raid_mode get/set behavior */
int cur = raid_mode(RAID_MODE_GET);
int prev = raid_mode(RAID_MODE_CAUCHY_AES);
if (prev != cur) {
/* LCOV_EXCL_START */
log_fatal(EINTERNAL, "raid_mode set did not return previous mode\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (raid_mode(RAID_MODE_GET) != RAID_MODE_CAUCHY_AES) {
/* LCOV_EXCL_START */
log_fatal(EINTERNAL, "raid_mode GET did not return active mode\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* restore default mode */
/* verify RAID -> AES -> RAID switches all mode-dependent GF/Cauchy tables */
raid_mode(RAID_MODE_CAUCHY_RAID);
if (raid_gfmul != raid_gfmul_raid || raid_gfinv != raid_gfinv_raid
|| raid_gfcauchy != raid_gfcauchy_raid || raid_gfgen != raid_gfcauchy_raid) {
/* LCOV_EXCL_START */
log_fatal(EINTERNAL, "raid_mode did not select RAID tables\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32)
if (raid_gfcauchypshufb != raid_gfcauchypshufb_raid
|| raid_gfmulpshufb != raid_gfmulpshufb_raid) {
log_fatal(EINTERNAL, "raid_mode did not select RAID SIMD tables\n");
exit(EXIT_FAILURE);
}
#endif
if (raid_mode(RAID_MODE_CAUCHY_AES) != RAID_MODE_CAUCHY_RAID
|| raid_mode(RAID_MODE_GET) != RAID_MODE_CAUCHY_AES
|| raid_gfmul != raid_gfmul_aes || raid_gfinv != raid_gfinv_aes
|| raid_gfcauchy != raid_gfcauchy_aes || raid_gfgen != raid_gfcauchy_aes) {
/* LCOV_EXCL_START */
log_fatal(EINTERNAL, "raid_mode did not select AES tables\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32)
if (raid_gfcauchypshufb != raid_gfcauchypshufb_aes
|| raid_gfmulpshufb != raid_gfmulpshufb_aes) {
log_fatal(EINTERNAL, "raid_mode did not select AES SIMD tables\n");
exit(EXIT_FAILURE);
}
#endif
if (raid_mode(RAID_MODE_CAUCHY_RAID) != RAID_MODE_CAUCHY_AES
|| raid_gfmul != raid_gfmul_raid || raid_gfinv != raid_gfinv_raid
|| raid_gfcauchy != raid_gfcauchy_raid || raid_gfgen != raid_gfcauchy_raid) {
/* LCOV_EXCL_START */
log_fatal(EINTERNAL, "raid_mode did not restore RAID tables\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32)
if (raid_gfcauchypshufb != raid_gfcauchypshufb_raid
|| raid_gfmulpshufb != raid_gfmulpshufb_raid) {
log_fatal(EINTERNAL, "raid_mode did not restore RAID SIMD tables\n");
exit(EXIT_FAILURE);
}
#endif
}

32
cmdline/speed.c
View file

@ -330,12 +330,7 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
printf("%8s", raid_gen_tag(RAID_ALGO_CAUCHY_PAR2));
fflush(stdout);
SPEED_START {
raid_gen2_int8(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
printf("%8s", "");
if (mode == RAID_MODE_CAUCHY_AES) {
SPEED_START {
@ -844,15 +839,17 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
#endif
}
if (raid_cpu_has_ssse3()) {
if (mode == RAID_MODE_CAUCHY_RAID) {
if (mode == RAID_MODE_CAUCHY_AES) {
SPEED_START {
raid_gen5_ssse3_aes(nd, size, v);
} SPEED_STOP
} else {
SPEED_START {
raid_gen5_ssse3_raid(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
} else {
printf("%8s", "");
}
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
@ -984,17 +981,18 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
#endif
}
if (raid_cpu_has_ssse3()) {
if (mode == RAID_MODE_CAUCHY_RAID) {
if (mode == RAID_MODE_CAUCHY_AES) {
SPEED_START {
raid_gen6_ssse3_aes(nd, size, v);
} SPEED_STOP
} else {
SPEED_START {
raid_gen6_ssse3_raid(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
} else {
printf("%8s", "");
}
fflush(stdout);
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
if (mode == RAID_MODE_CAUCHY_AES) {
SPEED_START {

21
raid/README
View file

@ -120,17 +120,20 @@ Operating Modes and Polynomials
(Z parity). Optimized for low-end CPUs without vector instructions.
RAID_MODE_CAUCHY_AES (Mode = 2)
Uses Cauchy matrix with AES polynomial 0x1b (x^8 + x^4 + x^3 + x + 1,
0x11b) and primitive generator g = 3. On CPUs supporting Intel GFNI,
generation and recovery are accelerated directly using VGF2P8MULB.
Uses a 6x251 Extended Cauchy matrix with AES polynomial 0x1b
(x^8 + x^4 + x^3 + x + 1, 0x11b). Its Q row uses the G23 sequence,
with multiplication by 2 for almost every coefficient transition and
multiplication by 3 only between its five cosets. On CPUs supporting
Intel GFNI, generation and recovery are accelerated directly using VGF2P8MULB.
This native GFNI multiplication is typically faster than the affine GFNI
implementation required by the standard RAID polynomial. Parity data is
incompatible with standard 0x11d polynomial arrays.
incompatible with standard 0x11d polynomial arrays and with the previous
RAID_MODE_CAUCHY_AES parity layout.
Both Cauchy modes use the same Extended Cauchy construction with coefficients
based on powers of their respective primitive generator. GFNI acceleration
is available for both fields: the RAID field uses affine transformations,
while the AES field uses the native GFNI multiplication instruction.
Both Cauchy modes support RAID_DATA_MAX = 251 data disks and
RAID_PARITY_MAX = 6 parity levels. GFNI acceleration is available for both
fields: the RAID field uses affine transformations, while the AES field uses
the native GFNI multiplication instruction.
Implementation Structure
@ -140,7 +143,7 @@ Implementation Structure
raid.c Cauchy matrix solver, Gauss-Jordan inversion, delta generation.
module.c Algorithm registration, CPU detection, and dynamic dispatch.
int.c Portable C fallback implementation (8/32/64-bit word loops).
tables.c Precomputed GF(2^8) log/exp, multiplication, Cauchy tables.
tables.c Precomputed GF(2^8) multiplication and Cauchy tables.
mktables.c Generator for GF(2^8), Cauchy, and GFNI affine tables.
internal.h Private header, assertion macros (BUG_ON), alignment helpers.

1512
raid/avx2.c
View file

File diff suppressed because it is too large Load diff

10
raid/gf.h
View file

@ -29,16 +29,6 @@ static __always_inline uint8_t inv(uint8_t v)
return raid_gfinv[v];
}
/*
* GF generator^a for the active Cauchy matrix.
*/
static __always_inline uint8_t powgen(int v)
{
BUG_ON(v < 0 || v > 254); /* invalid exponent */
return raid_gfexp[v];
}
/*
* Gets the multiplication table for a specified value.
*/

228
raid/int.c
View file

@ -61,9 +61,53 @@ void raid_gen1_int64(int nd, size_t size, void **vv)
}
/*
* GEN2 (double parity) 32bit C implementation
* GEN2 (double parity) 32bit C implementation using only Q *= 2.
*/
static __always_inline void raid_gen2_int32_gen(int nd, size_t size, void **vv, int generator)
static __always_inline void raid_gen2_int32_x2(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
uint8_t *q;
int d, l;
size_t i;
const uint32_t poly_32 = raid_poly_32;
uint32_t d0, q0, p0;
uint32_t d1, q1, p1;
l = nd - 1;
p = v[nd];
q = v[nd + 1];
for (i = 0; i < size; i += 8) {
q0 = p0 = v_read32(&v[l][i]);
q1 = p1 = v_read32(&v[l][i + 4]);
for (d = l - 1; d >= 0; --d) {
d0 = v_read32(&v[d][i]);
d1 = v_read32(&v[d][i + 4]);
p0 ^= d0;
p1 ^= d1;
q0 = x2_32(q0, poly_32);
q1 = x2_32(q1, poly_32);
q0 ^= d0;
q1 ^= d1;
}
v_write32(&p[i], p0);
v_write32(&p[i + 4], p1);
v_write32(&q[i], q0);
v_write32(&q[i + 4], q1);
}
}
/*
* GEN2 (double parity) 32bit C implementation using the AES G23 Q recurrence.
*/
static __always_inline void raid_gen2_int32_g23(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -80,10 +124,47 @@ static __always_inline void raid_gen2_int32_gen(int nd, size_t size, void **vv,
p = v[nd];
q = v[nd + 1];
int g23_boundary = raid_g23_boundary(l - 1);
for (i = 0; i < size; i += 8) {
int boundary = g23_boundary;
q0 = p0 = v_read32(&v[l][i]);
q1 = p1 = v_read32(&v[l][i + 4]);
for (d = l - 1; d >= 0; --d) {
d = l - 1;
for (;;) {
/*
* All transitions above 'boundary' are x2.
*/
for (; d > boundary; --d) {
d0 = v_read32(&v[d][i]);
d1 = v_read32(&v[d][i + 4]);
p0 ^= d0;
p1 ^= d1;
q0 = x2_32(q0, poly_32);
q1 = x2_32(q1, poly_32);
q0 ^= d0;
q1 ^= d1;
}
/*
* boundary == -1 identifies the final x2-only segment.
* At this point all remaining disks have been consumed.
*/
if (boundary < 0)
break;
/*
* d == boundary.
*
* Perform the fixed G23 x3 transition and then add
* the current disk to P and Q.
*/
d0 = v_read32(&v[d][i]);
d1 = v_read32(&v[d][i + 4]);
@ -92,17 +173,26 @@ static __always_inline void raid_gen2_int32_gen(int nd, size_t size, void **vv,
qold = q0;
q0 = x2_32(q0, poly_32);
if (generator == 3)
q0 ^= qold;
q0 ^= qold;
qold = q1;
q1 = x2_32(q1, poly_32);
if (generator == 3)
q1 ^= qold;
q1 ^= qold;
q0 ^= d0;
q1 ^= d1;
--d;
/*
* G23 boundaries are exactly 51 transitions apart.
* After d = 50 there are no more x3 transitions.
*/
boundary -= 51;
if (boundary < 50)
boundary = -1;
}
v_write32(&p[i], p0);
v_write32(&p[i + 4], p1);
v_write32(&q[i], q0);
@ -112,18 +202,65 @@ static __always_inline void raid_gen2_int32_gen(int nd, size_t size, void **vv,
void raid_gen2_int32_raid(int nd, size_t size, void **vv)
{
raid_gen2_int32_gen(nd, size, vv, 2);
raid_gen2_int32_x2(nd, size, vv);
}
void raid_gen2_int32_aes(int nd, size_t size, void **vv)
{
raid_gen2_int32_gen(nd, size, vv, 3);
if (raid_g23_x2_only(nd))
raid_gen2_int32_x2(nd, size, vv);
else
raid_gen2_int32_g23(nd, size, vv);
}
/*
* GEN2 (double parity) 64bit C implementation
* GEN2 (double parity) 64bit C implementation using only Q *= 2.
*/
static __always_inline void raid_gen2_int64_gen(int nd, size_t size, void **vv, int generator)
static __always_inline void raid_gen2_int64_x2(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
uint8_t *q;
int d, l;
size_t i;
const uint64_t poly_64 = raid_poly_64;
uint64_t d0, q0, p0;
uint64_t d1, q1, p1;
l = nd - 1;
p = v[nd];
q = v[nd + 1];
for (i = 0; i < size; i += 16) {
q0 = p0 = v_read64(&v[l][i]);
q1 = p1 = v_read64(&v[l][i + 8]);
for (d = l - 1; d >= 0; --d) {
d0 = v_read64(&v[d][i]);
d1 = v_read64(&v[d][i + 8]);
p0 ^= d0;
p1 ^= d1;
q0 = x2_64(q0, poly_64);
q1 = x2_64(q1, poly_64);
q0 ^= d0;
q1 ^= d1;
}
v_write64(&p[i], p0);
v_write64(&p[i + 8], p1);
v_write64(&q[i], q0);
v_write64(&q[i + 8], q1);
}
}
/*
* GEN2 (double parity) 64bit C implementation using the AES G23 Q recurrence.
*/
static __always_inline void raid_gen2_int64_g23(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -140,10 +277,47 @@ static __always_inline void raid_gen2_int64_gen(int nd, size_t size, void **vv,
p = v[nd];
q = v[nd + 1];
int g23_boundary = raid_g23_boundary(l - 1);
for (i = 0; i < size; i += 16) {
int boundary = g23_boundary;
q0 = p0 = v_read64(&v[l][i]);
q1 = p1 = v_read64(&v[l][i + 8]);
for (d = l - 1; d >= 0; --d) {
d = l - 1;
for (;;) {
/*
* All transitions above 'boundary' are x2.
*/
for (; d > boundary; --d) {
d0 = v_read64(&v[d][i]);
d1 = v_read64(&v[d][i + 8]);
p0 ^= d0;
p1 ^= d1;
q0 = x2_64(q0, poly_64);
q1 = x2_64(q1, poly_64);
q0 ^= d0;
q1 ^= d1;
}
/*
* boundary == -1 identifies the final x2-only segment.
* At this point all remaining disks have been consumed.
*/
if (boundary < 0)
break;
/*
* d == boundary.
*
* Perform the fixed G23 x3 transition and then add
* the current disk to P and Q.
*/
d0 = v_read64(&v[d][i]);
d1 = v_read64(&v[d][i + 8]);
@ -152,17 +326,26 @@ static __always_inline void raid_gen2_int64_gen(int nd, size_t size, void **vv,
qold = q0;
q0 = x2_64(q0, poly_64);
if (generator == 3)
q0 ^= qold;
q0 ^= qold;
qold = q1;
q1 = x2_64(q1, poly_64);
if (generator == 3)
q1 ^= qold;
q1 ^= qold;
q0 ^= d0;
q1 ^= d1;
--d;
/*
* G23 boundaries are exactly 51 transitions apart.
* After d = 50 there are no more x3 transitions.
*/
boundary -= 51;
if (boundary < 50)
boundary = -1;
}
v_write64(&p[i], p0);
v_write64(&p[i + 8], p1);
v_write64(&q[i], q0);
@ -172,17 +355,15 @@ static __always_inline void raid_gen2_int64_gen(int nd, size_t size, void **vv,
void raid_gen2_int64_raid(int nd, size_t size, void **vv)
{
raid_gen2_int64_gen(nd, size, vv, 2);
raid_gen2_int64_x2(nd, size, vv);
}
void raid_gen2_int64_aes(int nd, size_t size, void **vv)
{
raid_gen2_int64_gen(nd, size, vv, 3);
}
void raid_gen2_int8(int nd, size_t size, void **vv)
{
raid_gen_ref(nd, 2, size, vv);
if (raid_g23_x2_only(nd))
raid_gen2_int64_x2(nd, size, vv);
else
raid_gen2_int64_g23(nd, size, vv);
}
/*
@ -694,7 +875,6 @@ void raid_recX_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv)
void raid_register_int(void)
{
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "int8", raid_gen2_int8, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "int8", raid_gen3_int8, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "int8", raid_gen4_int8, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "int8", raid_gen5_int8, RAID_POLY_ANY);

82
raid/internal.h
View file

@ -74,6 +74,82 @@ static __always_inline void * __align_ptr(void *ptr, uintptr_t size)
return (void *)offset;
}
/*
* Check if the AES G23 Q recurrence contains only x2 transitions
* for the specified number of data disks.
*/
static __always_inline int raid_g23_x2_only(int nd)
{
return nd <= 51;
}
/*
* Find the next AES G23 x3 boundary in descending Horner order.
*
* The G23 Q coefficients are generated by multiplying by 2 at every step,
* except at the four boundaries between the five cosets, where the next
* coefficient is obtained by multiplying by 3 instead:
*
* q[i + 1] = 3 * q[i] for i = 50, 101, 152, 203
* q[i + 1] = 2 * q[i] otherwise
*
* The optimized parity generators evaluate Q in descending Horner order.
* If the current step is identified by disk index d, the x3 transitions are
* therefore encountered at:
*
* d = 203, 152, 101, 50
*
* i.e. every 51 descending steps.
*
* The generators can process all the x2 transitions between two consecutive
* boundaries in a branch-free loop, and handle the x3 transition separately.
* This function returns the first x3 boundary encountered by a descending
* traversal whose first processed step has index d.
*
* For d >= 50, (d - 50) % 51 gives the distance from the previous x3
* boundary in ascending index order. Subtracting this distance from d gives
* that boundary directly:
*
* boundary = d - (d - 50) % 51
*
* Since the Horner traversal proceeds toward decreasing indexes, this is also
* the next x3 transition that will be encountered.
*
* For d < 50 there is no remaining x3 transition, so -1 is returned. This
* value can also be used as the lower bound of the final x2-only segment,
* allowing all remaining steps down to d = 0 to be processed without a
* special G23 test.
*
* Examples:
*
* d = 249 -> 203 -> x2 down to 204, then x3 at 203
* d = 203 -> 203 -> x3 immediately
* d = 202 -> 152 -> x2 down to 153, then x3 at 152
* d = 152 -> 152 -> x3 immediately
* d = 151 -> 101 -> x2 down to 102, then x3 at 101
* d = 101 -> 101 -> x3 immediately
* d = 100 -> 50 -> x2 down to 51, then x3 at 50
* d = 50 -> 50 -> x3 immediately
* d = 49 -> -1 -> no further x3 transition
*
* After processing a boundary, the following one in descending order is
* exactly 51 positions lower. Thus generators can continue with:
*
* boundary -= 51;
* if (boundary < 50)
* boundary = -1;
*
* The returned value is normally computed once before each independent
* Horner traversal and then advanced between the fixed G23 segments.
*/
static __always_inline int raid_g23_boundary(int d)
{
if (d < 50)
return -1;
return d - (d - 50) % 51;
}
/*
* Includes the main interface headers.
*/
@ -95,7 +171,6 @@ void raid_gen1_int64(int nd, size_t size, void **vv);
void raid_gen1_sse2(int nd, size_t size, void **vv);
void raid_gen1_avx2(int nd, size_t size, void **vv);
void raid_gen1_avx512bw(int nd, size_t size, void **vv);
void raid_gen2_int8(int nd, size_t size, void **vv);
void raid_gen2_int32_raid(int nd, size_t size, void **vv);
void raid_gen2_int32_aes(int nd, size_t size, void **vv);
void raid_gen2_int64_raid(int nd, size_t size, void **vv);
@ -144,6 +219,7 @@ void raid_gen4_avx2gfni_aes(int nd, size_t size, void **vv);
void raid_gen4_avx512gfni_aes(int nd, size_t size, void **vv);
void raid_gen5_int8(int nd, size_t size, void **vv);
void raid_gen5_ssse3_raid(int nd, size_t size, void **vv);
void raid_gen5_ssse3_aes(int nd, size_t size, void **vv);
void raid_gen5_ssse3ext_raid(int nd, size_t size, void **vv);
void raid_gen5_ssse3ext_aes(int nd, size_t size, void **vv);
void raid_gen5_avx2ext_raid(int nd, size_t size, void **vv);
@ -155,6 +231,7 @@ void raid_gen5_avx2gfni_aes(int nd, size_t size, void **vv);
void raid_gen5_avx512gfni_aes(int nd, size_t size, void **vv);
void raid_gen6_int8(int nd, size_t size, void **vv);
void raid_gen6_ssse3_raid(int nd, size_t size, void **vv);
void raid_gen6_ssse3_aes(int nd, size_t size, void **vv);
void raid_gen6_ssse3ext_raid(int nd, size_t size, void **vv);
void raid_gen6_ssse3ext_aes(int nd, size_t size, void **vv);
void raid_gen6_avx2ext_raid(int nd, size_t size, void **vv);
@ -357,7 +434,6 @@ extern struct gfconst16 gfconst16 __aligned(16);
* Tables.
*/
extern const uint8_t raid_gfmul_raid[256][256] __aligned(256);
extern const uint8_t raid_gfexp_raid[256] __aligned(256);
extern const uint8_t raid_gfinv_raid[256] __aligned(256);
extern const uint8_t raid_gfvandermonde_raid[3][256] __aligned(256);
extern const uint8_t raid_gfcauchy_raid[6][256] __aligned(256);
@ -368,7 +444,6 @@ extern const uint8_t raid_gfmulpshufb_raid[256][2][16] __aligned(256);
#endif
extern const uint8_t raid_gfmul_aes[256][256] __aligned(256);
extern const uint8_t raid_gfexp_aes[256] __aligned(256);
extern const uint8_t raid_gfinv_aes[256] __aligned(256);
extern const uint8_t raid_gfcauchy_aes[6][256] __aligned(256);
#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32)
@ -377,7 +452,6 @@ extern const uint8_t raid_gfmulpshufb_aes[256][2][16] __aligned(256);
#endif
extern const uint8_t(*raid_gfmul)[256];
extern const uint8_t *raid_gfexp;
extern const uint8_t *raid_gfinv;
extern const uint8_t(*raid_gfvandermonde)[256];
extern const uint8_t(*raid_gfcauchy)[256];

104
raid/mktables.c
View file

@ -3,12 +3,13 @@
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
/*
* Galois field reduction polynomials.
*
* RAID_POLY_RAID - Standard RAID polynomial 0x1d (x^8 + x^4 + x^3 + x^2 + 1).
* RAID_POLY_AES - AES polynomial 0x1b (x^8 + x^4 + x^3 + x + 1) for GFNI acceleration.
* RAID_POLY_AES - AES polynomial 0x1b (x^8 + x^4 + x^3 + x + 1) used by the AES/G23 Cauchy mode.
*/
#define RAID_POLY_RAID 0x1d
#define RAID_POLY_AES 0x1b
@ -41,9 +42,10 @@ static uint8_t raid_gfmul(uint8_t poly, uint8_t a, uint8_t b)
/**
* Setup the GFNI affine matrix for multiplication by a constant.
*
* VGF2P8MULB is fixed to the AES polynomial 0x11b, while RAID parity is
* defined over GF(2^8) modulo 0x11d. VGF2P8AFFINEQB can implement the
* multiplication because multiplication by a constant is GF(2)-linear.
* VGF2P8MULB is fixed to the AES polynomial 0x11b, while
* RAID_MODE_CAUCHY_RAID parity is defined over GF(2^8) modulo 0x11d.
* VGF2P8AFFINEQB can implement the multiplication because multiplication by a
* constant is GF(2)-linear.
*
* VGF2P8AFFINEQB stores the row producing output bit j in byte 7-j.
* Within each row, bit k selects input bit k.
@ -84,9 +86,9 @@ uint8_t raid_gfinv[256];
#define DISK (257 - PARITY)
/**
* Setup the Cauchy matrix used to generate the parity.
* Setup the RAID/g=2 Cauchy matrix used to generate parity.
*/
static void set_cauchy(uint8_t poly, uint8_t generator, uint8_t *matrix)
static void set_cauchy_raid(uint8_t poly, uint8_t generator, uint8_t *matrix)
{
int i, j;
uint8_t inv_x, y;
@ -206,6 +208,72 @@ static void set_cauchy(uint8_t poly, uint8_t generator, uint8_t *matrix)
}
}
/**
* Setup the AES Extended Cauchy matrix using the G23 sequence.
*
* The canonical G23 sequence starts with q[0]=1, advances by multiplication
* by 2 within each 51-element coset, and uses multiplication by 3 between
* consecutive cosets.
*
* The data X set consumes the canonical G23 sequence q[0..254] from the
* beginning, with x[i]=inv(q[i]), while the Cauchy Y set starts with zero and
* consumes the sequence backward from the end, y[j]=inv(q[255-j]) for j>0.
* For PARITY=n, DISK=257-n; increasing parity removes one X element from the
* tail and appends that same element to Y, leaving all existing rows unchanged
* on the remaining data columns. This defines a nested matrix without
* implementing parity levels above the current limit.
*/
static void set_cauchy_aes(uint8_t *matrix)
{
uint8_t q[255];
uint8_t y[PARITY - 1];
int i, j, k;
q[0] = 1;
for (i = 0; i < 254; ++i) {
uint8_t generator = i == 50 || i == 101 || i == 152 || i == 203 ? 3 : 2;
q[i + 1] = raid_gfmul(RAID_POLY_AES, generator, q[i]);
}
/* The five cosets must enumerate GF(256)* exactly once. */
for (i = 0; i < 255; ++i) {
if (q[i] == 0) {
fprintf(stderr, "Invalid zero in AES G23 sequence at %d\n", i);
exit(EXIT_FAILURE);
}
for (k = i + 1; k < 255; ++k) {
if (q[i] == q[k]) {
fprintf(stderr, "Duplicate in AES G23 sequence at %d and %d\n", i, k);
exit(EXIT_FAILURE);
}
}
}
for (i = 0; i < DISK; ++i)
matrix[i] = 1;
y[0] = 0;
for (j = 1; j < PARITY - 1; ++j)
y[j] = raid_gfinv[q[255 - j]];
for (j = 0; j < PARITY - 1; ++j) {
for (i = 0; i < DISK; ++i) {
uint8_t x = raid_gfinv[q[i]];
matrix[(j + 1) * DISK + i] = raid_gfinv[x ^ y[j]];
}
}
/* Normalize R..U so every coefficient in the first column is 1. */
for (j = 1; j < PARITY - 1; ++j) {
uint8_t f = raid_gfinv[matrix[(j + 1) * DISK]];
for (i = 0; i < DISK; ++i)
matrix[(j + 1) * DISK + i] = raid_gfmul(RAID_POLY_AES, matrix[(j + 1) * DISK + i], f);
}
}
/**
* Setup the Power matrix used to generate the parity.
*/
@ -284,22 +352,6 @@ void tables(uint8_t poly, uint8_t generator, const char *tag)
printf("};\n\n");
}
/* generator^a */
printf("const uint8_t __aligned(256) raid_gfexp_%s[256] =\n", tag);
printf("{\n");
v = 1;
for (i = 0; i < 256; ++i) {
if (i % 8 == 0)
printf("\t");
printf("0x%02x,", v);
v = raid_gfmul(poly, v, generator);
if (i % 8 == 7)
printf("\n");
else
printf(" ");
}
printf("};\n\n");
/* 1/a */
printf("const uint8_t __aligned(256) raid_gfinv_%s[256] =\n", tag);
printf("{\n");
@ -320,7 +372,10 @@ void tables(uint8_t poly, uint8_t generator, const char *tag)
printf("};\n\n");
/* cauchy matrix */
set_cauchy(poly, generator, matrix);
if (poly == RAID_POLY_AES)
set_cauchy_aes(matrix);
else
set_cauchy_raid(poly, generator, matrix);
printf("/**\n");
printf(" * Cauchy matrix used to generate parity.\n");
@ -410,10 +465,11 @@ void tables(uint8_t poly, uint8_t generator, const char *tag)
printf("\t},\n");
}
printf("};\n");
printf("#endif\n\n");
printf("#endif\n");
if (poly == RAID_POLY_AES)
return;
printf("\n");
/* power matrix */
set_power(poly, matrix);

3
raid/module.c
View file

@ -40,7 +40,6 @@ static struct raid_rec_algo *raid_rec_algo;
int raid_mode_active = RAID_MODE_GET;
const uint8_t(*raid_gfmul)[256];
const uint8_t *raid_gfexp;
const uint8_t *raid_gfinv;
const uint8_t(*raid_gfvandermonde)[256];
const uint8_t(*raid_gfcauchy)[256];
@ -225,7 +224,6 @@ int raid_mode(int mode)
if (mode == RAID_MODE_CAUCHY_AES) {
raid_gfmul = raid_gfmul_aes;
raid_gfexp = raid_gfexp_aes;
raid_gfinv = raid_gfinv_aes;
raid_gfvandermonde = 0;
raid_gfcauchy = raid_gfcauchy_aes;
@ -245,7 +243,6 @@ int raid_mode(int mode)
raid_rec_algo = raid_rec_algo_aes;
} else {
raid_gfmul = raid_gfmul_raid;
raid_gfexp = raid_gfexp_raid;
raid_gfinv = raid_gfinv_raid;
raid_gfvandermonde = raid_gfvandermonde_raid;
raid_gfcauchy = raid_gfcauchy_raid;

792
raid/neon.c
View file

@ -60,9 +60,9 @@ void raid_gen1_neon(int nd, size_t size, void **vv)
}
/*
* GEN2 Cauchy NEON implementation using the active generator
* GEN2 Cauchy NEON implementation using only Q *= 2.
*/
static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, int generator)
static __always_inline void raid_gen2_neon_x2(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -93,13 +93,6 @@ static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, i
);
for (d = l - 1; d >= 0; --d) {
if (generator == 3) {
/* v14-v15 are otherwise unused and preserve Q across the AES xtime. */
asm volatile (
"mov v14.16b, v2.16b\n"
"mov v15.16b, v3.16b\n"
);
}
asm volatile (
/* double Q0-Q1 */
"sshr v16.16b, v2.16b, #7\n"
@ -122,12 +115,142 @@ static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, i
:
: "m" (v[d][i]), "m" (v[d][i + 16])
);
if (generator == 3) {
}
asm volatile (
"str q0, %0\n"
"str q1, %1\n"
"str q2, %2\n"
"str q3, %3\n"
: "=m" (p[i]), "=m" (p[i + 16]),
"=m" (q[i]), "=m" (q[i + 16])
);
}
raid_neon_end();
}
/*
* GEN2 Cauchy NEON implementation using the AES G23 Q recurrence.
*/
static __always_inline void raid_gen2_neon_g23(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
uint8_t *q;
int d, l;
size_t i;
l = nd - 1;
p = v[nd];
q = v[nd + 1];
int g23_boundary = raid_g23_boundary(l - 1);
raid_neon_begin();
asm volatile (
"ldr q29, %0\n"
:
: "m" (gfconst16.poly[0])
);
for (i = 0; i < size; i += 32) {
int boundary = g23_boundary;
asm volatile (
"ldr q0, %0\n"
"ldr q1, %1\n"
"mov v2.16b, v0.16b\n"
"mov v3.16b, v1.16b\n"
:
: "m" (v[l][i]), "m" (v[l][i + 16])
);
d = l - 1;
for (;;) {
/*
* All transitions above 'boundary' are x2.
*/
for (; d > boundary; --d) {
asm volatile (
"eor v2.16b, v2.16b, v14.16b\n"
"eor v3.16b, v3.16b, v15.16b\n"
/* double Q0-Q1 */
"sshr v16.16b, v2.16b, #7\n"
"shl v2.16b, v2.16b, #1\n"
"and v16.16b, v16.16b, v29.16b\n"
"eor v2.16b, v2.16b, v16.16b\n"
"sshr v16.16b, v3.16b, #7\n"
"shl v3.16b, v3.16b, #1\n"
"and v16.16b, v16.16b, v29.16b\n"
"eor v3.16b, v3.16b, v16.16b\n"
/* load and XOR */
"ldr q12, %0\n"
"ldr q13, %1\n"
"eor v0.16b, v0.16b, v12.16b\n"
"eor v1.16b, v1.16b, v13.16b\n"
"eor v2.16b, v2.16b, v12.16b\n"
"eor v3.16b, v3.16b, v13.16b\n"
:
: "m" (v[d][i]), "m" (v[d][i + 16])
);
}
/*
* boundary == -1 identifies the final x2-only segment.
* At this point all remaining disks have been consumed.
*/
if (boundary < 0)
break;
/*
* d == boundary.
*
* Perform the fixed G23 x3 transition and then add
* the current disk to P and Q.
*/
asm volatile (
/* preserve Q for Q *= 3 */
"mov v14.16b, v2.16b\n"
"mov v15.16b, v3.16b\n"
/* double Q0-Q1 */
"sshr v16.16b, v2.16b, #7\n"
"shl v2.16b, v2.16b, #1\n"
"and v16.16b, v16.16b, v29.16b\n"
"eor v2.16b, v2.16b, v16.16b\n"
"sshr v16.16b, v3.16b, #7\n"
"shl v3.16b, v3.16b, #1\n"
"and v16.16b, v16.16b, v29.16b\n"
"eor v3.16b, v3.16b, v16.16b\n"
/* load and XOR */
"ldr q12, %0\n"
"ldr q13, %1\n"
"eor v0.16b, v0.16b, v12.16b\n"
"eor v1.16b, v1.16b, v13.16b\n"
"eor v2.16b, v2.16b, v12.16b\n"
"eor v3.16b, v3.16b, v13.16b\n"
/* complete Q *= 3 */
"eor v2.16b, v2.16b, v14.16b\n"
"eor v3.16b, v3.16b, v15.16b\n"
:
: "m" (v[d][i]), "m" (v[d][i + 16])
);
--d;
/*
* G23 boundaries are exactly 51 transitions apart.
* After d = 50 there are no more x3 transitions.
*/
boundary -= 51;
if (boundary < 50)
boundary = -1;
}
asm volatile (
@ -145,12 +268,15 @@ static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, i
void raid_gen2_neon_raid(int nd, size_t size, void **vv)
{
raid_gen2_neon_gen(nd, size, vv, 2);
raid_gen2_neon_x2(nd, size, vv);
}
void raid_gen2_neon_aes(int nd, size_t size, void **vv)
{
raid_gen2_neon_gen(nd, size, vv, 3);
if (raid_g23_x2_only(nd))
raid_gen2_neon_x2(nd, size, vv);
else
raid_gen2_neon_g23(nd, size, vv);
}
/*
@ -248,9 +374,10 @@ void raid_genz_neon_raid(int nd, size_t size, void **vv)
}
/*
* GENX NEON implementation
* GENX NEON implementation using only Q *= 2.
*/
static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int np, int generator)
static __always_inline void raid_genX_neon_x2(int nd, size_t size,
void **vv, int np)
{
uint8_t **v = (uint8_t **)vv;
size_t i;
@ -276,7 +403,7 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
);
for (i = 0; i < size; i += 32) {
/* last disk without the generator multiplication */
/* last disk initializes P/Q without a preceding Q transition */
asm volatile (
"ldr q0, %0\n"
"ldr q1, %1\n"
@ -308,9 +435,11 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
"tbl v20.16b, {v23.16b}, v19.16b\n"
"eor v5.16b, v5.16b, v20.16b\n"
:
: "m" (raid_gfcauchypshufb[l][1][0][0]), "m" (raid_gfcauchypshufb[l][1][1][0])
: "m" (raid_gfcauchypshufb[l][1][0][0]),
"m" (raid_gfcauchypshufb[l][1][1][0])
);
}
if (np >= 4) {
asm volatile (
"ldr q22, %0\n"
@ -323,9 +452,11 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
"tbl v20.16b, {v23.16b}, v19.16b\n"
"eor v7.16b, v7.16b, v20.16b\n"
:
: "m" (raid_gfcauchypshufb[l][2][0][0]), "m" (raid_gfcauchypshufb[l][2][1][0])
: "m" (raid_gfcauchypshufb[l][2][0][0]),
"m" (raid_gfcauchypshufb[l][2][1][0])
);
}
if (np >= 5) {
asm volatile (
"ldr q22, %0\n"
@ -338,9 +469,11 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
"tbl v20.16b, {v23.16b}, v19.16b\n"
"eor v9.16b, v9.16b, v20.16b\n"
:
: "m" (raid_gfcauchypshufb[l][3][0][0]), "m" (raid_gfcauchypshufb[l][3][1][0])
: "m" (raid_gfcauchypshufb[l][3][0][0]),
"m" (raid_gfcauchypshufb[l][3][1][0])
);
}
if (np >= 6) {
asm volatile (
"ldr q22, %0\n"
@ -353,18 +486,13 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
"tbl v20.16b, {v23.16b}, v19.16b\n"
"eor v11.16b, v11.16b, v20.16b\n"
:
: "m" (raid_gfcauchypshufb[l][4][0][0]), "m" (raid_gfcauchypshufb[l][4][1][0])
: "m" (raid_gfcauchypshufb[l][4][0][0]),
"m" (raid_gfcauchypshufb[l][4][1][0])
);
}
/* intermediate disks */
for (d = l - 1; d > 0; --d) {
if (generator == 3) {
asm volatile (
"mov v14.16b, v2.16b\n"
"mov v15.16b, v3.16b\n"
);
}
asm volatile (
/* double Q0-Q1 */
"sshr v20.16b, v2.16b, #7\n"
@ -387,12 +515,6 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
:
: "m" (v[d][i]), "m" (v[d][i + 16])
);
if (generator == 3) {
asm volatile (
"eor v2.16b, v2.16b, v14.16b\n"
"eor v3.16b, v3.16b, v15.16b\n"
);
}
if (np >= 3) {
asm volatile (
@ -418,9 +540,11 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
"eor v20.16b, v20.16b, v21.16b\n"
"eor v5.16b, v5.16b, v20.16b\n"
:
: "m" (raid_gfcauchypshufb[d][1][0][0]), "m" (raid_gfcauchypshufb[d][1][1][0])
: "m" (raid_gfcauchypshufb[d][1][0][0]),
"m" (raid_gfcauchypshufb[d][1][1][0])
);
}
if (np >= 4) {
asm volatile (
"ldr q22, %0\n"
@ -435,9 +559,11 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
"eor v20.16b, v20.16b, v21.16b\n"
"eor v7.16b, v7.16b, v20.16b\n"
:
: "m" (raid_gfcauchypshufb[d][2][0][0]), "m" (raid_gfcauchypshufb[d][2][1][0])
: "m" (raid_gfcauchypshufb[d][2][0][0]),
"m" (raid_gfcauchypshufb[d][2][1][0])
);
}
if (np >= 5) {
asm volatile (
"ldr q22, %0\n"
@ -452,9 +578,11 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
"eor v20.16b, v20.16b, v21.16b\n"
"eor v9.16b, v9.16b, v20.16b\n"
:
: "m" (raid_gfcauchypshufb[d][3][0][0]), "m" (raid_gfcauchypshufb[d][3][1][0])
: "m" (raid_gfcauchypshufb[d][3][0][0]),
"m" (raid_gfcauchypshufb[d][3][1][0])
);
}
if (np >= 6) {
asm volatile (
"ldr q22, %0\n"
@ -469,18 +597,13 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
"eor v20.16b, v20.16b, v21.16b\n"
"eor v11.16b, v11.16b, v20.16b\n"
:
: "m" (raid_gfcauchypshufb[d][4][0][0]), "m" (raid_gfcauchypshufb[d][4][1][0])
: "m" (raid_gfcauchypshufb[d][4][0][0]),
"m" (raid_gfcauchypshufb[d][4][1][0])
);
}
}
/* first disk with all coefficients at 1 */
if (generator == 3) {
asm volatile (
"mov v14.16b, v2.16b\n"
"mov v15.16b, v3.16b\n"
);
}
asm volatile (
/* double Q0-Q1 */
"sshr v20.16b, v2.16b, #7\n"
@ -503,12 +626,6 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
:
: "m" (v[0][i]), "m" (v[0][i + 16])
);
if (generator == 3) {
asm volatile (
"eor v2.16b, v2.16b, v14.16b\n"
"eor v3.16b, v3.16b, v15.16b\n"
);
}
if (np >= 3) {
asm volatile (
@ -535,13 +652,560 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
);
}
/* write parity in increasing order */
asm volatile (
"str q0, %0\n"
"str q1, %1\n"
"str q2, %2\n"
"str q3, %3\n"
: "=m" (v[nd][i]), "=m" (v[nd][i + 16]),
"=m" (v[nd + 1][i]), "=m" (v[nd + 1][i + 16])
"=m" (v[nd + 1][i]), "=m" (v[nd + 1][i + 16])
);
if (np >= 3) {
asm volatile (
"str q4, %0\n"
"str q5, %1\n"
: "=m" (v[nd + 2][i]), "=m" (v[nd + 2][i + 16])
);
}
if (np >= 4) {
asm volatile (
"str q6, %0\n"
"str q7, %1\n"
: "=m" (v[nd + 3][i]), "=m" (v[nd + 3][i + 16])
);
}
if (np >= 5) {
asm volatile (
"str q8, %0\n"
"str q9, %1\n"
: "=m" (v[nd + 4][i]), "=m" (v[nd + 4][i + 16])
);
}
if (np >= 6) {
asm volatile (
"str q10, %0\n"
"str q11, %1\n"
: "=m" (v[nd + 5][i]), "=m" (v[nd + 5][i + 16])
);
}
}
raid_neon_end();
}
/*
* GENX NEON implementation using the AES G23 Q recurrence.
*/
static __always_inline void raid_genX_neon_g23(int nd, size_t size,
void **vv, int np)
{
uint8_t **v = (uint8_t **)vv;
size_t i;
int d, l;
l = nd - 1;
/* special case with only one data disk */
if (l == 0) {
for (d = 0; d < np; ++d)
memcpy(v[1 + d], v[0], size);
return;
}
int g23_boundary = raid_g23_boundary(l - 1);
raid_neon_begin();
asm volatile (
"ldr q29, %0\n"
"ldr q28, %1\n"
:
: "m" (gfconst16.poly[0]), "m" (gfconst16.low4[0])
);
for (i = 0; i < size; i += 32) {
int boundary = g23_boundary;
/* last disk initializes P/Q without a preceding Q transition */
asm volatile (
"ldr q0, %0\n"
"ldr q1, %1\n"
"mov v2.16b, v0.16b\n"
"mov v3.16b, v1.16b\n"
:
: "m" (v[l][i]), "m" (v[l][i + 16])
);
if (np >= 3) {
asm volatile (
"ushr v17.16b, v0.16b, #4\n"
"and v16.16b, v0.16b, v28.16b\n"
"and v17.16b, v17.16b, v28.16b\n"
"ushr v19.16b, v1.16b, #4\n"
"and v18.16b, v1.16b, v28.16b\n"
"and v19.16b, v19.16b, v28.16b\n"
);
asm volatile (
"ldr q22, %0\n"
"ldr q23, %1\n"
"tbl v4.16b, {v22.16b}, v16.16b\n"
"tbl v20.16b, {v23.16b}, v17.16b\n"
"eor v4.16b, v4.16b, v20.16b\n"
"tbl v5.16b, {v22.16b}, v18.16b\n"
"tbl v20.16b, {v23.16b}, v19.16b\n"
"eor v5.16b, v5.16b, v20.16b\n"
:
: "m" (raid_gfcauchypshufb[l][1][0][0]),
"m" (raid_gfcauchypshufb[l][1][1][0])
);
}
if (np >= 4) {
asm volatile (
"ldr q22, %0\n"
"ldr q23, %1\n"
"tbl v6.16b, {v22.16b}, v16.16b\n"
"tbl v20.16b, {v23.16b}, v17.16b\n"
"eor v6.16b, v6.16b, v20.16b\n"
"tbl v7.16b, {v22.16b}, v18.16b\n"
"tbl v20.16b, {v23.16b}, v19.16b\n"
"eor v7.16b, v7.16b, v20.16b\n"
:
: "m" (raid_gfcauchypshufb[l][2][0][0]),
"m" (raid_gfcauchypshufb[l][2][1][0])
);
}
if (np >= 5) {
asm volatile (
"ldr q22, %0\n"
"ldr q23, %1\n"
"tbl v8.16b, {v22.16b}, v16.16b\n"
"tbl v20.16b, {v23.16b}, v17.16b\n"
"eor v8.16b, v8.16b, v20.16b\n"
"tbl v9.16b, {v22.16b}, v18.16b\n"
"tbl v20.16b, {v23.16b}, v19.16b\n"
"eor v9.16b, v9.16b, v20.16b\n"
:
: "m" (raid_gfcauchypshufb[l][3][0][0]),
"m" (raid_gfcauchypshufb[l][3][1][0])
);
}
if (np >= 6) {
asm volatile (
"ldr q22, %0\n"
"ldr q23, %1\n"
"tbl v10.16b, {v22.16b}, v16.16b\n"
"tbl v20.16b, {v23.16b}, v17.16b\n"
"eor v10.16b, v10.16b, v20.16b\n"
"tbl v11.16b, {v22.16b}, v18.16b\n"
"tbl v20.16b, {v23.16b}, v19.16b\n"
"eor v11.16b, v11.16b, v20.16b\n"
:
: "m" (raid_gfcauchypshufb[l][4][0][0]),
"m" (raid_gfcauchypshufb[l][4][1][0])
);
}
d = l - 1;
while (boundary >= 0) {
/*
* All transitions above the next G23 boundary use Q *= 2.
*/
for (; d > boundary; --d) {
asm volatile (
"sshr v20.16b, v2.16b, #7\n"
"shl v2.16b, v2.16b, #1\n"
"and v20.16b, v20.16b, v29.16b\n"
"eor v2.16b, v2.16b, v20.16b\n"
"sshr v20.16b, v3.16b, #7\n"
"shl v3.16b, v3.16b, #1\n"
"and v20.16b, v20.16b, v29.16b\n"
"eor v3.16b, v3.16b, v20.16b\n"
"ldr q12, %0\n"
"ldr q13, %1\n"
"eor v0.16b, v0.16b, v12.16b\n"
"eor v1.16b, v1.16b, v13.16b\n"
"eor v2.16b, v2.16b, v12.16b\n"
"eor v3.16b, v3.16b, v13.16b\n"
:
: "m" (v[d][i]), "m" (v[d][i + 16])
);
if (np >= 3) {
asm volatile (
"ushr v17.16b, v12.16b, #4\n"
"and v16.16b, v12.16b, v28.16b\n"
"and v17.16b, v17.16b, v28.16b\n"
"ushr v19.16b, v13.16b, #4\n"
"and v18.16b, v13.16b, v28.16b\n"
"and v19.16b, v19.16b, v28.16b\n"
);
asm volatile (
"ldr q22, %0\n"
"ldr q23, %1\n"
"tbl v20.16b, {v22.16b}, v16.16b\n"
"tbl v21.16b, {v23.16b}, v17.16b\n"
"eor v20.16b, v20.16b, v21.16b\n"
"eor v4.16b, v4.16b, v20.16b\n"
"tbl v20.16b, {v22.16b}, v18.16b\n"
"tbl v21.16b, {v23.16b}, v19.16b\n"
"eor v20.16b, v20.16b, v21.16b\n"
"eor v5.16b, v5.16b, v20.16b\n"
:
: "m" (raid_gfcauchypshufb[d][1][0][0]),
"m" (raid_gfcauchypshufb[d][1][1][0])
);
}
if (np >= 4) {
asm volatile (
"ldr q22, %0\n"
"ldr q23, %1\n"
"tbl v20.16b, {v22.16b}, v16.16b\n"
"tbl v21.16b, {v23.16b}, v17.16b\n"
"eor v20.16b, v20.16b, v21.16b\n"
"eor v6.16b, v6.16b, v20.16b\n"
"tbl v20.16b, {v22.16b}, v18.16b\n"
"tbl v21.16b, {v23.16b}, v19.16b\n"
"eor v20.16b, v20.16b, v21.16b\n"
"eor v7.16b, v7.16b, v20.16b\n"
:
: "m" (raid_gfcauchypshufb[d][2][0][0]),
"m" (raid_gfcauchypshufb[d][2][1][0])
);
}
if (np >= 5) {
asm volatile (
"ldr q22, %0\n"
"ldr q23, %1\n"
"tbl v20.16b, {v22.16b}, v16.16b\n"
"tbl v21.16b, {v23.16b}, v17.16b\n"
"eor v20.16b, v20.16b, v21.16b\n"
"eor v8.16b, v8.16b, v20.16b\n"
"tbl v20.16b, {v22.16b}, v18.16b\n"
"tbl v21.16b, {v23.16b}, v19.16b\n"
"eor v20.16b, v20.16b, v21.16b\n"
"eor v9.16b, v9.16b, v20.16b\n"
:
: "m" (raid_gfcauchypshufb[d][3][0][0]),
"m" (raid_gfcauchypshufb[d][3][1][0])
);
}
if (np >= 6) {
asm volatile (
"ldr q22, %0\n"
"ldr q23, %1\n"
"tbl v20.16b, {v22.16b}, v16.16b\n"
"tbl v21.16b, {v23.16b}, v17.16b\n"
"eor v20.16b, v20.16b, v21.16b\n"
"eor v10.16b, v10.16b, v20.16b\n"
"tbl v20.16b, {v22.16b}, v18.16b\n"
"tbl v21.16b, {v23.16b}, v19.16b\n"
"eor v20.16b, v20.16b, v21.16b\n"
"eor v11.16b, v11.16b, v20.16b\n"
:
: "m" (raid_gfcauchypshufb[d][4][0][0]),
"m" (raid_gfcauchypshufb[d][4][1][0])
);
}
}
/*
* d == boundary.
*
* Preserve Q in v14/v15, perform Q *= 2, then XOR
* the old value back to obtain Q *= 3.
*/
asm volatile (
"mov v14.16b, v2.16b\n"
"mov v15.16b, v3.16b\n"
"sshr v20.16b, v2.16b, #7\n"
"shl v2.16b, v2.16b, #1\n"
"and v20.16b, v20.16b, v29.16b\n"
"eor v2.16b, v2.16b, v20.16b\n"
"sshr v20.16b, v3.16b, #7\n"
"shl v3.16b, v3.16b, #1\n"
"and v20.16b, v20.16b, v29.16b\n"
"eor v3.16b, v3.16b, v20.16b\n"
"ldr q12, %0\n"
"ldr q13, %1\n"
"eor v0.16b, v0.16b, v12.16b\n"
"eor v1.16b, v1.16b, v13.16b\n"
"eor v2.16b, v2.16b, v12.16b\n"
"eor v3.16b, v3.16b, v13.16b\n"
"eor v2.16b, v2.16b, v14.16b\n"
"eor v3.16b, v3.16b, v15.16b\n"
:
: "m" (v[d][i]), "m" (v[d][i + 16])
);
if (np >= 3) {
asm volatile (
"ushr v17.16b, v12.16b, #4\n"
"and v16.16b, v12.16b, v28.16b\n"
"and v17.16b, v17.16b, v28.16b\n"
"ushr v19.16b, v13.16b, #4\n"
"and v18.16b, v13.16b, v28.16b\n"
"and v19.16b, v19.16b, v28.16b\n"
);
asm volatile (
"ldr q22, %0\n"
"ldr q23, %1\n"
"tbl v20.16b, {v22.16b}, v16.16b\n"
"tbl v21.16b, {v23.16b}, v17.16b\n"
"eor v20.16b, v20.16b, v21.16b\n"
"eor v4.16b, v4.16b, v20.16b\n"
"tbl v20.16b, {v22.16b}, v18.16b\n"
"tbl v21.16b, {v23.16b}, v19.16b\n"
"eor v20.16b, v20.16b, v21.16b\n"
"eor v5.16b, v5.16b, v20.16b\n"
:
: "m" (raid_gfcauchypshufb[d][1][0][0]),
"m" (raid_gfcauchypshufb[d][1][1][0])
);
}
if (np >= 4) {
asm volatile (
"ldr q22, %0\n"
"ldr q23, %1\n"
"tbl v20.16b, {v22.16b}, v16.16b\n"
"tbl v21.16b, {v23.16b}, v17.16b\n"
"eor v20.16b, v20.16b, v21.16b\n"
"eor v6.16b, v6.16b, v20.16b\n"
"tbl v20.16b, {v22.16b}, v18.16b\n"
"tbl v21.16b, {v23.16b}, v19.16b\n"
"eor v20.16b, v20.16b, v21.16b\n"
"eor v7.16b, v7.16b, v20.16b\n"
:
: "m" (raid_gfcauchypshufb[d][2][0][0]),
"m" (raid_gfcauchypshufb[d][2][1][0])
);
}
if (np >= 5) {
asm volatile (
"ldr q22, %0\n"
"ldr q23, %1\n"
"tbl v20.16b, {v22.16b}, v16.16b\n"
"tbl v21.16b, {v23.16b}, v17.16b\n"
"eor v20.16b, v20.16b, v21.16b\n"
"eor v8.16b, v8.16b, v20.16b\n"
"tbl v20.16b, {v22.16b}, v18.16b\n"
"tbl v21.16b, {v23.16b}, v19.16b\n"
"eor v20.16b, v20.16b, v21.16b\n"
"eor v9.16b, v9.16b, v20.16b\n"
:
: "m" (raid_gfcauchypshufb[d][3][0][0]),
"m" (raid_gfcauchypshufb[d][3][1][0])
);
}
if (np >= 6) {
asm volatile (
"ldr q22, %0\n"
"ldr q23, %1\n"
"tbl v20.16b, {v22.16b}, v16.16b\n"
"tbl v21.16b, {v23.16b}, v17.16b\n"
"eor v20.16b, v20.16b, v21.16b\n"
"eor v10.16b, v10.16b, v20.16b\n"
"tbl v20.16b, {v22.16b}, v18.16b\n"
"tbl v21.16b, {v23.16b}, v19.16b\n"
"eor v20.16b, v20.16b, v21.16b\n"
"eor v11.16b, v11.16b, v20.16b\n"
:
: "m" (raid_gfcauchypshufb[d][4][0][0]),
"m" (raid_gfcauchypshufb[d][4][1][0])
);
}
--d;
boundary -= 51;
if (boundary < 50)
boundary = -1;
}
/*
* No G23 boundaries remain. Process D[d] ... D1 using
* only Q *= 2.
*/
for (; d > 0; --d) {
asm volatile (
"sshr v20.16b, v2.16b, #7\n"
"shl v2.16b, v2.16b, #1\n"
"and v20.16b, v20.16b, v29.16b\n"
"eor v2.16b, v2.16b, v20.16b\n"
"sshr v20.16b, v3.16b, #7\n"
"shl v3.16b, v3.16b, #1\n"
"and v20.16b, v20.16b, v29.16b\n"
"eor v3.16b, v3.16b, v20.16b\n"
"ldr q12, %0\n"
"ldr q13, %1\n"
"eor v0.16b, v0.16b, v12.16b\n"
"eor v1.16b, v1.16b, v13.16b\n"
"eor v2.16b, v2.16b, v12.16b\n"
"eor v3.16b, v3.16b, v13.16b\n"
:
: "m" (v[d][i]), "m" (v[d][i + 16])
);
if (np >= 3) {
asm volatile (
"ushr v17.16b, v12.16b, #4\n"
"and v16.16b, v12.16b, v28.16b\n"
"and v17.16b, v17.16b, v28.16b\n"
"ushr v19.16b, v13.16b, #4\n"
"and v18.16b, v13.16b, v28.16b\n"
"and v19.16b, v19.16b, v28.16b\n"
);
asm volatile (
"ldr q22, %0\n"
"ldr q23, %1\n"
"tbl v20.16b, {v22.16b}, v16.16b\n"
"tbl v21.16b, {v23.16b}, v17.16b\n"
"eor v20.16b, v20.16b, v21.16b\n"
"eor v4.16b, v4.16b, v20.16b\n"
"tbl v20.16b, {v22.16b}, v18.16b\n"
"tbl v21.16b, {v23.16b}, v19.16b\n"
"eor v20.16b, v20.16b, v21.16b\n"
"eor v5.16b, v5.16b, v20.16b\n"
:
: "m" (raid_gfcauchypshufb[d][1][0][0]),
"m" (raid_gfcauchypshufb[d][1][1][0])
);
}
if (np >= 4) {
asm volatile (
"ldr q22, %0\n"
"ldr q23, %1\n"
"tbl v20.16b, {v22.16b}, v16.16b\n"
"tbl v21.16b, {v23.16b}, v17.16b\n"
"eor v20.16b, v20.16b, v21.16b\n"
"eor v6.16b, v6.16b, v20.16b\n"
"tbl v20.16b, {v22.16b}, v18.16b\n"
"tbl v21.16b, {v23.16b}, v19.16b\n"
"eor v20.16b, v20.16b, v21.16b\n"
"eor v7.16b, v7.16b, v20.16b\n"
:
: "m" (raid_gfcauchypshufb[d][2][0][0]),
"m" (raid_gfcauchypshufb[d][2][1][0])
);
}
if (np >= 5) {
asm volatile (
"ldr q22, %0\n"
"ldr q23, %1\n"
"tbl v20.16b, {v22.16b}, v16.16b\n"
"tbl v21.16b, {v23.16b}, v17.16b\n"
"eor v20.16b, v20.16b, v21.16b\n"
"eor v8.16b, v8.16b, v20.16b\n"
"tbl v20.16b, {v22.16b}, v18.16b\n"
"tbl v21.16b, {v23.16b}, v19.16b\n"
"eor v20.16b, v20.16b, v21.16b\n"
"eor v9.16b, v9.16b, v20.16b\n"
:
: "m" (raid_gfcauchypshufb[d][3][0][0]),
"m" (raid_gfcauchypshufb[d][3][1][0])
);
}
if (np >= 6) {
asm volatile (
"ldr q22, %0\n"
"ldr q23, %1\n"
"tbl v20.16b, {v22.16b}, v16.16b\n"
"tbl v21.16b, {v23.16b}, v17.16b\n"
"eor v20.16b, v20.16b, v21.16b\n"
"eor v10.16b, v10.16b, v20.16b\n"
"tbl v20.16b, {v22.16b}, v18.16b\n"
"tbl v21.16b, {v23.16b}, v19.16b\n"
"eor v20.16b, v20.16b, v21.16b\n"
"eor v11.16b, v11.16b, v20.16b\n"
:
: "m" (raid_gfcauchypshufb[d][4][0][0]),
"m" (raid_gfcauchypshufb[d][4][1][0])
);
}
}
/*
* D0 always follows Q *= 2. The lowest G23 boundary is 50.
*/
asm volatile (
"sshr v20.16b, v2.16b, #7\n"
"shl v2.16b, v2.16b, #1\n"
"and v20.16b, v20.16b, v29.16b\n"
"eor v2.16b, v2.16b, v20.16b\n"
"sshr v20.16b, v3.16b, #7\n"
"shl v3.16b, v3.16b, #1\n"
"and v20.16b, v20.16b, v29.16b\n"
"eor v3.16b, v3.16b, v20.16b\n"
"ldr q12, %0\n"
"ldr q13, %1\n"
"eor v0.16b, v0.16b, v12.16b\n"
"eor v1.16b, v1.16b, v13.16b\n"
"eor v2.16b, v2.16b, v12.16b\n"
"eor v3.16b, v3.16b, v13.16b\n"
:
: "m" (v[0][i]), "m" (v[0][i + 16])
);
if (np >= 3) {
asm volatile (
"eor v4.16b, v4.16b, v12.16b\n"
"eor v5.16b, v5.16b, v13.16b\n"
);
}
if (np >= 4) {
asm volatile (
"eor v6.16b, v6.16b, v12.16b\n"
"eor v7.16b, v7.16b, v13.16b\n"
);
}
if (np >= 5) {
asm volatile (
"eor v8.16b, v8.16b, v12.16b\n"
"eor v9.16b, v9.16b, v13.16b\n"
);
}
if (np >= 6) {
asm volatile (
"eor v10.16b, v10.16b, v12.16b\n"
"eor v11.16b, v11.16b, v13.16b\n"
);
}
/* write parity in increasing order */
asm volatile (
"str q0, %0\n"
"str q1, %1\n"
"str q2, %2\n"
"str q3, %3\n"
: "=m" (v[nd][i]), "=m" (v[nd][i + 16]),
"=m" (v[nd + 1][i]), "=m" (v[nd + 1][i + 16])
);
if (np >= 3) {
@ -582,12 +1246,15 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
*/
void raid_gen3_neon_raid(int nd, size_t size, void **vv)
{
raid_genX_neon(nd, size, vv, 3, 2);
raid_genX_neon_x2(nd, size, vv, 3);
}
void raid_gen3_neon_aes(int nd, size_t size, void **vv)
{
raid_genX_neon(nd, size, vv, 3, 3);
if (raid_g23_x2_only(nd))
raid_genX_neon_x2(nd, size, vv, 3);
else
raid_genX_neon_g23(nd, size, vv, 3);
}
/*
@ -595,12 +1262,15 @@ void raid_gen3_neon_aes(int nd, size_t size, void **vv)
*/
void raid_gen4_neon_raid(int nd, size_t size, void **vv)
{
raid_genX_neon(nd, size, vv, 4, 2);
raid_genX_neon_x2(nd, size, vv, 4);
}
void raid_gen4_neon_aes(int nd, size_t size, void **vv)
{
raid_genX_neon(nd, size, vv, 4, 3);
if (raid_g23_x2_only(nd))
raid_genX_neon_x2(nd, size, vv, 4);
else
raid_genX_neon_g23(nd, size, vv, 4);
}
/*
@ -608,12 +1278,15 @@ void raid_gen4_neon_aes(int nd, size_t size, void **vv)
*/
void raid_gen5_neon_raid(int nd, size_t size, void **vv)
{
raid_genX_neon(nd, size, vv, 5, 2);
raid_genX_neon_x2(nd, size, vv, 5);
}
void raid_gen5_neon_aes(int nd, size_t size, void **vv)
{
raid_genX_neon(nd, size, vv, 5, 3);
if (raid_g23_x2_only(nd))
raid_genX_neon_x2(nd, size, vv, 5);
else
raid_genX_neon_g23(nd, size, vv, 5);
}
/*
@ -621,12 +1294,15 @@ void raid_gen5_neon_aes(int nd, size_t size, void **vv)
*/
void raid_gen6_neon_raid(int nd, size_t size, void **vv)
{
raid_genX_neon(nd, size, vv, 6, 2);
raid_genX_neon_x2(nd, size, vv, 6);
}
void raid_gen6_neon_aes(int nd, size_t size, void **vv)
{
raid_genX_neon(nd, size, vv, 6, 3);
if (raid_g23_x2_only(nd))
raid_genX_neon_x2(nd, size, vv, 6);
else
raid_genX_neon_g23(nd, size, vv, 6);
}
/*

700
raid/neon32.c
View file

@ -63,10 +63,9 @@ void raid_gen1_neon32(int nd, size_t size, void **vv)
}
/*
* GEN2 Cauchy AArch32 NEON implementation using the active generator.
* GEN2 Cauchy AArch32 NEON implementation using only Q *= 2.
*/
static __always_inline void raid_gen2_neon32_gen(int nd, size_t size,
void **vv, int generator)
static __always_inline void raid_gen2_neon32_x2(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -98,21 +97,19 @@ static __always_inline void raid_gen2_neon32_gen(int nd, size_t size,
);
for (d = l - 1; d >= 0; --d) {
if (generator == 3) {
asm volatile (
"vmov q10, q2\n"
"vmov q11, q3\n"
);
}
asm volatile (
/* double Q0-Q1 */
"vshr.s8 q8, q2, #7\n"
"vshl.i8 q2, q2, #1\n"
"vand q8, q8, q14\n"
"veor q2, q2, q8\n"
"vshr.s8 q8, q3, #7\n"
"vshl.i8 q3, q3, #1\n"
"vand q8, q8, q14\n"
"veor q3, q3, q8\n"
/* load and XOR */
"vld1.8 {q12}, %0\n"
"vld1.8 {q13}, %1\n"
"veor q0, q0, q12\n"
@ -122,12 +119,143 @@ static __always_inline void raid_gen2_neon32_gen(int nd, size_t size,
:
: "Q" (v[d][i]), "Q" (v[d][i + 16])
);
if (generator == 3) {
}
asm volatile (
"vst1.8 {q0}, %0\n"
"vst1.8 {q1}, %1\n"
"vst1.8 {q2}, %2\n"
"vst1.8 {q3}, %3\n"
: "=Q" (p[i]), "=Q" (p[i + 16]),
"=Q" (q[i]), "=Q" (q[i + 16])
);
}
raid_neon32_end();
}
/*
* GEN2 Cauchy AArch32 NEON implementation using the AES G23 Q recurrence.
*/
static __always_inline void raid_gen2_neon32_g23(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
uint8_t *q;
int d, l;
size_t i;
l = nd - 1;
p = v[nd];
q = v[nd + 1];
int g23_boundary = raid_g23_boundary(l - 1);
raid_neon32_begin();
/* q14 contains the active reduction polynomial. */
asm volatile (
"vld1.8 {q14}, %0\n"
:
: "Q" (gfconst16.poly[0])
);
for (i = 0; i < size; i += 32) {
int boundary = g23_boundary;
asm volatile (
"vld1.8 {q0}, %0\n"
"vld1.8 {q1}, %1\n"
"vmov q2, q0\n"
"vmov q3, q1\n"
:
: "Q" (v[l][i]), "Q" (v[l][i + 16])
);
d = l - 1;
for (;;) {
/*
* All transitions above 'boundary' are x2.
*/
for (; d > boundary; --d) {
asm volatile (
"veor q2, q2, q10\n"
"veor q3, q3, q11\n"
/* double Q0-Q1 */
"vshr.s8 q8, q2, #7\n"
"vshl.i8 q2, q2, #1\n"
"vand q8, q8, q14\n"
"veor q2, q2, q8\n"
"vshr.s8 q8, q3, #7\n"
"vshl.i8 q3, q3, #1\n"
"vand q8, q8, q14\n"
"veor q3, q3, q8\n"
/* load and XOR */
"vld1.8 {q12}, %0\n"
"vld1.8 {q13}, %1\n"
"veor q0, q0, q12\n"
"veor q1, q1, q13\n"
"veor q2, q2, q12\n"
"veor q3, q3, q13\n"
:
: "Q" (v[d][i]), "Q" (v[d][i + 16])
);
}
/*
* boundary == -1 identifies the final x2-only segment.
* At this point all remaining disks have been consumed.
*/
if (boundary < 0)
break;
/*
* d == boundary.
*
* Perform the fixed G23 x3 transition and then add
* the current disk to P and Q.
*/
asm volatile (
/* preserve Q for Q *= 3 */
"vmov q10, q2\n"
"vmov q11, q3\n"
/* double Q0-Q1 */
"vshr.s8 q8, q2, #7\n"
"vshl.i8 q2, q2, #1\n"
"vand q8, q8, q14\n"
"veor q2, q2, q8\n"
"vshr.s8 q8, q3, #7\n"
"vshl.i8 q3, q3, #1\n"
"vand q8, q8, q14\n"
"veor q3, q3, q8\n"
/* load and XOR */
"vld1.8 {q12}, %0\n"
"vld1.8 {q13}, %1\n"
"veor q0, q0, q12\n"
"veor q1, q1, q13\n"
"veor q2, q2, q12\n"
"veor q3, q3, q13\n"
/* complete Q *= 3 */
"veor q2, q2, q10\n"
"veor q3, q3, q11\n"
:
: "Q" (v[d][i]), "Q" (v[d][i + 16])
);
--d;
/*
* G23 boundaries are exactly 51 transitions apart.
* After d = 50 there are no more x3 transitions.
*/
boundary -= 51;
if (boundary < 50)
boundary = -1;
}
asm volatile (
@ -145,12 +273,15 @@ static __always_inline void raid_gen2_neon32_gen(int nd, size_t size,
void raid_gen2_neon32_raid(int nd, size_t size, void **vv)
{
raid_gen2_neon32_gen(nd, size, vv, 2);
raid_gen2_neon32_x2(nd, size, vv);
}
void raid_gen2_neon32_aes(int nd, size_t size, void **vv)
{
raid_gen2_neon32_gen(nd, size, vv, 3);
if (raid_g23_x2_only(nd))
raid_gen2_neon32_x2(nd, size, vv);
else
raid_gen2_neon32_g23(nd, size, vv);
}
/*
@ -241,10 +372,10 @@ void raid_genz_neon32_raid(int nd, size_t size, void **vv)
}
/*
* GENX AArch32 NEON implementation.
* GENX AArch32 NEON implementation using only Q *= 2.
*/
static __always_inline void raid_genX_neon32(int nd, size_t size,
void **vv, int np, int generator)
static __always_inline void raid_genX_neon32_x2(int nd, size_t size,
void **vv, int np)
{
uint8_t **v = (uint8_t **)vv;
size_t i;
@ -269,7 +400,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
);
for (i = 0; i < size; i += 16) {
/* last disk without the generator multiplication */
/* last disk initializes P/Q without a preceding Q transition */
asm volatile (
"vld1.8 {q0}, %0\n"
"vld1.8 {q1}, %0\n"
@ -296,7 +427,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
"veor q2, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][1][0][0]),
"Q" (raid_gfcauchypshufb[l][1][1][0])
"Q" (raid_gfcauchypshufb[l][1][1][0])
);
}
@ -313,7 +444,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
"veor q3, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][2][0][0]),
"Q" (raid_gfcauchypshufb[l][2][1][0])
"Q" (raid_gfcauchypshufb[l][2][1][0])
);
}
@ -330,7 +461,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
"veor q4, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][3][0][0]),
"Q" (raid_gfcauchypshufb[l][3][1][0])
"Q" (raid_gfcauchypshufb[l][3][1][0])
);
}
@ -347,33 +478,26 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
"veor q5, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][4][0][0]),
"Q" (raid_gfcauchypshufb[l][4][1][0])
"Q" (raid_gfcauchypshufb[l][4][1][0])
);
}
/* intermediate disks */
for (d = l - 1; d > 0; --d) {
if (generator == 3) {
asm volatile (
"vmov q13, q1\n"
);
}
asm volatile (
/* Q *= 2 */
"vshr.s8 q11, q1, #7\n"
"vshl.i8 q1, q1, #1\n"
"vand q11, q11, q14\n"
"veor q1, q1, q11\n"
/* load disk and update P/Q */
"vld1.8 {q6}, %0\n"
"veor q0, q0, q6\n"
"veor q1, q1, q6\n"
:
: "Q" (v[d][i])
);
if (generator == 3) {
asm volatile (
"veor q1, q1, q13\n"
);
}
if (np >= 3) {
asm volatile (
@ -381,6 +505,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
"vand q7, q6, q15\n"
"vand q8, q8, q15\n"
);
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
@ -394,7 +519,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
"veor q2, q2, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][1][0][0]),
"Q" (raid_gfcauchypshufb[d][1][1][0])
"Q" (raid_gfcauchypshufb[d][1][1][0])
);
}
@ -410,7 +535,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
"veor q3, q3, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][2][0][0]),
"Q" (raid_gfcauchypshufb[d][2][1][0])
"Q" (raid_gfcauchypshufb[d][2][1][0])
);
}
@ -426,7 +551,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
"veor q4, q4, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][3][0][0]),
"Q" (raid_gfcauchypshufb[d][3][1][0])
"Q" (raid_gfcauchypshufb[d][3][1][0])
);
}
@ -442,87 +567,484 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
"veor q5, q5, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][4][0][0]),
"Q" (raid_gfcauchypshufb[d][4][1][0])
"Q" (raid_gfcauchypshufb[d][4][1][0])
);
}
}
/* first disk with all coefficients at 1 */
if (generator == 3) {
asm volatile (
"vmov q13, q1\n"
);
}
asm volatile (
"vshr.s8 q11, q1, #7\n"
"vshl.i8 q1, q1, #1\n"
"vand q11, q11, q14\n"
"veor q1, q1, q11\n"
"vld1.8 {q6}, %0\n"
"veor q0, q0, q6\n"
"veor q1, q1, q6\n"
:
: "Q" (v[0][i])
);
if (generator == 3) {
asm volatile (
"veor q1, q1, q13\n"
);
}
if (np >= 3) {
asm volatile (
"veor q2, q2, q6\n"
);
}
if (np >= 4) {
asm volatile (
"veor q3, q3, q6\n"
);
}
if (np >= 5) {
asm volatile (
"veor q4, q4, q6\n"
);
}
if (np >= 6) {
asm volatile (
"veor q5, q5, q6\n"
);
}
if (np >= 3)
asm volatile ("veor q2, q2, q6\n");
if (np >= 4)
asm volatile ("veor q3, q3, q6\n");
if (np >= 5)
asm volatile ("veor q4, q4, q6\n");
if (np >= 6)
asm volatile ("veor q5, q5, q6\n");
/* write parity in increasing order */
asm volatile (
"vst1.8 {q0}, %0\n"
"vst1.8 {q1}, %1\n"
: "=Q" (v[nd][i]),
"=Q" (v[nd + 1][i])
"=Q" (v[nd + 1][i])
);
if (np >= 3)
asm volatile ("vst1.8 {q2}, %0\n" : "=Q" (v[nd + 2][i]));
if (np >= 4)
asm volatile ("vst1.8 {q3}, %0\n" : "=Q" (v[nd + 3][i]));
if (np >= 5)
asm volatile ("vst1.8 {q4}, %0\n" : "=Q" (v[nd + 4][i]));
if (np >= 6)
asm volatile ("vst1.8 {q5}, %0\n" : "=Q" (v[nd + 5][i]));
}
raid_neon32_end();
}
/*
* GENX AArch32 NEON implementation using the AES G23 Q recurrence.
*/
static __always_inline void raid_genX_neon32_g23(int nd, size_t size,
void **vv, int np)
{
uint8_t **v = (uint8_t **)vv;
size_t i;
int d, l;
l = nd - 1;
/* special case with only one data disk */
if (l == 0) {
for (d = 0; d < np; ++d)
memcpy(v[1 + d], v[0], size);
return;
}
int g23_boundary = raid_g23_boundary(l - 1);
raid_neon32_begin();
asm volatile (
"vld1.8 {q14}, %0\n"
"vld1.8 {q15}, %1\n"
:
: "Q" (gfconst16.poly[0]), "Q" (gfconst16.low4[0])
);
for (i = 0; i < size; i += 16) {
int boundary = g23_boundary;
/* last disk initializes P/Q without a preceding Q transition */
asm volatile (
"vld1.8 {q0}, %0\n"
"vld1.8 {q1}, %0\n"
:
: "Q" (v[l][i])
);
if (np >= 3) {
asm volatile (
"vst1.8 {q2}, %0\n"
: "=Q" (v[nd + 2][i])
"vshr.u8 q8, q0, #4\n"
"vand q7, q0, q15\n"
"vand q8, q8, q15\n"
);
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q2, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][1][0][0]),
"Q" (raid_gfcauchypshufb[l][1][1][0])
);
}
if (np >= 4) {
asm volatile (
"vst1.8 {q3}, %0\n"
: "=Q" (v[nd + 3][i])
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q3, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][2][0][0]),
"Q" (raid_gfcauchypshufb[l][2][1][0])
);
}
if (np >= 5) {
asm volatile (
"vst1.8 {q4}, %0\n"
: "=Q" (v[nd + 4][i])
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q4, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][3][0][0]),
"Q" (raid_gfcauchypshufb[l][3][1][0])
);
}
if (np >= 6) {
asm volatile (
"vst1.8 {q5}, %0\n"
: "=Q" (v[nd + 5][i])
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q5, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][4][0][0]),
"Q" (raid_gfcauchypshufb[l][4][1][0])
);
}
d = l - 1;
while (boundary >= 0) {
/*
* Everything above the next G23 boundary is Q *= 2.
*/
for (; d > boundary; --d) {
asm volatile (
"vshr.s8 q11, q1, #7\n"
"vshl.i8 q1, q1, #1\n"
"vand q11, q11, q14\n"
"veor q1, q1, q11\n"
"vld1.8 {q6}, %0\n"
"veor q0, q0, q6\n"
"veor q1, q1, q6\n"
:
: "Q" (v[d][i])
);
if (np >= 3) {
asm volatile (
"vshr.u8 q8, q6, #4\n"
"vand q7, q6, q15\n"
"vand q8, q8, q15\n"
);
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q2, q2, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][1][0][0]),
"Q" (raid_gfcauchypshufb[d][1][1][0])
);
}
if (np >= 4) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q3, q3, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][2][0][0]),
"Q" (raid_gfcauchypshufb[d][2][1][0])
);
}
if (np >= 5) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q4, q4, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][3][0][0]),
"Q" (raid_gfcauchypshufb[d][3][1][0])
);
}
if (np >= 6) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q5, q5, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][4][0][0]),
"Q" (raid_gfcauchypshufb[d][4][1][0])
);
}
}
/*
* d == boundary: Q *= 3.
*
* q13 is otherwise unused here, so preserve the old Q
* there and XOR it back after the normal x2 transition.
*/
asm volatile (
"vmov q13, q1\n"
"vshr.s8 q11, q1, #7\n"
"vshl.i8 q1, q1, #1\n"
"vand q11, q11, q14\n"
"veor q1, q1, q11\n"
"vld1.8 {q6}, %0\n"
"veor q0, q0, q6\n"
"veor q1, q1, q6\n"
/* complete Q *= 3 */
"veor q1, q1, q13\n"
:
: "Q" (v[d][i])
);
if (np >= 3) {
asm volatile (
"vshr.u8 q8, q6, #4\n"
"vand q7, q6, q15\n"
"vand q8, q8, q15\n"
);
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q2, q2, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][1][0][0]),
"Q" (raid_gfcauchypshufb[d][1][1][0])
);
}
if (np >= 4) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q3, q3, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][2][0][0]),
"Q" (raid_gfcauchypshufb[d][2][1][0])
);
}
if (np >= 5) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q4, q4, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][3][0][0]),
"Q" (raid_gfcauchypshufb[d][3][1][0])
);
}
if (np >= 6) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q5, q5, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][4][0][0]),
"Q" (raid_gfcauchypshufb[d][4][1][0])
);
}
--d;
boundary -= 51;
if (boundary < 50)
boundary = -1;
}
/*
* Remaining intermediate disks contain no more G23
* boundaries, so this loop is pure x2.
*/
for (; d > 0; --d) {
asm volatile (
"vshr.s8 q11, q1, #7\n"
"vshl.i8 q1, q1, #1\n"
"vand q11, q11, q14\n"
"veor q1, q1, q11\n"
"vld1.8 {q6}, %0\n"
"veor q0, q0, q6\n"
"veor q1, q1, q6\n"
:
: "Q" (v[d][i])
);
if (np >= 3) {
asm volatile (
"vshr.u8 q8, q6, #4\n"
"vand q7, q6, q15\n"
"vand q8, q8, q15\n"
);
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q2, q2, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][1][0][0]),
"Q" (raid_gfcauchypshufb[d][1][1][0])
);
}
if (np >= 4) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q3, q3, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][2][0][0]),
"Q" (raid_gfcauchypshufb[d][2][1][0])
);
}
if (np >= 5) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q4, q4, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][3][0][0]),
"Q" (raid_gfcauchypshufb[d][3][1][0])
);
}
if (np >= 6) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q5, q5, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][4][0][0]),
"Q" (raid_gfcauchypshufb[d][4][1][0])
);
}
}
/*
* D0 always follows Q *= 2. The lowest G23 boundary is 50.
*/
asm volatile (
"vshr.s8 q11, q1, #7\n"
"vshl.i8 q1, q1, #1\n"
"vand q11, q11, q14\n"
"veor q1, q1, q11\n"
"vld1.8 {q6}, %0\n"
"veor q0, q0, q6\n"
"veor q1, q1, q6\n"
:
: "Q" (v[0][i])
);
if (np >= 3)
asm volatile ("veor q2, q2, q6\n");
if (np >= 4)
asm volatile ("veor q3, q3, q6\n");
if (np >= 5)
asm volatile ("veor q4, q4, q6\n");
if (np >= 6)
asm volatile ("veor q5, q5, q6\n");
/* write parity in increasing order */
asm volatile (
"vst1.8 {q0}, %0\n"
"vst1.8 {q1}, %1\n"
: "=Q" (v[nd][i]),
"=Q" (v[nd + 1][i])
);
if (np >= 3)
asm volatile ("vst1.8 {q2}, %0\n" : "=Q" (v[nd + 2][i]));
if (np >= 4)
asm volatile ("vst1.8 {q3}, %0\n" : "=Q" (v[nd + 3][i]));
if (np >= 5)
asm volatile ("vst1.8 {q4}, %0\n" : "=Q" (v[nd + 4][i]));
if (np >= 6)
asm volatile ("vst1.8 {q5}, %0\n" : "=Q" (v[nd + 5][i]));
}
raid_neon32_end();
@ -533,12 +1055,15 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
*/
void raid_gen3_neon32_raid(int nd, size_t size, void **vv)
{
raid_genX_neon32(nd, size, vv, 3, 2);
raid_genX_neon32_x2(nd, size, vv, 3);
}
void raid_gen3_neon32_aes(int nd, size_t size, void **vv)
{
raid_genX_neon32(nd, size, vv, 3, 3);
if (raid_g23_x2_only(nd))
raid_genX_neon32_x2(nd, size, vv, 3);
else
raid_genX_neon32_g23(nd, size, vv, 3);
}
/*
@ -546,12 +1071,15 @@ void raid_gen3_neon32_aes(int nd, size_t size, void **vv)
*/
void raid_gen4_neon32_raid(int nd, size_t size, void **vv)
{
raid_genX_neon32(nd, size, vv, 4, 2);
raid_genX_neon32_x2(nd, size, vv, 4);
}
void raid_gen4_neon32_aes(int nd, size_t size, void **vv)
{
raid_genX_neon32(nd, size, vv, 4, 3);
if (raid_g23_x2_only(nd))
raid_genX_neon32_x2(nd, size, vv, 4);
else
raid_genX_neon32_g23(nd, size, vv, 4);
}
/*
@ -559,12 +1087,15 @@ void raid_gen4_neon32_aes(int nd, size_t size, void **vv)
*/
void raid_gen5_neon32_raid(int nd, size_t size, void **vv)
{
raid_genX_neon32(nd, size, vv, 5, 2);
raid_genX_neon32_x2(nd, size, vv, 5);
}
void raid_gen5_neon32_aes(int nd, size_t size, void **vv)
{
raid_genX_neon32(nd, size, vv, 5, 3);
if (raid_g23_x2_only(nd))
raid_genX_neon32_x2(nd, size, vv, 5);
else
raid_genX_neon32_g23(nd, size, vv, 5);
}
/*
@ -572,12 +1103,15 @@ void raid_gen5_neon32_aes(int nd, size_t size, void **vv)
*/
void raid_gen6_neon32_raid(int nd, size_t size, void **vv)
{
raid_genX_neon32(nd, size, vv, 6, 2);
raid_genX_neon32_x2(nd, size, vv, 6);
}
void raid_gen6_neon32_aes(int nd, size_t size, void **vv)
{
raid_genX_neon32(nd, size, vv, 6, 3);
if (raid_g23_x2_only(nd))
raid_genX_neon32_x2(nd, size, vv, 6);
else
raid_genX_neon32_g23(nd, size, vv, 6);
}
/*

85
raid/raid.c
View file

@ -5,11 +5,11 @@
#include "gf.h"
/*
* Cauchy Matrix Construction for MDS RAID with Arbitrary Parity Levels in GF(2^8)
* Extended Cauchy Matrix Construction for MDS RAID in GF(2^8)
*
* This is a RAID implementation operating in the Galois Field GF(2^8) with
* the primitive polynomial x^8 + x^4 + x^3 + x^2 + 1 (285 decimal), supporting
* up to six parity levels.
* The Cauchy RAID modes operate in GF(2^8) and support up to six parity
* levels. RAID_MODE_CAUCHY_RAID uses polynomial 0x11d, while
* RAID_MODE_CAUCHY_AES uses polynomial 0x11b.
*
* For RAID5 and RAID6, it follows the method described in H. Peter Anvin's
* paper "The mathematics of RAID-6" [1]. Please refer to this paper for a
@ -38,20 +38,42 @@
* nonsingular, so the linear system is always solvable for any combination
* of missing disks. This guarantees an MDS code.
*
* The Cauchy construction below uses a primitive generator g.
* RAID_MODE_CAUCHY_RAID uses polynomial 0x11d and the existing primitive
* generator g=2 construction described below.
*
* RAID_MODE_CAUCHY_RAID uses:
* polynomial 0x11d
* generator g = 2
* RAID_MODE_CAUCHY_AES instead uses polynomial 0x11b and a G23 Extended
* Cauchy construction. In the AES field, element 2 has multiplicative order
* 51 while element 3 is primitive. Therefore H=<2> has five cosets in
* GF(256)*. The Q coefficients q[0..254] enumerate those cosets, starting
* with q[0]=1. Within a coset q advances by multiplication by 2. The four
* transitions between cosets advance by multiplication by 3, after the
* coefficients indexed 50, 101, 152, and 203.
*
* RAID_MODE_CAUCHY_AES uses:
* polynomial 0x11b
* generator g = 3
* The AES data X set consumes inverses of the G23 sequence from the beginning:
*
* X[i] = inv(q[i])
*
* The Cauchy Y set starts with zero and consumes inverses backward from the
* end of the sequence:
*
* Y[0] = 0
* Y[j] = inv(q[255-j]) for j > 0
*
* Thus the Y values associated with R, S, T, and U are inv(q[254]),
* inv(q[253]), inv(q[252]), and inv(q[251]), respectively. For PARITY=n and
* DISK=257-n, increasing parity removes one X element from the tail and
* appends it to Y, leaving the existing rows unchanged on the remaining
* columns. X and Y are disjoint and together contain all 256 field elements, so
* C[j,i]=1/(X[i]+Y[j]) defines valid Cauchy rows. Adding P as an all-ones row
* forms an Extended Cauchy matrix. Q is the row for Y[0]=0 and therefore
* Q=sum(q[i]*Di). Rows R through U are scaled so their first coefficient is 1.
* This nonzero row scaling preserves the MDS property, and the resulting 6x251
* matrix is MDS.
*
* The numerical examples below refer to RAID_MODE_CAUCHY_RAID,
* using polynomial 0x11d and generator g = 2.
*
* How the matrix is obtained:
* How the RAID/g=2 matrix is obtained:
*
* The matrix is constructed to match Linux RAID coefficients for the
* first two rows, while ensuring that all square submatrices are nonsingular.
@ -121,9 +143,10 @@
* data disks, one for each column. All the 377,342,351,231 square submatrices
* are nonsingular, verified also by brute-force testing.
*
* This matrix can be extended to support any number of parities by simply
* adding additional rows and removing one column for each new row to maintain
* the condition i + j < 255.
* The RAID/g=2 construction can be extended beyond six parity levels by
* adding rows and removing one data column for each additional row, while
* preserving the field-size constraint i + j < 255. The current
* implementation remains limited to RAID_PARITY_MAX = 6.
* (See mktables.c for more details on how the matrix is generated.)
*
* The downside is that generic multiplications are required to compute the
@ -132,27 +155,27 @@
* or AVX2 instructions [1][5] make this approach competitive with the triple
* parity computation using power coefficients.
*
* Another advantage of the Cauchy matrix is that the first two rows can
* Another advantage of the RAID/g=2 Cauchy matrix is that the first two rows can
* replicate the RAID5 and RAID6 approach, resulting in a compatible extension.
* SSSE3 or AVX2 instructions are only needed for triple parity or beyond.
*
* Parity computation is as follows:
*
* P = sum(Di)
* Q = sum(g^i * Di)
* Q = sum(A[1,i] * Di)
* R = sum(A[2,i] * Di)
* S = sum(A[3,i] * Di)
* T = sum(A[4,i] * Di)
* U = sum(A[5,i] * Di) for 0 <= i < N
*
* where g=2 for RAID_MODE_CAUCHY_RAID and
* g=3 for RAID_MODE_CAUCHY_AES.
* For RAID_MODE_CAUCHY_RAID, A[1,i]=2^i. For RAID_MODE_CAUCHY_AES,
* A[1,i]=q[i] from the G23 sequence.
*
* Recovery from six disk failures at indices x, y, z, h, v, w (0 <= x < y < z < h < v < w < N)
* involves computing parity of the remaining N-6 disks:
*
* Pa = sum(Di)
* Qa = sum(g^i * Di)
* Qa = sum(A[1,i] * Di)
* Ra = sum(A[2,i] * Di)
* Sa = sum(A[3,i] * Di)
* Ta = sum(A[4,i] * Di)
@ -170,15 +193,12 @@
* yields:
*
* Pd = Dx + Dy + Dz + Dh + Dv + Dw
* Qd = g^x * Dx + g^y * Dy + g^z * Dz + g^h * Dh + g^v * Dv + g^w * Dw
* Qd = A[1,x] * Dx + A[1,y] * Dy + A[1,z] * Dz + A[1,h] * Dh + A[1,v] * Dv + A[1,w] * Dw
* Rd = A[2,x] * Dx + A[2,y] * Dy + A[2,z] * Dz + A[2,h] * Dh + A[2,v] * Dv + A[2,w] * Dw
* Sd = A[3,x] * Dx + A[3,y] * Dy + A[3,z] * Dz + A[3,h] * Dh + A[3,v] * Dv + A[3,w] * Dw
* Td = A[4,x] * Dx + A[4,y] * Dy + A[4,z] * Dz + A[4,h] * Dh + A[4,v] * Dv + A[4,w] * Dw
* Ud = A[5,x] * Dx + A[5,y] * Dy + A[5,z] * Dz + A[5,h] * Dh + A[5,v] * Dv + A[5,w] * Dw
*
* with g=2 for RAID_MODE_CAUCHY_RAID and
* g=3 for RAID_MODE_CAUCHY_AES.
*
* This linear system is always solvable since the coefficient matrix is
* nonsingular due to the properties of A[].
*
@ -422,20 +442,19 @@ void raid_rec1of1(int *id, int nd, size_t size, void **v)
* Starting from the equations:
*
* Pd = Dx + Dy
* Qd = g^id[0] * Dx + g^id[1] * Dy
* Qd = A[1,id[0]] * Dx + A[1,id[1]] * Dy
*
* and solving we get:
*
* 1 g^(-id[0])
* Dy = ------------------- * Pd + ------------------- * Qd
* g^(id[1]-id[0]) + 1 g^(id[1]-id[0]) + 1
* Qd + A[1,id[0]] * Pd
* Dy = ------------------------
* A[1,id[0]] + A[1,id[1]]
*
* Dx = Dy + Pd
*
* with conditions:
*
* g^id[0] != 0
* g^(id[1]-id[0]) + 1 != 0
* A[1,id[0]] + A[1,id[1]] != 0
*
* That are always satisfied for any 0<=id[0]<id[1]<255.
*/
@ -449,9 +468,9 @@ void raid_rec2of2_int8(int *id, int *ip, int nd, size_t size, void **vv)
uint8_t *qa;
const uint8_t *T[2];
/* get multiplication tables */
T[0] = table(inv(powgen(id[1] - id[0]) ^ 1));
T[1] = table(inv(powgen(id[0]) ^ powgen(id[1])));
/* get multiplication tables from the actual active Q coefficients */
T[0] = table(mul(A(1, id[0]), inv(A(1, id[0]) ^ A(1, id[1]))));
T[1] = table(inv(A(1, id[0]) ^ A(1, id[1])));
/* compute delta parity */
raid_delta_gen(2, id, ip, nd, size, vv);

24
raid/raid.h
View file

@ -5,13 +5,13 @@
#define __RAID_H
/*
* GF(2^8) reducing polynomial selection.
* RAID operating modes and GF(2^8) field selection.
*
* RAID parity computation operates in GF(2^8), a finite field where
* arithmetic is performed modulo an irreducible polynomial of degree 8.
* The choice of operating mode affects both correctness and performance:
* all tables, coefficients, and parity data are tied to the selected
* polynomial and primitive generator and cannot be mixed between
* polynomial and generator matrix and cannot be mixed between
* incompatible operating modes.
*
* Two polynomials are supported:
@ -20,8 +20,9 @@
*
* The polynomial used by the original RAID-6 specification and the
* Linux kernel RAID implementation, used with primitive generator g=2.
* Choosing this polynomial ensures on-disk compatibility with existing
* RAID arrays and interchange with other RAID implementations.
* The P and Q rows preserve the standard RAID5/RAID6 coefficient layout,
* providing compatibility with the corresponding Linux RAID parity.
* Higher parity rows are SnapRAID's Extended Cauchy extension.
*
* On CPUs supporting Intel GFNI (Galois Field New Instructions),
* generation and recovery are accelerated with vgf2p8affineqb.
@ -31,7 +32,8 @@
* 0x1b (x^8 + x^4 + x^3 + x + 1) -- AES polynomial
*
* The polynomial used by the AES encryption standard (0x11b including
* the x^8 term), used with primitive generator g=3.
* the x^8 term). Parity uses a 6x251 Extended Cauchy matrix whose Q row
* follows the G23 sequence.
*
* On CPUs supporting Intel GFNI, generation and recovery can use the
* vgf2p8mulb instruction, which performs GF(2^8) multiplication directly
@ -76,14 +78,14 @@
#define RAID_MODE_VANDERMONDE_RAID 1
/**
* RAID mode supporting up to 6 parities using AES polynomial 0x1b (0x11b) and primitive generator g=3.
* RAID mode supporting up to 6 parities and 251 data disks using AES polynomial 0x1b (0x11b).
*
* It has slightly lower performance than RAID_MODE_CAUCHY_RAID on CPUs without GFNI due
* to the additional cost of generator g=3. On CPUs supporting GFNI, it uses native
* VGF2P8MULB multiplication and is typically faster than the affine GFNI implementation
* required by RAID_MODE_CAUCHY_RAID.
* It uses an Extended Cauchy matrix with a G23 Q sequence, allowing optimized
* generators to use multiplication by 2 for almost every disk transition.
* On CPUs supporting GFNI, it uses native VGF2P8MULB multiplication.
*
* Parity data is incompatible with RAID_MODE_CAUCHY_RAID.
* This parity layout is incompatible with RAID_MODE_CAUCHY_RAID and with the
* previous RAID_MODE_CAUCHY_AES layout.
*/
#define RAID_MODE_CAUCHY_AES 2

579
raid/sse2.c
View file

@ -39,9 +39,10 @@
* - D0 Cauchy Bypass: The final disk processed (D0) always has a
* coefficient of 1 for all parities. It is handled separately to bypass
* the expensive Cauchy matrix lookup and `pshufb` multiplications.
* - Pipeline Dependency Breaking: Galois Field doubling (Horner's method
* multiplication by 2) is strategically placed relative to the XOR
* operations to break serial dependency chains. This allows the CPU's
* - Pipeline Dependency Breaking: The Q Horner transition, normally
* multiplication by 2 and multiplication by 3 at G23 boundaries, is
* strategically placed relative to the XOR operations to break serial
* dependency chains. This allows the CPU's
* out-of-order execution engine to hide the latency of the GF math
* behind memory loads.
*/
@ -52,11 +53,11 @@
* ================================================================
*
* This file implements different strategies for computing the Q parity.
* Q parity requires scaling each data disk by a descending power of the
* primitive generator g (g=2 for RAID_MODE_CAUCHY_RAID, g=3 for
* RAID_MODE_CAUCHY_AES) in the Galois Field:
* Q parity uses coefficients connected by cheap GF transitions. RAID mode
* always uses multiplication by 2. AES mode also uses multiplication by 2,
* except for the four G23 coset transitions that use multiplication by 3.
*
* Q = (D_l * g^l) + (D_{l-1} * g^{l-1}) + ... + (D_1 * g^1) + (D_0 * 1)
* Q = (D_l * q[l]) + ... + (D_1 * q[1]) + (D_0 * q[0])
*
* To efficiently calculate this sequentially using SIMD instructions, we
* use Horner's method. Depending on hardware architecture and whether we
@ -67,15 +68,15 @@
* CASE 1: Multiplication BEFORE XOR, handling all disks inside the loop
* -------------------------------------------------------------------------
* Used in GEN2 where all disks can be processed uniformly.
* The accumulator is scaled by the active generator BEFORE adding the current disk.
* The accumulator is scaled by the active Q transition BEFORE adding the current disk.
* Because the loop runs all the way down to D0 (d >= 0), the final disk is XORed
* into the accumulator and the loop terminates. This naturally leaves D0 with
* a coefficient of 1, as it escapes the scaling step.
* into the accumulator and the loop terminates. Since D0 is added after the
* final scaling operation, its coefficient is naturally 1.
*
* Simplified Code:
* Q = D_l;
* for (d = l - 1; d >= 0; --d) {
* Q = Q * g; // Scale accumulator by active generator
* Q = Q * step[d]; // Scale by 2, or by 3 at a G23 transition
* Q = Q ^ D_d; // Add current disk
* }
*
@ -90,12 +91,13 @@
* Simplified Code:
* Q = D_l;
* for (d = l - 1; d > 0; --d) {
* Q = Q * g; // Scale accumulator by active generator
* Q = Q * step[d]; // Scale by 2, or by 3 at a G23 transition
* Q = Q ^ D_d; // Add current disk
* }
*
* // Final disk (D0) processed outside the loop to skip Cauchy math
* Q = Q * g;
* // Final disk (D0) processed outside the loop to skip Cauchy math.
* // D0 is below every G23 boundary, so this transition is always x2.
* Q = Q * 2;
* Q = Q ^ D_0;
*/
@ -137,9 +139,9 @@ void raid_gen1_sse2(int nd, size_t size, void **vv)
}
/*
* GEN2 Cauchy SSE2 implementation using the active generator
* GEN2 Cauchy SSE2 implementation using only Q *= 2.
*/
static __always_inline void raid_gen2_sse2_gen(int nd, size_t size, void **vv, int generator)
static __always_inline void raid_gen2_sse2_x2(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -162,34 +164,17 @@ static __always_inline void raid_gen2_sse2_gen(int nd, size_t size, void **vv, i
asm volatile ("movdqa %xmm1,%xmm3");
for (d = l - 1; d >= 0; --d) {
/* scale Q by the active generator before adding the current disk */
if (generator == 3) {
asm volatile ("movdqa %xmm2,%xmm4");
asm volatile ("movdqa %xmm3,%xmm5");
asm volatile ("pxor %xmm6,%xmm6");
asm volatile ("pcmpgtb %xmm4,%xmm6");
asm volatile ("paddb %xmm2,%xmm2");
asm volatile ("paddb %xmm3,%xmm3");
asm volatile ("pxor %xmm4,%xmm2");
asm volatile ("pxor %xmm4,%xmm4");
asm volatile ("pcmpgtb %xmm5,%xmm4");
asm volatile ("pand %xmm7,%xmm6");
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pxor %xmm6,%xmm2");
asm volatile ("pxor %xmm4,%xmm3");
asm volatile ("pxor %xmm5,%xmm3");
} else {
asm volatile ("pxor %xmm4,%xmm4");
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pcmpgtb %xmm2,%xmm4");
asm volatile ("pcmpgtb %xmm3,%xmm5");
asm volatile ("paddb %xmm2,%xmm2");
asm volatile ("paddb %xmm3,%xmm3");
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("pxor %xmm4,%xmm2");
asm volatile ("pxor %xmm5,%xmm3");
}
/* scale Q before adding the current disk */
asm volatile ("pxor %xmm4,%xmm4");
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pcmpgtb %xmm2,%xmm4");
asm volatile ("pcmpgtb %xmm3,%xmm5");
asm volatile ("paddb %xmm2,%xmm2");
asm volatile ("paddb %xmm3,%xmm3");
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("pxor %xmm4,%xmm2");
asm volatile ("pxor %xmm5,%xmm3");
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
asm volatile ("movdqa %0,%%xmm5" : : "m" (v[d][i + 16]));
@ -208,23 +193,140 @@ static __always_inline void raid_gen2_sse2_gen(int nd, size_t size, void **vv, i
raid_sse_end();
}
/*
* GEN2 Cauchy SSE2 implementation using the AES G23 Q recurrence.
*/
static __always_inline void raid_gen2_sse2_g23(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
uint8_t *q;
int d, l;
size_t i;
l = nd - 1;
p = v[nd];
q = v[nd + 1];
int g23_boundary = raid_g23_boundary(l - 1);
raid_sse_begin();
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
for (i = 0; i < size; i += 32) {
int boundary = g23_boundary;
asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i]));
asm volatile ("movdqa %0,%%xmm1" : : "m" (v[l][i + 16]));
asm volatile ("movdqa %xmm0,%xmm2");
asm volatile ("movdqa %xmm1,%xmm3");
d = l - 1;
for (;;) {
/*
* All transitions above 'boundary' are x2.
*/
for (; d > boundary; --d) {
/* scale Q before adding the current disk */
asm volatile ("pxor %xmm4,%xmm4");
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pcmpgtb %xmm2,%xmm4");
asm volatile ("pcmpgtb %xmm3,%xmm5");
asm volatile ("paddb %xmm2,%xmm2");
asm volatile ("paddb %xmm3,%xmm3");
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("pxor %xmm4,%xmm2");
asm volatile ("pxor %xmm5,%xmm3");
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
asm volatile ("movdqa %0,%%xmm5" : : "m" (v[d][i + 16]));
asm volatile ("pxor %xmm4,%xmm0");
asm volatile ("pxor %xmm5,%xmm1");
asm volatile ("pxor %xmm4,%xmm2");
asm volatile ("pxor %xmm5,%xmm3");
}
/*
* boundary == -1 identifies the final x2-only segment.
* At this point all remaining disks have been consumed.
*/
if (boundary < 0)
break;
/*
* d == boundary.
*
* Perform the fixed G23 x3 transition and then add
* the current disk to P and Q.
*/
asm volatile ("movdqa %xmm2,%xmm4");
asm volatile ("movdqa %xmm3,%xmm5");
asm volatile ("pxor %xmm6,%xmm6");
asm volatile ("pcmpgtb %xmm4,%xmm6");
asm volatile ("paddb %xmm2,%xmm2");
asm volatile ("paddb %xmm3,%xmm3");
asm volatile ("pxor %xmm4,%xmm2");
asm volatile ("pxor %xmm4,%xmm4");
asm volatile ("pcmpgtb %xmm5,%xmm4");
asm volatile ("pand %xmm7,%xmm6");
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pxor %xmm6,%xmm2");
asm volatile ("pxor %xmm4,%xmm3");
asm volatile ("pxor %xmm5,%xmm3");
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
asm volatile ("movdqa %0,%%xmm5" : : "m" (v[d][i + 16]));
asm volatile ("pxor %xmm4,%xmm0");
asm volatile ("pxor %xmm5,%xmm1");
asm volatile ("pxor %xmm4,%xmm2");
asm volatile ("pxor %xmm5,%xmm3");
--d;
/*
* G23 boundaries are exactly 51 transitions apart.
* After d = 50 there are no more x3 transitions.
*/
boundary -= 51;
if (boundary < 50)
boundary = -1;
}
asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i]));
asm volatile ("movntdq %%xmm1,%0" : "=m" (p[i + 16]));
asm volatile ("movntdq %%xmm2,%0" : "=m" (q[i]));
asm volatile ("movntdq %%xmm3,%0" : "=m" (q[i + 16]));
}
raid_sse_end();
}
void raid_gen2_sse2_raid(int nd, size_t size, void **vv)
{
raid_gen2_sse2_gen(nd, size, vv, 2);
raid_gen2_sse2_x2(nd, size, vv);
}
void raid_gen2_sse2_aes(int nd, size_t size, void **vv)
{
raid_gen2_sse2_gen(nd, size, vv, 3);
if (raid_g23_x2_only(nd))
raid_gen2_sse2_x2(nd, size, vv);
else
raid_gen2_sse2_g23(nd, size, vv);
}
#ifdef CONFIG_X86_64
/*
* GEN2 Cauchy SSE2 implementation using the active generator
* GEN2 Cauchy SSE2 implementation using only Q *= 2.
*
* Process two data disks at a time and process the four 16-byte lanes
* sequentially.
*
* Note that it uses 16 registers, meaning that x64 is required.
*/
static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv, int generator)
static __always_inline void raid_gen2_sse2ext_x2(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -252,12 +354,15 @@ static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv
/* process two disks per iteration */
for (d = l - 1; d >= 1; d -= 2) {
/* lane 0 */
/*
* Lane 0.
*
* Compute both x2 Horner transitions and update P
* once with D[d] ^ D[d - 1].
*/
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i]));
if (generator == 3)
asm volatile ("movdqa %xmm4,%xmm10");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm4,%xmm11");
asm volatile ("paddb %xmm4,%xmm4");
@ -267,25 +372,18 @@ static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv
asm volatile ("pxor %xmm9,%xmm8");
asm volatile ("pxor %xmm8,%xmm0");
if (generator == 3)
asm volatile ("pxor %xmm10,%xmm8");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm4,%xmm11");
asm volatile ("paddb %xmm4,%xmm4");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm4");
if (generator == 3)
asm volatile ("pxor %xmm8,%xmm4");
else
asm volatile ("pxor %xmm9,%xmm4");
asm volatile ("pxor %xmm9,%xmm4");
/* lane 1 */
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i + 16]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i + 16]));
if (generator == 3)
asm volatile ("movdqa %xmm5,%xmm10");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm5,%xmm11");
asm volatile ("paddb %xmm5,%xmm5");
@ -295,25 +393,18 @@ static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv
asm volatile ("pxor %xmm9,%xmm8");
asm volatile ("pxor %xmm8,%xmm1");
if (generator == 3)
asm volatile ("pxor %xmm10,%xmm8");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm5,%xmm11");
asm volatile ("paddb %xmm5,%xmm5");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm5");
if (generator == 3)
asm volatile ("pxor %xmm8,%xmm5");
else
asm volatile ("pxor %xmm9,%xmm5");
asm volatile ("pxor %xmm9,%xmm5");
/* lane 2 */
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i + 32]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i + 32]));
if (generator == 3)
asm volatile ("movdqa %xmm6,%xmm10");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm6,%xmm11");
asm volatile ("paddb %xmm6,%xmm6");
@ -323,25 +414,18 @@ static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv
asm volatile ("pxor %xmm9,%xmm8");
asm volatile ("pxor %xmm8,%xmm2");
if (generator == 3)
asm volatile ("pxor %xmm10,%xmm8");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm6,%xmm11");
asm volatile ("paddb %xmm6,%xmm6");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm6");
if (generator == 3)
asm volatile ("pxor %xmm8,%xmm6");
else
asm volatile ("pxor %xmm9,%xmm6");
asm volatile ("pxor %xmm9,%xmm6");
/* lane 3 */
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i + 48]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i + 48]));
if (generator == 3)
asm volatile ("movdqa %xmm7,%xmm10");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm7,%xmm11");
asm volatile ("paddb %xmm7,%xmm7");
@ -351,73 +435,37 @@ static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv
asm volatile ("pxor %xmm9,%xmm8");
asm volatile ("pxor %xmm8,%xmm3");
if (generator == 3)
asm volatile ("pxor %xmm10,%xmm8");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm7,%xmm11");
asm volatile ("paddb %xmm7,%xmm7");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm7");
if (generator == 3)
asm volatile ("pxor %xmm8,%xmm7");
else
asm volatile ("pxor %xmm9,%xmm7");
asm volatile ("pxor %xmm9,%xmm7");
}
/* single remaining disk */
if (d == 0) {
if (generator == 3) {
asm volatile ("movdqa %xmm4,%xmm8");
asm volatile ("movdqa %xmm5,%xmm9");
asm volatile ("movdqa %xmm6,%xmm10");
asm volatile ("movdqa %xmm7,%xmm11");
asm volatile ("pxor %xmm12,%xmm12");
asm volatile ("pxor %xmm13,%xmm13");
asm volatile ("pxor %xmm14,%xmm14");
asm volatile ("pcmpgtb %xmm8,%xmm12");
asm volatile ("pcmpgtb %xmm9,%xmm13");
asm volatile ("pcmpgtb %xmm10,%xmm14");
asm volatile ("paddb %xmm4,%xmm4");
asm volatile ("paddb %xmm5,%xmm5");
asm volatile ("paddb %xmm6,%xmm6");
asm volatile ("paddb %xmm7,%xmm7");
asm volatile ("pxor %xmm8,%xmm4");
asm volatile ("pxor %xmm8,%xmm8");
asm volatile ("pcmpgtb %xmm11,%xmm8");
asm volatile ("pand %xmm15,%xmm12");
asm volatile ("pand %xmm15,%xmm13");
asm volatile ("pand %xmm15,%xmm14");
asm volatile ("pand %xmm15,%xmm8");
asm volatile ("pxor %xmm12,%xmm4");
asm volatile ("pxor %xmm13,%xmm5");
asm volatile ("pxor %xmm14,%xmm6");
asm volatile ("pxor %xmm8,%xmm7");
asm volatile ("pxor %xmm9,%xmm5");
asm volatile ("pxor %xmm10,%xmm6");
asm volatile ("pxor %xmm11,%xmm7");
} else {
asm volatile ("pxor %xmm8,%xmm8");
asm volatile ("pxor %xmm9,%xmm9");
asm volatile ("pxor %xmm10,%xmm10");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm4,%xmm8");
asm volatile ("pcmpgtb %xmm5,%xmm9");
asm volatile ("pcmpgtb %xmm6,%xmm10");
asm volatile ("pcmpgtb %xmm7,%xmm11");
asm volatile ("paddb %xmm4,%xmm4");
asm volatile ("paddb %xmm5,%xmm5");
asm volatile ("paddb %xmm6,%xmm6");
asm volatile ("paddb %xmm7,%xmm7");
asm volatile ("pand %xmm15,%xmm8");
asm volatile ("pand %xmm15,%xmm9");
asm volatile ("pand %xmm15,%xmm10");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm8,%xmm4");
asm volatile ("pxor %xmm9,%xmm5");
asm volatile ("pxor %xmm10,%xmm6");
asm volatile ("pxor %xmm11,%xmm7");
}
asm volatile ("pxor %xmm8,%xmm8");
asm volatile ("pxor %xmm9,%xmm9");
asm volatile ("pxor %xmm10,%xmm10");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm4,%xmm8");
asm volatile ("pcmpgtb %xmm5,%xmm9");
asm volatile ("pcmpgtb %xmm6,%xmm10");
asm volatile ("pcmpgtb %xmm7,%xmm11");
asm volatile ("paddb %xmm4,%xmm4");
asm volatile ("paddb %xmm5,%xmm5");
asm volatile ("paddb %xmm6,%xmm6");
asm volatile ("paddb %xmm7,%xmm7");
asm volatile ("pand %xmm15,%xmm8");
asm volatile ("pand %xmm15,%xmm9");
asm volatile ("pand %xmm15,%xmm10");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm8,%xmm4");
asm volatile ("pxor %xmm9,%xmm5");
asm volatile ("pxor %xmm10,%xmm6");
asm volatile ("pxor %xmm11,%xmm7");
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[0][i]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[0][i + 16]));
@ -446,14 +494,277 @@ static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv
raid_sse_end();
}
/*
* GEN2 Cauchy SSE2 implementation using the AES G23 Q recurrence.
*
* Process two data disks at a time without crossing G23 boundaries.
*
* Note that it uses 16 registers, meaning that x64 is required.
*/
static __always_inline void raid_gen2_sse2ext_g23(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
uint8_t *q;
int d, l;
size_t i;
l = nd - 1;
p = v[nd];
q = v[nd + 1];
int g23_boundary = raid_g23_boundary(l - 1);
raid_sse_begin();
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.poly[0]));
for (i = 0; i < size; i += 64) {
int boundary = g23_boundary;
asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i]));
asm volatile ("movdqa %0,%%xmm1" : : "m" (v[l][i + 16]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (v[l][i + 32]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (v[l][i + 48]));
asm volatile ("movdqa %xmm0,%xmm4");
asm volatile ("movdqa %xmm1,%xmm5");
asm volatile ("movdqa %xmm2,%xmm6");
asm volatile ("movdqa %xmm3,%xmm7");
d = l - 1;
for (;;) {
/*
* All transitions above 'boundary' are x2.
* Process two disks per iteration without crossing
* the next G23 boundary.
*/
for (; d >= boundary + 2; d -= 2) {
/*
* Lane 0.
*
* Q = 2 * Q + D[d]
* Q = 2 * Q + D[d - 1]
*
* Compute D[d] ^ D[d - 1] independently and use
* it to update P with a single dependent XOR.
*/
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i]));
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm4,%xmm11");
asm volatile ("paddb %xmm4,%xmm4");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm4");
asm volatile ("pxor %xmm8,%xmm4");
asm volatile ("pxor %xmm9,%xmm8");
asm volatile ("pxor %xmm8,%xmm0");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm4,%xmm11");
asm volatile ("paddb %xmm4,%xmm4");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm4");
asm volatile ("pxor %xmm9,%xmm4");
/* lane 1 */
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i + 16]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i + 16]));
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm5,%xmm11");
asm volatile ("paddb %xmm5,%xmm5");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm5");
asm volatile ("pxor %xmm8,%xmm5");
asm volatile ("pxor %xmm9,%xmm8");
asm volatile ("pxor %xmm8,%xmm1");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm5,%xmm11");
asm volatile ("paddb %xmm5,%xmm5");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm5");
asm volatile ("pxor %xmm9,%xmm5");
/* lane 2 */
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i + 32]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i + 32]));
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm6,%xmm11");
asm volatile ("paddb %xmm6,%xmm6");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm6");
asm volatile ("pxor %xmm8,%xmm6");
asm volatile ("pxor %xmm9,%xmm8");
asm volatile ("pxor %xmm8,%xmm2");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm6,%xmm11");
asm volatile ("paddb %xmm6,%xmm6");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm6");
asm volatile ("pxor %xmm9,%xmm6");
/* lane 3 */
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i + 48]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i + 48]));
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm7,%xmm11");
asm volatile ("paddb %xmm7,%xmm7");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm7");
asm volatile ("pxor %xmm8,%xmm7");
asm volatile ("pxor %xmm9,%xmm8");
asm volatile ("pxor %xmm8,%xmm3");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm7,%xmm11");
asm volatile ("paddb %xmm7,%xmm7");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm11,%xmm7");
asm volatile ("pxor %xmm9,%xmm7");
}
/*
* The first x2 segment can have odd length.
* Consume its last x2 transition separately.
*/
if (d > boundary) {
asm volatile ("pxor %xmm8,%xmm8");
asm volatile ("pxor %xmm9,%xmm9");
asm volatile ("pxor %xmm10,%xmm10");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm4,%xmm8");
asm volatile ("pcmpgtb %xmm5,%xmm9");
asm volatile ("pcmpgtb %xmm6,%xmm10");
asm volatile ("pcmpgtb %xmm7,%xmm11");
asm volatile ("paddb %xmm4,%xmm4");
asm volatile ("paddb %xmm5,%xmm5");
asm volatile ("paddb %xmm6,%xmm6");
asm volatile ("paddb %xmm7,%xmm7");
asm volatile ("pand %xmm15,%xmm8");
asm volatile ("pand %xmm15,%xmm9");
asm volatile ("pand %xmm15,%xmm10");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm8,%xmm4");
asm volatile ("pxor %xmm9,%xmm5");
asm volatile ("pxor %xmm10,%xmm6");
asm volatile ("pxor %xmm11,%xmm7");
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d][i + 16]));
asm volatile ("movdqa %0,%%xmm10" : : "m" (v[d][i + 32]));
asm volatile ("movdqa %0,%%xmm11" : : "m" (v[d][i + 48]));
asm volatile ("pxor %xmm8,%xmm0");
asm volatile ("pxor %xmm9,%xmm1");
asm volatile ("pxor %xmm10,%xmm2");
asm volatile ("pxor %xmm11,%xmm3");
asm volatile ("pxor %xmm8,%xmm4");
asm volatile ("pxor %xmm9,%xmm5");
asm volatile ("pxor %xmm10,%xmm6");
asm volatile ("pxor %xmm11,%xmm7");
--d;
}
/*
* boundary == -1 identifies the final x2-only segment.
* At this point all remaining disks have been consumed.
*/
if (boundary < 0)
break;
/*
* d == boundary.
*
* Perform the fixed G23 x3 transition and then add
* the current disk to P and Q.
*/
asm volatile ("movdqa %xmm4,%xmm8");
asm volatile ("movdqa %xmm5,%xmm9");
asm volatile ("movdqa %xmm6,%xmm10");
asm volatile ("movdqa %xmm7,%xmm11");
asm volatile ("pxor %xmm12,%xmm12");
asm volatile ("pxor %xmm13,%xmm13");
asm volatile ("pxor %xmm14,%xmm14");
asm volatile ("pcmpgtb %xmm8,%xmm12");
asm volatile ("pcmpgtb %xmm9,%xmm13");
asm volatile ("pcmpgtb %xmm10,%xmm14");
asm volatile ("paddb %xmm4,%xmm4");
asm volatile ("paddb %xmm5,%xmm5");
asm volatile ("paddb %xmm6,%xmm6");
asm volatile ("paddb %xmm7,%xmm7");
asm volatile ("pxor %xmm8,%xmm4");
asm volatile ("pxor %xmm8,%xmm8");
asm volatile ("pcmpgtb %xmm11,%xmm8");
asm volatile ("pand %xmm15,%xmm12");
asm volatile ("pand %xmm15,%xmm13");
asm volatile ("pand %xmm15,%xmm14");
asm volatile ("pand %xmm15,%xmm8");
asm volatile ("pxor %xmm12,%xmm4");
asm volatile ("pxor %xmm13,%xmm5");
asm volatile ("pxor %xmm14,%xmm6");
asm volatile ("pxor %xmm8,%xmm7");
asm volatile ("pxor %xmm9,%xmm5");
asm volatile ("pxor %xmm10,%xmm6");
asm volatile ("pxor %xmm11,%xmm7");
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d][i + 16]));
asm volatile ("movdqa %0,%%xmm10" : : "m" (v[d][i + 32]));
asm volatile ("movdqa %0,%%xmm11" : : "m" (v[d][i + 48]));
asm volatile ("pxor %xmm8,%xmm0");
asm volatile ("pxor %xmm9,%xmm1");
asm volatile ("pxor %xmm10,%xmm2");
asm volatile ("pxor %xmm11,%xmm3");
asm volatile ("pxor %xmm8,%xmm4");
asm volatile ("pxor %xmm9,%xmm5");
asm volatile ("pxor %xmm10,%xmm6");
asm volatile ("pxor %xmm11,%xmm7");
--d;
/*
* G23 boundaries are exactly 51 transitions apart.
* After d = 50 there are no more x3 transitions.
*/
boundary -= 51;
if (boundary < 50)
boundary = -1;
}
asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i]));
asm volatile ("movntdq %%xmm1,%0" : "=m" (p[i + 16]));
asm volatile ("movntdq %%xmm2,%0" : "=m" (p[i + 32]));
asm volatile ("movntdq %%xmm3,%0" : "=m" (p[i + 48]));
asm volatile ("movntdq %%xmm4,%0" : "=m" (q[i]));
asm volatile ("movntdq %%xmm5,%0" : "=m" (q[i + 16]));
asm volatile ("movntdq %%xmm6,%0" : "=m" (q[i + 32]));
asm volatile ("movntdq %%xmm7,%0" : "=m" (q[i + 48]));
}
raid_sse_end();
}
void raid_gen2_sse2ext_raid(int nd, size_t size, void **vv)
{
raid_gen2_sse2ext_gen(nd, size, vv, 2);
raid_gen2_sse2ext_x2(nd, size, vv);
}
void raid_gen2_sse2ext_aes(int nd, size_t size, void **vv)
{
raid_gen2_sse2ext_gen(nd, size, vv, 3);
if (raid_g23_x2_only(nd))
raid_gen2_sse2ext_x2(nd, size, vv);
else
raid_gen2_sse2ext_g23(nd, size, vv);
}
#endif

2108
raid/ssse3.c
View file

File diff suppressed because it is too large Load diff

10354
raid/tables.c
View file

File diff suppressed because it is too large Load diff

216
raid/test.c
View file

@ -189,19 +189,22 @@ int raid_test_sort(void)
*
* RAID_MODE_CAUCHY_AES:
* polynomial 0x11b
* primitive generator g=3
* G23 Extended Cauchy construction
*
* The parity generation tests compare optimized implementations against
* raid_gen_ref(), but both use the same active generator matrix. Therefore
* they can agree even if the matrix itself is mathematically unsuitable.
*
* This test instead checks a few fundamental properties of the field
* generator and of the generated Cauchy matrix.
* This test instead constructs the expected Q sequence independently and
* checks the generated matrix coefficients.
*/
int raid_test_poly(unsigned mode)
{
uint8_t q[255];
uint8_t v;
int i, j;
int p, d;
int g23_count;
/*
* This test is defined only for the two Cauchy modes.
@ -216,7 +219,6 @@ int raid_test_poly(unsigned mode)
/*
* Select the mode first so that:
*
* raid_gfexp
* raid_gfcauchy
* raid_gfmul
* raid_gfinv
@ -229,92 +231,104 @@ int raid_test_poly(unsigned mode)
raid_mode(mode);
/*
* Primitive order alone is not enough to protect parity format
* compatibility. Each Cauchy mode has a fixed reducing polynomial
* and primitive generator, which determine the actual matrix coefficients.
* Each Cauchy mode has a fixed reducing polynomial and matrix
* construction, which together determine the parity format.
*/
if (mode == RAID_MODE_CAUCHY_RAID) {
if (raid_poly_byte != RAID_POLY_RAID)
return -1;
if (raid_gfexp[1] != 2)
return -1;
} else if (mode == RAID_MODE_CAUCHY_AES) {
} else {
if (raid_poly_byte != RAID_POLY_AES)
return -1;
if (raid_gfexp[1] != 3)
return -1;
}
/*
* raid_gfexp[a] contains g^a for the primitive generator associated
* with the active Cauchy mode.
*
* The multiplicative group GF(2^8)* contains 255 elements:
*
* 255 = 3 * 5 * 17
*
* The order of every nonzero field element divides 255.
*
* To prove that g has order exactly 255, it is therefore sufficient
* to verify:
*
* g^255 == 1
*
* and to exclude the three maximal proper divisors:
*
* 255 / 3 = 85
* 255 / 5 = 51
* 255 / 17 = 15
*
* Hence a primitive generator must satisfy:
*
* g^85 != 1
* g^51 != 1
* g^15 != 1
* Build the expected Q sequence independently of raid_g23_boundary()
* and the optimized generator implementations.
*/
if (raid_gfexp[0] != 1)
return -1;
if (raid_gfexp[255] != 1)
return -1;
if (raid_gfexp[85] == 1)
return -1;
if (raid_gfexp[51] == 1)
return -1;
if (raid_gfexp[15] == 1)
return -1;
q[0] = 1;
g23_count = 51;
for (i = 0; i < 254; ++i) {
uint8_t f = 2;
/*
* The Q row of the Extended Cauchy matrix contains:
*
* 1, g, g^2, g^3, ...
*
* SnapRAID supports RAID_DATA_MAX data disks, so all generator powers
* corresponding to those columns must be distinct.
*
* A primitive generator of order 255 guarantees this, but checking it
* explicitly is inexpensive and also verifies the generated
* raid_gfexp[] table itself.
*/
for (i = 0; i < RAID_DATA_MAX; ++i)
for (j = i + 1; j < RAID_DATA_MAX; ++j)
if (raid_gfexp[i] == raid_gfexp[j])
if (mode == RAID_MODE_CAUCHY_AES && --g23_count == 0) {
f = 3;
g23_count = 51;
}
q[i + 1] = mul(f, q[i]);
}
/* Both constructions enumerate distinct nonzero Q coefficients. */
for (i = 0; i < 255; ++i) {
if (q[i] == 0)
return -1;
for (j = i + 1; j < 255; ++j)
if (q[i] == q[j])
return -1;
}
/*
* The second row of the Extended Cauchy matrix is exactly g^d.
*
* Verify that the exponent table and the actual matrix used for
* parity generation agree for every supported data column.
*
* This makes sure that the primitive generator checked above is the
* same generator actually used by the Q row.
*/
for (d = 0; d < RAID_DATA_MAX; ++d)
if (raid_gfcauchy[1][d] != raid_gfexp[d])
for (d = 0; d < RAID_DATA_MAX; ++d) {
if (raid_gfcauchy[0][d] != 1)
return -1;
if (raid_gfcauchy[1][d] != q[d])
return -1;
}
if (mode == RAID_MODE_CAUCHY_AES) {
/* In AES, 2 has order 51 and 3 has order 255. */
v = 1;
for (i = 1; i <= 51; ++i) {
v = mul(2, v);
if ((i == 3 || i == 17) && v == 1)
return -1;
}
if (v != 1)
return -1;
v = 1;
for (i = 1; i <= 255; ++i) {
v = mul(3, v);
if ((i == 15 || i == 51 || i == 85) && v == 1)
return -1;
}
if (v != 1)
return -1;
if (q[251] != 0xb8 || q[252] != 0x6b
|| q[253] != 0xd6 || q[254] != 0xb7)
return -1;
if (inv(q[251]) != 0xa5 || inv(q[252]) != 0xdf
|| inv(q[253]) != 0xe2 || inv(q[254]) != 0x71)
return -1;
/* the descending Y allocation keeps hypothetical higher-parity matrices nested */
for (i = 6; i <= 9; ++i) {
int disk = 257 - i;
for (j = 1; j < i - 1; ++j)
if (255 - j < disk || 255 - j > 254)
return -1;
if (255 - (i - 2) != disk)
return -1;
}
/* independently reconstruct and normalize the four Cauchy rows */
for (p = 2; p < RAID_PARITY_MAX; ++p) {
uint8_t y = inv(q[256 - p]);
uint8_t f = inv(inv(q[0]) ^ y);
uint8_t scale = inv(f);
for (d = 0; d < RAID_DATA_MAX; ++d) {
uint8_t c = inv(inv(q[d]) ^ y);
if (raid_gfcauchy[p][d] != mul(c, scale))
return -1;
}
}
}
/*
* Every coefficient of the active 6x251 Cauchy generator matrix must
* Every coefficient of the active 6x251 Extended Cauchy generator matrix must
* be nonzero.
*
* A single coefficient is itself a 1x1 submatrix, so a zero
@ -550,6 +564,57 @@ bail:
/* LCOV_EXCL_STOP */
}
int raid_test_rec2_g23(size_t size)
{
static const int missing[][2] = {
{ 1, 20 }, /* within the first coset */
{ 50, 51 }, /* across the first G23 boundary */
{ 10, 204 } /* across multiple cosets */
};
void *v_alloc;
void **v;
void *save[2];
int ip[2] = { 0, 1 };
int nv = RAID_DATA_MAX + RAID_PARITY_MAX + 3;
int i, j;
raid_mode(RAID_MODE_CAUCHY_AES);
v = raid_malloc_vector(nv, size, &v_alloc);
if (!v)
return -1;
memset(v[nv - 1], 0, size);
raid_zero(v[nv - 1]);
raid_mrand_vector(3, RAID_DATA_MAX, size, v);
raid_gen_ref(RAID_DATA_MAX, RAID_PARITY_MAX, size, v);
for (j = 0; j < (int)(sizeof(missing) / sizeof(missing[0])); ++j) {
int id[2] = { missing[j][0], missing[j][1] };
for (i = 0; i < 2; ++i) {
save[i] = v[id[i]];
v[id[i]] = v[RAID_DATA_MAX + RAID_PARITY_MAX + i];
}
raid_rec2_int8(2, id, ip, RAID_DATA_MAX, size, v);
for (i = 0; i < 2; ++i) {
if (memcmp(v[id[i]], save[i], size) != 0)
goto bail;
v[id[i]] = save[i];
}
}
free(v_alloc);
free(v);
return 0;
bail:
free(v_alloc);
free(v);
return -1;
}
int raid_test_tail(int mode, int nd_max, size_t size)
{
void (*f[RAID_PARITY_MAX][32])(int nr, int *id, int *ip, int nd, size_t size, void **vbuf);
@ -826,7 +891,6 @@ int raid_test_par(int mode, int nd, size_t size)
/* setup all the available functions */
test_setup(1) = raid_gen1_int32;
test_setup(1) = raid_gen1_int64;
test_setup(2) = raid_gen2_int8;
if (mode == RAID_MODE_CAUCHY_AES) {
test_setup(2) = raid_gen2_int32_aes;
test_setup(2) = raid_gen2_int64_aes;
@ -935,6 +999,8 @@ int raid_test_par(int mode, int nd, size_t size)
if (mode == RAID_MODE_CAUCHY_AES) {
test_setup(3) = raid_gen3_ssse3_aes;
test_setup(4) = raid_gen4_ssse3_aes;
test_setup(5) = raid_gen5_ssse3_aes;
test_setup(6) = raid_gen6_ssse3_aes;
} else {
test_setup(3) = raid_gen3_ssse3_raid;
test_setup(4) = raid_gen4_ssse3_raid;

10
raid/test.h
View file

@ -41,7 +41,7 @@ int raid_test_gfaffine(void);
/**
* Tests the Galois field and Cauchy matrix properties.
*
* Verifies the expected polynomial and primitive generator and basic
* Verifies the expected polynomial and matrix construction and basic
* properties of the Extended Cauchy matrix for the specified RAID mode.
*
* Returns 0 on success.
@ -60,6 +60,14 @@ int raid_test_poly(unsigned mode);
*/
int raid_test_rec(unsigned mode, int nd, size_t size);
/**
* Tests AES P/Q double-disk recovery for failures within one G23 coset,
* across a G23 boundary, and across multiple cosets.
*
* Returns 0 on success.
*/
int raid_test_rec2_g23(size_t size);
/**
* Tests recovering functions on the tail data disks.
*

7
raid/test/fulltest.c
View file

@ -151,6 +151,13 @@ int main(void)
/* LCOV_EXCL_STOP */
}
printf("Test Cauchy AES P/Q recovery within and across G23 cosets...\n");
if (raid_test_rec2_g23(TEST_SIZE) != 0) {
/* LCOV_EXCL_START */
goto bail;
/* LCOV_EXCL_STOP */
}
printf("OK\n");
return 0;

2
raid/test/invtest.c
View file

@ -8,7 +8,7 @@
* nonsingular (invertible) for both operating modes:
*
* RAID: GF polynomial 0x11d, primitive generator g=2
* AES: GF polynomial 0x11b, primitive generator g=3
* AES: GF polynomial 0x11b, G23 Extended Cauchy construction
*/
#include "internal.h"

2
raid/test/invtestx.c
View file

@ -8,7 +8,7 @@
* nonsingular (invertible) for both operating modes:
*
* RAID: GF polynomial 0x11d, primitive generator g=2
* AES: GF polynomial 0x11b, primitive generator g=3
* AES: GF polynomial 0x11b, G23 Extended Cauchy construction
*/
#include "internal.h"

31
raid/test/speedtest.c
View file

@ -232,12 +232,7 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
printf("%8s", raid_gen_tag(RAID_ALGO_CAUCHY_PAR2));
fflush(stdout);
SPEED_START {
raid_gen2_int8(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
printf("%8s", "");
if (mode == RAID_MODE_CAUCHY_AES) {
SPEED_START {
@ -742,15 +737,17 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
#endif
}
if (raid_cpu_has_ssse3()) {
if (mode == RAID_MODE_CAUCHY_RAID) {
if (mode == RAID_MODE_CAUCHY_AES) {
SPEED_START {
raid_gen5_ssse3_aes(nd, size, v);
} SPEED_STOP
} else {
SPEED_START {
raid_gen5_ssse3_raid(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
} else {
printf("%8s", "");
}
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
if (mode == RAID_MODE_CAUCHY_AES) {
@ -881,15 +878,17 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
#endif
}
if (raid_cpu_has_ssse3()) {
if (mode == RAID_MODE_CAUCHY_RAID) {
if (mode == RAID_MODE_CAUCHY_AES) {
SPEED_START {
raid_gen6_ssse3_aes(nd, size, v);
} SPEED_STOP
} else {
SPEED_START {
raid_gen6_ssse3_raid(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
} else {
printf("%8s", "");
}
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
if (mode == RAID_MODE_CAUCHY_AES) {