mirror of
https://github.com/amadvance/snapraid.git
synced 2026-10-26 19:01:57 -04:00
Implement G23 Extended Cauchy construction for AES RAID mode
Some checks are pending
Build / build-mac (push) Waiting to run
Build / build-linux (push) Waiting to run
Build / build-arm32 (push) Waiting to run
Build / build-win (push) Waiting to run
CodeQL Advanced / Analyze (c-cpp) (push) Waiting to run
CodeQL Advanced / Analyze (actions) (push) Waiting to run
Some checks are pending
Build / build-mac (push) Waiting to run
Build / build-linux (push) Waiting to run
Build / build-arm32 (push) Waiting to run
Build / build-win (push) Waiting to run
CodeQL Advanced / Analyze (c-cpp) (push) Waiting to run
CodeQL Advanced / Analyze (actions) (push) Waiting to run
This commit is contained in:
Andrea Mazzoleni
parent
a831853092
commit
6a786a42b6
20 changed files with 6105 additions and 5626 deletions
|
|
@ -1237,6 +1237,12 @@ void selftest(void)
|
|||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
if (raid_test_rec2_g23(256) != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "Failed G23 P/Q recovery test\n");
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
if (raid_test_tail(RAID_MODE_CAUCHY_AES, RAID_DATA_MAX, 256) != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "Failed TAIL Cauchy AES test\n");
|
||||
|
|
@ -1244,23 +1250,52 @@ void selftest(void)
|
|||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
|
||||
/* verify raid_mode get/set behavior */
|
||||
int cur = raid_mode(RAID_MODE_GET);
|
||||
int prev = raid_mode(RAID_MODE_CAUCHY_AES);
|
||||
if (prev != cur) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "raid_mode set did not return previous mode\n");
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
if (raid_mode(RAID_MODE_GET) != RAID_MODE_CAUCHY_AES) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "raid_mode GET did not return active mode\n");
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
|
||||
/* restore default mode */
|
||||
/* verify RAID -> AES -> RAID switches every active table */
|
||||
raid_mode(RAID_MODE_CAUCHY_RAID);
|
||||
if (raid_gfmul != raid_gfmul_raid || raid_gfinv != raid_gfinv_raid
|
||||
|| raid_gfcauchy != raid_gfcauchy_raid || raid_gfgen != raid_gfcauchy_raid) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "raid_mode did not select RAID tables\n");
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32)
|
||||
if (raid_gfcauchypshufb != raid_gfcauchypshufb_raid
|
||||
|| raid_gfmulpshufb != raid_gfmulpshufb_raid) {
|
||||
log_fatal(EINTERNAL, "raid_mode did not select RAID SIMD tables\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
#endif
|
||||
if (raid_mode(RAID_MODE_CAUCHY_AES) != RAID_MODE_CAUCHY_RAID
|
||||
|| raid_mode(RAID_MODE_GET) != RAID_MODE_CAUCHY_AES
|
||||
|| raid_gfmul != raid_gfmul_aes || raid_gfinv != raid_gfinv_aes
|
||||
|| raid_gfcauchy != raid_gfcauchy_aes || raid_gfgen != raid_gfcauchy_aes) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "raid_mode did not select AES tables\n");
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32)
|
||||
if (raid_gfcauchypshufb != raid_gfcauchypshufb_aes
|
||||
|| raid_gfmulpshufb != raid_gfmulpshufb_aes) {
|
||||
log_fatal(EINTERNAL, "raid_mode did not select AES SIMD tables\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
#endif
|
||||
if (raid_mode(RAID_MODE_CAUCHY_RAID) != RAID_MODE_CAUCHY_AES
|
||||
|| raid_gfmul != raid_gfmul_raid || raid_gfinv != raid_gfinv_raid
|
||||
|| raid_gfcauchy != raid_gfcauchy_raid || raid_gfgen != raid_gfcauchy_raid) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "raid_mode did not restore RAID tables\n");
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32)
|
||||
if (raid_gfcauchypshufb != raid_gfcauchypshufb_raid
|
||||
|| raid_gfmulpshufb != raid_gfmulpshufb_raid) {
|
||||
log_fatal(EINTERNAL, "raid_mode did not restore RAID SIMD tables\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
|
|
|
|||
21
raid/README
21
raid/README
|
|
@ -120,17 +120,20 @@ Operating Modes and Polynomials
|
|||
(Z parity). Optimized for low-end CPUs without vector instructions.
|
||||
|
||||
RAID_MODE_CAUCHY_AES (Mode = 2)
|
||||
Uses Cauchy matrix with AES polynomial 0x1b (x^8 + x^4 + x^3 + x + 1,
|
||||
0x11b) and primitive generator g = 3. On CPUs supporting Intel GFNI,
|
||||
generation and recovery are accelerated directly using VGF2P8MULB.
|
||||
Uses a 6x251 Extended Cauchy matrix with AES polynomial 0x1b
|
||||
(x^8 + x^4 + x^3 + x + 1, 0x11b). Its Q row uses the G23 sequence,
|
||||
with multiplication by 2 for almost every coefficient transition and
|
||||
multiplication by 3 only between its five cosets. On CPUs supporting
|
||||
Intel GFNI, generation and recovery are accelerated directly using VGF2P8MULB.
|
||||
This native GFNI multiplication is typically faster than the affine GFNI
|
||||
implementation required by the standard RAID polynomial. Parity data is
|
||||
incompatible with standard 0x11d polynomial arrays.
|
||||
incompatible with standard 0x11d polynomial arrays and with the previous
|
||||
RAID_MODE_CAUCHY_AES parity layout.
|
||||
|
||||
Both Cauchy modes use the same Extended Cauchy construction with coefficients
|
||||
based on powers of their respective primitive generator. GFNI acceleration
|
||||
is available for both fields: the RAID field uses affine transformations,
|
||||
while the AES field uses the native GFNI multiplication instruction.
|
||||
Both Cauchy modes support RAID_DATA_MAX = 251 data disks and
|
||||
RAID_PARITY_MAX = 6 parity levels. GFNI acceleration is available for both
|
||||
fields: the RAID field uses affine transformations, while the AES field uses
|
||||
the native GFNI multiplication instruction.
|
||||
|
||||
|
||||
Implementation Structure
|
||||
|
|
@ -140,7 +143,7 @@ Implementation Structure
|
|||
raid.c Cauchy matrix solver, Gauss-Jordan inversion, delta generation.
|
||||
module.c Algorithm registration, CPU detection, and dynamic dispatch.
|
||||
int.c Portable C fallback implementation (8/32/64-bit word loops).
|
||||
tables.c Precomputed GF(2^8) log/exp, multiplication, Cauchy tables.
|
||||
tables.c Precomputed GF(2^8) multiplication and Cauchy tables.
|
||||
mktables.c Generator for GF(2^8), Cauchy, and GFNI affine tables.
|
||||
internal.h Private header, assertion macros (BUG_ON), alignment helpers.
|
||||
|
||||
|
|
|
|||
269
raid/avx2.c
269
raid/avx2.c
|
|
@ -39,9 +39,9 @@ void raid_gen1_avx2(int nd, size_t size, void **vv)
|
|||
}
|
||||
|
||||
/*
|
||||
* GEN2 Cauchy AVX2 implementation using the active generator
|
||||
* GEN2 Cauchy AVX2 implementation using the active Q transition
|
||||
*/
|
||||
static __always_inline void raid_gen2_avx2_gen(int nd, size_t size, void **vv, int generator)
|
||||
static __always_inline void raid_gen2_avx2_gen(int nd, size_t size, void **vv, int g23)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
|
|
@ -57,15 +57,26 @@ static __always_inline void raid_gen2_avx2_gen(int nd, size_t size, void **vv, i
|
|||
|
||||
asm volatile ("vpbroadcastb %0, %%ymm7" : : "m" (gfconst16.poly[0]));
|
||||
asm volatile ("vpxor %ymm6,%ymm6,%ymm6");
|
||||
int g23_start = raid_g23_count(l - 1);
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
int g23_count = g23_start;
|
||||
int g23_x3;
|
||||
|
||||
asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[l][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[l][i + 32]));
|
||||
asm volatile ("vmovdqa %ymm0,%ymm2");
|
||||
asm volatile ("vmovdqa %ymm1,%ymm3");
|
||||
|
||||
for (d = l - 1; d >= 0; --d) {
|
||||
if (generator == 3) {
|
||||
g23_x3 = 0;
|
||||
if (g23) {
|
||||
g23_x3 = --g23_count == 0;
|
||||
if (g23_x3)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
if (g23 && g23_x3) {
|
||||
asm volatile ("vmovdqa %ymm2,%ymm4");
|
||||
asm volatile ("vmovdqa %ymm3,%ymm5");
|
||||
asm volatile ("vpaddb %ymm2,%ymm2,%ymm2");
|
||||
|
|
@ -108,12 +119,12 @@ static __always_inline void raid_gen2_avx2_gen(int nd, size_t size, void **vv, i
|
|||
|
||||
void raid_gen2_avx2_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen2_avx2_gen(nd, size, vv, 2);
|
||||
raid_gen2_avx2_gen(nd, size, vv, 0);
|
||||
}
|
||||
|
||||
void raid_gen2_avx2_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen2_avx2_gen(nd, size, vv, 3);
|
||||
raid_gen2_avx2_gen(nd, size, vv, 1);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
|
|
@ -126,7 +137,7 @@ void raid_gen2_avx2_aes(int nd, size_t size, void **vv)
|
|||
* Note that it uses 16 registers, meaning that x64 is required.
|
||||
*/
|
||||
static __always_inline void raid_gen2_avx2ext_gen(int nd, size_t size,
|
||||
void **vv, int generator)
|
||||
void **vv, int g23)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
|
|
@ -142,8 +153,13 @@ static __always_inline void raid_gen2_avx2ext_gen(int nd, size_t size,
|
|||
|
||||
asm volatile ("vpxor %ymm14,%ymm14,%ymm14");
|
||||
asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.poly[0]));
|
||||
int g23_start = raid_g23_count(l - 1);
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
int g23_count = g23_start;
|
||||
int g23_x3_d;
|
||||
int g23_x3_d1;
|
||||
|
||||
asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[l][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[l][i + 32]));
|
||||
asm volatile ("vmovdqa %ymm0,%ymm2");
|
||||
|
|
@ -151,61 +167,88 @@ static __always_inline void raid_gen2_avx2ext_gen(int nd, size_t size,
|
|||
|
||||
/* process two disks per iteration */
|
||||
for (d = l - 1; d >= 1; d -= 2) {
|
||||
g23_x3_d = 0;
|
||||
g23_x3_d1 = 0;
|
||||
if (g23) {
|
||||
g23_x3_d = --g23_count == 0;
|
||||
if (g23_x3_d)
|
||||
g23_count = 51;
|
||||
g23_x3_d1 = --g23_count == 0;
|
||||
if (g23_x3_d1)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm5" : : "m" (v[d][i + 32]));
|
||||
asm volatile ("vmovdqa %0,%%ymm6" : : "m" (v[d - 1][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm7" : : "m" (v[d - 1][i + 32]));
|
||||
|
||||
if (g23 && g23_x3_d) {
|
||||
asm volatile ("vmovdqa %ymm2,%ymm12");
|
||||
asm volatile ("vmovdqa %ymm3,%ymm13");
|
||||
}
|
||||
asm volatile ("vpcmpgtb %ymm2,%ymm14,%ymm10");
|
||||
asm volatile ("vpaddb %ymm2,%ymm2,%ymm8");
|
||||
asm volatile ("vpxor %ymm6,%ymm4,%ymm12");
|
||||
asm volatile ("vpand %ymm15,%ymm10,%ymm10");
|
||||
asm volatile ("vpxor %ymm10,%ymm8,%ymm8");
|
||||
asm volatile ("vpxor %ymm4,%ymm8,%ymm8");
|
||||
asm volatile ("vpcmpgtb %ymm8,%ymm14,%ymm10");
|
||||
asm volatile ("vpaddb %ymm8,%ymm8,%ymm8");
|
||||
asm volatile ("vpxor %ymm12,%ymm0,%ymm0");
|
||||
if (generator == 3)
|
||||
asm volatile ("vpxor %ymm2,%ymm12,%ymm12");
|
||||
asm volatile ("vpand %ymm15,%ymm10,%ymm10");
|
||||
asm volatile ("vpxor %ymm10,%ymm8,%ymm8");
|
||||
if (generator == 3)
|
||||
asm volatile ("vpxor %ymm12,%ymm8,%ymm2");
|
||||
else
|
||||
asm volatile ("vpxor %ymm6,%ymm8,%ymm2");
|
||||
|
||||
asm volatile ("vpcmpgtb %ymm3,%ymm14,%ymm11");
|
||||
asm volatile ("vpaddb %ymm3,%ymm3,%ymm9");
|
||||
asm volatile ("vpxor %ymm7,%ymm5,%ymm13");
|
||||
asm volatile ("vpaddb %ymm2,%ymm2,%ymm2");
|
||||
asm volatile ("vpaddb %ymm3,%ymm3,%ymm3");
|
||||
asm volatile ("vpand %ymm15,%ymm10,%ymm10");
|
||||
asm volatile ("vpand %ymm15,%ymm11,%ymm11");
|
||||
asm volatile ("vpxor %ymm11,%ymm9,%ymm9");
|
||||
asm volatile ("vpxor %ymm5,%ymm9,%ymm9");
|
||||
asm volatile ("vpcmpgtb %ymm9,%ymm14,%ymm11");
|
||||
asm volatile ("vpaddb %ymm9,%ymm9,%ymm9");
|
||||
asm volatile ("vpxor %ymm13,%ymm1,%ymm1");
|
||||
if (generator == 3)
|
||||
asm volatile ("vpxor %ymm3,%ymm13,%ymm13");
|
||||
asm volatile ("vpxor %ymm10,%ymm2,%ymm2");
|
||||
asm volatile ("vpxor %ymm11,%ymm3,%ymm3");
|
||||
if (g23 && g23_x3_d)
|
||||
asm volatile ("vpxor %ymm12,%ymm2,%ymm2");
|
||||
if (g23 && g23_x3_d)
|
||||
asm volatile ("vpxor %ymm13,%ymm3,%ymm3");
|
||||
asm volatile ("vpxor %ymm4,%ymm2,%ymm2");
|
||||
asm volatile ("vpxor %ymm5,%ymm3,%ymm3");
|
||||
|
||||
if (g23 && g23_x3_d1) {
|
||||
asm volatile ("vmovdqa %ymm2,%ymm12");
|
||||
asm volatile ("vmovdqa %ymm3,%ymm13");
|
||||
}
|
||||
asm volatile ("vpcmpgtb %ymm2,%ymm14,%ymm10");
|
||||
asm volatile ("vpcmpgtb %ymm3,%ymm14,%ymm11");
|
||||
asm volatile ("vpaddb %ymm2,%ymm2,%ymm2");
|
||||
asm volatile ("vpaddb %ymm3,%ymm3,%ymm3");
|
||||
asm volatile ("vpand %ymm15,%ymm10,%ymm10");
|
||||
asm volatile ("vpand %ymm15,%ymm11,%ymm11");
|
||||
asm volatile ("vpxor %ymm11,%ymm9,%ymm9");
|
||||
if (generator == 3)
|
||||
asm volatile ("vpxor %ymm13,%ymm9,%ymm3");
|
||||
else
|
||||
asm volatile ("vpxor %ymm7,%ymm9,%ymm3");
|
||||
asm volatile ("vpxor %ymm10,%ymm2,%ymm2");
|
||||
asm volatile ("vpxor %ymm11,%ymm3,%ymm3");
|
||||
if (g23 && g23_x3_d1)
|
||||
asm volatile ("vpxor %ymm12,%ymm2,%ymm2");
|
||||
if (g23 && g23_x3_d1)
|
||||
asm volatile ("vpxor %ymm13,%ymm3,%ymm3");
|
||||
asm volatile ("vpxor %ymm6,%ymm2,%ymm2");
|
||||
asm volatile ("vpxor %ymm7,%ymm3,%ymm3");
|
||||
|
||||
asm volatile ("vpxor %ymm4,%ymm0,%ymm0");
|
||||
asm volatile ("vpxor %ymm6,%ymm0,%ymm0");
|
||||
asm volatile ("vpxor %ymm5,%ymm1,%ymm1");
|
||||
asm volatile ("vpxor %ymm7,%ymm1,%ymm1");
|
||||
}
|
||||
|
||||
/* single remaining disk */
|
||||
if (d == 0) {
|
||||
int g23_x3;
|
||||
|
||||
g23_x3 = 0;
|
||||
if (g23) {
|
||||
g23_x3 = --g23_count == 0;
|
||||
if (g23_x3)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[0][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm5" : : "m" (v[0][i + 32]));
|
||||
|
||||
asm volatile ("vpcmpgtb %ymm2,%ymm14,%ymm10");
|
||||
asm volatile ("vpaddb %ymm2,%ymm2,%ymm8");
|
||||
asm volatile ("vpxor %ymm4,%ymm0,%ymm0");
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3)
|
||||
asm volatile ("vpxor %ymm4,%ymm2,%ymm12");
|
||||
asm volatile ("vpand %ymm15,%ymm10,%ymm10");
|
||||
asm volatile ("vpxor %ymm10,%ymm8,%ymm8");
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3)
|
||||
asm volatile ("vpxor %ymm12,%ymm8,%ymm2");
|
||||
else
|
||||
asm volatile ("vpxor %ymm4,%ymm8,%ymm2");
|
||||
|
|
@ -213,11 +256,11 @@ static __always_inline void raid_gen2_avx2ext_gen(int nd, size_t size,
|
|||
asm volatile ("vpcmpgtb %ymm3,%ymm14,%ymm11");
|
||||
asm volatile ("vpaddb %ymm3,%ymm3,%ymm9");
|
||||
asm volatile ("vpxor %ymm5,%ymm1,%ymm1");
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3)
|
||||
asm volatile ("vpxor %ymm5,%ymm3,%ymm13");
|
||||
asm volatile ("vpand %ymm15,%ymm11,%ymm11");
|
||||
asm volatile ("vpxor %ymm11,%ymm9,%ymm9");
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3)
|
||||
asm volatile ("vpxor %ymm13,%ymm9,%ymm3");
|
||||
else
|
||||
asm volatile ("vpxor %ymm5,%ymm9,%ymm3");
|
||||
|
|
@ -234,12 +277,12 @@ static __always_inline void raid_gen2_avx2ext_gen(int nd, size_t size,
|
|||
|
||||
void raid_gen2_avx2ext_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen2_avx2ext_gen(nd, size, vv, 2);
|
||||
raid_gen2_avx2ext_gen(nd, size, vv, 0);
|
||||
}
|
||||
|
||||
void raid_gen2_avx2ext_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen2_avx2ext_gen(nd, size, vv, 3);
|
||||
raid_gen2_avx2ext_gen(nd, size, vv, 1);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -324,7 +367,7 @@ void raid_genz_avx2ext_raid(int nd, size_t size, void **vv)
|
|||
*
|
||||
* Note that it uses 16 registers, meaning that x64 is required.
|
||||
*/
|
||||
static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv, int generator)
|
||||
static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv, int g23)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
|
|
@ -344,6 +387,7 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv
|
|||
memcpy(v[1 + i], v[0], size);
|
||||
return;
|
||||
}
|
||||
int g23_start = raid_g23_count(l - 1);
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
|
|
@ -352,6 +396,10 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv
|
|||
asm volatile ("vpbroadcastb %0, %%ymm11" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
int g23_count = g23_start;
|
||||
int g23_x3_d;
|
||||
int g23_x3_d1;
|
||||
|
||||
/* last disk without the generator multiplication */
|
||||
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[l][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[l][i + 32]));
|
||||
|
|
@ -379,20 +427,31 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv
|
|||
|
||||
/* process two intermediate disks per iteration */
|
||||
for (d = l - 1; d > 1; d -= 2) {
|
||||
g23_x3_d = 0;
|
||||
g23_x3_d1 = 0;
|
||||
if (g23) {
|
||||
g23_x3_d = --g23_count == 0;
|
||||
if (g23_x3_d)
|
||||
g23_count = 51;
|
||||
g23_x3_d1 = --g23_count == 0;
|
||||
if (g23_x3_d1)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i + 32]));
|
||||
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
|
||||
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3_d)
|
||||
asm volatile ("vmovdqa %ymm1,%ymm6");
|
||||
asm volatile ("vpxor %ymm5,%ymm5,%ymm5");
|
||||
asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5");
|
||||
asm volatile ("vpaddb %ymm1,%ymm1,%ymm1");
|
||||
asm volatile ("vpand %ymm3,%ymm5,%ymm5");
|
||||
asm volatile ("vpxor %ymm5,%ymm1,%ymm1");
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3_d)
|
||||
asm volatile ("vpxor %ymm6,%ymm1,%ymm1");
|
||||
|
||||
asm volatile ("vpxor %ymm4,%ymm0,%ymm0");
|
||||
|
|
@ -407,14 +466,14 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv
|
|||
asm volatile ("vpxor %ymm7,%ymm6,%ymm6");
|
||||
asm volatile ("vpxor %ymm6,%ymm2,%ymm2");
|
||||
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3_d)
|
||||
asm volatile ("vmovdqa %ymm9,%ymm6");
|
||||
asm volatile ("vpxor %ymm13,%ymm13,%ymm13");
|
||||
asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13");
|
||||
asm volatile ("vpaddb %ymm9,%ymm9,%ymm9");
|
||||
asm volatile ("vpand %ymm3,%ymm13,%ymm13");
|
||||
asm volatile ("vpxor %ymm13,%ymm9,%ymm9");
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3_d)
|
||||
asm volatile ("vpxor %ymm6,%ymm9,%ymm9");
|
||||
|
||||
asm volatile ("vpxor %ymm12,%ymm8,%ymm8");
|
||||
|
|
@ -435,14 +494,14 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv
|
|||
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfcauchypshufb[d - 1][1][0][0]));
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfcauchypshufb[d - 1][1][1][0]));
|
||||
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3_d1)
|
||||
asm volatile ("vmovdqa %ymm1,%ymm6");
|
||||
asm volatile ("vpxor %ymm5,%ymm5,%ymm5");
|
||||
asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5");
|
||||
asm volatile ("vpaddb %ymm1,%ymm1,%ymm1");
|
||||
asm volatile ("vpand %ymm3,%ymm5,%ymm5");
|
||||
asm volatile ("vpxor %ymm5,%ymm1,%ymm1");
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3_d1)
|
||||
asm volatile ("vpxor %ymm6,%ymm1,%ymm1");
|
||||
|
||||
asm volatile ("vpxor %ymm4,%ymm0,%ymm0");
|
||||
|
|
@ -457,14 +516,14 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv
|
|||
asm volatile ("vpxor %ymm7,%ymm6,%ymm6");
|
||||
asm volatile ("vpxor %ymm6,%ymm2,%ymm2");
|
||||
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3_d1)
|
||||
asm volatile ("vmovdqa %ymm9,%ymm6");
|
||||
asm volatile ("vpxor %ymm13,%ymm13,%ymm13");
|
||||
asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13");
|
||||
asm volatile ("vpaddb %ymm9,%ymm9,%ymm9");
|
||||
asm volatile ("vpand %ymm3,%ymm13,%ymm13");
|
||||
asm volatile ("vpxor %ymm13,%ymm9,%ymm9");
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3_d1)
|
||||
asm volatile ("vpxor %ymm6,%ymm9,%ymm9");
|
||||
|
||||
asm volatile ("vpxor %ymm12,%ymm8,%ymm8");
|
||||
|
|
@ -482,17 +541,24 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv
|
|||
|
||||
/* single remaining intermediate disk */
|
||||
if (d == 1) {
|
||||
g23_x3_d = 0;
|
||||
if (g23) {
|
||||
g23_x3_d = --g23_count == 0;
|
||||
if (g23_x3_d)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[1][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[1][i + 32]));
|
||||
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3_d)
|
||||
asm volatile ("vmovdqa %ymm1,%ymm6");
|
||||
asm volatile ("vpxor %ymm5,%ymm5,%ymm5");
|
||||
asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5");
|
||||
asm volatile ("vpaddb %ymm1,%ymm1,%ymm1");
|
||||
asm volatile ("vpand %ymm3,%ymm5,%ymm5");
|
||||
asm volatile ("vpxor %ymm5,%ymm1,%ymm1");
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3_d)
|
||||
asm volatile ("vpxor %ymm6,%ymm1,%ymm1");
|
||||
|
||||
asm volatile ("vpxor %ymm4,%ymm0,%ymm0");
|
||||
|
|
@ -510,14 +576,14 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv
|
|||
asm volatile ("vpxor %ymm7,%ymm6,%ymm6");
|
||||
asm volatile ("vpxor %ymm6,%ymm2,%ymm2");
|
||||
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3_d)
|
||||
asm volatile ("vmovdqa %ymm9,%ymm6");
|
||||
asm volatile ("vpxor %ymm13,%ymm13,%ymm13");
|
||||
asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13");
|
||||
asm volatile ("vpaddb %ymm9,%ymm9,%ymm9");
|
||||
asm volatile ("vpand %ymm3,%ymm13,%ymm13");
|
||||
asm volatile ("vpxor %ymm13,%ymm9,%ymm9");
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3_d)
|
||||
asm volatile ("vpxor %ymm6,%ymm9,%ymm9");
|
||||
|
||||
asm volatile ("vpxor %ymm12,%ymm8,%ymm8");
|
||||
|
|
@ -537,7 +603,14 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv
|
|||
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[0][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[0][i + 32]));
|
||||
|
||||
if (generator == 3) {
|
||||
g23_x3_d = 0;
|
||||
if (g23) {
|
||||
g23_x3_d = --g23_count == 0;
|
||||
if (g23_x3_d)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
if (g23 && g23_x3_d) {
|
||||
asm volatile ("vmovdqa %ymm1,%ymm6");
|
||||
asm volatile ("vmovdqa %ymm9,%ymm14");
|
||||
}
|
||||
|
|
@ -551,7 +624,7 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv
|
|||
asm volatile ("vpand %ymm3,%ymm13,%ymm13");
|
||||
asm volatile ("vpxor %ymm5,%ymm1,%ymm1");
|
||||
asm volatile ("vpxor %ymm13,%ymm9,%ymm9");
|
||||
if (generator == 3) {
|
||||
if (g23 && g23_x3_d) {
|
||||
asm volatile ("vpxor %ymm6,%ymm1,%ymm1");
|
||||
asm volatile ("vpxor %ymm14,%ymm9,%ymm9");
|
||||
}
|
||||
|
|
@ -576,12 +649,12 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv
|
|||
|
||||
void raid_gen3_avx2ext_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen3_avx2ext_gen(nd, size, vv, 2);
|
||||
raid_gen3_avx2ext_gen(nd, size, vv, 0);
|
||||
}
|
||||
|
||||
void raid_gen3_avx2ext_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen3_avx2ext_gen(nd, size, vv, 3);
|
||||
raid_gen3_avx2ext_gen(nd, size, vv, 1);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -591,7 +664,7 @@ void raid_gen3_avx2ext_aes(int nd, size_t size, void **vv)
|
|||
*
|
||||
* Note that it uses 16 registers, meaning that x64 is required.
|
||||
*/
|
||||
static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv, int generator)
|
||||
static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv, int g23)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
|
|
@ -613,6 +686,7 @@ static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv
|
|||
memcpy(v[1 + i], v[0], size);
|
||||
return;
|
||||
}
|
||||
int g23_start = raid_g23_count(l - 1);
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
|
|
@ -622,6 +696,9 @@ static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv
|
|||
|
||||
/* generic case with at least two data disks */
|
||||
for (i = 0; i < size; i += 64) {
|
||||
int g23_count = g23_start;
|
||||
int g23_x3;
|
||||
|
||||
/* last disk without the generator multiplication */
|
||||
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[l][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[l][i + 32]));
|
||||
|
|
@ -658,14 +735,21 @@ static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv
|
|||
|
||||
/* intermediate disks */
|
||||
for (d = l - 1; d > 0; --d) {
|
||||
if (generator == 3)
|
||||
g23_x3 = 0;
|
||||
if (g23) {
|
||||
g23_x3 = --g23_count == 0;
|
||||
if (g23_x3)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
if (g23 && g23_x3)
|
||||
asm volatile ("vmovdqa %ymm1,%ymm4");
|
||||
asm volatile ("vpxor %ymm5,%ymm5,%ymm5");
|
||||
asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5");
|
||||
asm volatile ("vpaddb %ymm1,%ymm1,%ymm1");
|
||||
asm volatile ("vpand %ymm6,%ymm5,%ymm5");
|
||||
asm volatile ("vpxor %ymm5,%ymm1,%ymm1");
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3)
|
||||
asm volatile ("vpxor %ymm4,%ymm1,%ymm1");
|
||||
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i]));
|
||||
asm volatile ("vpxor %ymm4,%ymm0,%ymm0");
|
||||
|
|
@ -673,14 +757,14 @@ static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv
|
|||
asm volatile ("vpsrlw $4,%ymm4,%ymm5");
|
||||
asm volatile ("vpand %ymm7,%ymm4,%ymm4");
|
||||
asm volatile ("vpand %ymm7,%ymm5,%ymm5");
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3)
|
||||
asm volatile ("vmovdqa %ymm9,%ymm12");
|
||||
asm volatile ("vpxor %ymm13,%ymm13,%ymm13");
|
||||
asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13");
|
||||
asm volatile ("vpaddb %ymm9,%ymm9,%ymm9");
|
||||
asm volatile ("vpand %ymm6,%ymm13,%ymm13");
|
||||
asm volatile ("vpxor %ymm13,%ymm9,%ymm9");
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3)
|
||||
asm volatile ("vpxor %ymm12,%ymm9,%ymm9");
|
||||
|
||||
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i + 32]));
|
||||
|
|
@ -715,28 +799,35 @@ static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv
|
|||
}
|
||||
|
||||
/* first disk with all coefficients at 1 */
|
||||
if (generator == 3)
|
||||
g23_x3 = 0;
|
||||
if (g23) {
|
||||
g23_x3 = --g23_count == 0;
|
||||
if (g23_x3)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
if (g23 && g23_x3)
|
||||
asm volatile ("vmovdqa %ymm1,%ymm4");
|
||||
asm volatile ("vpxor %ymm5,%ymm5,%ymm5");
|
||||
asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5");
|
||||
asm volatile ("vpaddb %ymm1,%ymm1,%ymm1");
|
||||
asm volatile ("vpand %ymm6,%ymm5,%ymm5");
|
||||
asm volatile ("vpxor %ymm5,%ymm1,%ymm1");
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3)
|
||||
asm volatile ("vpxor %ymm4,%ymm1,%ymm1");
|
||||
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[0][i]));
|
||||
asm volatile ("vpxor %ymm4,%ymm0,%ymm0");
|
||||
asm volatile ("vpxor %ymm4,%ymm1,%ymm1");
|
||||
asm volatile ("vpxor %ymm4,%ymm2,%ymm2");
|
||||
asm volatile ("vpxor %ymm4,%ymm3,%ymm3");
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3)
|
||||
asm volatile ("vmovdqa %ymm9,%ymm12");
|
||||
asm volatile ("vpxor %ymm13,%ymm13,%ymm13");
|
||||
asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13");
|
||||
asm volatile ("vpaddb %ymm9,%ymm9,%ymm9");
|
||||
asm volatile ("vpand %ymm6,%ymm13,%ymm13");
|
||||
asm volatile ("vpxor %ymm13,%ymm9,%ymm9");
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3)
|
||||
asm volatile ("vpxor %ymm12,%ymm9,%ymm9");
|
||||
|
||||
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[0][i + 32]));
|
||||
|
|
@ -760,12 +851,12 @@ static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv
|
|||
|
||||
void raid_gen4_avx2ext_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen4_avx2ext_gen(nd, size, vv, 2);
|
||||
raid_gen4_avx2ext_gen(nd, size, vv, 0);
|
||||
}
|
||||
|
||||
void raid_gen4_avx2ext_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen4_avx2ext_gen(nd, size, vv, 3);
|
||||
raid_gen4_avx2ext_gen(nd, size, vv, 1);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -773,7 +864,7 @@ void raid_gen4_avx2ext_aes(int nd, size_t size, void **vv)
|
|||
/*
|
||||
* GENX AVX2EXT implementation
|
||||
*/
|
||||
static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, int np, int generator)
|
||||
static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, int np, int g23)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
size_t i;
|
||||
|
|
@ -787,6 +878,7 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
|
|||
memcpy(v[1 + d], v[0], size);
|
||||
return;
|
||||
}
|
||||
int g23_start = raid_g23_count(l - 1);
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
|
|
@ -796,6 +888,9 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
|
|||
asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
for (i = 0; i < size; i += 32) {
|
||||
int g23_count = g23_start;
|
||||
int g23_x3;
|
||||
|
||||
/* last disk without the generator multiplication */
|
||||
asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[l][i]));
|
||||
|
||||
|
|
@ -836,15 +931,22 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
|
|||
|
||||
/* intermediate disks */
|
||||
for (d = l - 1; d > 0; --d) {
|
||||
g23_x3 = 0;
|
||||
if (g23) {
|
||||
g23_x3 = --g23_count == 0;
|
||||
if (g23_x3)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[d][i]));
|
||||
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3)
|
||||
asm volatile ("vmovdqa %ymm1,%ymm12");
|
||||
asm volatile ("vpcmpgtb %ymm1,%ymm6,%ymm11");
|
||||
asm volatile ("vpaddb %ymm1,%ymm1,%ymm1");
|
||||
asm volatile ("vpand %ymm14,%ymm11,%ymm11");
|
||||
asm volatile ("vpxor %ymm11,%ymm1,%ymm1");
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3)
|
||||
asm volatile ("vpxor %ymm12,%ymm1,%ymm1");
|
||||
|
||||
asm volatile ("vpxor %ymm10,%ymm0,%ymm0");
|
||||
|
|
@ -890,13 +992,20 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
|
|||
/* first disk with all coefficients at 1 */
|
||||
asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[0][i]));
|
||||
|
||||
if (generator == 3)
|
||||
g23_x3 = 0;
|
||||
if (g23) {
|
||||
g23_x3 = --g23_count == 0;
|
||||
if (g23_x3)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
if (g23 && g23_x3)
|
||||
asm volatile ("vmovdqa %ymm1,%ymm12");
|
||||
asm volatile ("vpcmpgtb %ymm1,%ymm6,%ymm11");
|
||||
asm volatile ("vpaddb %ymm1,%ymm1,%ymm1");
|
||||
asm volatile ("vpand %ymm14,%ymm11,%ymm11");
|
||||
asm volatile ("vpxor %ymm11,%ymm1,%ymm1");
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3)
|
||||
asm volatile ("vpxor %ymm12,%ymm1,%ymm1");
|
||||
|
||||
asm volatile ("vpxor %ymm10,%ymm0,%ymm0");
|
||||
|
|
@ -934,12 +1043,12 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
|
|||
*/
|
||||
void raid_gen5_avx2ext_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx2ext(nd, size, vv, 5, 2);
|
||||
raid_genX_avx2ext(nd, size, vv, 5, 0);
|
||||
}
|
||||
|
||||
void raid_gen5_avx2ext_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx2ext(nd, size, vv, 5, 3);
|
||||
raid_genX_avx2ext(nd, size, vv, 5, 1);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -951,12 +1060,12 @@ void raid_gen5_avx2ext_aes(int nd, size_t size, void **vv)
|
|||
*/
|
||||
void raid_gen6_avx2ext_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx2ext(nd, size, vv, 6, 2);
|
||||
raid_genX_avx2ext(nd, size, vv, 6, 0);
|
||||
}
|
||||
|
||||
void raid_gen6_avx2ext_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx2ext(nd, size, vv, 6, 3);
|
||||
raid_genX_avx2ext(nd, size, vv, 6, 1);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
|
|||
10
raid/gf.h
10
raid/gf.h
|
|
@ -29,16 +29,6 @@ static __always_inline uint8_t inv(uint8_t v)
|
|||
return raid_gfinv[v];
|
||||
}
|
||||
|
||||
/*
|
||||
* GF generator^a for the active Cauchy matrix.
|
||||
*/
|
||||
static __always_inline uint8_t powgen(int v)
|
||||
{
|
||||
BUG_ON(v < 0 || v > 254); /* invalid exponent */
|
||||
|
||||
return raid_gfexp[v];
|
||||
}
|
||||
|
||||
/*
|
||||
* Gets the multiplication table for a specified value.
|
||||
*/
|
||||
|
|
|
|||
66
raid/int.c
66
raid/int.c
|
|
@ -63,7 +63,7 @@ void raid_gen1_int64(int nd, size_t size, void **vv)
|
|||
/*
|
||||
* GEN2 (double parity) 32bit C implementation
|
||||
*/
|
||||
static __always_inline void raid_gen2_int32_gen(int nd, size_t size, void **vv, int generator)
|
||||
static __always_inline void raid_gen2_int32_gen(int nd, size_t size, void **vv, int g23)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
|
|
@ -79,26 +79,39 @@ static __always_inline void raid_gen2_int32_gen(int nd, size_t size, void **vv,
|
|||
l = nd - 1;
|
||||
p = v[nd];
|
||||
q = v[nd + 1];
|
||||
int g23_start = raid_g23_count(l - 1);
|
||||
|
||||
for (i = 0; i < size; i += 8) {
|
||||
int g23_count = g23_start;
|
||||
int g23_x3;
|
||||
|
||||
q0 = p0 = v_read32(&v[l][i]);
|
||||
q1 = p1 = v_read32(&v[l][i + 4]);
|
||||
for (d = l - 1; d >= 0; --d) {
|
||||
g23_x3 = 0;
|
||||
if (g23) {
|
||||
g23_x3 = --g23_count == 0;
|
||||
if (g23_x3)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
d0 = v_read32(&v[d][i]);
|
||||
d1 = v_read32(&v[d][i + 4]);
|
||||
|
||||
p0 ^= d0;
|
||||
p1 ^= d1;
|
||||
|
||||
qold = q0;
|
||||
q0 = x2_32(q0, poly_32);
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3) {
|
||||
qold = q0;
|
||||
q0 = x2_32(q0, poly_32);
|
||||
q0 ^= qold;
|
||||
|
||||
qold = q1;
|
||||
q1 = x2_32(q1, poly_32);
|
||||
if (generator == 3)
|
||||
qold = q1;
|
||||
q1 = x2_32(q1, poly_32);
|
||||
q1 ^= qold;
|
||||
} else {
|
||||
q0 = x2_32(q0, poly_32);
|
||||
q1 = x2_32(q1, poly_32);
|
||||
}
|
||||
|
||||
q0 ^= d0;
|
||||
q1 ^= d1;
|
||||
|
|
@ -112,18 +125,18 @@ static __always_inline void raid_gen2_int32_gen(int nd, size_t size, void **vv,
|
|||
|
||||
void raid_gen2_int32_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen2_int32_gen(nd, size, vv, 2);
|
||||
raid_gen2_int32_gen(nd, size, vv, 0);
|
||||
}
|
||||
|
||||
void raid_gen2_int32_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen2_int32_gen(nd, size, vv, 3);
|
||||
raid_gen2_int32_gen(nd, size, vv, 1);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN2 (double parity) 64bit C implementation
|
||||
*/
|
||||
static __always_inline void raid_gen2_int64_gen(int nd, size_t size, void **vv, int generator)
|
||||
static __always_inline void raid_gen2_int64_gen(int nd, size_t size, void **vv, int g23)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
|
|
@ -139,26 +152,39 @@ static __always_inline void raid_gen2_int64_gen(int nd, size_t size, void **vv,
|
|||
l = nd - 1;
|
||||
p = v[nd];
|
||||
q = v[nd + 1];
|
||||
int g23_start = raid_g23_count(l - 1);
|
||||
|
||||
for (i = 0; i < size; i += 16) {
|
||||
int g23_count = g23_start;
|
||||
int g23_x3;
|
||||
|
||||
q0 = p0 = v_read64(&v[l][i]);
|
||||
q1 = p1 = v_read64(&v[l][i + 8]);
|
||||
for (d = l - 1; d >= 0; --d) {
|
||||
g23_x3 = 0;
|
||||
if (g23) {
|
||||
g23_x3 = --g23_count == 0;
|
||||
if (g23_x3)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
d0 = v_read64(&v[d][i]);
|
||||
d1 = v_read64(&v[d][i + 8]);
|
||||
|
||||
p0 ^= d0;
|
||||
p1 ^= d1;
|
||||
|
||||
qold = q0;
|
||||
q0 = x2_64(q0, poly_64);
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3) {
|
||||
qold = q0;
|
||||
q0 = x2_64(q0, poly_64);
|
||||
q0 ^= qold;
|
||||
|
||||
qold = q1;
|
||||
q1 = x2_64(q1, poly_64);
|
||||
if (generator == 3)
|
||||
qold = q1;
|
||||
q1 = x2_64(q1, poly_64);
|
||||
q1 ^= qold;
|
||||
} else {
|
||||
q0 = x2_64(q0, poly_64);
|
||||
q1 = x2_64(q1, poly_64);
|
||||
}
|
||||
|
||||
q0 ^= d0;
|
||||
q1 ^= d1;
|
||||
|
|
@ -172,12 +198,12 @@ static __always_inline void raid_gen2_int64_gen(int nd, size_t size, void **vv,
|
|||
|
||||
void raid_gen2_int64_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen2_int64_gen(nd, size, vv, 2);
|
||||
raid_gen2_int64_gen(nd, size, vv, 0);
|
||||
}
|
||||
|
||||
void raid_gen2_int64_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen2_int64_gen(nd, size, vv, 3);
|
||||
raid_gen2_int64_gen(nd, size, vv, 1);
|
||||
}
|
||||
|
||||
void raid_gen2_int8(int nd, size_t size, void **vv)
|
||||
|
|
|
|||
|
|
@ -74,6 +74,63 @@ static __always_inline void * __align_ptr(void *ptr, uintptr_t size)
|
|||
return (void *)offset;
|
||||
}
|
||||
|
||||
/*
|
||||
* Initialize the countdown used by the AES G23 Horner recurrence.
|
||||
*
|
||||
* The G23 Q coefficients are generated by multiplying by 2 at every step,
|
||||
* except at the four boundaries between the five cosets, where the next
|
||||
* coefficient is obtained by multiplying by 3 instead:
|
||||
*
|
||||
* q[i + 1] = 3 * q[i] for i = 50, 101, 152, 203
|
||||
* q[i + 1] = 2 * q[i] otherwise
|
||||
*
|
||||
* The optimized parity generators evaluate Q in descending Horner order.
|
||||
* If the current step is identified by disk index d, the x3 transitions are
|
||||
* therefore encountered at:
|
||||
*
|
||||
* d = 203, 152, 101, 50
|
||||
*
|
||||
* i.e. every 51 descending steps.
|
||||
*
|
||||
* Rather than comparing d against those four values in the hot loop, we keep
|
||||
* a countdown. Before processing each Horner step the generator does:
|
||||
*
|
||||
* if (--g23_count == 0) {
|
||||
* Q = 3 * Q;
|
||||
* g23_count = 51;
|
||||
* } else {
|
||||
* Q = 2 * Q;
|
||||
* }
|
||||
*
|
||||
* This function returns the initial countdown for a traversal whose first
|
||||
* processed step has index d. For d >= 50, (d - 50) % 51 gives the distance
|
||||
* from the previous x3 boundary, so adding 1 gives the number of decrements
|
||||
* required before the next descending x3 transition is reached.
|
||||
*
|
||||
* For d < 50 there is no remaining x3 transition. Returning 51 is safe
|
||||
* because fewer than 51 Horner steps remain, so the countdown cannot reach
|
||||
* zero before the traversal ends.
|
||||
*
|
||||
* Examples:
|
||||
*
|
||||
* d = 203 -> 1 -> x3 immediately
|
||||
* d = 204 -> 2 -> x2 at 204, x3 at 203
|
||||
* d = 250 -> 48 -> x3 at 203
|
||||
* d = 152 -> 1 -> x3 immediately
|
||||
* d = 50 -> 1 -> x3 immediately
|
||||
* d = 49 -> 51 -> no further x3 transition
|
||||
*
|
||||
* The returned value is normally computed once as g23_start and copied into
|
||||
* g23_count at the beginning of each independent Horner traversal.
|
||||
*/
|
||||
static __always_inline int raid_g23_count(int d)
|
||||
{
|
||||
if (d < 50)
|
||||
return 51;
|
||||
|
||||
return (d - 50) % 51 + 1;
|
||||
}
|
||||
|
||||
/*
|
||||
* Includes the main interface headers.
|
||||
*/
|
||||
|
|
@ -357,7 +414,6 @@ extern struct gfconst16 gfconst16 __aligned(16);
|
|||
* Tables.
|
||||
*/
|
||||
extern const uint8_t raid_gfmul_raid[256][256] __aligned(256);
|
||||
extern const uint8_t raid_gfexp_raid[256] __aligned(256);
|
||||
extern const uint8_t raid_gfinv_raid[256] __aligned(256);
|
||||
extern const uint8_t raid_gfvandermonde_raid[3][256] __aligned(256);
|
||||
extern const uint8_t raid_gfcauchy_raid[6][256] __aligned(256);
|
||||
|
|
@ -368,7 +424,6 @@ extern const uint8_t raid_gfmulpshufb_raid[256][2][16] __aligned(256);
|
|||
#endif
|
||||
|
||||
extern const uint8_t raid_gfmul_aes[256][256] __aligned(256);
|
||||
extern const uint8_t raid_gfexp_aes[256] __aligned(256);
|
||||
extern const uint8_t raid_gfinv_aes[256] __aligned(256);
|
||||
extern const uint8_t raid_gfcauchy_aes[6][256] __aligned(256);
|
||||
#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32)
|
||||
|
|
@ -377,7 +432,6 @@ extern const uint8_t raid_gfmulpshufb_aes[256][2][16] __aligned(256);
|
|||
#endif
|
||||
|
||||
extern const uint8_t(*raid_gfmul)[256];
|
||||
extern const uint8_t *raid_gfexp;
|
||||
extern const uint8_t *raid_gfinv;
|
||||
extern const uint8_t(*raid_gfvandermonde)[256];
|
||||
extern const uint8_t(*raid_gfcauchy)[256];
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
|
||||
#include <stdio.h>
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
/*
|
||||
* Galois field reduction polynomials.
|
||||
|
|
@ -84,9 +85,9 @@ uint8_t raid_gfinv[256];
|
|||
#define DISK (257 - PARITY)
|
||||
|
||||
/**
|
||||
* Setup the Cauchy matrix used to generate the parity.
|
||||
* Setup the RAID/g=2 Cauchy matrix used to generate parity.
|
||||
*/
|
||||
static void set_cauchy(uint8_t poly, uint8_t generator, uint8_t *matrix)
|
||||
static void set_cauchy_raid(uint8_t poly, uint8_t generator, uint8_t *matrix)
|
||||
{
|
||||
int i, j;
|
||||
uint8_t inv_x, y;
|
||||
|
|
@ -206,6 +207,68 @@ static void set_cauchy(uint8_t poly, uint8_t generator, uint8_t *matrix)
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup the AES Extended Cauchy matrix using the G23 sequence.
|
||||
*
|
||||
* The data X set consumes the canonical sequence from the beginning,
|
||||
* x[i]=inv(q[i]), while the Cauchy Y set starts with zero and consumes the
|
||||
* sequence backward from the end, y[j]=inv(q[255-j]) for j>0. For PARITY=n,
|
||||
* DISK=257-n; increasing parity removes one X element from the tail and
|
||||
* appends that same element to Y, leaving all existing rows unchanged on the
|
||||
* remaining data columns. This defines a nested matrix without implementing
|
||||
* parity levels above the current limit.
|
||||
*/
|
||||
static void set_cauchy_aes(uint8_t *matrix)
|
||||
{
|
||||
uint8_t q[255];
|
||||
uint8_t y[PARITY - 1];
|
||||
int i, j, k;
|
||||
|
||||
q[0] = 1;
|
||||
for (i = 0; i < 254; ++i) {
|
||||
uint8_t generator = i == 50 || i == 101 || i == 152 || i == 203 ? 3 : 2;
|
||||
|
||||
q[i + 1] = raid_gfmul(RAID_POLY_AES, generator, q[i]);
|
||||
}
|
||||
|
||||
/* The five cosets must enumerate GF(256)* exactly once. */
|
||||
for (i = 0; i < 255; ++i) {
|
||||
if (q[i] == 0) {
|
||||
fprintf(stderr, "Invalid zero in AES G23 sequence at %d\n", i);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
for (k = i + 1; k < 255; ++k) {
|
||||
if (q[i] == q[k]) {
|
||||
fprintf(stderr, "Duplicate in AES G23 sequence at %d and %d\n", i, k);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (i = 0; i < DISK; ++i)
|
||||
matrix[i] = 1;
|
||||
|
||||
y[0] = 0;
|
||||
for (j = 1; j < PARITY - 1; ++j)
|
||||
y[j] = raid_gfinv[q[255 - j]];
|
||||
|
||||
for (j = 0; j < PARITY - 1; ++j) {
|
||||
for (i = 0; i < DISK; ++i) {
|
||||
uint8_t x = raid_gfinv[q[i]];
|
||||
|
||||
matrix[(j + 1) * DISK + i] = raid_gfinv[x ^ y[j]];
|
||||
}
|
||||
}
|
||||
|
||||
/* Normalize R..U so every coefficient in the first column is 1. */
|
||||
for (j = 1; j < PARITY - 1; ++j) {
|
||||
uint8_t f = raid_gfinv[matrix[(j + 1) * DISK]];
|
||||
|
||||
for (i = 0; i < DISK; ++i)
|
||||
matrix[(j + 1) * DISK + i] = raid_gfmul(RAID_POLY_AES, matrix[(j + 1) * DISK + i], f);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup the Power matrix used to generate the parity.
|
||||
*/
|
||||
|
|
@ -284,22 +347,6 @@ void tables(uint8_t poly, uint8_t generator, const char *tag)
|
|||
printf("};\n\n");
|
||||
}
|
||||
|
||||
/* generator^a */
|
||||
printf("const uint8_t __aligned(256) raid_gfexp_%s[256] =\n", tag);
|
||||
printf("{\n");
|
||||
v = 1;
|
||||
for (i = 0; i < 256; ++i) {
|
||||
if (i % 8 == 0)
|
||||
printf("\t");
|
||||
printf("0x%02x,", v);
|
||||
v = raid_gfmul(poly, v, generator);
|
||||
if (i % 8 == 7)
|
||||
printf("\n");
|
||||
else
|
||||
printf(" ");
|
||||
}
|
||||
printf("};\n\n");
|
||||
|
||||
/* 1/a */
|
||||
printf("const uint8_t __aligned(256) raid_gfinv_%s[256] =\n", tag);
|
||||
printf("{\n");
|
||||
|
|
@ -320,7 +367,10 @@ void tables(uint8_t poly, uint8_t generator, const char *tag)
|
|||
printf("};\n\n");
|
||||
|
||||
/* cauchy matrix */
|
||||
set_cauchy(poly, generator, matrix);
|
||||
if (poly == RAID_POLY_AES)
|
||||
set_cauchy_aes(matrix);
|
||||
else
|
||||
set_cauchy_raid(poly, generator, matrix);
|
||||
|
||||
printf("/**\n");
|
||||
printf(" * Cauchy matrix used to generate parity.\n");
|
||||
|
|
@ -410,10 +460,11 @@ void tables(uint8_t poly, uint8_t generator, const char *tag)
|
|||
printf("\t},\n");
|
||||
}
|
||||
printf("};\n");
|
||||
printf("#endif\n\n");
|
||||
printf("#endif\n");
|
||||
|
||||
if (poly == RAID_POLY_AES)
|
||||
return;
|
||||
printf("\n");
|
||||
|
||||
/* power matrix */
|
||||
set_power(poly, matrix);
|
||||
|
|
|
|||
|
|
@ -40,7 +40,6 @@ static struct raid_rec_algo *raid_rec_algo;
|
|||
int raid_mode_active = RAID_MODE_GET;
|
||||
|
||||
const uint8_t(*raid_gfmul)[256];
|
||||
const uint8_t *raid_gfexp;
|
||||
const uint8_t *raid_gfinv;
|
||||
const uint8_t(*raid_gfvandermonde)[256];
|
||||
const uint8_t(*raid_gfcauchy)[256];
|
||||
|
|
@ -225,7 +224,6 @@ int raid_mode(int mode)
|
|||
|
||||
if (mode == RAID_MODE_CAUCHY_AES) {
|
||||
raid_gfmul = raid_gfmul_aes;
|
||||
raid_gfexp = raid_gfexp_aes;
|
||||
raid_gfinv = raid_gfinv_aes;
|
||||
raid_gfvandermonde = 0;
|
||||
raid_gfcauchy = raid_gfcauchy_aes;
|
||||
|
|
@ -245,7 +243,6 @@ int raid_mode(int mode)
|
|||
raid_rec_algo = raid_rec_algo_aes;
|
||||
} else {
|
||||
raid_gfmul = raid_gfmul_raid;
|
||||
raid_gfexp = raid_gfexp_raid;
|
||||
raid_gfinv = raid_gfinv_raid;
|
||||
raid_gfvandermonde = raid_gfvandermonde_raid;
|
||||
raid_gfcauchy = raid_gfcauchy_raid;
|
||||
|
|
|
|||
67
raid/neon.c
67
raid/neon.c
|
|
@ -60,9 +60,9 @@ void raid_gen1_neon(int nd, size_t size, void **vv)
|
|||
}
|
||||
|
||||
/*
|
||||
* GEN2 Cauchy NEON implementation using the active generator
|
||||
* GEN2 Cauchy NEON implementation using the active Q transition
|
||||
*/
|
||||
static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, int generator)
|
||||
static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, int g23)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
|
|
@ -81,8 +81,12 @@ static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, i
|
|||
:
|
||||
: "m" (gfconst16.poly[0])
|
||||
);
|
||||
int g23_start = raid_g23_count(l - 1);
|
||||
|
||||
for (i = 0; i < size; i += 32) {
|
||||
int g23_count = g23_start;
|
||||
int g23_x3;
|
||||
|
||||
asm volatile (
|
||||
"ldr q0, %0\n"
|
||||
"ldr q1, %1\n"
|
||||
|
|
@ -93,7 +97,14 @@ static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, i
|
|||
);
|
||||
|
||||
for (d = l - 1; d >= 0; --d) {
|
||||
if (generator == 3) {
|
||||
g23_x3 = 0;
|
||||
if (g23) {
|
||||
g23_x3 = --g23_count == 0;
|
||||
if (g23_x3)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
if (g23 && g23_x3) {
|
||||
/* v14-v15 are otherwise unused and preserve Q across the AES xtime. */
|
||||
asm volatile (
|
||||
"mov v14.16b, v2.16b\n"
|
||||
|
|
@ -122,7 +133,7 @@ static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, i
|
|||
:
|
||||
: "m" (v[d][i]), "m" (v[d][i + 16])
|
||||
);
|
||||
if (generator == 3) {
|
||||
if (g23 && g23_x3) {
|
||||
asm volatile (
|
||||
"eor v2.16b, v2.16b, v14.16b\n"
|
||||
"eor v3.16b, v3.16b, v15.16b\n"
|
||||
|
|
@ -145,12 +156,12 @@ static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, i
|
|||
|
||||
void raid_gen2_neon_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen2_neon_gen(nd, size, vv, 2);
|
||||
raid_gen2_neon_gen(nd, size, vv, 0);
|
||||
}
|
||||
|
||||
void raid_gen2_neon_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen2_neon_gen(nd, size, vv, 3);
|
||||
raid_gen2_neon_gen(nd, size, vv, 1);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -250,7 +261,7 @@ void raid_genz_neon_raid(int nd, size_t size, void **vv)
|
|||
/*
|
||||
* GENX NEON implementation
|
||||
*/
|
||||
static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int np, int generator)
|
||||
static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int np, int g23)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
size_t i;
|
||||
|
|
@ -264,6 +275,7 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
|
|||
memcpy(v[1 + d], v[0], size);
|
||||
return;
|
||||
}
|
||||
int g23_start = raid_g23_count(l - 1);
|
||||
|
||||
raid_neon_begin();
|
||||
|
||||
|
|
@ -276,6 +288,9 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
|
|||
);
|
||||
|
||||
for (i = 0; i < size; i += 32) {
|
||||
int g23_count = g23_start;
|
||||
int g23_x3;
|
||||
|
||||
/* last disk without the generator multiplication */
|
||||
asm volatile (
|
||||
"ldr q0, %0\n"
|
||||
|
|
@ -359,7 +374,14 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
|
|||
|
||||
/* intermediate disks */
|
||||
for (d = l - 1; d > 0; --d) {
|
||||
if (generator == 3) {
|
||||
g23_x3 = 0;
|
||||
if (g23) {
|
||||
g23_x3 = --g23_count == 0;
|
||||
if (g23_x3)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
if (g23 && g23_x3) {
|
||||
asm volatile (
|
||||
"mov v14.16b, v2.16b\n"
|
||||
"mov v15.16b, v3.16b\n"
|
||||
|
|
@ -387,7 +409,7 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
|
|||
:
|
||||
: "m" (v[d][i]), "m" (v[d][i + 16])
|
||||
);
|
||||
if (generator == 3) {
|
||||
if (g23 && g23_x3) {
|
||||
asm volatile (
|
||||
"eor v2.16b, v2.16b, v14.16b\n"
|
||||
"eor v3.16b, v3.16b, v15.16b\n"
|
||||
|
|
@ -475,7 +497,14 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
|
|||
}
|
||||
|
||||
/* first disk with all coefficients at 1 */
|
||||
if (generator == 3) {
|
||||
g23_x3 = 0;
|
||||
if (g23) {
|
||||
g23_x3 = --g23_count == 0;
|
||||
if (g23_x3)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
if (g23 && g23_x3) {
|
||||
asm volatile (
|
||||
"mov v14.16b, v2.16b\n"
|
||||
"mov v15.16b, v3.16b\n"
|
||||
|
|
@ -503,7 +532,7 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
|
|||
:
|
||||
: "m" (v[0][i]), "m" (v[0][i + 16])
|
||||
);
|
||||
if (generator == 3) {
|
||||
if (g23 && g23_x3) {
|
||||
asm volatile (
|
||||
"eor v2.16b, v2.16b, v14.16b\n"
|
||||
"eor v3.16b, v3.16b, v15.16b\n"
|
||||
|
|
@ -582,12 +611,12 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
|
|||
*/
|
||||
void raid_gen3_neon_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_neon(nd, size, vv, 3, 2);
|
||||
raid_genX_neon(nd, size, vv, 3, 0);
|
||||
}
|
||||
|
||||
void raid_gen3_neon_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_neon(nd, size, vv, 3, 3);
|
||||
raid_genX_neon(nd, size, vv, 3, 1);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -595,12 +624,12 @@ void raid_gen3_neon_aes(int nd, size_t size, void **vv)
|
|||
*/
|
||||
void raid_gen4_neon_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_neon(nd, size, vv, 4, 2);
|
||||
raid_genX_neon(nd, size, vv, 4, 0);
|
||||
}
|
||||
|
||||
void raid_gen4_neon_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_neon(nd, size, vv, 4, 3);
|
||||
raid_genX_neon(nd, size, vv, 4, 1);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -608,12 +637,12 @@ void raid_gen4_neon_aes(int nd, size_t size, void **vv)
|
|||
*/
|
||||
void raid_gen5_neon_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_neon(nd, size, vv, 5, 2);
|
||||
raid_genX_neon(nd, size, vv, 5, 0);
|
||||
}
|
||||
|
||||
void raid_gen5_neon_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_neon(nd, size, vv, 5, 3);
|
||||
raid_genX_neon(nd, size, vv, 5, 1);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -621,12 +650,12 @@ void raid_gen5_neon_aes(int nd, size_t size, void **vv)
|
|||
*/
|
||||
void raid_gen6_neon_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_neon(nd, size, vv, 6, 2);
|
||||
raid_genX_neon(nd, size, vv, 6, 0);
|
||||
}
|
||||
|
||||
void raid_gen6_neon_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_neon(nd, size, vv, 6, 3);
|
||||
raid_genX_neon(nd, size, vv, 6, 1);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -63,10 +63,10 @@ void raid_gen1_neon32(int nd, size_t size, void **vv)
|
|||
}
|
||||
|
||||
/*
|
||||
* GEN2 Cauchy AArch32 NEON implementation using the active generator.
|
||||
* GEN2 Cauchy AArch32 NEON implementation using the active Q transition.
|
||||
*/
|
||||
static __always_inline void raid_gen2_neon32_gen(int nd, size_t size,
|
||||
void **vv, int generator)
|
||||
void **vv, int g23)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
|
|
@ -86,8 +86,12 @@ static __always_inline void raid_gen2_neon32_gen(int nd, size_t size,
|
|||
:
|
||||
: "Q" (gfconst16.poly[0])
|
||||
);
|
||||
int g23_start = raid_g23_count(l - 1);
|
||||
|
||||
for (i = 0; i < size; i += 32) {
|
||||
int g23_count = g23_start;
|
||||
int g23_x3;
|
||||
|
||||
asm volatile (
|
||||
"vld1.8 {q0}, %0\n"
|
||||
"vld1.8 {q1}, %1\n"
|
||||
|
|
@ -98,7 +102,14 @@ static __always_inline void raid_gen2_neon32_gen(int nd, size_t size,
|
|||
);
|
||||
|
||||
for (d = l - 1; d >= 0; --d) {
|
||||
if (generator == 3) {
|
||||
g23_x3 = 0;
|
||||
if (g23) {
|
||||
g23_x3 = --g23_count == 0;
|
||||
if (g23_x3)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
if (g23 && g23_x3) {
|
||||
asm volatile (
|
||||
"vmov q10, q2\n"
|
||||
"vmov q11, q3\n"
|
||||
|
|
@ -122,7 +133,7 @@ static __always_inline void raid_gen2_neon32_gen(int nd, size_t size,
|
|||
:
|
||||
: "Q" (v[d][i]), "Q" (v[d][i + 16])
|
||||
);
|
||||
if (generator == 3) {
|
||||
if (g23 && g23_x3) {
|
||||
asm volatile (
|
||||
"veor q2, q2, q10\n"
|
||||
"veor q3, q3, q11\n"
|
||||
|
|
@ -145,12 +156,12 @@ static __always_inline void raid_gen2_neon32_gen(int nd, size_t size,
|
|||
|
||||
void raid_gen2_neon32_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen2_neon32_gen(nd, size, vv, 2);
|
||||
raid_gen2_neon32_gen(nd, size, vv, 0);
|
||||
}
|
||||
|
||||
void raid_gen2_neon32_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen2_neon32_gen(nd, size, vv, 3);
|
||||
raid_gen2_neon32_gen(nd, size, vv, 1);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -244,7 +255,7 @@ void raid_genz_neon32_raid(int nd, size_t size, void **vv)
|
|||
* GENX AArch32 NEON implementation.
|
||||
*/
|
||||
static __always_inline void raid_genX_neon32(int nd, size_t size,
|
||||
void **vv, int np, int generator)
|
||||
void **vv, int np, int g23)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
size_t i;
|
||||
|
|
@ -258,6 +269,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
|
|||
memcpy(v[1 + d], v[0], size);
|
||||
return;
|
||||
}
|
||||
int g23_start = raid_g23_count(l - 1);
|
||||
|
||||
raid_neon32_begin();
|
||||
|
||||
|
|
@ -269,6 +281,9 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
|
|||
);
|
||||
|
||||
for (i = 0; i < size; i += 16) {
|
||||
int g23_count = g23_start;
|
||||
int g23_x3;
|
||||
|
||||
/* last disk without the generator multiplication */
|
||||
asm volatile (
|
||||
"vld1.8 {q0}, %0\n"
|
||||
|
|
@ -353,7 +368,14 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
|
|||
|
||||
/* intermediate disks */
|
||||
for (d = l - 1; d > 0; --d) {
|
||||
if (generator == 3) {
|
||||
g23_x3 = 0;
|
||||
if (g23) {
|
||||
g23_x3 = --g23_count == 0;
|
||||
if (g23_x3)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
if (g23 && g23_x3) {
|
||||
asm volatile (
|
||||
"vmov q13, q1\n"
|
||||
);
|
||||
|
|
@ -369,7 +391,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
|
|||
:
|
||||
: "Q" (v[d][i])
|
||||
);
|
||||
if (generator == 3) {
|
||||
if (g23 && g23_x3) {
|
||||
asm volatile (
|
||||
"veor q1, q1, q13\n"
|
||||
);
|
||||
|
|
@ -448,7 +470,14 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
|
|||
}
|
||||
|
||||
/* first disk with all coefficients at 1 */
|
||||
if (generator == 3) {
|
||||
g23_x3 = 0;
|
||||
if (g23) {
|
||||
g23_x3 = --g23_count == 0;
|
||||
if (g23_x3)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
if (g23 && g23_x3) {
|
||||
asm volatile (
|
||||
"vmov q13, q1\n"
|
||||
);
|
||||
|
|
@ -464,7 +493,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
|
|||
:
|
||||
: "Q" (v[0][i])
|
||||
);
|
||||
if (generator == 3) {
|
||||
if (g23 && g23_x3) {
|
||||
asm volatile (
|
||||
"veor q1, q1, q13\n"
|
||||
);
|
||||
|
|
@ -533,12 +562,12 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
|
|||
*/
|
||||
void raid_gen3_neon32_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_neon32(nd, size, vv, 3, 2);
|
||||
raid_genX_neon32(nd, size, vv, 3, 0);
|
||||
}
|
||||
|
||||
void raid_gen3_neon32_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_neon32(nd, size, vv, 3, 3);
|
||||
raid_genX_neon32(nd, size, vv, 3, 1);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -546,12 +575,12 @@ void raid_gen3_neon32_aes(int nd, size_t size, void **vv)
|
|||
*/
|
||||
void raid_gen4_neon32_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_neon32(nd, size, vv, 4, 2);
|
||||
raid_genX_neon32(nd, size, vv, 4, 0);
|
||||
}
|
||||
|
||||
void raid_gen4_neon32_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_neon32(nd, size, vv, 4, 3);
|
||||
raid_genX_neon32(nd, size, vv, 4, 1);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -559,12 +588,12 @@ void raid_gen4_neon32_aes(int nd, size_t size, void **vv)
|
|||
*/
|
||||
void raid_gen5_neon32_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_neon32(nd, size, vv, 5, 2);
|
||||
raid_genX_neon32(nd, size, vv, 5, 0);
|
||||
}
|
||||
|
||||
void raid_gen5_neon32_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_neon32(nd, size, vv, 5, 3);
|
||||
raid_genX_neon32(nd, size, vv, 5, 1);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -572,12 +601,12 @@ void raid_gen5_neon32_aes(int nd, size_t size, void **vv)
|
|||
*/
|
||||
void raid_gen6_neon32_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_neon32(nd, size, vv, 6, 2);
|
||||
raid_genX_neon32(nd, size, vv, 6, 0);
|
||||
}
|
||||
|
||||
void raid_gen6_neon32_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_neon32(nd, size, vv, 6, 3);
|
||||
raid_genX_neon32(nd, size, vv, 6, 1);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
|
|||
64
raid/raid.c
64
raid/raid.c
|
|
@ -38,20 +38,34 @@
|
|||
* nonsingular, so the linear system is always solvable for any combination
|
||||
* of missing disks. This guarantees an MDS code.
|
||||
*
|
||||
* The Cauchy construction below uses a primitive generator g.
|
||||
* RAID_MODE_CAUCHY_RAID uses polynomial 0x11d and the existing primitive
|
||||
* generator g=2 construction described below.
|
||||
*
|
||||
* RAID_MODE_CAUCHY_RAID uses:
|
||||
* polynomial 0x11d
|
||||
* generator g = 2
|
||||
* RAID_MODE_CAUCHY_AES instead uses polynomial 0x11b and a G23 Extended
|
||||
* Cauchy construction. In the AES field, element 2 has multiplicative order
|
||||
* 51 while element 3 is primitive. Therefore H=<2> has five cosets in
|
||||
* GF(256)*. The Q coefficients q[0..254] enumerate those cosets, starting
|
||||
* with q[0]=1. Within a coset q advances by multiplication by 2. The four
|
||||
* transitions between cosets advance by multiplication by 3, after the
|
||||
* coefficients indexed 50, 101, 152, and 203.
|
||||
*
|
||||
* RAID_MODE_CAUCHY_AES uses:
|
||||
* polynomial 0x11b
|
||||
* generator g = 3
|
||||
* The AES data X set consumes inverses of the G23 sequence from the beginning,
|
||||
* q[0], q[1], ..., while the Cauchy Y set starts with zero and consumes
|
||||
* inverses backward from its end: Y[0]=0 and Y[j]=inv(q[255-j]) for j>0.
|
||||
* Thus R, S, T, and U use inv(q[254]), inv(q[253]), inv(q[252]), and
|
||||
* inv(q[251]), respectively. For PARITY=n and DISK=257-n, increasing parity
|
||||
* removes one X element from the tail and appends it to Y, leaving the existing
|
||||
* rows unchanged on the remaining columns. X and Y are disjoint and together
|
||||
* contain all 256 field elements, so C[j,i]=1/(X[i]+Y[j]) defines valid Cauchy
|
||||
* rows. Adding P as an all-ones row forms an Extended Cauchy matrix. Q is the
|
||||
* row for Y[0]=0 and therefore Q=sum(q[i]*Di). Rows R through U are scaled so
|
||||
* their first coefficient is 1. This nonzero row scaling preserves the MDS
|
||||
* property, and the resulting 6x251 matrix is MDS.
|
||||
*
|
||||
* The numerical examples below refer to RAID_MODE_CAUCHY_RAID,
|
||||
* using polynomial 0x11d and generator g = 2.
|
||||
*
|
||||
* How the matrix is obtained:
|
||||
* How the RAID/g=2 matrix is obtained:
|
||||
*
|
||||
* The matrix is constructed to match Linux RAID coefficients for the
|
||||
* first two rows, while ensuring that all square submatrices are nonsingular.
|
||||
|
|
@ -121,7 +135,7 @@
|
|||
* data disks, one for each column. All the 377,342,351,231 square submatrices
|
||||
* are nonsingular, verified also by brute-force testing.
|
||||
*
|
||||
* This matrix can be extended to support any number of parities by simply
|
||||
* The RAID/g=2 matrix can be extended to support any number of parities by simply
|
||||
* adding additional rows and removing one column for each new row to maintain
|
||||
* the condition i + j < 255.
|
||||
* (See mktables.c for more details on how the matrix is generated.)
|
||||
|
|
@ -132,27 +146,27 @@
|
|||
* or AVX2 instructions [1][5] make this approach competitive with the triple
|
||||
* parity computation using power coefficients.
|
||||
*
|
||||
* Another advantage of the Cauchy matrix is that the first two rows can
|
||||
* Another advantage of the RAID/g=2 Cauchy matrix is that the first two rows can
|
||||
* replicate the RAID5 and RAID6 approach, resulting in a compatible extension.
|
||||
* SSSE3 or AVX2 instructions are only needed for triple parity or beyond.
|
||||
*
|
||||
* Parity computation is as follows:
|
||||
*
|
||||
* P = sum(Di)
|
||||
* Q = sum(g^i * Di)
|
||||
* Q = sum(A[1,i] * Di)
|
||||
* R = sum(A[2,i] * Di)
|
||||
* S = sum(A[3,i] * Di)
|
||||
* T = sum(A[4,i] * Di)
|
||||
* U = sum(A[5,i] * Di) for 0 <= i < N
|
||||
*
|
||||
* where g=2 for RAID_MODE_CAUCHY_RAID and
|
||||
* g=3 for RAID_MODE_CAUCHY_AES.
|
||||
* For RAID_MODE_CAUCHY_RAID, A[1,i]=2^i. For RAID_MODE_CAUCHY_AES,
|
||||
* A[1,i]=q[i] from the G23 sequence.
|
||||
*
|
||||
* Recovery from six disk failures at indices x, y, z, h, v, w (0 <= x < y < z < h < v < w < N)
|
||||
* involves computing parity of the remaining N-6 disks:
|
||||
*
|
||||
* Pa = sum(Di)
|
||||
* Qa = sum(g^i * Di)
|
||||
* Qa = sum(A[1,i] * Di)
|
||||
* Ra = sum(A[2,i] * Di)
|
||||
* Sa = sum(A[3,i] * Di)
|
||||
* Ta = sum(A[4,i] * Di)
|
||||
|
|
@ -170,15 +184,12 @@
|
|||
* yields:
|
||||
*
|
||||
* Pd = Dx + Dy + Dz + Dh + Dv + Dw
|
||||
* Qd = g^x * Dx + g^y * Dy + g^z * Dz + g^h * Dh + g^v * Dv + g^w * Dw
|
||||
* Qd = A[1,x] * Dx + A[1,y] * Dy + A[1,z] * Dz + A[1,h] * Dh + A[1,v] * Dv + A[1,w] * Dw
|
||||
* Rd = A[2,x] * Dx + A[2,y] * Dy + A[2,z] * Dz + A[2,h] * Dh + A[2,v] * Dv + A[2,w] * Dw
|
||||
* Sd = A[3,x] * Dx + A[3,y] * Dy + A[3,z] * Dz + A[3,h] * Dh + A[3,v] * Dv + A[3,w] * Dw
|
||||
* Td = A[4,x] * Dx + A[4,y] * Dy + A[4,z] * Dz + A[4,h] * Dh + A[4,v] * Dv + A[4,w] * Dw
|
||||
* Ud = A[5,x] * Dx + A[5,y] * Dy + A[5,z] * Dz + A[5,h] * Dh + A[5,v] * Dv + A[5,w] * Dw
|
||||
*
|
||||
* with g=2 for RAID_MODE_CAUCHY_RAID and
|
||||
* g=3 for RAID_MODE_CAUCHY_AES.
|
||||
*
|
||||
* This linear system is always solvable since the coefficient matrix is
|
||||
* nonsingular due to the properties of A[].
|
||||
*
|
||||
|
|
@ -422,20 +433,19 @@ void raid_rec1of1(int *id, int nd, size_t size, void **v)
|
|||
* Starting from the equations:
|
||||
*
|
||||
* Pd = Dx + Dy
|
||||
* Qd = g^id[0] * Dx + g^id[1] * Dy
|
||||
* Qd = A[1,id[0]] * Dx + A[1,id[1]] * Dy
|
||||
*
|
||||
* and solving we get:
|
||||
*
|
||||
* 1 g^(-id[0])
|
||||
* Dy = ------------------- * Pd + ------------------- * Qd
|
||||
* g^(id[1]-id[0]) + 1 g^(id[1]-id[0]) + 1
|
||||
* Qd + A[1,id[0]] * Pd
|
||||
* Dy = ------------------------
|
||||
* A[1,id[0]] + A[1,id[1]]
|
||||
*
|
||||
* Dx = Dy + Pd
|
||||
*
|
||||
* with conditions:
|
||||
*
|
||||
* g^id[0] != 0
|
||||
* g^(id[1]-id[0]) + 1 != 0
|
||||
* A[1,id[0]] + A[1,id[1]] != 0
|
||||
*
|
||||
* That are always satisfied for any 0<=id[0]<id[1]<255.
|
||||
*/
|
||||
|
|
@ -449,9 +459,9 @@ void raid_rec2of2_int8(int *id, int *ip, int nd, size_t size, void **vv)
|
|||
uint8_t *qa;
|
||||
const uint8_t *T[2];
|
||||
|
||||
/* get multiplication tables */
|
||||
T[0] = table(inv(powgen(id[1] - id[0]) ^ 1));
|
||||
T[1] = table(inv(powgen(id[0]) ^ powgen(id[1])));
|
||||
/* get multiplication tables from the actual active Q coefficients */
|
||||
T[0] = table(mul(A(1, id[0]), inv(A(1, id[0]) ^ A(1, id[1]))));
|
||||
T[1] = table(inv(A(1, id[0]) ^ A(1, id[1])));
|
||||
|
||||
/* compute delta parity */
|
||||
raid_delta_gen(2, id, ip, nd, size, vv);
|
||||
|
|
|
|||
17
raid/raid.h
17
raid/raid.h
|
|
@ -11,7 +11,7 @@
|
|||
* arithmetic is performed modulo an irreducible polynomial of degree 8.
|
||||
* The choice of operating mode affects both correctness and performance:
|
||||
* all tables, coefficients, and parity data are tied to the selected
|
||||
* polynomial and primitive generator and cannot be mixed between
|
||||
* polynomial and generator matrix and cannot be mixed between
|
||||
* incompatible operating modes.
|
||||
*
|
||||
* Two polynomials are supported:
|
||||
|
|
@ -31,7 +31,8 @@
|
|||
* 0x1b (x^8 + x^4 + x^3 + x + 1) -- AES polynomial
|
||||
*
|
||||
* The polynomial used by the AES encryption standard (0x11b including
|
||||
* the x^8 term), used with primitive generator g=3.
|
||||
* the x^8 term). Parity uses a 6x251 Extended Cauchy matrix whose Q row
|
||||
* follows the G23 sequence.
|
||||
*
|
||||
* On CPUs supporting Intel GFNI, generation and recovery can use the
|
||||
* vgf2p8mulb instruction, which performs GF(2^8) multiplication directly
|
||||
|
|
@ -76,14 +77,14 @@
|
|||
#define RAID_MODE_VANDERMONDE_RAID 1
|
||||
|
||||
/**
|
||||
* RAID mode supporting up to 6 parities using AES polynomial 0x1b (0x11b) and primitive generator g=3.
|
||||
* RAID mode supporting up to 6 parities and 251 data disks using AES polynomial 0x1b (0x11b).
|
||||
*
|
||||
* It has slightly lower performance than RAID_MODE_CAUCHY_RAID on CPUs without GFNI due
|
||||
* to the additional cost of generator g=3. On CPUs supporting GFNI, it uses native
|
||||
* VGF2P8MULB multiplication and is typically faster than the affine GFNI implementation
|
||||
* required by RAID_MODE_CAUCHY_RAID.
|
||||
* It uses an Extended Cauchy matrix with a G23 Q sequence, allowing optimized
|
||||
* generators to use multiplication by 2 for almost every disk transition.
|
||||
* On CPUs supporting GFNI, it uses native VGF2P8MULB multiplication.
|
||||
*
|
||||
* Parity data is incompatible with RAID_MODE_CAUCHY_RAID.
|
||||
* This parity layout is incompatible with RAID_MODE_CAUCHY_RAID and with the
|
||||
* previous RAID_MODE_CAUCHY_AES layout.
|
||||
*/
|
||||
#define RAID_MODE_CAUCHY_AES 2
|
||||
|
||||
|
|
|
|||
156
raid/sse2.c
156
raid/sse2.c
|
|
@ -52,11 +52,11 @@
|
|||
* ================================================================
|
||||
*
|
||||
* This file implements different strategies for computing the Q parity.
|
||||
* Q parity requires scaling each data disk by a descending power of the
|
||||
* primitive generator g (g=2 for RAID_MODE_CAUCHY_RAID, g=3 for
|
||||
* RAID_MODE_CAUCHY_AES) in the Galois Field:
|
||||
* Q parity uses coefficients connected by cheap GF transitions. RAID mode
|
||||
* always uses multiplication by 2. AES mode also uses multiplication by 2,
|
||||
* except for the four G23 coset transitions that use multiplication by 3.
|
||||
*
|
||||
* Q = (D_l * g^l) + (D_{l-1} * g^{l-1}) + ... + (D_1 * g^1) + (D_0 * 1)
|
||||
* Q = (D_l * q[l]) + ... + (D_1 * q[1]) + (D_0 * q[0])
|
||||
*
|
||||
* To efficiently calculate this sequentially using SIMD instructions, we
|
||||
* use Horner's method. Depending on hardware architecture and whether we
|
||||
|
|
@ -67,7 +67,7 @@
|
|||
* CASE 1: Multiplication BEFORE XOR, handling all disks inside the loop
|
||||
* -------------------------------------------------------------------------
|
||||
* Used in GEN2 where all disks can be processed uniformly.
|
||||
* The accumulator is scaled by the active generator BEFORE adding the current disk.
|
||||
* The accumulator is scaled by the active Q transition BEFORE adding the current disk.
|
||||
* Because the loop runs all the way down to D0 (d >= 0), the final disk is XORed
|
||||
* into the accumulator and the loop terminates. This naturally leaves D0 with
|
||||
* a coefficient of 1, as it escapes the scaling step.
|
||||
|
|
@ -75,7 +75,7 @@
|
|||
* Simplified Code:
|
||||
* Q = D_l;
|
||||
* for (d = l - 1; d >= 0; --d) {
|
||||
* Q = Q * g; // Scale accumulator by active generator
|
||||
* Q = Q * step[d]; // Scale by 2, or by 3 at a G23 transition
|
||||
* Q = Q ^ D_d; // Add current disk
|
||||
* }
|
||||
*
|
||||
|
|
@ -90,7 +90,7 @@
|
|||
* Simplified Code:
|
||||
* Q = D_l;
|
||||
* for (d = l - 1; d > 0; --d) {
|
||||
* Q = Q * g; // Scale accumulator by active generator
|
||||
* Q = Q * step[d]; // Scale by 2, or by 3 at a G23 transition
|
||||
* Q = Q ^ D_d; // Add current disk
|
||||
* }
|
||||
*
|
||||
|
|
@ -137,9 +137,9 @@ void raid_gen1_sse2(int nd, size_t size, void **vv)
|
|||
}
|
||||
|
||||
/*
|
||||
* GEN2 Cauchy SSE2 implementation using the active generator
|
||||
* GEN2 Cauchy SSE2 implementation using the active Q transition
|
||||
*/
|
||||
static __always_inline void raid_gen2_sse2_gen(int nd, size_t size, void **vv, int generator)
|
||||
static __always_inline void raid_gen2_sse2_gen(int nd, size_t size, void **vv, int g23)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
|
|
@ -154,16 +154,27 @@ static __always_inline void raid_gen2_sse2_gen(int nd, size_t size, void **vv, i
|
|||
raid_sse_begin();
|
||||
|
||||
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
|
||||
int g23_start = raid_g23_count(l - 1);
|
||||
|
||||
for (i = 0; i < size; i += 32) {
|
||||
int g23_count = g23_start;
|
||||
int g23_x3;
|
||||
|
||||
asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i]));
|
||||
asm volatile ("movdqa %0,%%xmm1" : : "m" (v[l][i + 16]));
|
||||
asm volatile ("movdqa %xmm0,%xmm2");
|
||||
asm volatile ("movdqa %xmm1,%xmm3");
|
||||
|
||||
for (d = l - 1; d >= 0; --d) {
|
||||
/* scale Q by the active generator before adding the current disk */
|
||||
if (generator == 3) {
|
||||
g23_x3 = 0;
|
||||
if (g23) {
|
||||
g23_x3 = --g23_count == 0;
|
||||
if (g23_x3)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
/* scale Q before adding the current disk */
|
||||
if (g23 && g23_x3) {
|
||||
asm volatile ("movdqa %xmm2,%xmm4");
|
||||
asm volatile ("movdqa %xmm3,%xmm5");
|
||||
asm volatile ("pxor %xmm6,%xmm6");
|
||||
|
|
@ -210,21 +221,21 @@ static __always_inline void raid_gen2_sse2_gen(int nd, size_t size, void **vv, i
|
|||
|
||||
void raid_gen2_sse2_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen2_sse2_gen(nd, size, vv, 2);
|
||||
raid_gen2_sse2_gen(nd, size, vv, 0);
|
||||
}
|
||||
|
||||
void raid_gen2_sse2_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen2_sse2_gen(nd, size, vv, 3);
|
||||
raid_gen2_sse2_gen(nd, size, vv, 1);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
/*
|
||||
* GEN2 Cauchy SSE2 implementation using the active generator
|
||||
* GEN2 Cauchy SSE2 implementation using the active Q transition
|
||||
*
|
||||
* Note that it uses 16 registers, meaning that x64 is required.
|
||||
*/
|
||||
static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv, int generator)
|
||||
static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv, int g23)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
|
|
@ -239,8 +250,13 @@ static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv
|
|||
raid_sse_begin();
|
||||
|
||||
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.poly[0]));
|
||||
int g23_start = raid_g23_count(l - 1);
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
int g23_count = g23_start;
|
||||
int g23_x3_d;
|
||||
int g23_x3_d1;
|
||||
|
||||
asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i]));
|
||||
asm volatile ("movdqa %0,%%xmm1" : : "m" (v[l][i + 16]));
|
||||
asm volatile ("movdqa %0,%%xmm2" : : "m" (v[l][i + 32]));
|
||||
|
|
@ -252,122 +268,146 @@ static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv
|
|||
|
||||
/* process two disks per iteration */
|
||||
for (d = l - 1; d >= 1; d -= 2) {
|
||||
g23_x3_d = 0;
|
||||
g23_x3_d1 = 0;
|
||||
if (g23) {
|
||||
g23_x3_d = --g23_count == 0;
|
||||
if (g23_x3_d)
|
||||
g23_count = 51;
|
||||
g23_x3_d1 = --g23_count == 0;
|
||||
if (g23_x3_d1)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
/* lane 0 */
|
||||
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i]));
|
||||
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i]));
|
||||
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3_d)
|
||||
asm volatile ("movdqa %xmm4,%xmm10");
|
||||
asm volatile ("pxor %xmm11,%xmm11");
|
||||
asm volatile ("pcmpgtb %xmm4,%xmm11");
|
||||
asm volatile ("paddb %xmm4,%xmm4");
|
||||
asm volatile ("pand %xmm15,%xmm11");
|
||||
asm volatile ("pxor %xmm11,%xmm4");
|
||||
if (g23 && g23_x3_d)
|
||||
asm volatile ("pxor %xmm10,%xmm4");
|
||||
asm volatile ("pxor %xmm8,%xmm4");
|
||||
|
||||
asm volatile ("pxor %xmm9,%xmm8");
|
||||
asm volatile ("pxor %xmm8,%xmm0");
|
||||
if (generator == 3)
|
||||
asm volatile ("pxor %xmm10,%xmm8");
|
||||
|
||||
if (g23 && g23_x3_d1)
|
||||
asm volatile ("movdqa %xmm4,%xmm10");
|
||||
asm volatile ("pxor %xmm11,%xmm11");
|
||||
asm volatile ("pcmpgtb %xmm4,%xmm11");
|
||||
asm volatile ("paddb %xmm4,%xmm4");
|
||||
asm volatile ("pand %xmm15,%xmm11");
|
||||
asm volatile ("pxor %xmm11,%xmm4");
|
||||
if (generator == 3)
|
||||
asm volatile ("pxor %xmm8,%xmm4");
|
||||
else
|
||||
asm volatile ("pxor %xmm9,%xmm4");
|
||||
if (g23 && g23_x3_d1)
|
||||
asm volatile ("pxor %xmm10,%xmm4");
|
||||
asm volatile ("pxor %xmm9,%xmm4");
|
||||
|
||||
asm volatile ("pxor %xmm8,%xmm0");
|
||||
asm volatile ("pxor %xmm9,%xmm0");
|
||||
|
||||
/* lane 1 */
|
||||
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i + 16]));
|
||||
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i + 16]));
|
||||
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3_d)
|
||||
asm volatile ("movdqa %xmm5,%xmm10");
|
||||
asm volatile ("pxor %xmm11,%xmm11");
|
||||
asm volatile ("pcmpgtb %xmm5,%xmm11");
|
||||
asm volatile ("paddb %xmm5,%xmm5");
|
||||
asm volatile ("pand %xmm15,%xmm11");
|
||||
asm volatile ("pxor %xmm11,%xmm5");
|
||||
if (g23 && g23_x3_d)
|
||||
asm volatile ("pxor %xmm10,%xmm5");
|
||||
asm volatile ("pxor %xmm8,%xmm5");
|
||||
|
||||
asm volatile ("pxor %xmm9,%xmm8");
|
||||
asm volatile ("pxor %xmm8,%xmm1");
|
||||
if (generator == 3)
|
||||
asm volatile ("pxor %xmm10,%xmm8");
|
||||
|
||||
if (g23 && g23_x3_d1)
|
||||
asm volatile ("movdqa %xmm5,%xmm10");
|
||||
asm volatile ("pxor %xmm11,%xmm11");
|
||||
asm volatile ("pcmpgtb %xmm5,%xmm11");
|
||||
asm volatile ("paddb %xmm5,%xmm5");
|
||||
asm volatile ("pand %xmm15,%xmm11");
|
||||
asm volatile ("pxor %xmm11,%xmm5");
|
||||
if (generator == 3)
|
||||
asm volatile ("pxor %xmm8,%xmm5");
|
||||
else
|
||||
asm volatile ("pxor %xmm9,%xmm5");
|
||||
if (g23 && g23_x3_d1)
|
||||
asm volatile ("pxor %xmm10,%xmm5");
|
||||
asm volatile ("pxor %xmm9,%xmm5");
|
||||
|
||||
asm volatile ("pxor %xmm8,%xmm1");
|
||||
asm volatile ("pxor %xmm9,%xmm1");
|
||||
|
||||
/* lane 2 */
|
||||
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i + 32]));
|
||||
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i + 32]));
|
||||
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3_d)
|
||||
asm volatile ("movdqa %xmm6,%xmm10");
|
||||
asm volatile ("pxor %xmm11,%xmm11");
|
||||
asm volatile ("pcmpgtb %xmm6,%xmm11");
|
||||
asm volatile ("paddb %xmm6,%xmm6");
|
||||
asm volatile ("pand %xmm15,%xmm11");
|
||||
asm volatile ("pxor %xmm11,%xmm6");
|
||||
if (g23 && g23_x3_d)
|
||||
asm volatile ("pxor %xmm10,%xmm6");
|
||||
asm volatile ("pxor %xmm8,%xmm6");
|
||||
|
||||
asm volatile ("pxor %xmm9,%xmm8");
|
||||
asm volatile ("pxor %xmm8,%xmm2");
|
||||
if (generator == 3)
|
||||
asm volatile ("pxor %xmm10,%xmm8");
|
||||
|
||||
if (g23 && g23_x3_d1)
|
||||
asm volatile ("movdqa %xmm6,%xmm10");
|
||||
asm volatile ("pxor %xmm11,%xmm11");
|
||||
asm volatile ("pcmpgtb %xmm6,%xmm11");
|
||||
asm volatile ("paddb %xmm6,%xmm6");
|
||||
asm volatile ("pand %xmm15,%xmm11");
|
||||
asm volatile ("pxor %xmm11,%xmm6");
|
||||
if (generator == 3)
|
||||
asm volatile ("pxor %xmm8,%xmm6");
|
||||
else
|
||||
asm volatile ("pxor %xmm9,%xmm6");
|
||||
if (g23 && g23_x3_d1)
|
||||
asm volatile ("pxor %xmm10,%xmm6");
|
||||
asm volatile ("pxor %xmm9,%xmm6");
|
||||
|
||||
asm volatile ("pxor %xmm8,%xmm2");
|
||||
asm volatile ("pxor %xmm9,%xmm2");
|
||||
|
||||
/* lane 3 */
|
||||
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i + 48]));
|
||||
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d - 1][i + 48]));
|
||||
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3_d)
|
||||
asm volatile ("movdqa %xmm7,%xmm10");
|
||||
asm volatile ("pxor %xmm11,%xmm11");
|
||||
asm volatile ("pcmpgtb %xmm7,%xmm11");
|
||||
asm volatile ("paddb %xmm7,%xmm7");
|
||||
asm volatile ("pand %xmm15,%xmm11");
|
||||
asm volatile ("pxor %xmm11,%xmm7");
|
||||
if (g23 && g23_x3_d)
|
||||
asm volatile ("pxor %xmm10,%xmm7");
|
||||
asm volatile ("pxor %xmm8,%xmm7");
|
||||
|
||||
asm volatile ("pxor %xmm9,%xmm8");
|
||||
asm volatile ("pxor %xmm8,%xmm3");
|
||||
if (generator == 3)
|
||||
asm volatile ("pxor %xmm10,%xmm8");
|
||||
|
||||
if (g23 && g23_x3_d1)
|
||||
asm volatile ("movdqa %xmm7,%xmm10");
|
||||
asm volatile ("pxor %xmm11,%xmm11");
|
||||
asm volatile ("pcmpgtb %xmm7,%xmm11");
|
||||
asm volatile ("paddb %xmm7,%xmm7");
|
||||
asm volatile ("pand %xmm15,%xmm11");
|
||||
asm volatile ("pxor %xmm11,%xmm7");
|
||||
if (generator == 3)
|
||||
asm volatile ("pxor %xmm8,%xmm7");
|
||||
else
|
||||
asm volatile ("pxor %xmm9,%xmm7");
|
||||
if (g23 && g23_x3_d1)
|
||||
asm volatile ("pxor %xmm10,%xmm7");
|
||||
asm volatile ("pxor %xmm9,%xmm7");
|
||||
|
||||
asm volatile ("pxor %xmm8,%xmm3");
|
||||
asm volatile ("pxor %xmm9,%xmm3");
|
||||
}
|
||||
|
||||
/* single remaining disk */
|
||||
if (d == 0) {
|
||||
if (generator == 3) {
|
||||
int g23_x3;
|
||||
|
||||
g23_x3 = 0;
|
||||
if (g23) {
|
||||
g23_x3 = --g23_count == 0;
|
||||
if (g23_x3)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
if (g23 && g23_x3) {
|
||||
asm volatile ("movdqa %xmm4,%xmm8");
|
||||
asm volatile ("movdqa %xmm5,%xmm9");
|
||||
asm volatile ("movdqa %xmm6,%xmm10");
|
||||
|
|
@ -448,12 +488,12 @@ static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv
|
|||
|
||||
void raid_gen2_sse2ext_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen2_sse2ext_gen(nd, size, vv, 2);
|
||||
raid_gen2_sse2ext_gen(nd, size, vv, 0);
|
||||
}
|
||||
|
||||
void raid_gen2_sse2ext_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen2_sse2ext_gen(nd, size, vv, 3);
|
||||
raid_gen2_sse2ext_gen(nd, size, vv, 1);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
|
|||
184
raid/ssse3.c
184
raid/ssse3.c
|
|
@ -9,7 +9,7 @@
|
|||
/*
|
||||
* GEN3 (triple parity with Cauchy matrix) SSSE3 implementation
|
||||
*/
|
||||
static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv, int generator)
|
||||
static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv, int g23)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
|
|
@ -29,6 +29,7 @@ static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv,
|
|||
memcpy(v[1 + i], v[0], size);
|
||||
return;
|
||||
}
|
||||
int g23_start = raid_g23_count(l - 1);
|
||||
|
||||
raid_sse_begin();
|
||||
|
||||
|
|
@ -37,6 +38,9 @@ static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv,
|
|||
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
for (i = 0; i < size; i += 16) {
|
||||
int g23_count = g23_start;
|
||||
int g23_x3;
|
||||
|
||||
/* last disk without the generator multiplication */
|
||||
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));
|
||||
|
||||
|
|
@ -56,16 +60,23 @@ static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv,
|
|||
|
||||
/* intermediate disks */
|
||||
for (d = l - 1; d > 0; --d) {
|
||||
g23_x3 = 0;
|
||||
if (g23) {
|
||||
g23_x3 = --g23_count == 0;
|
||||
if (g23_x3)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
|
||||
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3)
|
||||
asm volatile ("movdqa %xmm1,%xmm6");
|
||||
asm volatile ("pxor %xmm5,%xmm5");
|
||||
asm volatile ("pcmpgtb %xmm1,%xmm5");
|
||||
asm volatile ("paddb %xmm1,%xmm1");
|
||||
asm volatile ("pand %xmm3,%xmm5");
|
||||
asm volatile ("pxor %xmm5,%xmm1");
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3)
|
||||
asm volatile ("pxor %xmm6,%xmm1");
|
||||
|
||||
asm volatile ("pxor %xmm4,%xmm0");
|
||||
|
|
@ -87,14 +98,21 @@ static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv,
|
|||
/* first disk with all coefficients at 1 */
|
||||
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));
|
||||
|
||||
if (generator == 3)
|
||||
g23_x3 = 0;
|
||||
if (g23) {
|
||||
g23_x3 = --g23_count == 0;
|
||||
if (g23_x3)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
if (g23 && g23_x3)
|
||||
asm volatile ("movdqa %xmm1,%xmm6");
|
||||
asm volatile ("pxor %xmm5,%xmm5");
|
||||
asm volatile ("pcmpgtb %xmm1,%xmm5");
|
||||
asm volatile ("paddb %xmm1,%xmm1");
|
||||
asm volatile ("pand %xmm3,%xmm5");
|
||||
asm volatile ("pxor %xmm5,%xmm1");
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3)
|
||||
asm volatile ("pxor %xmm6,%xmm1");
|
||||
|
||||
asm volatile ("pxor %xmm4,%xmm0");
|
||||
|
|
@ -111,12 +129,12 @@ static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv,
|
|||
|
||||
void raid_gen3_ssse3_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen3_ssse3_gen(nd, size, vv, 2);
|
||||
raid_gen3_ssse3_gen(nd, size, vv, 0);
|
||||
}
|
||||
|
||||
void raid_gen3_ssse3_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen3_ssse3_gen(nd, size, vv, 3);
|
||||
raid_gen3_ssse3_gen(nd, size, vv, 1);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
|
|
@ -125,7 +143,7 @@ void raid_gen3_ssse3_aes(int nd, size_t size, void **vv)
|
|||
*
|
||||
* Note that it uses 16 registers, meaning that x64 is required.
|
||||
*/
|
||||
static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **vv, int generator)
|
||||
static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **vv, int g23)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
|
|
@ -145,6 +163,7 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v
|
|||
memcpy(v[1 + i], v[0], size);
|
||||
return;
|
||||
}
|
||||
int g23_start = raid_g23_count(l - 1);
|
||||
|
||||
raid_sse_begin();
|
||||
|
||||
|
|
@ -153,6 +172,10 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v
|
|||
asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
for (i = 0; i < size; i += 32) {
|
||||
int g23_count = g23_start;
|
||||
int g23_x3_d;
|
||||
int g23_x3_d1;
|
||||
|
||||
/* last disk without the generator multiplication */
|
||||
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));
|
||||
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[l][i + 16]));
|
||||
|
|
@ -184,6 +207,17 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v
|
|||
|
||||
/* process two intermediate disks per iteration */
|
||||
for (d = l - 1; d > 1; d -= 2) {
|
||||
g23_x3_d = 0;
|
||||
g23_x3_d1 = 0;
|
||||
if (g23) {
|
||||
g23_x3_d = --g23_count == 0;
|
||||
if (g23_x3_d)
|
||||
g23_count = 51;
|
||||
g23_x3_d1 = --g23_count == 0;
|
||||
if (g23_x3_d1)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
/* disk d */
|
||||
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
|
||||
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16]));
|
||||
|
|
@ -191,7 +225,7 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v
|
|||
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
|
||||
asm volatile ("movdqa %0,%%xmm15" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
|
||||
|
||||
if (generator == 3) {
|
||||
if (g23 && g23_x3_d) {
|
||||
asm volatile ("movdqa %xmm1,%xmm6");
|
||||
asm volatile ("movdqa %xmm9,%xmm14");
|
||||
}
|
||||
|
|
@ -205,7 +239,7 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v
|
|||
asm volatile ("pand %xmm3,%xmm13");
|
||||
asm volatile ("pxor %xmm5,%xmm1");
|
||||
asm volatile ("pxor %xmm13,%xmm9");
|
||||
if (generator == 3) {
|
||||
if (g23 && g23_x3_d) {
|
||||
asm volatile ("pxor %xmm6,%xmm1");
|
||||
asm volatile ("pxor %xmm14,%xmm9");
|
||||
}
|
||||
|
|
@ -241,7 +275,7 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v
|
|||
asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d - 1][1][0][0]));
|
||||
asm volatile ("movdqa %0,%%xmm15" : : "m" (raid_gfcauchypshufb[d - 1][1][1][0]));
|
||||
|
||||
if (generator == 3) {
|
||||
if (g23 && g23_x3_d1) {
|
||||
asm volatile ("movdqa %xmm1,%xmm6");
|
||||
asm volatile ("movdqa %xmm9,%xmm14");
|
||||
}
|
||||
|
|
@ -255,7 +289,7 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v
|
|||
asm volatile ("pand %xmm3,%xmm13");
|
||||
asm volatile ("pxor %xmm5,%xmm1");
|
||||
asm volatile ("pxor %xmm13,%xmm9");
|
||||
if (generator == 3) {
|
||||
if (g23 && g23_x3_d1) {
|
||||
asm volatile ("pxor %xmm6,%xmm1");
|
||||
asm volatile ("pxor %xmm14,%xmm9");
|
||||
}
|
||||
|
|
@ -287,10 +321,17 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v
|
|||
}
|
||||
|
||||
if (d == 1) {
|
||||
g23_x3_d = 0;
|
||||
if (g23) {
|
||||
g23_x3_d = --g23_count == 0;
|
||||
if (g23_x3_d)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[1][i]));
|
||||
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[1][i + 16]));
|
||||
|
||||
if (generator == 3) {
|
||||
if (g23 && g23_x3_d) {
|
||||
asm volatile ("movdqa %xmm1,%xmm6");
|
||||
asm volatile ("movdqa %xmm9,%xmm14");
|
||||
}
|
||||
|
|
@ -304,7 +345,7 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v
|
|||
asm volatile ("pand %xmm3,%xmm13");
|
||||
asm volatile ("pxor %xmm5,%xmm1");
|
||||
asm volatile ("pxor %xmm13,%xmm9");
|
||||
if (generator == 3) {
|
||||
if (g23 && g23_x3_d) {
|
||||
asm volatile ("pxor %xmm6,%xmm1");
|
||||
asm volatile ("pxor %xmm14,%xmm9");
|
||||
}
|
||||
|
|
@ -341,7 +382,14 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v
|
|||
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));
|
||||
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[0][i + 16]));
|
||||
|
||||
if (generator == 3) {
|
||||
g23_x3_d = 0;
|
||||
if (g23) {
|
||||
g23_x3_d = --g23_count == 0;
|
||||
if (g23_x3_d)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
if (g23 && g23_x3_d) {
|
||||
asm volatile ("movdqa %xmm1,%xmm6");
|
||||
asm volatile ("movdqa %xmm9,%xmm14");
|
||||
}
|
||||
|
|
@ -355,7 +403,7 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v
|
|||
asm volatile ("pand %xmm3,%xmm13");
|
||||
asm volatile ("pxor %xmm5,%xmm1");
|
||||
asm volatile ("pxor %xmm13,%xmm9");
|
||||
if (generator == 3) {
|
||||
if (g23 && g23_x3_d) {
|
||||
asm volatile ("pxor %xmm6,%xmm1");
|
||||
asm volatile ("pxor %xmm14,%xmm9");
|
||||
}
|
||||
|
|
@ -380,19 +428,19 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v
|
|||
|
||||
void raid_gen3_ssse3ext_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen3_ssse3ext_gen(nd, size, vv, 2);
|
||||
raid_gen3_ssse3ext_gen(nd, size, vv, 0);
|
||||
}
|
||||
|
||||
void raid_gen3_ssse3ext_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen3_ssse3ext_gen(nd, size, vv, 3);
|
||||
raid_gen3_ssse3ext_gen(nd, size, vv, 1);
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
* GEN4 (quad parity with Cauchy matrix) SSSE3 implementation
|
||||
*/
|
||||
static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv, int generator)
|
||||
static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv, int g23)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
|
|
@ -414,11 +462,15 @@ static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv,
|
|||
memcpy(v[1 + i], v[0], size);
|
||||
return;
|
||||
}
|
||||
int g23_start = raid_g23_count(l - 1);
|
||||
|
||||
raid_sse_begin();
|
||||
|
||||
/* generic case with at least two data disks */
|
||||
for (i = 0; i < size; i += 16) {
|
||||
int g23_count = g23_start;
|
||||
int g23_x3;
|
||||
|
||||
/* last disk without the generator multiplication */
|
||||
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
|
||||
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));
|
||||
|
|
@ -445,17 +497,24 @@ static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv,
|
|||
|
||||
/* intermediate disks */
|
||||
for (d = l - 1; d > 0; --d) {
|
||||
g23_x3 = 0;
|
||||
if (g23) {
|
||||
g23_x3 = --g23_count == 0;
|
||||
if (g23_x3)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
|
||||
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
|
||||
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3)
|
||||
asm volatile ("movdqa %xmm1,%xmm6");
|
||||
asm volatile ("pxor %xmm5,%xmm5");
|
||||
asm volatile ("pcmpgtb %xmm1,%xmm5");
|
||||
asm volatile ("paddb %xmm1,%xmm1");
|
||||
asm volatile ("pand %xmm7,%xmm5");
|
||||
asm volatile ("pxor %xmm5,%xmm1");
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3)
|
||||
asm volatile ("pxor %xmm6,%xmm1");
|
||||
|
||||
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
|
||||
|
|
@ -487,14 +546,21 @@ static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv,
|
|||
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
|
||||
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));
|
||||
|
||||
if (generator == 3)
|
||||
g23_x3 = 0;
|
||||
if (g23) {
|
||||
g23_x3 = --g23_count == 0;
|
||||
if (g23_x3)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
if (g23 && g23_x3)
|
||||
asm volatile ("movdqa %xmm1,%xmm6");
|
||||
asm volatile ("pxor %xmm5,%xmm5");
|
||||
asm volatile ("pcmpgtb %xmm1,%xmm5");
|
||||
asm volatile ("paddb %xmm1,%xmm1");
|
||||
asm volatile ("pand %xmm7,%xmm5");
|
||||
asm volatile ("pxor %xmm5,%xmm1");
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3)
|
||||
asm volatile ("pxor %xmm6,%xmm1");
|
||||
|
||||
asm volatile ("pxor %xmm4,%xmm0");
|
||||
|
|
@ -513,12 +579,12 @@ static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv,
|
|||
|
||||
void raid_gen4_ssse3_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen4_ssse3_gen(nd, size, vv, 2);
|
||||
raid_gen4_ssse3_gen(nd, size, vv, 0);
|
||||
}
|
||||
|
||||
void raid_gen4_ssse3_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen4_ssse3_gen(nd, size, vv, 3);
|
||||
raid_gen4_ssse3_gen(nd, size, vv, 1);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
|
|
@ -527,7 +593,7 @@ void raid_gen4_ssse3_aes(int nd, size_t size, void **vv)
|
|||
*
|
||||
* Note that it uses 16 registers, meaning that x64 is required.
|
||||
*/
|
||||
static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **vv, int generator)
|
||||
static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **vv, int g23)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
|
|
@ -549,11 +615,15 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v
|
|||
memcpy(v[1 + i], v[0], size);
|
||||
return;
|
||||
}
|
||||
int g23_start = raid_g23_count(l - 1);
|
||||
|
||||
raid_sse_begin();
|
||||
|
||||
/* generic case with at least two data disks */
|
||||
for (i = 0; i < size; i += 32) {
|
||||
int g23_count = g23_start;
|
||||
int g23_x3;
|
||||
|
||||
/* last disk without the generator multiplication */
|
||||
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0]));
|
||||
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));
|
||||
|
|
@ -597,12 +667,19 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v
|
|||
|
||||
/* intermediate disks */
|
||||
for (d = l - 1; d > 0; --d) {
|
||||
g23_x3 = 0;
|
||||
if (g23) {
|
||||
g23_x3 = --g23_count == 0;
|
||||
if (g23_x3)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
|
||||
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0]));
|
||||
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
|
||||
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16]));
|
||||
|
||||
if (generator == 3) {
|
||||
if (g23 && g23_x3) {
|
||||
asm volatile ("movdqa %xmm1,%xmm6");
|
||||
asm volatile ("movdqa %xmm9,%xmm14");
|
||||
}
|
||||
|
|
@ -616,7 +693,7 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v
|
|||
asm volatile ("pand %xmm7,%xmm13");
|
||||
asm volatile ("pxor %xmm5,%xmm1");
|
||||
asm volatile ("pxor %xmm13,%xmm9");
|
||||
if (generator == 3) {
|
||||
if (g23 && g23_x3) {
|
||||
asm volatile ("pxor %xmm6,%xmm1");
|
||||
asm volatile ("pxor %xmm14,%xmm9");
|
||||
}
|
||||
|
|
@ -668,7 +745,14 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v
|
|||
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));
|
||||
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[0][i + 16]));
|
||||
|
||||
if (generator == 3) {
|
||||
g23_x3 = 0;
|
||||
if (g23) {
|
||||
g23_x3 = --g23_count == 0;
|
||||
if (g23_x3)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
if (g23 && g23_x3) {
|
||||
asm volatile ("movdqa %xmm1,%xmm6");
|
||||
asm volatile ("movdqa %xmm9,%xmm14");
|
||||
}
|
||||
|
|
@ -682,7 +766,7 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v
|
|||
asm volatile ("pand %xmm7,%xmm13");
|
||||
asm volatile ("pxor %xmm5,%xmm1");
|
||||
asm volatile ("pxor %xmm13,%xmm9");
|
||||
if (generator == 3) {
|
||||
if (g23 && g23_x3) {
|
||||
asm volatile ("pxor %xmm6,%xmm1");
|
||||
asm volatile ("pxor %xmm14,%xmm9");
|
||||
}
|
||||
|
|
@ -711,12 +795,12 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v
|
|||
|
||||
void raid_gen4_ssse3ext_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen4_ssse3ext_gen(nd, size, vv, 2);
|
||||
raid_gen4_ssse3ext_gen(nd, size, vv, 0);
|
||||
}
|
||||
|
||||
void raid_gen4_ssse3ext_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen4_ssse3ext_gen(nd, size, vv, 3);
|
||||
raid_gen4_ssse3ext_gen(nd, size, vv, 1);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -859,7 +943,7 @@ void raid_gen5_ssse3_raid(int nd, size_t size, void **vv)
|
|||
/*
|
||||
* GENX SSSE3ext implementation
|
||||
*/
|
||||
static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, int np, int generator)
|
||||
static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, int np, int g23)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
|
|
@ -886,6 +970,7 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i
|
|||
memcpy(v[1 + d], v[0], size);
|
||||
return;
|
||||
}
|
||||
int g23_start = raid_g23_count(l - 1);
|
||||
|
||||
raid_sse_begin();
|
||||
|
||||
|
|
@ -894,6 +979,9 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i
|
|||
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
for (i = 0; i < size; i += 16) {
|
||||
int g23_count = g23_start;
|
||||
int g23_x3;
|
||||
|
||||
/* last disk without the generator multiplication */
|
||||
asm volatile ("movdqa %0,%%xmm10" : : "m" (v[l][i]));
|
||||
|
||||
|
|
@ -939,16 +1027,23 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i
|
|||
|
||||
/* intermediate disks */
|
||||
for (d = l - 1; d > 0; --d) {
|
||||
g23_x3 = 0;
|
||||
if (g23) {
|
||||
g23_x3 = --g23_count == 0;
|
||||
if (g23_x3)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
asm volatile ("movdqa %0,%%xmm10" : : "m" (v[d][i]));
|
||||
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3)
|
||||
asm volatile ("movdqa %xmm1,%xmm12");
|
||||
asm volatile ("pxor %xmm11,%xmm11");
|
||||
asm volatile ("pcmpgtb %xmm1,%xmm11");
|
||||
asm volatile ("paddb %xmm1,%xmm1");
|
||||
asm volatile ("pand %xmm14,%xmm11");
|
||||
asm volatile ("pxor %xmm11,%xmm1");
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3)
|
||||
asm volatile ("pxor %xmm12,%xmm1");
|
||||
|
||||
asm volatile ("pxor %xmm10,%xmm0");
|
||||
|
|
@ -999,14 +1094,21 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i
|
|||
/* first disk with all coefficients at 1 */
|
||||
asm volatile ("movdqa %0,%%xmm10" : : "m" (v[0][i]));
|
||||
|
||||
if (generator == 3)
|
||||
g23_x3 = 0;
|
||||
if (g23) {
|
||||
g23_x3 = --g23_count == 0;
|
||||
if (g23_x3)
|
||||
g23_count = 51;
|
||||
}
|
||||
|
||||
if (g23 && g23_x3)
|
||||
asm volatile ("movdqa %xmm1,%xmm12");
|
||||
asm volatile ("pxor %xmm11,%xmm11");
|
||||
asm volatile ("pcmpgtb %xmm1,%xmm11");
|
||||
asm volatile ("paddb %xmm1,%xmm1");
|
||||
asm volatile ("pand %xmm14,%xmm11");
|
||||
asm volatile ("pxor %xmm11,%xmm1");
|
||||
if (generator == 3)
|
||||
if (g23 && g23_x3)
|
||||
asm volatile ("pxor %xmm12,%xmm1");
|
||||
|
||||
asm volatile ("pxor %xmm10,%xmm0");
|
||||
|
|
@ -1042,12 +1144,12 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i
|
|||
*/
|
||||
void raid_gen5_ssse3ext_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_ssse3ext(nd, size, vv, 5, 2);
|
||||
raid_genX_ssse3ext(nd, size, vv, 5, 0);
|
||||
}
|
||||
|
||||
void raid_gen5_ssse3ext_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_ssse3ext(nd, size, vv, 5, 3);
|
||||
raid_genX_ssse3ext(nd, size, vv, 5, 1);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -1215,12 +1317,12 @@ void raid_gen6_ssse3_raid(int nd, size_t size, void **vv)
|
|||
*/
|
||||
void raid_gen6_ssse3ext_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_ssse3ext(nd, size, vv, 6, 2);
|
||||
raid_genX_ssse3ext(nd, size, vv, 6, 0);
|
||||
}
|
||||
|
||||
void raid_gen6_ssse3ext_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_ssse3ext(nd, size, vv, 6, 3);
|
||||
raid_genX_ssse3ext(nd, size, vv, 6, 1);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
|
|||
10354
raid/tables.c
10354
raid/tables.c
File diff suppressed because it is too large
Load diff
213
raid/test.c
213
raid/test.c
|
|
@ -189,19 +189,22 @@ int raid_test_sort(void)
|
|||
*
|
||||
* RAID_MODE_CAUCHY_AES:
|
||||
* polynomial 0x11b
|
||||
* primitive generator g=3
|
||||
* G23 Extended Cauchy construction
|
||||
*
|
||||
* The parity generation tests compare optimized implementations against
|
||||
* raid_gen_ref(), but both use the same active generator matrix. Therefore
|
||||
* they can agree even if the matrix itself is mathematically unsuitable.
|
||||
*
|
||||
* This test instead checks a few fundamental properties of the field
|
||||
* generator and of the generated Cauchy matrix.
|
||||
* This test instead constructs the expected Q sequence independently and
|
||||
* checks the generated matrix coefficients.
|
||||
*/
|
||||
int raid_test_poly(unsigned mode)
|
||||
{
|
||||
uint8_t q[255];
|
||||
uint8_t v;
|
||||
int i, j;
|
||||
int p, d;
|
||||
int g23_count;
|
||||
|
||||
/*
|
||||
* This test is defined only for the two Cauchy modes.
|
||||
|
|
@ -216,7 +219,6 @@ int raid_test_poly(unsigned mode)
|
|||
/*
|
||||
* Select the mode first so that:
|
||||
*
|
||||
* raid_gfexp
|
||||
* raid_gfcauchy
|
||||
* raid_gfmul
|
||||
* raid_gfinv
|
||||
|
|
@ -229,89 +231,97 @@ int raid_test_poly(unsigned mode)
|
|||
raid_mode(mode);
|
||||
|
||||
/*
|
||||
* Primitive order alone is not enough to protect parity format
|
||||
* compatibility. Each Cauchy mode has a fixed reducing polynomial
|
||||
* and primitive generator, which determine the actual matrix coefficients.
|
||||
* Each Cauchy mode has a fixed reducing polynomial and matrix
|
||||
* construction, which together determine the parity format.
|
||||
*/
|
||||
if (mode == RAID_MODE_CAUCHY_RAID) {
|
||||
if (raid_poly_byte != RAID_POLY_RAID)
|
||||
return -1;
|
||||
if (raid_gfexp[1] != 2)
|
||||
return -1;
|
||||
} else if (mode == RAID_MODE_CAUCHY_AES) {
|
||||
} else {
|
||||
if (raid_poly_byte != RAID_POLY_AES)
|
||||
return -1;
|
||||
if (raid_gfexp[1] != 3)
|
||||
return -1;
|
||||
}
|
||||
|
||||
/*
|
||||
* raid_gfexp[a] contains g^a for the primitive generator associated
|
||||
* with the active Cauchy mode.
|
||||
*
|
||||
* The multiplicative group GF(2^8)* contains 255 elements:
|
||||
*
|
||||
* 255 = 3 * 5 * 17
|
||||
*
|
||||
* The order of every nonzero field element divides 255.
|
||||
*
|
||||
* To prove that g has order exactly 255, it is therefore sufficient
|
||||
* to verify:
|
||||
*
|
||||
* g^255 == 1
|
||||
*
|
||||
* and to exclude the three maximal proper divisors:
|
||||
*
|
||||
* 255 / 3 = 85
|
||||
* 255 / 5 = 51
|
||||
* 255 / 17 = 15
|
||||
*
|
||||
* Hence a primitive generator must satisfy:
|
||||
*
|
||||
* g^85 != 1
|
||||
* g^51 != 1
|
||||
* g^15 != 1
|
||||
*/
|
||||
if (raid_gfexp[0] != 1)
|
||||
return -1;
|
||||
if (raid_gfexp[255] != 1)
|
||||
return -1;
|
||||
if (raid_gfexp[85] == 1)
|
||||
return -1;
|
||||
if (raid_gfexp[51] == 1)
|
||||
return -1;
|
||||
if (raid_gfexp[15] == 1)
|
||||
return -1;
|
||||
q[0] = 1;
|
||||
g23_count = 51;
|
||||
for (i = 0; i < 254; ++i) {
|
||||
uint8_t f = 2;
|
||||
|
||||
/*
|
||||
* The Q row of the Extended Cauchy matrix contains:
|
||||
*
|
||||
* 1, g, g^2, g^3, ...
|
||||
*
|
||||
* SnapRAID supports RAID_DATA_MAX data disks, so all generator powers
|
||||
* corresponding to those columns must be distinct.
|
||||
*
|
||||
* A primitive generator of order 255 guarantees this, but checking it
|
||||
* explicitly is inexpensive and also verifies the generated
|
||||
* raid_gfexp[] table itself.
|
||||
*/
|
||||
for (i = 0; i < RAID_DATA_MAX; ++i)
|
||||
for (j = i + 1; j < RAID_DATA_MAX; ++j)
|
||||
if (raid_gfexp[i] == raid_gfexp[j])
|
||||
return -1;
|
||||
if (mode == RAID_MODE_CAUCHY_AES && --g23_count == 0) {
|
||||
f = 3;
|
||||
g23_count = 51;
|
||||
}
|
||||
q[i + 1] = mul(f, q[i]);
|
||||
}
|
||||
|
||||
/*
|
||||
* The second row of the Extended Cauchy matrix is exactly g^d.
|
||||
*
|
||||
* Verify that the exponent table and the actual matrix used for
|
||||
* parity generation agree for every supported data column.
|
||||
*
|
||||
* This makes sure that the primitive generator checked above is the
|
||||
* same generator actually used by the Q row.
|
||||
*/
|
||||
for (d = 0; d < RAID_DATA_MAX; ++d)
|
||||
if (raid_gfcauchy[1][d] != raid_gfexp[d])
|
||||
/* Both constructions enumerate distinct nonzero Q coefficients. */
|
||||
for (i = 0; i < 255; ++i) {
|
||||
if (q[i] == 0)
|
||||
return -1;
|
||||
for (j = i + 1; j < 255; ++j)
|
||||
if (q[i] == q[j])
|
||||
return -1;
|
||||
}
|
||||
|
||||
for (d = 0; d < RAID_DATA_MAX; ++d) {
|
||||
if (raid_gfcauchy[0][d] != 1)
|
||||
return -1;
|
||||
if (raid_gfcauchy[1][d] != q[d])
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (mode == RAID_MODE_CAUCHY_AES) {
|
||||
/* In AES, 2 has order 51 and 3 has order 255. */
|
||||
v = 1;
|
||||
for (i = 1; i <= 51; ++i) {
|
||||
v = mul(2, v);
|
||||
if ((i == 3 || i == 17) && v == 1)
|
||||
return -1;
|
||||
}
|
||||
if (v != 1)
|
||||
return -1;
|
||||
|
||||
v = 1;
|
||||
for (i = 1; i <= 255; ++i) {
|
||||
v = mul(3, v);
|
||||
if ((i == 15 || i == 51 || i == 85) && v == 1)
|
||||
return -1;
|
||||
}
|
||||
if (v != 1)
|
||||
return -1;
|
||||
|
||||
if (q[251] != 0xb8 || q[252] != 0x6b
|
||||
|| q[253] != 0xd6 || q[254] != 0xb7)
|
||||
return -1;
|
||||
if (inv(q[251]) != 0xa5 || inv(q[252]) != 0xdf
|
||||
|| inv(q[253]) != 0xe2 || inv(q[254]) != 0x71)
|
||||
return -1;
|
||||
|
||||
/* the descending Y allocation keeps future matrices nested */
|
||||
for (i = 6; i <= 9; ++i) {
|
||||
int disk = 257 - i;
|
||||
|
||||
for (j = 1; j < i - 1; ++j)
|
||||
if (255 - j < disk || 255 - j > 254)
|
||||
return -1;
|
||||
if (255 - (i - 2) != disk)
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* independently reconstruct and normalize the four Cauchy rows */
|
||||
for (p = 2; p < RAID_PARITY_MAX; ++p) {
|
||||
uint8_t y = inv(q[256 - p]);
|
||||
uint8_t f = inv(inv(q[0]) ^ y);
|
||||
uint8_t scale = inv(f);
|
||||
|
||||
for (d = 0; d < RAID_DATA_MAX; ++d) {
|
||||
uint8_t c = inv(inv(q[d]) ^ y);
|
||||
|
||||
if (raid_gfcauchy[p][d] != mul(c, scale))
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Every coefficient of the active 6x251 Cauchy generator matrix must
|
||||
|
|
@ -550,6 +560,57 @@ bail:
|
|||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
|
||||
int raid_test_rec2_g23(size_t size)
|
||||
{
|
||||
static const int missing[][2] = {
|
||||
{ 1, 20 },
|
||||
{ 50, 51 },
|
||||
{ 10, 204 }
|
||||
};
|
||||
void *v_alloc;
|
||||
void **v;
|
||||
void *save[2];
|
||||
int ip[2] = { 0, 1 };
|
||||
int nv = RAID_DATA_MAX + RAID_PARITY_MAX + 3;
|
||||
int i, j;
|
||||
|
||||
raid_mode(RAID_MODE_CAUCHY_AES);
|
||||
v = raid_malloc_vector(nv, size, &v_alloc);
|
||||
if (!v)
|
||||
return -1;
|
||||
|
||||
memset(v[nv - 1], 0, size);
|
||||
raid_zero(v[nv - 1]);
|
||||
raid_mrand_vector(3, RAID_DATA_MAX, size, v);
|
||||
raid_gen_ref(RAID_DATA_MAX, RAID_PARITY_MAX, size, v);
|
||||
|
||||
for (j = 0; j < (int)(sizeof(missing) / sizeof(missing[0])); ++j) {
|
||||
int id[2] = { missing[j][0], missing[j][1] };
|
||||
|
||||
for (i = 0; i < 2; ++i) {
|
||||
save[i] = v[id[i]];
|
||||
v[id[i]] = v[RAID_DATA_MAX + RAID_PARITY_MAX + i];
|
||||
}
|
||||
|
||||
raid_rec2_int8(2, id, ip, RAID_DATA_MAX, size, v);
|
||||
|
||||
for (i = 0; i < 2; ++i) {
|
||||
if (memcmp(v[id[i]], save[i], size) != 0)
|
||||
goto bail;
|
||||
v[id[i]] = save[i];
|
||||
}
|
||||
}
|
||||
|
||||
free(v_alloc);
|
||||
free(v);
|
||||
return 0;
|
||||
|
||||
bail:
|
||||
free(v_alloc);
|
||||
free(v);
|
||||
return -1;
|
||||
}
|
||||
|
||||
int raid_test_tail(int mode, int nd_max, size_t size)
|
||||
{
|
||||
void (*f[RAID_PARITY_MAX][32])(int nr, int *id, int *ip, int nd, size_t size, void **vbuf);
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ int raid_test_gfaffine(void);
|
|||
/**
|
||||
* Tests the Galois field and Cauchy matrix properties.
|
||||
*
|
||||
* Verifies the expected polynomial and primitive generator and basic
|
||||
* Verifies the expected polynomial and matrix construction and basic
|
||||
* properties of the Extended Cauchy matrix for the specified RAID mode.
|
||||
*
|
||||
* Returns 0 on success.
|
||||
|
|
@ -60,6 +60,13 @@ int raid_test_poly(unsigned mode);
|
|||
*/
|
||||
int raid_test_rec(unsigned mode, int nd, size_t size);
|
||||
|
||||
/**
|
||||
* Tests P/Q double-disk recovery at selected G23 positions.
|
||||
*
|
||||
* Returns 0 on success.
|
||||
*/
|
||||
int raid_test_rec2_g23(size_t size);
|
||||
|
||||
/**
|
||||
* Tests recovering functions on the tail data disks.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -151,6 +151,13 @@ int main(void)
|
|||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
|
||||
printf("Test Cauchy AES P/Q recovery within and across G23 cosets...\n");
|
||||
if (raid_test_rec2_g23(TEST_SIZE) != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
goto bail;
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
|
||||
printf("OK\n");
|
||||
return 0;
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
* nonsingular (invertible) for both operating modes:
|
||||
*
|
||||
* RAID: GF polynomial 0x11d, primitive generator g=2
|
||||
* AES: GF polynomial 0x11b, primitive generator g=3
|
||||
* AES: GF polynomial 0x11b, G23 Extended Cauchy construction
|
||||
*/
|
||||
|
||||
#include "internal.h"
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
* nonsingular (invertible) for both operating modes:
|
||||
*
|
||||
* RAID: GF polynomial 0x11d, primitive generator g=2
|
||||
* AES: GF polynomial 0x11b, primitive generator g=3
|
||||
* AES: GF polynomial 0x11b, G23 Extended Cauchy construction
|
||||
*/
|
||||
|
||||
#include "internal.h"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue