rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 19:01:57 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Optimize scalar 8-bit RAID parity generation and recovery in int.c

Browse source
This commit is contained in:
Andrea Mazzoleni 2026-08-27 19:19:31 +02:00
parent 69a1ed1125
commit e0c0ea3dcc
7 changed files with 777 additions and 228 deletions
Download patch file Download diff file Expand all files Collapse all files

18
cmdline/speed.c
View file

@ -1218,7 +1218,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
printf("RAID polynomial functions used for recovering with 'fix':\n");
printf("%8s", "");
printf("%8s", "best");
printf("%8s", "int8");
printf("%8s", "int");
#ifdef CONFIG_NEON
printf("%8s", "neon");
#endif
@ -1911,9 +1911,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
fflush(stdout);
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(3, raid_gen3_int8);
raid_recX_int8(3, id, ip, nd, size, v);
raid_rec3_int8(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -2004,9 +2002,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
fflush(stdout);
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(4, raid_gen4_int8);
raid_recX_int8(4, id, ip, nd, size, v);
raid_rec4_int8(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -2097,9 +2093,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
fflush(stdout);
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(5, raid_gen5_int8);
raid_recX_int8(5, id, ip, nd, size, v);
raid_rec5_int8(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -2190,9 +2184,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
fflush(stdout);
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(6, raid_gen6_int8);
raid_recX_int8(6, id, ip, nd, size, v);
raid_rec6_int8(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);

52
raid/NOTES
View file

@ -45,6 +45,9 @@ Generation:
* RAID GFNI generation uses direct Cauchy coefficients. Their 8-byte affine
representations are precomputed in a disk-major table, avoiding a dependent
Cauchy-coefficient -> affine-matrix lookup in the hot generation loop.
Similarly, scalar int8 generation precomputes per-disk 2D multiplication table
pointers T[d][p] before the hot loop to eliminate two-level dependent table
lookups.
* AVX512 128-byte processing, non-GFNI AVX512BW Horner, standalone
VPTERNLOGQ transformations, VPBLENDVB, cached parity stores, and large
@ -69,8 +72,8 @@ Recovery:
AArch32 NEON uses fused syndrome generation for all non-P REC1 and REC2 cases
and for REC3..REC6. SSSE3 and SSSE3 extended use it for REC3..REC6.
* Scalar recovery, SSSE3 REC1/REC2, and AArch32 NEON P+Q continue to use
raid_delta_gen().
* Scalar REC1/REC2, SSSE3 REC1/REC2, and AArch32 NEON P+Q continue to use
raid_delta_gen(), while scalar REC3..REC6 uses fused syndrome recovery.
* Non-GFNI fused recovery splits every completed syndrome into low/high nibbles
once and reuses the split values across all reconstructed outputs.
@ -86,13 +89,14 @@ Recovery:
* Specialized P+Q rec2of2 recovery reduces 4 GF multiplications to 2 GF
multiplications and 1 XOR; AVX512BW, GFNI, AVX2, and AArch64 NEON handle this
directly inside the 1-pass fused kernel, while scalar int8, SSSE3, SSSE3ext,
and AArch32 NEON backends implement dedicated rec2of2 helpers.
directly inside the 1-pass fused kernel, while SSSE3, SSSE3ext,
and AArch32 NEON backends implement dedicated rec2of2 helpers, and scalar int8
handles it via has_p in raid_rec2_int8.
* When P is among the selected parity blocks, optimized SIMD RECX paths
reconstruct only nr - 1 missing blocks through the inverse matrix and derive
the final missing block from the raw P delta by XORing the blocks already
reconstructed.
* When P is among the selected parity blocks, optimized SIMD and scalar RECX
paths reconstruct only nr - 1 missing blocks through the inverse matrix and
derive the final missing block from the raw P delta by XORing the blocks
already reconstructed.
Architecture and validation:
@ -251,8 +255,8 @@ REC2OF2:
directly in the 1-pass fused kernel, which computes P with XOR, Q with GF
multiplication, and derives the second missing block via XOR.
* Scalar int8, SSSE3, SSSE3ext, and AArch32 NEON call their specialized
raid_rec2of2_*() delta helpers.
* SSSE3, SSSE3ext, and AArch32 NEON call their specialized
raid_rec2of2_*() delta helpers, while scalar int8 leverages has_p in raid_rec2_int8.
* This makes the row a direct benchmark for the specialized P+Q recovery
implementation.
@ -271,8 +275,8 @@ REC2OF3 and REC2OF4:
REC3OF3 through REC6OF6:
* These rows measure multi-failure recovery for each supported failure count
using the first N parity levels. SIMD columns use fused recovery; the scalar
column retains raid_delta_gen().
using the first N parity levels. SIMD backends and scalar int8 use fused
syndrome recovery.
Delta generation:
@ -316,7 +320,7 @@ raid_delta_gen() fusion
XOR streaming within the fused kernel (no GF matrix multiplication), while on
scalar, SSSE3, and AVX2 it uses the raid_rec1of1() pointer remapping path.
* raid_delta_gen() remains in the scalar implementation, SSSE3 REC1/REC2, and
* raid_delta_gen() remains in scalar REC1/REC2, SSSE3 REC1/REC2, and
AArch32 NEON P+Q.
@ -390,6 +394,11 @@ Generation: GFNI direct coefficients
difference is VGF2P8AFFINEQB with an 8-byte RAID matrix versus
VGF2P8MULB with an AES coefficient byte.
* Scalar int8 generation (raid_gen2_int8 .. raid_gen6_int8) similarly precomputes
per-disk multiplication table pointers T[d][p] in a 2D array before entering
the hot loop, replacing the double-indexed raid_gfmul[d0][raid_gfgen[j][d]]
lookup with a single direct table access T[d][p][d0].
Generation: coefficient residency
---------------------------------
@ -695,8 +704,8 @@ REC2 status:
* Generic REC2 uses the fused path across AVX2, AVX512BW, GFNI, and NEON.
* AVX2, AVX512BW, GFNI, and AArch64 NEON handle P+Q recovery directly inside the
1-pass fused kernel, while scalar int8, SSSE3, SSSE3ext, and AArch32 NEON
backends implement dedicated rec2of2 delta helpers.
1-pass fused kernel, while SSSE3, SSSE3ext, and AArch32 NEON backends implement
dedicated rec2of2 delta helpers, and scalar int8 handles it via has_p.
Specialized rec2of2 P+Q
@ -717,7 +726,7 @@ Backends implement this specialization as follows:
derivation directly into the unified 1-pass fused kernel (P is accumulated via
XOR, Q via GF multiplication, Dy is solved with inverse coefficients, and Dx
derived by XOR with Pdelta).
* Scalar int8 (raid_rec2of2_int8)
* Scalar int8 (has_p branch in raid_rec2_int8)
* SSSE3 and SSSE3 ext (raid_rec2of2_ssse3, raid_rec2of2_ssse3ext)
* AArch32 NEON (raid_rec2of2_neon32)
@ -782,6 +791,10 @@ Users include:
* AVX512BW RECX
* SSSE3 REC3..REC6
* AArch64 and AArch32 NEON REC1..REC6
* Scalar int8 RECX (raid_recX_int8), which precomputes 256-byte raid_gfmul
pointers S[s][j] and R[j][k], and structures inner loops with explicit
if (nr >= N) branches so the compiler fully unrolls syndrome accumulation
and inverse-matrix reconstruction when nr is a compile-time constant.
The remaining cost is that the low/high tables are still loaded or broadcast for
every coefficient of every data chunk.
@ -1032,9 +1045,10 @@ Short list of ideas already tried or rejected
existing traversal instead.
* raid_delta_gen() + REC fusion:
implemented for all SIMD REC1..REC6 on AVX512BW, GFNI, and AArch64 NEON, and for
all AVX2 REC except rec1of1; SSSE3 retains delta for REC1/REC2; only scalar
paths, SSSE3 REC1/REC2, and AArch32 NEON P+Q retain raid_delta_gen().
implemented for all SIMD REC1..REC6 on AVX512BW, GFNI, and AArch64 NEON, for
all AVX2 REC except rec1of1, and for scalar int8 REC3..REC6; SSSE3 retains delta
for REC1/REC2; only scalar REC1/REC2, SSSE3 REC1/REC2, and AArch32 NEON P+Q retain
raid_delta_gen().
* Software prefetch:
no strong reason to add it given the existing long streams and proven

846
raid/int.c
View file

File diff suppressed because it is too large Load diff

6
raid/internal.h
View file

@ -91,7 +91,6 @@ void raid_gen_ref(int nd, int np, size_t size, void **vv);
void raid_invert(uint8_t *M, uint8_t *V, int n);
void raid_delta_gen(int nr, int *id, int *ip, int nd, size_t size, void **v);
void raid_rec1of1(int *id, int nd, size_t size, void **v);
void raid_rec2of2_int8(int *id, int *ip, int nd, size_t size, void **vv);
void raid_gen1_int32(int nd, size_t size, void **vv);
void raid_gen1_int64(int nd, size_t size, void **vv);
void raid_gen1_sse2(int nd, size_t size, void **vv);
@ -168,7 +167,10 @@ void raid_gen6_avx2gfni_aes(int nd, size_t size, void **vv);
void raid_gen6_avx512gfni_aes(int nd, size_t size, void **vv);
void raid_rec1_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec2_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_recX_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec3_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec4_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec5_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec6_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec3_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);

60
raid/raid.c
View file

@ -413,63 +413,3 @@ void raid_rec1of1(int *id, int nd, size_t size, void **v)
v[id[0]] = pa;
v[nd] = p;
}
/**
* Recover failure of two data blocks for PAR2.
*
* Starting from the equations:
*
* Pd = Dx + Dy
* Qd = g^id[0] * Dx + g^id[1] * Dy
*
* and solving we get:
*
* 1 g^(-id[0])
* Dy = ------------------- * Pd + ------------------- * Qd
* g^(id[1]-id[0]) + 1 g^(id[1]-id[0]) + 1
*
* Dx = Dy + Pd
*
* with conditions:
*
* g^id[0] != 0
* g^(id[1]-id[0]) + 1 != 0
*
* That are always satisfied for any 0<=id[0]<id[1]<255.
*/
void raid_rec2of2_int8(int *id, int *ip, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
size_t i;
uint8_t *p;
uint8_t *pa;
uint8_t *q;
uint8_t *qa;
const uint8_t *T[2];
/* get multiplication tables */
T[0] = table(inv(powgen(id[1] - id[0]) ^ 1));
T[1] = table(inv(powgen(id[0]) ^ powgen(id[1])));
/* compute delta parity */
raid_delta_gen(2, id, ip, nd, size, vv);
p = v[nd];
q = v[nd + 1];
pa = v[id[0]];
qa = v[id[1]];
for (i = 0; i < size; ++i) {
/* delta */
uint8_t Pd = p[i] ^ pa[i];
uint8_t Qd = q[i] ^ qa[i];
/* reconstruct */
uint8_t Dy = T[0][Pd] ^ T[1][Qd];
uint8_t Dx = Pd ^ Dy;
/* set */
pa[i] = Dx;
qa[i] = Dy;
}
}

13
raid/test.c
View file

@ -367,8 +367,17 @@ static void raid_test_setup_rec(int mode, int np, raid_rec_fn *f[RAID_PARITY_MAX
case 2:
test_setup(i) = raid_rec2_int8;
break;
default:
test_setup(i) = raid_recX_int8;
case 3:
test_setup(i) = raid_rec3_int8;
break;
case 4:
test_setup(i) = raid_rec4_int8;
break;
case 5:
test_setup(i) = raid_rec5_int8;
break;
case 6:
test_setup(i) = raid_rec6_int8;
break;
}
#ifdef CONFIG_X86

10
raid/test/speedtest.c
View file

@ -1114,7 +1114,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
printf("RAID polynomial functions used for recovering:\n");
printf("%8s", "");
printf("%8s", "best");
printf("%8s", "int8");
printf("%8s", "int");
#ifdef CONFIG_NEON
printf("%8s", "neon");
#endif
@ -1796,7 +1796,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
SPEED_START {
raid_gen_force(3, raid_gen3_int8);
raid_recX_int8(3, id, ip, nd, size, v);
raid_rec3_int8(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1889,7 +1889,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
SPEED_START {
raid_gen_force(4, raid_gen4_int8);
raid_recX_int8(4, id, ip, nd, size, v);
raid_rec4_int8(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1982,7 +1982,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
SPEED_START {
raid_gen_force(5, raid_gen5_int8);
raid_recX_int8(5, id, ip, nd, size, v);
raid_rec5_int8(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -2075,7 +2075,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
SPEED_START {
raid_gen_force(6, raid_gen6_int8);
raid_recX_int8(6, id, ip, nd, size, v);
raid_rec6_int8(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);