From ec1daddc8eb6c492d084e130f2da798e9a01f8a4 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Wed, 2 Sep 2026 18:19:26 +0200 Subject: [PATCH] Eliminate raid_delta_gen and convert int/neon32 REC1/REC2 to single-pass fused recovery --- cmdline/speed.c | 16 ------ raid/NOTES | 72 ++++++++++------------- raid/OTHERS | 14 ++--- raid/avx2.c | 6 +- raid/avx512.c | 2 +- raid/gfni.c | 8 +-- raid/int.c | 129 ++++-------------------------------------- raid/internal.h | 5 -- raid/neon.c | 2 +- raid/neon32.c | 107 +---------------------------------- raid/raid.c | 98 -------------------------------- raid/ssse3.c | 20 +++---- raid/test/speedtest.c | 20 ------- 13 files changed, 70 insertions(+), 429 deletions(-) diff --git a/cmdline/speed.c b/cmdline/speed.c index 637a0900..4ce4186c 100644 --- a/cmdline/speed.c +++ b/cmdline/speed.c @@ -1367,8 +1367,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period) fflush(stdout); SPEED_START { - /* force the parity generator used by raid_delta_gen() */ - raid_gen_force(2, sizeof(void*) == 8 ? raid_gen2_int64_raid : raid_gen2_int32_raid); /* +1 to avoid GEN1 optimized case */ raid_rec1_int8(1, id, ip + 1, nd, size, v); } SPEED_STOP @@ -1471,8 +1469,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period) fflush(stdout); SPEED_START { - /* force the parity generator used by raid_delta_gen() */ - raid_gen_force(3, raid_gen3_int8); /* +2 to select R instead of Q */ raid_rec1_int8(1, id, ip + 2, nd, size, v); } SPEED_STOP @@ -1575,8 +1571,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period) fflush(stdout); SPEED_START { - /* force the parity generator used by raid_delta_gen() */ - raid_gen_force(2, sizeof(void*) == 8 ? raid_gen2_int64_raid : raid_gen2_int32_raid); raid_rec2_int8(2, id, ip, nd, size, v); } SPEED_STOP @@ -1593,8 +1587,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #endif #ifdef CONFIG_NEON32 SPEED_START { - /* force the parity generator used by raid_delta_gen() */ - raid_gen_force(2, raid_gen2_neon32_raid); raid_rec2_neon32(2, id, ip, nd, size, v); } SPEED_STOP @@ -1605,8 +1597,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { SPEED_START { - /* force the parity generator used by raid_delta_gen() */ - raid_gen_force(2, raid_gen2_sse2_raid); raid_rec2_ssse3(2, id, ip, nd, size, v); } SPEED_STOP @@ -1615,8 +1605,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #ifdef CONFIG_X86_64 SPEED_START { - /* force the parity generator used by raid_delta_gen() */ - raid_gen_force(2, raid_gen2_sse2ext_raid); raid_rec2_ssse3ext(2, id, ip, nd, size, v); } SPEED_STOP @@ -1675,8 +1663,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period) fflush(stdout); SPEED_START { - /* force the parity generator used by raid_delta_gen() */ - raid_gen_force(3, raid_gen3_int8); /* +1 to avoid GEN2 optimized case */ raid_rec2_int8(2, id, ip + 1, nd, size, v); } SPEED_STOP @@ -1779,8 +1765,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period) fflush(stdout); SPEED_START { - /* force the parity generator used by raid_delta_gen() */ - raid_gen_force(4, raid_gen4_int8); /* +2 to select R,S instead of Q,R */ raid_rec2_int8(2, id, ip + 2, nd, size, v); } SPEED_STOP diff --git a/raid/NOTES b/raid/NOTES index 7c407a14..a373a5b8 100644 --- a/raid/NOTES +++ b/raid/NOTES @@ -73,12 +73,11 @@ Recovery: SSSE3 and SSSE3 extended likewise use raid_rec1of1() for P, use specialized 1-pass Horner kernels for Q-only REC1 and P+Q REC2, and use fused selected-syndrome recovery for the remaining REC1..REC6 cases. AArch64 NEON - uses raid_rec1of1() for P and fused syndrome generation for non-P REC1 and all - REC2..REC6 cases. AArch32 NEON uses raid_rec1of1() for P, fused syndrome - generation for non-P REC1, all REC2 cases except P+Q, and REC3..REC6. + and AArch32 NEON use raid_rec1of1() for P and fused syndrome generation for + non-P REC1 and all REC2..REC6 cases. -* Scalar non-P REC1 and scalar REC2, plus AArch32 NEON P+Q, continue to use - raid_delta_gen(), while scalar REC3..REC6 uses fused syndrome recovery. +* Scalar recovery uses raid_rec1of1() for P and fused syndrome recovery for + non-P REC1 and all REC2..REC6 cases. raid_delta_gen() is completely eliminated. * Non-GFNI fused recovery splits every completed syndrome into low/high nibbles once and reuses the split values across all reconstructed outputs. @@ -177,10 +176,10 @@ MOVNT / non-temporal stores vs cached generation MOVNT avoids normal cache allocation/write-allocate traffic and reduces cache pressure. -* Conversely, in-memory parity verification in scrub and check, delta parity - generation in raid_delta_gen(), and single-parity recovery in raid_rec1of1() - use cached generation (streaming = 0) so that computed parity or recovered - data remains hot in cache for immediate CPU consumption. +* Conversely, in-memory parity verification in scrub and check, and + single-parity recovery in raid_rec1of1() use cached generation (streaming = 0) + so that computed parity or recovered data remains hot in cache for immediate + CPU consumption. Recovery temporal stores @@ -233,8 +232,8 @@ Typical recovery case failed data buffer is used as the GEN1 output, then calls raid_gen(nd, 1, ..., 0). All backends use this path with temporal stores for optimal P single-disk recovery. -* Therefore the common single-disk recovery path does NOT call - raid_delta_gen(). +* Therefore the common single-disk recovery path executes with no syndrome or GF + overhead. * The data traffic is already close to the minimum: @@ -276,24 +275,22 @@ REC1OF2 and REC1OF3: * The rows use Q and R respectively, avoiding the PAR1 fast path. -* SIMD backends use single-pass recovery for these non-P cases. AVX2, +* All backends use single-pass recovery for these non-P cases. AVX2, AVX2 extended, SSSE3, and SSSE3 extended use specialized Horner kernels for Q and fused selected-syndrome kernels for the other parity rows. AVX512BW, - GFNI, and NEON use their fused recovery paths. Scalar REC1 remains - delta-based. + GFNI, NEON, AArch32 NEON, and Scalar use their fused recovery paths. REC2OF2: * The row uses ip = { 0, 1 }, meaning P and Q. -* On AVX2, AVX512BW, GFNI (AVX2 and AVX512), and AArch64 NEON, REC2 executes - directly in the 1-pass fused kernel, which computes P with XOR, Q with GF - multiplication, and derives the second missing block via XOR. +* On AVX2, AVX512BW, GFNI (AVX2 and AVX512), AArch64 NEON, AArch32 NEON, and + Scalar int8, REC2 executes directly in the 1-pass fused kernel, which computes + P with XOR, Q with GF multiplication, and derives the second missing block + via XOR. * SSSE3 and SSSE3ext call dedicated 1-pass Horner raid_rec2of2_*() helpers that - compute P and Q directly from the surviving data without raid_delta_gen(). - AArch32 NEON uses its specialized delta-based raid_rec2of2_neon32() helper, - while scalar int8 leverages has_p in raid_rec2_int8. + compute P and Q directly from the surviving data in a single pass. * This makes the row a direct benchmark for the specialized P+Q recovery implementation. @@ -306,8 +303,7 @@ REC2OF3 and REC2OF4: ip = { 0, 1 } and therefore continue to exercise the generic REC2 paths. * They measure generic 2x2 recovery with different selected-parity positions, - exposing the difference between exact selected-syndrome fusion and - prefix-based delta generation. + verifying exact selected-syndrome fusion. REC3OF3 through REC6OF6: @@ -318,32 +314,29 @@ REC3OF3 through REC6OF6: Delta generation: * A benchmark forces a matching generation backend only when the measured - wrapper can call raid_rec1of1() or raid_delta_gen(). + wrapper can call raid_rec1of1() (specifically in REC1OF1). * Fused and specialized single-pass recovery rows do not depend on raid_gen() because they generate the required syndromes directly inside the recovery kernel. -raid_delta_gen() fusion ------------------------ +Single-pass fused recovery +-------------------------- * Fused SIMD recovery accumulates exactly the parity rows selected by ip[] directly from the stored parity and surviving data. -* This avoids the prefix constraint of raid_delta_gen(), which computes: - - np = ip[nr - 1] + 1 - - and raid_gen() can generate only a prefix of parity rows. +* This avoids the prefix constraint of the old delta-based approach, which + computed np = ip[nr - 1] + 1 via raid_gen(). * Example: nr = 3 ip = { 0, 2, 5 } - makes a delta-based path generate through GEN6, while a fused path calculates - only P, R and U. + makes an old delta-based path generate through GEN6, while a fused path + calculates only P, R and U. * For each SIMD chunk, the fused path starts the selected syndrome accumulators from stored parity, scans every surviving data source once, and then applies @@ -358,8 +351,7 @@ raid_delta_gen() fusion calling the GEN1 generator with streaming = 0 to write the recovered data block using temporal stores with no syndrome or GF recovery setup. -* raid_delta_gen() remains in scalar non-P REC1, scalar REC2, and AArch32 NEON - P+Q. +* raid_delta_gen() has been completely eliminated from the codebase. Generation: AVX512 width and unrolling @@ -1087,13 +1079,11 @@ Short list of ideas already tried or rejected poor fit when they add source rereads. Use register residency inside the existing traversal instead. -* raid_delta_gen() + REC fusion: - implemented for the generic SIMD recovery paths on AVX2, AVX512BW, GFNI, - SSSE3, SSSE3ext, AArch64 NEON, and AArch32 NEON. AVX2/AVX2ext and - SSSE3/SSSE3ext use dedicated 1-pass Horner kernels for Q-only REC1, and - SSSE3/SSSE3ext also use dedicated 1-pass Horner kernels for P+Q REC2. - All backends use raid_rec1of1() for rec1of1. Only scalar non-P REC1/scalar REC2 - and AArch32 NEON P+Q retain raid_delta_gen(). +* Two-pass delta generation (raid_delta_gen()): + completely eliminated in favor of generic single-pass fused recovery and + dedicated 1-pass Horner kernels. All backends use raid_rec1of1() for rec1of1, + and fused/Horner recovery for all other combinations. raid_delta_gen() has + been removed. * Software prefetch: no strong reason to add it given the existing long streams and proven diff --git a/raid/OTHERS b/raid/OTHERS index 3c19fcc5..65acc087 100644 --- a/raid/OTHERS +++ b/raid/OTHERS @@ -592,9 +592,10 @@ SnapRAID: With one missing data disk and P available, rec1of1 remaps pointers exactly the selected syndromes from stored parity and one scan of the surviving data, and reconstructs all missing blocks in the same kernel. -* The fused paths avoid raid_delta_gen(), generation of unused parity-prefix - rows, and full-size temporary syndrome buffers. Scalar and selected - SSSE3/AVX2 REC1/REC2 paths retain delta-based recovery. +* The fused paths avoid the two-pass delta generation approach, generation of + unused parity-prefix rows, and full-size temporary syndrome buffers. All + recovery backends (SIMD and scalar) use single-pass fused or specialized + Horner kernels. * N is at most six, so scalar matrix inversion setup is negligible compared with processing 256 KiB blocks. @@ -751,10 +752,9 @@ SnapRAID: During sync, large source blocks and a deep writer queue make without achieving a useful cache hit. Non-temporal stores (VMOVNTDQ) bypass write-allocate and avoid cache pollution. Conversely, in-memory parity verification during scrub and check, - delta parity generation in raid_delta_gen(), and data recovery in - raid_rec1of1() use temporal cached stores (streaming = 0) so that - computed parity or reconstructed data remains hot in cache for - immediate CPU consumption. + and data recovery in raid_rec1of1() use temporal cached stores + (streaming = 0) so that computed parity or reconstructed data remains + hot in cache for immediate CPU consumption. Linux RAID6: Optimized generation (gen_syndrome) is very similar: source prefetch and non-temporal final P/Q stores (VMOVNTDQ) are used in diff --git a/raid/avx2.c b/raid/avx2.c index a2373145..22c2739d 100644 --- a/raid/avx2.c +++ b/raid/avx2.c @@ -1079,7 +1079,7 @@ static __always_inline void raid_rec1_avx2_1(int *id, int *ip, int nd, size_t si * * Computes Q of all surviving data directly with Horner's method and reconstructs the missing block from Qdelta. * - * Processes two 32-byte lanes per iteration, avoiding raid_delta_gen(), temporary parity buffers, and the extra pass over the data. + * Processes two 32-byte lanes per iteration, avoiding temporary parity buffers, and the extra pass over the data. */ static __always_inline void raid_rec1_avx2_q(int *id, int *ip, int nd, size_t size, void **vv) { @@ -1624,7 +1624,7 @@ static __always_inline void raid_recX_avx2(int nr, int has_p, int *id, int *ip, * * Computes Q of all surviving data directly with Horner's method and reconstructs the missing block from Qdelta. * - * Processes two 32-byte lanes and two data disks per Horner iteration, avoiding raid_delta_gen(), temporary parity buffers, and the extra pass over the data. + * Processes two 32-byte lanes and two data disks per Horner iteration, avoiding temporary parity buffers, and the extra pass over the data. * * Note that it uses 16 registers, meaning that x64 is required. */ @@ -2373,7 +2373,7 @@ static __always_inline void raid_recX_avx2ext_123(int nr, int has_p, int *id, in * Recover multiple data failures using selected parity blocks with AVX2 extended. * * Compute only the selected syndromes, keeping them in registers. - * This avoids raid_delta_gen(), temporary syndrome buffers, and the + * This avoids temporary syndrome buffers and the * generation of unused parity rows. * * If P is available, preserve the complete P delta syndrome and diff --git a/raid/avx512.c b/raid/avx512.c index f7934776..4d214568 100644 --- a/raid/avx512.c +++ b/raid/avx512.c @@ -612,7 +612,7 @@ static __always_inline void raid_genX_avx512bw(int nd, size_t size, void **vv, i * Recover multiple data failures using selected parity blocks with AVX512BW. * * Compute only the selected syndromes, keeping them in registers. - * This avoids raid_delta_gen(), temporary syndrome buffers, recomputation of + * This avoids temporary syndrome buffers, recomputation of * parity blocks, and generation of unused parity rows. * * AVX512 provides enough registers to keep all six supported syndromes diff --git a/raid/gfni.c b/raid/gfni.c index 4c5f9237..0fd66832 100644 --- a/raid/gfni.c +++ b/raid/gfni.c @@ -1448,7 +1448,7 @@ static __always_inline void raid_genX_avx512gfni_aes(int nd, size_t size, void * * Recover multiple data failures using selected parity blocks with AVX2 GFNI. * * Compute only the selected syndromes, keeping them in registers. - * This avoids raid_delta_gen(), temporary syndrome buffers, recomputation of + * This avoids temporary syndrome buffers, recomputation of * parity blocks, and generation of unused parity rows. * * If recovering a single failure from P, the P delta directly contains @@ -1663,7 +1663,7 @@ static __always_inline void raid_recX_avx2gfni_raid(int nr, int has_p, int *id, * Recover multiple data failures using selected parity blocks with AVX512 GFNI. * * Compute only the selected syndromes, keeping them in registers. - * This avoids raid_delta_gen(), temporary syndrome buffers, recomputation of + * This avoids temporary syndrome buffers, recomputation of * parity blocks, and generation of unused parity rows. * * If P is available, reconstruct only nr - 1 missing blocks through the @@ -1841,7 +1841,7 @@ static __always_inline void raid_recX_avx512gfni_raid(int nr, int has_p, int *id * Recover multiple data failures for the AES polynomial using selected parity blocks with AVX2 GFNI. * * Compute only the selected syndromes, keeping them in registers. - * This avoids raid_delta_gen(), temporary syndrome buffers, recomputation of + * This avoids temporary syndrome buffers, recomputation of * parity blocks, and generation of unused parity rows. * * If recovering a single failure from P, the P delta directly contains @@ -2046,7 +2046,7 @@ static __always_inline void raid_recX_avx2gfni_aes(int nr, int has_p, int *id, i * Recover multiple data failures for the AES polynomial using selected parity blocks with AVX512 GFNI. * * Compute only the selected syndromes, keeping them in registers. - * This avoids raid_delta_gen(), temporary syndrome buffers, recomputation of + * This avoids temporary syndrome buffers, recomputation of * parity blocks, and generation of unused parity rows. * * If P is available, reconstruct only nr - 1 missing blocks through the diff --git a/raid/int.c b/raid/int.c index 36b06c54..86de9e49 100644 --- a/raid/int.c +++ b/raid/int.c @@ -666,121 +666,6 @@ void raid_gen6_int8(int nd, size_t size, void **vv, int streaming) } } -/* - * Recover failure of one data block at index id[0] using parity at index - * ip[0] for any RAID level using 8bit C implementation. - * - * Starting from the equation: - * - * Pd = A[ip[0],id[0]] * Dx - * - * and solving we get: - * - * Dx = A[ip[0],id[0]]^-1 * Pd - */ -void raid_rec1_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *pa; - const uint8_t *T; - uint8_t G; - uint8_t V; - size_t i; - - (void)nr; /* unused, it's always 1 */ - - /* if recovering with P uses the delta function */ - if (ip[0] == 0) { - raid_rec1of1(id, nd, size, vv); - return; - } - - /* setup the coefficients matrix */ - G = A(ip[0], id[0]); - - /* invert it to solve the system of linear equations */ - V = inv(G); - - /* get multiplication tables */ - T = table(V); - - /* compute delta parity */ - raid_delta_gen(1, id, ip, nd, size, vv); - - p = v[nd + ip[0]]; - pa = v[id[0]]; - - for (i = 0; i < size; ++i) { - /* delta */ - uint8_t Pd = p[i] ^ pa[i]; - - /* reconstruct */ - pa[i] = T[Pd]; - } -} - -/* - * Recover failure of two data blocks at indexes id[0],id[1] using parity at - * indexes ip[0],ip[1] for any RAID level using 8bit C implementation. - * - * If P is available, reconstructs only the first missing block through the - * inverse matrix and derives the second missing block from the P delta by XOR. - */ -static __always_inline void raid_rec2_int8_gen(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *q; - uint8_t *pa; - uint8_t *qa; - const uint8_t *T[2][2]; - uint8_t G[2 * 2]; - uint8_t V[2 * 2]; - size_t i; - int j, k; - - (void)nr; /* unused, it's always 2 */ - - /* setup the coefficients matrix */ - for (j = 0; j < 2; ++j) - for (k = 0; k < 2; ++k) - G[j * 2 + k] = A(ip[j], id[k]); - - /* invert it to solve the system of linear equations */ - raid_invert(G, V, 2); - - /* get multiplication tables */ - for (j = 0; j < 2 - has_p; ++j) - for (k = 0; k < 2; ++k) - T[j][k] = table(V[j * 2 + k]); - - /* compute delta parity */ - raid_delta_gen(2, id, ip, nd, size, vv); - - p = v[nd + ip[0]]; - q = v[nd + ip[1]]; - pa = v[id[0]]; - qa = v[id[1]]; - - for (i = 0; i < size; ++i) { - /* delta */ - uint8_t Pd = p[i] ^ pa[i]; - uint8_t Qd = q[i] ^ qa[i]; - - /* reconstruct */ - if (has_p) { - uint8_t Dx = T[0][0][Pd] ^ T[0][1][Qd]; - - pa[i] = Dx; - qa[i] = Pd ^ Dx; - } else { - pa[i] = T[0][0][Pd] ^ T[0][1][Qd]; - qa[i] = T[1][0][Pd] ^ T[1][1][Qd]; - } - } -} - /* * Recover failure of N data blocks at indexes id[N] using parity at indexes * ip[N] for any RAID level using 8bit C implementation. @@ -1259,14 +1144,24 @@ void raid_gen2_int64_aes(int nd, size_t size, void **vv, int streaming) raid_gen2_int64_gen(nd, size, vv, 3); } +void raid_rec1_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 1); + + if (ip[0] == 0) + raid_rec1of1(id, nd, size, vv); + else + raid_recX_int8(1, 0, id, ip, nd, size, vv); +} + void raid_rec2_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 2); if (ip[0] == 0) - raid_rec2_int8_gen(2, 1, id, ip, nd, size, vv); + raid_recX_int8(2, 1, id, ip, nd, size, vv); else - raid_rec2_int8_gen(2, 0, id, ip, nd, size, vv); + raid_recX_int8(2, 0, id, ip, nd, size, vv); } void raid_rec3_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv) diff --git a/raid/internal.h b/raid/internal.h index 12ff9b0a..05a148fb 100644 --- a/raid/internal.h +++ b/raid/internal.h @@ -89,7 +89,6 @@ static __always_inline void * __align_ptr(void *ptr, uintptr_t size) */ void raid_gen_ref(int nd, int np, size_t size, void **vv); void raid_invert(uint8_t *M, uint8_t *V, int n); -void raid_delta_gen(int nr, int *id, int *ip, int nd, size_t size, void **v); void raid_rec1of1(int *id, int nd, size_t size, void **v); void raid_gen1_int32(int nd, size_t size, void **vv, int streaming); void raid_gen1_int64(int nd, size_t size, void **vv, int streaming); @@ -272,10 +271,6 @@ void raid_rec6_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv); * * All these functions give the guarantee that parities are written * in order. First parity P, then parity Q, and so on. - * This allows to specify the same memory buffer for multiple parities - * knowing that you'll get the latest written one. - * This characteristic is used by the raid_delta_gen() function to - * avoid damaging unused parities during recovery. * * @nd Number of data blocks * @size Size of the blocks pointed to by @vv. It must be a multiple of 64. diff --git a/raid/neon.c b/raid/neon.c index 033cdcea..3dc412ed 100644 --- a/raid/neon.c +++ b/raid/neon.c @@ -922,7 +922,7 @@ static __always_inline void raid_recX_neon_12(int nr, int has_p, int *id, int *i /* * Recover multiple data failures using selected parity blocks with NEON. * - * This avoids raid_delta_gen(), temporary syndrome buffers, and the + * This avoids temporary syndrome buffers and the * generation of unused parity rows. * * If P is available, preserve the complete P delta syndrome and diff --git a/raid/neon32.c b/raid/neon32.c index 980bbe81..d63df997 100644 --- a/raid/neon32.c +++ b/raid/neon32.c @@ -830,7 +830,7 @@ static __always_inline void raid_recX_neon32_123(int nr, int has_p, int *id, int /* * Recover multiple data failures using selected parity blocks with AArch32 NEON. * - * This avoids raid_delta_gen(), temporary syndrome buffers, and the + * This avoids temporary syndrome buffers and the * generation of unused parity rows. * * If P is available, preserve the complete P delta syndrome and @@ -1241,115 +1241,10 @@ void raid_rec1_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv) raid_recX_neon32_123(1, 0, id, ip, nd, size, vv); } -/* - * Recover failure of two data blocks using P and Q AArch32 NEON implementation. - * - * Uses raid_delta_gen() and computes the analytical solution with polynomial reduction. - */ -static __always_inline void raid_rec2of2_neon32(int *id, int *ip, int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *pa; - uint8_t *q; - uint8_t *qa; - uint8_t C[2]; - size_t i; - - C[0] = inv(powgen(id[1] - id[0]) ^ 1); - C[1] = inv(powgen(id[0]) ^ powgen(id[1])); - - raid_delta_gen(2, id, ip, nd, size, vv); - - p = v[nd]; - q = v[nd + 1]; - pa = v[id[0]]; - qa = v[id[1]]; - - raid_neon32_begin(); - - /* q15 = nibble mask, q9-q12 = C0/C1 low/high tables */ - asm volatile ( - "vmov.i8 q15, #0x0f\n" - "vld1.8 {q9}, %0\n" - "vld1.8 {q10}, %1\n" - "vld1.8 {q11}, %2\n" - "vld1.8 {q12}, %3\n" - : - : "Q" (raid_gfmulpshufb[C[0]][0][0]), - "Q" (raid_gfmulpshufb[C[0]][1][0]), - "Q" (raid_gfmulpshufb[C[1]][0][0]), - "Q" (raid_gfmulpshufb[C[1]][1][0]) - ); - - for (i = 0; i < size; i += 16) { - asm volatile ( - /* Pd */ - "vld1.8 {q0}, %2\n" - "vld1.8 {q13}, %4\n" - "veor q0, q0, q13\n" - - /* Qd */ - "vld1.8 {q1}, %3\n" - "vld1.8 {q13}, %5\n" - "veor q1, q1, q13\n" - - /* split Pd */ - "vmov q2, q0\n" - "vshr.u8 q3, q0, #4\n" - "vand q2, q2, q15\n" - "vand q3, q3, q15\n" - - /* split Qd */ - "vmov q4, q1\n" - "vshr.u8 q5, q1, #4\n" - "vand q4, q4, q15\n" - "vand q5, q5, q15\n" - - /* C0 low * Pd low -> q6 */ - "vtbl.8 d12, {d18-d19}, d4\n" - "vtbl.8 d13, {d18-d19}, d5\n" - - /* C0 high * Pd high -> q7 */ - "vtbl.8 d14, {d20-d21}, d6\n" - "vtbl.8 d15, {d20-d21}, d7\n" - "veor q6, q6, q7\n" - - /* C1 low * Qd low -> q7 */ - "vtbl.8 d14, {d22-d23}, d8\n" - "vtbl.8 d15, {d22-d23}, d9\n" - - /* C1 high * Qd high -> q8 */ - "vtbl.8 d16, {d24-d25}, d10\n" - "vtbl.8 d17, {d24-d25}, d11\n" - "veor q7, q7, q8\n" - - /* Dy */ - "veor q6, q6, q7\n" - - /* Dx = Pd ^ Dy */ - "veor q0, q0, q6\n" - - "vst1.8 {q0}, %0\n" - "vst1.8 {q6}, %1\n" - : "=Q" (pa[i]), "=Q" (qa[i]) - : "Q" (p[i]), "Q" (q[i]), - "Q" (pa[i]), "Q" (qa[i]) - ); - } - - raid_neon32_end(); -} - void raid_rec2_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 2); - if (ip[0] == 0 && ip[1] == 1) { - raid_rec2of2_neon32(id, ip, nd, size, vv); - return; - } - if (ip[0] == 0) raid_recX_neon32_123(2, 1, id, ip, nd, size, vv); else diff --git a/raid/raid.c b/raid/raid.c index 827fc516..b931ef07 100644 --- a/raid/raid.c +++ b/raid/raid.c @@ -279,104 +279,6 @@ void raid_invert(uint8_t *M, uint8_t *V, int n) } } -/** - * Computes the parity without the missing data blocks - * and store it in the buffers of such data blocks. - * - * This is the parity expressed as Pa,Qa,Ra,Sa,Ta,Ua in the equations. - */ -void raid_delta_gen(int nr, int *id, int *ip, int nd, size_t size, void **v) -{ - void *p[RAID_PARITY_MAX]; - void *pa[RAID_PARITY_MAX]; - int i, j; - int np; - void *latest; - - /* zero buffer must be initialized prior to delta recovery */ - BUG_ON(raid_zero_block == 0); - - /* - * Total number of parities we are going to process - * they are both the used and the unused ones - */ - np = ip[nr - 1] + 1; - - /* latest missing data block */ - latest = v[id[nr - 1]]; - - /* setup pointers for delta computation */ - for (i = 0, j = 0; i < np; ++i) { - /* keep a copy of the original parity vector */ - p[i] = v[nd + i]; - - if (ip[j] == i) { - /* - * Set used parities to point to the missing - * data blocks. - * - * The related data blocks are instead set - * to point to the "zero" buffer. - */ - - /* - * The latest parity to use ends the for loop and - * then it cannot happen to process more of them - */ - BUG_ON(j >= nr); - - /* buffer for missing data blocks */ - pa[j] = v[id[j]]; - - /* set at zero the missing data blocks */ - v[id[j]] = raid_zero_block; - - /* compute the parity over the missing data blocks */ - v[nd + i] = pa[j]; - - /* check for the next used entry */ - ++j; - } else { - /* - * Unused parities are going to be rewritten with - * not significant data, because we don't have - * functions able to compute only a subset of - * parities. - * - * To avoid this, we reuse parity buffers, - * assuming that all the parity functions write - * parities in order. - * - * We assign the unused parity block to the same - * block of the latest used parity that we know it - * will be written. - * - * This means that this block will be written - * multiple times and only the latest write will - * contain the correct data. - */ - v[nd + i] = latest; - } - } - - /* all the parities have to be processed */ - BUG_ON(j != nr); - - /* - * Recompute the parity, note that np may be smaller than the - * total number of parities available - */ - raid_gen(nd, np, size, v, 0); - - /* restore data buffers as before */ - for (j = 0; j < nr; ++j) - v[id[j]] = pa[j]; - - /* restore parity buffers as before */ - for (i = 0; i < np; ++i) - v[nd + i] = p[i]; -} - /** * Recover failure of one data block for PAR1. * diff --git a/raid/ssse3.c b/raid/ssse3.c index ee3853c0..de1fb159 100644 --- a/raid/ssse3.c +++ b/raid/ssse3.c @@ -1250,7 +1250,7 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i * Recover failure of one data block using selected parity with SSSE3, optimized for one failure. * * Computes the selected syndrome in a single survivor scan and reconstructs - * the missing block directly, avoiding raid_delta_gen() and the extra pass. + * the missing block directly, avoiding extra memory passes. */ static __always_inline void raid_rec1_ssse3_1(int *id, int *ip, int nd, size_t size, void **vv) { @@ -1338,8 +1338,8 @@ static __always_inline void raid_rec1_ssse3_1(int *id, int *ip, int nd, size_t s * Computes Q of all surviving data directly with Horner's method and * reconstructs the missing block from Qdelta. * - * Processes two 16-byte lanes per iteration, avoiding raid_delta_gen(), - * temporary parity buffers, and the extra pass over the data. + * Processes two 16-byte lanes per iteration, avoiding temporary parity + * buffers and the extra pass over the data. */ static __always_inline void raid_rec1_ssse3_q(int *id, int *ip, int nd, size_t size, void **vv) { @@ -1472,8 +1472,8 @@ static __always_inline void raid_rec1_ssse3_q(int *id, int *ip, int nd, size_t s * optimized specifically for two failures. * * Computes only the two selected syndromes in a single survivor scan and - * reconstructs the missing blocks directly, avoiding raid_delta_gen(), - * temporary syndrome buffers, and the generation of unused parity rows. + * reconstructs the missing blocks directly, avoiding temporary syndrome + * buffers and the generation of unused parity rows. * * has_p is expected to be a compile-time constant after inlining. * @@ -1661,8 +1661,8 @@ static __always_inline void raid_rec2_ssse3_2(int has_p, int *id, int *ip, int n * Computes Pdelta and Qdelta directly in a single survivor scan. * Q is computed with Horner's method using the active field generator. * - * Processes two 16-byte lanes per iteration, avoiding raid_delta_gen(), - * temporary parity buffers, and the second pass over the data. + * Processes two 16-byte lanes per iteration, avoiding temporary parity + * buffers and the second pass over the data. */ static __always_inline void raid_rec2of2_ssse3(int *id, int *ip, int nd, size_t size, void **vv) { @@ -2395,8 +2395,8 @@ static __always_inline void raid_rec1_ssse3ext_1(int *id, int *ip, int nd, size_ * Computes Q of all surviving data directly with Horner's method and * reconstructs the missing block from Qdelta. * - * Processes four 16-byte lanes per iteration, avoiding raid_delta_gen(), - * temporary parity buffers, and the extra pass over the data. + * Processes four 16-byte lanes per iteration, avoiding temporary parity + * buffers and the extra pass over the data. */ static __always_inline void raid_rec1_ssse3ext_q(int *id, int *ip, int nd, size_t size, void **vv) { @@ -3799,7 +3799,7 @@ static __always_inline void raid_recX_ssse3ext_12345(int nr, int has_p, int *id, /* * Recover multiple data failures using selected parity blocks with SSSE3 extended. * - * This avoids raid_delta_gen(), temporary syndrome buffers, and the + * This avoids temporary syndrome buffers and the * generation of unused parity rows. * * If P is available, preserve the complete P delta syndrome and diff --git a/raid/test/speedtest.c b/raid/test/speedtest.c index cea6227b..d7c6a744 100644 --- a/raid/test/speedtest.c +++ b/raid/test/speedtest.c @@ -1263,8 +1263,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) fflush(stdout); SPEED_START { - /* force the parity generator used by raid_delta_gen() */ - raid_gen_force(2, sizeof(void *) == 8 ? raid_gen2_int64_raid : raid_gen2_int32_raid); /* +1 to avoid GEN1 optimized case */ raid_rec1_int8(1, id, ip + 1, nd, size, v); } SPEED_STOP @@ -1367,8 +1365,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) fflush(stdout); SPEED_START { - /* force the parity generator used by raid_delta_gen() */ - raid_gen_force(3, raid_gen3_int8); /* +2 to select R instead of Q */ raid_rec1_int8(1, id, ip + 2, nd, size, v); } SPEED_STOP @@ -1471,8 +1467,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) fflush(stdout); SPEED_START { - /* force the parity generator used by raid_delta_gen() */ - raid_gen_force(2, sizeof(void *) == 8 ? raid_gen2_int64_raid : raid_gen2_int32_raid); raid_rec2_int8(2, id, ip, nd, size, v); } SPEED_STOP @@ -1489,8 +1483,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #endif #ifdef CONFIG_NEON32 SPEED_START { - /* force the parity generator used by raid_delta_gen() */ - raid_gen_force(2, raid_gen2_neon32_raid); raid_rec2_neon32(2, id, ip, nd, size, v); } SPEED_STOP @@ -1501,8 +1493,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { SPEED_START { - /* force the parity generator used by raid_delta_gen() */ - raid_gen_force(2, raid_gen2_sse2_raid); raid_rec2_ssse3(2, id, ip, nd, size, v); } SPEED_STOP @@ -1511,8 +1501,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #ifdef CONFIG_X86_64 SPEED_START { - /* force the parity generator used by raid_delta_gen() */ - raid_gen_force(2, raid_gen2_sse2ext_raid); raid_rec2_ssse3ext(2, id, ip, nd, size, v); } SPEED_STOP @@ -1571,8 +1559,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) fflush(stdout); SPEED_START { - /* force the parity generator used by raid_delta_gen() */ - raid_gen_force(3, raid_gen3_int8); /* +1 to avoid GEN2 optimized case */ raid_rec2_int8(2, id, ip + 1, nd, size, v); } SPEED_STOP @@ -1675,8 +1661,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) fflush(stdout); SPEED_START { - /* force the parity generator used by raid_delta_gen() */ - raid_gen_force(4, raid_gen4_int8); /* +2 to select R,S instead of Q,R */ raid_rec2_int8(2, id, ip + 2, nd, size, v); } SPEED_STOP @@ -1779,7 +1763,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) fflush(stdout); SPEED_START { - raid_gen_force(3, raid_gen3_int8); raid_rec3_int8(3, id, ip, nd, size, v); } SPEED_STOP @@ -1872,7 +1855,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) fflush(stdout); SPEED_START { - raid_gen_force(4, raid_gen4_int8); raid_rec4_int8(4, id, ip, nd, size, v); } SPEED_STOP @@ -1965,7 +1947,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) fflush(stdout); SPEED_START { - raid_gen_force(5, raid_gen5_int8); raid_rec5_int8(5, id, ip, nd, size, v); } SPEED_STOP @@ -2058,7 +2039,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) fflush(stdout); SPEED_START { - raid_gen_force(6, raid_gen6_int8); raid_rec6_int8(6, id, ip, nd, size, v); } SPEED_STOP