mirror of
https://github.com/amadvance/snapraid.git
synced 2026-10-26 10:51:54 -04:00
Eliminate raid_delta_gen and convert int/neon32 REC1/REC2 to single-pass fused recovery
This commit is contained in:
Andrea Mazzoleni
parent
0ce89aa35d
commit
ec1daddc8e
13 changed files with 70 additions and 429 deletions
72
raid/NOTES
72
raid/NOTES
|
|
@ -73,12 +73,11 @@ Recovery:
|
|||
SSSE3 and SSSE3 extended likewise use raid_rec1of1() for P, use specialized
|
||||
1-pass Horner kernels for Q-only REC1 and P+Q REC2, and use fused
|
||||
selected-syndrome recovery for the remaining REC1..REC6 cases. AArch64 NEON
|
||||
uses raid_rec1of1() for P and fused syndrome generation for non-P REC1 and all
|
||||
REC2..REC6 cases. AArch32 NEON uses raid_rec1of1() for P, fused syndrome
|
||||
generation for non-P REC1, all REC2 cases except P+Q, and REC3..REC6.
|
||||
and AArch32 NEON use raid_rec1of1() for P and fused syndrome generation for
|
||||
non-P REC1 and all REC2..REC6 cases.
|
||||
|
||||
* Scalar non-P REC1 and scalar REC2, plus AArch32 NEON P+Q, continue to use
|
||||
raid_delta_gen(), while scalar REC3..REC6 uses fused syndrome recovery.
|
||||
* Scalar recovery uses raid_rec1of1() for P and fused syndrome recovery for
|
||||
non-P REC1 and all REC2..REC6 cases. raid_delta_gen() is completely eliminated.
|
||||
|
||||
* Non-GFNI fused recovery splits every completed syndrome into low/high nibbles
|
||||
once and reuses the split values across all reconstructed outputs.
|
||||
|
|
@ -177,10 +176,10 @@ MOVNT / non-temporal stores vs cached generation
|
|||
MOVNT avoids normal cache allocation/write-allocate traffic and reduces cache
|
||||
pressure.
|
||||
|
||||
* Conversely, in-memory parity verification in scrub and check, delta parity
|
||||
generation in raid_delta_gen(), and single-parity recovery in raid_rec1of1()
|
||||
use cached generation (streaming = 0) so that computed parity or recovered
|
||||
data remains hot in cache for immediate CPU consumption.
|
||||
* Conversely, in-memory parity verification in scrub and check, and
|
||||
single-parity recovery in raid_rec1of1() use cached generation (streaming = 0)
|
||||
so that computed parity or recovered data remains hot in cache for immediate
|
||||
CPU consumption.
|
||||
|
||||
|
||||
Recovery temporal stores
|
||||
|
|
@ -233,8 +232,8 @@ Typical recovery case
|
|||
failed data buffer is used as the GEN1 output, then calls raid_gen(nd, 1, ..., 0).
|
||||
All backends use this path with temporal stores for optimal P single-disk recovery.
|
||||
|
||||
* Therefore the common single-disk recovery path does NOT call
|
||||
raid_delta_gen().
|
||||
* Therefore the common single-disk recovery path executes with no syndrome or GF
|
||||
overhead.
|
||||
|
||||
* The data traffic is already close to the minimum:
|
||||
|
||||
|
|
@ -276,24 +275,22 @@ REC1OF2 and REC1OF3:
|
|||
|
||||
* The rows use Q and R respectively, avoiding the PAR1 fast path.
|
||||
|
||||
* SIMD backends use single-pass recovery for these non-P cases. AVX2,
|
||||
* All backends use single-pass recovery for these non-P cases. AVX2,
|
||||
AVX2 extended, SSSE3, and SSSE3 extended use specialized Horner kernels for
|
||||
Q and fused selected-syndrome kernels for the other parity rows. AVX512BW,
|
||||
GFNI, and NEON use their fused recovery paths. Scalar REC1 remains
|
||||
delta-based.
|
||||
GFNI, NEON, AArch32 NEON, and Scalar use their fused recovery paths.
|
||||
|
||||
REC2OF2:
|
||||
|
||||
* The row uses ip = { 0, 1 }, meaning P and Q.
|
||||
|
||||
* On AVX2, AVX512BW, GFNI (AVX2 and AVX512), and AArch64 NEON, REC2 executes
|
||||
directly in the 1-pass fused kernel, which computes P with XOR, Q with GF
|
||||
multiplication, and derives the second missing block via XOR.
|
||||
* On AVX2, AVX512BW, GFNI (AVX2 and AVX512), AArch64 NEON, AArch32 NEON, and
|
||||
Scalar int8, REC2 executes directly in the 1-pass fused kernel, which computes
|
||||
P with XOR, Q with GF multiplication, and derives the second missing block
|
||||
via XOR.
|
||||
|
||||
* SSSE3 and SSSE3ext call dedicated 1-pass Horner raid_rec2of2_*() helpers that
|
||||
compute P and Q directly from the surviving data without raid_delta_gen().
|
||||
AArch32 NEON uses its specialized delta-based raid_rec2of2_neon32() helper,
|
||||
while scalar int8 leverages has_p in raid_rec2_int8.
|
||||
compute P and Q directly from the surviving data in a single pass.
|
||||
|
||||
* This makes the row a direct benchmark for the specialized P+Q recovery
|
||||
implementation.
|
||||
|
|
@ -306,8 +303,7 @@ REC2OF3 and REC2OF4:
|
|||
ip = { 0, 1 } and therefore continue to exercise the generic REC2 paths.
|
||||
|
||||
* They measure generic 2x2 recovery with different selected-parity positions,
|
||||
exposing the difference between exact selected-syndrome fusion and
|
||||
prefix-based delta generation.
|
||||
verifying exact selected-syndrome fusion.
|
||||
|
||||
REC3OF3 through REC6OF6:
|
||||
|
||||
|
|
@ -318,32 +314,29 @@ REC3OF3 through REC6OF6:
|
|||
Delta generation:
|
||||
|
||||
* A benchmark forces a matching generation backend only when the measured
|
||||
wrapper can call raid_rec1of1() or raid_delta_gen().
|
||||
wrapper can call raid_rec1of1() (specifically in REC1OF1).
|
||||
|
||||
* Fused and specialized single-pass recovery rows do not depend on raid_gen()
|
||||
because they generate the required syndromes directly inside the recovery
|
||||
kernel.
|
||||
|
||||
|
||||
raid_delta_gen() fusion
|
||||
-----------------------
|
||||
Single-pass fused recovery
|
||||
--------------------------
|
||||
|
||||
* Fused SIMD recovery accumulates exactly the parity rows selected by ip[]
|
||||
directly from the stored parity and surviving data.
|
||||
|
||||
* This avoids the prefix constraint of raid_delta_gen(), which computes:
|
||||
|
||||
np = ip[nr - 1] + 1
|
||||
|
||||
and raid_gen() can generate only a prefix of parity rows.
|
||||
* This avoids the prefix constraint of the old delta-based approach, which
|
||||
computed np = ip[nr - 1] + 1 via raid_gen().
|
||||
|
||||
* Example:
|
||||
|
||||
nr = 3
|
||||
ip = { 0, 2, 5 }
|
||||
|
||||
makes a delta-based path generate through GEN6, while a fused path calculates
|
||||
only P, R and U.
|
||||
makes an old delta-based path generate through GEN6, while a fused path
|
||||
calculates only P, R and U.
|
||||
|
||||
* For each SIMD chunk, the fused path starts the selected syndrome accumulators
|
||||
from stored parity, scans every surviving data source once, and then applies
|
||||
|
|
@ -358,8 +351,7 @@ raid_delta_gen() fusion
|
|||
calling the GEN1 generator with streaming = 0 to write the recovered data
|
||||
block using temporal stores with no syndrome or GF recovery setup.
|
||||
|
||||
* raid_delta_gen() remains in scalar non-P REC1, scalar REC2, and AArch32 NEON
|
||||
P+Q.
|
||||
* raid_delta_gen() has been completely eliminated from the codebase.
|
||||
|
||||
|
||||
Generation: AVX512 width and unrolling
|
||||
|
|
@ -1087,13 +1079,11 @@ Short list of ideas already tried or rejected
|
|||
poor fit when they add source rereads. Use register residency inside the
|
||||
existing traversal instead.
|
||||
|
||||
* raid_delta_gen() + REC fusion:
|
||||
implemented for the generic SIMD recovery paths on AVX2, AVX512BW, GFNI,
|
||||
SSSE3, SSSE3ext, AArch64 NEON, and AArch32 NEON. AVX2/AVX2ext and
|
||||
SSSE3/SSSE3ext use dedicated 1-pass Horner kernels for Q-only REC1, and
|
||||
SSSE3/SSSE3ext also use dedicated 1-pass Horner kernels for P+Q REC2.
|
||||
All backends use raid_rec1of1() for rec1of1. Only scalar non-P REC1/scalar REC2
|
||||
and AArch32 NEON P+Q retain raid_delta_gen().
|
||||
* Two-pass delta generation (raid_delta_gen()):
|
||||
completely eliminated in favor of generic single-pass fused recovery and
|
||||
dedicated 1-pass Horner kernels. All backends use raid_rec1of1() for rec1of1,
|
||||
and fused/Horner recovery for all other combinations. raid_delta_gen() has
|
||||
been removed.
|
||||
|
||||
* Software prefetch:
|
||||
no strong reason to add it given the existing long streams and proven
|
||||
|
|
|
|||
14
raid/OTHERS
14
raid/OTHERS
|
|
@ -592,9 +592,10 @@ SnapRAID: With one missing data disk and P available, rec1of1 remaps pointers
|
|||
exactly the selected syndromes from stored parity and one scan of the
|
||||
surviving data, and reconstructs all missing blocks in the same kernel.
|
||||
|
||||
* The fused paths avoid raid_delta_gen(), generation of unused parity-prefix
|
||||
rows, and full-size temporary syndrome buffers. Scalar and selected
|
||||
SSSE3/AVX2 REC1/REC2 paths retain delta-based recovery.
|
||||
* The fused paths avoid the two-pass delta generation approach, generation of
|
||||
unused parity-prefix rows, and full-size temporary syndrome buffers. All
|
||||
recovery backends (SIMD and scalar) use single-pass fused or specialized
|
||||
Horner kernels.
|
||||
|
||||
* N is at most six, so scalar matrix inversion setup is negligible compared
|
||||
with processing 256 KiB blocks.
|
||||
|
|
@ -751,10 +752,9 @@ SnapRAID: During sync, large source blocks and a deep writer queue make
|
|||
without achieving a useful cache hit. Non-temporal stores (VMOVNTDQ)
|
||||
bypass write-allocate and avoid cache pollution.
|
||||
Conversely, in-memory parity verification during scrub and check,
|
||||
delta parity generation in raid_delta_gen(), and data recovery in
|
||||
raid_rec1of1() use temporal cached stores (streaming = 0) so that
|
||||
computed parity or reconstructed data remains hot in cache for
|
||||
immediate CPU consumption.
|
||||
and data recovery in raid_rec1of1() use temporal cached stores
|
||||
(streaming = 0) so that computed parity or reconstructed data remains
|
||||
hot in cache for immediate CPU consumption.
|
||||
|
||||
Linux RAID6: Optimized generation (gen_syndrome) is very similar: source
|
||||
prefetch and non-temporal final P/Q stores (VMOVNTDQ) are used in
|
||||
|
|
|
|||
|
|
@ -1079,7 +1079,7 @@ static __always_inline void raid_rec1_avx2_1(int *id, int *ip, int nd, size_t si
|
|||
*
|
||||
* Computes Q of all surviving data directly with Horner's method and reconstructs the missing block from Qdelta.
|
||||
*
|
||||
* Processes two 32-byte lanes per iteration, avoiding raid_delta_gen(), temporary parity buffers, and the extra pass over the data.
|
||||
* Processes two 32-byte lanes per iteration, avoiding temporary parity buffers, and the extra pass over the data.
|
||||
*/
|
||||
static __always_inline void raid_rec1_avx2_q(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -1624,7 +1624,7 @@ static __always_inline void raid_recX_avx2(int nr, int has_p, int *id, int *ip,
|
|||
*
|
||||
* Computes Q of all surviving data directly with Horner's method and reconstructs the missing block from Qdelta.
|
||||
*
|
||||
* Processes two 32-byte lanes and two data disks per Horner iteration, avoiding raid_delta_gen(), temporary parity buffers, and the extra pass over the data.
|
||||
* Processes two 32-byte lanes and two data disks per Horner iteration, avoiding temporary parity buffers, and the extra pass over the data.
|
||||
*
|
||||
* Note that it uses 16 registers, meaning that x64 is required.
|
||||
*/
|
||||
|
|
@ -2373,7 +2373,7 @@ static __always_inline void raid_recX_avx2ext_123(int nr, int has_p, int *id, in
|
|||
* Recover multiple data failures using selected parity blocks with AVX2 extended.
|
||||
*
|
||||
* Compute only the selected syndromes, keeping them in registers.
|
||||
* This avoids raid_delta_gen(), temporary syndrome buffers, and the
|
||||
* This avoids temporary syndrome buffers and the
|
||||
* generation of unused parity rows.
|
||||
*
|
||||
* If P is available, preserve the complete P delta syndrome and
|
||||
|
|
|
|||
|
|
@ -612,7 +612,7 @@ static __always_inline void raid_genX_avx512bw(int nd, size_t size, void **vv, i
|
|||
* Recover multiple data failures using selected parity blocks with AVX512BW.
|
||||
*
|
||||
* Compute only the selected syndromes, keeping them in registers.
|
||||
* This avoids raid_delta_gen(), temporary syndrome buffers, recomputation of
|
||||
* This avoids temporary syndrome buffers, recomputation of
|
||||
* parity blocks, and generation of unused parity rows.
|
||||
*
|
||||
* AVX512 provides enough registers to keep all six supported syndromes
|
||||
|
|
|
|||
|
|
@ -1448,7 +1448,7 @@ static __always_inline void raid_genX_avx512gfni_aes(int nd, size_t size, void *
|
|||
* Recover multiple data failures using selected parity blocks with AVX2 GFNI.
|
||||
*
|
||||
* Compute only the selected syndromes, keeping them in registers.
|
||||
* This avoids raid_delta_gen(), temporary syndrome buffers, recomputation of
|
||||
* This avoids temporary syndrome buffers, recomputation of
|
||||
* parity blocks, and generation of unused parity rows.
|
||||
*
|
||||
* If recovering a single failure from P, the P delta directly contains
|
||||
|
|
@ -1663,7 +1663,7 @@ static __always_inline void raid_recX_avx2gfni_raid(int nr, int has_p, int *id,
|
|||
* Recover multiple data failures using selected parity blocks with AVX512 GFNI.
|
||||
*
|
||||
* Compute only the selected syndromes, keeping them in registers.
|
||||
* This avoids raid_delta_gen(), temporary syndrome buffers, recomputation of
|
||||
* This avoids temporary syndrome buffers, recomputation of
|
||||
* parity blocks, and generation of unused parity rows.
|
||||
*
|
||||
* If P is available, reconstruct only nr - 1 missing blocks through the
|
||||
|
|
@ -1841,7 +1841,7 @@ static __always_inline void raid_recX_avx512gfni_raid(int nr, int has_p, int *id
|
|||
* Recover multiple data failures for the AES polynomial using selected parity blocks with AVX2 GFNI.
|
||||
*
|
||||
* Compute only the selected syndromes, keeping them in registers.
|
||||
* This avoids raid_delta_gen(), temporary syndrome buffers, recomputation of
|
||||
* This avoids temporary syndrome buffers, recomputation of
|
||||
* parity blocks, and generation of unused parity rows.
|
||||
*
|
||||
* If recovering a single failure from P, the P delta directly contains
|
||||
|
|
@ -2046,7 +2046,7 @@ static __always_inline void raid_recX_avx2gfni_aes(int nr, int has_p, int *id, i
|
|||
* Recover multiple data failures for the AES polynomial using selected parity blocks with AVX512 GFNI.
|
||||
*
|
||||
* Compute only the selected syndromes, keeping them in registers.
|
||||
* This avoids raid_delta_gen(), temporary syndrome buffers, recomputation of
|
||||
* This avoids temporary syndrome buffers, recomputation of
|
||||
* parity blocks, and generation of unused parity rows.
|
||||
*
|
||||
* If P is available, reconstruct only nr - 1 missing blocks through the
|
||||
|
|
|
|||
129
raid/int.c
129
raid/int.c
|
|
@ -666,121 +666,6 @@ void raid_gen6_int8(int nd, size_t size, void **vv, int streaming)
|
|||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Recover failure of one data block at index id[0] using parity at index
|
||||
* ip[0] for any RAID level using 8bit C implementation.
|
||||
*
|
||||
* Starting from the equation:
|
||||
*
|
||||
* Pd = A[ip[0],id[0]] * Dx
|
||||
*
|
||||
* and solving we get:
|
||||
*
|
||||
* Dx = A[ip[0],id[0]]^-1 * Pd
|
||||
*/
|
||||
void raid_rec1_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
uint8_t *pa;
|
||||
const uint8_t *T;
|
||||
uint8_t G;
|
||||
uint8_t V;
|
||||
size_t i;
|
||||
|
||||
(void)nr; /* unused, it's always 1 */
|
||||
|
||||
/* if recovering with P uses the delta function */
|
||||
if (ip[0] == 0) {
|
||||
raid_rec1of1(id, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
/* setup the coefficients matrix */
|
||||
G = A(ip[0], id[0]);
|
||||
|
||||
/* invert it to solve the system of linear equations */
|
||||
V = inv(G);
|
||||
|
||||
/* get multiplication tables */
|
||||
T = table(V);
|
||||
|
||||
/* compute delta parity */
|
||||
raid_delta_gen(1, id, ip, nd, size, vv);
|
||||
|
||||
p = v[nd + ip[0]];
|
||||
pa = v[id[0]];
|
||||
|
||||
for (i = 0; i < size; ++i) {
|
||||
/* delta */
|
||||
uint8_t Pd = p[i] ^ pa[i];
|
||||
|
||||
/* reconstruct */
|
||||
pa[i] = T[Pd];
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Recover failure of two data blocks at indexes id[0],id[1] using parity at
|
||||
* indexes ip[0],ip[1] for any RAID level using 8bit C implementation.
|
||||
*
|
||||
* If P is available, reconstructs only the first missing block through the
|
||||
* inverse matrix and derives the second missing block from the P delta by XOR.
|
||||
*/
|
||||
static __always_inline void raid_rec2_int8_gen(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
uint8_t *q;
|
||||
uint8_t *pa;
|
||||
uint8_t *qa;
|
||||
const uint8_t *T[2][2];
|
||||
uint8_t G[2 * 2];
|
||||
uint8_t V[2 * 2];
|
||||
size_t i;
|
||||
int j, k;
|
||||
|
||||
(void)nr; /* unused, it's always 2 */
|
||||
|
||||
/* setup the coefficients matrix */
|
||||
for (j = 0; j < 2; ++j)
|
||||
for (k = 0; k < 2; ++k)
|
||||
G[j * 2 + k] = A(ip[j], id[k]);
|
||||
|
||||
/* invert it to solve the system of linear equations */
|
||||
raid_invert(G, V, 2);
|
||||
|
||||
/* get multiplication tables */
|
||||
for (j = 0; j < 2 - has_p; ++j)
|
||||
for (k = 0; k < 2; ++k)
|
||||
T[j][k] = table(V[j * 2 + k]);
|
||||
|
||||
/* compute delta parity */
|
||||
raid_delta_gen(2, id, ip, nd, size, vv);
|
||||
|
||||
p = v[nd + ip[0]];
|
||||
q = v[nd + ip[1]];
|
||||
pa = v[id[0]];
|
||||
qa = v[id[1]];
|
||||
|
||||
for (i = 0; i < size; ++i) {
|
||||
/* delta */
|
||||
uint8_t Pd = p[i] ^ pa[i];
|
||||
uint8_t Qd = q[i] ^ qa[i];
|
||||
|
||||
/* reconstruct */
|
||||
if (has_p) {
|
||||
uint8_t Dx = T[0][0][Pd] ^ T[0][1][Qd];
|
||||
|
||||
pa[i] = Dx;
|
||||
qa[i] = Pd ^ Dx;
|
||||
} else {
|
||||
pa[i] = T[0][0][Pd] ^ T[0][1][Qd];
|
||||
qa[i] = T[1][0][Pd] ^ T[1][1][Qd];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Recover failure of N data blocks at indexes id[N] using parity at indexes
|
||||
* ip[N] for any RAID level using 8bit C implementation.
|
||||
|
|
@ -1259,14 +1144,24 @@ void raid_gen2_int64_aes(int nd, size_t size, void **vv, int streaming)
|
|||
raid_gen2_int64_gen(nd, size, vv, 3);
|
||||
}
|
||||
|
||||
void raid_rec1_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 1);
|
||||
|
||||
if (ip[0] == 0)
|
||||
raid_rec1of1(id, nd, size, vv);
|
||||
else
|
||||
raid_recX_int8(1, 0, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
void raid_rec2_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 2);
|
||||
|
||||
if (ip[0] == 0)
|
||||
raid_rec2_int8_gen(2, 1, id, ip, nd, size, vv);
|
||||
raid_recX_int8(2, 1, id, ip, nd, size, vv);
|
||||
else
|
||||
raid_rec2_int8_gen(2, 0, id, ip, nd, size, vv);
|
||||
raid_recX_int8(2, 0, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
void raid_rec3_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
|
|
|
|||
|
|
@ -89,7 +89,6 @@ static __always_inline void * __align_ptr(void *ptr, uintptr_t size)
|
|||
*/
|
||||
void raid_gen_ref(int nd, int np, size_t size, void **vv);
|
||||
void raid_invert(uint8_t *M, uint8_t *V, int n);
|
||||
void raid_delta_gen(int nr, int *id, int *ip, int nd, size_t size, void **v);
|
||||
void raid_rec1of1(int *id, int nd, size_t size, void **v);
|
||||
void raid_gen1_int32(int nd, size_t size, void **vv, int streaming);
|
||||
void raid_gen1_int64(int nd, size_t size, void **vv, int streaming);
|
||||
|
|
@ -272,10 +271,6 @@ void raid_rec6_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
|||
*
|
||||
* All these functions give the guarantee that parities are written
|
||||
* in order. First parity P, then parity Q, and so on.
|
||||
* This allows to specify the same memory buffer for multiple parities
|
||||
* knowing that you'll get the latest written one.
|
||||
* This characteristic is used by the raid_delta_gen() function to
|
||||
* avoid damaging unused parities during recovery.
|
||||
*
|
||||
* @nd Number of data blocks
|
||||
* @size Size of the blocks pointed to by @vv. It must be a multiple of 64.
|
||||
|
|
|
|||
|
|
@ -922,7 +922,7 @@ static __always_inline void raid_recX_neon_12(int nr, int has_p, int *id, int *i
|
|||
/*
|
||||
* Recover multiple data failures using selected parity blocks with NEON.
|
||||
*
|
||||
* This avoids raid_delta_gen(), temporary syndrome buffers, and the
|
||||
* This avoids temporary syndrome buffers and the
|
||||
* generation of unused parity rows.
|
||||
*
|
||||
* If P is available, preserve the complete P delta syndrome and
|
||||
|
|
|
|||
107
raid/neon32.c
107
raid/neon32.c
|
|
@ -830,7 +830,7 @@ static __always_inline void raid_recX_neon32_123(int nr, int has_p, int *id, int
|
|||
/*
|
||||
* Recover multiple data failures using selected parity blocks with AArch32 NEON.
|
||||
*
|
||||
* This avoids raid_delta_gen(), temporary syndrome buffers, and the
|
||||
* This avoids temporary syndrome buffers and the
|
||||
* generation of unused parity rows.
|
||||
*
|
||||
* If P is available, preserve the complete P delta syndrome and
|
||||
|
|
@ -1241,115 +1241,10 @@ void raid_rec1_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|||
raid_recX_neon32_123(1, 0, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
/*
|
||||
* Recover failure of two data blocks using P and Q AArch32 NEON implementation.
|
||||
*
|
||||
* Uses raid_delta_gen() and computes the analytical solution with polynomial reduction.
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_neon32(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
uint8_t *pa;
|
||||
uint8_t *q;
|
||||
uint8_t *qa;
|
||||
uint8_t C[2];
|
||||
size_t i;
|
||||
|
||||
C[0] = inv(powgen(id[1] - id[0]) ^ 1);
|
||||
C[1] = inv(powgen(id[0]) ^ powgen(id[1]));
|
||||
|
||||
raid_delta_gen(2, id, ip, nd, size, vv);
|
||||
|
||||
p = v[nd];
|
||||
q = v[nd + 1];
|
||||
pa = v[id[0]];
|
||||
qa = v[id[1]];
|
||||
|
||||
raid_neon32_begin();
|
||||
|
||||
/* q15 = nibble mask, q9-q12 = C0/C1 low/high tables */
|
||||
asm volatile (
|
||||
"vmov.i8 q15, #0x0f\n"
|
||||
"vld1.8 {q9}, %0\n"
|
||||
"vld1.8 {q10}, %1\n"
|
||||
"vld1.8 {q11}, %2\n"
|
||||
"vld1.8 {q12}, %3\n"
|
||||
:
|
||||
: "Q" (raid_gfmulpshufb[C[0]][0][0]),
|
||||
"Q" (raid_gfmulpshufb[C[0]][1][0]),
|
||||
"Q" (raid_gfmulpshufb[C[1]][0][0]),
|
||||
"Q" (raid_gfmulpshufb[C[1]][1][0])
|
||||
);
|
||||
|
||||
for (i = 0; i < size; i += 16) {
|
||||
asm volatile (
|
||||
/* Pd */
|
||||
"vld1.8 {q0}, %2\n"
|
||||
"vld1.8 {q13}, %4\n"
|
||||
"veor q0, q0, q13\n"
|
||||
|
||||
/* Qd */
|
||||
"vld1.8 {q1}, %3\n"
|
||||
"vld1.8 {q13}, %5\n"
|
||||
"veor q1, q1, q13\n"
|
||||
|
||||
/* split Pd */
|
||||
"vmov q2, q0\n"
|
||||
"vshr.u8 q3, q0, #4\n"
|
||||
"vand q2, q2, q15\n"
|
||||
"vand q3, q3, q15\n"
|
||||
|
||||
/* split Qd */
|
||||
"vmov q4, q1\n"
|
||||
"vshr.u8 q5, q1, #4\n"
|
||||
"vand q4, q4, q15\n"
|
||||
"vand q5, q5, q15\n"
|
||||
|
||||
/* C0 low * Pd low -> q6 */
|
||||
"vtbl.8 d12, {d18-d19}, d4\n"
|
||||
"vtbl.8 d13, {d18-d19}, d5\n"
|
||||
|
||||
/* C0 high * Pd high -> q7 */
|
||||
"vtbl.8 d14, {d20-d21}, d6\n"
|
||||
"vtbl.8 d15, {d20-d21}, d7\n"
|
||||
"veor q6, q6, q7\n"
|
||||
|
||||
/* C1 low * Qd low -> q7 */
|
||||
"vtbl.8 d14, {d22-d23}, d8\n"
|
||||
"vtbl.8 d15, {d22-d23}, d9\n"
|
||||
|
||||
/* C1 high * Qd high -> q8 */
|
||||
"vtbl.8 d16, {d24-d25}, d10\n"
|
||||
"vtbl.8 d17, {d24-d25}, d11\n"
|
||||
"veor q7, q7, q8\n"
|
||||
|
||||
/* Dy */
|
||||
"veor q6, q6, q7\n"
|
||||
|
||||
/* Dx = Pd ^ Dy */
|
||||
"veor q0, q0, q6\n"
|
||||
|
||||
"vst1.8 {q0}, %0\n"
|
||||
"vst1.8 {q6}, %1\n"
|
||||
: "=Q" (pa[i]), "=Q" (qa[i])
|
||||
: "Q" (p[i]), "Q" (q[i]),
|
||||
"Q" (pa[i]), "Q" (qa[i])
|
||||
);
|
||||
}
|
||||
|
||||
raid_neon32_end();
|
||||
}
|
||||
|
||||
void raid_rec2_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 2);
|
||||
|
||||
if (ip[0] == 0 && ip[1] == 1) {
|
||||
raid_rec2of2_neon32(id, ip, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
if (ip[0] == 0)
|
||||
raid_recX_neon32_123(2, 1, id, ip, nd, size, vv);
|
||||
else
|
||||
|
|
|
|||
98
raid/raid.c
98
raid/raid.c
|
|
@ -279,104 +279,6 @@ void raid_invert(uint8_t *M, uint8_t *V, int n)
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the parity without the missing data blocks
|
||||
* and store it in the buffers of such data blocks.
|
||||
*
|
||||
* This is the parity expressed as Pa,Qa,Ra,Sa,Ta,Ua in the equations.
|
||||
*/
|
||||
void raid_delta_gen(int nr, int *id, int *ip, int nd, size_t size, void **v)
|
||||
{
|
||||
void *p[RAID_PARITY_MAX];
|
||||
void *pa[RAID_PARITY_MAX];
|
||||
int i, j;
|
||||
int np;
|
||||
void *latest;
|
||||
|
||||
/* zero buffer must be initialized prior to delta recovery */
|
||||
BUG_ON(raid_zero_block == 0);
|
||||
|
||||
/*
|
||||
* Total number of parities we are going to process
|
||||
* they are both the used and the unused ones
|
||||
*/
|
||||
np = ip[nr - 1] + 1;
|
||||
|
||||
/* latest missing data block */
|
||||
latest = v[id[nr - 1]];
|
||||
|
||||
/* setup pointers for delta computation */
|
||||
for (i = 0, j = 0; i < np; ++i) {
|
||||
/* keep a copy of the original parity vector */
|
||||
p[i] = v[nd + i];
|
||||
|
||||
if (ip[j] == i) {
|
||||
/*
|
||||
* Set used parities to point to the missing
|
||||
* data blocks.
|
||||
*
|
||||
* The related data blocks are instead set
|
||||
* to point to the "zero" buffer.
|
||||
*/
|
||||
|
||||
/*
|
||||
* The latest parity to use ends the for loop and
|
||||
* then it cannot happen to process more of them
|
||||
*/
|
||||
BUG_ON(j >= nr);
|
||||
|
||||
/* buffer for missing data blocks */
|
||||
pa[j] = v[id[j]];
|
||||
|
||||
/* set at zero the missing data blocks */
|
||||
v[id[j]] = raid_zero_block;
|
||||
|
||||
/* compute the parity over the missing data blocks */
|
||||
v[nd + i] = pa[j];
|
||||
|
||||
/* check for the next used entry */
|
||||
++j;
|
||||
} else {
|
||||
/*
|
||||
* Unused parities are going to be rewritten with
|
||||
* not significant data, because we don't have
|
||||
* functions able to compute only a subset of
|
||||
* parities.
|
||||
*
|
||||
* To avoid this, we reuse parity buffers,
|
||||
* assuming that all the parity functions write
|
||||
* parities in order.
|
||||
*
|
||||
* We assign the unused parity block to the same
|
||||
* block of the latest used parity that we know it
|
||||
* will be written.
|
||||
*
|
||||
* This means that this block will be written
|
||||
* multiple times and only the latest write will
|
||||
* contain the correct data.
|
||||
*/
|
||||
v[nd + i] = latest;
|
||||
}
|
||||
}
|
||||
|
||||
/* all the parities have to be processed */
|
||||
BUG_ON(j != nr);
|
||||
|
||||
/*
|
||||
* Recompute the parity, note that np may be smaller than the
|
||||
* total number of parities available
|
||||
*/
|
||||
raid_gen(nd, np, size, v, 0);
|
||||
|
||||
/* restore data buffers as before */
|
||||
for (j = 0; j < nr; ++j)
|
||||
v[id[j]] = pa[j];
|
||||
|
||||
/* restore parity buffers as before */
|
||||
for (i = 0; i < np; ++i)
|
||||
v[nd + i] = p[i];
|
||||
}
|
||||
|
||||
/**
|
||||
* Recover failure of one data block for PAR1.
|
||||
*
|
||||
|
|
|
|||
20
raid/ssse3.c
20
raid/ssse3.c
|
|
@ -1250,7 +1250,7 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i
|
|||
* Recover failure of one data block using selected parity with SSSE3, optimized for one failure.
|
||||
*
|
||||
* Computes the selected syndrome in a single survivor scan and reconstructs
|
||||
* the missing block directly, avoiding raid_delta_gen() and the extra pass.
|
||||
* the missing block directly, avoiding extra memory passes.
|
||||
*/
|
||||
static __always_inline void raid_rec1_ssse3_1(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -1338,8 +1338,8 @@ static __always_inline void raid_rec1_ssse3_1(int *id, int *ip, int nd, size_t s
|
|||
* Computes Q of all surviving data directly with Horner's method and
|
||||
* reconstructs the missing block from Qdelta.
|
||||
*
|
||||
* Processes two 16-byte lanes per iteration, avoiding raid_delta_gen(),
|
||||
* temporary parity buffers, and the extra pass over the data.
|
||||
* Processes two 16-byte lanes per iteration, avoiding temporary parity
|
||||
* buffers and the extra pass over the data.
|
||||
*/
|
||||
static __always_inline void raid_rec1_ssse3_q(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -1472,8 +1472,8 @@ static __always_inline void raid_rec1_ssse3_q(int *id, int *ip, int nd, size_t s
|
|||
* optimized specifically for two failures.
|
||||
*
|
||||
* Computes only the two selected syndromes in a single survivor scan and
|
||||
* reconstructs the missing blocks directly, avoiding raid_delta_gen(),
|
||||
* temporary syndrome buffers, and the generation of unused parity rows.
|
||||
* reconstructs the missing blocks directly, avoiding temporary syndrome
|
||||
* buffers and the generation of unused parity rows.
|
||||
*
|
||||
* has_p is expected to be a compile-time constant after inlining.
|
||||
*
|
||||
|
|
@ -1661,8 +1661,8 @@ static __always_inline void raid_rec2_ssse3_2(int has_p, int *id, int *ip, int n
|
|||
* Computes Pdelta and Qdelta directly in a single survivor scan.
|
||||
* Q is computed with Horner's method using the active field generator.
|
||||
*
|
||||
* Processes two 16-byte lanes per iteration, avoiding raid_delta_gen(),
|
||||
* temporary parity buffers, and the second pass over the data.
|
||||
* Processes two 16-byte lanes per iteration, avoiding temporary parity
|
||||
* buffers and the second pass over the data.
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_ssse3(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -2395,8 +2395,8 @@ static __always_inline void raid_rec1_ssse3ext_1(int *id, int *ip, int nd, size_
|
|||
* Computes Q of all surviving data directly with Horner's method and
|
||||
* reconstructs the missing block from Qdelta.
|
||||
*
|
||||
* Processes four 16-byte lanes per iteration, avoiding raid_delta_gen(),
|
||||
* temporary parity buffers, and the extra pass over the data.
|
||||
* Processes four 16-byte lanes per iteration, avoiding temporary parity
|
||||
* buffers and the extra pass over the data.
|
||||
*/
|
||||
static __always_inline void raid_rec1_ssse3ext_q(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -3799,7 +3799,7 @@ static __always_inline void raid_recX_ssse3ext_12345(int nr, int has_p, int *id,
|
|||
/*
|
||||
* Recover multiple data failures using selected parity blocks with SSSE3 extended.
|
||||
*
|
||||
* This avoids raid_delta_gen(), temporary syndrome buffers, and the
|
||||
* This avoids temporary syndrome buffers and the
|
||||
* generation of unused parity rows.
|
||||
*
|
||||
* If P is available, preserve the complete P delta syndrome and
|
||||
|
|
|
|||
|
|
@ -1263,8 +1263,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
fflush(stdout);
|
||||
|
||||
SPEED_START {
|
||||
/* force the parity generator used by raid_delta_gen() */
|
||||
raid_gen_force(2, sizeof(void *) == 8 ? raid_gen2_int64_raid : raid_gen2_int32_raid);
|
||||
/* +1 to avoid GEN1 optimized case */
|
||||
raid_rec1_int8(1, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
|
@ -1367,8 +1365,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
fflush(stdout);
|
||||
|
||||
SPEED_START {
|
||||
/* force the parity generator used by raid_delta_gen() */
|
||||
raid_gen_force(3, raid_gen3_int8);
|
||||
/* +2 to select R instead of Q */
|
||||
raid_rec1_int8(1, id, ip + 2, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
|
@ -1471,8 +1467,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
fflush(stdout);
|
||||
|
||||
SPEED_START {
|
||||
/* force the parity generator used by raid_delta_gen() */
|
||||
raid_gen_force(2, sizeof(void *) == 8 ? raid_gen2_int64_raid : raid_gen2_int32_raid);
|
||||
raid_rec2_int8(2, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
|
|
@ -1489,8 +1483,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
#endif
|
||||
#ifdef CONFIG_NEON32
|
||||
SPEED_START {
|
||||
/* force the parity generator used by raid_delta_gen() */
|
||||
raid_gen_force(2, raid_gen2_neon32_raid);
|
||||
raid_rec2_neon32(2, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
|
|
@ -1501,8 +1493,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
#ifdef CONFIG_X86
|
||||
if (raid_cpu_has_ssse3()) {
|
||||
SPEED_START {
|
||||
/* force the parity generator used by raid_delta_gen() */
|
||||
raid_gen_force(2, raid_gen2_sse2_raid);
|
||||
raid_rec2_ssse3(2, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
|
|
@ -1511,8 +1501,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
|
||||
#ifdef CONFIG_X86_64
|
||||
SPEED_START {
|
||||
/* force the parity generator used by raid_delta_gen() */
|
||||
raid_gen_force(2, raid_gen2_sse2ext_raid);
|
||||
raid_rec2_ssse3ext(2, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
|
|
@ -1571,8 +1559,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
fflush(stdout);
|
||||
|
||||
SPEED_START {
|
||||
/* force the parity generator used by raid_delta_gen() */
|
||||
raid_gen_force(3, raid_gen3_int8);
|
||||
/* +1 to avoid GEN2 optimized case */
|
||||
raid_rec2_int8(2, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
|
@ -1675,8 +1661,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
fflush(stdout);
|
||||
|
||||
SPEED_START {
|
||||
/* force the parity generator used by raid_delta_gen() */
|
||||
raid_gen_force(4, raid_gen4_int8);
|
||||
/* +2 to select R,S instead of Q,R */
|
||||
raid_rec2_int8(2, id, ip + 2, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
|
@ -1779,7 +1763,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
fflush(stdout);
|
||||
|
||||
SPEED_START {
|
||||
raid_gen_force(3, raid_gen3_int8);
|
||||
raid_rec3_int8(3, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
|
|
@ -1872,7 +1855,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
fflush(stdout);
|
||||
|
||||
SPEED_START {
|
||||
raid_gen_force(4, raid_gen4_int8);
|
||||
raid_rec4_int8(4, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
|
|
@ -1965,7 +1947,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
fflush(stdout);
|
||||
|
||||
SPEED_START {
|
||||
raid_gen_force(5, raid_gen5_int8);
|
||||
raid_rec5_int8(5, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
|
|
@ -2058,7 +2039,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
fflush(stdout);
|
||||
|
||||
SPEED_START {
|
||||
raid_gen_force(6, raid_gen6_int8);
|
||||
raid_rec6_int8(6, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue