diff --git a/raid/avx2.c b/raid/avx2.c index 3354afa0..7c890271 100644 --- a/raid/avx2.c +++ b/raid/avx2.c @@ -7,7 +7,9 @@ #ifdef CONFIG_X86 /* - * GEN1 (RAID5 with xor) AVX2 implementation + * Generate one parity block (RAID5 with XOR) using AVX2 implementation. + * + * Uses 64-byte chunks across two 32-byte YMM lanes. */ void raid_gen1_avx2(int nd, size_t size, void **vv) { @@ -38,7 +40,9 @@ void raid_gen1_avx2(int nd, size_t size, void **vv) } /* - * GEN2 Cauchy AVX2 implementation using the active generator + * Generate two parity blocks (RAID6 with Cauchy matrix) using AVX2 implementation. + * + * Uses Horner's method with 64-byte chunks across two 32-byte YMM lanes. */ static __always_inline void raid_gen2_avx2_gen(int nd, size_t size, void **vv, int generator) { @@ -107,7 +111,7 @@ static __always_inline void raid_gen2_avx2_gen(int nd, size_t size, void **vv, i #ifdef CONFIG_X86_64 /* - * GEN2 Cauchy AVX2 implementation using the extended register set. + * Generate two parity blocks (RAID6 with Cauchy matrix) using AVX2 extended implementation. * * Process two data disks at a time and process the two 32-byte lanes * sequentially. @@ -221,7 +225,7 @@ static __always_inline void raid_gen2_avx2ext_gen(int nd, size_t size, void **vv } /* - * GENz (triple parity with powers of 2^-1) AVX2 implementation + * Generate three parity blocks with powers of 2^-1 using AVX2 extended implementation. * * Note that it uses 16 registers, meaning that x64 is required. */ @@ -296,7 +300,10 @@ void raid_genz_avx2ext_raid(int nd, size_t size, void **vv) } /* - * GEN3 (triple parity with Cauchy matrix) AVX2 implementation + * Generate three parity blocks with Cauchy matrix using AVX2 extended implementation. + * + * Uses the extended register set (16 YMM registers) and processes two disks + * per iteration across two 32-byte YMM lanes (64 bytes/step). * * Note that it uses 16 registers, meaning that x64 is required. */ @@ -552,7 +559,10 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv } /* - * GEN4 (quad parity with Cauchy matrix) AVX2 implementation + * Generate four parity blocks with Cauchy matrix using AVX2 extended implementation. + * + * Uses the extended register set (16 YMM registers) and processes two disks + * per iteration across two 32-byte YMM lanes (64 bytes/step). * * Note that it uses 16 registers, meaning that x64 is required. */ @@ -725,7 +735,7 @@ static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv } /* - * GENX AVX2EXT implementation + * Generate N parity blocks with Cauchy matrix using AVX2 extended implementation. */ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, int np, int generator) { @@ -883,7 +893,9 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in #endif /* - * RAID recovering for one disk AVX2 implementation + * Recover failure of one data block using AVX2 delta implementation. + * + * Uses raid_delta_gen() and multiplies by the inverted coefficient table. */ static __always_inline void raid_rec1_avx2_delta(int *id, int *ip, int nd, size_t size, void **vv) { @@ -945,7 +957,9 @@ static __always_inline void raid_rec1_avx2_delta(int *id, int *ip, int nd, size_ } /* - * RAID recovering for two disks AVX2 implementation + * Recover failure of two data blocks using AVX2 delta implementation. + * + * Uses raid_delta_gen() and solves the 2x2 system directly on delta blocks. */ static __always_inline void raid_rec2_avx2_delta(int *id, int *ip, int nd, size_t size, void **vv) { @@ -1039,6 +1053,8 @@ static __always_inline void raid_rec2_avx2_delta(int *id, int *ip, int nd, size_ /* * Recover failure of two data blocks using P and Q AVX2 implementation. + * + * Uses raid_delta_gen() and computes the analytical solution. */ static __always_inline void raid_rec2of2_avx2(int *id, int *ip, int nd, size_t size, void **vv) { @@ -1111,7 +1127,7 @@ static __always_inline void raid_rec2of2_avx2(int *id, int *ip, int nd, size_t s } /* - * RAID recovering AVX2 implementation optimized for up to four failures. + * Recover multiple data failures using selected parity blocks with AVX2 optimized for up to four failures. * * Compute all selected syndromes in one scan of the surviving data, then * retain only their low/high nibbles while reconstructing the missing data. @@ -1320,7 +1336,7 @@ static __always_inline void raid_recX_avx2_1234(int nr, int *id, int *ip, int nd } /* - * RAID recovering AVX2 implementation with all syndromes in memory. + * Recover multiple data failures using selected parity blocks with AVX2 and all syndromes in memory. * * Compute all selected syndromes in a single pass over the surviving data. * Syndrome accumulators are kept in memory to minimize register pressure. @@ -1544,7 +1560,10 @@ static __always_inline void raid_recX_avx2(int nr, int *id, int *ip, int nd, siz #ifdef CONFIG_X86_64 /* - * RAID recovering for two disks with P and Q AVX2 implementation + * Recover failure of two data blocks using P and Q AVX2 extended implementation. + * + * Uses raid_delta_gen() and keeps all four multiplication tables resident + * in YMM registers (ymm8..ymm11). */ static __always_inline void raid_rec2of2_avx2ext(int *id, int *ip, int nd, size_t size, void **vv) { @@ -1616,7 +1635,7 @@ static __always_inline void raid_rec2of2_avx2ext(int *id, int *ip, int nd, size_ } /* - * RAID recovering AVX2 implementation optimized for up to three failures. + * Recover multiple data failures using selected parity blocks with AVX2 extended optimized for up to three failures. * * Process 64 bytes at a time as two independent 32-byte lanes. * Multiplication tables are broadcast once and shared between both lanes. @@ -1919,7 +1938,7 @@ static __always_inline void raid_recX_avx2ext_123(int nr, int *id, int *ip, int } /* - * RAID recovering AVX2 extended implementation. + * Recover multiple data failures using selected parity blocks with AVX2 extended. * * Compute only the selected syndromes, keeping them in registers. * This avoids raid_delta_gen(), temporary syndrome buffers, and the @@ -2300,11 +2319,6 @@ void raid_gen4_avx2ext_aes(int nd, size_t size, void **vv) raid_gen4_avx2ext_gen(nd, size, vv, 3); } -/* - * GEN5 (penta parity with Cauchy matrix) AVX2 implementation - * - * Note that it uses 16 registers, meaning that x64 is required. - */ void raid_gen5_avx2ext_raid(int nd, size_t size, void **vv) { raid_genX_avx2ext(nd, size, vv, 5, 2); @@ -2315,11 +2329,6 @@ void raid_gen5_avx2ext_aes(int nd, size_t size, void **vv) raid_genX_avx2ext(nd, size, vv, 5, 3); } -/* - * GEN6 (hexa parity with Cauchy matrix) AVX2 implementation - * - * Note that it uses 16 registers, meaning that x64 is required. - */ void raid_gen6_avx2ext_raid(int nd, size_t size, void **vv) { raid_genX_avx2ext(nd, size, vv, 6, 2); diff --git a/raid/avx512.c b/raid/avx512.c index d23f67d6..fdfa84fd 100644 --- a/raid/avx512.c +++ b/raid/avx512.c @@ -7,7 +7,9 @@ #ifdef CONFIG_X86_64 /* - * GEN1 (RAID5 with xor) AVX512BW implementation + * Generate one parity block (RAID5 with XOR) using AVX512BW implementation. + * + * Processes two disks per iteration using vpternlogq over 64-byte blocks. * * Note that in true AVX512F would suffice, but we don't want to add * specific support for AVX512F because this would be the only function @@ -48,7 +50,10 @@ void raid_gen1_avx512bw(int nd, size_t size, void **vv) } /* - * GEN2 Cauchy AVX512BW implementation + * Generate two parity blocks (RAID6 with Cauchy matrix) using AVX512BW implementation. + * + * Preloads 13 pairs of Q coefficient tables in ZMM registers (zmm4..zmm29) + * with an unrolled disk loop and early exits. */ void raid_gen2_avx512bw(int nd, size_t size, void **vv) { @@ -274,7 +279,10 @@ store: } /* - * GEN3 (triple parity with Cauchy matrix) AVX512BW implementation + * Generate three parity blocks with Cauchy matrix using AVX512BW implementation. + * + * Preloads 6 sets of Q and R coefficient tables in ZMM registers (zmm7..zmm30) + * with an unrolled disk loop and early exits. */ void raid_gen3_avx512bw(int nd, size_t size, void **vv) { @@ -445,7 +453,7 @@ store: } /* - * GENX AVX512BW implementation + * Generate N parity blocks with Cauchy matrix using AVX512BW implementation. */ static __always_inline void raid_genX_avx512bw(int nd, size_t size, void **vv, int np) { @@ -885,25 +893,16 @@ static __always_inline void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, raid_avx_end(); } -/* - * GEN4 (quad parity with Cauchy matrix) AVX512BW implementation - */ void raid_gen4_avx512bw(int nd, size_t size, void **vv) { raid_genX_avx512bw(nd, size, vv, 4); } -/* - * GEN5 (penta parity with Cauchy matrix) AVX512BW implementation - */ void raid_gen5_avx512bw(int nd, size_t size, void **vv) { raid_genX_avx512bw(nd, size, vv, 5); } -/* - * GEN6 (hexa parity with Cauchy matrix) AVX512BW implementation - */ void raid_gen6_avx512bw(int nd, size_t size, void **vv) { raid_genX_avx512bw(nd, size, vv, 6); @@ -924,6 +923,9 @@ void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv /* * Recover failure of two data blocks using P and Q AVX512BW implementation. + * + * Uses raid_delta_gen() and keeps both multiplication-table pairs resident + * in ZMM registers (zmm16..zmm19). */ static __always_inline void raid_rec2of2_avx512bw(int *id, int *ip, int nd, size_t size, void **vv) { diff --git a/raid/gfni.c b/raid/gfni.c index 03766e97..319ca2ab 100644 --- a/raid/gfni.c +++ b/raid/gfni.c @@ -7,8 +7,7 @@ #ifdef CONFIG_X86_64 /* - * GENX AVX2 GFNI implementation for the RAID polynomial using direct - * Cauchy coefficients. + * Generate N parity blocks for the RAID polynomial using direct Cauchy coefficients with AVX2 GFNI. */ static __always_inline void raid_genX_avx2gfni_raid(int nd, size_t size, void **vv, int np) { @@ -119,7 +118,10 @@ static __always_inline void raid_genX_avx2gfni_raid(int nd, size_t size, void ** } /* - * GEN2 (RAID6 with powers of 2) AVX512 GFNI implementation + * Generate two parity blocks (RAID6 with powers of 2) using AVX512 GFNI implementation. + * + * Preloads up to 28 Q affine matrix coefficients in ZMM registers (zmm4..zmm31) + * with an unrolled disk loop and early exits. */ static __always_inline void raid_gen2_avx512gfni_raid_gen(int nd, size_t size, void **vv) { @@ -388,7 +390,10 @@ store: } /* - * GEN3 (triple parity with Cauchy matrix) AVX512 GFNI implementation + * Generate three parity blocks with Cauchy matrix using AVX512 GFNI implementation. + * + * Preloads up to 13 pairs of Q and R affine matrix coefficients in ZMM + * registers (zmm6..zmm31) with an unrolled disk loop and early exits. */ static __always_inline void raid_gen3_avx512gfni_raid_gen(int nd, size_t size, void **vv) { @@ -585,8 +590,7 @@ store: } /* - * GENX AVX512 GFNI implementation for the RAID polynomial using direct - * Cauchy coefficients. + * Generate N parity blocks for the RAID polynomial using direct Cauchy coefficients with AVX512 GFNI. */ static __always_inline void raid_genX_avx512gfni_raid(int nd, size_t size, void **vv, int np) { @@ -669,8 +673,7 @@ static __always_inline void raid_genX_avx512gfni_raid(int nd, size_t size, void } /* - * GENX AVX2 GFNI implementation for the AES polynomial using direct - * Cauchy coefficients. + * Generate N parity blocks for the AES polynomial using direct Cauchy coefficients with AVX2 GFNI. */ static __always_inline void raid_genX_avx2gfni_aes(int nd, size_t size, void **vv, int np) { @@ -788,7 +791,10 @@ static __always_inline void raid_genX_avx2gfni_aes(int nd, size_t size, void **v } /* - * GEN2 (RAID6 with powers of 3) AVX512 GFNI implementation + * Generate two parity blocks (RAID6 with powers of 3) using AVX512 GFNI implementation. + * + * Preloads up to 28 Q multiplier coefficients in ZMM registers (zmm4..zmm31) + * with an unrolled disk loop and early exits. */ static __always_inline void raid_gen2_avx512gfni_aes_gen(int nd, size_t size, void **vv) { @@ -1057,7 +1063,10 @@ store: } /* - * GEN3 (triple parity with Cauchy matrix) AVX512 GFNI implementation + * Generate three parity blocks with Cauchy matrix using AVX512 GFNI implementation. + * + * Preloads up to 13 pairs of Q and R multiplier coefficients in ZMM + * registers (zmm6..zmm31) with an unrolled disk loop and early exits. */ static __always_inline void raid_gen3_avx512gfni_aes_gen(int nd, size_t size, void **vv) { @@ -1254,8 +1263,7 @@ store: } /* - * GENX AVX512 GFNI implementation for the AES polynomial using direct - * Cauchy coefficients. + * Generate N parity blocks for the AES polynomial using direct Cauchy coefficients with AVX512 GFNI. */ static __always_inline void raid_genX_avx512gfni_aes(int nd, size_t size, void **vv, int np) { @@ -1337,6 +1345,16 @@ static __always_inline void raid_genX_avx512gfni_aes(int nd, size_t size, void * raid_avx_end(); } +/* + * Recover multiple data failures using selected parity blocks with AVX2 GFNI. + * + * Compute only the selected syndromes, keeping them in registers. + * This avoids raid_delta_gen(), temporary syndrome buffers, recomputation of + * parity blocks, and generation of unused parity rows. + * + * If P is available, reconstruct only nr - 1 missing blocks through the + * inverse matrix and derive the last missing block from the P delta by XOR. + */ static __always_inline void raid_recX_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; @@ -1716,6 +1734,16 @@ static __always_inline void raid_recX_avx512gfni_raid(int nr, int *id, int *ip, raid_avx_end(); } +/* + * Recover multiple data failures for the AES polynomial using selected parity blocks with AVX2 GFNI. + * + * Compute only the selected syndromes, keeping them in registers. + * This avoids raid_delta_gen(), temporary syndrome buffers, recomputation of + * parity blocks, and generation of unused parity rows. + * + * If P is available, reconstruct only nr - 1 missing blocks through the + * inverse matrix and derive the last missing block from the P delta by XOR. + */ static __always_inline void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; @@ -1906,8 +1934,7 @@ static __always_inline void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int } /* - * Recover multiple data failures using selected parity blocks with AVX512 GFNI - * and the AES polynomial. + * Recover multiple data failures for the AES polynomial using selected parity blocks with AVX512 GFNI. * * Compute only the selected syndromes, keeping them in registers. * This avoids raid_delta_gen(), temporary syndrome buffers, recomputation of @@ -2073,161 +2100,101 @@ static __always_inline void raid_recX_avx512gfni_aes(int nr, int *id, int *ip, i raid_avx_end(); } -/* - * GEN2 (RAID6 with powers of 2) AVX2 GFNI implementation - */ void raid_gen2_avx2gfni_raid(int nd, size_t size, void **vv) { raid_genX_avx2gfni_raid(nd, size, vv, 2); } -/* - * GEN2 (RAID6 with powers of 2) AVX512 GFNI implementation - */ void raid_gen2_avx512gfni_raid(int nd, size_t size, void **vv) { raid_gen2_avx512gfni_raid_gen(nd, size, vv); } -/* - * GEN3 (triple parity with Cauchy matrix) AVX2 GFNI implementation - */ void raid_gen3_avx2gfni_raid(int nd, size_t size, void **vv) { raid_genX_avx2gfni_raid(nd, size, vv, 3); } -/* - * GEN3 (triple parity with Cauchy matrix) AVX512 GFNI implementation - */ void raid_gen3_avx512gfni_raid(int nd, size_t size, void **vv) { raid_gen3_avx512gfni_raid_gen(nd, size, vv); } -/* - * GEN4 (quad parity with Cauchy matrix) AVX2 GFNI implementation - */ void raid_gen4_avx2gfni_raid(int nd, size_t size, void **vv) { raid_genX_avx2gfni_raid(nd, size, vv, 4); } -/* - * GEN4 (quad parity with Cauchy matrix) GFNI implementation - */ void raid_gen4_avx512gfni_raid(int nd, size_t size, void **vv) { raid_genX_avx512gfni_raid(nd, size, vv, 4); } -/* - * GEN5 (penta parity with Cauchy matrix) AVX2 GFNI implementation - */ void raid_gen5_avx2gfni_raid(int nd, size_t size, void **vv) { raid_genX_avx2gfni_raid(nd, size, vv, 5); } -/* - * GEN5 (penta parity with Cauchy matrix) GFNI implementation - */ void raid_gen5_avx512gfni_raid(int nd, size_t size, void **vv) { raid_genX_avx512gfni_raid(nd, size, vv, 5); } -/* - * GEN6 (hexa parity with Cauchy matrix) AVX2 GFNI implementation - */ void raid_gen6_avx2gfni_raid(int nd, size_t size, void **vv) { raid_genX_avx2gfni_raid(nd, size, vv, 6); } -/* - * GEN6 (hexa parity with Cauchy matrix) GFNI implementation - */ void raid_gen6_avx512gfni_raid(int nd, size_t size, void **vv) { raid_genX_avx512gfni_raid(nd, size, vv, 6); } -/* - * GEN2 (RAID6 with powers of 3) AVX2 GFNI implementation - */ void raid_gen2_avx2gfni_aes(int nd, size_t size, void **vv) { raid_genX_avx2gfni_aes(nd, size, vv, 2); } -/* - * GEN2 (RAID6 with powers of 3) AVX512 GFNI implementation - */ void raid_gen2_avx512gfni_aes(int nd, size_t size, void **vv) { raid_gen2_avx512gfni_aes_gen(nd, size, vv); } -/* - * GEN3 (triple parity with Cauchy matrix) AVX2 GFNI implementation - */ void raid_gen3_avx2gfni_aes(int nd, size_t size, void **vv) { raid_genX_avx2gfni_aes(nd, size, vv, 3); } -/* - * GEN3 (triple parity with Cauchy matrix) AVX512 GFNI implementation - */ void raid_gen3_avx512gfni_aes(int nd, size_t size, void **vv) { raid_gen3_avx512gfni_aes_gen(nd, size, vv); } -/* - * GEN4 (quad parity with Cauchy matrix) AVX2 GFNI implementation - */ void raid_gen4_avx2gfni_aes(int nd, size_t size, void **vv) { raid_genX_avx2gfni_aes(nd, size, vv, 4); } -/* - * GEN4 (quad parity with Cauchy matrix) GFNI implementation - */ void raid_gen4_avx512gfni_aes(int nd, size_t size, void **vv) { raid_genX_avx512gfni_aes(nd, size, vv, 4); } -/* - * GEN5 (penta parity with Cauchy matrix) AVX2 GFNI implementation - */ void raid_gen5_avx2gfni_aes(int nd, size_t size, void **vv) { raid_genX_avx2gfni_aes(nd, size, vv, 5); } -/* - * GEN5 (penta parity with Cauchy matrix) GFNI implementation - */ void raid_gen5_avx512gfni_aes(int nd, size_t size, void **vv) { raid_genX_avx512gfni_aes(nd, size, vv, 5); } -/* - * GEN6 (hexa parity with Cauchy matrix) AVX2 GFNI implementation - */ void raid_gen6_avx2gfni_aes(int nd, size_t size, void **vv) { raid_genX_avx2gfni_aes(nd, size, vv, 6); } -/* - * GEN6 (hexa parity with Cauchy matrix) GFNI implementation - */ void raid_gen6_avx512gfni_aes(int nd, size_t size, void **vv) { raid_genX_avx512gfni_aes(nd, size, vv, 6); @@ -2248,6 +2215,8 @@ void raid_rec1_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void /* * Recover failure of two data blocks using P and Q AVX2 RAID GFNI implementation. + * + * Uses raid_delta_gen() and keeps both affine matrices resident in YMM registers. */ static __always_inline void raid_rec2of2_avx2gfni_raid(int *id, int *ip, int nd, size_t size, void **vv) { @@ -2364,6 +2333,8 @@ void raid_rec1_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, vo /* * Recover failure of two data blocks using P and Q AVX512 RAID GFNI implementation. + * + * Uses raid_delta_gen() and keeps both affine matrices resident in ZMM registers. */ static __always_inline void raid_rec2of2_avx512gfni_raid(int *id, int *ip, int nd, size_t size, void **vv) { @@ -2466,6 +2437,8 @@ void raid_rec1_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void /* * Recover failure of two data blocks using P and Q AVX2 AES GFNI implementation. + * + * Uses raid_delta_gen() and keeps both byte multipliers resident in YMM registers. */ static __always_inline void raid_rec2of2_avx2gfni_aes(int *id, int *ip, int nd, size_t size, void **vv) { @@ -2573,6 +2546,8 @@ void raid_rec1_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, voi /* * Recover failure of two data blocks using P and Q AVX512 AES GFNI implementation. + * + * Uses raid_delta_gen() and keeps both byte multipliers resident in ZMM registers. */ static __always_inline void raid_rec2of2_avx512gfni_aes(int *id, int *ip, int nd, size_t size, void **vv) { diff --git a/raid/int.c b/raid/int.c index 7c27e436..62d7fbb7 100644 --- a/raid/int.c +++ b/raid/int.c @@ -5,7 +5,7 @@ #include "gf.h" /* - * GEN1 (RAID5 with xor) 32bit C implementation + * Generate one parity block (RAID5 with XOR) using 32bit C implementation. */ void raid_gen1_int32(int nd, size_t size, void **vv) { @@ -33,7 +33,7 @@ void raid_gen1_int32(int nd, size_t size, void **vv) } /* - * GEN1 (RAID5 with xor) 64bit C implementation + * Generate one parity block (RAID5 with XOR) using 64bit C implementation. */ void raid_gen1_int64(int nd, size_t size, void **vv) { @@ -61,7 +61,7 @@ void raid_gen1_int64(int nd, size_t size, void **vv) } /* - * GEN2 (double parity) 32bit C implementation + * Generate two parity blocks (RAID6 with powers of 2 or 3) using 32bit C implementation. */ static __always_inline void raid_gen2_int32_gen(int nd, size_t size, void **vv, int generator) { @@ -121,7 +121,7 @@ void raid_gen2_int32_aes(int nd, size_t size, void **vv) } /* - * GEN2 (double parity) 64bit C implementation + * Generate two parity blocks (RAID6 with powers of 2 or 3) using 64bit C implementation. */ static __always_inline void raid_gen2_int64_gen(int nd, size_t size, void **vv, int generator) { @@ -186,7 +186,7 @@ void raid_gen2_int8(int nd, size_t size, void **vv) } /* - * GENz (triple parity with powers of 2^-1) 32bit C implementation + * Generate three parity blocks with powers of 2^-1 using 32bit C implementation. */ void raid_genz_int32_raid(int nd, size_t size, void **vv) { @@ -239,7 +239,7 @@ void raid_genz_int32_raid(int nd, size_t size, void **vv) } /* - * GENz (triple parity with powers of 2^-1) 64bit C implementation + * Generate three parity blocks with powers of 2^-1 using 64bit C implementation. */ void raid_genz_int64_raid(int nd, size_t size, void **vv) { @@ -292,7 +292,7 @@ void raid_genz_int64_raid(int nd, size_t size, void **vv) } /* - * GEN3 (triple parity with Cauchy matrix) 8bit C implementation + * Generate three parity blocks with Cauchy matrix using 8bit C implementation. * * Note that instead of a generic multiplication table, likely resulting * in multiple cache misses, a precomputed table could be used. @@ -339,7 +339,7 @@ void raid_gen3_int8(int nd, size_t size, void **vv) } /* - * GEN4 (quad parity with Cauchy matrix) 8bit C implementation + * Generate four parity blocks with Cauchy matrix using 8bit C implementation. * * Note that instead of a generic multiplication table, likely resulting * in multiple cache misses, a precomputed table could be used. @@ -391,7 +391,7 @@ void raid_gen4_int8(int nd, size_t size, void **vv) } /* - * GEN5 (penta parity with Cauchy matrix) 8bit C implementation + * Generate five parity blocks with Cauchy matrix using 8bit C implementation. * * Note that instead of a generic multiplication table, likely resulting * in multiple cache misses, a precomputed table could be used. @@ -448,7 +448,7 @@ void raid_gen5_int8(int nd, size_t size, void **vv) } /* - * GEN6 (hexa parity with Cauchy matrix) 8bit C implementation + * Generate six parity blocks with Cauchy matrix using 8bit C implementation. * * Note that instead of a generic multiplication table, likely resulting * in multiple cache misses, a precomputed table could be used. diff --git a/raid/neon.c b/raid/neon.c index 7af6ae34..ef88c790 100644 --- a/raid/neon.c +++ b/raid/neon.c @@ -7,7 +7,9 @@ #ifdef CONFIG_NEON /* - * GEN1 (RAID5 with xor) NEON implementation + * Generate one parity block (RAID5 with XOR) using NEON implementation. + * + * Uses 64-byte chunks across four 16-byte vectors. */ void raid_gen1_neon(int nd, size_t size, void **vv) { @@ -59,7 +61,7 @@ void raid_gen1_neon(int nd, size_t size, void **vv) } /* - * GEN2 Cauchy NEON implementation using the active generator + * Generate two parity blocks (RAID6 with Cauchy matrix) using NEON implementation. */ static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, int generator) { @@ -143,7 +145,7 @@ static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, i } /* - * GENz (triple parity with powers of 2^-1) NEON implementation + * Generate three parity blocks with powers of 2^-1 using NEON implementation. */ void raid_genz_neon_raid(int nd, size_t size, void **vv) { @@ -237,7 +239,7 @@ void raid_genz_neon_raid(int nd, size_t size, void **vv) } /* - * GENX NEON implementation + * Generate N parity blocks with Cauchy matrix using NEON implementation. */ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int np, int generator) { @@ -567,6 +569,17 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n raid_neon_end(); } +/* + * Recover multiple data failures using selected parity blocks with NEON. + * + * This avoids raid_delta_gen(), temporary syndrome buffers, and the + * generation of unused parity rows. + * + * If P is available, preserve the complete P delta syndrome and + * reconstruct only nr - 1 missing blocks through the inverse matrix. + * The last missing block is obtained by XORing the reconstructed blocks + * out of Pdelta. + */ static __always_inline void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; @@ -980,9 +993,6 @@ void raid_gen2_neon_aes(int nd, size_t size, void **vv) raid_gen2_neon_gen(nd, size, vv, 3); } -/* - * GEN3 (triple parity with Cauchy matrix) NEON implementation - */ void raid_gen3_neon_raid(int nd, size_t size, void **vv) { raid_genX_neon(nd, size, vv, 3, 2); @@ -993,9 +1003,6 @@ void raid_gen3_neon_aes(int nd, size_t size, void **vv) raid_genX_neon(nd, size, vv, 3, 3); } -/* - * GEN4 (quad parity with Cauchy matrix) NEON implementation - */ void raid_gen4_neon_raid(int nd, size_t size, void **vv) { raid_genX_neon(nd, size, vv, 4, 2); @@ -1006,9 +1013,6 @@ void raid_gen4_neon_aes(int nd, size_t size, void **vv) raid_genX_neon(nd, size, vv, 4, 3); } -/* - * GEN5 (penta parity with Cauchy matrix) NEON implementation - */ void raid_gen5_neon_raid(int nd, size_t size, void **vv) { raid_genX_neon(nd, size, vv, 5, 2); @@ -1019,9 +1023,6 @@ void raid_gen5_neon_aes(int nd, size_t size, void **vv) raid_genX_neon(nd, size, vv, 5, 3); } -/* - * GEN6 (hexa parity with Cauchy matrix) NEON implementation - */ void raid_gen6_neon_raid(int nd, size_t size, void **vv) { raid_genX_neon(nd, size, vv, 6, 2); @@ -1047,6 +1048,9 @@ void raid_rec1_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) /* * Recover failure of two data blocks using P and Q AArch64 NEON implementation. + * + * Uses raid_delta_gen() and keeps all four multiplication tables resident + * in NEON registers (q20..q23) with a 32-byte two-lane unroll. */ static __always_inline void raid_rec2of2_neon(int *id, int *ip, int nd, size_t size, void **vv) { diff --git a/raid/neon32.c b/raid/neon32.c index 1483a6ff..91c2448c 100644 --- a/raid/neon32.c +++ b/raid/neon32.c @@ -7,7 +7,9 @@ #ifdef CONFIG_NEON32 /* - * GEN1 (RAID5 with xor) AArch32 NEON implementation + * Generate one parity block (RAID5 with XOR) using AArch32 NEON implementation. + * + * Uses 64-byte chunks across four 16-byte vectors. */ void raid_gen1_neon32(int nd, size_t size, void **vv) { @@ -62,7 +64,7 @@ void raid_gen1_neon32(int nd, size_t size, void **vv) } /* - * GEN2 Cauchy AArch32 NEON implementation using the active generator + * Generate two parity blocks (RAID6 with Cauchy matrix) using AArch32 NEON implementation. */ static __always_inline void raid_gen2_neon32_gen(int nd, size_t size, void **vv, int generator) @@ -143,7 +145,7 @@ static __always_inline void raid_gen2_neon32_gen(int nd, size_t size, } /* - * GENz (triple parity with powers of 2^-1) AArch32 NEON implementation + * Generate three parity blocks with powers of 2^-1 using AArch32 NEON implementation. */ void raid_genz_neon32_raid(int nd, size_t size, void **vv) { @@ -230,7 +232,7 @@ void raid_genz_neon32_raid(int nd, size_t size, void **vv) } /* - * GENX AArch32 NEON implementation + * Generate N parity blocks with Cauchy matrix using AArch32 NEON implementation. */ static __always_inline void raid_genX_neon32(int nd, size_t size, void **vv, int np, int generator) @@ -519,6 +521,13 @@ static __always_inline void raid_genX_neon32(int nd, size_t size, raid_neon32_end(); } +/* + * Recover multiple data failures using selected parity blocks with AArch32 NEON optimized for up to three failures. + * + * If P is available, keep the complete P delta syndrome in q6. + * Reconstruct only nr - 1 missing blocks through the inverse matrix and + * obtain the last block by XORing the reconstructed blocks out of Pdelta. + */ static __always_inline void raid_recX_neon32_123(int nr, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; @@ -816,6 +825,17 @@ static __always_inline void raid_recX_neon32_123(int nr, int *id, int *ip, int n raid_neon32_end(); } +/* + * Recover multiple data failures using selected parity blocks with AArch32 NEON. + * + * This avoids raid_delta_gen(), temporary syndrome buffers, and the + * generation of unused parity rows. + * + * If P is available, preserve the complete P delta syndrome and + * reconstruct only nr - 1 missing blocks through the inverse matrix. + * The last missing block is obtained by XORing the reconstructed blocks + * out of Pdelta. + */ static __always_inline void raid_recX_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; @@ -1159,9 +1179,6 @@ void raid_gen2_neon32_aes(int nd, size_t size, void **vv) raid_gen2_neon32_gen(nd, size, vv, 3); } -/* - * GEN3 (triple parity with Cauchy matrix) AArch32 NEON implementation - */ void raid_gen3_neon32_raid(int nd, size_t size, void **vv) { raid_genX_neon32(nd, size, vv, 3, 2); @@ -1172,9 +1189,6 @@ void raid_gen3_neon32_aes(int nd, size_t size, void **vv) raid_genX_neon32(nd, size, vv, 3, 3); } -/* - * GEN4 (quad parity with Cauchy matrix) AArch32 NEON implementation - */ void raid_gen4_neon32_raid(int nd, size_t size, void **vv) { raid_genX_neon32(nd, size, vv, 4, 2); @@ -1185,9 +1199,6 @@ void raid_gen4_neon32_aes(int nd, size_t size, void **vv) raid_genX_neon32(nd, size, vv, 4, 3); } -/* - * GEN5 (penta parity with Cauchy matrix) AArch32 NEON implementation - */ void raid_gen5_neon32_raid(int nd, size_t size, void **vv) { raid_genX_neon32(nd, size, vv, 5, 2); @@ -1198,9 +1209,6 @@ void raid_gen5_neon32_aes(int nd, size_t size, void **vv) raid_genX_neon32(nd, size, vv, 5, 3); } -/* - * GEN6 (hexa parity with Cauchy matrix) AArch32 NEON implementation - */ void raid_gen6_neon32_raid(int nd, size_t size, void **vv) { raid_genX_neon32(nd, size, vv, 6, 2); @@ -1226,6 +1234,8 @@ void raid_rec1_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv) /* * Recover failure of two data blocks using P and Q AArch32 NEON implementation. + * + * Uses raid_delta_gen() and computes the analytical solution with polynomial reduction. */ static __always_inline void raid_rec2of2_neon32(int *id, int *ip, int nd, size_t size, void **vv) { diff --git a/raid/sse2.c b/raid/sse2.c index 6a93e880..0ca2fb3e 100644 --- a/raid/sse2.c +++ b/raid/sse2.c @@ -100,7 +100,9 @@ */ /* - * GEN1 (RAID5 with xor) SSE2 implementation + * Generate one parity block (RAID5 with XOR) using SSE2 implementation. + * + * Uses 64-byte chunks across four 16-byte XMM vectors. */ void raid_gen1_sse2(int nd, size_t size, void **vv) { @@ -137,7 +139,7 @@ void raid_gen1_sse2(int nd, size_t size, void **vv) } /* - * GEN2 Cauchy SSE2 implementation using the active generator + * Generate two parity blocks (RAID6 with Cauchy matrix) using SSE2 implementation. */ static __always_inline void raid_gen2_sse2_gen(int nd, size_t size, void **vv, int generator) { @@ -220,7 +222,10 @@ void raid_gen2_sse2_aes(int nd, size_t size, void **vv) #ifdef CONFIG_X86_64 /* - * GEN2 Cauchy SSE2 implementation using the active generator + * Generate two parity blocks (RAID6 with Cauchy matrix) using SSE2 extended implementation. + * + * Uses the extended register set (16 XMM registers), unrolling 64-byte chunks + * across four 16-byte lanes and processing two disks per iteration. * * Note that it uses 16 registers, meaning that x64 is required. */ @@ -458,7 +463,7 @@ void raid_gen2_sse2ext_aes(int nd, size_t size, void **vv) #endif /* - * GENz (triple parity with powers of 2^-1) SSE2 implementation + * Generate three parity blocks with powers of 2^-1 using SSE2 implementation. */ void raid_genz_sse2_raid(int nd, size_t size, void **vv) { @@ -515,7 +520,7 @@ void raid_genz_sse2_raid(int nd, size_t size, void **vv) #ifdef CONFIG_X86_64 /* - * GENz (triple parity with powers of 2^-1) SSE2 implementation + * Generate three parity blocks with powers of 2^-1 using SSE2 extended implementation. * * Note that it uses 16 registers, meaning that x64 is required. */ diff --git a/raid/ssse3.c b/raid/ssse3.c index c5165bf1..3457d260 100644 --- a/raid/ssse3.c +++ b/raid/ssse3.c @@ -7,7 +7,7 @@ #ifdef CONFIG_X86 /* - * GEN3 (triple parity with Cauchy matrix) SSSE3 implementation + * Generate three parity blocks with Cauchy matrix using SSSE3 implementation. */ static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv, int generator) { @@ -111,7 +111,7 @@ static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv, } /* - * GEN4 (quad parity with Cauchy matrix) SSSE3 implementation + * Generate four parity blocks with Cauchy matrix using SSSE3 implementation. */ static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv, int generator) { @@ -234,7 +234,9 @@ static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv, } /* - * GEN5 (penta parity with Cauchy matrix) SSSE3 implementation + * Generate five parity blocks with Cauchy matrix using SSSE3 implementation. + * + * Uses an aligned stack buffer for parity accumulators under 8-register pressure. */ void raid_gen5_ssse3_raid(int nd, size_t size, void **vv) { @@ -370,7 +372,9 @@ void raid_gen5_ssse3_raid(int nd, size_t size, void **vv) } /* - * GEN6 (hexa parity with Cauchy matrix) SSSE3 implementation + * Generate six parity blocks with Cauchy matrix using SSSE3 implementation. + * + * Uses an aligned stack buffer for parity accumulators under 8-register pressure. */ void raid_gen6_ssse3_raid(int nd, size_t size, void **vv) { @@ -528,7 +532,10 @@ void raid_gen6_ssse3_raid(int nd, size_t size, void **vv) #ifdef CONFIG_X86_64 /* - * GEN3 (triple parity with Cauchy matrix) SSSE3 implementation + * Generate three parity blocks with Cauchy matrix using SSSE3 extended implementation. + * + * Uses the extended register set (16 XMM registers) and processes two disks + * per iteration across two 16-byte XMM lanes (32 bytes/step). * * Note that it uses 16 registers, meaning that x64 is required. */ @@ -787,7 +794,10 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v } /* - * GEN4 (quad parity with Cauchy matrix) SSSE3 implementation + * Generate four parity blocks with Cauchy matrix using SSSE3 extended implementation. + * + * Uses the extended register set (16 XMM registers) and processes two disks + * per iteration across two 16-byte XMM lanes (32 bytes/step). * * Note that it uses 16 registers, meaning that x64 is required. */ @@ -975,7 +985,7 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v } /* - * GENX SSSE3ext implementation + * Generate N parity blocks with Cauchy matrix using SSSE3 extended implementation. */ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, int np, int generator) { @@ -1157,7 +1167,9 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i #endif /* - * RAID recovering for one disk SSSE3 implementation + * Recover failure of one data block using SSSE3 delta implementation. + * + * Uses raid_delta_gen() and multiplies by the inverted coefficient table. */ static __always_inline void raid_rec1_ssse3_delta(int *id, int *ip, int nd, size_t size, void **vv) { @@ -1206,7 +1218,9 @@ static __always_inline void raid_rec1_ssse3_delta(int *id, int *ip, int nd, size } /* - * RAID recovering for two disks SSSE3 implementation + * Recover failure of two data blocks using SSSE3 delta implementation. + * + * Uses raid_delta_gen() and solves the 2x2 system directly on delta blocks. */ static __always_inline void raid_rec2_ssse3_delta(int *id, int *ip, int nd, size_t size, void **vv) { @@ -1308,6 +1322,8 @@ static __always_inline void raid_rec2_ssse3_delta(int *id, int *ip, int nd, size /* * Recover failure of two data blocks using P and Q SSSE3 implementation. + * + * Uses raid_delta_gen() and computes the analytical solution. */ static __always_inline void raid_rec2of2_ssse3(int *id, int *ip, int nd, size_t size, void **vv) { @@ -1384,7 +1400,7 @@ static __always_inline void raid_rec2of2_ssse3(int *id, int *ip, int nd, size_t } /* - * RAID recovering SSSE3 implementation optimized for up to four failures. + * Recover multiple data failures using selected parity blocks with SSSE3 optimized for up to four failures. * * If P is available, keep the complete P delta syndrome in xmm6. * Reconstruct only nr - 1 missing blocks through the inverse matrix and @@ -1609,7 +1625,7 @@ static __always_inline void raid_recX_ssse3_1234(int nr, int *id, int *ip, int n } /* - * RAID recovering SSSE3 implementation. + * Recover multiple data failures using selected parity blocks with SSSE3. * * If P is available, keep the complete P delta syndrome in xmm6. * Reconstruct only nr - 1 missing blocks through the inverse matrix and @@ -1776,7 +1792,7 @@ static __always_inline void raid_recX_ssse3(int nr, int *id, int *ip, int nd, si #ifdef CONFIG_X86_64 /* - * RAID recovering for one disk SSSE3 extended implementation. + * Recover failure of one data block using SSSE3 extended delta implementation. * * Process two 16-byte blocks per iteration, grouping equal operations across * both lanes to expose instruction-level parallelism. @@ -1856,7 +1872,10 @@ static __always_inline void raid_rec1_ssse3ext_delta(int *id, int *ip, int nd, s } /* - * RAID recovering for two disks SSSE3 extended implementation + * Recover failure of two data blocks using SSSE3 extended delta implementation. + * + * Uses raid_delta_gen() and keeps all 8 inverse-matrix tables resident + * in XMM registers (xmm8..xmm15). */ static __always_inline void raid_rec2_ssse3ext_delta(int *id, int *ip, int nd, size_t size, void **vv) { @@ -1964,6 +1983,9 @@ static __always_inline void raid_rec2_ssse3ext_delta(int *id, int *ip, int nd, s /* * Recover failure of two data blocks using P and Q SSSE3 extended implementation. + * + * Uses raid_delta_gen() and keeps all four multiplication tables resident + * in XMM registers (xmm8..xmm11). */ static __always_inline void raid_rec2of2_ssse3ext(int *id, int *ip, int nd, size_t size, void **vv) { @@ -2043,6 +2065,13 @@ static __always_inline void raid_rec2of2_ssse3ext(int *id, int *ip, int nd, size raid_sse_end(); } +/* + * Recover multiple data failures using selected parity blocks with SSSE3 extended optimized for up to three failures. + * + * If P is available, keep the complete P delta syndrome in xmm6. + * Reconstruct only nr - 1 missing blocks through the inverse matrix and + * obtain the last block by XORing the reconstructed blocks out of Pdelta. + */ static __always_inline void raid_recX_ssse3ext_123(int nr, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; @@ -2293,8 +2322,7 @@ static __always_inline void raid_recX_ssse3ext_123(int nr, int *id, int *ip, int } /* - * RAID recovering SSSE3 extended implementation optimized for up to five - * failures. + * Recover multiple data failures using selected parity blocks with SSSE3 extended optimized for up to five failures. * * If P is available, keep the complete P delta syndrome in xmm10. * Reconstruct only nr - 1 missing blocks through the inverse matrix and @@ -2593,6 +2621,17 @@ static __always_inline void raid_recX_ssse3ext_12345(int nr, int *id, int *ip, i raid_sse_end(); } +/* + * Recover multiple data failures using selected parity blocks with SSSE3 extended. + * + * This avoids raid_delta_gen(), temporary syndrome buffers, and the + * generation of unused parity rows. + * + * If P is available, preserve the complete P delta syndrome and + * reconstruct only nr - 1 missing blocks through the inverse matrix. + * The last missing block is obtained by XORing the reconstructed blocks + * out of Pdelta. + */ static __always_inline void raid_recX_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; @@ -2895,11 +2934,6 @@ void raid_gen4_ssse3ext_aes(int nd, size_t size, void **vv) raid_gen4_ssse3ext_gen(nd, size, vv, 3); } -/* - * GEN5 (penta parity with Cauchy matrix) SSSE3 implementation - * - * Note that it uses 16 registers, meaning that x64 is required. - */ void raid_gen5_ssse3ext_raid(int nd, size_t size, void **vv) { raid_genX_ssse3ext(nd, size, vv, 5, 2); @@ -2910,11 +2944,6 @@ void raid_gen5_ssse3ext_aes(int nd, size_t size, void **vv) raid_genX_ssse3ext(nd, size, vv, 5, 3); } -/* - * GEN6 (hexa parity with Cauchy matrix) SSSE3 implementation - * - * Note that it uses 16 registers, meaning that x64 is required. - */ void raid_gen6_ssse3ext_raid(int nd, size_t size, void **vv) { raid_genX_ssse3ext(nd, size, vv, 6, 2);