mirror of
https://github.com/amadvance/snapraid.git
synced 2026-10-26 19:01:57 -04:00
Add and standardize top comments across SIMD assembly functions
This commit is contained in:
Andrea Mazzoleni
parent
3759c9356b
commit
9bd2f68046
8 changed files with 217 additions and 183 deletions
57
raid/avx2.c
57
raid/avx2.c
|
|
@ -7,7 +7,9 @@
|
|||
|
||||
#ifdef CONFIG_X86
|
||||
/*
|
||||
* GEN1 (RAID5 with xor) AVX2 implementation
|
||||
* Generate one parity block (RAID5 with XOR) using AVX2 implementation.
|
||||
*
|
||||
* Uses 64-byte chunks across two 32-byte YMM lanes.
|
||||
*/
|
||||
void raid_gen1_avx2(int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -38,7 +40,9 @@ void raid_gen1_avx2(int nd, size_t size, void **vv)
|
|||
}
|
||||
|
||||
/*
|
||||
* GEN2 Cauchy AVX2 implementation using the active generator
|
||||
* Generate two parity blocks (RAID6 with Cauchy matrix) using AVX2 implementation.
|
||||
*
|
||||
* Uses Horner's method with 64-byte chunks across two 32-byte YMM lanes.
|
||||
*/
|
||||
static __always_inline void raid_gen2_avx2_gen(int nd, size_t size, void **vv, int generator)
|
||||
{
|
||||
|
|
@ -107,7 +111,7 @@ static __always_inline void raid_gen2_avx2_gen(int nd, size_t size, void **vv, i
|
|||
|
||||
#ifdef CONFIG_X86_64
|
||||
/*
|
||||
* GEN2 Cauchy AVX2 implementation using the extended register set.
|
||||
* Generate two parity blocks (RAID6 with Cauchy matrix) using AVX2 extended implementation.
|
||||
*
|
||||
* Process two data disks at a time and process the two 32-byte lanes
|
||||
* sequentially.
|
||||
|
|
@ -221,7 +225,7 @@ static __always_inline void raid_gen2_avx2ext_gen(int nd, size_t size, void **vv
|
|||
}
|
||||
|
||||
/*
|
||||
* GENz (triple parity with powers of 2^-1) AVX2 implementation
|
||||
* Generate three parity blocks with powers of 2^-1 using AVX2 extended implementation.
|
||||
*
|
||||
* Note that it uses 16 registers, meaning that x64 is required.
|
||||
*/
|
||||
|
|
@ -296,7 +300,10 @@ void raid_genz_avx2ext_raid(int nd, size_t size, void **vv)
|
|||
}
|
||||
|
||||
/*
|
||||
* GEN3 (triple parity with Cauchy matrix) AVX2 implementation
|
||||
* Generate three parity blocks with Cauchy matrix using AVX2 extended implementation.
|
||||
*
|
||||
* Uses the extended register set (16 YMM registers) and processes two disks
|
||||
* per iteration across two 32-byte YMM lanes (64 bytes/step).
|
||||
*
|
||||
* Note that it uses 16 registers, meaning that x64 is required.
|
||||
*/
|
||||
|
|
@ -552,7 +559,10 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv
|
|||
}
|
||||
|
||||
/*
|
||||
* GEN4 (quad parity with Cauchy matrix) AVX2 implementation
|
||||
* Generate four parity blocks with Cauchy matrix using AVX2 extended implementation.
|
||||
*
|
||||
* Uses the extended register set (16 YMM registers) and processes two disks
|
||||
* per iteration across two 32-byte YMM lanes (64 bytes/step).
|
||||
*
|
||||
* Note that it uses 16 registers, meaning that x64 is required.
|
||||
*/
|
||||
|
|
@ -725,7 +735,7 @@ static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv
|
|||
}
|
||||
|
||||
/*
|
||||
* GENX AVX2EXT implementation
|
||||
* Generate N parity blocks with Cauchy matrix using AVX2 extended implementation.
|
||||
*/
|
||||
static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, int np, int generator)
|
||||
{
|
||||
|
|
@ -883,7 +893,9 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
|
|||
#endif
|
||||
|
||||
/*
|
||||
* RAID recovering for one disk AVX2 implementation
|
||||
* Recover failure of one data block using AVX2 delta implementation.
|
||||
*
|
||||
* Uses raid_delta_gen() and multiplies by the inverted coefficient table.
|
||||
*/
|
||||
static __always_inline void raid_rec1_avx2_delta(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -945,7 +957,9 @@ static __always_inline void raid_rec1_avx2_delta(int *id, int *ip, int nd, size_
|
|||
}
|
||||
|
||||
/*
|
||||
* RAID recovering for two disks AVX2 implementation
|
||||
* Recover failure of two data blocks using AVX2 delta implementation.
|
||||
*
|
||||
* Uses raid_delta_gen() and solves the 2x2 system directly on delta blocks.
|
||||
*/
|
||||
static __always_inline void raid_rec2_avx2_delta(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -1039,6 +1053,8 @@ static __always_inline void raid_rec2_avx2_delta(int *id, int *ip, int nd, size_
|
|||
|
||||
/*
|
||||
* Recover failure of two data blocks using P and Q AVX2 implementation.
|
||||
*
|
||||
* Uses raid_delta_gen() and computes the analytical solution.
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_avx2(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -1111,7 +1127,7 @@ static __always_inline void raid_rec2of2_avx2(int *id, int *ip, int nd, size_t s
|
|||
}
|
||||
|
||||
/*
|
||||
* RAID recovering AVX2 implementation optimized for up to four failures.
|
||||
* Recover multiple data failures using selected parity blocks with AVX2 optimized for up to four failures.
|
||||
*
|
||||
* Compute all selected syndromes in one scan of the surviving data, then
|
||||
* retain only their low/high nibbles while reconstructing the missing data.
|
||||
|
|
@ -1320,7 +1336,7 @@ static __always_inline void raid_recX_avx2_1234(int nr, int *id, int *ip, int nd
|
|||
}
|
||||
|
||||
/*
|
||||
* RAID recovering AVX2 implementation with all syndromes in memory.
|
||||
* Recover multiple data failures using selected parity blocks with AVX2 and all syndromes in memory.
|
||||
*
|
||||
* Compute all selected syndromes in a single pass over the surviving data.
|
||||
* Syndrome accumulators are kept in memory to minimize register pressure.
|
||||
|
|
@ -1544,7 +1560,10 @@ static __always_inline void raid_recX_avx2(int nr, int *id, int *ip, int nd, siz
|
|||
|
||||
#ifdef CONFIG_X86_64
|
||||
/*
|
||||
* RAID recovering for two disks with P and Q AVX2 implementation
|
||||
* Recover failure of two data blocks using P and Q AVX2 extended implementation.
|
||||
*
|
||||
* Uses raid_delta_gen() and keeps all four multiplication tables resident
|
||||
* in YMM registers (ymm8..ymm11).
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_avx2ext(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -1616,7 +1635,7 @@ static __always_inline void raid_rec2of2_avx2ext(int *id, int *ip, int nd, size_
|
|||
}
|
||||
|
||||
/*
|
||||
* RAID recovering AVX2 implementation optimized for up to three failures.
|
||||
* Recover multiple data failures using selected parity blocks with AVX2 extended optimized for up to three failures.
|
||||
*
|
||||
* Process 64 bytes at a time as two independent 32-byte lanes.
|
||||
* Multiplication tables are broadcast once and shared between both lanes.
|
||||
|
|
@ -1919,7 +1938,7 @@ static __always_inline void raid_recX_avx2ext_123(int nr, int *id, int *ip, int
|
|||
}
|
||||
|
||||
/*
|
||||
* RAID recovering AVX2 extended implementation.
|
||||
* Recover multiple data failures using selected parity blocks with AVX2 extended.
|
||||
*
|
||||
* Compute only the selected syndromes, keeping them in registers.
|
||||
* This avoids raid_delta_gen(), temporary syndrome buffers, and the
|
||||
|
|
@ -2300,11 +2319,6 @@ void raid_gen4_avx2ext_aes(int nd, size_t size, void **vv)
|
|||
raid_gen4_avx2ext_gen(nd, size, vv, 3);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN5 (penta parity with Cauchy matrix) AVX2 implementation
|
||||
*
|
||||
* Note that it uses 16 registers, meaning that x64 is required.
|
||||
*/
|
||||
void raid_gen5_avx2ext_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx2ext(nd, size, vv, 5, 2);
|
||||
|
|
@ -2315,11 +2329,6 @@ void raid_gen5_avx2ext_aes(int nd, size_t size, void **vv)
|
|||
raid_genX_avx2ext(nd, size, vv, 5, 3);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN6 (hexa parity with Cauchy matrix) AVX2 implementation
|
||||
*
|
||||
* Note that it uses 16 registers, meaning that x64 is required.
|
||||
*/
|
||||
void raid_gen6_avx2ext_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx2ext(nd, size, vv, 6, 2);
|
||||
|
|
|
|||
|
|
@ -7,7 +7,9 @@
|
|||
|
||||
#ifdef CONFIG_X86_64
|
||||
/*
|
||||
* GEN1 (RAID5 with xor) AVX512BW implementation
|
||||
* Generate one parity block (RAID5 with XOR) using AVX512BW implementation.
|
||||
*
|
||||
* Processes two disks per iteration using vpternlogq over 64-byte blocks.
|
||||
*
|
||||
* Note that in true AVX512F would suffice, but we don't want to add
|
||||
* specific support for AVX512F because this would be the only function
|
||||
|
|
@ -48,7 +50,10 @@ void raid_gen1_avx512bw(int nd, size_t size, void **vv)
|
|||
}
|
||||
|
||||
/*
|
||||
* GEN2 Cauchy AVX512BW implementation
|
||||
* Generate two parity blocks (RAID6 with Cauchy matrix) using AVX512BW implementation.
|
||||
*
|
||||
* Preloads 13 pairs of Q coefficient tables in ZMM registers (zmm4..zmm29)
|
||||
* with an unrolled disk loop and early exits.
|
||||
*/
|
||||
void raid_gen2_avx512bw(int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -274,7 +279,10 @@ store:
|
|||
}
|
||||
|
||||
/*
|
||||
* GEN3 (triple parity with Cauchy matrix) AVX512BW implementation
|
||||
* Generate three parity blocks with Cauchy matrix using AVX512BW implementation.
|
||||
*
|
||||
* Preloads 6 sets of Q and R coefficient tables in ZMM registers (zmm7..zmm30)
|
||||
* with an unrolled disk loop and early exits.
|
||||
*/
|
||||
void raid_gen3_avx512bw(int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -445,7 +453,7 @@ store:
|
|||
}
|
||||
|
||||
/*
|
||||
* GENX AVX512BW implementation
|
||||
* Generate N parity blocks with Cauchy matrix using AVX512BW implementation.
|
||||
*/
|
||||
static __always_inline void raid_genX_avx512bw(int nd, size_t size, void **vv, int np)
|
||||
{
|
||||
|
|
@ -885,25 +893,16 @@ static __always_inline void raid_recX_avx512bw(int nr, int *id, int *ip, int nd,
|
|||
raid_avx_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN4 (quad parity with Cauchy matrix) AVX512BW implementation
|
||||
*/
|
||||
void raid_gen4_avx512bw(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx512bw(nd, size, vv, 4);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN5 (penta parity with Cauchy matrix) AVX512BW implementation
|
||||
*/
|
||||
void raid_gen5_avx512bw(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx512bw(nd, size, vv, 5);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN6 (hexa parity with Cauchy matrix) AVX512BW implementation
|
||||
*/
|
||||
void raid_gen6_avx512bw(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx512bw(nd, size, vv, 6);
|
||||
|
|
@ -924,6 +923,9 @@ void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv
|
|||
|
||||
/*
|
||||
* Recover failure of two data blocks using P and Q AVX512BW implementation.
|
||||
*
|
||||
* Uses raid_delta_gen() and keeps both multiplication-table pairs resident
|
||||
* in ZMM registers (zmm16..zmm19).
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_avx512bw(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
|
|||
123
raid/gfni.c
123
raid/gfni.c
|
|
@ -7,8 +7,7 @@
|
|||
|
||||
#ifdef CONFIG_X86_64
|
||||
/*
|
||||
* GENX AVX2 GFNI implementation for the RAID polynomial using direct
|
||||
* Cauchy coefficients.
|
||||
* Generate N parity blocks for the RAID polynomial using direct Cauchy coefficients with AVX2 GFNI.
|
||||
*/
|
||||
static __always_inline void raid_genX_avx2gfni_raid(int nd, size_t size, void **vv, int np)
|
||||
{
|
||||
|
|
@ -119,7 +118,10 @@ static __always_inline void raid_genX_avx2gfni_raid(int nd, size_t size, void **
|
|||
}
|
||||
|
||||
/*
|
||||
* GEN2 (RAID6 with powers of 2) AVX512 GFNI implementation
|
||||
* Generate two parity blocks (RAID6 with powers of 2) using AVX512 GFNI implementation.
|
||||
*
|
||||
* Preloads up to 28 Q affine matrix coefficients in ZMM registers (zmm4..zmm31)
|
||||
* with an unrolled disk loop and early exits.
|
||||
*/
|
||||
static __always_inline void raid_gen2_avx512gfni_raid_gen(int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -388,7 +390,10 @@ store:
|
|||
}
|
||||
|
||||
/*
|
||||
* GEN3 (triple parity with Cauchy matrix) AVX512 GFNI implementation
|
||||
* Generate three parity blocks with Cauchy matrix using AVX512 GFNI implementation.
|
||||
*
|
||||
* Preloads up to 13 pairs of Q and R affine matrix coefficients in ZMM
|
||||
* registers (zmm6..zmm31) with an unrolled disk loop and early exits.
|
||||
*/
|
||||
static __always_inline void raid_gen3_avx512gfni_raid_gen(int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -585,8 +590,7 @@ store:
|
|||
}
|
||||
|
||||
/*
|
||||
* GENX AVX512 GFNI implementation for the RAID polynomial using direct
|
||||
* Cauchy coefficients.
|
||||
* Generate N parity blocks for the RAID polynomial using direct Cauchy coefficients with AVX512 GFNI.
|
||||
*/
|
||||
static __always_inline void raid_genX_avx512gfni_raid(int nd, size_t size, void **vv, int np)
|
||||
{
|
||||
|
|
@ -669,8 +673,7 @@ static __always_inline void raid_genX_avx512gfni_raid(int nd, size_t size, void
|
|||
}
|
||||
|
||||
/*
|
||||
* GENX AVX2 GFNI implementation for the AES polynomial using direct
|
||||
* Cauchy coefficients.
|
||||
* Generate N parity blocks for the AES polynomial using direct Cauchy coefficients with AVX2 GFNI.
|
||||
*/
|
||||
static __always_inline void raid_genX_avx2gfni_aes(int nd, size_t size, void **vv, int np)
|
||||
{
|
||||
|
|
@ -788,7 +791,10 @@ static __always_inline void raid_genX_avx2gfni_aes(int nd, size_t size, void **v
|
|||
}
|
||||
|
||||
/*
|
||||
* GEN2 (RAID6 with powers of 3) AVX512 GFNI implementation
|
||||
* Generate two parity blocks (RAID6 with powers of 3) using AVX512 GFNI implementation.
|
||||
*
|
||||
* Preloads up to 28 Q multiplier coefficients in ZMM registers (zmm4..zmm31)
|
||||
* with an unrolled disk loop and early exits.
|
||||
*/
|
||||
static __always_inline void raid_gen2_avx512gfni_aes_gen(int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -1057,7 +1063,10 @@ store:
|
|||
}
|
||||
|
||||
/*
|
||||
* GEN3 (triple parity with Cauchy matrix) AVX512 GFNI implementation
|
||||
* Generate three parity blocks with Cauchy matrix using AVX512 GFNI implementation.
|
||||
*
|
||||
* Preloads up to 13 pairs of Q and R multiplier coefficients in ZMM
|
||||
* registers (zmm6..zmm31) with an unrolled disk loop and early exits.
|
||||
*/
|
||||
static __always_inline void raid_gen3_avx512gfni_aes_gen(int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -1254,8 +1263,7 @@ store:
|
|||
}
|
||||
|
||||
/*
|
||||
* GENX AVX512 GFNI implementation for the AES polynomial using direct
|
||||
* Cauchy coefficients.
|
||||
* Generate N parity blocks for the AES polynomial using direct Cauchy coefficients with AVX512 GFNI.
|
||||
*/
|
||||
static __always_inline void raid_genX_avx512gfni_aes(int nd, size_t size, void **vv, int np)
|
||||
{
|
||||
|
|
@ -1337,6 +1345,16 @@ static __always_inline void raid_genX_avx512gfni_aes(int nd, size_t size, void *
|
|||
raid_avx_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* Recover multiple data failures using selected parity blocks with AVX2 GFNI.
|
||||
*
|
||||
* Compute only the selected syndromes, keeping them in registers.
|
||||
* This avoids raid_delta_gen(), temporary syndrome buffers, recomputation of
|
||||
* parity blocks, and generation of unused parity rows.
|
||||
*
|
||||
* If P is available, reconstruct only nr - 1 missing blocks through the
|
||||
* inverse matrix and derive the last missing block from the P delta by XOR.
|
||||
*/
|
||||
static __always_inline void raid_recX_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
|
|
@ -1716,6 +1734,16 @@ static __always_inline void raid_recX_avx512gfni_raid(int nr, int *id, int *ip,
|
|||
raid_avx_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* Recover multiple data failures for the AES polynomial using selected parity blocks with AVX2 GFNI.
|
||||
*
|
||||
* Compute only the selected syndromes, keeping them in registers.
|
||||
* This avoids raid_delta_gen(), temporary syndrome buffers, recomputation of
|
||||
* parity blocks, and generation of unused parity rows.
|
||||
*
|
||||
* If P is available, reconstruct only nr - 1 missing blocks through the
|
||||
* inverse matrix and derive the last missing block from the P delta by XOR.
|
||||
*/
|
||||
static __always_inline void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
|
|
@ -1906,8 +1934,7 @@ static __always_inline void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int
|
|||
}
|
||||
|
||||
/*
|
||||
* Recover multiple data failures using selected parity blocks with AVX512 GFNI
|
||||
* and the AES polynomial.
|
||||
* Recover multiple data failures for the AES polynomial using selected parity blocks with AVX512 GFNI.
|
||||
*
|
||||
* Compute only the selected syndromes, keeping them in registers.
|
||||
* This avoids raid_delta_gen(), temporary syndrome buffers, recomputation of
|
||||
|
|
@ -2073,161 +2100,101 @@ static __always_inline void raid_recX_avx512gfni_aes(int nr, int *id, int *ip, i
|
|||
raid_avx_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN2 (RAID6 with powers of 2) AVX2 GFNI implementation
|
||||
*/
|
||||
void raid_gen2_avx2gfni_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx2gfni_raid(nd, size, vv, 2);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN2 (RAID6 with powers of 2) AVX512 GFNI implementation
|
||||
*/
|
||||
void raid_gen2_avx512gfni_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen2_avx512gfni_raid_gen(nd, size, vv);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN3 (triple parity with Cauchy matrix) AVX2 GFNI implementation
|
||||
*/
|
||||
void raid_gen3_avx2gfni_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx2gfni_raid(nd, size, vv, 3);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN3 (triple parity with Cauchy matrix) AVX512 GFNI implementation
|
||||
*/
|
||||
void raid_gen3_avx512gfni_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen3_avx512gfni_raid_gen(nd, size, vv);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN4 (quad parity with Cauchy matrix) AVX2 GFNI implementation
|
||||
*/
|
||||
void raid_gen4_avx2gfni_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx2gfni_raid(nd, size, vv, 4);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN4 (quad parity with Cauchy matrix) GFNI implementation
|
||||
*/
|
||||
void raid_gen4_avx512gfni_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx512gfni_raid(nd, size, vv, 4);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN5 (penta parity with Cauchy matrix) AVX2 GFNI implementation
|
||||
*/
|
||||
void raid_gen5_avx2gfni_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx2gfni_raid(nd, size, vv, 5);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN5 (penta parity with Cauchy matrix) GFNI implementation
|
||||
*/
|
||||
void raid_gen5_avx512gfni_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx512gfni_raid(nd, size, vv, 5);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN6 (hexa parity with Cauchy matrix) AVX2 GFNI implementation
|
||||
*/
|
||||
void raid_gen6_avx2gfni_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx2gfni_raid(nd, size, vv, 6);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN6 (hexa parity with Cauchy matrix) GFNI implementation
|
||||
*/
|
||||
void raid_gen6_avx512gfni_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx512gfni_raid(nd, size, vv, 6);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN2 (RAID6 with powers of 3) AVX2 GFNI implementation
|
||||
*/
|
||||
void raid_gen2_avx2gfni_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx2gfni_aes(nd, size, vv, 2);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN2 (RAID6 with powers of 3) AVX512 GFNI implementation
|
||||
*/
|
||||
void raid_gen2_avx512gfni_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen2_avx512gfni_aes_gen(nd, size, vv);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN3 (triple parity with Cauchy matrix) AVX2 GFNI implementation
|
||||
*/
|
||||
void raid_gen3_avx2gfni_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx2gfni_aes(nd, size, vv, 3);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN3 (triple parity with Cauchy matrix) AVX512 GFNI implementation
|
||||
*/
|
||||
void raid_gen3_avx512gfni_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen3_avx512gfni_aes_gen(nd, size, vv);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN4 (quad parity with Cauchy matrix) AVX2 GFNI implementation
|
||||
*/
|
||||
void raid_gen4_avx2gfni_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx2gfni_aes(nd, size, vv, 4);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN4 (quad parity with Cauchy matrix) GFNI implementation
|
||||
*/
|
||||
void raid_gen4_avx512gfni_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx512gfni_aes(nd, size, vv, 4);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN5 (penta parity with Cauchy matrix) AVX2 GFNI implementation
|
||||
*/
|
||||
void raid_gen5_avx2gfni_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx2gfni_aes(nd, size, vv, 5);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN5 (penta parity with Cauchy matrix) GFNI implementation
|
||||
*/
|
||||
void raid_gen5_avx512gfni_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx512gfni_aes(nd, size, vv, 5);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN6 (hexa parity with Cauchy matrix) AVX2 GFNI implementation
|
||||
*/
|
||||
void raid_gen6_avx2gfni_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx2gfni_aes(nd, size, vv, 6);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN6 (hexa parity with Cauchy matrix) GFNI implementation
|
||||
*/
|
||||
void raid_gen6_avx512gfni_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx512gfni_aes(nd, size, vv, 6);
|
||||
|
|
@ -2248,6 +2215,8 @@ void raid_rec1_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void
|
|||
|
||||
/*
|
||||
* Recover failure of two data blocks using P and Q AVX2 RAID GFNI implementation.
|
||||
*
|
||||
* Uses raid_delta_gen() and keeps both affine matrices resident in YMM registers.
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_avx2gfni_raid(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -2364,6 +2333,8 @@ void raid_rec1_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, vo
|
|||
|
||||
/*
|
||||
* Recover failure of two data blocks using P and Q AVX512 RAID GFNI implementation.
|
||||
*
|
||||
* Uses raid_delta_gen() and keeps both affine matrices resident in ZMM registers.
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_avx512gfni_raid(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -2466,6 +2437,8 @@ void raid_rec1_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void
|
|||
|
||||
/*
|
||||
* Recover failure of two data blocks using P and Q AVX2 AES GFNI implementation.
|
||||
*
|
||||
* Uses raid_delta_gen() and keeps both byte multipliers resident in YMM registers.
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_avx2gfni_aes(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -2573,6 +2546,8 @@ void raid_rec1_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, voi
|
|||
|
||||
/*
|
||||
* Recover failure of two data blocks using P and Q AVX512 AES GFNI implementation.
|
||||
*
|
||||
* Uses raid_delta_gen() and keeps both byte multipliers resident in ZMM registers.
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_avx512gfni_aes(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
|
|||
20
raid/int.c
20
raid/int.c
|
|
@ -5,7 +5,7 @@
|
|||
#include "gf.h"
|
||||
|
||||
/*
|
||||
* GEN1 (RAID5 with xor) 32bit C implementation
|
||||
* Generate one parity block (RAID5 with XOR) using 32bit C implementation.
|
||||
*/
|
||||
void raid_gen1_int32(int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -33,7 +33,7 @@ void raid_gen1_int32(int nd, size_t size, void **vv)
|
|||
}
|
||||
|
||||
/*
|
||||
* GEN1 (RAID5 with xor) 64bit C implementation
|
||||
* Generate one parity block (RAID5 with XOR) using 64bit C implementation.
|
||||
*/
|
||||
void raid_gen1_int64(int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -61,7 +61,7 @@ void raid_gen1_int64(int nd, size_t size, void **vv)
|
|||
}
|
||||
|
||||
/*
|
||||
* GEN2 (double parity) 32bit C implementation
|
||||
* Generate two parity blocks (RAID6 with powers of 2 or 3) using 32bit C implementation.
|
||||
*/
|
||||
static __always_inline void raid_gen2_int32_gen(int nd, size_t size, void **vv, int generator)
|
||||
{
|
||||
|
|
@ -121,7 +121,7 @@ void raid_gen2_int32_aes(int nd, size_t size, void **vv)
|
|||
}
|
||||
|
||||
/*
|
||||
* GEN2 (double parity) 64bit C implementation
|
||||
* Generate two parity blocks (RAID6 with powers of 2 or 3) using 64bit C implementation.
|
||||
*/
|
||||
static __always_inline void raid_gen2_int64_gen(int nd, size_t size, void **vv, int generator)
|
||||
{
|
||||
|
|
@ -186,7 +186,7 @@ void raid_gen2_int8(int nd, size_t size, void **vv)
|
|||
}
|
||||
|
||||
/*
|
||||
* GENz (triple parity with powers of 2^-1) 32bit C implementation
|
||||
* Generate three parity blocks with powers of 2^-1 using 32bit C implementation.
|
||||
*/
|
||||
void raid_genz_int32_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -239,7 +239,7 @@ void raid_genz_int32_raid(int nd, size_t size, void **vv)
|
|||
}
|
||||
|
||||
/*
|
||||
* GENz (triple parity with powers of 2^-1) 64bit C implementation
|
||||
* Generate three parity blocks with powers of 2^-1 using 64bit C implementation.
|
||||
*/
|
||||
void raid_genz_int64_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -292,7 +292,7 @@ void raid_genz_int64_raid(int nd, size_t size, void **vv)
|
|||
}
|
||||
|
||||
/*
|
||||
* GEN3 (triple parity with Cauchy matrix) 8bit C implementation
|
||||
* Generate three parity blocks with Cauchy matrix using 8bit C implementation.
|
||||
*
|
||||
* Note that instead of a generic multiplication table, likely resulting
|
||||
* in multiple cache misses, a precomputed table could be used.
|
||||
|
|
@ -339,7 +339,7 @@ void raid_gen3_int8(int nd, size_t size, void **vv)
|
|||
}
|
||||
|
||||
/*
|
||||
* GEN4 (quad parity with Cauchy matrix) 8bit C implementation
|
||||
* Generate four parity blocks with Cauchy matrix using 8bit C implementation.
|
||||
*
|
||||
* Note that instead of a generic multiplication table, likely resulting
|
||||
* in multiple cache misses, a precomputed table could be used.
|
||||
|
|
@ -391,7 +391,7 @@ void raid_gen4_int8(int nd, size_t size, void **vv)
|
|||
}
|
||||
|
||||
/*
|
||||
* GEN5 (penta parity with Cauchy matrix) 8bit C implementation
|
||||
* Generate five parity blocks with Cauchy matrix using 8bit C implementation.
|
||||
*
|
||||
* Note that instead of a generic multiplication table, likely resulting
|
||||
* in multiple cache misses, a precomputed table could be used.
|
||||
|
|
@ -448,7 +448,7 @@ void raid_gen5_int8(int nd, size_t size, void **vv)
|
|||
}
|
||||
|
||||
/*
|
||||
* GEN6 (hexa parity with Cauchy matrix) 8bit C implementation
|
||||
* Generate six parity blocks with Cauchy matrix using 8bit C implementation.
|
||||
*
|
||||
* Note that instead of a generic multiplication table, likely resulting
|
||||
* in multiple cache misses, a precomputed table could be used.
|
||||
|
|
|
|||
36
raid/neon.c
36
raid/neon.c
|
|
@ -7,7 +7,9 @@
|
|||
|
||||
#ifdef CONFIG_NEON
|
||||
/*
|
||||
* GEN1 (RAID5 with xor) NEON implementation
|
||||
* Generate one parity block (RAID5 with XOR) using NEON implementation.
|
||||
*
|
||||
* Uses 64-byte chunks across four 16-byte vectors.
|
||||
*/
|
||||
void raid_gen1_neon(int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -59,7 +61,7 @@ void raid_gen1_neon(int nd, size_t size, void **vv)
|
|||
}
|
||||
|
||||
/*
|
||||
* GEN2 Cauchy NEON implementation using the active generator
|
||||
* Generate two parity blocks (RAID6 with Cauchy matrix) using NEON implementation.
|
||||
*/
|
||||
static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, int generator)
|
||||
{
|
||||
|
|
@ -143,7 +145,7 @@ static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, i
|
|||
}
|
||||
|
||||
/*
|
||||
* GENz (triple parity with powers of 2^-1) NEON implementation
|
||||
* Generate three parity blocks with powers of 2^-1 using NEON implementation.
|
||||
*/
|
||||
void raid_genz_neon_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -237,7 +239,7 @@ void raid_genz_neon_raid(int nd, size_t size, void **vv)
|
|||
}
|
||||
|
||||
/*
|
||||
* GENX NEON implementation
|
||||
* Generate N parity blocks with Cauchy matrix using NEON implementation.
|
||||
*/
|
||||
static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int np, int generator)
|
||||
{
|
||||
|
|
@ -567,6 +569,17 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
|
|||
raid_neon_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* Recover multiple data failures using selected parity blocks with NEON.
|
||||
*
|
||||
* This avoids raid_delta_gen(), temporary syndrome buffers, and the
|
||||
* generation of unused parity rows.
|
||||
*
|
||||
* If P is available, preserve the complete P delta syndrome and
|
||||
* reconstruct only nr - 1 missing blocks through the inverse matrix.
|
||||
* The last missing block is obtained by XORing the reconstructed blocks
|
||||
* out of Pdelta.
|
||||
*/
|
||||
static __always_inline void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
|
|
@ -980,9 +993,6 @@ void raid_gen2_neon_aes(int nd, size_t size, void **vv)
|
|||
raid_gen2_neon_gen(nd, size, vv, 3);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN3 (triple parity with Cauchy matrix) NEON implementation
|
||||
*/
|
||||
void raid_gen3_neon_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_neon(nd, size, vv, 3, 2);
|
||||
|
|
@ -993,9 +1003,6 @@ void raid_gen3_neon_aes(int nd, size_t size, void **vv)
|
|||
raid_genX_neon(nd, size, vv, 3, 3);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN4 (quad parity with Cauchy matrix) NEON implementation
|
||||
*/
|
||||
void raid_gen4_neon_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_neon(nd, size, vv, 4, 2);
|
||||
|
|
@ -1006,9 +1013,6 @@ void raid_gen4_neon_aes(int nd, size_t size, void **vv)
|
|||
raid_genX_neon(nd, size, vv, 4, 3);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN5 (penta parity with Cauchy matrix) NEON implementation
|
||||
*/
|
||||
void raid_gen5_neon_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_neon(nd, size, vv, 5, 2);
|
||||
|
|
@ -1019,9 +1023,6 @@ void raid_gen5_neon_aes(int nd, size_t size, void **vv)
|
|||
raid_genX_neon(nd, size, vv, 5, 3);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN6 (hexa parity with Cauchy matrix) NEON implementation
|
||||
*/
|
||||
void raid_gen6_neon_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_neon(nd, size, vv, 6, 2);
|
||||
|
|
@ -1047,6 +1048,9 @@ void raid_rec1_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|||
|
||||
/*
|
||||
* Recover failure of two data blocks using P and Q AArch64 NEON implementation.
|
||||
*
|
||||
* Uses raid_delta_gen() and keeps all four multiplication tables resident
|
||||
* in NEON registers (q20..q23) with a 32-byte two-lane unroll.
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_neon(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -7,7 +7,9 @@
|
|||
|
||||
#ifdef CONFIG_NEON32
|
||||
/*
|
||||
* GEN1 (RAID5 with xor) AArch32 NEON implementation
|
||||
* Generate one parity block (RAID5 with XOR) using AArch32 NEON implementation.
|
||||
*
|
||||
* Uses 64-byte chunks across four 16-byte vectors.
|
||||
*/
|
||||
void raid_gen1_neon32(int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -62,7 +64,7 @@ void raid_gen1_neon32(int nd, size_t size, void **vv)
|
|||
}
|
||||
|
||||
/*
|
||||
* GEN2 Cauchy AArch32 NEON implementation using the active generator
|
||||
* Generate two parity blocks (RAID6 with Cauchy matrix) using AArch32 NEON implementation.
|
||||
*/
|
||||
static __always_inline void raid_gen2_neon32_gen(int nd, size_t size,
|
||||
void **vv, int generator)
|
||||
|
|
@ -143,7 +145,7 @@ static __always_inline void raid_gen2_neon32_gen(int nd, size_t size,
|
|||
}
|
||||
|
||||
/*
|
||||
* GENz (triple parity with powers of 2^-1) AArch32 NEON implementation
|
||||
* Generate three parity blocks with powers of 2^-1 using AArch32 NEON implementation.
|
||||
*/
|
||||
void raid_genz_neon32_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -230,7 +232,7 @@ void raid_genz_neon32_raid(int nd, size_t size, void **vv)
|
|||
}
|
||||
|
||||
/*
|
||||
* GENX AArch32 NEON implementation
|
||||
* Generate N parity blocks with Cauchy matrix using AArch32 NEON implementation.
|
||||
*/
|
||||
static __always_inline void raid_genX_neon32(int nd, size_t size,
|
||||
void **vv, int np, int generator)
|
||||
|
|
@ -519,6 +521,13 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
|
|||
raid_neon32_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* Recover multiple data failures using selected parity blocks with AArch32 NEON optimized for up to three failures.
|
||||
*
|
||||
* If P is available, keep the complete P delta syndrome in q6.
|
||||
* Reconstruct only nr - 1 missing blocks through the inverse matrix and
|
||||
* obtain the last block by XORing the reconstructed blocks out of Pdelta.
|
||||
*/
|
||||
static __always_inline void raid_recX_neon32_123(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
|
|
@ -816,6 +825,17 @@ static __always_inline void raid_recX_neon32_123(int nr, int *id, int *ip, int n
|
|||
raid_neon32_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* Recover multiple data failures using selected parity blocks with AArch32 NEON.
|
||||
*
|
||||
* This avoids raid_delta_gen(), temporary syndrome buffers, and the
|
||||
* generation of unused parity rows.
|
||||
*
|
||||
* If P is available, preserve the complete P delta syndrome and
|
||||
* reconstruct only nr - 1 missing blocks through the inverse matrix.
|
||||
* The last missing block is obtained by XORing the reconstructed blocks
|
||||
* out of Pdelta.
|
||||
*/
|
||||
static __always_inline void raid_recX_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
|
|
@ -1159,9 +1179,6 @@ void raid_gen2_neon32_aes(int nd, size_t size, void **vv)
|
|||
raid_gen2_neon32_gen(nd, size, vv, 3);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN3 (triple parity with Cauchy matrix) AArch32 NEON implementation
|
||||
*/
|
||||
void raid_gen3_neon32_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_neon32(nd, size, vv, 3, 2);
|
||||
|
|
@ -1172,9 +1189,6 @@ void raid_gen3_neon32_aes(int nd, size_t size, void **vv)
|
|||
raid_genX_neon32(nd, size, vv, 3, 3);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN4 (quad parity with Cauchy matrix) AArch32 NEON implementation
|
||||
*/
|
||||
void raid_gen4_neon32_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_neon32(nd, size, vv, 4, 2);
|
||||
|
|
@ -1185,9 +1199,6 @@ void raid_gen4_neon32_aes(int nd, size_t size, void **vv)
|
|||
raid_genX_neon32(nd, size, vv, 4, 3);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN5 (penta parity with Cauchy matrix) AArch32 NEON implementation
|
||||
*/
|
||||
void raid_gen5_neon32_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_neon32(nd, size, vv, 5, 2);
|
||||
|
|
@ -1198,9 +1209,6 @@ void raid_gen5_neon32_aes(int nd, size_t size, void **vv)
|
|||
raid_genX_neon32(nd, size, vv, 5, 3);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN6 (hexa parity with Cauchy matrix) AArch32 NEON implementation
|
||||
*/
|
||||
void raid_gen6_neon32_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_neon32(nd, size, vv, 6, 2);
|
||||
|
|
@ -1226,6 +1234,8 @@ void raid_rec1_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|||
|
||||
/*
|
||||
* Recover failure of two data blocks using P and Q AArch32 NEON implementation.
|
||||
*
|
||||
* Uses raid_delta_gen() and computes the analytical solution with polynomial reduction.
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_neon32(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
|
|||
15
raid/sse2.c
15
raid/sse2.c
|
|
@ -100,7 +100,9 @@
|
|||
*/
|
||||
|
||||
/*
|
||||
* GEN1 (RAID5 with xor) SSE2 implementation
|
||||
* Generate one parity block (RAID5 with XOR) using SSE2 implementation.
|
||||
*
|
||||
* Uses 64-byte chunks across four 16-byte XMM vectors.
|
||||
*/
|
||||
void raid_gen1_sse2(int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -137,7 +139,7 @@ void raid_gen1_sse2(int nd, size_t size, void **vv)
|
|||
}
|
||||
|
||||
/*
|
||||
* GEN2 Cauchy SSE2 implementation using the active generator
|
||||
* Generate two parity blocks (RAID6 with Cauchy matrix) using SSE2 implementation.
|
||||
*/
|
||||
static __always_inline void raid_gen2_sse2_gen(int nd, size_t size, void **vv, int generator)
|
||||
{
|
||||
|
|
@ -220,7 +222,10 @@ void raid_gen2_sse2_aes(int nd, size_t size, void **vv)
|
|||
|
||||
#ifdef CONFIG_X86_64
|
||||
/*
|
||||
* GEN2 Cauchy SSE2 implementation using the active generator
|
||||
* Generate two parity blocks (RAID6 with Cauchy matrix) using SSE2 extended implementation.
|
||||
*
|
||||
* Uses the extended register set (16 XMM registers), unrolling 64-byte chunks
|
||||
* across four 16-byte lanes and processing two disks per iteration.
|
||||
*
|
||||
* Note that it uses 16 registers, meaning that x64 is required.
|
||||
*/
|
||||
|
|
@ -458,7 +463,7 @@ void raid_gen2_sse2ext_aes(int nd, size_t size, void **vv)
|
|||
#endif
|
||||
|
||||
/*
|
||||
* GENz (triple parity with powers of 2^-1) SSE2 implementation
|
||||
* Generate three parity blocks with powers of 2^-1 using SSE2 implementation.
|
||||
*/
|
||||
void raid_genz_sse2_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -515,7 +520,7 @@ void raid_genz_sse2_raid(int nd, size_t size, void **vv)
|
|||
|
||||
#ifdef CONFIG_X86_64
|
||||
/*
|
||||
* GENz (triple parity with powers of 2^-1) SSE2 implementation
|
||||
* Generate three parity blocks with powers of 2^-1 using SSE2 extended implementation.
|
||||
*
|
||||
* Note that it uses 16 registers, meaning that x64 is required.
|
||||
*/
|
||||
|
|
|
|||
79
raid/ssse3.c
79
raid/ssse3.c
|
|
@ -7,7 +7,7 @@
|
|||
|
||||
#ifdef CONFIG_X86
|
||||
/*
|
||||
* GEN3 (triple parity with Cauchy matrix) SSSE3 implementation
|
||||
* Generate three parity blocks with Cauchy matrix using SSSE3 implementation.
|
||||
*/
|
||||
static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv, int generator)
|
||||
{
|
||||
|
|
@ -111,7 +111,7 @@ static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv,
|
|||
}
|
||||
|
||||
/*
|
||||
* GEN4 (quad parity with Cauchy matrix) SSSE3 implementation
|
||||
* Generate four parity blocks with Cauchy matrix using SSSE3 implementation.
|
||||
*/
|
||||
static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv, int generator)
|
||||
{
|
||||
|
|
@ -234,7 +234,9 @@ static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv,
|
|||
}
|
||||
|
||||
/*
|
||||
* GEN5 (penta parity with Cauchy matrix) SSSE3 implementation
|
||||
* Generate five parity blocks with Cauchy matrix using SSSE3 implementation.
|
||||
*
|
||||
* Uses an aligned stack buffer for parity accumulators under 8-register pressure.
|
||||
*/
|
||||
void raid_gen5_ssse3_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -370,7 +372,9 @@ void raid_gen5_ssse3_raid(int nd, size_t size, void **vv)
|
|||
}
|
||||
|
||||
/*
|
||||
* GEN6 (hexa parity with Cauchy matrix) SSSE3 implementation
|
||||
* Generate six parity blocks with Cauchy matrix using SSSE3 implementation.
|
||||
*
|
||||
* Uses an aligned stack buffer for parity accumulators under 8-register pressure.
|
||||
*/
|
||||
void raid_gen6_ssse3_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -528,7 +532,10 @@ void raid_gen6_ssse3_raid(int nd, size_t size, void **vv)
|
|||
|
||||
#ifdef CONFIG_X86_64
|
||||
/*
|
||||
* GEN3 (triple parity with Cauchy matrix) SSSE3 implementation
|
||||
* Generate three parity blocks with Cauchy matrix using SSSE3 extended implementation.
|
||||
*
|
||||
* Uses the extended register set (16 XMM registers) and processes two disks
|
||||
* per iteration across two 16-byte XMM lanes (32 bytes/step).
|
||||
*
|
||||
* Note that it uses 16 registers, meaning that x64 is required.
|
||||
*/
|
||||
|
|
@ -787,7 +794,10 @@ static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **v
|
|||
}
|
||||
|
||||
/*
|
||||
* GEN4 (quad parity with Cauchy matrix) SSSE3 implementation
|
||||
* Generate four parity blocks with Cauchy matrix using SSSE3 extended implementation.
|
||||
*
|
||||
* Uses the extended register set (16 XMM registers) and processes two disks
|
||||
* per iteration across two 16-byte XMM lanes (32 bytes/step).
|
||||
*
|
||||
* Note that it uses 16 registers, meaning that x64 is required.
|
||||
*/
|
||||
|
|
@ -975,7 +985,7 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v
|
|||
}
|
||||
|
||||
/*
|
||||
* GENX SSSE3ext implementation
|
||||
* Generate N parity blocks with Cauchy matrix using SSSE3 extended implementation.
|
||||
*/
|
||||
static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, int np, int generator)
|
||||
{
|
||||
|
|
@ -1157,7 +1167,9 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i
|
|||
#endif
|
||||
|
||||
/*
|
||||
* RAID recovering for one disk SSSE3 implementation
|
||||
* Recover failure of one data block using SSSE3 delta implementation.
|
||||
*
|
||||
* Uses raid_delta_gen() and multiplies by the inverted coefficient table.
|
||||
*/
|
||||
static __always_inline void raid_rec1_ssse3_delta(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -1206,7 +1218,9 @@ static __always_inline void raid_rec1_ssse3_delta(int *id, int *ip, int nd, size
|
|||
}
|
||||
|
||||
/*
|
||||
* RAID recovering for two disks SSSE3 implementation
|
||||
* Recover failure of two data blocks using SSSE3 delta implementation.
|
||||
*
|
||||
* Uses raid_delta_gen() and solves the 2x2 system directly on delta blocks.
|
||||
*/
|
||||
static __always_inline void raid_rec2_ssse3_delta(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -1308,6 +1322,8 @@ static __always_inline void raid_rec2_ssse3_delta(int *id, int *ip, int nd, size
|
|||
|
||||
/*
|
||||
* Recover failure of two data blocks using P and Q SSSE3 implementation.
|
||||
*
|
||||
* Uses raid_delta_gen() and computes the analytical solution.
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_ssse3(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -1384,7 +1400,7 @@ static __always_inline void raid_rec2of2_ssse3(int *id, int *ip, int nd, size_t
|
|||
}
|
||||
|
||||
/*
|
||||
* RAID recovering SSSE3 implementation optimized for up to four failures.
|
||||
* Recover multiple data failures using selected parity blocks with SSSE3 optimized for up to four failures.
|
||||
*
|
||||
* If P is available, keep the complete P delta syndrome in xmm6.
|
||||
* Reconstruct only nr - 1 missing blocks through the inverse matrix and
|
||||
|
|
@ -1609,7 +1625,7 @@ static __always_inline void raid_recX_ssse3_1234(int nr, int *id, int *ip, int n
|
|||
}
|
||||
|
||||
/*
|
||||
* RAID recovering SSSE3 implementation.
|
||||
* Recover multiple data failures using selected parity blocks with SSSE3.
|
||||
*
|
||||
* If P is available, keep the complete P delta syndrome in xmm6.
|
||||
* Reconstruct only nr - 1 missing blocks through the inverse matrix and
|
||||
|
|
@ -1776,7 +1792,7 @@ static __always_inline void raid_recX_ssse3(int nr, int *id, int *ip, int nd, si
|
|||
|
||||
#ifdef CONFIG_X86_64
|
||||
/*
|
||||
* RAID recovering for one disk SSSE3 extended implementation.
|
||||
* Recover failure of one data block using SSSE3 extended delta implementation.
|
||||
*
|
||||
* Process two 16-byte blocks per iteration, grouping equal operations across
|
||||
* both lanes to expose instruction-level parallelism.
|
||||
|
|
@ -1856,7 +1872,10 @@ static __always_inline void raid_rec1_ssse3ext_delta(int *id, int *ip, int nd, s
|
|||
}
|
||||
|
||||
/*
|
||||
* RAID recovering for two disks SSSE3 extended implementation
|
||||
* Recover failure of two data blocks using SSSE3 extended delta implementation.
|
||||
*
|
||||
* Uses raid_delta_gen() and keeps all 8 inverse-matrix tables resident
|
||||
* in XMM registers (xmm8..xmm15).
|
||||
*/
|
||||
static __always_inline void raid_rec2_ssse3ext_delta(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -1964,6 +1983,9 @@ static __always_inline void raid_rec2_ssse3ext_delta(int *id, int *ip, int nd, s
|
|||
|
||||
/*
|
||||
* Recover failure of two data blocks using P and Q SSSE3 extended implementation.
|
||||
*
|
||||
* Uses raid_delta_gen() and keeps all four multiplication tables resident
|
||||
* in XMM registers (xmm8..xmm11).
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_ssse3ext(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -2043,6 +2065,13 @@ static __always_inline void raid_rec2of2_ssse3ext(int *id, int *ip, int nd, size
|
|||
raid_sse_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* Recover multiple data failures using selected parity blocks with SSSE3 extended optimized for up to three failures.
|
||||
*
|
||||
* If P is available, keep the complete P delta syndrome in xmm6.
|
||||
* Reconstruct only nr - 1 missing blocks through the inverse matrix and
|
||||
* obtain the last block by XORing the reconstructed blocks out of Pdelta.
|
||||
*/
|
||||
static __always_inline void raid_recX_ssse3ext_123(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
|
|
@ -2293,8 +2322,7 @@ static __always_inline void raid_recX_ssse3ext_123(int nr, int *id, int *ip, int
|
|||
}
|
||||
|
||||
/*
|
||||
* RAID recovering SSSE3 extended implementation optimized for up to five
|
||||
* failures.
|
||||
* Recover multiple data failures using selected parity blocks with SSSE3 extended optimized for up to five failures.
|
||||
*
|
||||
* If P is available, keep the complete P delta syndrome in xmm10.
|
||||
* Reconstruct only nr - 1 missing blocks through the inverse matrix and
|
||||
|
|
@ -2593,6 +2621,17 @@ static __always_inline void raid_recX_ssse3ext_12345(int nr, int *id, int *ip, i
|
|||
raid_sse_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* Recover multiple data failures using selected parity blocks with SSSE3 extended.
|
||||
*
|
||||
* This avoids raid_delta_gen(), temporary syndrome buffers, and the
|
||||
* generation of unused parity rows.
|
||||
*
|
||||
* If P is available, preserve the complete P delta syndrome and
|
||||
* reconstruct only nr - 1 missing blocks through the inverse matrix.
|
||||
* The last missing block is obtained by XORing the reconstructed blocks
|
||||
* out of Pdelta.
|
||||
*/
|
||||
static __always_inline void raid_recX_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
|
|
@ -2895,11 +2934,6 @@ void raid_gen4_ssse3ext_aes(int nd, size_t size, void **vv)
|
|||
raid_gen4_ssse3ext_gen(nd, size, vv, 3);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN5 (penta parity with Cauchy matrix) SSSE3 implementation
|
||||
*
|
||||
* Note that it uses 16 registers, meaning that x64 is required.
|
||||
*/
|
||||
void raid_gen5_ssse3ext_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_ssse3ext(nd, size, vv, 5, 2);
|
||||
|
|
@ -2910,11 +2944,6 @@ void raid_gen5_ssse3ext_aes(int nd, size_t size, void **vv)
|
|||
raid_genX_ssse3ext(nd, size, vv, 5, 3);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN6 (hexa parity with Cauchy matrix) SSSE3 implementation
|
||||
*
|
||||
* Note that it uses 16 registers, meaning that x64 is required.
|
||||
*/
|
||||
void raid_gen6_ssse3ext_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_ssse3ext(nd, size, vv, 6, 2);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue