rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 19:01:57 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Specialize RAID recovery functions by passing has_p as a constant argument

Browse source
This commit is contained in:
Andrea Mazzoleni 2026-08-27 18:54:10 +02:00
parent e0e3618240
commit 69a1ed1125
6 changed files with 256 additions and 146 deletions
Download patch file Download diff file Expand all files Collapse all files

81
raid/avx2.c
View file

@ -1009,7 +1009,7 @@ static __always_inline void raid_rec1_avx2_1(int *id, int *ip, int nd, size_t si
* Reconstruct only nr - 1 missing blocks through the inverse matrix and
* obtain the last block by XORing the reconstructed blocks out of Pdelta.
*/
static __always_inline void raid_recX_avx2_1234(int nr, int *id, int *ip, int nd, size_t size, void **vv)
static __always_inline void raid_recX_avx2_1234(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p[RAID_PARITY_MAX];
@ -1026,7 +1026,6 @@ static __always_inline void raid_recX_avx2_1234(int nr, int *id, int *ip, int nd
size_t i;
int d, j, k, s;
int ns;
int has_p;
BUG_ON(nr < 1 || nr > 4);
@ -1044,9 +1043,6 @@ static __always_inline void raid_recX_avx2_1234(int nr, int *id, int *ip, int nd
pa[j] = v[id[j]];
}
/* ip[] is ordered. If P is available, it is always ip[0] */
has_p = ip[0] == 0;
/* build the compact surviving-data list and its syndrome tables */
ns = 0;
k = 0;
@ -1218,7 +1214,7 @@ static __always_inline void raid_recX_avx2_1234(int nr, int *id, int *ip, int nd
* Reconstruct only nr - 1 missing blocks through the inverse matrix and
* obtain the last block by XORing the reconstructed blocks out of Pdelta.
*/
static __always_inline void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
static __always_inline void raid_recX_avx2(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p[RAID_PARITY_MAX];
@ -1235,7 +1231,6 @@ static __always_inline void raid_recX_avx2(int nr, int *id, int *ip, int nd, siz
size_t i;
int d, j, k, s;
int ns;
int has_p;
BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
@ -1253,9 +1248,6 @@ static __always_inline void raid_recX_avx2(int nr, int *id, int *ip, int nd, siz
pa[j] = v[id[j]];
}
/* ip[] is ordered. If P is available, it is always ip[0] */
has_p = ip[0] == 0;
/* build the compact surviving-data list and its syndrome tables */
ns = 0;
k = 0;
@ -1446,7 +1438,7 @@ static __always_inline void raid_recX_avx2(int nr, int *id, int *ip, int nd, siz
*
* Note that it uses 16 registers, meaning that x64 is required.
*/
static __always_inline void raid_recX_avx2ext_12(int nr, int *id, int *ip, int nd, size_t size, void **vv)
static __always_inline void raid_recX_avx2ext_12(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p[RAID_PARITY_MAX];
@ -1459,7 +1451,6 @@ static __always_inline void raid_recX_avx2ext_12(int nr, int *id, int *ip, int n
size_t i;
int d, j, k, s;
int ns;
int has_p;
BUG_ON(nr < 1 || nr > 2);
@ -1477,9 +1468,6 @@ static __always_inline void raid_recX_avx2ext_12(int nr, int *id, int *ip, int n
pa[j] = v[id[j]];
}
/* ip[] is ordered. If P is available, it is always ip[0] */
has_p = ip[0] == 0;
/* build the compact surviving-data list and its syndrome tables */
ns = 0;
k = 0;
@ -1716,7 +1704,7 @@ static __always_inline void raid_recX_avx2ext_12(int nr, int *id, int *ip, int n
*
* Note that it uses 16 registers, meaning that x64 is required.
*/
static __always_inline void raid_recX_avx2ext_123(int nr, int *id, int *ip, int nd, size_t size, void **vv)
static __always_inline void raid_recX_avx2ext_123(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p[RAID_PARITY_MAX];
@ -1729,7 +1717,6 @@ static __always_inline void raid_recX_avx2ext_123(int nr, int *id, int *ip, int
size_t i;
int d, j, k, s;
int ns;
int has_p;
BUG_ON(nr < 1 || nr > 3);
@ -1747,9 +1734,6 @@ static __always_inline void raid_recX_avx2ext_123(int nr, int *id, int *ip, int
pa[j] = v[id[j]];
}
/* ip[] is ordered. If P is available, it is always ip[0] */
has_p = ip[0] == 0;
/* build the compact surviving-data list and its syndrome tables */
ns = 0;
k = 0;
@ -2023,7 +2007,7 @@ static __always_inline void raid_recX_avx2ext_123(int nr, int *id, int *ip, int
* The last missing block is obtained by XORing the reconstructed blocks
* out of Pdelta.
*/
static __always_inline void raid_recX_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
static __always_inline void raid_recX_avx2ext(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p[RAID_PARITY_MAX];
@ -2036,7 +2020,6 @@ static __always_inline void raid_recX_avx2ext(int nr, int *id, int *ip, int nd,
size_t i;
int d, j, k, s;
int ns;
int has_p;
BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
@ -2054,9 +2037,6 @@ static __always_inline void raid_recX_avx2ext(int nr, int *id, int *ip, int nd,
pa[j] = v[id[j]];
}
/* ip[] is ordered. If P is available, it is always ip[0] */
has_p = ip[0] == 0;
/* build the compact surviving-data list and its syndrome tables */
ns = 0;
k = 0;
@ -2431,31 +2411,46 @@ void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 2);
raid_recX_avx2_1234(2, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx2_1234(2, 1, id, ip, nd, size, vv);
else
raid_recX_avx2_1234(2, 0, id, ip, nd, size, vv);
}
void raid_rec3_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 3);
raid_recX_avx2_1234(3, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx2_1234(3, 1, id, ip, nd, size, vv);
else
raid_recX_avx2_1234(3, 0, id, ip, nd, size, vv);
}
void raid_rec4_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 4);
raid_recX_avx2_1234(4, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx2_1234(4, 1, id, ip, nd, size, vv);
else
raid_recX_avx2_1234(4, 0, id, ip, nd, size, vv);
}
void raid_rec5_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 5);
raid_recX_avx2(5, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx2(5, 1, id, ip, nd, size, vv);
else
raid_recX_avx2(5, 0, id, ip, nd, size, vv);
}
void raid_rec6_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 6);
raid_recX_avx2(6, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx2(6, 1, id, ip, nd, size, vv);
else
raid_recX_avx2(6, 0, id, ip, nd, size, vv);
}
#ifdef CONFIG_X86_64
@ -2475,33 +2470,47 @@ void raid_rec1_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
void raid_rec2_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 2);
raid_recX_avx2ext_12(2, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx2ext_12(2, 1, id, ip, nd, size, vv);
else
raid_recX_avx2ext_12(2, 0, id, ip, nd, size, vv);
}
void raid_rec3_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 3);
raid_recX_avx2ext_123(3, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx2ext_123(3, 1, id, ip, nd, size, vv);
else
raid_recX_avx2ext_123(3, 0, id, ip, nd, size, vv);
}
void raid_rec4_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 4);
raid_recX_avx2ext(4, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx2ext(4, 1, id, ip, nd, size, vv);
else
raid_recX_avx2ext(4, 0, id, ip, nd, size, vv);
}
void raid_rec5_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 5);
raid_recX_avx2ext(5, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx2ext(5, 1, id, ip, nd, size, vv);
else
raid_recX_avx2ext(5, 0, id, ip, nd, size, vv);
}
void raid_rec6_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 6);
raid_recX_avx2ext(6, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx2ext(6, 1, id, ip, nd, size, vv);
else
raid_recX_avx2ext(6, 0, id, ip, nd, size, vv);
}
#endif
void raid_register_avx2(void)

36
raid/avx512.c
View file

@ -567,7 +567,7 @@ static __always_inline void raid_genX_avx512bw(int nd, size_t size, void **vv, i
* AVX512 provides enough registers to keep all six supported syndromes
* in registers in a single surviving-data scan.
*/
static __always_inline void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
static __always_inline void raid_recX_avx512bw(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p[RAID_PARITY_MAX];
@ -580,7 +580,6 @@ static __always_inline void raid_recX_avx512bw(int nr, int *id, int *ip, int nd,
size_t i;
int d, j, k, s;
int ns;
int has_p;
BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
@ -598,9 +597,6 @@ static __always_inline void raid_recX_avx512bw(int nr, int *id, int *ip, int nd,
pa[j] = v[id[j]];
}
/* ip[] is ordered. If P is available, it is always ip[0] */
has_p = ip[0] == 0;
/*
* Build the compact list of surviving data blocks and precompute
* the multiplication-table pointers for each selected syndrome.
@ -911,37 +907,55 @@ void raid_gen6_avx512bw(int nd, size_t size, void **vv)
void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 1);
raid_recX_avx512bw(1, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx512bw(1, 1, id, ip, nd, size, vv);
else
raid_recX_avx512bw(1, 0, id, ip, nd, size, vv);
}
void raid_rec2_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 2);
raid_recX_avx512bw(2, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx512bw(2, 1, id, ip, nd, size, vv);
else
raid_recX_avx512bw(2, 0, id, ip, nd, size, vv);
}
void raid_rec3_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 3);
raid_recX_avx512bw(3, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx512bw(3, 1, id, ip, nd, size, vv);
else
raid_recX_avx512bw(3, 0, id, ip, nd, size, vv);
}
void raid_rec4_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 4);
raid_recX_avx512bw(4, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx512bw(4, 1, id, ip, nd, size, vv);
else
raid_recX_avx512bw(4, 0, id, ip, nd, size, vv);
}
void raid_rec5_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 5);
raid_recX_avx512bw(5, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx512bw(5, 1, id, ip, nd, size, vv);
else
raid_recX_avx512bw(5, 0, id, ip, nd, size, vv);
}
void raid_rec6_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 6);
raid_recX_avx512bw(6, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx512bw(6, 1, id, ip, nd, size, vv);
else
raid_recX_avx512bw(6, 0, id, ip, nd, size, vv);
}
void raid_register_avx512(void)

140
raid/gfni.c
View file

@ -1355,7 +1355,7 @@ static __always_inline void raid_genX_avx512gfni_aes(int nd, size_t size, void *
* If P is available, reconstruct only nr - 1 missing blocks through the
* inverse matrix and derive the last missing block from the P delta by XOR.
*/
static __always_inline void raid_recX_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
static __always_inline void raid_recX_avx2gfni_raid(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p[RAID_PARITY_MAX];
@ -1368,7 +1368,6 @@ static __always_inline void raid_recX_avx2gfni_raid(int nr, int *id, int *ip, in
size_t i;
int d, j, k, s;
int ns;
int has_p;
BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
@ -1383,8 +1382,6 @@ static __always_inline void raid_recX_avx2gfni_raid(int nr, int *id, int *ip, in
pa[j] = v[id[j]];
}
has_p = ip[0] == 0;
ns = 0;
k = 0;
@ -1563,7 +1560,7 @@ static __always_inline void raid_recX_avx2gfni_raid(int nr, int *id, int *ip, in
* If P is available, reconstruct only nr - 1 missing blocks through the
* inverse matrix and derive the last missing block from the P delta by XOR.
*/
static __always_inline void raid_recX_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
static __always_inline void raid_recX_avx512gfni_raid(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p[RAID_PARITY_MAX];
@ -1576,7 +1573,6 @@ static __always_inline void raid_recX_avx512gfni_raid(int nr, int *id, int *ip,
size_t i;
int d, j, k, s;
int ns;
int has_p;
BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
@ -1591,8 +1587,6 @@ static __always_inline void raid_recX_avx512gfni_raid(int nr, int *id, int *ip,
pa[j] = v[id[j]];
}
has_p = ip[0] == 0;
ns = 0;
k = 0;
@ -1744,7 +1738,7 @@ static __always_inline void raid_recX_avx512gfni_raid(int nr, int *id, int *ip,
* If P is available, reconstruct only nr - 1 missing blocks through the
* inverse matrix and derive the last missing block from the P delta by XOR.
*/
static __always_inline void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
static __always_inline void raid_recX_avx2gfni_aes(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p[RAID_PARITY_MAX];
@ -1756,7 +1750,6 @@ static __always_inline void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int
size_t i;
int d, j, k, s;
int ns;
int has_p;
BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
@ -1771,8 +1764,6 @@ static __always_inline void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int
pa[j] = v[id[j]];
}
has_p = ip[0] == 0;
ns = 0;
k = 0;
@ -1943,7 +1934,7 @@ static __always_inline void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int
* If P is available, reconstruct only nr - 1 missing blocks through the
* inverse matrix and derive the last missing block from the P delta by XOR.
*/
static __always_inline void raid_recX_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
static __always_inline void raid_recX_avx512gfni_aes(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p[RAID_PARITY_MAX];
@ -1955,7 +1946,6 @@ static __always_inline void raid_recX_avx512gfni_aes(int nr, int *id, int *ip, i
size_t i;
int d, j, k, s;
int ns;
int has_p;
BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
@ -1970,8 +1960,6 @@ static __always_inline void raid_recX_avx512gfni_aes(int nr, int *id, int *ip, i
pa[j] = v[id[j]];
}
has_p = ip[0] == 0;
ns = 0;
k = 0;
@ -2203,145 +2191,217 @@ void raid_gen6_avx512gfni_aes(int nd, size_t size, void **vv)
void raid_rec1_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 1);
raid_recX_avx2gfni_raid(1, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx2gfni_raid(1, 1, id, ip, nd, size, vv);
else
raid_recX_avx2gfni_raid(1, 0, id, ip, nd, size, vv);
}
void raid_rec2_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 2);
raid_recX_avx2gfni_raid(2, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx2gfni_raid(2, 1, id, ip, nd, size, vv);
else
raid_recX_avx2gfni_raid(2, 0, id, ip, nd, size, vv);
}
void raid_rec3_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 3);
raid_recX_avx2gfni_raid(3, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx2gfni_raid(3, 1, id, ip, nd, size, vv);
else
raid_recX_avx2gfni_raid(3, 0, id, ip, nd, size, vv);
}
void raid_rec4_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 4);
raid_recX_avx2gfni_raid(4, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx2gfni_raid(4, 1, id, ip, nd, size, vv);
else
raid_recX_avx2gfni_raid(4, 0, id, ip, nd, size, vv);
}
void raid_rec5_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 5);
raid_recX_avx2gfni_raid(5, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx2gfni_raid(5, 1, id, ip, nd, size, vv);
else
raid_recX_avx2gfni_raid(5, 0, id, ip, nd, size, vv);
}
void raid_rec6_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 6);
raid_recX_avx2gfni_raid(6, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx2gfni_raid(6, 1, id, ip, nd, size, vv);
else
raid_recX_avx2gfni_raid(6, 0, id, ip, nd, size, vv);
}
void raid_rec1_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 1);
raid_recX_avx512gfni_raid(1, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx512gfni_raid(1, 1, id, ip, nd, size, vv);
else
raid_recX_avx512gfni_raid(1, 0, id, ip, nd, size, vv);
}
void raid_rec2_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 2);
raid_recX_avx512gfni_raid(2, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx512gfni_raid(2, 1, id, ip, nd, size, vv);
else
raid_recX_avx512gfni_raid(2, 0, id, ip, nd, size, vv);
}
void raid_rec3_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 3);
raid_recX_avx512gfni_raid(3, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx512gfni_raid(3, 1, id, ip, nd, size, vv);
else
raid_recX_avx512gfni_raid(3, 0, id, ip, nd, size, vv);
}
void raid_rec4_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 4);
raid_recX_avx512gfni_raid(4, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx512gfni_raid(4, 1, id, ip, nd, size, vv);
else
raid_recX_avx512gfni_raid(4, 0, id, ip, nd, size, vv);
}
void raid_rec5_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 5);
raid_recX_avx512gfni_raid(5, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx512gfni_raid(5, 1, id, ip, nd, size, vv);
else
raid_recX_avx512gfni_raid(5, 0, id, ip, nd, size, vv);
}
void raid_rec6_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 6);
raid_recX_avx512gfni_raid(6, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx512gfni_raid(6, 1, id, ip, nd, size, vv);
else
raid_recX_avx512gfni_raid(6, 0, id, ip, nd, size, vv);
}
void raid_rec1_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 1);
raid_recX_avx2gfni_aes(1, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx2gfni_aes(1, 1, id, ip, nd, size, vv);
else
raid_recX_avx2gfni_aes(1, 0, id, ip, nd, size, vv);
}
void raid_rec2_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 2);
raid_recX_avx2gfni_aes(2, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx2gfni_aes(2, 1, id, ip, nd, size, vv);
else
raid_recX_avx2gfni_aes(2, 0, id, ip, nd, size, vv);
}
void raid_rec3_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 3);
raid_recX_avx2gfni_aes(3, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx2gfni_aes(3, 1, id, ip, nd, size, vv);
else
raid_recX_avx2gfni_aes(3, 0, id, ip, nd, size, vv);
}
void raid_rec4_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 4);
raid_recX_avx2gfni_aes(4, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx2gfni_aes(4, 1, id, ip, nd, size, vv);
else
raid_recX_avx2gfni_aes(4, 0, id, ip, nd, size, vv);
}
void raid_rec5_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 5);
raid_recX_avx2gfni_aes(5, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx2gfni_aes(5, 1, id, ip, nd, size, vv);
else
raid_recX_avx2gfni_aes(5, 0, id, ip, nd, size, vv);
}
void raid_rec6_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 6);
raid_recX_avx2gfni_aes(6, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx2gfni_aes(6, 1, id, ip, nd, size, vv);
else
raid_recX_avx2gfni_aes(6, 0, id, ip, nd, size, vv);
}
void raid_rec1_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 1);
raid_recX_avx512gfni_aes(1, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx512gfni_aes(1, 1, id, ip, nd, size, vv);
else
raid_recX_avx512gfni_aes(1, 0, id, ip, nd, size, vv);
}
void raid_rec2_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 2);
raid_recX_avx512gfni_aes(2, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx512gfni_aes(2, 1, id, ip, nd, size, vv);
else
raid_recX_avx512gfni_aes(2, 0, id, ip, nd, size, vv);
}
void raid_rec3_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 3);
raid_recX_avx512gfni_aes(3, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx512gfni_aes(3, 1, id, ip, nd, size, vv);
else
raid_recX_avx512gfni_aes(3, 0, id, ip, nd, size, vv);
}
void raid_rec4_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 4);
raid_recX_avx512gfni_aes(4, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx512gfni_aes(4, 1, id, ip, nd, size, vv);
else
raid_recX_avx512gfni_aes(4, 0, id, ip, nd, size, vv);
}
void raid_rec5_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 5);
raid_recX_avx512gfni_aes(5, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx512gfni_aes(5, 1, id, ip, nd, size, vv);
else
raid_recX_avx512gfni_aes(5, 0, id, ip, nd, size, vv);
}
void raid_rec6_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 6);
raid_recX_avx512gfni_aes(6, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_avx512gfni_aes(6, 1, id, ip, nd, size, vv);
else
raid_recX_avx512gfni_aes(6, 0, id, ip, nd, size, vv);
}
void raid_register_avx2gfni(void)

42
raid/neon.c
View file

@ -580,7 +580,7 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
* The last missing block is obtained by XORing the reconstructed blocks
* out of Pdelta.
*/
static __always_inline void raid_recX_neon_12(int nr, int *id, int *ip, int nd, size_t size, void **vv)
static __always_inline void raid_recX_neon_12(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p[RAID_PARITY_MAX];
@ -593,7 +593,6 @@ static __always_inline void raid_recX_neon_12(int nr, int *id, int *ip, int nd,
size_t i;
int d, j, k, s;
int ns;
int has_p;
BUG_ON(nr < 1 || nr > 2);
@ -611,9 +610,6 @@ static __always_inline void raid_recX_neon_12(int nr, int *id, int *ip, int nd,
pa[j] = v[id[j]];
}
/* ip[] is ordered. If P is available, it is always ip[0] */
has_p = ip[0] == 0;
/*
* Build the compact list of surviving data blocks and precompute
* the multiplication-table pointers for each selected syndrome.
@ -929,7 +925,7 @@ static __always_inline void raid_recX_neon_12(int nr, int *id, int *ip, int nd,
* The last missing block is obtained by XORing the reconstructed blocks
* out of Pdelta.
*/
static __always_inline void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
static __always_inline void raid_recX_neon(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p[RAID_PARITY_MAX];
@ -942,7 +938,6 @@ static __always_inline void raid_recX_neon(int nr, int *id, int *ip, int nd, siz
size_t i;
int d, j, k, s;
int ns;
int has_p;
BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
@ -960,9 +955,6 @@ static __always_inline void raid_recX_neon(int nr, int *id, int *ip, int nd, siz
pa[j] = v[id[j]];
}
/* ip[] is ordered. If P is available, it is always ip[0] */
has_p = ip[0] == 0;
/*
* Build the compact list of surviving data blocks and precompute
* the multiplication-table pointers for each selected syndrome.
@ -1385,37 +1377,55 @@ void raid_gen6_neon_aes(int nd, size_t size, void **vv)
void raid_rec1_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 1);
raid_recX_neon_12(1, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_neon_12(1, 1, id, ip, nd, size, vv);
else
raid_recX_neon_12(1, 0, id, ip, nd, size, vv);
}
void raid_rec2_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 2);
raid_recX_neon_12(2, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_neon_12(2, 1, id, ip, nd, size, vv);
else
raid_recX_neon_12(2, 0, id, ip, nd, size, vv);
}
void raid_rec3_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 3);
raid_recX_neon(3, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_neon(3, 1, id, ip, nd, size, vv);
else
raid_recX_neon(3, 0, id, ip, nd, size, vv);
}
void raid_rec4_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 4);
raid_recX_neon(4, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_neon(4, 1, id, ip, nd, size, vv);
else
raid_recX_neon(4, 0, id, ip, nd, size, vv);
}
void raid_rec5_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 5);
raid_recX_neon(5, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_neon(5, 1, id, ip, nd, size, vv);
else
raid_recX_neon(5, 0, id, ip, nd, size, vv);
}
void raid_rec6_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 6);
raid_recX_neon(6, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_neon(6, 1, id, ip, nd, size, vv);
else
raid_recX_neon(6, 0, id, ip, nd, size, vv);
}
void raid_register_neon(void)

37
raid/neon32.c
View file

@ -528,7 +528,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
* Reconstruct only nr - 1 missing blocks through the inverse matrix and
* obtain the last block by XORing the reconstructed blocks out of Pdelta.
*/
static __always_inline void raid_recX_neon32_123(int nr, int *id, int *ip, int nd, size_t size, void **vv)
static __always_inline void raid_recX_neon32_123(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p[RAID_PARITY_MAX];
@ -541,7 +541,6 @@ static __always_inline void raid_recX_neon32_123(int nr, int *id, int *ip, int n
size_t i;
int d, j, k, s;
int ns;
int has_p;
BUG_ON(nr < 1 || nr > 3);
@ -556,8 +555,6 @@ static __always_inline void raid_recX_neon32_123(int nr, int *id, int *ip, int n
pa[j] = v[id[j]];
}
has_p = ip[0] == 0;
ns = 0;
k = 0;
@ -836,7 +833,7 @@ static __always_inline void raid_recX_neon32_123(int nr, int *id, int *ip, int n
* The last missing block is obtained by XORing the reconstructed blocks
* out of Pdelta.
*/
static __always_inline void raid_recX_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
static __always_inline void raid_recX_neon32(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p[RAID_PARITY_MAX];
@ -849,7 +846,6 @@ static __always_inline void raid_recX_neon32(int nr, int *id, int *ip, int nd, s
size_t i;
int d, j, k, s;
int ns;
int has_p;
BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
@ -864,8 +860,6 @@ static __always_inline void raid_recX_neon32(int nr, int *id, int *ip, int nd, s
pa[j] = v[id[j]];
}
has_p = ip[0] == 0;
ns = 0;
k = 0;
@ -1229,7 +1223,7 @@ void raid_rec1_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
return;
}
raid_recX_neon32_123(1, id, ip, nd, size, vv);
raid_recX_neon32_123(1, 0, id, ip, nd, size, vv);
}
/*
@ -1342,31 +1336,46 @@ void raid_rec2_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
return;
}
raid_recX_neon32_123(2, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_neon32_123(2, 1, id, ip, nd, size, vv);
else
raid_recX_neon32_123(2, 0, id, ip, nd, size, vv);
}
void raid_rec3_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 3);
raid_recX_neon32_123(3, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_neon32_123(3, 1, id, ip, nd, size, vv);
else
raid_recX_neon32_123(3, 0, id, ip, nd, size, vv);
}
void raid_rec4_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 4);
raid_recX_neon32(4, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_neon32(4, 1, id, ip, nd, size, vv);
else
raid_recX_neon32(4, 0, id, ip, nd, size, vv);
}
void raid_rec5_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 5);
raid_recX_neon32(5, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_neon32(5, 1, id, ip, nd, size, vv);
else
raid_recX_neon32(5, 0, id, ip, nd, size, vv);
}
void raid_rec6_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 6);
raid_recX_neon32(6, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_neon32(6, 1, id, ip, nd, size, vv);
else
raid_recX_neon32(6, 0, id, ip, nd, size, vv);
}
void raid_register_neon32(void)

66
raid/ssse3.c
View file

@ -1409,7 +1409,7 @@ static __always_inline void raid_rec2of2_ssse3(int *id, int *ip, int nd, size_t
* After the survivor scan xmm6 is no longer needed, leaving it available
* for the P delta accumulator.
*/
static __always_inline void raid_recX_ssse3_1234(int nr, int *id, int *ip, int nd, size_t size, void **vv)
static __always_inline void raid_recX_ssse3_1234(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p[RAID_PARITY_MAX];
@ -1426,7 +1426,6 @@ static __always_inline void raid_recX_ssse3_1234(int nr, int *id, int *ip, int n
size_t i;
int d, j, k, s;
int ns;
int has_p;
BUG_ON(nr < 1 || nr > 4);
@ -1441,8 +1440,6 @@ static __always_inline void raid_recX_ssse3_1234(int nr, int *id, int *ip, int n
pa[j] = v[id[j]];
}
has_p = ip[0] == 0;
ns = 0;
k = 0;
@ -1634,7 +1631,7 @@ static __always_inline void raid_recX_ssse3_1234(int nr, int *id, int *ip, int n
* The completed syndromes are kept in temporary memory, leaving xmm6
* available for the P delta accumulator during reconstruction.
*/
static __always_inline void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
static __always_inline void raid_recX_ssse3(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p[RAID_PARITY_MAX];
@ -1651,7 +1648,6 @@ static __always_inline void raid_recX_ssse3(int nr, int *id, int *ip, int nd, si
size_t i;
int d, j, k, s;
int ns;
int has_p;
BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
@ -1666,8 +1662,6 @@ static __always_inline void raid_recX_ssse3(int nr, int *id, int *ip, int nd, si
pa[j] = v[id[j]];
}
has_p = ip[0] == 0;
ns = 0;
k = 0;
@ -2072,7 +2066,7 @@ static __always_inline void raid_rec2of2_ssse3ext(int *id, int *ip, int nd, size
* Reconstruct only nr - 1 missing blocks through the inverse matrix and
* obtain the last block by XORing the reconstructed blocks out of Pdelta.
*/
static __always_inline void raid_recX_ssse3ext_123(int nr, int *id, int *ip, int nd, size_t size, void **vv)
static __always_inline void raid_recX_ssse3ext_123(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p[RAID_PARITY_MAX];
@ -2085,7 +2079,6 @@ static __always_inline void raid_recX_ssse3ext_123(int nr, int *id, int *ip, int
size_t i;
int d, j, k, s;
int ns;
int has_p;
BUG_ON(nr < 1 || nr > 3);
@ -2100,8 +2093,6 @@ static __always_inline void raid_recX_ssse3ext_123(int nr, int *id, int *ip, int
pa[j] = v[id[j]];
}
has_p = ip[0] == 0;
ns = 0;
k = 0;
@ -2331,7 +2322,7 @@ static __always_inline void raid_recX_ssse3ext_123(int nr, int *id, int *ip, int
* With at most five failures the low/high syndrome pairs use xmm0..xmm9,
* leaving xmm10 available for the P delta accumulator.
*/
static __always_inline void raid_recX_ssse3ext_12345(int nr, int *id, int *ip, int nd, size_t size, void **vv)
static __always_inline void raid_recX_ssse3ext_12345(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p[RAID_PARITY_MAX];
@ -2344,7 +2335,6 @@ static __always_inline void raid_recX_ssse3ext_12345(int nr, int *id, int *ip, i
size_t i;
int d, j, k, s;
int ns;
int has_p;
BUG_ON(nr < 1 || nr > 5);
@ -2359,9 +2349,6 @@ static __always_inline void raid_recX_ssse3ext_12345(int nr, int *id, int *ip, i
pa[j] = v[id[j]];
}
/* ip[] is ordered. If P is available, it is always ip[0] */
has_p = ip[0] == 0;
ns = 0;
k = 0;
@ -2632,7 +2619,7 @@ static __always_inline void raid_recX_ssse3ext_12345(int nr, int *id, int *ip, i
* The last missing block is obtained by XORing the reconstructed blocks
* out of Pdelta.
*/
static __always_inline void raid_recX_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
static __always_inline void raid_recX_ssse3ext(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p[RAID_PARITY_MAX];
@ -2645,7 +2632,6 @@ static __always_inline void raid_recX_ssse3ext(int nr, int *id, int *ip, int nd,
size_t i;
int d, j, k, s;
int ns;
int has_p;
BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
@ -2660,8 +2646,6 @@ static __always_inline void raid_recX_ssse3ext(int nr, int *id, int *ip, int nd,
pa[j] = v[id[j]];
}
has_p = ip[0] == 0;
ns = 0;
k = 0;
@ -2985,25 +2969,37 @@ void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
void raid_rec3_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 3);
raid_recX_ssse3_1234(3, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_ssse3_1234(3, 1, id, ip, nd, size, vv);
else
raid_recX_ssse3_1234(3, 0, id, ip, nd, size, vv);
}
void raid_rec4_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 4);
raid_recX_ssse3_1234(4, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_ssse3_1234(4, 1, id, ip, nd, size, vv);
else
raid_recX_ssse3_1234(4, 0, id, ip, nd, size, vv);
}
void raid_rec5_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 5);
raid_recX_ssse3(5, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_ssse3(5, 1, id, ip, nd, size, vv);
else
raid_recX_ssse3(5, 0, id, ip, nd, size, vv);
}
void raid_rec6_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 6);
raid_recX_ssse3(6, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_ssse3(6, 1, id, ip, nd, size, vv);
else
raid_recX_ssse3(6, 0, id, ip, nd, size, vv);
}
#ifdef CONFIG_X86_64
@ -3036,25 +3032,37 @@ void raid_rec2_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv
void raid_rec3_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 3);
raid_recX_ssse3ext_123(3, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_ssse3ext_123(3, 1, id, ip, nd, size, vv);
else
raid_recX_ssse3ext_123(3, 0, id, ip, nd, size, vv);
}
void raid_rec4_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 4);
raid_recX_ssse3ext_12345(4, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_ssse3ext_12345(4, 1, id, ip, nd, size, vv);
else
raid_recX_ssse3ext_12345(4, 0, id, ip, nd, size, vv);
}
void raid_rec5_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 5);
raid_recX_ssse3ext_12345(5, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_ssse3ext_12345(5, 1, id, ip, nd, size, vv);
else
raid_recX_ssse3ext_12345(5, 0, id, ip, nd, size, vv);
}
void raid_rec6_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 6);
raid_recX_ssse3ext(6, id, ip, nd, size, vv);
if (ip[0] == 0)
raid_recX_ssse3ext(6, 1, id, ip, nd, size, vv);
else
raid_recX_ssse3ext(6, 0, id, ip, nd, size, vv);
}
#endif