diff --git a/raid/avx2.c b/raid/avx2.c index 6d0614ad..2b1bcdaf 100644 --- a/raid/avx2.c +++ b/raid/avx2.c @@ -1009,7 +1009,7 @@ static __always_inline void raid_rec1_avx2_1(int *id, int *ip, int nd, size_t si * Reconstruct only nr - 1 missing blocks through the inverse matrix and * obtain the last block by XORing the reconstructed blocks out of Pdelta. */ -static __always_inline void raid_recX_avx2_1234(int nr, int *id, int *ip, int nd, size_t size, void **vv) +static __always_inline void raid_recX_avx2_1234(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p[RAID_PARITY_MAX]; @@ -1026,7 +1026,6 @@ static __always_inline void raid_recX_avx2_1234(int nr, int *id, int *ip, int nd size_t i; int d, j, k, s; int ns; - int has_p; BUG_ON(nr < 1 || nr > 4); @@ -1044,9 +1043,6 @@ static __always_inline void raid_recX_avx2_1234(int nr, int *id, int *ip, int nd pa[j] = v[id[j]]; } - /* ip[] is ordered. If P is available, it is always ip[0] */ - has_p = ip[0] == 0; - /* build the compact surviving-data list and its syndrome tables */ ns = 0; k = 0; @@ -1218,7 +1214,7 @@ static __always_inline void raid_recX_avx2_1234(int nr, int *id, int *ip, int nd * Reconstruct only nr - 1 missing blocks through the inverse matrix and * obtain the last block by XORing the reconstructed blocks out of Pdelta. */ -static __always_inline void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) +static __always_inline void raid_recX_avx2(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p[RAID_PARITY_MAX]; @@ -1235,7 +1231,6 @@ static __always_inline void raid_recX_avx2(int nr, int *id, int *ip, int nd, siz size_t i; int d, j, k, s; int ns; - int has_p; BUG_ON(nr < 1 || nr > RAID_PARITY_MAX); @@ -1253,9 +1248,6 @@ static __always_inline void raid_recX_avx2(int nr, int *id, int *ip, int nd, siz pa[j] = v[id[j]]; } - /* ip[] is ordered. If P is available, it is always ip[0] */ - has_p = ip[0] == 0; - /* build the compact surviving-data list and its syndrome tables */ ns = 0; k = 0; @@ -1446,7 +1438,7 @@ static __always_inline void raid_recX_avx2(int nr, int *id, int *ip, int nd, siz * * Note that it uses 16 registers, meaning that x64 is required. */ -static __always_inline void raid_recX_avx2ext_12(int nr, int *id, int *ip, int nd, size_t size, void **vv) +static __always_inline void raid_recX_avx2ext_12(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p[RAID_PARITY_MAX]; @@ -1459,7 +1451,6 @@ static __always_inline void raid_recX_avx2ext_12(int nr, int *id, int *ip, int n size_t i; int d, j, k, s; int ns; - int has_p; BUG_ON(nr < 1 || nr > 2); @@ -1477,9 +1468,6 @@ static __always_inline void raid_recX_avx2ext_12(int nr, int *id, int *ip, int n pa[j] = v[id[j]]; } - /* ip[] is ordered. If P is available, it is always ip[0] */ - has_p = ip[0] == 0; - /* build the compact surviving-data list and its syndrome tables */ ns = 0; k = 0; @@ -1716,7 +1704,7 @@ static __always_inline void raid_recX_avx2ext_12(int nr, int *id, int *ip, int n * * Note that it uses 16 registers, meaning that x64 is required. */ -static __always_inline void raid_recX_avx2ext_123(int nr, int *id, int *ip, int nd, size_t size, void **vv) +static __always_inline void raid_recX_avx2ext_123(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p[RAID_PARITY_MAX]; @@ -1729,7 +1717,6 @@ static __always_inline void raid_recX_avx2ext_123(int nr, int *id, int *ip, int size_t i; int d, j, k, s; int ns; - int has_p; BUG_ON(nr < 1 || nr > 3); @@ -1747,9 +1734,6 @@ static __always_inline void raid_recX_avx2ext_123(int nr, int *id, int *ip, int pa[j] = v[id[j]]; } - /* ip[] is ordered. If P is available, it is always ip[0] */ - has_p = ip[0] == 0; - /* build the compact surviving-data list and its syndrome tables */ ns = 0; k = 0; @@ -2023,7 +2007,7 @@ static __always_inline void raid_recX_avx2ext_123(int nr, int *id, int *ip, int * The last missing block is obtained by XORing the reconstructed blocks * out of Pdelta. */ -static __always_inline void raid_recX_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv) +static __always_inline void raid_recX_avx2ext(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p[RAID_PARITY_MAX]; @@ -2036,7 +2020,6 @@ static __always_inline void raid_recX_avx2ext(int nr, int *id, int *ip, int nd, size_t i; int d, j, k, s; int ns; - int has_p; BUG_ON(nr < 1 || nr > RAID_PARITY_MAX); @@ -2054,9 +2037,6 @@ static __always_inline void raid_recX_avx2ext(int nr, int *id, int *ip, int nd, pa[j] = v[id[j]]; } - /* ip[] is ordered. If P is available, it is always ip[0] */ - has_p = ip[0] == 0; - /* build the compact surviving-data list and its syndrome tables */ ns = 0; k = 0; @@ -2431,31 +2411,46 @@ void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 2); - raid_recX_avx2_1234(2, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx2_1234(2, 1, id, ip, nd, size, vv); + else + raid_recX_avx2_1234(2, 0, id, ip, nd, size, vv); } void raid_rec3_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 3); - raid_recX_avx2_1234(3, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx2_1234(3, 1, id, ip, nd, size, vv); + else + raid_recX_avx2_1234(3, 0, id, ip, nd, size, vv); } void raid_rec4_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 4); - raid_recX_avx2_1234(4, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx2_1234(4, 1, id, ip, nd, size, vv); + else + raid_recX_avx2_1234(4, 0, id, ip, nd, size, vv); } void raid_rec5_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 5); - raid_recX_avx2(5, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx2(5, 1, id, ip, nd, size, vv); + else + raid_recX_avx2(5, 0, id, ip, nd, size, vv); } void raid_rec6_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 6); - raid_recX_avx2(6, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx2(6, 1, id, ip, nd, size, vv); + else + raid_recX_avx2(6, 0, id, ip, nd, size, vv); } #ifdef CONFIG_X86_64 @@ -2475,33 +2470,47 @@ void raid_rec1_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv) void raid_rec2_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 2); - raid_recX_avx2ext_12(2, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx2ext_12(2, 1, id, ip, nd, size, vv); + else + raid_recX_avx2ext_12(2, 0, id, ip, nd, size, vv); } void raid_rec3_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 3); - raid_recX_avx2ext_123(3, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx2ext_123(3, 1, id, ip, nd, size, vv); + else + raid_recX_avx2ext_123(3, 0, id, ip, nd, size, vv); } void raid_rec4_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 4); - raid_recX_avx2ext(4, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx2ext(4, 1, id, ip, nd, size, vv); + else + raid_recX_avx2ext(4, 0, id, ip, nd, size, vv); } void raid_rec5_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 5); - raid_recX_avx2ext(5, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx2ext(5, 1, id, ip, nd, size, vv); + else + raid_recX_avx2ext(5, 0, id, ip, nd, size, vv); } void raid_rec6_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 6); - raid_recX_avx2ext(6, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx2ext(6, 1, id, ip, nd, size, vv); + else + raid_recX_avx2ext(6, 0, id, ip, nd, size, vv); } - #endif void raid_register_avx2(void) diff --git a/raid/avx512.c b/raid/avx512.c index 0afe04e7..17c11286 100644 --- a/raid/avx512.c +++ b/raid/avx512.c @@ -567,7 +567,7 @@ static __always_inline void raid_genX_avx512bw(int nd, size_t size, void **vv, i * AVX512 provides enough registers to keep all six supported syndromes * in registers in a single surviving-data scan. */ -static __always_inline void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv) +static __always_inline void raid_recX_avx512bw(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p[RAID_PARITY_MAX]; @@ -580,7 +580,6 @@ static __always_inline void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t i; int d, j, k, s; int ns; - int has_p; BUG_ON(nr < 1 || nr > RAID_PARITY_MAX); @@ -598,9 +597,6 @@ static __always_inline void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, pa[j] = v[id[j]]; } - /* ip[] is ordered. If P is available, it is always ip[0] */ - has_p = ip[0] == 0; - /* * Build the compact list of surviving data blocks and precompute * the multiplication-table pointers for each selected syndrome. @@ -911,37 +907,55 @@ void raid_gen6_avx512bw(int nd, size_t size, void **vv) void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 1); - raid_recX_avx512bw(1, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx512bw(1, 1, id, ip, nd, size, vv); + else + raid_recX_avx512bw(1, 0, id, ip, nd, size, vv); } void raid_rec2_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 2); - raid_recX_avx512bw(2, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx512bw(2, 1, id, ip, nd, size, vv); + else + raid_recX_avx512bw(2, 0, id, ip, nd, size, vv); } void raid_rec3_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 3); - raid_recX_avx512bw(3, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx512bw(3, 1, id, ip, nd, size, vv); + else + raid_recX_avx512bw(3, 0, id, ip, nd, size, vv); } void raid_rec4_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 4); - raid_recX_avx512bw(4, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx512bw(4, 1, id, ip, nd, size, vv); + else + raid_recX_avx512bw(4, 0, id, ip, nd, size, vv); } void raid_rec5_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 5); - raid_recX_avx512bw(5, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx512bw(5, 1, id, ip, nd, size, vv); + else + raid_recX_avx512bw(5, 0, id, ip, nd, size, vv); } void raid_rec6_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 6); - raid_recX_avx512bw(6, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx512bw(6, 1, id, ip, nd, size, vv); + else + raid_recX_avx512bw(6, 0, id, ip, nd, size, vv); } void raid_register_avx512(void) diff --git a/raid/gfni.c b/raid/gfni.c index fc7da939..3932e2ac 100644 --- a/raid/gfni.c +++ b/raid/gfni.c @@ -1355,7 +1355,7 @@ static __always_inline void raid_genX_avx512gfni_aes(int nd, size_t size, void * * If P is available, reconstruct only nr - 1 missing blocks through the * inverse matrix and derive the last missing block from the P delta by XOR. */ -static __always_inline void raid_recX_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) +static __always_inline void raid_recX_avx2gfni_raid(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p[RAID_PARITY_MAX]; @@ -1368,7 +1368,6 @@ static __always_inline void raid_recX_avx2gfni_raid(int nr, int *id, int *ip, in size_t i; int d, j, k, s; int ns; - int has_p; BUG_ON(nr < 1 || nr > RAID_PARITY_MAX); @@ -1383,8 +1382,6 @@ static __always_inline void raid_recX_avx2gfni_raid(int nr, int *id, int *ip, in pa[j] = v[id[j]]; } - has_p = ip[0] == 0; - ns = 0; k = 0; @@ -1563,7 +1560,7 @@ static __always_inline void raid_recX_avx2gfni_raid(int nr, int *id, int *ip, in * If P is available, reconstruct only nr - 1 missing blocks through the * inverse matrix and derive the last missing block from the P delta by XOR. */ -static __always_inline void raid_recX_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) +static __always_inline void raid_recX_avx512gfni_raid(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p[RAID_PARITY_MAX]; @@ -1576,7 +1573,6 @@ static __always_inline void raid_recX_avx512gfni_raid(int nr, int *id, int *ip, size_t i; int d, j, k, s; int ns; - int has_p; BUG_ON(nr < 1 || nr > RAID_PARITY_MAX); @@ -1591,8 +1587,6 @@ static __always_inline void raid_recX_avx512gfni_raid(int nr, int *id, int *ip, pa[j] = v[id[j]]; } - has_p = ip[0] == 0; - ns = 0; k = 0; @@ -1744,7 +1738,7 @@ static __always_inline void raid_recX_avx512gfni_raid(int nr, int *id, int *ip, * If P is available, reconstruct only nr - 1 missing blocks through the * inverse matrix and derive the last missing block from the P delta by XOR. */ -static __always_inline void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) +static __always_inline void raid_recX_avx2gfni_aes(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p[RAID_PARITY_MAX]; @@ -1756,7 +1750,6 @@ static __always_inline void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int size_t i; int d, j, k, s; int ns; - int has_p; BUG_ON(nr < 1 || nr > RAID_PARITY_MAX); @@ -1771,8 +1764,6 @@ static __always_inline void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int pa[j] = v[id[j]]; } - has_p = ip[0] == 0; - ns = 0; k = 0; @@ -1943,7 +1934,7 @@ static __always_inline void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int * If P is available, reconstruct only nr - 1 missing blocks through the * inverse matrix and derive the last missing block from the P delta by XOR. */ -static __always_inline void raid_recX_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) +static __always_inline void raid_recX_avx512gfni_aes(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p[RAID_PARITY_MAX]; @@ -1955,7 +1946,6 @@ static __always_inline void raid_recX_avx512gfni_aes(int nr, int *id, int *ip, i size_t i; int d, j, k, s; int ns; - int has_p; BUG_ON(nr < 1 || nr > RAID_PARITY_MAX); @@ -1970,8 +1960,6 @@ static __always_inline void raid_recX_avx512gfni_aes(int nr, int *id, int *ip, i pa[j] = v[id[j]]; } - has_p = ip[0] == 0; - ns = 0; k = 0; @@ -2203,145 +2191,217 @@ void raid_gen6_avx512gfni_aes(int nd, size_t size, void **vv) void raid_rec1_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 1); - raid_recX_avx2gfni_raid(1, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx2gfni_raid(1, 1, id, ip, nd, size, vv); + else + raid_recX_avx2gfni_raid(1, 0, id, ip, nd, size, vv); } void raid_rec2_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 2); - raid_recX_avx2gfni_raid(2, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx2gfni_raid(2, 1, id, ip, nd, size, vv); + else + raid_recX_avx2gfni_raid(2, 0, id, ip, nd, size, vv); } void raid_rec3_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 3); - raid_recX_avx2gfni_raid(3, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx2gfni_raid(3, 1, id, ip, nd, size, vv); + else + raid_recX_avx2gfni_raid(3, 0, id, ip, nd, size, vv); } void raid_rec4_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 4); - raid_recX_avx2gfni_raid(4, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx2gfni_raid(4, 1, id, ip, nd, size, vv); + else + raid_recX_avx2gfni_raid(4, 0, id, ip, nd, size, vv); } void raid_rec5_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 5); - raid_recX_avx2gfni_raid(5, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx2gfni_raid(5, 1, id, ip, nd, size, vv); + else + raid_recX_avx2gfni_raid(5, 0, id, ip, nd, size, vv); } void raid_rec6_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 6); - raid_recX_avx2gfni_raid(6, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx2gfni_raid(6, 1, id, ip, nd, size, vv); + else + raid_recX_avx2gfni_raid(6, 0, id, ip, nd, size, vv); } void raid_rec1_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 1); - raid_recX_avx512gfni_raid(1, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx512gfni_raid(1, 1, id, ip, nd, size, vv); + else + raid_recX_avx512gfni_raid(1, 0, id, ip, nd, size, vv); } void raid_rec2_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 2); - raid_recX_avx512gfni_raid(2, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx512gfni_raid(2, 1, id, ip, nd, size, vv); + else + raid_recX_avx512gfni_raid(2, 0, id, ip, nd, size, vv); } void raid_rec3_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 3); - raid_recX_avx512gfni_raid(3, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx512gfni_raid(3, 1, id, ip, nd, size, vv); + else + raid_recX_avx512gfni_raid(3, 0, id, ip, nd, size, vv); } void raid_rec4_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 4); - raid_recX_avx512gfni_raid(4, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx512gfni_raid(4, 1, id, ip, nd, size, vv); + else + raid_recX_avx512gfni_raid(4, 0, id, ip, nd, size, vv); } void raid_rec5_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 5); - raid_recX_avx512gfni_raid(5, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx512gfni_raid(5, 1, id, ip, nd, size, vv); + else + raid_recX_avx512gfni_raid(5, 0, id, ip, nd, size, vv); } void raid_rec6_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 6); - raid_recX_avx512gfni_raid(6, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx512gfni_raid(6, 1, id, ip, nd, size, vv); + else + raid_recX_avx512gfni_raid(6, 0, id, ip, nd, size, vv); } void raid_rec1_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 1); - raid_recX_avx2gfni_aes(1, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx2gfni_aes(1, 1, id, ip, nd, size, vv); + else + raid_recX_avx2gfni_aes(1, 0, id, ip, nd, size, vv); } void raid_rec2_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 2); - raid_recX_avx2gfni_aes(2, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx2gfni_aes(2, 1, id, ip, nd, size, vv); + else + raid_recX_avx2gfni_aes(2, 0, id, ip, nd, size, vv); } void raid_rec3_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 3); - raid_recX_avx2gfni_aes(3, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx2gfni_aes(3, 1, id, ip, nd, size, vv); + else + raid_recX_avx2gfni_aes(3, 0, id, ip, nd, size, vv); } void raid_rec4_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 4); - raid_recX_avx2gfni_aes(4, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx2gfni_aes(4, 1, id, ip, nd, size, vv); + else + raid_recX_avx2gfni_aes(4, 0, id, ip, nd, size, vv); } void raid_rec5_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 5); - raid_recX_avx2gfni_aes(5, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx2gfni_aes(5, 1, id, ip, nd, size, vv); + else + raid_recX_avx2gfni_aes(5, 0, id, ip, nd, size, vv); } void raid_rec6_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 6); - raid_recX_avx2gfni_aes(6, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx2gfni_aes(6, 1, id, ip, nd, size, vv); + else + raid_recX_avx2gfni_aes(6, 0, id, ip, nd, size, vv); } void raid_rec1_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 1); - raid_recX_avx512gfni_aes(1, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx512gfni_aes(1, 1, id, ip, nd, size, vv); + else + raid_recX_avx512gfni_aes(1, 0, id, ip, nd, size, vv); } void raid_rec2_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 2); - raid_recX_avx512gfni_aes(2, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx512gfni_aes(2, 1, id, ip, nd, size, vv); + else + raid_recX_avx512gfni_aes(2, 0, id, ip, nd, size, vv); } void raid_rec3_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 3); - raid_recX_avx512gfni_aes(3, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx512gfni_aes(3, 1, id, ip, nd, size, vv); + else + raid_recX_avx512gfni_aes(3, 0, id, ip, nd, size, vv); } void raid_rec4_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 4); - raid_recX_avx512gfni_aes(4, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx512gfni_aes(4, 1, id, ip, nd, size, vv); + else + raid_recX_avx512gfni_aes(4, 0, id, ip, nd, size, vv); } void raid_rec5_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 5); - raid_recX_avx512gfni_aes(5, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx512gfni_aes(5, 1, id, ip, nd, size, vv); + else + raid_recX_avx512gfni_aes(5, 0, id, ip, nd, size, vv); } void raid_rec6_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 6); - raid_recX_avx512gfni_aes(6, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_avx512gfni_aes(6, 1, id, ip, nd, size, vv); + else + raid_recX_avx512gfni_aes(6, 0, id, ip, nd, size, vv); } void raid_register_avx2gfni(void) diff --git a/raid/neon.c b/raid/neon.c index af45671b..b779db41 100644 --- a/raid/neon.c +++ b/raid/neon.c @@ -580,7 +580,7 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n * The last missing block is obtained by XORing the reconstructed blocks * out of Pdelta. */ -static __always_inline void raid_recX_neon_12(int nr, int *id, int *ip, int nd, size_t size, void **vv) +static __always_inline void raid_recX_neon_12(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p[RAID_PARITY_MAX]; @@ -593,7 +593,6 @@ static __always_inline void raid_recX_neon_12(int nr, int *id, int *ip, int nd, size_t i; int d, j, k, s; int ns; - int has_p; BUG_ON(nr < 1 || nr > 2); @@ -611,9 +610,6 @@ static __always_inline void raid_recX_neon_12(int nr, int *id, int *ip, int nd, pa[j] = v[id[j]]; } - /* ip[] is ordered. If P is available, it is always ip[0] */ - has_p = ip[0] == 0; - /* * Build the compact list of surviving data blocks and precompute * the multiplication-table pointers for each selected syndrome. @@ -929,7 +925,7 @@ static __always_inline void raid_recX_neon_12(int nr, int *id, int *ip, int nd, * The last missing block is obtained by XORing the reconstructed blocks * out of Pdelta. */ -static __always_inline void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) +static __always_inline void raid_recX_neon(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p[RAID_PARITY_MAX]; @@ -942,7 +938,6 @@ static __always_inline void raid_recX_neon(int nr, int *id, int *ip, int nd, siz size_t i; int d, j, k, s; int ns; - int has_p; BUG_ON(nr < 1 || nr > RAID_PARITY_MAX); @@ -960,9 +955,6 @@ static __always_inline void raid_recX_neon(int nr, int *id, int *ip, int nd, siz pa[j] = v[id[j]]; } - /* ip[] is ordered. If P is available, it is always ip[0] */ - has_p = ip[0] == 0; - /* * Build the compact list of surviving data blocks and precompute * the multiplication-table pointers for each selected syndrome. @@ -1385,37 +1377,55 @@ void raid_gen6_neon_aes(int nd, size_t size, void **vv) void raid_rec1_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 1); - raid_recX_neon_12(1, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_neon_12(1, 1, id, ip, nd, size, vv); + else + raid_recX_neon_12(1, 0, id, ip, nd, size, vv); } void raid_rec2_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 2); - raid_recX_neon_12(2, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_neon_12(2, 1, id, ip, nd, size, vv); + else + raid_recX_neon_12(2, 0, id, ip, nd, size, vv); } void raid_rec3_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 3); - raid_recX_neon(3, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_neon(3, 1, id, ip, nd, size, vv); + else + raid_recX_neon(3, 0, id, ip, nd, size, vv); } void raid_rec4_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 4); - raid_recX_neon(4, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_neon(4, 1, id, ip, nd, size, vv); + else + raid_recX_neon(4, 0, id, ip, nd, size, vv); } void raid_rec5_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 5); - raid_recX_neon(5, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_neon(5, 1, id, ip, nd, size, vv); + else + raid_recX_neon(5, 0, id, ip, nd, size, vv); } void raid_rec6_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 6); - raid_recX_neon(6, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_neon(6, 1, id, ip, nd, size, vv); + else + raid_recX_neon(6, 0, id, ip, nd, size, vv); } void raid_register_neon(void) diff --git a/raid/neon32.c b/raid/neon32.c index 91c2448c..ba2c7fec 100644 --- a/raid/neon32.c +++ b/raid/neon32.c @@ -528,7 +528,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size, * Reconstruct only nr - 1 missing blocks through the inverse matrix and * obtain the last block by XORing the reconstructed blocks out of Pdelta. */ -static __always_inline void raid_recX_neon32_123(int nr, int *id, int *ip, int nd, size_t size, void **vv) +static __always_inline void raid_recX_neon32_123(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p[RAID_PARITY_MAX]; @@ -541,7 +541,6 @@ static __always_inline void raid_recX_neon32_123(int nr, int *id, int *ip, int n size_t i; int d, j, k, s; int ns; - int has_p; BUG_ON(nr < 1 || nr > 3); @@ -556,8 +555,6 @@ static __always_inline void raid_recX_neon32_123(int nr, int *id, int *ip, int n pa[j] = v[id[j]]; } - has_p = ip[0] == 0; - ns = 0; k = 0; @@ -836,7 +833,7 @@ static __always_inline void raid_recX_neon32_123(int nr, int *id, int *ip, int n * The last missing block is obtained by XORing the reconstructed blocks * out of Pdelta. */ -static __always_inline void raid_recX_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv) +static __always_inline void raid_recX_neon32(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p[RAID_PARITY_MAX]; @@ -849,7 +846,6 @@ static __always_inline void raid_recX_neon32(int nr, int *id, int *ip, int nd, s size_t i; int d, j, k, s; int ns; - int has_p; BUG_ON(nr < 1 || nr > RAID_PARITY_MAX); @@ -864,8 +860,6 @@ static __always_inline void raid_recX_neon32(int nr, int *id, int *ip, int nd, s pa[j] = v[id[j]]; } - has_p = ip[0] == 0; - ns = 0; k = 0; @@ -1229,7 +1223,7 @@ void raid_rec1_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv) return; } - raid_recX_neon32_123(1, id, ip, nd, size, vv); + raid_recX_neon32_123(1, 0, id, ip, nd, size, vv); } /* @@ -1342,31 +1336,46 @@ void raid_rec2_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv) return; } - raid_recX_neon32_123(2, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_neon32_123(2, 1, id, ip, nd, size, vv); + else + raid_recX_neon32_123(2, 0, id, ip, nd, size, vv); } void raid_rec3_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 3); - raid_recX_neon32_123(3, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_neon32_123(3, 1, id, ip, nd, size, vv); + else + raid_recX_neon32_123(3, 0, id, ip, nd, size, vv); } void raid_rec4_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 4); - raid_recX_neon32(4, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_neon32(4, 1, id, ip, nd, size, vv); + else + raid_recX_neon32(4, 0, id, ip, nd, size, vv); } void raid_rec5_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 5); - raid_recX_neon32(5, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_neon32(5, 1, id, ip, nd, size, vv); + else + raid_recX_neon32(5, 0, id, ip, nd, size, vv); } void raid_rec6_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 6); - raid_recX_neon32(6, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_neon32(6, 1, id, ip, nd, size, vv); + else + raid_recX_neon32(6, 0, id, ip, nd, size, vv); } void raid_register_neon32(void) diff --git a/raid/ssse3.c b/raid/ssse3.c index 3457d260..af22cc53 100644 --- a/raid/ssse3.c +++ b/raid/ssse3.c @@ -1409,7 +1409,7 @@ static __always_inline void raid_rec2of2_ssse3(int *id, int *ip, int nd, size_t * After the survivor scan xmm6 is no longer needed, leaving it available * for the P delta accumulator. */ -static __always_inline void raid_recX_ssse3_1234(int nr, int *id, int *ip, int nd, size_t size, void **vv) +static __always_inline void raid_recX_ssse3_1234(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p[RAID_PARITY_MAX]; @@ -1426,7 +1426,6 @@ static __always_inline void raid_recX_ssse3_1234(int nr, int *id, int *ip, int n size_t i; int d, j, k, s; int ns; - int has_p; BUG_ON(nr < 1 || nr > 4); @@ -1441,8 +1440,6 @@ static __always_inline void raid_recX_ssse3_1234(int nr, int *id, int *ip, int n pa[j] = v[id[j]]; } - has_p = ip[0] == 0; - ns = 0; k = 0; @@ -1634,7 +1631,7 @@ static __always_inline void raid_recX_ssse3_1234(int nr, int *id, int *ip, int n * The completed syndromes are kept in temporary memory, leaving xmm6 * available for the P delta accumulator during reconstruction. */ -static __always_inline void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) +static __always_inline void raid_recX_ssse3(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p[RAID_PARITY_MAX]; @@ -1651,7 +1648,6 @@ static __always_inline void raid_recX_ssse3(int nr, int *id, int *ip, int nd, si size_t i; int d, j, k, s; int ns; - int has_p; BUG_ON(nr < 1 || nr > RAID_PARITY_MAX); @@ -1666,8 +1662,6 @@ static __always_inline void raid_recX_ssse3(int nr, int *id, int *ip, int nd, si pa[j] = v[id[j]]; } - has_p = ip[0] == 0; - ns = 0; k = 0; @@ -2072,7 +2066,7 @@ static __always_inline void raid_rec2of2_ssse3ext(int *id, int *ip, int nd, size * Reconstruct only nr - 1 missing blocks through the inverse matrix and * obtain the last block by XORing the reconstructed blocks out of Pdelta. */ -static __always_inline void raid_recX_ssse3ext_123(int nr, int *id, int *ip, int nd, size_t size, void **vv) +static __always_inline void raid_recX_ssse3ext_123(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p[RAID_PARITY_MAX]; @@ -2085,7 +2079,6 @@ static __always_inline void raid_recX_ssse3ext_123(int nr, int *id, int *ip, int size_t i; int d, j, k, s; int ns; - int has_p; BUG_ON(nr < 1 || nr > 3); @@ -2100,8 +2093,6 @@ static __always_inline void raid_recX_ssse3ext_123(int nr, int *id, int *ip, int pa[j] = v[id[j]]; } - has_p = ip[0] == 0; - ns = 0; k = 0; @@ -2331,7 +2322,7 @@ static __always_inline void raid_recX_ssse3ext_123(int nr, int *id, int *ip, int * With at most five failures the low/high syndrome pairs use xmm0..xmm9, * leaving xmm10 available for the P delta accumulator. */ -static __always_inline void raid_recX_ssse3ext_12345(int nr, int *id, int *ip, int nd, size_t size, void **vv) +static __always_inline void raid_recX_ssse3ext_12345(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p[RAID_PARITY_MAX]; @@ -2344,7 +2335,6 @@ static __always_inline void raid_recX_ssse3ext_12345(int nr, int *id, int *ip, i size_t i; int d, j, k, s; int ns; - int has_p; BUG_ON(nr < 1 || nr > 5); @@ -2359,9 +2349,6 @@ static __always_inline void raid_recX_ssse3ext_12345(int nr, int *id, int *ip, i pa[j] = v[id[j]]; } - /* ip[] is ordered. If P is available, it is always ip[0] */ - has_p = ip[0] == 0; - ns = 0; k = 0; @@ -2632,7 +2619,7 @@ static __always_inline void raid_recX_ssse3ext_12345(int nr, int *id, int *ip, i * The last missing block is obtained by XORing the reconstructed blocks * out of Pdelta. */ -static __always_inline void raid_recX_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv) +static __always_inline void raid_recX_ssse3ext(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p[RAID_PARITY_MAX]; @@ -2645,7 +2632,6 @@ static __always_inline void raid_recX_ssse3ext(int nr, int *id, int *ip, int nd, size_t i; int d, j, k, s; int ns; - int has_p; BUG_ON(nr < 1 || nr > RAID_PARITY_MAX); @@ -2660,8 +2646,6 @@ static __always_inline void raid_recX_ssse3ext(int nr, int *id, int *ip, int nd, pa[j] = v[id[j]]; } - has_p = ip[0] == 0; - ns = 0; k = 0; @@ -2985,25 +2969,37 @@ void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) void raid_rec3_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 3); - raid_recX_ssse3_1234(3, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_ssse3_1234(3, 1, id, ip, nd, size, vv); + else + raid_recX_ssse3_1234(3, 0, id, ip, nd, size, vv); } void raid_rec4_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 4); - raid_recX_ssse3_1234(4, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_ssse3_1234(4, 1, id, ip, nd, size, vv); + else + raid_recX_ssse3_1234(4, 0, id, ip, nd, size, vv); } void raid_rec5_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 5); - raid_recX_ssse3(5, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_ssse3(5, 1, id, ip, nd, size, vv); + else + raid_recX_ssse3(5, 0, id, ip, nd, size, vv); } void raid_rec6_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 6); - raid_recX_ssse3(6, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_ssse3(6, 1, id, ip, nd, size, vv); + else + raid_recX_ssse3(6, 0, id, ip, nd, size, vv); } #ifdef CONFIG_X86_64 @@ -3036,25 +3032,37 @@ void raid_rec2_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv void raid_rec3_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 3); - raid_recX_ssse3ext_123(3, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_ssse3ext_123(3, 1, id, ip, nd, size, vv); + else + raid_recX_ssse3ext_123(3, 0, id, ip, nd, size, vv); } void raid_rec4_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 4); - raid_recX_ssse3ext_12345(4, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_ssse3ext_12345(4, 1, id, ip, nd, size, vv); + else + raid_recX_ssse3ext_12345(4, 0, id, ip, nd, size, vv); } void raid_rec5_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 5); - raid_recX_ssse3ext_12345(5, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_ssse3ext_12345(5, 1, id, ip, nd, size, vv); + else + raid_recX_ssse3ext_12345(5, 0, id, ip, nd, size, vv); } void raid_rec6_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 6); - raid_recX_ssse3ext(6, id, ip, nd, size, vv); + if (ip[0] == 0) + raid_recX_ssse3ext(6, 1, id, ip, nd, size, vv); + else + raid_recX_ssse3ext(6, 0, id, ip, nd, size, vv); } #endif