mirror of
https://github.com/amadvance/snapraid.git
synced 2026-10-26 02:49:02 -04:00
Optimize recovery with P by deriving the final missing block via XOR
This commit is contained in:
Andrea Mazzoleni
parent
d3e67b2fdf
commit
4d97df38ce
6 changed files with 550 additions and 146 deletions
210
raid/avx2.c
210
raid/avx2.c
|
|
@ -1112,6 +1112,10 @@ static __always_inline void raid_rec2of2_avx2(int *id, int *ip, int nd, size_t s
|
|||
*
|
||||
* Compute all selected syndromes in one scan of the surviving data, then
|
||||
* retain only their low/high nibbles while reconstructing the missing data.
|
||||
*
|
||||
* If P is available, keep the complete P delta syndrome in ymm6.
|
||||
* Reconstruct only nr - 1 missing blocks through the inverse matrix and
|
||||
* obtain the last block by XORing the reconstructed blocks out of Pdelta.
|
||||
*/
|
||||
static __always_inline void raid_recX_avx2_1234(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -1170,7 +1174,7 @@ static __always_inline void raid_recX_avx2_1234(int nr, int *id, int *ip, int nd
|
|||
BUG_ON(ns != nd - nr);
|
||||
|
||||
/* precompute inverse-matrix multiplication table pointers */
|
||||
for (j = 0; j < nr; ++j)
|
||||
for (j = 0; j < nr - has_p; ++j)
|
||||
for (k = 0; k < nr; ++k)
|
||||
R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
|
||||
|
||||
|
|
@ -1240,7 +1244,11 @@ static __always_inline void raid_recX_avx2_1234(int nr, int *id, int *ip, int nd
|
|||
}
|
||||
}
|
||||
|
||||
/* preserve only aligned low/high nibbles, never full delta blocks */
|
||||
/* preserve the complete P delta before splitting syndrome 0 */
|
||||
if (has_p)
|
||||
asm volatile ("vmovdqa %ymm0,%ymm6");
|
||||
|
||||
/* preserve only aligned low/high nibbles */
|
||||
asm volatile ("vpsrlw $4,%ymm0,%ymm4");
|
||||
asm volatile ("vpand %ymm7,%ymm0,%ymm0");
|
||||
asm volatile ("vpand %ymm7,%ymm4,%ymm4");
|
||||
|
|
@ -1271,8 +1279,8 @@ static __always_inline void raid_recX_avx2_1234(int nr, int *id, int *ip, int nd
|
|||
asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pd_high[96]));
|
||||
}
|
||||
|
||||
/* reconstruct every missing block, initializing from coefficient zero */
|
||||
for (j = 0; j < nr; ++j) {
|
||||
/* reconstruct all but the last missing block when P is available */
|
||||
for (j = 0; j < nr - has_p; ++j) {
|
||||
const uint8_t **t = R[j];
|
||||
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm0" : : "m" (t[0][0]));
|
||||
|
|
@ -1294,8 +1302,15 @@ static __always_inline void raid_recX_avx2_1234(int nr, int *id, int *ip, int nd
|
|||
}
|
||||
|
||||
asm volatile ("vpxor %ymm1,%ymm0,%ymm0");
|
||||
|
||||
if (has_p)
|
||||
asm volatile ("vpxor %ymm0,%ymm6,%ymm6");
|
||||
|
||||
asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[j][i]));
|
||||
}
|
||||
|
||||
if (has_p)
|
||||
asm volatile ("vmovdqa %%ymm6,%0" : "=m" (pa[nr - 1][i]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
|
|
@ -1306,6 +1321,10 @@ static __always_inline void raid_recX_avx2_1234(int nr, int *id, int *ip, int nd
|
|||
*
|
||||
* Compute all selected syndromes in a single pass over the surviving data.
|
||||
* Syndrome accumulators are kept in memory to minimize register pressure.
|
||||
*
|
||||
* If P is available, keep the complete P delta syndrome in ymm6.
|
||||
* Reconstruct only nr - 1 missing blocks through the inverse matrix and
|
||||
* obtain the last block by XORing the reconstructed blocks out of Pdelta.
|
||||
*/
|
||||
static __always_inline void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -1367,7 +1386,7 @@ static __always_inline void raid_recX_avx2(int nr, int *id, int *ip, int nd, siz
|
|||
BUG_ON(ns != nd - nr);
|
||||
|
||||
/* precompute inverse-matrix multiplication table pointers */
|
||||
for (j = 0; j < nr; ++j)
|
||||
for (j = 0; j < nr - has_p; ++j)
|
||||
for (k = 0; k < nr; ++k)
|
||||
R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
|
||||
|
||||
|
|
@ -1442,6 +1461,13 @@ static __always_inline void raid_recX_avx2(int nr, int *id, int *ip, int nd, siz
|
|||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Preserve the complete P delta before pd_low[0] is
|
||||
* destructively converted to low/high nibble form.
|
||||
*/
|
||||
if (has_p)
|
||||
asm volatile ("vmovdqa %0,%%ymm6" : : "m" (pd_low[0]));
|
||||
|
||||
/*
|
||||
* All survivor reads for this chunk are now complete.
|
||||
*
|
||||
|
|
@ -1458,7 +1484,8 @@ static __always_inline void raid_recX_avx2(int nr, int *id, int *ip, int nd, siz
|
|||
}
|
||||
|
||||
/*
|
||||
* Reconstruct each missing block.
|
||||
* Reconstruct all but the last missing block when P is
|
||||
* available.
|
||||
*
|
||||
* Keep independent low/high accumulation chains:
|
||||
*
|
||||
|
|
@ -1468,8 +1495,9 @@ static __always_inline void raid_recX_avx2(int nr, int *id, int *ip, int nd, siz
|
|||
* ymm3 = high table/result
|
||||
* ymm4 = syndrome low
|
||||
* ymm5 = syndrome high
|
||||
* ymm6 = remaining P delta
|
||||
*/
|
||||
for (j = 0; j < nr; ++j) {
|
||||
for (j = 0; j < nr - has_p; ++j) {
|
||||
const uint8_t **t = R[j];
|
||||
|
||||
/* coefficient 0 initializes the accumulators */
|
||||
|
|
@ -1497,111 +1525,21 @@ static __always_inline void raid_recX_avx2(int nr, int *id, int *ip, int nd, siz
|
|||
}
|
||||
|
||||
asm volatile ("vpxor %ymm1,%ymm0,%ymm0");
|
||||
|
||||
if (has_p)
|
||||
asm volatile ("vpxor %ymm0,%ymm6,%ymm6");
|
||||
|
||||
asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[j][i]));
|
||||
}
|
||||
|
||||
if (has_p)
|
||||
asm volatile ("vmovdqa %%ymm6,%0" : "=m" (pa[nr - 1][i]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
/*
|
||||
* RAID recovering for two disks AVX2 extended implementation
|
||||
*/
|
||||
static __always_inline void raid_rec2_avx2ext_delta(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p[2];
|
||||
uint8_t *pa[2];
|
||||
uint8_t G[2 * 2];
|
||||
uint8_t V[2 * 2];
|
||||
size_t i;
|
||||
int j, k;
|
||||
|
||||
/* setup the coefficients matrix */
|
||||
for (j = 0; j < 2; ++j)
|
||||
for (k = 0; k < 2; ++k)
|
||||
G[j * 2 + k] = A(ip[j], id[k]);
|
||||
|
||||
/* invert it to solve the system of linear equations */
|
||||
raid_invert(G, V, 2);
|
||||
|
||||
/* compute delta parity */
|
||||
raid_delta_gen(2, id, ip, nd, size, vv);
|
||||
|
||||
for (j = 0; j < 2; ++j) {
|
||||
p[j] = v[nd + ip[j]];
|
||||
pa[j] = v[id[j]];
|
||||
}
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm6" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
/* the inverse matrix V[] is constant for the whole recovery */
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm8" : : "m" (raid_gfmulpshufb[V[0]][0][0]));
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm9" : : "m" (raid_gfmulpshufb[V[0]][1][0]));
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm10" : : "m" (raid_gfmulpshufb[V[1]][0][0]));
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm11" : : "m" (raid_gfmulpshufb[V[1]][1][0]));
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfmulpshufb[V[2]][0][0]));
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfmulpshufb[V[2]][1][0]));
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfmulpshufb[V[3]][0][0]));
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfmulpshufb[V[3]][1][0]));
|
||||
|
||||
for (i = 0; i < size; i += 32) {
|
||||
/* d0 = p[0] ^ pa[0] */
|
||||
asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pa[0][i]));
|
||||
asm volatile ("vpxor %ymm4,%ymm0,%ymm0");
|
||||
|
||||
/* d1 = p[1] ^ pa[1] */
|
||||
asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[1][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pa[1][i]));
|
||||
asm volatile ("vpxor %ymm4,%ymm1,%ymm1");
|
||||
|
||||
/*
|
||||
* Split both deltas into low/high nibbles once.
|
||||
*
|
||||
* ymm0 = d0 low
|
||||
* ymm2 = d0 high
|
||||
* ymm1 = d1 low
|
||||
* ymm3 = d1 high
|
||||
*/
|
||||
asm volatile ("vpsrlw $4,%ymm0,%ymm2");
|
||||
asm volatile ("vpsrlw $4,%ymm1,%ymm3");
|
||||
asm volatile ("vpand %ymm6,%ymm0,%ymm0");
|
||||
asm volatile ("vpand %ymm6,%ymm2,%ymm2");
|
||||
asm volatile ("vpand %ymm6,%ymm1,%ymm1");
|
||||
asm volatile ("vpand %ymm6,%ymm3,%ymm3");
|
||||
|
||||
/* pa[0] = V[0] * d0 ^ V[1] * d1 */
|
||||
asm volatile ("vpshufb %ymm0,%ymm8,%ymm4");
|
||||
asm volatile ("vpshufb %ymm2,%ymm9,%ymm5");
|
||||
asm volatile ("vpxor %ymm5,%ymm4,%ymm4");
|
||||
|
||||
asm volatile ("vpshufb %ymm1,%ymm10,%ymm5");
|
||||
asm volatile ("vpxor %ymm5,%ymm4,%ymm4");
|
||||
asm volatile ("vpshufb %ymm3,%ymm11,%ymm5");
|
||||
asm volatile ("vpxor %ymm5,%ymm4,%ymm4");
|
||||
|
||||
asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pa[0][i]));
|
||||
|
||||
/* pa[1] = V[2] * d0 ^ V[3] * d1 */
|
||||
asm volatile ("vpshufb %ymm0,%ymm12,%ymm4");
|
||||
asm volatile ("vpshufb %ymm2,%ymm13,%ymm5");
|
||||
asm volatile ("vpxor %ymm5,%ymm4,%ymm4");
|
||||
|
||||
asm volatile ("vpshufb %ymm1,%ymm14,%ymm5");
|
||||
asm volatile ("vpxor %ymm5,%ymm4,%ymm4");
|
||||
asm volatile ("vpshufb %ymm3,%ymm15,%ymm5");
|
||||
asm volatile ("vpxor %ymm5,%ymm4,%ymm4");
|
||||
|
||||
asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pa[1][i]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* RAID recovering for two disks with P and Q AVX2 implementation
|
||||
*/
|
||||
|
|
@ -1978,11 +1916,16 @@ static __always_inline void raid_recX_avx2ext_123(int nr, int *id, int *ip, int
|
|||
}
|
||||
|
||||
/*
|
||||
* RAID recovering AVX2 implementation.
|
||||
* RAID recovering AVX2 extended implementation.
|
||||
*
|
||||
* Compute only the selected syndromes, keeping them in registers.
|
||||
* This avoids raid_delta_gen(), temporary syndrome buffers, and the
|
||||
* generation of unused parity rows.
|
||||
*
|
||||
* If P is available, preserve the complete P delta syndrome and
|
||||
* reconstruct only nr - 1 missing blocks through the inverse matrix.
|
||||
* The last missing block is obtained by XORing the reconstructed blocks
|
||||
* out of Pdelta.
|
||||
*/
|
||||
static __always_inline void raid_recX_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -1999,6 +1942,8 @@ static __always_inline void raid_recX_avx2ext(int nr, int *id, int *ip, int nd,
|
|||
int ns;
|
||||
int has_p;
|
||||
|
||||
BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
|
||||
|
||||
/* setup the coefficients matrix */
|
||||
for (j = 0; j < nr; ++j)
|
||||
for (k = 0; k < nr; ++k)
|
||||
|
|
@ -2037,16 +1982,18 @@ static __always_inline void raid_recX_avx2ext(int nr, int *id, int *ip, int nd,
|
|||
BUG_ON(k != nr);
|
||||
BUG_ON(ns != nd - nr);
|
||||
|
||||
/* precompute inverse-matrix multiplication table pointers */
|
||||
for (j = 0; j < nr; ++j)
|
||||
/*
|
||||
* If P is available, the last inverse-matrix row is not needed.
|
||||
* The corresponding missing block is reconstructed by XOR.
|
||||
*/
|
||||
for (j = 0; j < nr - has_p; ++j)
|
||||
for (k = 0; k < nr; ++k)
|
||||
R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
for (i = 0; i < size; i += 32) {
|
||||
asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0]));
|
||||
/*
|
||||
* Start each syndrome with the corresponding stored parity.
|
||||
*
|
||||
|
|
@ -2160,6 +2107,16 @@ static __always_inline void raid_recX_avx2ext(int nr, int *id, int *ip, int nd,
|
|||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Preserve the complete P delta before splitting syndrome 0.
|
||||
*
|
||||
* ymm14 is no longer needed by syndrome generation and is not
|
||||
* used by the split. After the split, Pdelta is moved to ymm15,
|
||||
* freeing ymm14 again for reconstruction.
|
||||
*/
|
||||
if (has_p)
|
||||
asm volatile ("vmovdqa %ymm0,%ymm14");
|
||||
|
||||
/*
|
||||
* Split every completed syndrome once.
|
||||
*
|
||||
|
|
@ -2206,8 +2163,30 @@ static __always_inline void raid_recX_avx2ext(int nr, int *id, int *ip, int nd,
|
|||
asm volatile ("vpand %ymm15,%ymm11,%ymm11");
|
||||
}
|
||||
|
||||
/* reconstruct every missing data block */
|
||||
for (j = 0; j < nr; ++j) {
|
||||
/*
|
||||
* The nibble mask is no longer needed.
|
||||
*
|
||||
* Move the complete P delta from the temporary register to
|
||||
* ymm15, which remains the P accumulator for reconstruction.
|
||||
* ymm14 is now free again as the multiplication temporary.
|
||||
*/
|
||||
if (has_p)
|
||||
asm volatile ("vmovdqa %ymm14,%ymm15");
|
||||
|
||||
/*
|
||||
* Reconstruct the missing data blocks.
|
||||
*
|
||||
* If P is available, only nr - 1 outputs are reconstructed
|
||||
* through the inverse matrix. Each reconstructed output is
|
||||
* XORed out of ymm15. The remaining value is the final
|
||||
* missing block.
|
||||
*
|
||||
* ymm12 = low accumulator
|
||||
* ymm13 = high accumulator
|
||||
* ymm14 = multiplication table / result
|
||||
* ymm15 = remaining Pdelta, if has_p
|
||||
*/
|
||||
for (j = 0; j < nr - has_p; ++j) {
|
||||
const uint8_t **t = R[j];
|
||||
|
||||
/* first coefficient initializes the low/high accumulators */
|
||||
|
|
@ -2256,20 +2235,25 @@ static __always_inline void raid_recX_avx2ext(int nr, int *id, int *ip, int nd,
|
|||
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[5][0]));
|
||||
asm volatile ("vpshufb %ymm10,%ymm14,%ymm14");
|
||||
asm volatile ("vpxor %ymm14,%ymm12,%ymm12");
|
||||
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[5][16]));
|
||||
asm volatile ("vpshufb %ymm11,%ymm14,%ymm14");
|
||||
asm volatile ("vpxor %ymm14,%ymm13,%ymm13");
|
||||
}
|
||||
|
||||
asm volatile ("vpxor %ymm13,%ymm12,%ymm12");
|
||||
|
||||
if (has_p)
|
||||
asm volatile ("vpxor %ymm12,%ymm15,%ymm15");
|
||||
|
||||
asm volatile ("vmovdqa %%ymm12,%0" : "=m" (pa[j][i]));
|
||||
}
|
||||
|
||||
if (has_p)
|
||||
asm volatile ("vmovdqa %%ymm15,%0" : "=m" (pa[nr - 1][i]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
void raid_gen2_avx2_raid(int nd, size_t size, void **vv)
|
||||
|
|
|
|||
|
|
@ -544,10 +544,10 @@ static __always_inline void raid_genX_avx512bw(int nd, size_t size, void **vv, i
|
|||
}
|
||||
|
||||
/*
|
||||
* RAID recovering AVX512BW implementation.
|
||||
* Recover multiple data failures using selected parity blocks with AVX512BW.
|
||||
*
|
||||
* Compute only the selected syndromes directly from the stored parity
|
||||
* and surviving data. This avoids raid_delta_gen(), temporary delta
|
||||
* Compute only the selected syndromes, keeping them in registers.
|
||||
* This avoids raid_delta_gen(), temporary syndrome buffers, recomputation of
|
||||
* parity blocks, and generation of unused parity rows.
|
||||
*
|
||||
* AVX512 provides enough registers to keep all six supported syndromes
|
||||
|
|
@ -611,8 +611,11 @@ static __always_inline void raid_recX_avx512bw(int nr, int *id, int *ip, int nd,
|
|||
BUG_ON(k != nr);
|
||||
BUG_ON(ns != nd - nr);
|
||||
|
||||
/* precompute inverse-matrix multiplication table pointers */
|
||||
for (j = 0; j < nr; ++j)
|
||||
/*
|
||||
* If P is available, the last missing block is reconstructed
|
||||
* directly from Pdelta, so the last inverse row isn't needed.
|
||||
*/
|
||||
for (j = 0; j < nr - has_p; ++j)
|
||||
for (k = 0; k < nr; ++k)
|
||||
R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
|
||||
|
||||
|
|
@ -632,6 +635,7 @@ static __always_inline void raid_recX_avx512bw(int nr, int *id, int *ip, int nd,
|
|||
* zmm13 source high nibble
|
||||
* zmm14 low multiplication table / result
|
||||
* zmm15 high multiplication table / result
|
||||
* zmm30 remaining P delta, if has_p
|
||||
* zmm31 low-nibble mask
|
||||
*
|
||||
* After syndrome splitting:
|
||||
|
|
@ -740,6 +744,13 @@ static __always_inline void raid_recX_avx512bw(int nr, int *id, int *ip, int nd,
|
|||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Preserve the complete P delta before syndrome 0 is
|
||||
* destructively split into low/high nibbles.
|
||||
*/
|
||||
if (has_p)
|
||||
asm volatile ("vmovdqa64 %zmm0,%zmm30");
|
||||
|
||||
/* split every completed syndrome once into low/high nibbles */
|
||||
asm volatile ("vpsrlw $4,%zmm0,%zmm1");
|
||||
asm volatile ("vpandq %zmm31,%zmm0,%zmm0");
|
||||
|
|
@ -776,7 +787,11 @@ static __always_inline void raid_recX_avx512bw(int nr, int *id, int *ip, int nd,
|
|||
}
|
||||
|
||||
/*
|
||||
* Reconstruct every missing data block.
|
||||
* Reconstruct the missing data blocks.
|
||||
*
|
||||
* If P is available, reconstruct only nr - 1 blocks through
|
||||
* the inverse matrix. XOR each result out of zmm30, leaving
|
||||
* the final missing block in zmm30.
|
||||
*
|
||||
* Keep independent low/high dependency chains:
|
||||
*
|
||||
|
|
@ -784,11 +799,12 @@ static __always_inline void raid_recX_avx512bw(int nr, int *id, int *ip, int nd,
|
|||
* zmm13 high accumulator
|
||||
* zmm14 low multiplication table/result
|
||||
* zmm15 high multiplication table/result
|
||||
* zmm30 remaining P delta, if has_p
|
||||
*
|
||||
* This is intentional. Do not serialize low/high products
|
||||
* through a single temporary register.
|
||||
*/
|
||||
for (j = 0; j < nr; ++j) {
|
||||
for (j = 0; j < nr - has_p; ++j) {
|
||||
const uint8_t **t = R[j];
|
||||
|
||||
/* coefficient 0 initializes both accumulators */
|
||||
|
|
@ -842,9 +858,22 @@ static __always_inline void raid_recX_avx512bw(int nr, int *id, int *ip, int nd,
|
|||
asm volatile ("vpxorq %zmm15,%zmm13,%zmm13");
|
||||
}
|
||||
|
||||
/* combine low/high products into the reconstructed block */
|
||||
asm volatile ("vpxorq %zmm13,%zmm12,%zmm12");
|
||||
|
||||
/*
|
||||
* Remove the reconstructed block from Pdelta.
|
||||
* After nr - 1 iterations zmm30 contains the final
|
||||
* missing data block.
|
||||
*/
|
||||
if (has_p)
|
||||
asm volatile ("vpxorq %zmm12,%zmm30,%zmm30");
|
||||
|
||||
asm volatile ("vmovdqa64 %%zmm12,%0" : "=m" (pa[j][i]));
|
||||
}
|
||||
|
||||
if (has_p)
|
||||
asm volatile ("vmovdqa64 %%zmm30,%0" : "=m" (pa[nr - 1][i]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
|
|
|
|||
54
raid/gfni.c
54
raid/gfni.c
|
|
@ -599,6 +599,16 @@ static __always_inline void raid_recX_avx2gfni_raid(int nr, int *id, int *ip, in
|
|||
raid_avx_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* Recover multiple data failures using selected parity blocks with AVX512 GFNI.
|
||||
*
|
||||
* Compute only the selected syndromes, keeping them in registers.
|
||||
* This avoids raid_delta_gen(), temporary syndrome buffers, recomputation of
|
||||
* parity blocks, and generation of unused parity rows.
|
||||
*
|
||||
* If P is available, reconstruct only nr - 1 missing blocks through the
|
||||
* inverse matrix and derive the last missing block from the P delta by XOR.
|
||||
*/
|
||||
static __always_inline void raid_recX_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
|
|
@ -649,7 +659,8 @@ static __always_inline void raid_recX_avx512gfni_raid(int nr, int *id, int *ip,
|
|||
BUG_ON(k != nr);
|
||||
BUG_ON(ns != nd - nr);
|
||||
|
||||
for (j = 0; j < nr; ++j)
|
||||
/* the last inverse row isn't needed when P is available */
|
||||
for (j = 0; j < nr - has_p; ++j)
|
||||
for (k = 0; k < nr; ++k)
|
||||
R[j][k] = raid_gfaffine_raid[V[j * nr + k]];
|
||||
|
||||
|
|
@ -712,7 +723,15 @@ static __always_inline void raid_recX_avx512gfni_raid(int nr, int *id, int *ip,
|
|||
}
|
||||
}
|
||||
|
||||
for (j = 0; j < nr; ++j) {
|
||||
/* preserve the complete P delta */
|
||||
if (has_p)
|
||||
asm volatile ("vmovdqa64 %zmm0,%zmm30");
|
||||
|
||||
/*
|
||||
* If P is available, reconstruct only nr - 1 blocks through
|
||||
* the inverse matrix. The final block remains in zmm30.
|
||||
*/
|
||||
for (j = 0; j < nr - has_p; ++j) {
|
||||
const uint8_t **t = R[j];
|
||||
|
||||
asm volatile ("vpbroadcastq %0,%%zmm6" : : "m" (t[0][0]));
|
||||
|
|
@ -748,13 +767,19 @@ static __always_inline void raid_recX_avx512gfni_raid(int nr, int *id, int *ip,
|
|||
asm volatile ("vpxorq %zmm8,%zmm7,%zmm7");
|
||||
}
|
||||
|
||||
if (has_p)
|
||||
asm volatile ("vpxorq %zmm7,%zmm30,%zmm30");
|
||||
|
||||
asm volatile ("vmovdqa64 %%zmm7,%0" : "=m" (pa[j][i]));
|
||||
}
|
||||
|
||||
if (has_p)
|
||||
asm volatile ("vmovdqa64 %%zmm30,%0" : "=m" (pa[nr - 1][i]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
|
||||
|
||||
static __always_inline void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
|
|
@ -944,6 +969,17 @@ static __always_inline void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int
|
|||
raid_avx_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* Recover multiple data failures using selected parity blocks with AVX512 GFNI
|
||||
* and the AES polynomial.
|
||||
*
|
||||
* Compute only the selected syndromes, keeping them in registers.
|
||||
* This avoids raid_delta_gen(), temporary syndrome buffers, recomputation of
|
||||
* parity blocks, and generation of unused parity rows.
|
||||
*
|
||||
* If P is available, reconstruct only nr - 1 missing blocks through the
|
||||
* inverse matrix and derive the last missing block from the P delta by XOR.
|
||||
*/
|
||||
static __always_inline void raid_recX_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
|
|
@ -1050,7 +1086,11 @@ static __always_inline void raid_recX_avx512gfni_aes(int nr, int *id, int *ip, i
|
|||
}
|
||||
}
|
||||
|
||||
for (j = 0; j < nr; ++j) {
|
||||
/* preserve the complete P delta */
|
||||
if (has_p)
|
||||
asm volatile ("vmovdqa64 %zmm0,%zmm30");
|
||||
|
||||
for (j = 0; j < nr - has_p; ++j) {
|
||||
asm volatile ("vpbroadcastb %0,%%zmm6" : : "m" (V[j * nr]));
|
||||
asm volatile ("vgf2p8mulb %zmm6,%zmm0,%zmm7");
|
||||
|
||||
|
|
@ -1084,8 +1124,14 @@ static __always_inline void raid_recX_avx512gfni_aes(int nr, int *id, int *ip, i
|
|||
asm volatile ("vpxorq %zmm8,%zmm7,%zmm7");
|
||||
}
|
||||
|
||||
if (has_p)
|
||||
asm volatile ("vpxorq %zmm7,%zmm30,%zmm30");
|
||||
|
||||
asm volatile ("vmovdqa64 %%zmm7,%0" : "=m" (pa[j][i]));
|
||||
}
|
||||
|
||||
if (has_p)
|
||||
asm volatile ("vmovdqa64 %%zmm30,%0" : "=m" (pa[nr - 1][i]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
|
|
|
|||
|
|
@ -566,8 +566,7 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
|
|||
raid_neon_end();
|
||||
}
|
||||
|
||||
static __always_inline void raid_recX_neon(int nr, int *id, int *ip,
|
||||
int nd, size_t size, void **vv)
|
||||
static __always_inline void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p[RAID_PARITY_MAX];
|
||||
|
|
|
|||
|
|
@ -518,8 +518,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
|
|||
raid_neon32_end();
|
||||
}
|
||||
|
||||
static __always_inline void raid_recX_neon32_123(int nr, int *id, int *ip,
|
||||
int nd, size_t size, void **vv)
|
||||
static __always_inline void raid_recX_neon32_123(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p[RAID_PARITY_MAX];
|
||||
|
|
@ -816,8 +815,7 @@ static __always_inline void raid_recX_neon32_123(int nr, int *id, int *ip,
|
|||
raid_neon32_end();
|
||||
}
|
||||
|
||||
static __always_inline void raid_recX_neon32(int nr, int *id, int *ip,
|
||||
int nd, size_t size, void **vv)
|
||||
static __always_inline void raid_recX_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p[RAID_PARITY_MAX];
|
||||
|
|
|
|||
380
raid/ssse3.c
380
raid/ssse3.c
|
|
@ -1376,8 +1376,17 @@ static __always_inline void raid_rec2of2_ssse3(int *id, int *ip, int nd, size_t
|
|||
raid_sse_end();
|
||||
}
|
||||
|
||||
static __always_inline void raid_recX_ssse3_1234(int nr, int *id, int *ip,
|
||||
int nd, size_t size, void **vv)
|
||||
/*
|
||||
* RAID recovering SSSE3 implementation optimized for up to four failures.
|
||||
*
|
||||
* If P is available, keep the complete P delta syndrome in xmm6.
|
||||
* Reconstruct only nr - 1 missing blocks through the inverse matrix and
|
||||
* obtain the last block by XORing the reconstructed blocks out of Pdelta.
|
||||
*
|
||||
* After the survivor scan xmm6 is no longer needed, leaving it available
|
||||
* for the P delta accumulator.
|
||||
*/
|
||||
static __always_inline void raid_recX_ssse3_1234(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p[RAID_PARITY_MAX];
|
||||
|
|
@ -1431,7 +1440,7 @@ static __always_inline void raid_recX_ssse3_1234(int nr, int *id, int *ip,
|
|||
BUG_ON(k != nr);
|
||||
BUG_ON(ns != nd - nr);
|
||||
|
||||
for (j = 0; j < nr; ++j)
|
||||
for (j = 0; j < nr - has_p; ++j)
|
||||
for (k = 0; k < nr; ++k)
|
||||
R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
|
||||
|
||||
|
|
@ -1505,6 +1514,10 @@ static __always_inline void raid_recX_ssse3_1234(int nr, int *id, int *ip,
|
|||
}
|
||||
}
|
||||
|
||||
/* preserve the complete P delta before splitting syndrome 0 */
|
||||
if (has_p)
|
||||
asm volatile ("movdqa %xmm0,%xmm6");
|
||||
|
||||
/*
|
||||
* Split completed syndromes once and store the low/high nibbles.
|
||||
* The survivor scan is already complete.
|
||||
|
|
@ -1548,7 +1561,11 @@ static __always_inline void raid_recX_ssse3_1234(int nr, int *id, int *ip,
|
|||
asm volatile ("movdqa %%xmm5,%0" : "=m" (pd_high[48]));
|
||||
}
|
||||
|
||||
for (j = 0; j < nr; ++j) {
|
||||
/*
|
||||
* Reconstruct all but the last missing block when P is
|
||||
* available. xmm6 keeps the remaining P delta.
|
||||
*/
|
||||
for (j = 0; j < nr - has_p; ++j) {
|
||||
const uint8_t **t = R[j];
|
||||
|
||||
asm volatile ("movdqa %0,%%xmm0" : : "m" (t[0][0]));
|
||||
|
|
@ -1570,15 +1587,31 @@ static __always_inline void raid_recX_ssse3_1234(int nr, int *id, int *ip,
|
|||
}
|
||||
|
||||
asm volatile ("pxor %xmm1,%xmm0");
|
||||
|
||||
if (has_p)
|
||||
asm volatile ("pxor %xmm0,%xmm6");
|
||||
|
||||
asm volatile ("movdqa %%xmm0,%0" : "=m" (pa[j][i]));
|
||||
}
|
||||
|
||||
if (has_p)
|
||||
asm volatile ("movdqa %%xmm6,%0" : "=m" (pa[nr - 1][i]));
|
||||
}
|
||||
|
||||
raid_sse_end();
|
||||
}
|
||||
|
||||
static __always_inline void raid_recX_ssse3(int nr, int *id, int *ip,
|
||||
int nd, size_t size, void **vv)
|
||||
/*
|
||||
* RAID recovering SSSE3 implementation.
|
||||
*
|
||||
* If P is available, keep the complete P delta syndrome in xmm6.
|
||||
* Reconstruct only nr - 1 missing blocks through the inverse matrix and
|
||||
* obtain the last block by XORing the reconstructed blocks out of Pdelta.
|
||||
*
|
||||
* The completed syndromes are kept in temporary memory, leaving xmm6
|
||||
* available for the P delta accumulator during reconstruction.
|
||||
*/
|
||||
static __always_inline void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p[RAID_PARITY_MAX];
|
||||
|
|
@ -1632,7 +1665,7 @@ static __always_inline void raid_recX_ssse3(int nr, int *id, int *ip,
|
|||
BUG_ON(k != nr);
|
||||
BUG_ON(ns != nd - nr);
|
||||
|
||||
for (j = 0; j < nr; ++j)
|
||||
for (j = 0; j < nr - has_p; ++j)
|
||||
for (k = 0; k < nr; ++k)
|
||||
R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
|
||||
|
||||
|
|
@ -1676,6 +1709,13 @@ static __always_inline void raid_recX_ssse3(int nr, int *id, int *ip,
|
|||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Preserve the complete P delta before pd_low[0] is converted
|
||||
* to its low-nibble representation.
|
||||
*/
|
||||
if (has_p)
|
||||
asm volatile ("movdqa %0,%%xmm6" : : "m" (pd_low[0]));
|
||||
|
||||
/* convert all completed raw syndromes to low/high nibble form */
|
||||
for (k = 0; k < nr; ++k) {
|
||||
asm volatile ("movdqa %0,%%xmm0" : : "m" (pd_low[k * 16]));
|
||||
|
|
@ -1687,8 +1727,11 @@ static __always_inline void raid_recX_ssse3(int nr, int *id, int *ip,
|
|||
asm volatile ("movdqa %%xmm1,%0" : "=m" (pd_high[k * 16]));
|
||||
}
|
||||
|
||||
/* reconstruct */
|
||||
for (j = 0; j < nr; ++j) {
|
||||
/*
|
||||
* Reconstruct all but the last missing block when P is
|
||||
* available. xmm6 keeps the remaining P delta.
|
||||
*/
|
||||
for (j = 0; j < nr - has_p; ++j) {
|
||||
const uint8_t **t = R[j];
|
||||
|
||||
asm volatile ("movdqa %0,%%xmm0" : : "m" (t[0][0]));
|
||||
|
|
@ -1710,8 +1753,15 @@ static __always_inline void raid_recX_ssse3(int nr, int *id, int *ip,
|
|||
}
|
||||
|
||||
asm volatile ("pxor %xmm1,%xmm0");
|
||||
|
||||
if (has_p)
|
||||
asm volatile ("pxor %xmm0,%xmm6");
|
||||
|
||||
asm volatile ("movdqa %%xmm0,%0" : "=m" (pa[j][i]));
|
||||
}
|
||||
|
||||
if (has_p)
|
||||
asm volatile ("movdqa %%xmm6,%0" : "=m" (pa[nr - 1][i]));
|
||||
}
|
||||
|
||||
raid_sse_end();
|
||||
|
|
@ -2006,8 +2056,7 @@ static __always_inline void raid_rec2of2_ssse3ext(int *id, int *ip, int nd, size
|
|||
raid_sse_end();
|
||||
}
|
||||
|
||||
static __always_inline void raid_recX_ssse3ext_123(int nr, int *id, int *ip,
|
||||
int nd, size_t size, void **vv)
|
||||
static __always_inline void raid_recX_ssse3ext_123(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p[RAID_PARITY_MAX];
|
||||
|
|
@ -2256,8 +2305,308 @@ static __always_inline void raid_recX_ssse3ext_123(int nr, int *id, int *ip,
|
|||
raid_sse_end();
|
||||
}
|
||||
|
||||
static __always_inline void raid_recX_ssse3ext(int nr, int *id, int *ip,
|
||||
int nd, size_t size, void **vv)
|
||||
/*
|
||||
* RAID recovering SSSE3 extended implementation optimized for up to five
|
||||
* failures.
|
||||
*
|
||||
* If P is available, keep the complete P delta syndrome in xmm10.
|
||||
* Reconstruct only nr - 1 missing blocks through the inverse matrix and
|
||||
* obtain the last block by XORing the reconstructed blocks out of Pdelta.
|
||||
*
|
||||
* With at most five failures the low/high syndrome pairs use xmm0..xmm9,
|
||||
* leaving xmm10 available for the P delta accumulator.
|
||||
*/
|
||||
static __always_inline void raid_recX_ssse3ext_12345(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p[RAID_PARITY_MAX];
|
||||
uint8_t *pa[RAID_PARITY_MAX];
|
||||
uint8_t *src[RAID_DATA_MAX];
|
||||
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
||||
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
||||
const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX];
|
||||
const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
|
||||
size_t i;
|
||||
int d, j, k, s;
|
||||
int ns;
|
||||
int has_p;
|
||||
|
||||
BUG_ON(nr < 1 || nr > 5);
|
||||
|
||||
for (j = 0; j < nr; ++j)
|
||||
for (k = 0; k < nr; ++k)
|
||||
G[j * nr + k] = A(ip[j], id[k]);
|
||||
|
||||
raid_invert(G, V, nr);
|
||||
|
||||
for (j = 0; j < nr; ++j) {
|
||||
p[j] = v[nd + ip[j]];
|
||||
pa[j] = v[id[j]];
|
||||
}
|
||||
|
||||
/* ip[] is ordered. If P is available, it is always ip[0] */
|
||||
has_p = ip[0] == 0;
|
||||
|
||||
ns = 0;
|
||||
k = 0;
|
||||
|
||||
for (d = 0; d < nd; ++d) {
|
||||
if (k < nr && d == id[k]) {
|
||||
++k;
|
||||
continue;
|
||||
}
|
||||
|
||||
src[ns] = v[d];
|
||||
|
||||
for (j = 0; j < nr; ++j)
|
||||
S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0];
|
||||
|
||||
++ns;
|
||||
}
|
||||
|
||||
BUG_ON(k != nr);
|
||||
BUG_ON(ns != nd - nr);
|
||||
|
||||
/*
|
||||
* If P is available the last inverse-matrix row isn't needed.
|
||||
* The corresponding missing block is obtained by XOR.
|
||||
*/
|
||||
for (j = 0; j < nr - has_p; ++j)
|
||||
for (k = 0; k < nr; ++k)
|
||||
R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
|
||||
|
||||
raid_sse_begin();
|
||||
|
||||
for (i = 0; i < size; i += 16) {
|
||||
/*
|
||||
* xmm15 is reused by reconstruction, therefore reload the
|
||||
* nibble mask on every iteration.
|
||||
*/
|
||||
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
/*
|
||||
* Raw syndrome accumulators:
|
||||
*
|
||||
* xmm0 syndrome 0
|
||||
* xmm1 syndrome 1
|
||||
* xmm2 syndrome 2
|
||||
* xmm3 syndrome 3
|
||||
* xmm4 syndrome 4
|
||||
*
|
||||
* xmm10 remains free for Pdelta.
|
||||
*/
|
||||
asm volatile ("movdqa %0,%%xmm0" : : "m" (p[0][i]));
|
||||
|
||||
if (nr >= 2)
|
||||
asm volatile ("movdqa %0,%%xmm1" : : "m" (p[1][i]));
|
||||
|
||||
if (nr >= 3)
|
||||
asm volatile ("movdqa %0,%%xmm2" : : "m" (p[2][i]));
|
||||
|
||||
if (nr >= 4)
|
||||
asm volatile ("movdqa %0,%%xmm3" : : "m" (p[3][i]));
|
||||
|
||||
if (nr >= 5)
|
||||
asm volatile ("movdqa %0,%%xmm4" : : "m" (p[4][i]));
|
||||
|
||||
/* add all surviving data contributions */
|
||||
for (s = 0; s < ns; ++s) {
|
||||
const uint8_t **t = S[s];
|
||||
|
||||
asm volatile ("movdqa %0,%%xmm6" : : "m" (src[s][i]));
|
||||
|
||||
/*
|
||||
* P has coefficient 1 for every data disk.
|
||||
* XOR the original source before splitting it.
|
||||
*/
|
||||
if (has_p) {
|
||||
asm volatile ("pxor %xmm6,%xmm0");
|
||||
|
||||
asm volatile ("movdqa %xmm6,%xmm7");
|
||||
asm volatile ("psrlw $4,%xmm7");
|
||||
asm volatile ("pand %xmm15,%xmm6");
|
||||
asm volatile ("pand %xmm15,%xmm7");
|
||||
} else {
|
||||
asm volatile ("movdqa %xmm6,%xmm7");
|
||||
asm volatile ("psrlw $4,%xmm7");
|
||||
asm volatile ("pand %xmm15,%xmm6");
|
||||
asm volatile ("pand %xmm15,%xmm7");
|
||||
|
||||
asm volatile ("movdqa %0,%%xmm8" : : "m" (t[0][0]));
|
||||
asm volatile ("movdqa %0,%%xmm9" : : "m" (t[0][16]));
|
||||
asm volatile ("pshufb %xmm6,%xmm8");
|
||||
asm volatile ("pshufb %xmm7,%xmm9");
|
||||
asm volatile ("pxor %xmm8,%xmm0");
|
||||
asm volatile ("pxor %xmm9,%xmm0");
|
||||
}
|
||||
|
||||
if (nr >= 2) {
|
||||
asm volatile ("movdqa %0,%%xmm8" : : "m" (t[1][0]));
|
||||
asm volatile ("movdqa %0,%%xmm9" : : "m" (t[1][16]));
|
||||
asm volatile ("pshufb %xmm6,%xmm8");
|
||||
asm volatile ("pshufb %xmm7,%xmm9");
|
||||
asm volatile ("pxor %xmm8,%xmm1");
|
||||
asm volatile ("pxor %xmm9,%xmm1");
|
||||
}
|
||||
|
||||
if (nr >= 3) {
|
||||
asm volatile ("movdqa %0,%%xmm8" : : "m" (t[2][0]));
|
||||
asm volatile ("movdqa %0,%%xmm9" : : "m" (t[2][16]));
|
||||
asm volatile ("pshufb %xmm6,%xmm8");
|
||||
asm volatile ("pshufb %xmm7,%xmm9");
|
||||
asm volatile ("pxor %xmm8,%xmm2");
|
||||
asm volatile ("pxor %xmm9,%xmm2");
|
||||
}
|
||||
|
||||
if (nr >= 4) {
|
||||
asm volatile ("movdqa %0,%%xmm8" : : "m" (t[3][0]));
|
||||
asm volatile ("movdqa %0,%%xmm9" : : "m" (t[3][16]));
|
||||
asm volatile ("pshufb %xmm6,%xmm8");
|
||||
asm volatile ("pshufb %xmm7,%xmm9");
|
||||
asm volatile ("pxor %xmm8,%xmm3");
|
||||
asm volatile ("pxor %xmm9,%xmm3");
|
||||
}
|
||||
|
||||
if (nr >= 5) {
|
||||
asm volatile ("movdqa %0,%%xmm8" : : "m" (t[4][0]));
|
||||
asm volatile ("movdqa %0,%%xmm9" : : "m" (t[4][16]));
|
||||
asm volatile ("pshufb %xmm6,%xmm8");
|
||||
asm volatile ("pshufb %xmm7,%xmm9");
|
||||
asm volatile ("pxor %xmm8,%xmm4");
|
||||
asm volatile ("pxor %xmm9,%xmm4");
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Preserve the complete P delta before splitting syndrome 0.
|
||||
*
|
||||
* xmm10 = M0 ^ M1 ^ ... ^ M(nr-1)
|
||||
*/
|
||||
if (has_p)
|
||||
asm volatile ("movdqa %xmm0,%xmm10");
|
||||
|
||||
/*
|
||||
* Expand backwards to low/high syndrome pairs:
|
||||
*
|
||||
* xmm0/xmm1 syndrome 0 low/high
|
||||
* xmm2/xmm3 syndrome 1 low/high
|
||||
* xmm4/xmm5 syndrome 2 low/high
|
||||
* xmm6/xmm7 syndrome 3 low/high
|
||||
* xmm8/xmm9 syndrome 4 low/high
|
||||
*
|
||||
* xmm10 remains the complete remaining P delta.
|
||||
*/
|
||||
if (nr >= 5) {
|
||||
asm volatile ("movdqa %xmm4,%xmm8");
|
||||
asm volatile ("movdqa %xmm4,%xmm9");
|
||||
asm volatile ("psrlw $4,%xmm9");
|
||||
asm volatile ("pand %xmm15,%xmm8");
|
||||
asm volatile ("pand %xmm15,%xmm9");
|
||||
}
|
||||
|
||||
if (nr >= 4) {
|
||||
asm volatile ("movdqa %xmm3,%xmm6");
|
||||
asm volatile ("movdqa %xmm3,%xmm7");
|
||||
asm volatile ("psrlw $4,%xmm7");
|
||||
asm volatile ("pand %xmm15,%xmm6");
|
||||
asm volatile ("pand %xmm15,%xmm7");
|
||||
}
|
||||
|
||||
if (nr >= 3) {
|
||||
asm volatile ("movdqa %xmm2,%xmm4");
|
||||
asm volatile ("movdqa %xmm2,%xmm5");
|
||||
asm volatile ("psrlw $4,%xmm5");
|
||||
asm volatile ("pand %xmm15,%xmm4");
|
||||
asm volatile ("pand %xmm15,%xmm5");
|
||||
}
|
||||
|
||||
if (nr >= 2) {
|
||||
asm volatile ("movdqa %xmm1,%xmm2");
|
||||
asm volatile ("movdqa %xmm1,%xmm3");
|
||||
asm volatile ("psrlw $4,%xmm3");
|
||||
asm volatile ("pand %xmm15,%xmm2");
|
||||
asm volatile ("pand %xmm15,%xmm3");
|
||||
}
|
||||
|
||||
asm volatile ("movdqa %xmm0,%xmm1");
|
||||
asm volatile ("psrlw $4,%xmm1");
|
||||
asm volatile ("pand %xmm15,%xmm0");
|
||||
asm volatile ("pand %xmm15,%xmm1");
|
||||
|
||||
/*
|
||||
* Reconstruct all but the last missing block when P is
|
||||
* available.
|
||||
*
|
||||
* xmm12/xmm13 = low/high output accumulators
|
||||
* xmm14/xmm15 = multiplication temporaries
|
||||
* xmm10 = remaining P delta
|
||||
*/
|
||||
for (j = 0; j < nr - has_p; ++j) {
|
||||
const uint8_t **t = R[j];
|
||||
|
||||
/* coefficient 0 initializes both accumulators */
|
||||
asm volatile ("movdqa %0,%%xmm12" : : "m" (t[0][0]));
|
||||
asm volatile ("movdqa %0,%%xmm13" : : "m" (t[0][16]));
|
||||
asm volatile ("pshufb %xmm0,%xmm12");
|
||||
asm volatile ("pshufb %xmm1,%xmm13");
|
||||
|
||||
if (nr >= 2) {
|
||||
asm volatile ("movdqa %0,%%xmm14" : : "m" (t[1][0]));
|
||||
asm volatile ("movdqa %0,%%xmm15" : : "m" (t[1][16]));
|
||||
asm volatile ("pshufb %xmm2,%xmm14");
|
||||
asm volatile ("pshufb %xmm3,%xmm15");
|
||||
asm volatile ("pxor %xmm14,%xmm12");
|
||||
asm volatile ("pxor %xmm15,%xmm13");
|
||||
}
|
||||
|
||||
if (nr >= 3) {
|
||||
asm volatile ("movdqa %0,%%xmm14" : : "m" (t[2][0]));
|
||||
asm volatile ("movdqa %0,%%xmm15" : : "m" (t[2][16]));
|
||||
asm volatile ("pshufb %xmm4,%xmm14");
|
||||
asm volatile ("pshufb %xmm5,%xmm15");
|
||||
asm volatile ("pxor %xmm14,%xmm12");
|
||||
asm volatile ("pxor %xmm15,%xmm13");
|
||||
}
|
||||
|
||||
if (nr >= 4) {
|
||||
asm volatile ("movdqa %0,%%xmm14" : : "m" (t[3][0]));
|
||||
asm volatile ("movdqa %0,%%xmm15" : : "m" (t[3][16]));
|
||||
asm volatile ("pshufb %xmm6,%xmm14");
|
||||
asm volatile ("pshufb %xmm7,%xmm15");
|
||||
asm volatile ("pxor %xmm14,%xmm12");
|
||||
asm volatile ("pxor %xmm15,%xmm13");
|
||||
}
|
||||
|
||||
if (nr >= 5) {
|
||||
asm volatile ("movdqa %0,%%xmm14" : : "m" (t[4][0]));
|
||||
asm volatile ("movdqa %0,%%xmm15" : : "m" (t[4][16]));
|
||||
asm volatile ("pshufb %xmm8,%xmm14");
|
||||
asm volatile ("pshufb %xmm9,%xmm15");
|
||||
asm volatile ("pxor %xmm14,%xmm12");
|
||||
asm volatile ("pxor %xmm15,%xmm13");
|
||||
}
|
||||
|
||||
asm volatile ("pxor %xmm13,%xmm12");
|
||||
|
||||
/*
|
||||
* Remove the reconstructed block from Pdelta.
|
||||
* After nr - 1 iterations xmm10 contains the final
|
||||
* missing block.
|
||||
*/
|
||||
if (has_p)
|
||||
asm volatile ("pxor %xmm12,%xmm10");
|
||||
|
||||
asm volatile ("movdqa %%xmm12,%0" : "=m" (pa[j][i]));
|
||||
}
|
||||
|
||||
if (has_p)
|
||||
asm volatile ("movdqa %%xmm10,%0" : "=m" (pa[nr - 1][i]));
|
||||
}
|
||||
|
||||
raid_sse_end();
|
||||
}
|
||||
|
||||
static __always_inline void raid_recX_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p[RAID_PARITY_MAX];
|
||||
|
|
@ -2677,13 +3026,13 @@ void raid_rec3_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv
|
|||
void raid_rec4_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 4);
|
||||
raid_recX_ssse3ext(4, id, ip, nd, size, vv);
|
||||
raid_recX_ssse3ext_12345(4, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
void raid_rec5_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 5);
|
||||
raid_recX_ssse3ext(5, id, ip, nd, size, vv);
|
||||
raid_recX_ssse3ext_12345(5, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
void raid_rec6_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
|
|
@ -2691,7 +3040,6 @@ void raid_rec6_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv
|
|||
BUG_ON(nr != 6);
|
||||
raid_recX_ssse3ext(6, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
void raid_register_ssse3(void)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue