rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 10:51:54 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions
snapraid/raid/neon32.c

1321 lines
32 KiB
C
Raw Blame History

// SPDX-License-Identifier: GPL-2.0-or-later
// Copyright (C) 2026 Andrea Mazzoleni
#include "internal.h"
#include "gf.h"
#include "cpu.h"
#ifdef CONFIG_NEON32
/*
* Generate one parity block (RAID5 with XOR) using AArch32 NEON implementation.
*
* Uses 64-byte chunks across four 16-byte vectors.
*/
static __always_inline void raid_gen1_neon32_gen(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
int d, l;
size_t i;
l = nd - 1;
p = v[nd];
raid_neon32_begin();
for (i = 0; i < size; i += 64) {
asm volatile (
"vld1.8 {q0}, %0\n"
"vld1.8 {q1}, %1\n"
"vld1.8 {q2}, %2\n"
"vld1.8 {q3}, %3\n"
:
: "Q" (v[0][i]), "Q" (v[0][i + 16]),
"Q" (v[0][i + 32]), "Q" (v[0][i + 48])
);
for (d = 1; d <= l; ++d) {
asm volatile (
"vld1.8 {q4}, %0\n"
"vld1.8 {q5}, %1\n"
"vld1.8 {q6}, %2\n"
"vld1.8 {q7}, %3\n"
"veor q0, q0, q4\n"
"veor q1, q1, q5\n"
"veor q2, q2, q6\n"
"veor q3, q3, q7\n"
:
: "Q" (v[d][i]), "Q" (v[d][i + 16]),
"Q" (v[d][i + 32]), "Q" (v[d][i + 48])
);
}
asm volatile (
"vst1.8 {q0}, %0\n"
"vst1.8 {q1}, %1\n"
"vst1.8 {q2}, %2\n"
"vst1.8 {q3}, %3\n"
: "=Q" (p[i]), "=Q" (p[i + 16]),
"=Q" (p[i + 32]), "=Q" (p[i + 48])
);
}
raid_neon32_end();
}
/*
* Generate two parity blocks (RAID6 with Cauchy matrix) using AArch32 NEON implementation.
*/
static __always_inline void raid_gen2_neon32_gen(int nd, size_t size,
void **vv, int generator)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
uint8_t *q;
int d, l;
size_t i;
l = nd - 1;
p = v[nd];
q = v[nd + 1];
raid_neon32_begin();
/* q14 contains the active reduction polynomial */
asm volatile (
"vld1.8 {q14}, %0\n"
:
: "Q" (gfconst16.poly[0])
);
for (i = 0; i < size; i += 32) {
asm volatile (
"vld1.8 {q0}, %0\n"
"vld1.8 {q1}, %1\n"
"vmov q2, q0\n"
"vmov q3, q1\n"
:
: "Q" (v[l][i]), "Q" (v[l][i + 16])
);
for (d = l - 1; d >= 0; --d) {
if (generator == 3) {
asm volatile (
"vmov q10, q2\n"
"vmov q11, q3\n"
);
}
asm volatile (
"vshr.s8 q8, q2, #7\n"
"vshl.i8 q2, q2, #1\n"
"vand q8, q8, q14\n"
"veor q2, q2, q8\n"
"vshr.s8 q8, q3, #7\n"
"vshl.i8 q3, q3, #1\n"
"vand q8, q8, q14\n"
"veor q3, q3, q8\n"
"vld1.8 {q12}, %0\n"
"vld1.8 {q13}, %1\n"
"veor q0, q0, q12\n"
"veor q1, q1, q13\n"
"veor q2, q2, q12\n"
"veor q3, q3, q13\n"
:
: "Q" (v[d][i]), "Q" (v[d][i + 16])
);
if (generator == 3) {
asm volatile (
"veor q2, q2, q10\n"
"veor q3, q3, q11\n"
);
}
}
asm volatile (
"vst1.8 {q0}, %0\n"
"vst1.8 {q1}, %1\n"
"vst1.8 {q2}, %2\n"
"vst1.8 {q3}, %3\n"
: "=Q" (p[i]), "=Q" (p[i + 16]),
"=Q" (q[i]), "=Q" (q[i + 16])
);
}
raid_neon32_end();
}
/*
* Generate three parity blocks with powers of 2^-1 using AArch32 NEON implementation.
*/
static __always_inline void raid_genz_neon32_gen(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
uint8_t *q;
uint8_t *r;
int d, l;
size_t i;
l = nd - 1;
p = v[nd];
q = v[nd + 1];
r = v[nd + 2];
raid_neon32_begin();
asm volatile (
"vld1.8 {q14}, %0\n"
"vld1.8 {q15}, %1\n"
:
: "Q" (gfconst16.poly[0]), "Q" (gfconst16.half[0])
);
for (i = 0; i < size; i += 32) {
asm volatile (
"vld1.8 {q0}, %0\n"
"vld1.8 {q1}, %1\n"
"vmov q2, q0\n"
"vmov q3, q1\n"
"vmov q4, q0\n"
"vmov q5, q1\n"
:
: "Q" (v[l][i]), "Q" (v[l][i + 16])
);
for (d = l - 1; d >= 0; --d) {
asm volatile (
"vshr.s8 q8, q2, #7\n"
"vshl.i8 q2, q2, #1\n"
"vand q8, q8, q14\n"
"veor q2, q2, q8\n"
"vshr.s8 q8, q3, #7\n"
"vshl.i8 q3, q3, #1\n"
"vand q8, q8, q14\n"
"veor q3, q3, q8\n"
"vshl.i8 q9, q4, #7\n"
"vshr.s8 q9, q9, #7\n"
"vshr.u8 q4, q4, #1\n"
"vand q9, q9, q15\n"
"veor q4, q4, q9\n"
"vshl.i8 q9, q5, #7\n"
"vshr.s8 q9, q9, #7\n"
"vshr.u8 q5, q5, #1\n"
"vand q9, q9, q15\n"
"veor q5, q5, q9\n"
"vld1.8 {q12}, %0\n"
"vld1.8 {q13}, %1\n"
"veor q0, q0, q12\n"
"veor q1, q1, q13\n"
"veor q2, q2, q12\n"
"veor q3, q3, q13\n"
"veor q4, q4, q12\n"
"veor q5, q5, q13\n"
:
: "Q" (v[d][i]), "Q" (v[d][i + 16])
);
}
asm volatile (
"vst1.8 {q0}, %0\n"
"vst1.8 {q1}, %1\n"
"vst1.8 {q2}, %2\n"
"vst1.8 {q3}, %3\n"
"vst1.8 {q4}, %4\n"
"vst1.8 {q5}, %5\n"
: "=Q" (p[i]), "=Q" (p[i + 16]),
"=Q" (q[i]), "=Q" (q[i + 16]),
"=Q" (r[i]), "=Q" (r[i + 16])
);
}
raid_neon32_end();
}
/*
* Generate N parity blocks with Cauchy matrix using AArch32 NEON implementation.
*/
static __always_inline void raid_genX_neon32(int nd, size_t size,
void **vv, int np, int generator)
{
uint8_t **v = (uint8_t **)vv;
size_t i;
int d, l;
l = nd - 1;
/* special case with only one data disk */
if (l == 0) {
for (d = 0; d < np; ++d)
if (v[1 + d] != v[0])
memcpy(v[1 + d], v[0], size);
return;
}
raid_neon32_begin();
/* generic case with at least two data disks */
asm volatile (
"vld1.8 {q14}, %0\n"
"vmov.i8 q15, #0x0f\n"
:
: "Q" (gfconst16.poly[0])
);
for (i = 0; i < size; i += 16) {
/* last disk without the generator multiplication */
asm volatile (
"vld1.8 {q0}, %0\n"
"vld1.8 {q1}, %0\n"
:
: "Q" (v[l][i])
);
if (np >= 3) {
asm volatile (
"vshr.u8 q8, q0, #4\n"
"vand q7, q0, q15\n"
"vand q8, q8, q15\n"
);
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q2, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][1][0][0]),
"Q" (raid_gfcauchypshufb[l][1][1][0])
);
}
if (np >= 4) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q3, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][2][0][0]),
"Q" (raid_gfcauchypshufb[l][2][1][0])
);
}
if (np >= 5) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q4, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][3][0][0]),
"Q" (raid_gfcauchypshufb[l][3][1][0])
);
}
if (np >= 6) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q5, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][4][0][0]),
"Q" (raid_gfcauchypshufb[l][4][1][0])
);
}
/* intermediate disks */
for (d = l - 1; d > 0; --d) {
if (generator == 3) {
asm volatile (
"vmov q13, q1\n"
);
}
asm volatile (
"vshr.s8 q11, q1, #7\n"
"vshl.i8 q1, q1, #1\n"
"vand q11, q11, q14\n"
"veor q1, q1, q11\n"
"vld1.8 {q6}, %0\n"
"veor q0, q0, q6\n"
"veor q1, q1, q6\n"
:
: "Q" (v[d][i])
);
if (generator == 3) {
asm volatile (
"veor q1, q1, q13\n"
);
}
if (np >= 3) {
asm volatile (
"vshr.u8 q8, q6, #4\n"
"vand q7, q6, q15\n"
"vand q8, q8, q15\n"
);
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q2, q2, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][1][0][0]),
"Q" (raid_gfcauchypshufb[d][1][1][0])
);
}
if (np >= 4) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q3, q3, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][2][0][0]),
"Q" (raid_gfcauchypshufb[d][2][1][0])
);
}
if (np >= 5) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q4, q4, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][3][0][0]),
"Q" (raid_gfcauchypshufb[d][3][1][0])
);
}
if (np >= 6) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q5, q5, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][4][0][0]),
"Q" (raid_gfcauchypshufb[d][4][1][0])
);
}
}
/* first disk with all coefficients at 1 */
if (generator == 3) {
asm volatile (
"vmov q13, q1\n"
);
}
asm volatile (
"vshr.s8 q11, q1, #7\n"
"vshl.i8 q1, q1, #1\n"
"vand q11, q11, q14\n"
"veor q1, q1, q11\n"
"vld1.8 {q6}, %0\n"
"veor q0, q0, q6\n"
"veor q1, q1, q6\n"
:
: "Q" (v[0][i])
);
if (generator == 3) {
asm volatile (
"veor q1, q1, q13\n"
);
}
if (np >= 3) {
asm volatile (
"veor q2, q2, q6\n"
);
}
if (np >= 4) {
asm volatile (
"veor q3, q3, q6\n"
);
}
if (np >= 5) {
asm volatile (
"veor q4, q4, q6\n"
);
}
if (np >= 6) {
asm volatile (
"veor q5, q5, q6\n"
);
}
/* write parity in increasing order */
asm volatile (
"vst1.8 {q0}, %0\n"
"vst1.8 {q1}, %1\n"
: "=Q" (v[nd][i]),
"=Q" (v[nd + 1][i])
);
if (np >= 3) {
asm volatile (
"vst1.8 {q2}, %0\n"
: "=Q" (v[nd + 2][i])
);
}
if (np >= 4) {
asm volatile (
"vst1.8 {q3}, %0\n"
: "=Q" (v[nd + 3][i])
);
}
if (np >= 5) {
asm volatile (
"vst1.8 {q4}, %0\n"
: "=Q" (v[nd + 4][i])
);
}
if (np >= 6) {
asm volatile (
"vst1.8 {q5}, %0\n"
: "=Q" (v[nd + 5][i])
);
}
}
raid_neon32_end();
}
/*
* Recover multiple data failures using selected parity blocks with AArch32 NEON optimized for up to three failures.
*
* If P is available, syndrome 0 is accumulated with XOR without table lookups.
*/
static __always_inline void raid_recX_neon32_123(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p[RAID_PARITY_MAX];
uint8_t *pa[RAID_PARITY_MAX];
uint8_t *src[RAID_DATA_MAX];
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX];
const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
size_t i;
int d, j, k, s;
int ns;
BUG_ON(nr < 1 || nr > 3);
for (j = 0; j < nr; ++j)
for (k = 0; k < nr; ++k)
G[j * nr + k] = A(ip[j], id[k]);
raid_invert(G, V, nr);
for (j = 0; j < nr; ++j) {
p[j] = v[nd + ip[j]];
pa[j] = v[id[j]];
}
ns = 0;
k = 0;
for (d = 0; d < nd; ++d) {
if (k < nr && d == id[k]) {
++k;
continue;
}
src[ns] = v[d];
for (j = 0; j < nr; ++j)
S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0];
++ns;
}
BUG_ON(k != nr);
BUG_ON(ns != nd - nr);
for (j = 0; j < nr; ++j)
for (k = 0; k < nr; ++k)
R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
raid_neon32_begin();
for (i = 0; i < size; i += 32) {
/*
* Q15 is the low-nibble mask during generation/splitting.
* Reconstruction later reuses q15 as the second accumulator,
* therefore reload it on every iteration.
*/
asm volatile ("vmov.i8 q15, #0x0f");
/* selected stored parity */
asm volatile ("vld1.8 {q0}, %0" : : "Q" (p[0][i]));
asm volatile ("vld1.8 {q1}, %0" : : "Q" (p[0][i + 16]));
if (nr >= 2) {
asm volatile ("vld1.8 {q2}, %0" : : "Q" (p[1][i]));
asm volatile ("vld1.8 {q3}, %0" : : "Q" (p[1][i + 16]));
}
if (nr >= 3) {
asm volatile ("vld1.8 {q4}, %0" : : "Q" (p[2][i]));
asm volatile ("vld1.8 {q5}, %0" : : "Q" (p[2][i + 16]));
}
for (s = 0; s < ns; ++s) {
const uint8_t **t = S[s];
asm volatile ("vld1.8 {q6}, %0" : : "Q" (src[s][i]));
asm volatile ("vld1.8 {q7}, %0" : : "Q" (src[s][i + 16]));
if (has_p) {
asm volatile ("veor q0, q0, q6");
asm volatile ("veor q1, q1, q7");
asm volatile ("vshr.u8 q8, q6, #4");
asm volatile ("vshr.u8 q9, q7, #4");
asm volatile ("vand q6, q6, q15");
asm volatile ("vand q7, q7, q15");
asm volatile ("vand q8, q8, q15");
asm volatile ("vand q9, q9, q15");
} else {
asm volatile ("vshr.u8 q8, q6, #4");
asm volatile ("vshr.u8 q9, q7, #4");
asm volatile ("vand q6, q6, q15");
asm volatile ("vand q7, q7, q15");
asm volatile ("vand q8, q8, q15");
asm volatile ("vand q9, q9, q15");
/*
* Q10 = low table
* q11 = high table
* q12/q13 = multiplication temporaries
*/
asm volatile ("vld1.8 {q10}, %0" : : "Q" (t[0][0]));
asm volatile ("vld1.8 {q11}, %0" : : "Q" (t[0][16]));
asm volatile ("vtbl.8 d24, {d20-d21}, d12");
asm volatile ("vtbl.8 d25, {d20-d21}, d13");
asm volatile ("vtbl.8 d26, {d22-d23}, d16");
asm volatile ("vtbl.8 d27, {d22-d23}, d17");
asm volatile ("veor q12, q12, q13");
asm volatile ("veor q0, q0, q12");
asm volatile ("vtbl.8 d24, {d20-d21}, d14");
asm volatile ("vtbl.8 d25, {d20-d21}, d15");
asm volatile ("vtbl.8 d26, {d22-d23}, d18");
asm volatile ("vtbl.8 d27, {d22-d23}, d19");
asm volatile ("veor q12, q12, q13");
asm volatile ("veor q1, q1, q12");
}
if (nr >= 2) {
asm volatile ("vld1.8 {q10}, %0" : : "Q" (t[1][0]));
asm volatile ("vld1.8 {q11}, %0" : : "Q" (t[1][16]));
asm volatile ("vtbl.8 d24, {d20-d21}, d12");
asm volatile ("vtbl.8 d25, {d20-d21}, d13");
asm volatile ("vtbl.8 d26, {d22-d23}, d16");
asm volatile ("vtbl.8 d27, {d22-d23}, d17");
asm volatile ("veor q12, q12, q13");
asm volatile ("veor q2, q2, q12");
asm volatile ("vtbl.8 d24, {d20-d21}, d14");
asm volatile ("vtbl.8 d25, {d20-d21}, d15");
asm volatile ("vtbl.8 d26, {d22-d23}, d18");
asm volatile ("vtbl.8 d27, {d22-d23}, d19");
asm volatile ("veor q12, q12, q13");
asm volatile ("veor q3, q3, q12");
}
if (nr >= 3) {
asm volatile ("vld1.8 {q10}, %0" : : "Q" (t[2][0]));
asm volatile ("vld1.8 {q11}, %0" : : "Q" (t[2][16]));
asm volatile ("vtbl.8 d24, {d20-d21}, d12");
asm volatile ("vtbl.8 d25, {d20-d21}, d13");
asm volatile ("vtbl.8 d26, {d22-d23}, d16");
asm volatile ("vtbl.8 d27, {d22-d23}, d17");
asm volatile ("veor q12, q12, q13");
asm volatile ("veor q4, q4, q12");
asm volatile ("vtbl.8 d24, {d20-d21}, d14");
asm volatile ("vtbl.8 d25, {d20-d21}, d15");
asm volatile ("vtbl.8 d26, {d22-d23}, d18");
asm volatile ("vtbl.8 d27, {d22-d23}, d19");
asm volatile ("veor q12, q12, q13");
asm volatile ("veor q5, q5, q12");
}
}
/*
* Expand raw syndromes backwards.
*
* Final layout:
*
* S0:
* q0 = lane0 low
* q1 = lane0 high
* q2 = lane1 low
* q3 = lane1 high
*
* S1:
* q4/q5 = lane0 low/high
* q6/q7 = lane1 low/high
*
* S2:
* q8/q9 = lane0 low/high
* q10/q11 = lane1 low/high
*/
if (nr >= 3) {
asm volatile ("vshr.u8 q9, q4, #4");
asm volatile ("vand q8, q4, q15");
asm volatile ("vand q9, q9, q15");
asm volatile ("vshr.u8 q11, q5, #4");
asm volatile ("vand q10, q5, q15");
asm volatile ("vand q11, q11, q15");
}
if (nr >= 2) {
asm volatile ("vshr.u8 q5, q2, #4");
asm volatile ("vand q4, q2, q15");
asm volatile ("vand q5, q5, q15");
asm volatile ("vshr.u8 q7, q3, #4");
asm volatile ("vand q6, q3, q15");
asm volatile ("vand q7, q7, q15");
}
/*
* Raw S0 lane1 is q1, so process it before q1 is overwritten
* by the high nibble of lane0.
*/
asm volatile ("vshr.u8 q3, q1, #4");
asm volatile ("vand q2, q1, q15");
asm volatile ("vand q3, q3, q15");
asm volatile ("vshr.u8 q1, q0, #4");
asm volatile ("vand q0, q0, q15");
asm volatile ("vand q1, q1, q15");
/*
* Reconstruction.
*
* q12 = multiplication table
* q13 = temporary result
* q14 = lane0 accumulator
* q15 = lane1 accumulator
*/
for (j = 0; j < nr; ++j) {
const uint8_t **t = R[j];
/* coefficient 0 initializes both accumulators */
asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[0][0]));
asm volatile ("vtbl.8 d28, {d24-d25}, d0");
asm volatile ("vtbl.8 d29, {d24-d25}, d1");
asm volatile ("vtbl.8 d30, {d24-d25}, d4");
asm volatile ("vtbl.8 d31, {d24-d25}, d5");
asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[0][16]));
asm volatile ("vtbl.8 d26, {d24-d25}, d2");
asm volatile ("vtbl.8 d27, {d24-d25}, d3");
asm volatile ("veor q14, q14, q13");
asm volatile ("vtbl.8 d26, {d24-d25}, d6");
asm volatile ("vtbl.8 d27, {d24-d25}, d7");
asm volatile ("veor q15, q15, q13");
if (nr >= 2) {
asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[1][0]));
asm volatile ("vtbl.8 d26, {d24-d25}, d8");
asm volatile ("vtbl.8 d27, {d24-d25}, d9");
asm volatile ("veor q14, q14, q13");
asm volatile ("vtbl.8 d26, {d24-d25}, d12");
asm volatile ("vtbl.8 d27, {d24-d25}, d13");
asm volatile ("veor q15, q15, q13");
asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[1][16]));
asm volatile ("vtbl.8 d26, {d24-d25}, d10");
asm volatile ("vtbl.8 d27, {d24-d25}, d11");
asm volatile ("veor q14, q14, q13");
asm volatile ("vtbl.8 d26, {d24-d25}, d14");
asm volatile ("vtbl.8 d27, {d24-d25}, d15");
asm volatile ("veor q15, q15, q13");
}
if (nr >= 3) {
asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[2][0]));
asm volatile ("vtbl.8 d26, {d24-d25}, d16");
asm volatile ("vtbl.8 d27, {d24-d25}, d17");
asm volatile ("veor q14, q14, q13");
asm volatile ("vtbl.8 d26, {d24-d25}, d20");
asm volatile ("vtbl.8 d27, {d24-d25}, d21");
asm volatile ("veor q15, q15, q13");
asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[2][16]));
asm volatile ("vtbl.8 d26, {d24-d25}, d18");
asm volatile ("vtbl.8 d27, {d24-d25}, d19");
asm volatile ("veor q14, q14, q13");
asm volatile ("vtbl.8 d26, {d24-d25}, d22");
asm volatile ("vtbl.8 d27, {d24-d25}, d23");
asm volatile ("veor q15, q15, q13");
}
asm volatile ("vst1.8 {q14}, %0" : "=Q" (pa[j][i]));
asm volatile ("vst1.8 {q15}, %0" : "=Q" (pa[j][i + 16]));
}
}
raid_neon32_end();
}
/*
* Recover multiple data failures using selected parity blocks with AArch32 NEON.
*
* This avoids temporary syndrome buffers and the
* generation of unused parity rows.
*
* If P is available, syndrome 0 is accumulated with XOR without table lookups.
*/
static __always_inline void raid_recX_neon32(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p[RAID_PARITY_MAX];
uint8_t *pa[RAID_PARITY_MAX];
uint8_t *src[RAID_DATA_MAX];
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX];
const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
size_t i;
int d, j, k, s;
int ns;
BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
for (j = 0; j < nr; ++j)
for (k = 0; k < nr; ++k)
G[j * nr + k] = A(ip[j], id[k]);
raid_invert(G, V, nr);
for (j = 0; j < nr; ++j) {
p[j] = v[nd + ip[j]];
pa[j] = v[id[j]];
}
ns = 0;
k = 0;
for (d = 0; d < nd; ++d) {
if (k < nr && d == id[k]) {
++k;
continue;
}
src[ns] = v[d];
for (j = 0; j < nr; ++j)
S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0];
++ns;
}
BUG_ON(k != nr);
BUG_ON(ns != nd - nr);
for (j = 0; j < nr; ++j)
for (k = 0; k < nr; ++k)
R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
raid_neon32_begin();
for (i = 0; i < size; i += 16) {
/*
* Q15 is the low-nibble mask during generation/splitting.
* Reconstruction later reuses q15 as a multiplication temporary.
*/
asm volatile ("vmov.i8 q15, #0x0f");
/*
* Raw syndrome registers:
*
* q0 = syndrome 0
* q1 = syndrome 1
* q2 = syndrome 2
* q3 = syndrome 3
* q4 = syndrome 4
* q5 = syndrome 5
*/
asm volatile ("vld1.8 {q0}, %0" : : "Q" (p[0][i]));
if (nr >= 2)
asm volatile ("vld1.8 {q1}, %0" : : "Q" (p[1][i]));
if (nr >= 3)
asm volatile ("vld1.8 {q2}, %0" : : "Q" (p[2][i]));
if (nr >= 4)
asm volatile ("vld1.8 {q3}, %0" : : "Q" (p[3][i]));
if (nr >= 5)
asm volatile ("vld1.8 {q4}, %0" : : "Q" (p[4][i]));
if (nr >= 6)
asm volatile ("vld1.8 {q5}, %0" : : "Q" (p[5][i]));
/*
* During survivor generation:
*
* q6 = source low
* q7 = source high
* q8 = low table
* q9 = high table
* q10 = low result
* q11 = high result
*/
for (s = 0; s < ns; ++s) {
const uint8_t **t = S[s];
asm volatile ("vld1.8 {q6}, %0" : : "Q" (src[s][i]));
if (has_p) {
asm volatile ("veor q0, q0, q6");
asm volatile ("vshr.u8 q7, q6, #4");
asm volatile ("vand q6, q6, q15");
asm volatile ("vand q7, q7, q15");
} else {
asm volatile ("vshr.u8 q7, q6, #4");
asm volatile ("vand q6, q6, q15");
asm volatile ("vand q7, q7, q15");
asm volatile ("vld1.8 {q8}, %0" : : "Q" (t[0][0]));
asm volatile ("vld1.8 {q9}, %0" : : "Q" (t[0][16]));
asm volatile ("vtbl.8 d20, {d16-d17}, d12");
asm volatile ("vtbl.8 d21, {d16-d17}, d13");
asm volatile ("vtbl.8 d22, {d18-d19}, d14");
asm volatile ("vtbl.8 d23, {d18-d19}, d15");
asm volatile ("veor q10, q10, q11");
asm volatile ("veor q0, q0, q10");
}
if (nr >= 2) {
asm volatile ("vld1.8 {q8}, %0" : : "Q" (t[1][0]));
asm volatile ("vld1.8 {q9}, %0" : : "Q" (t[1][16]));
asm volatile ("vtbl.8 d20, {d16-d17}, d12");
asm volatile ("vtbl.8 d21, {d16-d17}, d13");
asm volatile ("vtbl.8 d22, {d18-d19}, d14");
asm volatile ("vtbl.8 d23, {d18-d19}, d15");
asm volatile ("veor q10, q10, q11");
asm volatile ("veor q1, q1, q10");
}
if (nr >= 3) {
asm volatile ("vld1.8 {q8}, %0" : : "Q" (t[2][0]));
asm volatile ("vld1.8 {q9}, %0" : : "Q" (t[2][16]));
asm volatile ("vtbl.8 d20, {d16-d17}, d12");
asm volatile ("vtbl.8 d21, {d16-d17}, d13");
asm volatile ("vtbl.8 d22, {d18-d19}, d14");
asm volatile ("vtbl.8 d23, {d18-d19}, d15");
asm volatile ("veor q10, q10, q11");
asm volatile ("veor q2, q2, q10");
}
if (nr >= 4) {
asm volatile ("vld1.8 {q8}, %0" : : "Q" (t[3][0]));
asm volatile ("vld1.8 {q9}, %0" : : "Q" (t[3][16]));
asm volatile ("vtbl.8 d20, {d16-d17}, d12");
asm volatile ("vtbl.8 d21, {d16-d17}, d13");
asm volatile ("vtbl.8 d22, {d18-d19}, d14");
asm volatile ("vtbl.8 d23, {d18-d19}, d15");
asm volatile ("veor q10, q10, q11");
asm volatile ("veor q3, q3, q10");
}
if (nr >= 5) {
asm volatile ("vld1.8 {q8}, %0" : : "Q" (t[4][0]));
asm volatile ("vld1.8 {q9}, %0" : : "Q" (t[4][16]));
asm volatile ("vtbl.8 d20, {d16-d17}, d12");
asm volatile ("vtbl.8 d21, {d16-d17}, d13");
asm volatile ("vtbl.8 d22, {d18-d19}, d14");
asm volatile ("vtbl.8 d23, {d18-d19}, d15");
asm volatile ("veor q10, q10, q11");
asm volatile ("veor q4, q4, q10");
}
if (nr >= 6) {
asm volatile ("vld1.8 {q8}, %0" : : "Q" (t[5][0]));
asm volatile ("vld1.8 {q9}, %0" : : "Q" (t[5][16]));
asm volatile ("vtbl.8 d20, {d16-d17}, d12");
asm volatile ("vtbl.8 d21, {d16-d17}, d13");
asm volatile ("vtbl.8 d22, {d18-d19}, d14");
asm volatile ("vtbl.8 d23, {d18-d19}, d15");
asm volatile ("veor q10, q10, q11");
asm volatile ("veor q5, q5, q10");
}
}
/*
* Expand raw syndromes backwards.
*
* Final layout:
*
* q0/q1 syndrome 0 low/high
* q2/q3 syndrome 1 low/high
* q4/q5 syndrome 2 low/high
* q6/q7 syndrome 3 low/high
* q8/q9 syndrome 4 low/high
* q10/q11 syndrome 5 low/high
*/
if (nr >= 6) {
asm volatile ("vshr.u8 q11, q5, #4");
asm volatile ("vand q10, q5, q15");
asm volatile ("vand q11, q11, q15");
}
if (nr >= 5) {
asm volatile ("vshr.u8 q9, q4, #4");
asm volatile ("vand q8, q4, q15");
asm volatile ("vand q9, q9, q15");
}
if (nr >= 4) {
asm volatile ("vshr.u8 q7, q3, #4");
asm volatile ("vand q6, q3, q15");
asm volatile ("vand q7, q7, q15");
}
if (nr >= 3) {
asm volatile ("vshr.u8 q5, q2, #4");
asm volatile ("vand q4, q2, q15");
asm volatile ("vand q5, q5, q15");
}
if (nr >= 2) {
asm volatile ("vshr.u8 q3, q1, #4");
asm volatile ("vand q2, q1, q15");
asm volatile ("vand q3, q3, q15");
}
asm volatile ("vshr.u8 q1, q0, #4");
asm volatile ("vand q0, q0, q15");
asm volatile ("vand q1, q1, q15");
/*
* Reconstruction:
*
* q12 = low table
* q13 = high table
* q14 = output accumulator
* q15 = multiplication temporary
*/
for (j = 0; j < nr; ++j) {
const uint8_t **t = R[j];
/* coefficient 0 initializes q14 */
asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[0][0]));
asm volatile ("vld1.8 {q13}, %0" : : "Q" (t[0][16]));
asm volatile ("vtbl.8 d28, {d24-d25}, d0");
asm volatile ("vtbl.8 d29, {d24-d25}, d1");
asm volatile ("vtbl.8 d30, {d26-d27}, d2");
asm volatile ("vtbl.8 d31, {d26-d27}, d3");
asm volatile ("veor q14, q14, q15");
if (nr >= 2) {
asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[1][0]));
asm volatile ("vld1.8 {q13}, %0" : : "Q" (t[1][16]));
asm volatile ("vtbl.8 d30, {d24-d25}, d4");
asm volatile ("vtbl.8 d31, {d24-d25}, d5");
asm volatile ("veor q14, q14, q15");
asm volatile ("vtbl.8 d30, {d26-d27}, d6");
asm volatile ("vtbl.8 d31, {d26-d27}, d7");
asm volatile ("veor q14, q14, q15");
}
if (nr >= 3) {
asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[2][0]));
asm volatile ("vld1.8 {q13}, %0" : : "Q" (t[2][16]));
asm volatile ("vtbl.8 d30, {d24-d25}, d8");
asm volatile ("vtbl.8 d31, {d24-d25}, d9");
asm volatile ("veor q14, q14, q15");
asm volatile ("vtbl.8 d30, {d26-d27}, d10");
asm volatile ("vtbl.8 d31, {d26-d27}, d11");
asm volatile ("veor q14, q14, q15");
}
if (nr >= 4) {
asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[3][0]));
asm volatile ("vld1.8 {q13}, %0" : : "Q" (t[3][16]));
asm volatile ("vtbl.8 d30, {d24-d25}, d12");
asm volatile ("vtbl.8 d31, {d24-d25}, d13");
asm volatile ("veor q14, q14, q15");
asm volatile ("vtbl.8 d30, {d26-d27}, d14");
asm volatile ("vtbl.8 d31, {d26-d27}, d15");
asm volatile ("veor q14, q14, q15");
}
if (nr >= 5) {
asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[4][0]));
asm volatile ("vld1.8 {q13}, %0" : : "Q" (t[4][16]));
asm volatile ("vtbl.8 d30, {d24-d25}, d16");
asm volatile ("vtbl.8 d31, {d24-d25}, d17");
asm volatile ("veor q14, q14, q15");
asm volatile ("vtbl.8 d30, {d26-d27}, d18");
asm volatile ("vtbl.8 d31, {d26-d27}, d19");
asm volatile ("veor q14, q14, q15");
}
if (nr >= 6) {
asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[5][0]));
asm volatile ("vld1.8 {q13}, %0" : : "Q" (t[5][16]));
asm volatile ("vtbl.8 d30, {d24-d25}, d20");
asm volatile ("vtbl.8 d31, {d24-d25}, d21");
asm volatile ("veor q14, q14, q15");
asm volatile ("vtbl.8 d30, {d26-d27}, d22");
asm volatile ("vtbl.8 d31, {d26-d27}, d23");
asm volatile ("veor q14, q14, q15");
}
asm volatile ("vst1.8 {q14}, %0" : "=Q" (pa[j][i]));
}
}
raid_neon32_end();
}
void raid_gen1_neon32(int nd, size_t size, void **vv, int streaming)
{
(void)streaming;
raid_gen1_neon32_gen(nd, size, vv);
}
void raid_gen2_neon32_raid(int nd, size_t size, void **vv, int streaming)
{
(void)streaming;
raid_gen2_neon32_gen(nd, size, vv, 2);
}
void raid_gen2_neon32_aes(int nd, size_t size, void **vv, int streaming)
{
(void)streaming;
raid_gen2_neon32_gen(nd, size, vv, 3);
}
void raid_genz_neon32_raid(int nd, size_t size, void **vv, int streaming)
{
(void)streaming;
raid_genz_neon32_gen(nd, size, vv);
}
void raid_gen3_neon32_raid(int nd, size_t size, void **vv, int streaming)
{
(void)streaming;
raid_genX_neon32(nd, size, vv, 3, 2);
}
void raid_gen3_neon32_aes(int nd, size_t size, void **vv, int streaming)
{
(void)streaming;
raid_genX_neon32(nd, size, vv, 3, 3);
}
void raid_gen4_neon32_raid(int nd, size_t size, void **vv, int streaming)
{
(void)streaming;
raid_genX_neon32(nd, size, vv, 4, 2);
}
void raid_gen4_neon32_aes(int nd, size_t size, void **vv, int streaming)
{
(void)streaming;
raid_genX_neon32(nd, size, vv, 4, 3);
}
void raid_gen5_neon32_raid(int nd, size_t size, void **vv, int streaming)
{
(void)streaming;
raid_genX_neon32(nd, size, vv, 5, 2);
}
void raid_gen5_neon32_aes(int nd, size_t size, void **vv, int streaming)
{
(void)streaming;
raid_genX_neon32(nd, size, vv, 5, 3);
}
void raid_gen6_neon32_raid(int nd, size_t size, void **vv, int streaming)
{
(void)streaming;
raid_genX_neon32(nd, size, vv, 6, 2);
}
void raid_gen6_neon32_aes(int nd, size_t size, void **vv, int streaming)
{
(void)streaming;
raid_genX_neon32(nd, size, vv, 6, 3);
}
void raid_rec1_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 1);
/* if recovering with P uses the delta function */
if (ip[0] == 0) {
raid_rec1of1(id, nd, size, vv);
return;
}
raid_recX_neon32_123(1, 0, id, ip, nd, size, vv);
}
void raid_rec2_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 2);
if (ip[0] == 0)
raid_recX_neon32_123(2, 1, id, ip, nd, size, vv);
else
raid_recX_neon32_123(2, 0, id, ip, nd, size, vv);
}
void raid_rec3_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 3);
if (ip[0] == 0)
raid_recX_neon32_123(3, 1, id, ip, nd, size, vv);
else
raid_recX_neon32_123(3, 0, id, ip, nd, size, vv);
}
void raid_rec4_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 4);
if (ip[0] == 0)
raid_recX_neon32(4, 1, id, ip, nd, size, vv);
else
raid_recX_neon32(4, 0, id, ip, nd, size, vv);
}
void raid_rec5_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 5);
if (ip[0] == 0)
raid_recX_neon32(5, 1, id, ip, nd, size, vv);
else
raid_recX_neon32(5, 0, id, ip, nd, size, vv);
}
void raid_rec6_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 6);
if (ip[0] == 0)
raid_recX_neon32(6, 1, id, ip, nd, size, vv);
else
raid_recX_neon32(6, 0, id, ip, nd, size, vv);
}
void raid_register_neon32(void)
{
raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "neon32", raid_gen1_neon32, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "neon32", raid_gen2_neon32_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "neon32", raid_gen2_neon32_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "neon32", raid_genz_neon32_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "neon32", raid_gen3_neon32_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "neon32", raid_gen3_neon32_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "neon32", raid_gen4_neon32_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "neon32", raid_gen4_neon32_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "neon32", raid_gen5_neon32_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "neon32", raid_gen5_neon32_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "neon32", raid_gen6_neon32_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "neon32", raid_gen6_neon32_aes, RAID_POLY_AES);
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "neon32", raid_rec1_neon32, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "neon32", raid_rec2_neon32, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "neon32", raid_rec3_neon32, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "neon32", raid_rec4_neon32, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "neon32", raid_rec5_neon32, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "neon32", raid_rec6_neon32, RAID_POLY_ANY);
}
#endif
Reference in a new issue View git blame Copy permalink