mirror of
https://github.com/amadvance/snapraid.git
synced 2026-10-26 02:49:02 -04:00
b070cbb894
Some checks are pending
Build / build-mac (push) Waiting to run
Build / build-linux (push) Waiting to run
Build / build-arm32 (push) Waiting to run
Build / build-win (push) Waiting to run
Build / build-freebsd (push) Waiting to run
Build / build-musl (push) Waiting to run
CodeQL Advanced / Analyze (c-cpp) (push) Waiting to run
CodeQL Advanced / Analyze (actions) (push) Waiting to run
1321 lines
32 KiB
C
1321 lines
32 KiB
C
// SPDX-License-Identifier: GPL-2.0-or-later
|
|
// Copyright (C) 2026 Andrea Mazzoleni
|
|
|
|
#include "internal.h"
|
|
#include "gf.h"
|
|
#include "cpu.h"
|
|
|
|
#ifdef CONFIG_NEON32
|
|
/*
|
|
* Generate one parity block (RAID5 with XOR) using AArch32 NEON implementation.
|
|
*
|
|
* Uses 64-byte chunks across four 16-byte vectors.
|
|
*/
|
|
static __always_inline void raid_gen1_neon32_gen(int nd, size_t size, void **vv)
|
|
{
|
|
uint8_t **v = (uint8_t **)vv;
|
|
uint8_t *p;
|
|
int d, l;
|
|
size_t i;
|
|
|
|
l = nd - 1;
|
|
p = v[nd];
|
|
|
|
raid_neon32_begin();
|
|
|
|
for (i = 0; i < size; i += 64) {
|
|
asm volatile (
|
|
"vld1.8 {q0}, %0\n"
|
|
"vld1.8 {q1}, %1\n"
|
|
"vld1.8 {q2}, %2\n"
|
|
"vld1.8 {q3}, %3\n"
|
|
:
|
|
: "Q" (v[0][i]), "Q" (v[0][i + 16]),
|
|
"Q" (v[0][i + 32]), "Q" (v[0][i + 48])
|
|
);
|
|
|
|
for (d = 1; d <= l; ++d) {
|
|
asm volatile (
|
|
"vld1.8 {q4}, %0\n"
|
|
"vld1.8 {q5}, %1\n"
|
|
"vld1.8 {q6}, %2\n"
|
|
"vld1.8 {q7}, %3\n"
|
|
"veor q0, q0, q4\n"
|
|
"veor q1, q1, q5\n"
|
|
"veor q2, q2, q6\n"
|
|
"veor q3, q3, q7\n"
|
|
:
|
|
: "Q" (v[d][i]), "Q" (v[d][i + 16]),
|
|
"Q" (v[d][i + 32]), "Q" (v[d][i + 48])
|
|
);
|
|
}
|
|
|
|
asm volatile (
|
|
"vst1.8 {q0}, %0\n"
|
|
"vst1.8 {q1}, %1\n"
|
|
"vst1.8 {q2}, %2\n"
|
|
"vst1.8 {q3}, %3\n"
|
|
: "=Q" (p[i]), "=Q" (p[i + 16]),
|
|
"=Q" (p[i + 32]), "=Q" (p[i + 48])
|
|
);
|
|
}
|
|
|
|
raid_neon32_end();
|
|
}
|
|
|
|
/*
|
|
* Generate two parity blocks (RAID6 with Cauchy matrix) using AArch32 NEON implementation.
|
|
*/
|
|
static __always_inline void raid_gen2_neon32_gen(int nd, size_t size,
|
|
void **vv, int generator)
|
|
{
|
|
uint8_t **v = (uint8_t **)vv;
|
|
uint8_t *p;
|
|
uint8_t *q;
|
|
int d, l;
|
|
size_t i;
|
|
|
|
l = nd - 1;
|
|
p = v[nd];
|
|
q = v[nd + 1];
|
|
|
|
raid_neon32_begin();
|
|
|
|
/* q14 contains the active reduction polynomial */
|
|
asm volatile (
|
|
"vld1.8 {q14}, %0\n"
|
|
:
|
|
: "Q" (gfconst16.poly)
|
|
);
|
|
|
|
for (i = 0; i < size; i += 32) {
|
|
asm volatile (
|
|
"vld1.8 {q0}, %0\n"
|
|
"vld1.8 {q1}, %1\n"
|
|
"vmov q2, q0\n"
|
|
"vmov q3, q1\n"
|
|
:
|
|
: "Q" (v[l][i]), "Q" (v[l][i + 16])
|
|
);
|
|
|
|
for (d = l - 1; d >= 0; --d) {
|
|
if (generator == 3) {
|
|
asm volatile (
|
|
"vmov q10, q2\n"
|
|
"vmov q11, q3\n"
|
|
);
|
|
}
|
|
asm volatile (
|
|
"vshr.s8 q8, q2, #7\n"
|
|
"vshl.i8 q2, q2, #1\n"
|
|
"vand q8, q8, q14\n"
|
|
"veor q2, q2, q8\n"
|
|
"vshr.s8 q8, q3, #7\n"
|
|
"vshl.i8 q3, q3, #1\n"
|
|
"vand q8, q8, q14\n"
|
|
"veor q3, q3, q8\n"
|
|
"vld1.8 {q12}, %0\n"
|
|
"vld1.8 {q13}, %1\n"
|
|
"veor q0, q0, q12\n"
|
|
"veor q1, q1, q13\n"
|
|
"veor q2, q2, q12\n"
|
|
"veor q3, q3, q13\n"
|
|
:
|
|
: "Q" (v[d][i]), "Q" (v[d][i + 16])
|
|
);
|
|
if (generator == 3) {
|
|
asm volatile (
|
|
"veor q2, q2, q10\n"
|
|
"veor q3, q3, q11\n"
|
|
);
|
|
}
|
|
}
|
|
|
|
asm volatile (
|
|
"vst1.8 {q0}, %0\n"
|
|
"vst1.8 {q1}, %1\n"
|
|
"vst1.8 {q2}, %2\n"
|
|
"vst1.8 {q3}, %3\n"
|
|
: "=Q" (p[i]), "=Q" (p[i + 16]),
|
|
"=Q" (q[i]), "=Q" (q[i + 16])
|
|
);
|
|
}
|
|
|
|
raid_neon32_end();
|
|
}
|
|
|
|
/*
|
|
* Generate three parity blocks with powers of 2^-1 using AArch32 NEON implementation.
|
|
*/
|
|
static __always_inline void raid_genz_neon32_gen(int nd, size_t size, void **vv)
|
|
{
|
|
uint8_t **v = (uint8_t **)vv;
|
|
uint8_t *p;
|
|
uint8_t *q;
|
|
uint8_t *r;
|
|
int d, l;
|
|
size_t i;
|
|
|
|
l = nd - 1;
|
|
p = v[nd];
|
|
q = v[nd + 1];
|
|
r = v[nd + 2];
|
|
|
|
raid_neon32_begin();
|
|
|
|
asm volatile (
|
|
"vld1.8 {q14}, %0\n"
|
|
"vld1.8 {q15}, %1\n"
|
|
:
|
|
: "Q" (gfconst16.poly), "Q" (gfconst16.half)
|
|
);
|
|
|
|
for (i = 0; i < size; i += 32) {
|
|
asm volatile (
|
|
"vld1.8 {q0}, %0\n"
|
|
"vld1.8 {q1}, %1\n"
|
|
"vmov q2, q0\n"
|
|
"vmov q3, q1\n"
|
|
"vmov q4, q0\n"
|
|
"vmov q5, q1\n"
|
|
:
|
|
: "Q" (v[l][i]), "Q" (v[l][i + 16])
|
|
);
|
|
|
|
for (d = l - 1; d >= 0; --d) {
|
|
asm volatile (
|
|
"vshr.s8 q8, q2, #7\n"
|
|
"vshl.i8 q2, q2, #1\n"
|
|
"vand q8, q8, q14\n"
|
|
"veor q2, q2, q8\n"
|
|
"vshr.s8 q8, q3, #7\n"
|
|
"vshl.i8 q3, q3, #1\n"
|
|
"vand q8, q8, q14\n"
|
|
"veor q3, q3, q8\n"
|
|
"vshl.i8 q9, q4, #7\n"
|
|
"vshr.s8 q9, q9, #7\n"
|
|
"vshr.u8 q4, q4, #1\n"
|
|
"vand q9, q9, q15\n"
|
|
"veor q4, q4, q9\n"
|
|
"vshl.i8 q9, q5, #7\n"
|
|
"vshr.s8 q9, q9, #7\n"
|
|
"vshr.u8 q5, q5, #1\n"
|
|
"vand q9, q9, q15\n"
|
|
"veor q5, q5, q9\n"
|
|
"vld1.8 {q12}, %0\n"
|
|
"vld1.8 {q13}, %1\n"
|
|
"veor q0, q0, q12\n"
|
|
"veor q1, q1, q13\n"
|
|
"veor q2, q2, q12\n"
|
|
"veor q3, q3, q13\n"
|
|
"veor q4, q4, q12\n"
|
|
"veor q5, q5, q13\n"
|
|
:
|
|
: "Q" (v[d][i]), "Q" (v[d][i + 16])
|
|
);
|
|
}
|
|
|
|
asm volatile (
|
|
"vst1.8 {q0}, %0\n"
|
|
"vst1.8 {q1}, %1\n"
|
|
"vst1.8 {q2}, %2\n"
|
|
"vst1.8 {q3}, %3\n"
|
|
"vst1.8 {q4}, %4\n"
|
|
"vst1.8 {q5}, %5\n"
|
|
: "=Q" (p[i]), "=Q" (p[i + 16]),
|
|
"=Q" (q[i]), "=Q" (q[i + 16]),
|
|
"=Q" (r[i]), "=Q" (r[i + 16])
|
|
);
|
|
}
|
|
|
|
raid_neon32_end();
|
|
}
|
|
|
|
/*
|
|
* Generate N parity blocks with Cauchy matrix using AArch32 NEON implementation.
|
|
*/
|
|
static __always_inline void raid_genX_neon32(int nd, size_t size,
|
|
void **vv, int np, int generator)
|
|
{
|
|
uint8_t **v = (uint8_t **)vv;
|
|
size_t i;
|
|
int d, l;
|
|
|
|
l = nd - 1;
|
|
|
|
/* special case with only one data disk */
|
|
if (l == 0) {
|
|
for (d = 0; d < np; ++d)
|
|
if (v[1 + d] != v[0])
|
|
memcpy(v[1 + d], v[0], size);
|
|
return;
|
|
}
|
|
|
|
raid_neon32_begin();
|
|
|
|
/* generic case with at least two data disks */
|
|
asm volatile (
|
|
"vld1.8 {q14}, %0\n"
|
|
"vmov.i8 q15, #0x0f\n"
|
|
:
|
|
: "Q" (gfconst16.poly)
|
|
);
|
|
|
|
for (i = 0; i < size; i += 16) {
|
|
/* last disk without the generator multiplication */
|
|
asm volatile (
|
|
"vld1.8 {q0}, %0\n"
|
|
"vld1.8 {q1}, %0\n"
|
|
:
|
|
: "Q" (v[l][i])
|
|
);
|
|
|
|
if (np >= 3) {
|
|
asm volatile (
|
|
"vshr.u8 q8, q0, #4\n"
|
|
"vand q7, q0, q15\n"
|
|
"vand q8, q8, q15\n"
|
|
);
|
|
|
|
asm volatile (
|
|
"vld1.8 {q9}, %0\n"
|
|
"vld1.8 {q10}, %1\n"
|
|
|
|
"vtbl.8 d22, {d18-d19}, d14\n"
|
|
"vtbl.8 d23, {d18-d19}, d15\n"
|
|
"vtbl.8 d24, {d20-d21}, d16\n"
|
|
"vtbl.8 d25, {d20-d21}, d17\n"
|
|
|
|
"veor q2, q11, q12\n"
|
|
:
|
|
: "Q" (raid_gfcauchypshufb[l][1][0]),
|
|
"Q" (raid_gfcauchypshufb[l][1][1])
|
|
);
|
|
}
|
|
|
|
if (np >= 4) {
|
|
asm volatile (
|
|
"vld1.8 {q9}, %0\n"
|
|
"vld1.8 {q10}, %1\n"
|
|
|
|
"vtbl.8 d22, {d18-d19}, d14\n"
|
|
"vtbl.8 d23, {d18-d19}, d15\n"
|
|
"vtbl.8 d24, {d20-d21}, d16\n"
|
|
"vtbl.8 d25, {d20-d21}, d17\n"
|
|
|
|
"veor q3, q11, q12\n"
|
|
:
|
|
: "Q" (raid_gfcauchypshufb[l][2][0]),
|
|
"Q" (raid_gfcauchypshufb[l][2][1])
|
|
);
|
|
}
|
|
|
|
if (np >= 5) {
|
|
asm volatile (
|
|
"vld1.8 {q9}, %0\n"
|
|
"vld1.8 {q10}, %1\n"
|
|
|
|
"vtbl.8 d22, {d18-d19}, d14\n"
|
|
"vtbl.8 d23, {d18-d19}, d15\n"
|
|
"vtbl.8 d24, {d20-d21}, d16\n"
|
|
"vtbl.8 d25, {d20-d21}, d17\n"
|
|
|
|
"veor q4, q11, q12\n"
|
|
:
|
|
: "Q" (raid_gfcauchypshufb[l][3][0]),
|
|
"Q" (raid_gfcauchypshufb[l][3][1])
|
|
);
|
|
}
|
|
|
|
if (np >= 6) {
|
|
asm volatile (
|
|
"vld1.8 {q9}, %0\n"
|
|
"vld1.8 {q10}, %1\n"
|
|
|
|
"vtbl.8 d22, {d18-d19}, d14\n"
|
|
"vtbl.8 d23, {d18-d19}, d15\n"
|
|
"vtbl.8 d24, {d20-d21}, d16\n"
|
|
"vtbl.8 d25, {d20-d21}, d17\n"
|
|
|
|
"veor q5, q11, q12\n"
|
|
:
|
|
: "Q" (raid_gfcauchypshufb[l][4][0]),
|
|
"Q" (raid_gfcauchypshufb[l][4][1])
|
|
);
|
|
}
|
|
|
|
/* intermediate disks */
|
|
for (d = l - 1; d > 0; --d) {
|
|
if (generator == 3) {
|
|
asm volatile (
|
|
"vmov q13, q1\n"
|
|
);
|
|
}
|
|
asm volatile (
|
|
"vshr.s8 q11, q1, #7\n"
|
|
"vshl.i8 q1, q1, #1\n"
|
|
"vand q11, q11, q14\n"
|
|
"veor q1, q1, q11\n"
|
|
"vld1.8 {q6}, %0\n"
|
|
"veor q0, q0, q6\n"
|
|
"veor q1, q1, q6\n"
|
|
:
|
|
: "Q" (v[d][i])
|
|
);
|
|
if (generator == 3) {
|
|
asm volatile (
|
|
"veor q1, q1, q13\n"
|
|
);
|
|
}
|
|
|
|
if (np >= 3) {
|
|
asm volatile (
|
|
"vshr.u8 q8, q6, #4\n"
|
|
"vand q7, q6, q15\n"
|
|
"vand q8, q8, q15\n"
|
|
);
|
|
asm volatile (
|
|
"vld1.8 {q9}, %0\n"
|
|
"vld1.8 {q10}, %1\n"
|
|
|
|
"vtbl.8 d22, {d18-d19}, d14\n"
|
|
"vtbl.8 d23, {d18-d19}, d15\n"
|
|
"vtbl.8 d24, {d20-d21}, d16\n"
|
|
"vtbl.8 d25, {d20-d21}, d17\n"
|
|
|
|
"veor q11, q11, q12\n"
|
|
"veor q2, q2, q11\n"
|
|
:
|
|
: "Q" (raid_gfcauchypshufb[d][1][0]),
|
|
"Q" (raid_gfcauchypshufb[d][1][1])
|
|
);
|
|
}
|
|
|
|
if (np >= 4) {
|
|
asm volatile (
|
|
"vld1.8 {q9}, %0\n"
|
|
"vld1.8 {q10}, %1\n"
|
|
"vtbl.8 d22, {d18-d19}, d14\n"
|
|
"vtbl.8 d23, {d18-d19}, d15\n"
|
|
"vtbl.8 d24, {d20-d21}, d16\n"
|
|
"vtbl.8 d25, {d20-d21}, d17\n"
|
|
"veor q11, q11, q12\n"
|
|
"veor q3, q3, q11\n"
|
|
:
|
|
: "Q" (raid_gfcauchypshufb[d][2][0]),
|
|
"Q" (raid_gfcauchypshufb[d][2][1])
|
|
);
|
|
}
|
|
|
|
if (np >= 5) {
|
|
asm volatile (
|
|
"vld1.8 {q9}, %0\n"
|
|
"vld1.8 {q10}, %1\n"
|
|
"vtbl.8 d22, {d18-d19}, d14\n"
|
|
"vtbl.8 d23, {d18-d19}, d15\n"
|
|
"vtbl.8 d24, {d20-d21}, d16\n"
|
|
"vtbl.8 d25, {d20-d21}, d17\n"
|
|
"veor q11, q11, q12\n"
|
|
"veor q4, q4, q11\n"
|
|
:
|
|
: "Q" (raid_gfcauchypshufb[d][3][0]),
|
|
"Q" (raid_gfcauchypshufb[d][3][1])
|
|
);
|
|
}
|
|
|
|
if (np >= 6) {
|
|
asm volatile (
|
|
"vld1.8 {q9}, %0\n"
|
|
"vld1.8 {q10}, %1\n"
|
|
"vtbl.8 d22, {d18-d19}, d14\n"
|
|
"vtbl.8 d23, {d18-d19}, d15\n"
|
|
"vtbl.8 d24, {d20-d21}, d16\n"
|
|
"vtbl.8 d25, {d20-d21}, d17\n"
|
|
"veor q11, q11, q12\n"
|
|
"veor q5, q5, q11\n"
|
|
:
|
|
: "Q" (raid_gfcauchypshufb[d][4][0]),
|
|
"Q" (raid_gfcauchypshufb[d][4][1])
|
|
);
|
|
}
|
|
}
|
|
|
|
/* first disk with all coefficients at 1 */
|
|
if (generator == 3) {
|
|
asm volatile (
|
|
"vmov q13, q1\n"
|
|
);
|
|
}
|
|
asm volatile (
|
|
"vshr.s8 q11, q1, #7\n"
|
|
"vshl.i8 q1, q1, #1\n"
|
|
"vand q11, q11, q14\n"
|
|
"veor q1, q1, q11\n"
|
|
"vld1.8 {q6}, %0\n"
|
|
"veor q0, q0, q6\n"
|
|
"veor q1, q1, q6\n"
|
|
:
|
|
: "Q" (v[0][i])
|
|
);
|
|
if (generator == 3) {
|
|
asm volatile (
|
|
"veor q1, q1, q13\n"
|
|
);
|
|
}
|
|
|
|
if (np >= 3) {
|
|
asm volatile (
|
|
"veor q2, q2, q6\n"
|
|
);
|
|
}
|
|
if (np >= 4) {
|
|
asm volatile (
|
|
"veor q3, q3, q6\n"
|
|
);
|
|
}
|
|
if (np >= 5) {
|
|
asm volatile (
|
|
"veor q4, q4, q6\n"
|
|
);
|
|
}
|
|
if (np >= 6) {
|
|
asm volatile (
|
|
"veor q5, q5, q6\n"
|
|
);
|
|
}
|
|
|
|
/* write parity in increasing order */
|
|
asm volatile (
|
|
"vst1.8 {q0}, %0\n"
|
|
"vst1.8 {q1}, %1\n"
|
|
: "=Q" (v[nd][i]),
|
|
"=Q" (v[nd + 1][i])
|
|
);
|
|
|
|
if (np >= 3) {
|
|
asm volatile (
|
|
"vst1.8 {q2}, %0\n"
|
|
: "=Q" (v[nd + 2][i])
|
|
);
|
|
}
|
|
if (np >= 4) {
|
|
asm volatile (
|
|
"vst1.8 {q3}, %0\n"
|
|
: "=Q" (v[nd + 3][i])
|
|
);
|
|
}
|
|
if (np >= 5) {
|
|
asm volatile (
|
|
"vst1.8 {q4}, %0\n"
|
|
: "=Q" (v[nd + 4][i])
|
|
);
|
|
}
|
|
if (np >= 6) {
|
|
asm volatile (
|
|
"vst1.8 {q5}, %0\n"
|
|
: "=Q" (v[nd + 5][i])
|
|
);
|
|
}
|
|
}
|
|
|
|
raid_neon32_end();
|
|
}
|
|
|
|
/*
|
|
* Recover multiple data failures using selected parity blocks with AArch32 NEON optimized for up to three failures.
|
|
*
|
|
* If P is available, syndrome 0 is accumulated with XOR without table lookups.
|
|
*/
|
|
static __always_inline void raid_recX_neon32_123(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
uint8_t **v = (uint8_t **)vv;
|
|
uint8_t *p[RAID_PARITY_MAX];
|
|
uint8_t *pa[RAID_PARITY_MAX];
|
|
uint8_t *src[RAID_DATA_MAX];
|
|
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
|
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
|
const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX];
|
|
const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
|
|
size_t i;
|
|
int d, j, k, s;
|
|
int ns;
|
|
|
|
BUG_ON(nr < 1 || nr > 3);
|
|
|
|
for (j = 0; j < nr; ++j)
|
|
for (k = 0; k < nr; ++k)
|
|
G[j * nr + k] = A(ip[j], id[k]);
|
|
|
|
raid_invert(G, V, nr);
|
|
|
|
for (j = 0; j < nr; ++j) {
|
|
p[j] = v[nd + ip[j]];
|
|
pa[j] = v[id[j]];
|
|
}
|
|
|
|
ns = 0;
|
|
k = 0;
|
|
|
|
for (d = 0; d < nd; ++d) {
|
|
if (k < nr && d == id[k]) {
|
|
++k;
|
|
continue;
|
|
}
|
|
|
|
src[ns] = v[d];
|
|
|
|
for (j = 0; j < nr; ++j)
|
|
S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0];
|
|
|
|
++ns;
|
|
}
|
|
|
|
BUG_ON(k != nr);
|
|
BUG_ON(ns != nd - nr);
|
|
|
|
for (j = 0; j < nr; ++j)
|
|
for (k = 0; k < nr; ++k)
|
|
R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
|
|
|
|
raid_neon32_begin();
|
|
|
|
for (i = 0; i < size; i += 32) {
|
|
/*
|
|
* Q15 is the low-nibble mask during generation/splitting.
|
|
* Reconstruction later reuses q15 as the second accumulator,
|
|
* therefore reload it on every iteration.
|
|
*/
|
|
asm volatile ("vmov.i8 q15, #0x0f");
|
|
|
|
/* selected stored parity */
|
|
asm volatile ("vld1.8 {q0}, %0" : : "Q" (p[0][i]));
|
|
asm volatile ("vld1.8 {q1}, %0" : : "Q" (p[0][i + 16]));
|
|
|
|
if (nr >= 2) {
|
|
asm volatile ("vld1.8 {q2}, %0" : : "Q" (p[1][i]));
|
|
asm volatile ("vld1.8 {q3}, %0" : : "Q" (p[1][i + 16]));
|
|
}
|
|
|
|
if (nr >= 3) {
|
|
asm volatile ("vld1.8 {q4}, %0" : : "Q" (p[2][i]));
|
|
asm volatile ("vld1.8 {q5}, %0" : : "Q" (p[2][i + 16]));
|
|
}
|
|
|
|
for (s = 0; s < ns; ++s) {
|
|
const uint8_t **t = S[s];
|
|
|
|
asm volatile ("vld1.8 {q6}, %0" : : "Q" (src[s][i]));
|
|
asm volatile ("vld1.8 {q7}, %0" : : "Q" (src[s][i + 16]));
|
|
|
|
if (has_p) {
|
|
asm volatile ("veor q0, q0, q6");
|
|
asm volatile ("veor q1, q1, q7");
|
|
|
|
asm volatile ("vshr.u8 q8, q6, #4");
|
|
asm volatile ("vshr.u8 q9, q7, #4");
|
|
asm volatile ("vand q6, q6, q15");
|
|
asm volatile ("vand q7, q7, q15");
|
|
asm volatile ("vand q8, q8, q15");
|
|
asm volatile ("vand q9, q9, q15");
|
|
} else {
|
|
asm volatile ("vshr.u8 q8, q6, #4");
|
|
asm volatile ("vshr.u8 q9, q7, #4");
|
|
asm volatile ("vand q6, q6, q15");
|
|
asm volatile ("vand q7, q7, q15");
|
|
asm volatile ("vand q8, q8, q15");
|
|
asm volatile ("vand q9, q9, q15");
|
|
|
|
/*
|
|
* Q10 = low table
|
|
* q11 = high table
|
|
* q12/q13 = multiplication temporaries
|
|
*/
|
|
asm volatile ("vld1.8 {q10}, %0" : : "Q" (t[0][0]));
|
|
asm volatile ("vld1.8 {q11}, %0" : : "Q" (t[0][16]));
|
|
|
|
asm volatile ("vtbl.8 d24, {d20-d21}, d12");
|
|
asm volatile ("vtbl.8 d25, {d20-d21}, d13");
|
|
asm volatile ("vtbl.8 d26, {d22-d23}, d16");
|
|
asm volatile ("vtbl.8 d27, {d22-d23}, d17");
|
|
asm volatile ("veor q12, q12, q13");
|
|
asm volatile ("veor q0, q0, q12");
|
|
|
|
asm volatile ("vtbl.8 d24, {d20-d21}, d14");
|
|
asm volatile ("vtbl.8 d25, {d20-d21}, d15");
|
|
asm volatile ("vtbl.8 d26, {d22-d23}, d18");
|
|
asm volatile ("vtbl.8 d27, {d22-d23}, d19");
|
|
asm volatile ("veor q12, q12, q13");
|
|
asm volatile ("veor q1, q1, q12");
|
|
}
|
|
|
|
if (nr >= 2) {
|
|
asm volatile ("vld1.8 {q10}, %0" : : "Q" (t[1][0]));
|
|
asm volatile ("vld1.8 {q11}, %0" : : "Q" (t[1][16]));
|
|
|
|
asm volatile ("vtbl.8 d24, {d20-d21}, d12");
|
|
asm volatile ("vtbl.8 d25, {d20-d21}, d13");
|
|
asm volatile ("vtbl.8 d26, {d22-d23}, d16");
|
|
asm volatile ("vtbl.8 d27, {d22-d23}, d17");
|
|
asm volatile ("veor q12, q12, q13");
|
|
asm volatile ("veor q2, q2, q12");
|
|
|
|
asm volatile ("vtbl.8 d24, {d20-d21}, d14");
|
|
asm volatile ("vtbl.8 d25, {d20-d21}, d15");
|
|
asm volatile ("vtbl.8 d26, {d22-d23}, d18");
|
|
asm volatile ("vtbl.8 d27, {d22-d23}, d19");
|
|
asm volatile ("veor q12, q12, q13");
|
|
asm volatile ("veor q3, q3, q12");
|
|
}
|
|
|
|
if (nr >= 3) {
|
|
asm volatile ("vld1.8 {q10}, %0" : : "Q" (t[2][0]));
|
|
asm volatile ("vld1.8 {q11}, %0" : : "Q" (t[2][16]));
|
|
|
|
asm volatile ("vtbl.8 d24, {d20-d21}, d12");
|
|
asm volatile ("vtbl.8 d25, {d20-d21}, d13");
|
|
asm volatile ("vtbl.8 d26, {d22-d23}, d16");
|
|
asm volatile ("vtbl.8 d27, {d22-d23}, d17");
|
|
asm volatile ("veor q12, q12, q13");
|
|
asm volatile ("veor q4, q4, q12");
|
|
|
|
asm volatile ("vtbl.8 d24, {d20-d21}, d14");
|
|
asm volatile ("vtbl.8 d25, {d20-d21}, d15");
|
|
asm volatile ("vtbl.8 d26, {d22-d23}, d18");
|
|
asm volatile ("vtbl.8 d27, {d22-d23}, d19");
|
|
asm volatile ("veor q12, q12, q13");
|
|
asm volatile ("veor q5, q5, q12");
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Expand raw syndromes backwards.
|
|
*
|
|
* Final layout:
|
|
*
|
|
* S0:
|
|
* q0 = lane0 low
|
|
* q1 = lane0 high
|
|
* q2 = lane1 low
|
|
* q3 = lane1 high
|
|
*
|
|
* S1:
|
|
* q4/q5 = lane0 low/high
|
|
* q6/q7 = lane1 low/high
|
|
*
|
|
* S2:
|
|
* q8/q9 = lane0 low/high
|
|
* q10/q11 = lane1 low/high
|
|
*/
|
|
|
|
if (nr >= 3) {
|
|
asm volatile ("vshr.u8 q9, q4, #4");
|
|
asm volatile ("vand q8, q4, q15");
|
|
asm volatile ("vand q9, q9, q15");
|
|
|
|
asm volatile ("vshr.u8 q11, q5, #4");
|
|
asm volatile ("vand q10, q5, q15");
|
|
asm volatile ("vand q11, q11, q15");
|
|
}
|
|
|
|
if (nr >= 2) {
|
|
asm volatile ("vshr.u8 q5, q2, #4");
|
|
asm volatile ("vand q4, q2, q15");
|
|
asm volatile ("vand q5, q5, q15");
|
|
|
|
asm volatile ("vshr.u8 q7, q3, #4");
|
|
asm volatile ("vand q6, q3, q15");
|
|
asm volatile ("vand q7, q7, q15");
|
|
}
|
|
|
|
/*
|
|
* Raw S0 lane1 is q1, so process it before q1 is overwritten
|
|
* by the high nibble of lane0.
|
|
*/
|
|
asm volatile ("vshr.u8 q3, q1, #4");
|
|
asm volatile ("vand q2, q1, q15");
|
|
asm volatile ("vand q3, q3, q15");
|
|
|
|
asm volatile ("vshr.u8 q1, q0, #4");
|
|
asm volatile ("vand q0, q0, q15");
|
|
asm volatile ("vand q1, q1, q15");
|
|
|
|
/*
|
|
* Reconstruction.
|
|
*
|
|
* q12 = multiplication table
|
|
* q13 = temporary result
|
|
* q14 = lane0 accumulator
|
|
* q15 = lane1 accumulator
|
|
*/
|
|
for (j = 0; j < nr; ++j) {
|
|
const uint8_t **t = R[j];
|
|
|
|
/* coefficient 0 initializes both accumulators */
|
|
asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[0][0]));
|
|
|
|
asm volatile ("vtbl.8 d28, {d24-d25}, d0");
|
|
asm volatile ("vtbl.8 d29, {d24-d25}, d1");
|
|
asm volatile ("vtbl.8 d30, {d24-d25}, d4");
|
|
asm volatile ("vtbl.8 d31, {d24-d25}, d5");
|
|
|
|
asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[0][16]));
|
|
|
|
asm volatile ("vtbl.8 d26, {d24-d25}, d2");
|
|
asm volatile ("vtbl.8 d27, {d24-d25}, d3");
|
|
asm volatile ("veor q14, q14, q13");
|
|
|
|
asm volatile ("vtbl.8 d26, {d24-d25}, d6");
|
|
asm volatile ("vtbl.8 d27, {d24-d25}, d7");
|
|
asm volatile ("veor q15, q15, q13");
|
|
|
|
if (nr >= 2) {
|
|
asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[1][0]));
|
|
|
|
asm volatile ("vtbl.8 d26, {d24-d25}, d8");
|
|
asm volatile ("vtbl.8 d27, {d24-d25}, d9");
|
|
asm volatile ("veor q14, q14, q13");
|
|
|
|
asm volatile ("vtbl.8 d26, {d24-d25}, d12");
|
|
asm volatile ("vtbl.8 d27, {d24-d25}, d13");
|
|
asm volatile ("veor q15, q15, q13");
|
|
|
|
asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[1][16]));
|
|
|
|
asm volatile ("vtbl.8 d26, {d24-d25}, d10");
|
|
asm volatile ("vtbl.8 d27, {d24-d25}, d11");
|
|
asm volatile ("veor q14, q14, q13");
|
|
|
|
asm volatile ("vtbl.8 d26, {d24-d25}, d14");
|
|
asm volatile ("vtbl.8 d27, {d24-d25}, d15");
|
|
asm volatile ("veor q15, q15, q13");
|
|
}
|
|
|
|
if (nr >= 3) {
|
|
asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[2][0]));
|
|
|
|
asm volatile ("vtbl.8 d26, {d24-d25}, d16");
|
|
asm volatile ("vtbl.8 d27, {d24-d25}, d17");
|
|
asm volatile ("veor q14, q14, q13");
|
|
|
|
asm volatile ("vtbl.8 d26, {d24-d25}, d20");
|
|
asm volatile ("vtbl.8 d27, {d24-d25}, d21");
|
|
asm volatile ("veor q15, q15, q13");
|
|
|
|
asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[2][16]));
|
|
|
|
asm volatile ("vtbl.8 d26, {d24-d25}, d18");
|
|
asm volatile ("vtbl.8 d27, {d24-d25}, d19");
|
|
asm volatile ("veor q14, q14, q13");
|
|
|
|
asm volatile ("vtbl.8 d26, {d24-d25}, d22");
|
|
asm volatile ("vtbl.8 d27, {d24-d25}, d23");
|
|
asm volatile ("veor q15, q15, q13");
|
|
}
|
|
|
|
asm volatile ("vst1.8 {q14}, %0" : "=Q" (pa[j][i]));
|
|
asm volatile ("vst1.8 {q15}, %0" : "=Q" (pa[j][i + 16]));
|
|
}
|
|
}
|
|
|
|
raid_neon32_end();
|
|
}
|
|
|
|
/*
|
|
* Recover multiple data failures using selected parity blocks with AArch32 NEON.
|
|
*
|
|
* This avoids temporary syndrome buffers and the
|
|
* generation of unused parity rows.
|
|
*
|
|
* If P is available, syndrome 0 is accumulated with XOR without table lookups.
|
|
*/
|
|
static __always_inline void raid_recX_neon32(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
uint8_t **v = (uint8_t **)vv;
|
|
uint8_t *p[RAID_PARITY_MAX];
|
|
uint8_t *pa[RAID_PARITY_MAX];
|
|
uint8_t *src[RAID_DATA_MAX];
|
|
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
|
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
|
const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX];
|
|
const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
|
|
size_t i;
|
|
int d, j, k, s;
|
|
int ns;
|
|
|
|
BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
|
|
|
|
for (j = 0; j < nr; ++j)
|
|
for (k = 0; k < nr; ++k)
|
|
G[j * nr + k] = A(ip[j], id[k]);
|
|
|
|
raid_invert(G, V, nr);
|
|
|
|
for (j = 0; j < nr; ++j) {
|
|
p[j] = v[nd + ip[j]];
|
|
pa[j] = v[id[j]];
|
|
}
|
|
|
|
ns = 0;
|
|
k = 0;
|
|
|
|
for (d = 0; d < nd; ++d) {
|
|
if (k < nr && d == id[k]) {
|
|
++k;
|
|
continue;
|
|
}
|
|
|
|
src[ns] = v[d];
|
|
|
|
for (j = 0; j < nr; ++j)
|
|
S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0];
|
|
|
|
++ns;
|
|
}
|
|
|
|
BUG_ON(k != nr);
|
|
BUG_ON(ns != nd - nr);
|
|
|
|
for (j = 0; j < nr; ++j)
|
|
for (k = 0; k < nr; ++k)
|
|
R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
|
|
|
|
raid_neon32_begin();
|
|
|
|
for (i = 0; i < size; i += 16) {
|
|
/*
|
|
* Q15 is the low-nibble mask during generation/splitting.
|
|
* Reconstruction later reuses q15 as a multiplication temporary.
|
|
*/
|
|
asm volatile ("vmov.i8 q15, #0x0f");
|
|
|
|
/*
|
|
* Raw syndrome registers:
|
|
*
|
|
* q0 = syndrome 0
|
|
* q1 = syndrome 1
|
|
* q2 = syndrome 2
|
|
* q3 = syndrome 3
|
|
* q4 = syndrome 4
|
|
* q5 = syndrome 5
|
|
*/
|
|
|
|
asm volatile ("vld1.8 {q0}, %0" : : "Q" (p[0][i]));
|
|
|
|
if (nr >= 2)
|
|
asm volatile ("vld1.8 {q1}, %0" : : "Q" (p[1][i]));
|
|
|
|
if (nr >= 3)
|
|
asm volatile ("vld1.8 {q2}, %0" : : "Q" (p[2][i]));
|
|
|
|
if (nr >= 4)
|
|
asm volatile ("vld1.8 {q3}, %0" : : "Q" (p[3][i]));
|
|
|
|
if (nr >= 5)
|
|
asm volatile ("vld1.8 {q4}, %0" : : "Q" (p[4][i]));
|
|
|
|
if (nr >= 6)
|
|
asm volatile ("vld1.8 {q5}, %0" : : "Q" (p[5][i]));
|
|
|
|
/*
|
|
* During survivor generation:
|
|
*
|
|
* q6 = source low
|
|
* q7 = source high
|
|
* q8 = low table
|
|
* q9 = high table
|
|
* q10 = low result
|
|
* q11 = high result
|
|
*/
|
|
|
|
for (s = 0; s < ns; ++s) {
|
|
const uint8_t **t = S[s];
|
|
|
|
asm volatile ("vld1.8 {q6}, %0" : : "Q" (src[s][i]));
|
|
|
|
if (has_p) {
|
|
asm volatile ("veor q0, q0, q6");
|
|
|
|
asm volatile ("vshr.u8 q7, q6, #4");
|
|
asm volatile ("vand q6, q6, q15");
|
|
asm volatile ("vand q7, q7, q15");
|
|
} else {
|
|
asm volatile ("vshr.u8 q7, q6, #4");
|
|
asm volatile ("vand q6, q6, q15");
|
|
asm volatile ("vand q7, q7, q15");
|
|
|
|
asm volatile ("vld1.8 {q8}, %0" : : "Q" (t[0][0]));
|
|
asm volatile ("vld1.8 {q9}, %0" : : "Q" (t[0][16]));
|
|
|
|
asm volatile ("vtbl.8 d20, {d16-d17}, d12");
|
|
asm volatile ("vtbl.8 d21, {d16-d17}, d13");
|
|
asm volatile ("vtbl.8 d22, {d18-d19}, d14");
|
|
asm volatile ("vtbl.8 d23, {d18-d19}, d15");
|
|
asm volatile ("veor q10, q10, q11");
|
|
asm volatile ("veor q0, q0, q10");
|
|
}
|
|
|
|
if (nr >= 2) {
|
|
asm volatile ("vld1.8 {q8}, %0" : : "Q" (t[1][0]));
|
|
asm volatile ("vld1.8 {q9}, %0" : : "Q" (t[1][16]));
|
|
|
|
asm volatile ("vtbl.8 d20, {d16-d17}, d12");
|
|
asm volatile ("vtbl.8 d21, {d16-d17}, d13");
|
|
asm volatile ("vtbl.8 d22, {d18-d19}, d14");
|
|
asm volatile ("vtbl.8 d23, {d18-d19}, d15");
|
|
asm volatile ("veor q10, q10, q11");
|
|
asm volatile ("veor q1, q1, q10");
|
|
}
|
|
|
|
if (nr >= 3) {
|
|
asm volatile ("vld1.8 {q8}, %0" : : "Q" (t[2][0]));
|
|
asm volatile ("vld1.8 {q9}, %0" : : "Q" (t[2][16]));
|
|
|
|
asm volatile ("vtbl.8 d20, {d16-d17}, d12");
|
|
asm volatile ("vtbl.8 d21, {d16-d17}, d13");
|
|
asm volatile ("vtbl.8 d22, {d18-d19}, d14");
|
|
asm volatile ("vtbl.8 d23, {d18-d19}, d15");
|
|
asm volatile ("veor q10, q10, q11");
|
|
asm volatile ("veor q2, q2, q10");
|
|
}
|
|
|
|
if (nr >= 4) {
|
|
asm volatile ("vld1.8 {q8}, %0" : : "Q" (t[3][0]));
|
|
asm volatile ("vld1.8 {q9}, %0" : : "Q" (t[3][16]));
|
|
|
|
asm volatile ("vtbl.8 d20, {d16-d17}, d12");
|
|
asm volatile ("vtbl.8 d21, {d16-d17}, d13");
|
|
asm volatile ("vtbl.8 d22, {d18-d19}, d14");
|
|
asm volatile ("vtbl.8 d23, {d18-d19}, d15");
|
|
asm volatile ("veor q10, q10, q11");
|
|
asm volatile ("veor q3, q3, q10");
|
|
}
|
|
|
|
if (nr >= 5) {
|
|
asm volatile ("vld1.8 {q8}, %0" : : "Q" (t[4][0]));
|
|
asm volatile ("vld1.8 {q9}, %0" : : "Q" (t[4][16]));
|
|
|
|
asm volatile ("vtbl.8 d20, {d16-d17}, d12");
|
|
asm volatile ("vtbl.8 d21, {d16-d17}, d13");
|
|
asm volatile ("vtbl.8 d22, {d18-d19}, d14");
|
|
asm volatile ("vtbl.8 d23, {d18-d19}, d15");
|
|
asm volatile ("veor q10, q10, q11");
|
|
asm volatile ("veor q4, q4, q10");
|
|
}
|
|
|
|
if (nr >= 6) {
|
|
asm volatile ("vld1.8 {q8}, %0" : : "Q" (t[5][0]));
|
|
asm volatile ("vld1.8 {q9}, %0" : : "Q" (t[5][16]));
|
|
|
|
asm volatile ("vtbl.8 d20, {d16-d17}, d12");
|
|
asm volatile ("vtbl.8 d21, {d16-d17}, d13");
|
|
asm volatile ("vtbl.8 d22, {d18-d19}, d14");
|
|
asm volatile ("vtbl.8 d23, {d18-d19}, d15");
|
|
asm volatile ("veor q10, q10, q11");
|
|
asm volatile ("veor q5, q5, q10");
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Expand raw syndromes backwards.
|
|
*
|
|
* Final layout:
|
|
*
|
|
* q0/q1 syndrome 0 low/high
|
|
* q2/q3 syndrome 1 low/high
|
|
* q4/q5 syndrome 2 low/high
|
|
* q6/q7 syndrome 3 low/high
|
|
* q8/q9 syndrome 4 low/high
|
|
* q10/q11 syndrome 5 low/high
|
|
*/
|
|
|
|
if (nr >= 6) {
|
|
asm volatile ("vshr.u8 q11, q5, #4");
|
|
asm volatile ("vand q10, q5, q15");
|
|
asm volatile ("vand q11, q11, q15");
|
|
}
|
|
|
|
if (nr >= 5) {
|
|
asm volatile ("vshr.u8 q9, q4, #4");
|
|
asm volatile ("vand q8, q4, q15");
|
|
asm volatile ("vand q9, q9, q15");
|
|
}
|
|
|
|
if (nr >= 4) {
|
|
asm volatile ("vshr.u8 q7, q3, #4");
|
|
asm volatile ("vand q6, q3, q15");
|
|
asm volatile ("vand q7, q7, q15");
|
|
}
|
|
|
|
if (nr >= 3) {
|
|
asm volatile ("vshr.u8 q5, q2, #4");
|
|
asm volatile ("vand q4, q2, q15");
|
|
asm volatile ("vand q5, q5, q15");
|
|
}
|
|
|
|
if (nr >= 2) {
|
|
asm volatile ("vshr.u8 q3, q1, #4");
|
|
asm volatile ("vand q2, q1, q15");
|
|
asm volatile ("vand q3, q3, q15");
|
|
}
|
|
|
|
asm volatile ("vshr.u8 q1, q0, #4");
|
|
asm volatile ("vand q0, q0, q15");
|
|
asm volatile ("vand q1, q1, q15");
|
|
|
|
/*
|
|
* Reconstruction:
|
|
*
|
|
* q12 = low table
|
|
* q13 = high table
|
|
* q14 = output accumulator
|
|
* q15 = multiplication temporary
|
|
*/
|
|
|
|
for (j = 0; j < nr; ++j) {
|
|
const uint8_t **t = R[j];
|
|
|
|
/* coefficient 0 initializes q14 */
|
|
asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[0][0]));
|
|
asm volatile ("vld1.8 {q13}, %0" : : "Q" (t[0][16]));
|
|
|
|
asm volatile ("vtbl.8 d28, {d24-d25}, d0");
|
|
asm volatile ("vtbl.8 d29, {d24-d25}, d1");
|
|
|
|
asm volatile ("vtbl.8 d30, {d26-d27}, d2");
|
|
asm volatile ("vtbl.8 d31, {d26-d27}, d3");
|
|
|
|
asm volatile ("veor q14, q14, q15");
|
|
|
|
if (nr >= 2) {
|
|
asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[1][0]));
|
|
asm volatile ("vld1.8 {q13}, %0" : : "Q" (t[1][16]));
|
|
|
|
asm volatile ("vtbl.8 d30, {d24-d25}, d4");
|
|
asm volatile ("vtbl.8 d31, {d24-d25}, d5");
|
|
asm volatile ("veor q14, q14, q15");
|
|
|
|
asm volatile ("vtbl.8 d30, {d26-d27}, d6");
|
|
asm volatile ("vtbl.8 d31, {d26-d27}, d7");
|
|
asm volatile ("veor q14, q14, q15");
|
|
}
|
|
|
|
if (nr >= 3) {
|
|
asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[2][0]));
|
|
asm volatile ("vld1.8 {q13}, %0" : : "Q" (t[2][16]));
|
|
|
|
asm volatile ("vtbl.8 d30, {d24-d25}, d8");
|
|
asm volatile ("vtbl.8 d31, {d24-d25}, d9");
|
|
asm volatile ("veor q14, q14, q15");
|
|
|
|
asm volatile ("vtbl.8 d30, {d26-d27}, d10");
|
|
asm volatile ("vtbl.8 d31, {d26-d27}, d11");
|
|
asm volatile ("veor q14, q14, q15");
|
|
}
|
|
|
|
if (nr >= 4) {
|
|
asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[3][0]));
|
|
asm volatile ("vld1.8 {q13}, %0" : : "Q" (t[3][16]));
|
|
|
|
asm volatile ("vtbl.8 d30, {d24-d25}, d12");
|
|
asm volatile ("vtbl.8 d31, {d24-d25}, d13");
|
|
asm volatile ("veor q14, q14, q15");
|
|
|
|
asm volatile ("vtbl.8 d30, {d26-d27}, d14");
|
|
asm volatile ("vtbl.8 d31, {d26-d27}, d15");
|
|
asm volatile ("veor q14, q14, q15");
|
|
}
|
|
|
|
if (nr >= 5) {
|
|
asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[4][0]));
|
|
asm volatile ("vld1.8 {q13}, %0" : : "Q" (t[4][16]));
|
|
|
|
asm volatile ("vtbl.8 d30, {d24-d25}, d16");
|
|
asm volatile ("vtbl.8 d31, {d24-d25}, d17");
|
|
asm volatile ("veor q14, q14, q15");
|
|
|
|
asm volatile ("vtbl.8 d30, {d26-d27}, d18");
|
|
asm volatile ("vtbl.8 d31, {d26-d27}, d19");
|
|
asm volatile ("veor q14, q14, q15");
|
|
}
|
|
|
|
if (nr >= 6) {
|
|
asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[5][0]));
|
|
asm volatile ("vld1.8 {q13}, %0" : : "Q" (t[5][16]));
|
|
|
|
asm volatile ("vtbl.8 d30, {d24-d25}, d20");
|
|
asm volatile ("vtbl.8 d31, {d24-d25}, d21");
|
|
asm volatile ("veor q14, q14, q15");
|
|
|
|
asm volatile ("vtbl.8 d30, {d26-d27}, d22");
|
|
asm volatile ("vtbl.8 d31, {d26-d27}, d23");
|
|
asm volatile ("veor q14, q14, q15");
|
|
}
|
|
|
|
asm volatile ("vst1.8 {q14}, %0" : "=Q" (pa[j][i]));
|
|
}
|
|
}
|
|
|
|
raid_neon32_end();
|
|
}
|
|
|
|
void raid_gen1_neon32(int nd, size_t size, void **vv, int streaming)
|
|
{
|
|
(void)streaming;
|
|
raid_gen1_neon32_gen(nd, size, vv);
|
|
}
|
|
|
|
void raid_gen2_neon32_raid(int nd, size_t size, void **vv, int streaming)
|
|
{
|
|
(void)streaming;
|
|
raid_gen2_neon32_gen(nd, size, vv, 2);
|
|
}
|
|
|
|
void raid_gen2_neon32_aes(int nd, size_t size, void **vv, int streaming)
|
|
{
|
|
(void)streaming;
|
|
raid_gen2_neon32_gen(nd, size, vv, 3);
|
|
}
|
|
|
|
void raid_genz_neon32_raid(int nd, size_t size, void **vv, int streaming)
|
|
{
|
|
(void)streaming;
|
|
raid_genz_neon32_gen(nd, size, vv);
|
|
}
|
|
|
|
void raid_gen3_neon32_raid(int nd, size_t size, void **vv, int streaming)
|
|
{
|
|
(void)streaming;
|
|
raid_genX_neon32(nd, size, vv, 3, 2);
|
|
}
|
|
|
|
void raid_gen3_neon32_aes(int nd, size_t size, void **vv, int streaming)
|
|
{
|
|
(void)streaming;
|
|
raid_genX_neon32(nd, size, vv, 3, 3);
|
|
}
|
|
|
|
void raid_gen4_neon32_raid(int nd, size_t size, void **vv, int streaming)
|
|
{
|
|
(void)streaming;
|
|
raid_genX_neon32(nd, size, vv, 4, 2);
|
|
}
|
|
|
|
void raid_gen4_neon32_aes(int nd, size_t size, void **vv, int streaming)
|
|
{
|
|
(void)streaming;
|
|
raid_genX_neon32(nd, size, vv, 4, 3);
|
|
}
|
|
|
|
void raid_gen5_neon32_raid(int nd, size_t size, void **vv, int streaming)
|
|
{
|
|
(void)streaming;
|
|
raid_genX_neon32(nd, size, vv, 5, 2);
|
|
}
|
|
|
|
void raid_gen5_neon32_aes(int nd, size_t size, void **vv, int streaming)
|
|
{
|
|
(void)streaming;
|
|
raid_genX_neon32(nd, size, vv, 5, 3);
|
|
}
|
|
|
|
void raid_gen6_neon32_raid(int nd, size_t size, void **vv, int streaming)
|
|
{
|
|
(void)streaming;
|
|
raid_genX_neon32(nd, size, vv, 6, 2);
|
|
}
|
|
|
|
void raid_gen6_neon32_aes(int nd, size_t size, void **vv, int streaming)
|
|
{
|
|
(void)streaming;
|
|
raid_genX_neon32(nd, size, vv, 6, 3);
|
|
}
|
|
|
|
void raid_rec1_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
BUG_ON(nr != 1);
|
|
|
|
/* if recovering with P uses the delta function */
|
|
if (ip[0] == 0) {
|
|
raid_rec1of1(id, nd, size, vv);
|
|
return;
|
|
}
|
|
|
|
raid_recX_neon32_123(1, 0, id, ip, nd, size, vv);
|
|
}
|
|
|
|
void raid_rec2_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
BUG_ON(nr != 2);
|
|
|
|
if (ip[0] == 0)
|
|
raid_recX_neon32_123(2, 1, id, ip, nd, size, vv);
|
|
else
|
|
raid_recX_neon32_123(2, 0, id, ip, nd, size, vv);
|
|
}
|
|
|
|
void raid_rec3_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
BUG_ON(nr != 3);
|
|
if (ip[0] == 0)
|
|
raid_recX_neon32_123(3, 1, id, ip, nd, size, vv);
|
|
else
|
|
raid_recX_neon32_123(3, 0, id, ip, nd, size, vv);
|
|
}
|
|
|
|
void raid_rec4_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
BUG_ON(nr != 4);
|
|
if (ip[0] == 0)
|
|
raid_recX_neon32(4, 1, id, ip, nd, size, vv);
|
|
else
|
|
raid_recX_neon32(4, 0, id, ip, nd, size, vv);
|
|
}
|
|
|
|
void raid_rec5_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
BUG_ON(nr != 5);
|
|
if (ip[0] == 0)
|
|
raid_recX_neon32(5, 1, id, ip, nd, size, vv);
|
|
else
|
|
raid_recX_neon32(5, 0, id, ip, nd, size, vv);
|
|
}
|
|
|
|
void raid_rec6_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|
{
|
|
BUG_ON(nr != 6);
|
|
if (ip[0] == 0)
|
|
raid_recX_neon32(6, 1, id, ip, nd, size, vv);
|
|
else
|
|
raid_recX_neon32(6, 0, id, ip, nd, size, vv);
|
|
}
|
|
|
|
void raid_register_neon32(void)
|
|
{
|
|
raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "neon32", raid_gen1_neon32, RAID_POLY_ANY);
|
|
|
|
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "neon32", raid_gen2_neon32_raid, RAID_POLY_RAID);
|
|
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "neon32", raid_gen2_neon32_aes, RAID_POLY_AES);
|
|
|
|
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "neon32", raid_genz_neon32_raid, RAID_POLY_RAID);
|
|
|
|
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "neon32", raid_gen3_neon32_raid, RAID_POLY_RAID);
|
|
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "neon32", raid_gen3_neon32_aes, RAID_POLY_AES);
|
|
|
|
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "neon32", raid_gen4_neon32_raid, RAID_POLY_RAID);
|
|
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "neon32", raid_gen4_neon32_aes, RAID_POLY_AES);
|
|
|
|
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "neon32", raid_gen5_neon32_raid, RAID_POLY_RAID);
|
|
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "neon32", raid_gen5_neon32_aes, RAID_POLY_AES);
|
|
|
|
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "neon32", raid_gen6_neon32_raid, RAID_POLY_RAID);
|
|
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "neon32", raid_gen6_neon32_aes, RAID_POLY_AES);
|
|
|
|
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "neon32", raid_rec1_neon32, RAID_POLY_ANY);
|
|
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "neon32", raid_rec2_neon32, RAID_POLY_ANY);
|
|
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "neon32", raid_rec3_neon32, RAID_POLY_ANY);
|
|
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "neon32", raid_rec4_neon32, RAID_POLY_ANY);
|
|
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "neon32", raid_rec5_neon32, RAID_POLY_ANY);
|
|
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "neon32", raid_rec6_neon32, RAID_POLY_ANY);
|
|
}
|
|
|
|
#endif
|