rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 10:51:54 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions
snapraid/raid/neon32.c

1475 lines
31 KiB
C
Raw Permalink Blame History

// SPDX-License-Identifier: GPL-2.0-or-later
// Copyright (C) 2026 Andrea Mazzoleni
#include "internal.h"
#include "gf.h"
#include "cpu.h"
#ifdef CONFIG_NEON32
/*
* GEN1 (RAID5 with xor) AArch32 NEON implementation
*/
void raid_gen1_neon32(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
int d, l;
size_t i;
l = nd - 1;
p = v[nd];
raid_neon32_begin();
for (i = 0; i < size; i += 64) {
asm volatile (
"vld1.8 {q0}, %0\n"
"vld1.8 {q1}, %1\n"
"vld1.8 {q2}, %2\n"
"vld1.8 {q3}, %3\n"
:
: "Q" (v[0][i]), "Q" (v[0][i + 16]),
"Q" (v[0][i + 32]), "Q" (v[0][i + 48])
);
for (d = 1; d <= l; ++d) {
asm volatile (
"vld1.8 {q4}, %0\n"
"vld1.8 {q5}, %1\n"
"vld1.8 {q6}, %2\n"
"vld1.8 {q7}, %3\n"
"veor q0, q0, q4\n"
"veor q1, q1, q5\n"
"veor q2, q2, q6\n"
"veor q3, q3, q7\n"
:
: "Q" (v[d][i]), "Q" (v[d][i + 16]),
"Q" (v[d][i + 32]), "Q" (v[d][i + 48])
);
}
asm volatile (
"vst1.8 {q0}, %0\n"
"vst1.8 {q1}, %1\n"
"vst1.8 {q2}, %2\n"
"vst1.8 {q3}, %3\n"
: "=Q" (p[i]), "=Q" (p[i + 16]),
"=Q" (p[i + 32]), "=Q" (p[i + 48])
);
}
raid_neon32_end();
}
/*
* GEN2 Cauchy AArch32 NEON implementation using only Q *= 2.
*/
static __always_inline void raid_gen2_neon32_x2(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
uint8_t *q;
int d, l;
size_t i;
l = nd - 1;
p = v[nd];
q = v[nd + 1];
raid_neon32_begin();
/* q14 contains the active reduction polynomial. */
asm volatile (
"vld1.8 {q14}, %0\n"
:
: "Q" (gfconst16.poly[0])
);
for (i = 0; i < size; i += 32) {
asm volatile (
"vld1.8 {q0}, %0\n"
"vld1.8 {q1}, %1\n"
"vmov q2, q0\n"
"vmov q3, q1\n"
:
: "Q" (v[l][i]), "Q" (v[l][i + 16])
);
for (d = l - 1; d >= 0; --d) {
asm volatile (
/* double Q0-Q1 */
"vshr.s8 q8, q2, #7\n"
"vshl.i8 q2, q2, #1\n"
"vand q8, q8, q14\n"
"veor q2, q2, q8\n"
"vshr.s8 q8, q3, #7\n"
"vshl.i8 q3, q3, #1\n"
"vand q8, q8, q14\n"
"veor q3, q3, q8\n"
/* load and XOR */
"vld1.8 {q12}, %0\n"
"vld1.8 {q13}, %1\n"
"veor q0, q0, q12\n"
"veor q1, q1, q13\n"
"veor q2, q2, q12\n"
"veor q3, q3, q13\n"
:
: "Q" (v[d][i]), "Q" (v[d][i + 16])
);
}
asm volatile (
"vst1.8 {q0}, %0\n"
"vst1.8 {q1}, %1\n"
"vst1.8 {q2}, %2\n"
"vst1.8 {q3}, %3\n"
: "=Q" (p[i]), "=Q" (p[i + 16]),
"=Q" (q[i]), "=Q" (q[i + 16])
);
}
raid_neon32_end();
}
/*
* GEN2 Cauchy AArch32 NEON implementation using the AES G23 Q recurrence.
*/
static __always_inline void raid_gen2_neon32_g23(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
uint8_t *q;
int d, l;
size_t i;
l = nd - 1;
p = v[nd];
q = v[nd + 1];
int g23_boundary = raid_g23_boundary(l - 1);
raid_neon32_begin();
/* q14 contains the active reduction polynomial. */
asm volatile (
"vld1.8 {q14}, %0\n"
:
: "Q" (gfconst16.poly[0])
);
for (i = 0; i < size; i += 32) {
int boundary = g23_boundary;
asm volatile (
"vld1.8 {q0}, %0\n"
"vld1.8 {q1}, %1\n"
"vmov q2, q0\n"
"vmov q3, q1\n"
:
: "Q" (v[l][i]), "Q" (v[l][i + 16])
);
d = l - 1;
for (;;) {
/*
* All transitions above 'boundary' are x2.
*/
for (; d > boundary; --d) {
asm volatile (
/* double Q0-Q1 */
"vshr.s8 q8, q2, #7\n"
"vshl.i8 q2, q2, #1\n"
"vand q8, q8, q14\n"
"veor q2, q2, q8\n"
"vshr.s8 q8, q3, #7\n"
"vshl.i8 q3, q3, #1\n"
"vand q8, q8, q14\n"
"veor q3, q3, q8\n"
/* load and XOR */
"vld1.8 {q12}, %0\n"
"vld1.8 {q13}, %1\n"
"veor q0, q0, q12\n"
"veor q1, q1, q13\n"
"veor q2, q2, q12\n"
"veor q3, q3, q13\n"
:
: "Q" (v[d][i]), "Q" (v[d][i + 16])
);
}
/*
* boundary == -1 identifies the final x2-only segment.
* At this point all remaining disks have been consumed.
*/
if (boundary < 0)
break;
/*
* d == boundary.
*
* Perform the fixed G23 x3 transition and then add
* the current disk to P and Q.
*/
asm volatile (
/* preserve Q for Q *= 3 */
"vmov q10, q2\n"
"vmov q11, q3\n"
/* double Q0-Q1 */
"vshr.s8 q8, q2, #7\n"
"vshl.i8 q2, q2, #1\n"
"vand q8, q8, q14\n"
"veor q2, q2, q8\n"
"vshr.s8 q8, q3, #7\n"
"vshl.i8 q3, q3, #1\n"
"vand q8, q8, q14\n"
"veor q3, q3, q8\n"
/* load and XOR */
"vld1.8 {q12}, %0\n"
"vld1.8 {q13}, %1\n"
"veor q0, q0, q12\n"
"veor q1, q1, q13\n"
"veor q2, q2, q12\n"
"veor q3, q3, q13\n"
/* complete Q *= 3 */
"veor q2, q2, q10\n"
"veor q3, q3, q11\n"
:
: "Q" (v[d][i]), "Q" (v[d][i + 16])
);
--d;
/*
* G23 boundaries are exactly 51 transitions apart.
* After d = 50 there are no more x3 transitions.
*/
boundary -= 51;
if (boundary < 50)
boundary = -1;
}
asm volatile (
"vst1.8 {q0}, %0\n"
"vst1.8 {q1}, %1\n"
"vst1.8 {q2}, %2\n"
"vst1.8 {q3}, %3\n"
: "=Q" (p[i]), "=Q" (p[i + 16]),
"=Q" (q[i]), "=Q" (q[i + 16])
);
}
raid_neon32_end();
}
void raid_gen2_neon32_raid(int nd, size_t size, void **vv)
{
raid_gen2_neon32_x2(nd, size, vv);
}
void raid_gen2_neon32_aes(int nd, size_t size, void **vv)
{
if (raid_g23_x2_only(nd))
raid_gen2_neon32_x2(nd, size, vv);
else
raid_gen2_neon32_g23(nd, size, vv);
}
/*
* GENz (triple parity with powers of 2^-1) AArch32 NEON implementation
*/
void raid_genz_neon32_raid(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
uint8_t *q;
uint8_t *r;
int d, l;
size_t i;
l = nd - 1;
p = v[nd];
q = v[nd + 1];
r = v[nd + 2];
raid_neon32_begin();
asm volatile (
"vld1.8 {q14}, %0\n"
"vld1.8 {q15}, %1\n"
:
: "Q" (gfconst16.poly[0]), "Q" (gfconst16.half[0])
);
for (i = 0; i < size; i += 32) {
asm volatile (
"vld1.8 {q0}, %0\n"
"vld1.8 {q1}, %1\n"
"vmov q2, q0\n"
"vmov q3, q1\n"
"vmov q4, q0\n"
"vmov q5, q1\n"
:
: "Q" (v[l][i]), "Q" (v[l][i + 16])
);
for (d = l - 1; d >= 0; --d) {
asm volatile (
"vshr.s8 q8, q2, #7\n"
"vshl.i8 q2, q2, #1\n"
"vand q8, q8, q14\n"
"veor q2, q2, q8\n"
"vshr.s8 q8, q3, #7\n"
"vshl.i8 q3, q3, #1\n"
"vand q8, q8, q14\n"
"veor q3, q3, q8\n"
"vshl.i8 q9, q4, #7\n"
"vshr.s8 q9, q9, #7\n"
"vshr.u8 q4, q4, #1\n"
"vand q9, q9, q15\n"
"veor q4, q4, q9\n"
"vshl.i8 q9, q5, #7\n"
"vshr.s8 q9, q9, #7\n"
"vshr.u8 q5, q5, #1\n"
"vand q9, q9, q15\n"
"veor q5, q5, q9\n"
"vld1.8 {q12}, %0\n"
"vld1.8 {q13}, %1\n"
"veor q0, q0, q12\n"
"veor q1, q1, q13\n"
"veor q2, q2, q12\n"
"veor q3, q3, q13\n"
"veor q4, q4, q12\n"
"veor q5, q5, q13\n"
:
: "Q" (v[d][i]), "Q" (v[d][i + 16])
);
}
asm volatile (
"vst1.8 {q0}, %0\n"
"vst1.8 {q1}, %1\n"
"vst1.8 {q2}, %2\n"
"vst1.8 {q3}, %3\n"
"vst1.8 {q4}, %4\n"
"vst1.8 {q5}, %5\n"
: "=Q" (p[i]), "=Q" (p[i + 16]),
"=Q" (q[i]), "=Q" (q[i + 16]),
"=Q" (r[i]), "=Q" (r[i + 16])
);
}
raid_neon32_end();
}
/*
* GENX AArch32 NEON implementation using only Q *= 2.
*/
static __always_inline void raid_genX_neon32_x2(int nd, size_t size,
void **vv, int np)
{
uint8_t **v = (uint8_t **)vv;
size_t i;
int d, l;
l = nd - 1;
/* special case with only one data disk */
if (l == 0) {
for (d = 0; d < np; ++d)
memcpy(v[1 + d], v[0], size);
return;
}
raid_neon32_begin();
asm volatile (
"vld1.8 {q14}, %0\n"
"vld1.8 {q15}, %1\n"
:
: "Q" (gfconst16.poly[0]), "Q" (gfconst16.low4[0])
);
for (i = 0; i < size; i += 16) {
/* last disk initializes P/Q without a preceding Q transition */
asm volatile (
"vld1.8 {q0}, %0\n"
"vld1.8 {q1}, %0\n"
:
: "Q" (v[l][i])
);
if (np >= 3) {
asm volatile (
"vshr.u8 q8, q0, #4\n"
"vand q7, q0, q15\n"
"vand q8, q8, q15\n"
);
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q2, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][1][0][0]),
"Q" (raid_gfcauchypshufb[l][1][1][0])
);
}
if (np >= 4) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q3, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][2][0][0]),
"Q" (raid_gfcauchypshufb[l][2][1][0])
);
}
if (np >= 5) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q4, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][3][0][0]),
"Q" (raid_gfcauchypshufb[l][3][1][0])
);
}
if (np >= 6) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q5, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][4][0][0]),
"Q" (raid_gfcauchypshufb[l][4][1][0])
);
}
/* intermediate disks */
for (d = l - 1; d > 0; --d) {
asm volatile (
/* Q *= 2 */
"vshr.s8 q11, q1, #7\n"
"vshl.i8 q1, q1, #1\n"
"vand q11, q11, q14\n"
"veor q1, q1, q11\n"
/* load disk and update P/Q */
"vld1.8 {q6}, %0\n"
"veor q0, q0, q6\n"
"veor q1, q1, q6\n"
:
: "Q" (v[d][i])
);
if (np >= 3) {
asm volatile (
"vshr.u8 q8, q6, #4\n"
"vand q7, q6, q15\n"
"vand q8, q8, q15\n"
);
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q2, q2, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][1][0][0]),
"Q" (raid_gfcauchypshufb[d][1][1][0])
);
}
if (np >= 4) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q3, q3, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][2][0][0]),
"Q" (raid_gfcauchypshufb[d][2][1][0])
);
}
if (np >= 5) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q4, q4, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][3][0][0]),
"Q" (raid_gfcauchypshufb[d][3][1][0])
);
}
if (np >= 6) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q5, q5, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][4][0][0]),
"Q" (raid_gfcauchypshufb[d][4][1][0])
);
}
}
/* first disk with all coefficients at 1 */
asm volatile (
"vshr.s8 q11, q1, #7\n"
"vshl.i8 q1, q1, #1\n"
"vand q11, q11, q14\n"
"veor q1, q1, q11\n"
"vld1.8 {q6}, %0\n"
"veor q0, q0, q6\n"
"veor q1, q1, q6\n"
:
: "Q" (v[0][i])
);
if (np >= 3)
asm volatile ("veor q2, q2, q6\n");
if (np >= 4)
asm volatile ("veor q3, q3, q6\n");
if (np >= 5)
asm volatile ("veor q4, q4, q6\n");
if (np >= 6)
asm volatile ("veor q5, q5, q6\n");
/* write parity in increasing order */
asm volatile (
"vst1.8 {q0}, %0\n"
"vst1.8 {q1}, %1\n"
: "=Q" (v[nd][i]),
"=Q" (v[nd + 1][i])
);
if (np >= 3)
asm volatile ("vst1.8 {q2}, %0\n" : "=Q" (v[nd + 2][i]));
if (np >= 4)
asm volatile ("vst1.8 {q3}, %0\n" : "=Q" (v[nd + 3][i]));
if (np >= 5)
asm volatile ("vst1.8 {q4}, %0\n" : "=Q" (v[nd + 4][i]));
if (np >= 6)
asm volatile ("vst1.8 {q5}, %0\n" : "=Q" (v[nd + 5][i]));
}
raid_neon32_end();
}
/*
* GENX AArch32 NEON implementation using the AES G23 Q recurrence.
*/
static __always_inline void raid_genX_neon32_g23(int nd, size_t size,
void **vv, int np)
{
uint8_t **v = (uint8_t **)vv;
size_t i;
int d, l;
l = nd - 1;
/* special case with only one data disk */
if (l == 0) {
for (d = 0; d < np; ++d)
memcpy(v[1 + d], v[0], size);
return;
}
int g23_boundary = raid_g23_boundary(l - 1);
raid_neon32_begin();
asm volatile (
"vld1.8 {q14}, %0\n"
"vld1.8 {q15}, %1\n"
:
: "Q" (gfconst16.poly[0]), "Q" (gfconst16.low4[0])
);
for (i = 0; i < size; i += 16) {
int boundary = g23_boundary;
/* last disk initializes P/Q without a preceding Q transition */
asm volatile (
"vld1.8 {q0}, %0\n"
"vld1.8 {q1}, %0\n"
:
: "Q" (v[l][i])
);
if (np >= 3) {
asm volatile (
"vshr.u8 q8, q0, #4\n"
"vand q7, q0, q15\n"
"vand q8, q8, q15\n"
);
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q2, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][1][0][0]),
"Q" (raid_gfcauchypshufb[l][1][1][0])
);
}
if (np >= 4) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q3, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][2][0][0]),
"Q" (raid_gfcauchypshufb[l][2][1][0])
);
}
if (np >= 5) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q4, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][3][0][0]),
"Q" (raid_gfcauchypshufb[l][3][1][0])
);
}
if (np >= 6) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q5, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][4][0][0]),
"Q" (raid_gfcauchypshufb[l][4][1][0])
);
}
d = l - 1;
while (boundary >= 0) {
/*
* Everything above the next G23 boundary is Q *= 2.
*/
for (; d > boundary; --d) {
asm volatile (
"vshr.s8 q11, q1, #7\n"
"vshl.i8 q1, q1, #1\n"
"vand q11, q11, q14\n"
"veor q1, q1, q11\n"
"vld1.8 {q6}, %0\n"
"veor q0, q0, q6\n"
"veor q1, q1, q6\n"
:
: "Q" (v[d][i])
);
if (np >= 3) {
asm volatile (
"vshr.u8 q8, q6, #4\n"
"vand q7, q6, q15\n"
"vand q8, q8, q15\n"
);
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q2, q2, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][1][0][0]),
"Q" (raid_gfcauchypshufb[d][1][1][0])
);
}
if (np >= 4) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q3, q3, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][2][0][0]),
"Q" (raid_gfcauchypshufb[d][2][1][0])
);
}
if (np >= 5) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q4, q4, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][3][0][0]),
"Q" (raid_gfcauchypshufb[d][3][1][0])
);
}
if (np >= 6) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q5, q5, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][4][0][0]),
"Q" (raid_gfcauchypshufb[d][4][1][0])
);
}
}
/*
* d == boundary: Q *= 3.
*
* q13 is otherwise unused here, so preserve the old Q
* there and XOR it back after the normal x2 transition.
*/
asm volatile (
"vmov q13, q1\n"
"vshr.s8 q11, q1, #7\n"
"vshl.i8 q1, q1, #1\n"
"vand q11, q11, q14\n"
"veor q1, q1, q11\n"
"vld1.8 {q6}, %0\n"
"veor q0, q0, q6\n"
"veor q1, q1, q6\n"
/* complete Q *= 3 */
"veor q1, q1, q13\n"
:
: "Q" (v[d][i])
);
if (np >= 3) {
asm volatile (
"vshr.u8 q8, q6, #4\n"
"vand q7, q6, q15\n"
"vand q8, q8, q15\n"
);
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q2, q2, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][1][0][0]),
"Q" (raid_gfcauchypshufb[d][1][1][0])
);
}
if (np >= 4) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q3, q3, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][2][0][0]),
"Q" (raid_gfcauchypshufb[d][2][1][0])
);
}
if (np >= 5) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q4, q4, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][3][0][0]),
"Q" (raid_gfcauchypshufb[d][3][1][0])
);
}
if (np >= 6) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q5, q5, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][4][0][0]),
"Q" (raid_gfcauchypshufb[d][4][1][0])
);
}
--d;
boundary -= 51;
if (boundary < 50)
boundary = -1;
}
/*
* Remaining intermediate disks contain no more G23
* boundaries, so this loop is pure x2.
*/
for (; d > 0; --d) {
asm volatile (
"vshr.s8 q11, q1, #7\n"
"vshl.i8 q1, q1, #1\n"
"vand q11, q11, q14\n"
"veor q1, q1, q11\n"
"vld1.8 {q6}, %0\n"
"veor q0, q0, q6\n"
"veor q1, q1, q6\n"
:
: "Q" (v[d][i])
);
if (np >= 3) {
asm volatile (
"vshr.u8 q8, q6, #4\n"
"vand q7, q6, q15\n"
"vand q8, q8, q15\n"
);
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q2, q2, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][1][0][0]),
"Q" (raid_gfcauchypshufb[d][1][1][0])
);
}
if (np >= 4) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q3, q3, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][2][0][0]),
"Q" (raid_gfcauchypshufb[d][2][1][0])
);
}
if (np >= 5) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q4, q4, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][3][0][0]),
"Q" (raid_gfcauchypshufb[d][3][1][0])
);
}
if (np >= 6) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q5, q5, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][4][0][0]),
"Q" (raid_gfcauchypshufb[d][4][1][0])
);
}
}
/*
* D0 always follows Q *= 2. The lowest G23 boundary is 50.
*/
asm volatile (
"vshr.s8 q11, q1, #7\n"
"vshl.i8 q1, q1, #1\n"
"vand q11, q11, q14\n"
"veor q1, q1, q11\n"
"vld1.8 {q6}, %0\n"
"veor q0, q0, q6\n"
"veor q1, q1, q6\n"
:
: "Q" (v[0][i])
);
if (np >= 3)
asm volatile ("veor q2, q2, q6\n");
if (np >= 4)
asm volatile ("veor q3, q3, q6\n");
if (np >= 5)
asm volatile ("veor q4, q4, q6\n");
if (np >= 6)
asm volatile ("veor q5, q5, q6\n");
/* write parity in increasing order */
asm volatile (
"vst1.8 {q0}, %0\n"
"vst1.8 {q1}, %1\n"
: "=Q" (v[nd][i]),
"=Q" (v[nd + 1][i])
);
if (np >= 3)
asm volatile ("vst1.8 {q2}, %0\n" : "=Q" (v[nd + 2][i]));
if (np >= 4)
asm volatile ("vst1.8 {q3}, %0\n" : "=Q" (v[nd + 3][i]));
if (np >= 5)
asm volatile ("vst1.8 {q4}, %0\n" : "=Q" (v[nd + 4][i]));
if (np >= 6)
asm volatile ("vst1.8 {q5}, %0\n" : "=Q" (v[nd + 5][i]));
}
raid_neon32_end();
}
/*
* GEN3 (triple parity with Cauchy matrix) AArch32 NEON implementation
*/
void raid_gen3_neon32_raid(int nd, size_t size, void **vv)
{
raid_genX_neon32_x2(nd, size, vv, 3);
}
void raid_gen3_neon32_aes(int nd, size_t size, void **vv)
{
if (raid_g23_x2_only(nd))
raid_genX_neon32_x2(nd, size, vv, 3);
else
raid_genX_neon32_g23(nd, size, vv, 3);
}
/*
* GEN4 (quad parity with Cauchy matrix) AArch32 NEON implementation
*/
void raid_gen4_neon32_raid(int nd, size_t size, void **vv)
{
raid_genX_neon32_x2(nd, size, vv, 4);
}
void raid_gen4_neon32_aes(int nd, size_t size, void **vv)
{
if (raid_g23_x2_only(nd))
raid_genX_neon32_x2(nd, size, vv, 4);
else
raid_genX_neon32_g23(nd, size, vv, 4);
}
/*
* GEN5 (penta parity with Cauchy matrix) AArch32 NEON implementation
*/
void raid_gen5_neon32_raid(int nd, size_t size, void **vv)
{
raid_genX_neon32_x2(nd, size, vv, 5);
}
void raid_gen5_neon32_aes(int nd, size_t size, void **vv)
{
if (raid_g23_x2_only(nd))
raid_genX_neon32_x2(nd, size, vv, 5);
else
raid_genX_neon32_g23(nd, size, vv, 5);
}
/*
* GEN6 (hexa parity with Cauchy matrix) AArch32 NEON implementation
*/
void raid_gen6_neon32_raid(int nd, size_t size, void **vv)
{
raid_genX_neon32_x2(nd, size, vv, 6);
}
void raid_gen6_neon32_aes(int nd, size_t size, void **vv)
{
if (raid_g23_x2_only(nd))
raid_genX_neon32_x2(nd, size, vv, 6);
else
raid_genX_neon32_g23(nd, size, vv, 6);
}
/*
* RAID recovering for one disk AArch32 NEON implementation
*/
void raid_rec1_neon32(int nr, int *id, int *ip, int nd,
size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
uint8_t *pa;
uint8_t G;
uint8_t V;
size_t i;
(void)nr; /* unused, it's always 1 */
/* if it's RAID5 use the faster function */
if (ip[0] == 0) {
raid_rec1of1(id, nd, size, vv);
return;
}
/* setup the coefficients matrix */
G = A(ip[0], id[0]);
/* invert it to solve the system of linear equations */
V = inv(G);
/* compute delta parity */
raid_delta_gen(1, id, ip, nd, size, vv);
p = v[nd + ip[0]];
pa = v[id[0]];
raid_neon32_begin();
/*
* Preload the mask and multiplication tables, just as neon.c does.
*
* q12 = low table
* q13 = high table
* q14 = low4 mask
*/
asm volatile (
"vld1.8 {q14}, %0\n"
"vld1.8 {q12}, %1\n"
"vld1.8 {q13}, %2\n"
:
: "Q" (gfconst16.low4[0]),
"Q" (raid_gfmulpshufb[V][0][0]),
"Q" (raid_gfmulpshufb[V][1][0])
);
for (i = 0; i < size; i += 32) {
asm volatile (
"vld1.8 {q0}, %2\n"
"vld1.8 {q1}, %3\n"
"vld1.8 {q2}, %4\n"
"vld1.8 {q3}, %5\n"
"veor q0, q0, q2\n"
"veor q1, q1, q3\n"
"vshr.u8 q4, q0, #4\n"
"vshr.u8 q5, q1, #4\n"
"vand q0, q0, q14\n"
"vand q1, q1, q14\n"
"vand q4, q4, q14\n"
"vand q5, q5, q14\n"
/* low nibbles */
"vtbl.8 d12, {d24-d25}, d0\n"
"vtbl.8 d13, {d24-d25}, d1\n"
"vtbl.8 d14, {d24-d25}, d2\n"
"vtbl.8 d15, {d24-d25}, d3\n"
/* high nibbles */
"vtbl.8 d16, {d26-d27}, d8\n"
"vtbl.8 d17, {d26-d27}, d9\n"
"vtbl.8 d18, {d26-d27}, d10\n"
"vtbl.8 d19, {d26-d27}, d11\n"
"veor q6, q6, q8\n"
"veor q7, q7, q9\n"
"vst1.8 {q6}, %0\n"
"vst1.8 {q7}, %1\n"
: "=Q" (pa[i]), "=Q" (pa[i + 16])
: "Q" (p[i]), "Q" (p[i + 16]),
"Q" (pa[i]), "Q" (pa[i + 16])
);
}
raid_neon32_end();
}
/*
* RAID recovering for two disks AArch32 NEON implementation
*/
void raid_rec2_neon32(int nr, int *id, int *ip, int nd,
size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
const int N = 2;
uint8_t *p[2];
uint8_t *pa[2];
uint8_t G[2 * 2];
uint8_t V[2 * 2];
size_t i;
int j, k;
(void)nr; /* unused, it's always 2 */
/* setup the coefficients matrix */
for (j = 0; j < N; ++j)
for (k = 0; k < N; ++k)
G[j * N + k] = A(ip[j], id[k]);
/* invert it to solve the system of linear equations */
raid_invert(G, V, N);
/* compute delta parity */
raid_delta_gen(N, id, ip, nd, size, vv);
for (j = 0; j < N; ++j) {
p[j] = v[nd + ip[j]];
pa[j] = v[id[j]];
}
raid_neon32_begin();
/*
* q4-q11 contain the four low/high multiplication table pairs.
* q15 contains low4.
*/
asm volatile (
"vld1.8 {q15}, %0\n"
"vld1.8 {q4}, %1\n"
"vld1.8 {q5}, %2\n"
"vld1.8 {q6}, %3\n"
"vld1.8 {q7}, %4\n"
"vld1.8 {q8}, %5\n"
"vld1.8 {q9}, %6\n"
"vld1.8 {q10}, %7\n"
"vld1.8 {q11}, %8\n"
:
: "Q" (gfconst16.low4[0]),
"Q" (raid_gfmulpshufb[V[0]][0][0]),
"Q" (raid_gfmulpshufb[V[0]][1][0]),
"Q" (raid_gfmulpshufb[V[1]][0][0]),
"Q" (raid_gfmulpshufb[V[1]][1][0]),
"Q" (raid_gfmulpshufb[V[2]][0][0]),
"Q" (raid_gfmulpshufb[V[2]][1][0]),
"Q" (raid_gfmulpshufb[V[3]][0][0]),
"Q" (raid_gfmulpshufb[V[3]][1][0])
);
for (i = 0; i < size; i += 16) {
asm volatile (
/* delta 0 */
"vld1.8 {q0}, %2\n"
"vld1.8 {q12}, %4\n"
"veor q0, q0, q12\n"
"vshr.u8 q1, q0, #4\n"
"vand q0, q0, q15\n"
"vand q1, q1, q15\n"
/* delta 1 */
"vld1.8 {q2}, %3\n"
"vld1.8 {q12}, %5\n"
"veor q2, q2, q12\n"
"vshr.u8 q3, q2, #4\n"
"vand q2, q2, q15\n"
"vand q3, q3, q15\n"
/* pa[0] = V[0] * delta0 */
"vtbl.8 d24, {d8-d9}, d0\n"
"vtbl.8 d25, {d8-d9}, d1\n"
"vtbl.8 d26, {d10-d11}, d2\n"
"vtbl.8 d27, {d10-d11}, d3\n"
"veor q14, q12, q13\n"
/* ^ V[1] * delta1 */
"vtbl.8 d24, {d12-d13}, d4\n"
"vtbl.8 d25, {d12-d13}, d5\n"
"vtbl.8 d26, {d14-d15}, d6\n"
"vtbl.8 d27, {d14-d15}, d7\n"
"veor q12, q12, q13\n"
"veor q14, q14, q12\n"
"vst1.8 {q14}, %0\n"
/* pa[1] = V[2] * delta0 */
"vtbl.8 d24, {d16-d17}, d0\n"
"vtbl.8 d25, {d16-d17}, d1\n"
"vtbl.8 d26, {d18-d19}, d2\n"
"vtbl.8 d27, {d18-d19}, d3\n"
"veor q14, q12, q13\n"
/* ^ V[3] * delta1 */
"vtbl.8 d24, {d20-d21}, d4\n"
"vtbl.8 d25, {d20-d21}, d5\n"
"vtbl.8 d26, {d22-d23}, d6\n"
"vtbl.8 d27, {d22-d23}, d7\n"
"veor q12, q12, q13\n"
"veor q14, q14, q12\n"
"vst1.8 {q14}, %1\n"
: "=Q" (pa[0][i]), "=Q" (pa[1][i])
: "Q" (p[0][i]), "Q" (p[1][i]),
"Q" (pa[0][i]), "Q" (pa[1][i])
);
}
raid_neon32_end();
}
/*
* RAID recovering AArch32 NEON implementation
*/
void raid_recX_neon32(int nr, int *id, int *ip, int nd,
size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
int N = nr;
uint8_t *p[RAID_PARITY_MAX];
uint8_t *pa[RAID_PARITY_MAX];
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
const uint8_t *T[RAID_PARITY_MAX * RAID_PARITY_MAX];
uint8_t D[RAID_PARITY_MAX][2][16] __aligned(16);
size_t i;
int j, k;
/* setup the coefficients matrix */
for (j = 0; j < N; ++j)
for (k = 0; k < N; ++k)
G[j * N + k] = A(ip[j], id[k]);
/* invert it to solve the system of linear equations */
raid_invert(G, V, N);
/* precompute shuffle table pointers */
for (j = 0; j < N * N; ++j)
T[j] = &raid_gfmulpshufb[V[j]][0][0];
/* compute delta parity */
raid_delta_gen(N, id, ip, nd, size, vv);
for (j = 0; j < N; ++j) {
p[j] = v[nd + ip[j]];
pa[j] = v[id[j]];
}
raid_neon32_begin();
/* preload mask */
asm volatile (
"vld1.8 {q15}, %0\n"
:
: "Q" (gfconst16.low4[0])
);
for (i = 0; i < size; i += 16) {
for (k = 0; k < N; ++k) {
asm volatile (
"vld1.8 {q0}, %2\n"
"vld1.8 {q1}, %3\n"
"veor q0, q0, q1\n"
"vshr.u8 q1, q0, #4\n"
"vand q0, q0, q15\n"
"vand q1, q1, q15\n"
"vst1.8 {q0}, %0\n"
"vst1.8 {q1}, %1\n"
: "=Q" (D[k][0][0]), "=Q" (D[k][1][0])
: "Q" (p[k][i]), "Q" (pa[k][i])
);
}
/* reconstruct */
for (j = 0; j < N; ++j) {
asm volatile (
"veor q0, q0, q0\n"
);
for (k = 0; k < N; ++k) {
asm volatile (
"vld1.8 {q2}, %0\n"
"vld1.8 {q3}, %1\n"
"vld1.8 {q4}, %2\n"
"vld1.8 {q5}, %3\n"
"vtbl.8 d12, {d8-d9}, d4\n"
"vtbl.8 d13, {d8-d9}, d5\n"
"vtbl.8 d14, {d10-d11}, d6\n"
"vtbl.8 d15, {d10-d11}, d7\n"
"veor q6, q6, q7\n"
"veor q0, q0, q6\n"
:
: "Q" (D[k][0][0]),
"Q" (D[k][1][0]),
"Q" (T[j * N + k][0]),
"Q" (T[j * N + k][16])
);
}
asm volatile (
"vst1.8 {q0}, %0\n"
: "=Q" (pa[j][i])
);
}
}
raid_neon32_end();
}
void raid_register_neon32(void)
{
raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "neon32", raid_gen1_neon32, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "neon32", raid_gen2_neon32_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "neon32", raid_gen2_neon32_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "neon32", raid_genz_neon32_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "neon32", raid_gen3_neon32_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "neon32", raid_gen3_neon32_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "neon32", raid_gen4_neon32_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "neon32", raid_gen4_neon32_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "neon32", raid_gen5_neon32_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "neon32", raid_gen5_neon32_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "neon32", raid_gen6_neon32_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "neon32", raid_gen6_neon32_aes, RAID_POLY_AES);
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "neon32", raid_rec1_neon32, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "neon32", raid_rec2_neon32, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "neon32", raid_recX_neon32, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "neon32", raid_recX_neon32, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "neon32", raid_recX_neon32, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "neon32", raid_recX_neon32, RAID_POLY_ANY);
}
#endif
Reference in a new issue View git blame Copy permalink