rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-25 18:40:19 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add AArch32 (ARM32) NEON SIMD support for parity generation and recovery

Browse source
This commit is contained in:
Andrea Mazzoleni 2026-08-20 21:47:31 +02:00
parent e5ca862a37
commit 5c24968e6d
13 changed files with 1396 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

6
HISTORY
View file

@ -22,9 +22,9 @@ SnapRAID HISTORY
By scaling to 512-bit ZMM registers, the implementation now processes
64 bytes per iteration, effectively doubling the data width of the
previous AVX2 routines.
* Implemented AArch64 (ARM64) NEON support for parity generation and
data recovery, significantly improving performance on ARM-based
platforms.
* Implemented AArch64 (ARM64) and AArch32 (ARM32) NEON support for
parity generation and data recovery, significantly improving
performance on ARM-based platforms.
* Improved the chances of a successful recovery after an interrupted
sync. This is achieved by removing the sync optimization of not
recomputing the parity if the input data matches the expected hash.

1
Makefile.am
View file

@ -34,6 +34,7 @@ snapraid_SOURCES = \
raid/avx512.c \
raid/gfni.c \
raid/neon.c \
raid/neon32.c \
raid/helper.c \
raid/memory.c \
raid/test.c \

179
cmdline/speed.c
View file

@ -207,6 +207,9 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
#ifdef CONFIG_NEON
printf("%8s", "neon");
#endif
#ifdef CONFIG_NEON32
printf("%8s", "neon32");
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_sse2()) {
printf("%8s", "sse2");
@ -266,6 +269,14 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen1_neon32(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_sse2()) {
@ -370,6 +381,20 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
if (mode == RAID_MODE_CAUCHY_AES) {
SPEED_START {
raid_gen2_neon32_aes(nd, size, v);
} SPEED_STOP
} else {
SPEED_START {
raid_gen2_neon32_raid(nd, size, v);
} SPEED_STOP
}
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_sse2()) {
@ -510,6 +535,20 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
if (mode == RAID_MODE_CAUCHY_AES) {
SPEED_START {
raid_gen3_neon32_aes(nd, size, v);
} SPEED_STOP
} else {
SPEED_START {
raid_gen3_neon32_raid(nd, size, v);
} SPEED_STOP
}
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_sse2()) {
@ -639,6 +678,20 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
if (mode == RAID_MODE_CAUCHY_AES) {
SPEED_START {
raid_gen4_neon32_aes(nd, size, v);
} SPEED_STOP
} else {
SPEED_START {
raid_gen4_neon32_raid(nd, size, v);
} SPEED_STOP
}
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_sse2()) {
@ -768,6 +821,20 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
if (mode == RAID_MODE_CAUCHY_AES) {
SPEED_START {
raid_gen5_neon32_aes(nd, size, v);
} SPEED_STOP
} else {
SPEED_START {
raid_gen5_neon32_raid(nd, size, v);
} SPEED_STOP
}
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_sse2()) {
@ -894,6 +961,20 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
if (mode == RAID_MODE_CAUCHY_AES) {
SPEED_START {
raid_gen6_neon32_aes(nd, size, v);
} SPEED_STOP
} else {
SPEED_START {
raid_gen6_neon32_raid(nd, size, v);
} SPEED_STOP
}
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_sse2()) {
@ -1012,6 +1093,9 @@ void speed_genz(int nd, void** v, int size, int delta, int period)
#ifdef CONFIG_NEON
printf("%8s", "neon");
#endif
#ifdef CONFIG_NEON32
printf("%8s", "neon32");
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_sse2()) {
printf("%8s", "sse2");
@ -1063,6 +1147,14 @@ void speed_genz(int nd, void** v, int size, int delta, int period)
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_genz_neon32_raid(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_sse2()) {
@ -1130,6 +1222,9 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#ifdef CONFIG_NEON
printf("%8s", "neon");
#endif
#ifdef CONFIG_NEON32
printf("%8s", "neon32");
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3())
printf("%8s", "ssse3");
@ -1170,6 +1265,16 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, raid_gen1_neon32);
raid_rec1_neon32(1, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
@ -1253,6 +1358,17 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, raid_gen1_neon32);
/* +1 to avoid GEN1 optimized case */
raid_rec1_neon32(1, id, ip + 1, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
@ -1339,6 +1455,16 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_neon32_raid);
raid_rec2_neon32(2, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
@ -1430,6 +1556,17 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_neon32_raid);
/* +1 to avoid GEN2 optimized case */
raid_rec2_neon32(2, id, ip + 1, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
@ -1523,6 +1660,16 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(3, raid_gen3_neon32_raid);
raid_recX_neon32(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
@ -1612,6 +1759,16 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(4, raid_gen4_neon32_raid);
raid_recX_neon32(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
@ -1702,6 +1859,16 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(5, raid_gen5_neon32_raid);
raid_recX_neon32(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
@ -1791,6 +1958,16 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(6, raid_gen6_neon32_raid);
raid_recX_neon32(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
@ -2045,7 +2222,7 @@ void speed(int period, int nd, int size)
printf("CPU 64-bit ARM (AArch64)\n");
#endif
#elif defined(__arm__)
#if defined(CONFIG_NEON)
#if defined(CONFIG_NEON32)
printf("CPU 32-bit ARM, flags neon\n");
#else
printf("CPU 32-bit ARM\n");

3
raid/README
View file

@ -150,7 +150,8 @@ Implementation Structure
avx2.c x86 AVX2 (256-bit vectorization)
avx512.c x86 AVX-512BW (512-bit vectorization)
gfni.c x86 GFNI generation/recovery (RAID affine and AES multiply)
neon.c ARM NEON (128-bit SIMD vectorization)
neon.c AArch64 ARM NEON (128-bit SIMD vectorization)
neon32.c AArch32 ARM NEON (128-bit SIMD vectorization)
Testing and Benchmarks

54
raid/internal.h
View file

@ -23,11 +23,16 @@
#define CONFIG_X86 1
#define CONFIG_X86_64 1
#endif
#if defined(__arm__) || defined(__aarch64__)
#if defined(__aarch64__)
#if defined(__ARM_NEON) || defined(__ARM_NEON__)
#define CONFIG_NEON 1
#endif
#endif
#if defined(__arm__)
#if defined(__ARM_NEON) || defined(__ARM_NEON__)
#define CONFIG_NEON32 1
#endif
#endif
#endif
/*
@ -198,6 +203,21 @@ void raid_gen6_neon_aes(int nd, size_t size, void **vv);
void raid_rec1_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec2_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_gen1_neon32(int nd, size_t size, void **vv);
void raid_gen2_neon32_raid(int nd, size_t size, void **vv);
void raid_gen2_neon32_aes(int nd, size_t size, void **vv);
void raid_genz_neon32_raid(int nd, size_t size, void **vv);
void raid_gen3_neon32_raid(int nd, size_t size, void **vv);
void raid_gen3_neon32_aes(int nd, size_t size, void **vv);
void raid_gen4_neon32_raid(int nd, size_t size, void **vv);
void raid_gen4_neon32_aes(int nd, size_t size, void **vv);
void raid_gen5_neon32_raid(int nd, size_t size, void **vv);
void raid_gen5_neon32_aes(int nd, size_t size, void **vv);
void raid_gen6_neon32_raid(int nd, size_t size, void **vv);
void raid_gen6_neon32_aes(int nd, size_t size, void **vv);
void raid_rec1_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec2_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_recX_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv);
/*
* Functions for parity computation.
@ -323,7 +343,7 @@ const char * raid_rec_tag(int na);
*/
int raid_selftest(void);
#if defined(CONFIG_X86) || defined(CONFIG_NEON)
#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32)
struct gfconst16 {
uint8_t poly[16];
uint8_t low4[16];
@ -342,7 +362,7 @@ extern const uint8_t raid_gfinv_raid[256] __aligned(256);
extern const uint8_t raid_gfvandermonde_raid[3][256] __aligned(256);
extern const uint8_t raid_gfcauchy_raid[6][256] __aligned(256);
extern const uint8_t raid_gfaffine_raid[256][8] __aligned(256);
#if defined(CONFIG_X86) || defined(CONFIG_NEON)
#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32)
extern const uint8_t raid_gfcauchypshufb_raid[251][5][2][16] __aligned(256);
extern const uint8_t raid_gfmulpshufb_raid[256][2][16] __aligned(256);
#endif
@ -351,7 +371,7 @@ extern const uint8_t raid_gfmul_aes[256][256] __aligned(256);
extern const uint8_t raid_gfexp_aes[256] __aligned(256);
extern const uint8_t raid_gfinv_aes[256] __aligned(256);
extern const uint8_t raid_gfcauchy_aes[6][256] __aligned(256);
#if defined(CONFIG_X86) || defined(CONFIG_NEON)
#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32)
extern const uint8_t raid_gfcauchypshufb_aes[251][5][2][16] __aligned(256);
extern const uint8_t raid_gfmulpshufb_aes[256][2][16] __aligned(256);
#endif
@ -361,7 +381,7 @@ extern const uint8_t *raid_gfexp;
extern const uint8_t *raid_gfinv;
extern const uint8_t(*raid_gfvandermonde)[256];
extern const uint8_t(*raid_gfcauchy)[256];
#if defined(CONFIG_X86) || defined(CONFIG_NEON)
#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32)
extern const uint8_t(*raid_gfcauchypshufb)[5][2][16];
extern const uint8_t(*raid_gfmulpshufb)[2][16];
#endif
@ -475,4 +495,28 @@ static __always_inline void raid_neon_end(void)
void raid_register_neon(void);
#endif
#ifdef CONFIG_NEON32
static __always_inline void raid_neon32_begin(void)
{
}
static __always_inline void raid_neon32_end(void)
{
/*
* Clobbers registers used in AArch32 NEON asm operations.
*
* Like raid_neon_end(), the inner asm statements intentionally
* preserve hidden NEON state across adjacent asm blocks. Keep the
* dummy clobbers at the end of the function so the compiler preserves
* the AAPCS32 callee-saved NEON registers when necessary.
*/
asm volatile ("" : : : "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7");
asm volatile ("" : : : "d8", "d9", "d10", "d11", "d12", "d13", "d14", "d15");
asm volatile ("" : : : "d16", "d17", "d18", "d19", "d20", "d21", "d22", "d23");
asm volatile ("" : : : "d24", "d25", "d26", "d27", "d28", "d29", "d30", "d31");
}
void raid_register_neon32(void);
#endif
#endif

4
raid/mktables.c
View file

@ -352,7 +352,7 @@ void tables(uint8_t poly, uint8_t generator, const char *tag)
}
printf("};\n\n");
printf("#if defined(CONFIG_X86) || defined(CONFIG_NEON)\n");
printf("#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32)\n");
printf("/**\n");
printf(" * PSHUFB tables for the Cauchy matrix.\n");
printf(" *\n");
@ -384,7 +384,7 @@ void tables(uint8_t poly, uint8_t generator, const char *tag)
printf("};\n");
printf("#endif\n\n");
printf("#if defined(CONFIG_X86) || defined(CONFIG_NEON)\n");
printf("#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32)\n");
printf("/**\n");
printf(" * PSHUFB tables for generic multiplication.\n");
printf(" *\n");

13
raid/module.c
View file

@ -44,7 +44,7 @@ const uint8_t *raid_gfexp;
const uint8_t *raid_gfinv;
const uint8_t(*raid_gfvandermonde)[256];
const uint8_t(*raid_gfcauchy)[256];
#if defined(CONFIG_X86) || defined(CONFIG_NEON)
#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32)
const uint8_t(*raid_gfcauchypshufb)[5][2][16];
const uint8_t(*raid_gfmulpshufb)[2][16];
#endif
@ -57,7 +57,7 @@ uint64_t raid_poly_64;
uint32_t raid_inv2_32;
uint64_t raid_inv2_64;
#if defined(CONFIG_X86) || defined(CONFIG_NEON)
#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32)
struct gfconst16 __aligned(16) gfconst16 =
{
{ 0 },
@ -229,7 +229,7 @@ int raid_mode(int mode)
raid_gfinv = raid_gfinv_aes;
raid_gfvandermonde = 0;
raid_gfcauchy = raid_gfcauchy_aes;
#if defined(CONFIG_X86) || defined(CONFIG_NEON)
#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32)
raid_gfcauchypshufb = raid_gfcauchypshufb_aes;
raid_gfmulpshufb = raid_gfmulpshufb_aes;
#endif
@ -249,7 +249,7 @@ int raid_mode(int mode)
raid_gfinv = raid_gfinv_raid;
raid_gfvandermonde = raid_gfvandermonde_raid;
raid_gfcauchy = raid_gfcauchy_raid;
#if defined(CONFIG_X86) || defined(CONFIG_NEON)
#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32)
raid_gfcauchypshufb = raid_gfcauchypshufb_raid;
raid_gfmulpshufb = raid_gfmulpshufb_raid;
#endif
@ -265,7 +265,7 @@ int raid_mode(int mode)
raid_rec_algo = raid_rec_algo_raid;
}
#if defined(CONFIG_X86) || defined(CONFIG_NEON)
#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32)
memset(gfconst16.poly, raid_poly_byte, 16);
memset(gfconst16.half, raid_inv2_byte, 16);
#endif
@ -361,6 +361,9 @@ void raid_init(void)
#ifdef CONFIG_NEON
raid_register_neon();
#endif
#ifdef CONFIG_NEON32
raid_register_neon32();
#endif
/* set the default mode */
raid_mode(RAID_MODE_CAUCHY_RAID);

941
raid/neon32.c Normal file
View file

@ -0,0 +1,941 @@
// SPDX-License-Identifier: GPL-2.0-or-later
// Copyright (C) 2026 Andrea Mazzoleni
#include "internal.h"
#include "gf.h"
#include "cpu.h"
#ifdef CONFIG_NEON32
/*
* GEN1 (RAID5 with xor) AArch32 NEON implementation
*/
void raid_gen1_neon32(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
int d, l;
size_t i;
l = nd - 1;
p = v[nd];
raid_neon32_begin();
for (i = 0; i < size; i += 64) {
asm volatile (
"vld1.8 {q0}, %0\n"
"vld1.8 {q1}, %1\n"
"vld1.8 {q2}, %2\n"
"vld1.8 {q3}, %3\n"
:
: "Q" (v[0][i]), "Q" (v[0][i + 16]),
"Q" (v[0][i + 32]), "Q" (v[0][i + 48])
);
for (d = 1; d <= l; ++d) {
asm volatile (
"vld1.8 {q4}, %0\n"
"vld1.8 {q5}, %1\n"
"vld1.8 {q6}, %2\n"
"vld1.8 {q7}, %3\n"
"veor q0, q0, q4\n"
"veor q1, q1, q5\n"
"veor q2, q2, q6\n"
"veor q3, q3, q7\n"
:
: "Q" (v[d][i]), "Q" (v[d][i + 16]),
"Q" (v[d][i + 32]), "Q" (v[d][i + 48])
);
}
asm volatile (
"vst1.8 {q0}, %0\n"
"vst1.8 {q1}, %1\n"
"vst1.8 {q2}, %2\n"
"vst1.8 {q3}, %3\n"
: "=Q" (p[i]), "=Q" (p[i + 16]),
"=Q" (p[i + 32]), "=Q" (p[i + 48])
);
}
raid_neon32_end();
}
/*
* GEN2 Cauchy AArch32 NEON implementation using the active generator.
*/
static __always_inline void raid_gen2_neon32_gen(int nd, size_t size,
void **vv, int generator)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
uint8_t *q;
int d, l;
size_t i;
l = nd - 1;
p = v[nd];
q = v[nd + 1];
raid_neon32_begin();
/* q14 contains the active reduction polynomial. */
asm volatile (
"vld1.8 {q14}, %0\n"
:
: "Q" (gfconst16.poly[0])
);
for (i = 0; i < size; i += 32) {
asm volatile (
"vld1.8 {q0}, %0\n"
"vld1.8 {q1}, %1\n"
"vmov q2, q0\n"
"vmov q3, q1\n"
:
: "Q" (v[l][i]), "Q" (v[l][i + 16])
);
for (d = l - 1; d >= 0; --d) {
if (generator == 3) {
asm volatile (
"vmov q10, q2\n"
"vmov q11, q3\n"
);
}
asm volatile (
"vshr.s8 q8, q2, #7\n"
"vshl.i8 q2, q2, #1\n"
"vand q8, q8, q14\n"
"veor q2, q2, q8\n"
"vshr.s8 q8, q3, #7\n"
"vshl.i8 q3, q3, #1\n"
"vand q8, q8, q14\n"
"veor q3, q3, q8\n"
"vld1.8 {q12}, %0\n"
"vld1.8 {q13}, %1\n"
"veor q0, q0, q12\n"
"veor q1, q1, q13\n"
"veor q2, q2, q12\n"
"veor q3, q3, q13\n"
:
: "Q" (v[d][i]), "Q" (v[d][i + 16])
);
if (generator == 3) {
asm volatile (
"veor q2, q2, q10\n"
"veor q3, q3, q11\n"
);
}
}
asm volatile (
"vst1.8 {q0}, %0\n"
"vst1.8 {q1}, %1\n"
"vst1.8 {q2}, %2\n"
"vst1.8 {q3}, %3\n"
: "=Q" (p[i]), "=Q" (p[i + 16]),
"=Q" (q[i]), "=Q" (q[i + 16])
);
}
raid_neon32_end();
}
void raid_gen2_neon32_raid(int nd, size_t size, void **vv)
{
raid_gen2_neon32_gen(nd, size, vv, 2);
}
void raid_gen2_neon32_aes(int nd, size_t size, void **vv)
{
raid_gen2_neon32_gen(nd, size, vv, 3);
}
/*
* GENz (triple parity with powers of 2^-1) AArch32 NEON implementation
*/
void raid_genz_neon32_raid(int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
uint8_t *q;
uint8_t *r;
int d, l;
size_t i;
l = nd - 1;
p = v[nd];
q = v[nd + 1];
r = v[nd + 2];
raid_neon32_begin();
asm volatile (
"vld1.8 {q14}, %0\n"
"vld1.8 {q15}, %1\n"
:
: "Q" (gfconst16.poly[0]), "Q" (gfconst16.half[0])
);
for (i = 0; i < size; i += 32) {
asm volatile (
"vld1.8 {q0}, %0\n"
"vld1.8 {q1}, %1\n"
"vmov q2, q0\n"
"vmov q3, q1\n"
"vmov q4, q0\n"
"vmov q5, q1\n"
:
: "Q" (v[l][i]), "Q" (v[l][i + 16])
);
for (d = l - 1; d >= 0; --d) {
asm volatile (
"vshr.s8 q8, q2, #7\n"
"vshl.i8 q2, q2, #1\n"
"vand q8, q8, q14\n"
"veor q2, q2, q8\n"
"vshr.s8 q8, q3, #7\n"
"vshl.i8 q3, q3, #1\n"
"vand q8, q8, q14\n"
"veor q3, q3, q8\n"
"vshl.i8 q9, q4, #7\n"
"vshr.s8 q9, q9, #7\n"
"vshr.u8 q4, q4, #1\n"
"vand q9, q9, q15\n"
"veor q4, q4, q9\n"
"vshl.i8 q9, q5, #7\n"
"vshr.s8 q9, q9, #7\n"
"vshr.u8 q5, q5, #1\n"
"vand q9, q9, q15\n"
"veor q5, q5, q9\n"
"vld1.8 {q12}, %0\n"
"vld1.8 {q13}, %1\n"
"veor q0, q0, q12\n"
"veor q1, q1, q13\n"
"veor q2, q2, q12\n"
"veor q3, q3, q13\n"
"veor q4, q4, q12\n"
"veor q5, q5, q13\n"
:
: "Q" (v[d][i]), "Q" (v[d][i + 16])
);
}
asm volatile (
"vst1.8 {q0}, %0\n"
"vst1.8 {q1}, %1\n"
"vst1.8 {q2}, %2\n"
"vst1.8 {q3}, %3\n"
"vst1.8 {q4}, %4\n"
"vst1.8 {q5}, %5\n"
: "=Q" (p[i]), "=Q" (p[i + 16]),
"=Q" (q[i]), "=Q" (q[i + 16]),
"=Q" (r[i]), "=Q" (r[i + 16])
);
}
raid_neon32_end();
}
/*
* GENX AArch32 NEON implementation.
*/
static __always_inline void raid_genX_neon32(int nd, size_t size,
void **vv, int np, int generator)
{
uint8_t **v = (uint8_t **)vv;
size_t i;
int d, l;
l = nd - 1;
/* special case with only one data disk */
if (l == 0) {
for (d = 0; d < np; ++d)
memcpy(v[1 + d], v[0], size);
return;
}
raid_neon32_begin();
asm volatile (
"vld1.8 {q14}, %0\n"
"vld1.8 {q15}, %1\n"
:
: "Q" (gfconst16.poly[0]), "Q" (gfconst16.low4[0])
);
for (i = 0; i < size; i += 16) {
/* last disk without the generator multiplication */
asm volatile (
"vld1.8 {q0}, %0\n"
"vld1.8 {q1}, %0\n"
:
: "Q" (v[l][i])
);
if (np >= 3) {
asm volatile (
"vshr.u8 q8, q0, #4\n"
"vand q7, q0, q15\n"
"vand q8, q8, q15\n"
);
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q2, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][1][0][0]),
"Q" (raid_gfcauchypshufb[l][1][1][0])
);
}
if (np >= 4) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q3, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][2][0][0]),
"Q" (raid_gfcauchypshufb[l][2][1][0])
);
}
if (np >= 5) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q4, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][3][0][0]),
"Q" (raid_gfcauchypshufb[l][3][1][0])
);
}
if (np >= 6) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q5, q11, q12\n"
:
: "Q" (raid_gfcauchypshufb[l][4][0][0]),
"Q" (raid_gfcauchypshufb[l][4][1][0])
);
}
/* intermediate disks */
for (d = l - 1; d > 0; --d) {
if (generator == 3) {
asm volatile (
"vmov q13, q1\n"
);
}
asm volatile (
"vshr.s8 q11, q1, #7\n"
"vshl.i8 q1, q1, #1\n"
"vand q11, q11, q14\n"
"veor q1, q1, q11\n"
"vld1.8 {q6}, %0\n"
"veor q0, q0, q6\n"
"veor q1, q1, q6\n"
:
: "Q" (v[d][i])
);
if (generator == 3) {
asm volatile (
"veor q1, q1, q13\n"
);
}
if (np >= 3) {
asm volatile (
"vshr.u8 q8, q6, #4\n"
"vand q7, q6, q15\n"
"vand q8, q8, q15\n"
);
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q2, q2, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][1][0][0]),
"Q" (raid_gfcauchypshufb[d][1][1][0])
);
}
if (np >= 4) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q3, q3, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][2][0][0]),
"Q" (raid_gfcauchypshufb[d][2][1][0])
);
}
if (np >= 5) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q4, q4, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][3][0][0]),
"Q" (raid_gfcauchypshufb[d][3][1][0])
);
}
if (np >= 6) {
asm volatile (
"vld1.8 {q9}, %0\n"
"vld1.8 {q10}, %1\n"
"vtbl.8 d22, {d18-d19}, d14\n"
"vtbl.8 d23, {d18-d19}, d15\n"
"vtbl.8 d24, {d20-d21}, d16\n"
"vtbl.8 d25, {d20-d21}, d17\n"
"veor q11, q11, q12\n"
"veor q5, q5, q11\n"
:
: "Q" (raid_gfcauchypshufb[d][4][0][0]),
"Q" (raid_gfcauchypshufb[d][4][1][0])
);
}
}
/* first disk with all coefficients at 1 */
if (generator == 3) {
asm volatile (
"vmov q13, q1\n"
);
}
asm volatile (
"vshr.s8 q11, q1, #7\n"
"vshl.i8 q1, q1, #1\n"
"vand q11, q11, q14\n"
"veor q1, q1, q11\n"
"vld1.8 {q6}, %0\n"
"veor q0, q0, q6\n"
"veor q1, q1, q6\n"
:
: "Q" (v[0][i])
);
if (generator == 3) {
asm volatile (
"veor q1, q1, q13\n"
);
}
if (np >= 3) {
asm volatile (
"veor q2, q2, q6\n"
);
}
if (np >= 4) {
asm volatile (
"veor q3, q3, q6\n"
);
}
if (np >= 5) {
asm volatile (
"veor q4, q4, q6\n"
);
}
if (np >= 6) {
asm volatile (
"veor q5, q5, q6\n"
);
}
/* write parity in increasing order */
asm volatile (
"vst1.8 {q0}, %0\n"
"vst1.8 {q1}, %1\n"
: "=Q" (v[nd][i]),
"=Q" (v[nd + 1][i])
);
if (np >= 3) {
asm volatile (
"vst1.8 {q2}, %0\n"
: "=Q" (v[nd + 2][i])
);
}
if (np >= 4) {
asm volatile (
"vst1.8 {q3}, %0\n"
: "=Q" (v[nd + 3][i])
);
}
if (np >= 5) {
asm volatile (
"vst1.8 {q4}, %0\n"
: "=Q" (v[nd + 4][i])
);
}
if (np >= 6) {
asm volatile (
"vst1.8 {q5}, %0\n"
: "=Q" (v[nd + 5][i])
);
}
}
raid_neon32_end();
}
/*
* GEN3 (triple parity with Cauchy matrix) AArch32 NEON implementation
*/
void raid_gen3_neon32_raid(int nd, size_t size, void **vv)
{
raid_genX_neon32(nd, size, vv, 3, 2);
}
void raid_gen3_neon32_aes(int nd, size_t size, void **vv)
{
raid_genX_neon32(nd, size, vv, 3, 3);
}
/*
* GEN4 (quad parity with Cauchy matrix) AArch32 NEON implementation
*/
void raid_gen4_neon32_raid(int nd, size_t size, void **vv)
{
raid_genX_neon32(nd, size, vv, 4, 2);
}
void raid_gen4_neon32_aes(int nd, size_t size, void **vv)
{
raid_genX_neon32(nd, size, vv, 4, 3);
}
/*
* GEN5 (penta parity with Cauchy matrix) AArch32 NEON implementation
*/
void raid_gen5_neon32_raid(int nd, size_t size, void **vv)
{
raid_genX_neon32(nd, size, vv, 5, 2);
}
void raid_gen5_neon32_aes(int nd, size_t size, void **vv)
{
raid_genX_neon32(nd, size, vv, 5, 3);
}
/*
* GEN6 (hexa parity with Cauchy matrix) AArch32 NEON implementation
*/
void raid_gen6_neon32_raid(int nd, size_t size, void **vv)
{
raid_genX_neon32(nd, size, vv, 6, 2);
}
void raid_gen6_neon32_aes(int nd, size_t size, void **vv)
{
raid_genX_neon32(nd, size, vv, 6, 3);
}
/*
* RAID recovering for one disk AArch32 NEON implementation
*/
void raid_rec1_neon32(int nr, int *id, int *ip, int nd,
size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
uint8_t *pa;
uint8_t G;
uint8_t V;
size_t i;
(void)nr; /* unused, it's always 1 */
/* if it's RAID5 use the faster function */
if (ip[0] == 0) {
raid_rec1of1(id, nd, size, vv);
return;
}
/* setup the coefficients matrix */
G = A(ip[0], id[0]);
/* invert it to solve the system of linear equations */
V = inv(G);
/* compute delta parity */
raid_delta_gen(1, id, ip, nd, size, vv);
p = v[nd + ip[0]];
pa = v[id[0]];
raid_neon32_begin();
/*
* Preload the mask and multiplication tables, just as neon.c does.
*
* q12 = low table
* q13 = high table
* q14 = low4 mask
*/
asm volatile (
"vld1.8 {q14}, %0\n"
"vld1.8 {q12}, %1\n"
"vld1.8 {q13}, %2\n"
:
: "Q" (gfconst16.low4[0]),
"Q" (raid_gfmulpshufb[V][0][0]),
"Q" (raid_gfmulpshufb[V][1][0])
);
for (i = 0; i < size; i += 32) {
asm volatile (
"vld1.8 {q0}, %2\n"
"vld1.8 {q1}, %3\n"
"vld1.8 {q2}, %4\n"
"vld1.8 {q3}, %5\n"
"veor q0, q0, q2\n"
"veor q1, q1, q3\n"
"vshr.u8 q4, q0, #4\n"
"vshr.u8 q5, q1, #4\n"
"vand q0, q0, q14\n"
"vand q1, q1, q14\n"
"vand q4, q4, q14\n"
"vand q5, q5, q14\n"
/* low nibbles */
"vtbl.8 d12, {d24-d25}, d0\n"
"vtbl.8 d13, {d24-d25}, d1\n"
"vtbl.8 d14, {d24-d25}, d2\n"
"vtbl.8 d15, {d24-d25}, d3\n"
/* high nibbles */
"vtbl.8 d16, {d26-d27}, d8\n"
"vtbl.8 d17, {d26-d27}, d9\n"
"vtbl.8 d18, {d26-d27}, d10\n"
"vtbl.8 d19, {d26-d27}, d11\n"
"veor q6, q6, q8\n"
"veor q7, q7, q9\n"
"vst1.8 {q6}, %0\n"
"vst1.8 {q7}, %1\n"
: "=Q" (pa[i]), "=Q" (pa[i + 16])
: "Q" (p[i]), "Q" (p[i + 16]),
"Q" (pa[i]), "Q" (pa[i + 16])
);
}
raid_neon32_end();
}
/*
* RAID recovering for two disks AArch32 NEON implementation
*/
void raid_rec2_neon32(int nr, int *id, int *ip, int nd,
size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
const int N = 2;
uint8_t *p[2];
uint8_t *pa[2];
uint8_t G[2 * 2];
uint8_t V[2 * 2];
size_t i;
int j, k;
(void)nr; /* unused, it's always 2 */
/* setup the coefficients matrix */
for (j = 0; j < N; ++j)
for (k = 0; k < N; ++k)
G[j * N + k] = A(ip[j], id[k]);
/* invert it to solve the system of linear equations */
raid_invert(G, V, N);
/* compute delta parity */
raid_delta_gen(N, id, ip, nd, size, vv);
for (j = 0; j < N; ++j) {
p[j] = v[nd + ip[j]];
pa[j] = v[id[j]];
}
raid_neon32_begin();
/*
* q4-q11 contain the four low/high multiplication table pairs.
* q15 contains low4.
*/
asm volatile (
"vld1.8 {q15}, %0\n"
"vld1.8 {q4}, %1\n"
"vld1.8 {q5}, %2\n"
"vld1.8 {q6}, %3\n"
"vld1.8 {q7}, %4\n"
"vld1.8 {q8}, %5\n"
"vld1.8 {q9}, %6\n"
"vld1.8 {q10}, %7\n"
"vld1.8 {q11}, %8\n"
:
: "Q" (gfconst16.low4[0]),
"Q" (raid_gfmulpshufb[V[0]][0][0]),
"Q" (raid_gfmulpshufb[V[0]][1][0]),
"Q" (raid_gfmulpshufb[V[1]][0][0]),
"Q" (raid_gfmulpshufb[V[1]][1][0]),
"Q" (raid_gfmulpshufb[V[2]][0][0]),
"Q" (raid_gfmulpshufb[V[2]][1][0]),
"Q" (raid_gfmulpshufb[V[3]][0][0]),
"Q" (raid_gfmulpshufb[V[3]][1][0])
);
for (i = 0; i < size; i += 16) {
asm volatile (
/* delta 0 */
"vld1.8 {q0}, %2\n"
"vld1.8 {q12}, %4\n"
"veor q0, q0, q12\n"
"vshr.u8 q1, q0, #4\n"
"vand q0, q0, q15\n"
"vand q1, q1, q15\n"
/* delta 1 */
"vld1.8 {q2}, %3\n"
"vld1.8 {q12}, %5\n"
"veor q2, q2, q12\n"
"vshr.u8 q3, q2, #4\n"
"vand q2, q2, q15\n"
"vand q3, q3, q15\n"
/* pa[0] = V[0] * delta0 */
"vtbl.8 d24, {d8-d9}, d0\n"
"vtbl.8 d25, {d8-d9}, d1\n"
"vtbl.8 d26, {d10-d11}, d2\n"
"vtbl.8 d27, {d10-d11}, d3\n"
"veor q14, q12, q13\n"
/* ^ V[1] * delta1 */
"vtbl.8 d24, {d12-d13}, d4\n"
"vtbl.8 d25, {d12-d13}, d5\n"
"vtbl.8 d26, {d14-d15}, d6\n"
"vtbl.8 d27, {d14-d15}, d7\n"
"veor q12, q12, q13\n"
"veor q14, q14, q12\n"
"vst1.8 {q14}, %0\n"
/* pa[1] = V[2] * delta0 */
"vtbl.8 d24, {d16-d17}, d0\n"
"vtbl.8 d25, {d16-d17}, d1\n"
"vtbl.8 d26, {d18-d19}, d2\n"
"vtbl.8 d27, {d18-d19}, d3\n"
"veor q14, q12, q13\n"
/* ^ V[3] * delta1 */
"vtbl.8 d24, {d20-d21}, d4\n"
"vtbl.8 d25, {d20-d21}, d5\n"
"vtbl.8 d26, {d22-d23}, d6\n"
"vtbl.8 d27, {d22-d23}, d7\n"
"veor q12, q12, q13\n"
"veor q14, q14, q12\n"
"vst1.8 {q14}, %1\n"
: "=Q" (pa[0][i]), "=Q" (pa[1][i])
: "Q" (p[0][i]), "Q" (p[1][i]),
"Q" (pa[0][i]), "Q" (pa[1][i])
);
}
raid_neon32_end();
}
/*
* RAID recovering AArch32 NEON implementation
*/
void raid_recX_neon32(int nr, int *id, int *ip, int nd,
size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
int N = nr;
uint8_t *p[RAID_PARITY_MAX];
uint8_t *pa[RAID_PARITY_MAX];
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
const uint8_t *T[RAID_PARITY_MAX * RAID_PARITY_MAX];
uint8_t D[RAID_PARITY_MAX][2][16] __aligned(16);
size_t i;
int j, k;
/* setup the coefficients matrix */
for (j = 0; j < N; ++j)
for (k = 0; k < N; ++k)
G[j * N + k] = A(ip[j], id[k]);
/* invert it to solve the system of linear equations */
raid_invert(G, V, N);
/* precompute shuffle table pointers */
for (j = 0; j < N * N; ++j)
T[j] = &raid_gfmulpshufb[V[j]][0][0];
/* compute delta parity */
raid_delta_gen(N, id, ip, nd, size, vv);
for (j = 0; j < N; ++j) {
p[j] = v[nd + ip[j]];
pa[j] = v[id[j]];
}
raid_neon32_begin();
/* preload mask */
asm volatile (
"vld1.8 {q15}, %0\n"
:
: "Q" (gfconst16.low4[0])
);
for (i = 0; i < size; i += 16) {
for (k = 0; k < N; ++k) {
asm volatile (
"vld1.8 {q0}, %2\n"
"vld1.8 {q1}, %3\n"
"veor q0, q0, q1\n"
"vshr.u8 q1, q0, #4\n"
"vand q0, q0, q15\n"
"vand q1, q1, q15\n"
"vst1.8 {q0}, %0\n"
"vst1.8 {q1}, %1\n"
: "=Q" (D[k][0][0]), "=Q" (D[k][1][0])
: "Q" (p[k][i]), "Q" (pa[k][i])
);
}
/* reconstruct */
for (j = 0; j < N; ++j) {
asm volatile (
"veor q0, q0, q0\n"
);
for (k = 0; k < N; ++k) {
asm volatile (
"vld1.8 {q2}, %0\n"
"vld1.8 {q3}, %1\n"
"vld1.8 {q4}, %2\n"
"vld1.8 {q5}, %3\n"
"vtbl.8 d12, {d8-d9}, d4\n"
"vtbl.8 d13, {d8-d9}, d5\n"
"vtbl.8 d14, {d10-d11}, d6\n"
"vtbl.8 d15, {d10-d11}, d7\n"
"veor q6, q6, q7\n"
"veor q0, q0, q6\n"
:
: "Q" (D[k][0][0]),
"Q" (D[k][1][0]),
"Q" (T[j * N + k][0]),
"Q" (T[j * N + k][16])
);
}
asm volatile (
"vst1.8 {q0}, %0\n"
: "=Q" (pa[j][i])
);
}
}
raid_neon32_end();
}
void raid_register_neon32(void)
{
raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "neon32", raid_gen1_neon32, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "neon32", raid_gen2_neon32_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "neon32", raid_gen2_neon32_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "neon32", raid_genz_neon32_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "neon32", raid_gen3_neon32_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "neon32", raid_gen3_neon32_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "neon32", raid_gen4_neon32_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "neon32", raid_gen4_neon32_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "neon32", raid_gen5_neon32_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "neon32", raid_gen5_neon32_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "neon32", raid_gen6_neon32_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "neon32", raid_gen6_neon32_aes, RAID_POLY_AES);
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "neon32", raid_rec1_neon32, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "neon32", raid_rec2_neon32, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "neon32", raid_recX_neon32, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "neon32", raid_recX_neon32, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "neon32", raid_recX_neon32, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "neon32", raid_recX_neon32, RAID_POLY_ANY);
}
#endif

9
raid/tables.c
View file

@ -9272,7 +9272,7 @@ const uint8_t __aligned(256) raid_gfcauchy_raid[6][256] =
},
};
#if defined(CONFIG_X86) || defined(CONFIG_NEON)
#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32)
/**
* PSHUFB tables for the Cauchy matrix.
*
@ -14806,7 +14806,7 @@ const uint8_t __aligned(256) raid_gfcauchypshufb_raid[251][5][2][16] =
};
#endif
#if defined(CONFIG_X86) || defined(CONFIG_NEON)
#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32)
/**
* PSHUFB tables for generic multiplication.
*
@ -24957,7 +24957,7 @@ const uint8_t __aligned(256) raid_gfcauchy_aes[6][256] =
},
};
#if defined(CONFIG_X86) || defined(CONFIG_NEON)
#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32)
/**
* PSHUFB tables for the Cauchy matrix.
*
@ -30491,7 +30491,7 @@ const uint8_t __aligned(256) raid_gfcauchypshufb_aes[251][5][2][16] =
};
#endif
#if defined(CONFIG_X86) || defined(CONFIG_NEON)
#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32)
/**
* PSHUFB tables for generic multiplication.
*
@ -31526,4 +31526,3 @@ const uint8_t __aligned(256) raid_gfmulpshufb_aes[256][2][16] =
},
};
#endif

32
raid/test.c
View file

@ -417,6 +417,9 @@ int raid_test_rec(int mode, int nd, size_t size)
#endif
#ifdef CONFIG_NEON
test_setup(i) = raid_rec1_neon;
#endif
#ifdef CONFIG_NEON32
test_setup(i) = raid_rec1_neon32;
#endif
} else if (i == 2) {
test_setup(i) = raid_rec2_int8;
@ -446,6 +449,9 @@ int raid_test_rec(int mode, int nd, size_t size)
#endif
#ifdef CONFIG_NEON
test_setup(i) = raid_rec2_neon;
#endif
#ifdef CONFIG_NEON32
test_setup(i) = raid_rec2_neon32;
#endif
} else {
test_setup(i) = raid_recX_int8;
@ -475,6 +481,9 @@ int raid_test_rec(int mode, int nd, size_t size)
#endif
#ifdef CONFIG_NEON
test_setup(i) = raid_recX_neon;
#endif
#ifdef CONFIG_NEON32
test_setup(i) = raid_recX_neon32;
#endif
}
}
@ -606,6 +615,13 @@ int raid_test_par(int mode, int nd, size_t size)
else
test_setup(2) = raid_gen2_neon_raid;
#endif
#ifdef CONFIG_NEON32
test_setup(1) = raid_gen1_neon32;
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(2) = raid_gen2_neon32_aes;
else
test_setup(2) = raid_gen2_neon32_raid;
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_sse2()) {
@ -673,6 +689,19 @@ int raid_test_par(int mode, int nd, size_t size)
test_setup(6) = raid_gen6_neon_raid;
}
#endif
#ifdef CONFIG_NEON32
if (mode == RAID_MODE_CAUCHY_AES) {
test_setup(3) = raid_gen3_neon32_aes;
test_setup(4) = raid_gen4_neon32_aes;
test_setup(5) = raid_gen5_neon32_aes;
test_setup(6) = raid_gen6_neon32_aes;
} else {
test_setup(3) = raid_gen3_neon32_raid;
test_setup(4) = raid_gen4_neon32_raid;
test_setup(5) = raid_gen5_neon32_raid;
test_setup(6) = raid_gen6_neon32_raid;
}
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
@ -754,6 +783,9 @@ int raid_test_par(int mode, int nd, size_t size)
#ifdef CONFIG_NEON
test_setup(3) = raid_genz_neon_raid;
#endif
#ifdef CONFIG_NEON32
test_setup(3) = raid_genz_neon32_raid;
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_sse2()) {

2
raid/test/Makefile
View file

@ -36,7 +36,7 @@ CFLAGS += -O2
else
CFLAGS += -O0 --coverage -DCOVERAGE=1 -DNDEBUG=1
endif
OBJS = raid.o check.o int.o sse2.o ssse3.o avx2.o avx512.o gfni.o neon.o tables.o memory.o test.o helper.o module.o
OBJS = raid.o check.o int.o sse2.o ssse3.o avx2.o avx512.o gfni.o neon.o neon32.o tables.o memory.o test.o helper.o module.o
%.o: ../%.c
$(CC) $(CFLAGS) -c -o $@ $<

3
raid/test/fulltest.c
View file

@ -33,6 +33,9 @@ int main(void)
#ifdef CONFIG_NEON
printf("Including ARM NEON\n");
#endif
#ifdef CONFIG_NEON32
printf("Including ARM NEON\n");
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_sse2())
printf("Including x86 SSE2\n");

172
raid/test/speedtest.c
View file

@ -107,6 +107,9 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
#ifdef CONFIG_NEON
printf("%8s", "neon");
#endif
#ifdef CONFIG_NEON32
printf("%8s", "neon32");
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_sse2()) {
printf("%8s", "sse2");
@ -169,6 +172,14 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen1_neon32(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_sse2()) {
@ -272,6 +283,20 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
if (mode == RAID_MODE_CAUCHY_AES) {
SPEED_START {
raid_gen2_neon32_aes(nd, size, v);
} SPEED_STOP
} else {
SPEED_START {
raid_gen2_neon32_raid(nd, size, v);
} SPEED_STOP
}
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_sse2()) {
@ -412,6 +437,20 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
if (mode == RAID_MODE_CAUCHY_AES) {
SPEED_START {
raid_gen3_neon32_aes(nd, size, v);
} SPEED_STOP
} else {
SPEED_START {
raid_gen3_neon32_raid(nd, size, v);
} SPEED_STOP
}
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_sse2()) {
@ -539,6 +578,20 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
if (mode == RAID_MODE_CAUCHY_AES) {
SPEED_START {
raid_gen4_neon32_aes(nd, size, v);
} SPEED_STOP
} else {
SPEED_START {
raid_gen4_neon32_raid(nd, size, v);
} SPEED_STOP
}
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_sse2()) {
@ -666,6 +719,20 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
if (mode == RAID_MODE_CAUCHY_AES) {
SPEED_START {
raid_gen5_neon32_aes(nd, size, v);
} SPEED_STOP
} else {
SPEED_START {
raid_gen5_neon32_raid(nd, size, v);
} SPEED_STOP
}
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_sse2()) {
@ -791,6 +858,20 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
if (mode == RAID_MODE_CAUCHY_AES) {
SPEED_START {
raid_gen6_neon32_aes(nd, size, v);
} SPEED_STOP
} else {
SPEED_START {
raid_gen6_neon32_raid(nd, size, v);
} SPEED_STOP
}
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_sse2()) {
@ -908,6 +989,9 @@ void speed_genz(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_NEON
printf("%8s", "neon");
#endif
#ifdef CONFIG_NEON32
printf("%8s", "neon32");
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_sse2()) {
printf("%8s", "sse2");
@ -959,6 +1043,14 @@ void speed_genz(int nd, void **v, int size, int delta, int period)
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_genz_neon32_raid(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_sse2()) {
@ -1026,6 +1118,9 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_NEON
printf("%8s", "neon");
#endif
#ifdef CONFIG_NEON32
printf("%8s", "neon32");
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3())
printf("%8s", "ssse3");
@ -1064,6 +1159,15 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen_force(1, raid_gen1_neon32);
raid_rec1_neon32(1, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
@ -1140,6 +1244,16 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen_force(1, raid_gen1_neon32);
/* +1 to avoid GEN1 optimized case */
raid_rec1_neon32(1, id, ip + 1, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
@ -1219,6 +1333,15 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen_force(2, raid_gen2_neon32_raid);
raid_rec2_neon32(2, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
@ -1303,6 +1426,16 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen_force(2, raid_gen2_neon32_raid);
/* +1 to avoid GEN2 optimized case */
raid_rec2_neon32(2, id, ip + 1, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
@ -1390,6 +1523,15 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen_force(3, raid_gen3_neon32_raid);
raid_recX_neon32(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
@ -1473,6 +1615,15 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen_force(4, raid_gen4_neon32_raid);
raid_recX_neon32(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
@ -1556,6 +1707,15 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen_force(5, raid_gen5_neon32_raid);
raid_recX_neon32(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
@ -1639,6 +1799,15 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen_force(6, raid_gen6_neon32_raid);
raid_recX_neon32(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
@ -1761,6 +1930,9 @@ int main(void)
#ifdef CONFIG_NEON
printf("Including ARM NEON\n");
#endif
#ifdef CONFIG_NEON32
printf("Including ARM NEON\n");
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_sse2())
printf("Including x86 SSE2\n");