diff --git a/HISTORY b/HISTORY index bf203de5..18db39ff 100644 --- a/HISTORY +++ b/HISTORY @@ -22,9 +22,9 @@ SnapRAID HISTORY By scaling to 512-bit ZMM registers, the implementation now processes 64 bytes per iteration, effectively doubling the data width of the previous AVX2 routines. - * Implemented AArch64 (ARM64) NEON support for parity generation and - data recovery, significantly improving performance on ARM-based - platforms. + * Implemented AArch64 (ARM64) and AArch32 (ARM32) NEON support for + parity generation and data recovery, significantly improving + performance on ARM-based platforms. * Improved the chances of a successful recovery after an interrupted sync. This is achieved by removing the sync optimization of not recomputing the parity if the input data matches the expected hash. diff --git a/Makefile.am b/Makefile.am index 44303187..e2972069 100644 --- a/Makefile.am +++ b/Makefile.am @@ -34,6 +34,7 @@ snapraid_SOURCES = \ raid/avx512.c \ raid/gfni.c \ raid/neon.c \ + raid/neon32.c \ raid/helper.c \ raid/memory.c \ raid/test.c \ diff --git a/cmdline/speed.c b/cmdline/speed.c index ff9bec17..54b99846 100644 --- a/cmdline/speed.c +++ b/cmdline/speed.c @@ -207,6 +207,9 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms #ifdef CONFIG_NEON printf("%8s", "neon"); #endif +#ifdef CONFIG_NEON32 + printf("%8s", "neon32"); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { printf("%8s", "sse2"); @@ -266,6 +269,14 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + SPEED_START { + raid_gen1_neon32(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { @@ -370,6 +381,20 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + if (mode == RAID_MODE_CAUCHY_AES) { + SPEED_START { + raid_gen2_neon32_aes(nd, size, v); + } SPEED_STOP + } else { + SPEED_START { + raid_gen2_neon32_raid(nd, size, v); + } SPEED_STOP + } + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { @@ -510,6 +535,20 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + if (mode == RAID_MODE_CAUCHY_AES) { + SPEED_START { + raid_gen3_neon32_aes(nd, size, v); + } SPEED_STOP + } else { + SPEED_START { + raid_gen3_neon32_raid(nd, size, v); + } SPEED_STOP + } + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { @@ -639,6 +678,20 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + if (mode == RAID_MODE_CAUCHY_AES) { + SPEED_START { + raid_gen4_neon32_aes(nd, size, v); + } SPEED_STOP + } else { + SPEED_START { + raid_gen4_neon32_raid(nd, size, v); + } SPEED_STOP + } + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { @@ -768,6 +821,20 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + if (mode == RAID_MODE_CAUCHY_AES) { + SPEED_START { + raid_gen5_neon32_aes(nd, size, v); + } SPEED_STOP + } else { + SPEED_START { + raid_gen5_neon32_raid(nd, size, v); + } SPEED_STOP + } + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { @@ -894,6 +961,20 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + if (mode == RAID_MODE_CAUCHY_AES) { + SPEED_START { + raid_gen6_neon32_aes(nd, size, v); + } SPEED_STOP + } else { + SPEED_START { + raid_gen6_neon32_raid(nd, size, v); + } SPEED_STOP + } + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { @@ -1012,6 +1093,9 @@ void speed_genz(int nd, void** v, int size, int delta, int period) #ifdef CONFIG_NEON printf("%8s", "neon"); #endif +#ifdef CONFIG_NEON32 + printf("%8s", "neon32"); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { printf("%8s", "sse2"); @@ -1063,6 +1147,14 @@ void speed_genz(int nd, void** v, int size, int delta, int period) printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + SPEED_START { + raid_genz_neon32_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { @@ -1130,6 +1222,9 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #ifdef CONFIG_NEON printf("%8s", "neon"); #endif +#ifdef CONFIG_NEON32 + printf("%8s", "neon32"); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) printf("%8s", "ssse3"); @@ -1170,6 +1265,16 @@ void speed_rec(int nd, void** v, int size, int delta, int period) printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + SPEED_START { + /* ensure to use same hardware in the delta step */ + raid_gen_force(1, raid_gen1_neon32); + raid_rec1_neon32(1, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { @@ -1253,6 +1358,17 @@ void speed_rec(int nd, void** v, int size, int delta, int period) printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + SPEED_START { + /* ensure to use same hardware in the delta step */ + raid_gen_force(1, raid_gen1_neon32); + /* +1 to avoid GEN1 optimized case */ + raid_rec1_neon32(1, id, ip + 1, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { @@ -1339,6 +1455,16 @@ void speed_rec(int nd, void** v, int size, int delta, int period) printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + SPEED_START { + /* ensure to use same hardware in the delta step */ + raid_gen_force(2, raid_gen2_neon32_raid); + raid_rec2_neon32(2, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { @@ -1430,6 +1556,17 @@ void speed_rec(int nd, void** v, int size, int delta, int period) printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + SPEED_START { + /* ensure to use same hardware in the delta step */ + raid_gen_force(2, raid_gen2_neon32_raid); + /* +1 to avoid GEN2 optimized case */ + raid_rec2_neon32(2, id, ip + 1, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { @@ -1523,6 +1660,16 @@ void speed_rec(int nd, void** v, int size, int delta, int period) printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + SPEED_START { + /* ensure to use same hardware in the delta step */ + raid_gen_force(3, raid_gen3_neon32_raid); + raid_recX_neon32(3, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { @@ -1612,6 +1759,16 @@ void speed_rec(int nd, void** v, int size, int delta, int period) printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + SPEED_START { + /* ensure to use same hardware in the delta step */ + raid_gen_force(4, raid_gen4_neon32_raid); + raid_recX_neon32(4, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { @@ -1702,6 +1859,16 @@ void speed_rec(int nd, void** v, int size, int delta, int period) printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + SPEED_START { + /* ensure to use same hardware in the delta step */ + raid_gen_force(5, raid_gen5_neon32_raid); + raid_recX_neon32(5, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { @@ -1791,6 +1958,16 @@ void speed_rec(int nd, void** v, int size, int delta, int period) printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + SPEED_START { + /* ensure to use same hardware in the delta step */ + raid_gen_force(6, raid_gen6_neon32_raid); + raid_recX_neon32(6, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { @@ -2045,7 +2222,7 @@ void speed(int period, int nd, int size) printf("CPU 64-bit ARM (AArch64)\n"); #endif #elif defined(__arm__) -#if defined(CONFIG_NEON) +#if defined(CONFIG_NEON32) printf("CPU 32-bit ARM, flags neon\n"); #else printf("CPU 32-bit ARM\n"); diff --git a/raid/README b/raid/README index d67c4d0a..6f345b37 100644 --- a/raid/README +++ b/raid/README @@ -150,7 +150,8 @@ Implementation Structure avx2.c x86 AVX2 (256-bit vectorization) avx512.c x86 AVX-512BW (512-bit vectorization) gfni.c x86 GFNI generation/recovery (RAID affine and AES multiply) - neon.c ARM NEON (128-bit SIMD vectorization) + neon.c AArch64 ARM NEON (128-bit SIMD vectorization) + neon32.c AArch32 ARM NEON (128-bit SIMD vectorization) Testing and Benchmarks diff --git a/raid/internal.h b/raid/internal.h index 3851435c..0b7b0884 100644 --- a/raid/internal.h +++ b/raid/internal.h @@ -23,11 +23,16 @@ #define CONFIG_X86 1 #define CONFIG_X86_64 1 #endif -#if defined(__arm__) || defined(__aarch64__) +#if defined(__aarch64__) #if defined(__ARM_NEON) || defined(__ARM_NEON__) #define CONFIG_NEON 1 #endif #endif +#if defined(__arm__) +#if defined(__ARM_NEON) || defined(__ARM_NEON__) +#define CONFIG_NEON32 1 +#endif +#endif #endif /* @@ -198,6 +203,21 @@ void raid_gen6_neon_aes(int nd, size_t size, void **vv); void raid_rec1_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec2_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_gen1_neon32(int nd, size_t size, void **vv); +void raid_gen2_neon32_raid(int nd, size_t size, void **vv); +void raid_gen2_neon32_aes(int nd, size_t size, void **vv); +void raid_genz_neon32_raid(int nd, size_t size, void **vv); +void raid_gen3_neon32_raid(int nd, size_t size, void **vv); +void raid_gen3_neon32_aes(int nd, size_t size, void **vv); +void raid_gen4_neon32_raid(int nd, size_t size, void **vv); +void raid_gen4_neon32_aes(int nd, size_t size, void **vv); +void raid_gen5_neon32_raid(int nd, size_t size, void **vv); +void raid_gen5_neon32_aes(int nd, size_t size, void **vv); +void raid_gen6_neon32_raid(int nd, size_t size, void **vv); +void raid_gen6_neon32_aes(int nd, size_t size, void **vv); +void raid_rec1_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec2_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_recX_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv); /* * Functions for parity computation. @@ -323,7 +343,7 @@ const char * raid_rec_tag(int na); */ int raid_selftest(void); -#if defined(CONFIG_X86) || defined(CONFIG_NEON) +#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32) struct gfconst16 { uint8_t poly[16]; uint8_t low4[16]; @@ -342,7 +362,7 @@ extern const uint8_t raid_gfinv_raid[256] __aligned(256); extern const uint8_t raid_gfvandermonde_raid[3][256] __aligned(256); extern const uint8_t raid_gfcauchy_raid[6][256] __aligned(256); extern const uint8_t raid_gfaffine_raid[256][8] __aligned(256); -#if defined(CONFIG_X86) || defined(CONFIG_NEON) +#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32) extern const uint8_t raid_gfcauchypshufb_raid[251][5][2][16] __aligned(256); extern const uint8_t raid_gfmulpshufb_raid[256][2][16] __aligned(256); #endif @@ -351,7 +371,7 @@ extern const uint8_t raid_gfmul_aes[256][256] __aligned(256); extern const uint8_t raid_gfexp_aes[256] __aligned(256); extern const uint8_t raid_gfinv_aes[256] __aligned(256); extern const uint8_t raid_gfcauchy_aes[6][256] __aligned(256); -#if defined(CONFIG_X86) || defined(CONFIG_NEON) +#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32) extern const uint8_t raid_gfcauchypshufb_aes[251][5][2][16] __aligned(256); extern const uint8_t raid_gfmulpshufb_aes[256][2][16] __aligned(256); #endif @@ -361,7 +381,7 @@ extern const uint8_t *raid_gfexp; extern const uint8_t *raid_gfinv; extern const uint8_t(*raid_gfvandermonde)[256]; extern const uint8_t(*raid_gfcauchy)[256]; -#if defined(CONFIG_X86) || defined(CONFIG_NEON) +#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32) extern const uint8_t(*raid_gfcauchypshufb)[5][2][16]; extern const uint8_t(*raid_gfmulpshufb)[2][16]; #endif @@ -475,4 +495,28 @@ static __always_inline void raid_neon_end(void) void raid_register_neon(void); #endif +#ifdef CONFIG_NEON32 +static __always_inline void raid_neon32_begin(void) +{ +} + +static __always_inline void raid_neon32_end(void) +{ + /* + * Clobbers registers used in AArch32 NEON asm operations. + * + * Like raid_neon_end(), the inner asm statements intentionally + * preserve hidden NEON state across adjacent asm blocks. Keep the + * dummy clobbers at the end of the function so the compiler preserves + * the AAPCS32 callee-saved NEON registers when necessary. + */ + asm volatile ("" : : : "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7"); + asm volatile ("" : : : "d8", "d9", "d10", "d11", "d12", "d13", "d14", "d15"); + asm volatile ("" : : : "d16", "d17", "d18", "d19", "d20", "d21", "d22", "d23"); + asm volatile ("" : : : "d24", "d25", "d26", "d27", "d28", "d29", "d30", "d31"); +} + +void raid_register_neon32(void); +#endif + #endif diff --git a/raid/mktables.c b/raid/mktables.c index ef1fc63d..70e5c9bc 100644 --- a/raid/mktables.c +++ b/raid/mktables.c @@ -352,7 +352,7 @@ void tables(uint8_t poly, uint8_t generator, const char *tag) } printf("};\n\n"); - printf("#if defined(CONFIG_X86) || defined(CONFIG_NEON)\n"); + printf("#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32)\n"); printf("/**\n"); printf(" * PSHUFB tables for the Cauchy matrix.\n"); printf(" *\n"); @@ -384,7 +384,7 @@ void tables(uint8_t poly, uint8_t generator, const char *tag) printf("};\n"); printf("#endif\n\n"); - printf("#if defined(CONFIG_X86) || defined(CONFIG_NEON)\n"); + printf("#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32)\n"); printf("/**\n"); printf(" * PSHUFB tables for generic multiplication.\n"); printf(" *\n"); diff --git a/raid/module.c b/raid/module.c index 3ca187f3..4a0f9807 100644 --- a/raid/module.c +++ b/raid/module.c @@ -44,7 +44,7 @@ const uint8_t *raid_gfexp; const uint8_t *raid_gfinv; const uint8_t(*raid_gfvandermonde)[256]; const uint8_t(*raid_gfcauchy)[256]; -#if defined(CONFIG_X86) || defined(CONFIG_NEON) +#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32) const uint8_t(*raid_gfcauchypshufb)[5][2][16]; const uint8_t(*raid_gfmulpshufb)[2][16]; #endif @@ -57,7 +57,7 @@ uint64_t raid_poly_64; uint32_t raid_inv2_32; uint64_t raid_inv2_64; -#if defined(CONFIG_X86) || defined(CONFIG_NEON) +#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32) struct gfconst16 __aligned(16) gfconst16 = { { 0 }, @@ -229,7 +229,7 @@ int raid_mode(int mode) raid_gfinv = raid_gfinv_aes; raid_gfvandermonde = 0; raid_gfcauchy = raid_gfcauchy_aes; -#if defined(CONFIG_X86) || defined(CONFIG_NEON) +#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32) raid_gfcauchypshufb = raid_gfcauchypshufb_aes; raid_gfmulpshufb = raid_gfmulpshufb_aes; #endif @@ -249,7 +249,7 @@ int raid_mode(int mode) raid_gfinv = raid_gfinv_raid; raid_gfvandermonde = raid_gfvandermonde_raid; raid_gfcauchy = raid_gfcauchy_raid; -#if defined(CONFIG_X86) || defined(CONFIG_NEON) +#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32) raid_gfcauchypshufb = raid_gfcauchypshufb_raid; raid_gfmulpshufb = raid_gfmulpshufb_raid; #endif @@ -265,7 +265,7 @@ int raid_mode(int mode) raid_rec_algo = raid_rec_algo_raid; } -#if defined(CONFIG_X86) || defined(CONFIG_NEON) +#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32) memset(gfconst16.poly, raid_poly_byte, 16); memset(gfconst16.half, raid_inv2_byte, 16); #endif @@ -361,6 +361,9 @@ void raid_init(void) #ifdef CONFIG_NEON raid_register_neon(); #endif +#ifdef CONFIG_NEON32 + raid_register_neon32(); +#endif /* set the default mode */ raid_mode(RAID_MODE_CAUCHY_RAID); diff --git a/raid/neon32.c b/raid/neon32.c new file mode 100644 index 00000000..a959139f --- /dev/null +++ b/raid/neon32.c @@ -0,0 +1,941 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +// Copyright (C) 2026 Andrea Mazzoleni + +#include "internal.h" +#include "gf.h" +#include "cpu.h" + +#ifdef CONFIG_NEON32 + +/* + * GEN1 (RAID5 with xor) AArch32 NEON implementation + */ +void raid_gen1_neon32(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + + raid_neon32_begin(); + + for (i = 0; i < size; i += 64) { + asm volatile ( + "vld1.8 {q0}, %0\n" + "vld1.8 {q1}, %1\n" + "vld1.8 {q2}, %2\n" + "vld1.8 {q3}, %3\n" + : + : "Q" (v[0][i]), "Q" (v[0][i + 16]), + "Q" (v[0][i + 32]), "Q" (v[0][i + 48]) + ); + + for (d = 1; d <= l; ++d) { + asm volatile ( + "vld1.8 {q4}, %0\n" + "vld1.8 {q5}, %1\n" + "vld1.8 {q6}, %2\n" + "vld1.8 {q7}, %3\n" + "veor q0, q0, q4\n" + "veor q1, q1, q5\n" + "veor q2, q2, q6\n" + "veor q3, q3, q7\n" + : + : "Q" (v[d][i]), "Q" (v[d][i + 16]), + "Q" (v[d][i + 32]), "Q" (v[d][i + 48]) + ); + } + + asm volatile ( + "vst1.8 {q0}, %0\n" + "vst1.8 {q1}, %1\n" + "vst1.8 {q2}, %2\n" + "vst1.8 {q3}, %3\n" + : "=Q" (p[i]), "=Q" (p[i + 16]), + "=Q" (p[i + 32]), "=Q" (p[i + 48]) + ); + } + + raid_neon32_end(); +} + +/* + * GEN2 Cauchy AArch32 NEON implementation using the active generator. + */ +static __always_inline void raid_gen2_neon32_gen(int nd, size_t size, + void **vv, int generator) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + + raid_neon32_begin(); + + /* q14 contains the active reduction polynomial. */ + asm volatile ( + "vld1.8 {q14}, %0\n" + : + : "Q" (gfconst16.poly[0]) + ); + + for (i = 0; i < size; i += 32) { + asm volatile ( + "vld1.8 {q0}, %0\n" + "vld1.8 {q1}, %1\n" + "vmov q2, q0\n" + "vmov q3, q1\n" + : + : "Q" (v[l][i]), "Q" (v[l][i + 16]) + ); + + for (d = l - 1; d >= 0; --d) { + if (generator == 3) { + asm volatile ( + "vmov q10, q2\n" + "vmov q11, q3\n" + ); + } + asm volatile ( + "vshr.s8 q8, q2, #7\n" + "vshl.i8 q2, q2, #1\n" + "vand q8, q8, q14\n" + "veor q2, q2, q8\n" + "vshr.s8 q8, q3, #7\n" + "vshl.i8 q3, q3, #1\n" + "vand q8, q8, q14\n" + "veor q3, q3, q8\n" + "vld1.8 {q12}, %0\n" + "vld1.8 {q13}, %1\n" + "veor q0, q0, q12\n" + "veor q1, q1, q13\n" + "veor q2, q2, q12\n" + "veor q3, q3, q13\n" + : + : "Q" (v[d][i]), "Q" (v[d][i + 16]) + ); + if (generator == 3) { + asm volatile ( + "veor q2, q2, q10\n" + "veor q3, q3, q11\n" + ); + } + } + + asm volatile ( + "vst1.8 {q0}, %0\n" + "vst1.8 {q1}, %1\n" + "vst1.8 {q2}, %2\n" + "vst1.8 {q3}, %3\n" + : "=Q" (p[i]), "=Q" (p[i + 16]), + "=Q" (q[i]), "=Q" (q[i + 16]) + ); + } + + raid_neon32_end(); +} + +void raid_gen2_neon32_raid(int nd, size_t size, void **vv) +{ + raid_gen2_neon32_gen(nd, size, vv, 2); +} + +void raid_gen2_neon32_aes(int nd, size_t size, void **vv) +{ + raid_gen2_neon32_gen(nd, size, vv, 3); +} + +/* + * GENz (triple parity with powers of 2^-1) AArch32 NEON implementation + */ +void raid_genz_neon32_raid(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + + raid_neon32_begin(); + + asm volatile ( + "vld1.8 {q14}, %0\n" + "vld1.8 {q15}, %1\n" + : + : "Q" (gfconst16.poly[0]), "Q" (gfconst16.half[0]) + ); + + for (i = 0; i < size; i += 32) { + asm volatile ( + "vld1.8 {q0}, %0\n" + "vld1.8 {q1}, %1\n" + "vmov q2, q0\n" + "vmov q3, q1\n" + "vmov q4, q0\n" + "vmov q5, q1\n" + : + : "Q" (v[l][i]), "Q" (v[l][i + 16]) + ); + + for (d = l - 1; d >= 0; --d) { + asm volatile ( + "vshr.s8 q8, q2, #7\n" + "vshl.i8 q2, q2, #1\n" + "vand q8, q8, q14\n" + "veor q2, q2, q8\n" + "vshr.s8 q8, q3, #7\n" + "vshl.i8 q3, q3, #1\n" + "vand q8, q8, q14\n" + "veor q3, q3, q8\n" + "vshl.i8 q9, q4, #7\n" + "vshr.s8 q9, q9, #7\n" + "vshr.u8 q4, q4, #1\n" + "vand q9, q9, q15\n" + "veor q4, q4, q9\n" + "vshl.i8 q9, q5, #7\n" + "vshr.s8 q9, q9, #7\n" + "vshr.u8 q5, q5, #1\n" + "vand q9, q9, q15\n" + "veor q5, q5, q9\n" + "vld1.8 {q12}, %0\n" + "vld1.8 {q13}, %1\n" + "veor q0, q0, q12\n" + "veor q1, q1, q13\n" + "veor q2, q2, q12\n" + "veor q3, q3, q13\n" + "veor q4, q4, q12\n" + "veor q5, q5, q13\n" + : + : "Q" (v[d][i]), "Q" (v[d][i + 16]) + ); + } + + asm volatile ( + "vst1.8 {q0}, %0\n" + "vst1.8 {q1}, %1\n" + "vst1.8 {q2}, %2\n" + "vst1.8 {q3}, %3\n" + "vst1.8 {q4}, %4\n" + "vst1.8 {q5}, %5\n" + : "=Q" (p[i]), "=Q" (p[i + 16]), + "=Q" (q[i]), "=Q" (q[i + 16]), + "=Q" (r[i]), "=Q" (r[i + 16]) + ); + } + + raid_neon32_end(); +} + +/* + * GENX AArch32 NEON implementation. + */ +static __always_inline void raid_genX_neon32(int nd, size_t size, + void **vv, int np, int generator) +{ + uint8_t **v = (uint8_t **)vv; + size_t i; + int d, l; + + l = nd - 1; + + /* special case with only one data disk */ + if (l == 0) { + for (d = 0; d < np; ++d) + memcpy(v[1 + d], v[0], size); + return; + } + + raid_neon32_begin(); + + asm volatile ( + "vld1.8 {q14}, %0\n" + "vld1.8 {q15}, %1\n" + : + : "Q" (gfconst16.poly[0]), "Q" (gfconst16.low4[0]) + ); + + for (i = 0; i < size; i += 16) { + /* last disk without the generator multiplication */ + asm volatile ( + "vld1.8 {q0}, %0\n" + "vld1.8 {q1}, %0\n" + : + : "Q" (v[l][i]) + ); + + if (np >= 3) { + asm volatile ( + "vshr.u8 q8, q0, #4\n" + "vand q7, q0, q15\n" + "vand q8, q8, q15\n" + ); + + asm volatile ( + "vld1.8 {q9}, %0\n" + "vld1.8 {q10}, %1\n" + + "vtbl.8 d22, {d18-d19}, d14\n" + "vtbl.8 d23, {d18-d19}, d15\n" + "vtbl.8 d24, {d20-d21}, d16\n" + "vtbl.8 d25, {d20-d21}, d17\n" + + "veor q2, q11, q12\n" + : + : "Q" (raid_gfcauchypshufb[l][1][0][0]), + "Q" (raid_gfcauchypshufb[l][1][1][0]) + ); + } + + if (np >= 4) { + asm volatile ( + "vld1.8 {q9}, %0\n" + "vld1.8 {q10}, %1\n" + + "vtbl.8 d22, {d18-d19}, d14\n" + "vtbl.8 d23, {d18-d19}, d15\n" + "vtbl.8 d24, {d20-d21}, d16\n" + "vtbl.8 d25, {d20-d21}, d17\n" + + "veor q3, q11, q12\n" + : + : "Q" (raid_gfcauchypshufb[l][2][0][0]), + "Q" (raid_gfcauchypshufb[l][2][1][0]) + ); + } + + if (np >= 5) { + asm volatile ( + "vld1.8 {q9}, %0\n" + "vld1.8 {q10}, %1\n" + + "vtbl.8 d22, {d18-d19}, d14\n" + "vtbl.8 d23, {d18-d19}, d15\n" + "vtbl.8 d24, {d20-d21}, d16\n" + "vtbl.8 d25, {d20-d21}, d17\n" + + "veor q4, q11, q12\n" + : + : "Q" (raid_gfcauchypshufb[l][3][0][0]), + "Q" (raid_gfcauchypshufb[l][3][1][0]) + ); + } + + if (np >= 6) { + asm volatile ( + "vld1.8 {q9}, %0\n" + "vld1.8 {q10}, %1\n" + + "vtbl.8 d22, {d18-d19}, d14\n" + "vtbl.8 d23, {d18-d19}, d15\n" + "vtbl.8 d24, {d20-d21}, d16\n" + "vtbl.8 d25, {d20-d21}, d17\n" + + "veor q5, q11, q12\n" + : + : "Q" (raid_gfcauchypshufb[l][4][0][0]), + "Q" (raid_gfcauchypshufb[l][4][1][0]) + ); + } + + /* intermediate disks */ + for (d = l - 1; d > 0; --d) { + if (generator == 3) { + asm volatile ( + "vmov q13, q1\n" + ); + } + asm volatile ( + "vshr.s8 q11, q1, #7\n" + "vshl.i8 q1, q1, #1\n" + "vand q11, q11, q14\n" + "veor q1, q1, q11\n" + "vld1.8 {q6}, %0\n" + "veor q0, q0, q6\n" + "veor q1, q1, q6\n" + : + : "Q" (v[d][i]) + ); + if (generator == 3) { + asm volatile ( + "veor q1, q1, q13\n" + ); + } + + if (np >= 3) { + asm volatile ( + "vshr.u8 q8, q6, #4\n" + "vand q7, q6, q15\n" + "vand q8, q8, q15\n" + ); + asm volatile ( + "vld1.8 {q9}, %0\n" + "vld1.8 {q10}, %1\n" + + "vtbl.8 d22, {d18-d19}, d14\n" + "vtbl.8 d23, {d18-d19}, d15\n" + "vtbl.8 d24, {d20-d21}, d16\n" + "vtbl.8 d25, {d20-d21}, d17\n" + + "veor q11, q11, q12\n" + "veor q2, q2, q11\n" + : + : "Q" (raid_gfcauchypshufb[d][1][0][0]), + "Q" (raid_gfcauchypshufb[d][1][1][0]) + ); + } + + if (np >= 4) { + asm volatile ( + "vld1.8 {q9}, %0\n" + "vld1.8 {q10}, %1\n" + "vtbl.8 d22, {d18-d19}, d14\n" + "vtbl.8 d23, {d18-d19}, d15\n" + "vtbl.8 d24, {d20-d21}, d16\n" + "vtbl.8 d25, {d20-d21}, d17\n" + "veor q11, q11, q12\n" + "veor q3, q3, q11\n" + : + : "Q" (raid_gfcauchypshufb[d][2][0][0]), + "Q" (raid_gfcauchypshufb[d][2][1][0]) + ); + } + + if (np >= 5) { + asm volatile ( + "vld1.8 {q9}, %0\n" + "vld1.8 {q10}, %1\n" + "vtbl.8 d22, {d18-d19}, d14\n" + "vtbl.8 d23, {d18-d19}, d15\n" + "vtbl.8 d24, {d20-d21}, d16\n" + "vtbl.8 d25, {d20-d21}, d17\n" + "veor q11, q11, q12\n" + "veor q4, q4, q11\n" + : + : "Q" (raid_gfcauchypshufb[d][3][0][0]), + "Q" (raid_gfcauchypshufb[d][3][1][0]) + ); + } + + if (np >= 6) { + asm volatile ( + "vld1.8 {q9}, %0\n" + "vld1.8 {q10}, %1\n" + "vtbl.8 d22, {d18-d19}, d14\n" + "vtbl.8 d23, {d18-d19}, d15\n" + "vtbl.8 d24, {d20-d21}, d16\n" + "vtbl.8 d25, {d20-d21}, d17\n" + "veor q11, q11, q12\n" + "veor q5, q5, q11\n" + : + : "Q" (raid_gfcauchypshufb[d][4][0][0]), + "Q" (raid_gfcauchypshufb[d][4][1][0]) + ); + } + } + + /* first disk with all coefficients at 1 */ + if (generator == 3) { + asm volatile ( + "vmov q13, q1\n" + ); + } + asm volatile ( + "vshr.s8 q11, q1, #7\n" + "vshl.i8 q1, q1, #1\n" + "vand q11, q11, q14\n" + "veor q1, q1, q11\n" + "vld1.8 {q6}, %0\n" + "veor q0, q0, q6\n" + "veor q1, q1, q6\n" + : + : "Q" (v[0][i]) + ); + if (generator == 3) { + asm volatile ( + "veor q1, q1, q13\n" + ); + } + + if (np >= 3) { + asm volatile ( + "veor q2, q2, q6\n" + ); + } + if (np >= 4) { + asm volatile ( + "veor q3, q3, q6\n" + ); + } + if (np >= 5) { + asm volatile ( + "veor q4, q4, q6\n" + ); + } + if (np >= 6) { + asm volatile ( + "veor q5, q5, q6\n" + ); + } + + /* write parity in increasing order */ + asm volatile ( + "vst1.8 {q0}, %0\n" + "vst1.8 {q1}, %1\n" + : "=Q" (v[nd][i]), + "=Q" (v[nd + 1][i]) + ); + + if (np >= 3) { + asm volatile ( + "vst1.8 {q2}, %0\n" + : "=Q" (v[nd + 2][i]) + ); + } + if (np >= 4) { + asm volatile ( + "vst1.8 {q3}, %0\n" + : "=Q" (v[nd + 3][i]) + ); + } + if (np >= 5) { + asm volatile ( + "vst1.8 {q4}, %0\n" + : "=Q" (v[nd + 4][i]) + ); + } + if (np >= 6) { + asm volatile ( + "vst1.8 {q5}, %0\n" + : "=Q" (v[nd + 5][i]) + ); + } + } + + raid_neon32_end(); +} + +/* + * GEN3 (triple parity with Cauchy matrix) AArch32 NEON implementation + */ +void raid_gen3_neon32_raid(int nd, size_t size, void **vv) +{ + raid_genX_neon32(nd, size, vv, 3, 2); +} + +void raid_gen3_neon32_aes(int nd, size_t size, void **vv) +{ + raid_genX_neon32(nd, size, vv, 3, 3); +} + +/* + * GEN4 (quad parity with Cauchy matrix) AArch32 NEON implementation + */ +void raid_gen4_neon32_raid(int nd, size_t size, void **vv) +{ + raid_genX_neon32(nd, size, vv, 4, 2); +} + +void raid_gen4_neon32_aes(int nd, size_t size, void **vv) +{ + raid_genX_neon32(nd, size, vv, 4, 3); +} + +/* + * GEN5 (penta parity with Cauchy matrix) AArch32 NEON implementation + */ +void raid_gen5_neon32_raid(int nd, size_t size, void **vv) +{ + raid_genX_neon32(nd, size, vv, 5, 2); +} + +void raid_gen5_neon32_aes(int nd, size_t size, void **vv) +{ + raid_genX_neon32(nd, size, vv, 5, 3); +} + +/* + * GEN6 (hexa parity with Cauchy matrix) AArch32 NEON implementation + */ +void raid_gen6_neon32_raid(int nd, size_t size, void **vv) +{ + raid_genX_neon32(nd, size, vv, 6, 2); +} + +void raid_gen6_neon32_aes(int nd, size_t size, void **vv) +{ + raid_genX_neon32(nd, size, vv, 6, 3); +} + +/* + * RAID recovering for one disk AArch32 NEON implementation + */ +void raid_rec1_neon32(int nr, int *id, int *ip, int nd, + size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *pa; + uint8_t G; + uint8_t V; + size_t i; + + (void)nr; /* unused, it's always 1 */ + + /* if it's RAID5 use the faster function */ + if (ip[0] == 0) { + raid_rec1of1(id, nd, size, vv); + return; + } + + /* setup the coefficients matrix */ + G = A(ip[0], id[0]); + + /* invert it to solve the system of linear equations */ + V = inv(G); + + /* compute delta parity */ + raid_delta_gen(1, id, ip, nd, size, vv); + + p = v[nd + ip[0]]; + pa = v[id[0]]; + + raid_neon32_begin(); + + /* + * Preload the mask and multiplication tables, just as neon.c does. + * + * q12 = low table + * q13 = high table + * q14 = low4 mask + */ + asm volatile ( + "vld1.8 {q14}, %0\n" + "vld1.8 {q12}, %1\n" + "vld1.8 {q13}, %2\n" + : + : "Q" (gfconst16.low4[0]), + "Q" (raid_gfmulpshufb[V][0][0]), + "Q" (raid_gfmulpshufb[V][1][0]) + ); + + for (i = 0; i < size; i += 32) { + asm volatile ( + "vld1.8 {q0}, %2\n" + "vld1.8 {q1}, %3\n" + "vld1.8 {q2}, %4\n" + "vld1.8 {q3}, %5\n" + + "veor q0, q0, q2\n" + "veor q1, q1, q3\n" + + "vshr.u8 q4, q0, #4\n" + "vshr.u8 q5, q1, #4\n" + + "vand q0, q0, q14\n" + "vand q1, q1, q14\n" + "vand q4, q4, q14\n" + "vand q5, q5, q14\n" + + /* low nibbles */ + "vtbl.8 d12, {d24-d25}, d0\n" + "vtbl.8 d13, {d24-d25}, d1\n" + "vtbl.8 d14, {d24-d25}, d2\n" + "vtbl.8 d15, {d24-d25}, d3\n" + + /* high nibbles */ + "vtbl.8 d16, {d26-d27}, d8\n" + "vtbl.8 d17, {d26-d27}, d9\n" + "vtbl.8 d18, {d26-d27}, d10\n" + "vtbl.8 d19, {d26-d27}, d11\n" + + "veor q6, q6, q8\n" + "veor q7, q7, q9\n" + + "vst1.8 {q6}, %0\n" + "vst1.8 {q7}, %1\n" + : "=Q" (pa[i]), "=Q" (pa[i + 16]) + : "Q" (p[i]), "Q" (p[i + 16]), + "Q" (pa[i]), "Q" (pa[i + 16]) + ); + } + + raid_neon32_end(); +} + +/* + * RAID recovering for two disks AArch32 NEON implementation + */ +void raid_rec2_neon32(int nr, int *id, int *ip, int nd, + size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + const int N = 2; + uint8_t *p[2]; + uint8_t *pa[2]; + uint8_t G[2 * 2]; + uint8_t V[2 * 2]; + size_t i; + int j, k; + + (void)nr; /* unused, it's always 2 */ + + /* setup the coefficients matrix */ + for (j = 0; j < N; ++j) + for (k = 0; k < N; ++k) + G[j * N + k] = A(ip[j], id[k]); + + /* invert it to solve the system of linear equations */ + raid_invert(G, V, N); + + /* compute delta parity */ + raid_delta_gen(N, id, ip, nd, size, vv); + + for (j = 0; j < N; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + raid_neon32_begin(); + + /* + * q4-q11 contain the four low/high multiplication table pairs. + * q15 contains low4. + */ + asm volatile ( + "vld1.8 {q15}, %0\n" + + "vld1.8 {q4}, %1\n" + "vld1.8 {q5}, %2\n" + + "vld1.8 {q6}, %3\n" + "vld1.8 {q7}, %4\n" + + "vld1.8 {q8}, %5\n" + "vld1.8 {q9}, %6\n" + + "vld1.8 {q10}, %7\n" + "vld1.8 {q11}, %8\n" + : + : "Q" (gfconst16.low4[0]), + "Q" (raid_gfmulpshufb[V[0]][0][0]), + "Q" (raid_gfmulpshufb[V[0]][1][0]), + "Q" (raid_gfmulpshufb[V[1]][0][0]), + "Q" (raid_gfmulpshufb[V[1]][1][0]), + "Q" (raid_gfmulpshufb[V[2]][0][0]), + "Q" (raid_gfmulpshufb[V[2]][1][0]), + "Q" (raid_gfmulpshufb[V[3]][0][0]), + "Q" (raid_gfmulpshufb[V[3]][1][0]) + ); + + for (i = 0; i < size; i += 16) { + asm volatile ( + /* delta 0 */ + "vld1.8 {q0}, %2\n" + "vld1.8 {q12}, %4\n" + "veor q0, q0, q12\n" + "vshr.u8 q1, q0, #4\n" + "vand q0, q0, q15\n" + "vand q1, q1, q15\n" + + /* delta 1 */ + "vld1.8 {q2}, %3\n" + "vld1.8 {q12}, %5\n" + "veor q2, q2, q12\n" + "vshr.u8 q3, q2, #4\n" + "vand q2, q2, q15\n" + "vand q3, q3, q15\n" + + /* pa[0] = V[0] * delta0 */ + + "vtbl.8 d24, {d8-d9}, d0\n" + "vtbl.8 d25, {d8-d9}, d1\n" + "vtbl.8 d26, {d10-d11}, d2\n" + "vtbl.8 d27, {d10-d11}, d3\n" + "veor q14, q12, q13\n" + + /* ^ V[1] * delta1 */ + + "vtbl.8 d24, {d12-d13}, d4\n" + "vtbl.8 d25, {d12-d13}, d5\n" + "vtbl.8 d26, {d14-d15}, d6\n" + "vtbl.8 d27, {d14-d15}, d7\n" + "veor q12, q12, q13\n" + "veor q14, q14, q12\n" + + "vst1.8 {q14}, %0\n" + + /* pa[1] = V[2] * delta0 */ + + "vtbl.8 d24, {d16-d17}, d0\n" + "vtbl.8 d25, {d16-d17}, d1\n" + "vtbl.8 d26, {d18-d19}, d2\n" + "vtbl.8 d27, {d18-d19}, d3\n" + "veor q14, q12, q13\n" + + /* ^ V[3] * delta1 */ + + "vtbl.8 d24, {d20-d21}, d4\n" + "vtbl.8 d25, {d20-d21}, d5\n" + "vtbl.8 d26, {d22-d23}, d6\n" + "vtbl.8 d27, {d22-d23}, d7\n" + "veor q12, q12, q13\n" + "veor q14, q14, q12\n" + + "vst1.8 {q14}, %1\n" + : "=Q" (pa[0][i]), "=Q" (pa[1][i]) + : "Q" (p[0][i]), "Q" (p[1][i]), + "Q" (pa[0][i]), "Q" (pa[1][i]) + ); + } + + raid_neon32_end(); +} + +/* + * RAID recovering AArch32 NEON implementation + */ +void raid_recX_neon32(int nr, int *id, int *ip, int nd, + size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + int N = nr; + uint8_t *p[RAID_PARITY_MAX]; + uint8_t *pa[RAID_PARITY_MAX]; + uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; + const uint8_t *T[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t D[RAID_PARITY_MAX][2][16] __aligned(16); + size_t i; + int j, k; + + /* setup the coefficients matrix */ + for (j = 0; j < N; ++j) + for (k = 0; k < N; ++k) + G[j * N + k] = A(ip[j], id[k]); + + /* invert it to solve the system of linear equations */ + raid_invert(G, V, N); + + /* precompute shuffle table pointers */ + for (j = 0; j < N * N; ++j) + T[j] = &raid_gfmulpshufb[V[j]][0][0]; + + /* compute delta parity */ + raid_delta_gen(N, id, ip, nd, size, vv); + + for (j = 0; j < N; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + raid_neon32_begin(); + + /* preload mask */ + asm volatile ( + "vld1.8 {q15}, %0\n" + : + : "Q" (gfconst16.low4[0]) + ); + + for (i = 0; i < size; i += 16) { + for (k = 0; k < N; ++k) { + asm volatile ( + "vld1.8 {q0}, %2\n" + "vld1.8 {q1}, %3\n" + "veor q0, q0, q1\n" + + "vshr.u8 q1, q0, #4\n" + "vand q0, q0, q15\n" + "vand q1, q1, q15\n" + + "vst1.8 {q0}, %0\n" + "vst1.8 {q1}, %1\n" + : "=Q" (D[k][0][0]), "=Q" (D[k][1][0]) + : "Q" (p[k][i]), "Q" (pa[k][i]) + ); + } + + /* reconstruct */ + for (j = 0; j < N; ++j) { + asm volatile ( + "veor q0, q0, q0\n" + ); + + for (k = 0; k < N; ++k) { + asm volatile ( + "vld1.8 {q2}, %0\n" + "vld1.8 {q3}, %1\n" + + "vld1.8 {q4}, %2\n" + "vld1.8 {q5}, %3\n" + + "vtbl.8 d12, {d8-d9}, d4\n" + "vtbl.8 d13, {d8-d9}, d5\n" + + "vtbl.8 d14, {d10-d11}, d6\n" + "vtbl.8 d15, {d10-d11}, d7\n" + + "veor q6, q6, q7\n" + "veor q0, q0, q6\n" + : + : "Q" (D[k][0][0]), + "Q" (D[k][1][0]), + "Q" (T[j * N + k][0]), + "Q" (T[j * N + k][16]) + ); + } + + asm volatile ( + "vst1.8 {q0}, %0\n" + : "=Q" (pa[j][i]) + ); + } + } + + raid_neon32_end(); +} + +void raid_register_neon32(void) +{ + raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "neon32", raid_gen1_neon32, RAID_POLY_ANY); + + raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "neon32", raid_gen2_neon32_raid, RAID_POLY_RAID); + raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "neon32", raid_gen2_neon32_aes, RAID_POLY_AES); + + raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "neon32", raid_genz_neon32_raid, RAID_POLY_RAID); + + raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "neon32", raid_gen3_neon32_raid, RAID_POLY_RAID); + raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "neon32", raid_gen3_neon32_aes, RAID_POLY_AES); + + raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "neon32", raid_gen4_neon32_raid, RAID_POLY_RAID); + raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "neon32", raid_gen4_neon32_aes, RAID_POLY_AES); + + raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "neon32", raid_gen5_neon32_raid, RAID_POLY_RAID); + raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "neon32", raid_gen5_neon32_aes, RAID_POLY_AES); + + raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "neon32", raid_gen6_neon32_raid, RAID_POLY_RAID); + raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "neon32", raid_gen6_neon32_aes, RAID_POLY_AES); + + raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "neon32", raid_rec1_neon32, RAID_POLY_ANY); + raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "neon32", raid_rec2_neon32, RAID_POLY_ANY); + raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "neon32", raid_recX_neon32, RAID_POLY_ANY); + raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "neon32", raid_recX_neon32, RAID_POLY_ANY); + raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "neon32", raid_recX_neon32, RAID_POLY_ANY); + raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "neon32", raid_recX_neon32, RAID_POLY_ANY); +} + +#endif diff --git a/raid/tables.c b/raid/tables.c index 932599e0..2384ec9c 100644 --- a/raid/tables.c +++ b/raid/tables.c @@ -9272,7 +9272,7 @@ const uint8_t __aligned(256) raid_gfcauchy_raid[6][256] = }, }; -#if defined(CONFIG_X86) || defined(CONFIG_NEON) +#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32) /** * PSHUFB tables for the Cauchy matrix. * @@ -14806,7 +14806,7 @@ const uint8_t __aligned(256) raid_gfcauchypshufb_raid[251][5][2][16] = }; #endif -#if defined(CONFIG_X86) || defined(CONFIG_NEON) +#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32) /** * PSHUFB tables for generic multiplication. * @@ -24957,7 +24957,7 @@ const uint8_t __aligned(256) raid_gfcauchy_aes[6][256] = }, }; -#if defined(CONFIG_X86) || defined(CONFIG_NEON) +#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32) /** * PSHUFB tables for the Cauchy matrix. * @@ -30491,7 +30491,7 @@ const uint8_t __aligned(256) raid_gfcauchypshufb_aes[251][5][2][16] = }; #endif -#if defined(CONFIG_X86) || defined(CONFIG_NEON) +#if defined(CONFIG_X86) || defined(CONFIG_NEON) || defined(CONFIG_NEON32) /** * PSHUFB tables for generic multiplication. * @@ -31526,4 +31526,3 @@ const uint8_t __aligned(256) raid_gfmulpshufb_aes[256][2][16] = }, }; #endif - diff --git a/raid/test.c b/raid/test.c index a2155dee..c6c33276 100644 --- a/raid/test.c +++ b/raid/test.c @@ -417,6 +417,9 @@ int raid_test_rec(int mode, int nd, size_t size) #endif #ifdef CONFIG_NEON test_setup(i) = raid_rec1_neon; +#endif +#ifdef CONFIG_NEON32 + test_setup(i) = raid_rec1_neon32; #endif } else if (i == 2) { test_setup(i) = raid_rec2_int8; @@ -446,6 +449,9 @@ int raid_test_rec(int mode, int nd, size_t size) #endif #ifdef CONFIG_NEON test_setup(i) = raid_rec2_neon; +#endif +#ifdef CONFIG_NEON32 + test_setup(i) = raid_rec2_neon32; #endif } else { test_setup(i) = raid_recX_int8; @@ -475,6 +481,9 @@ int raid_test_rec(int mode, int nd, size_t size) #endif #ifdef CONFIG_NEON test_setup(i) = raid_recX_neon; +#endif +#ifdef CONFIG_NEON32 + test_setup(i) = raid_recX_neon32; #endif } } @@ -606,6 +615,13 @@ int raid_test_par(int mode, int nd, size_t size) else test_setup(2) = raid_gen2_neon_raid; #endif +#ifdef CONFIG_NEON32 + test_setup(1) = raid_gen1_neon32; + if (mode == RAID_MODE_CAUCHY_AES) + test_setup(2) = raid_gen2_neon32_aes; + else + test_setup(2) = raid_gen2_neon32_raid; +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { @@ -673,6 +689,19 @@ int raid_test_par(int mode, int nd, size_t size) test_setup(6) = raid_gen6_neon_raid; } #endif +#ifdef CONFIG_NEON32 + if (mode == RAID_MODE_CAUCHY_AES) { + test_setup(3) = raid_gen3_neon32_aes; + test_setup(4) = raid_gen4_neon32_aes; + test_setup(5) = raid_gen5_neon32_aes; + test_setup(6) = raid_gen6_neon32_aes; + } else { + test_setup(3) = raid_gen3_neon32_raid; + test_setup(4) = raid_gen4_neon32_raid; + test_setup(5) = raid_gen5_neon32_raid; + test_setup(6) = raid_gen6_neon32_raid; + } +#endif #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { @@ -754,6 +783,9 @@ int raid_test_par(int mode, int nd, size_t size) #ifdef CONFIG_NEON test_setup(3) = raid_genz_neon_raid; #endif +#ifdef CONFIG_NEON32 + test_setup(3) = raid_genz_neon32_raid; +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { diff --git a/raid/test/Makefile b/raid/test/Makefile index 78858ed4..29c51cb1 100644 --- a/raid/test/Makefile +++ b/raid/test/Makefile @@ -36,7 +36,7 @@ CFLAGS += -O2 else CFLAGS += -O0 --coverage -DCOVERAGE=1 -DNDEBUG=1 endif -OBJS = raid.o check.o int.o sse2.o ssse3.o avx2.o avx512.o gfni.o neon.o tables.o memory.o test.o helper.o module.o +OBJS = raid.o check.o int.o sse2.o ssse3.o avx2.o avx512.o gfni.o neon.o neon32.o tables.o memory.o test.o helper.o module.o %.o: ../%.c $(CC) $(CFLAGS) -c -o $@ $< diff --git a/raid/test/fulltest.c b/raid/test/fulltest.c index df53042c..5c898486 100644 --- a/raid/test/fulltest.c +++ b/raid/test/fulltest.c @@ -33,6 +33,9 @@ int main(void) #ifdef CONFIG_NEON printf("Including ARM NEON\n"); #endif +#ifdef CONFIG_NEON32 + printf("Including ARM NEON\n"); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) printf("Including x86 SSE2\n"); diff --git a/raid/test/speedtest.c b/raid/test/speedtest.c index eeab9157..591ff39a 100644 --- a/raid/test/speedtest.c +++ b/raid/test/speedtest.c @@ -107,6 +107,9 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms #ifdef CONFIG_NEON printf("%8s", "neon"); #endif +#ifdef CONFIG_NEON32 + printf("%8s", "neon32"); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { printf("%8s", "sse2"); @@ -169,6 +172,14 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + SPEED_START { + raid_gen1_neon32(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { @@ -272,6 +283,20 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + if (mode == RAID_MODE_CAUCHY_AES) { + SPEED_START { + raid_gen2_neon32_aes(nd, size, v); + } SPEED_STOP + } else { + SPEED_START { + raid_gen2_neon32_raid(nd, size, v); + } SPEED_STOP + } + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { @@ -412,6 +437,20 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + if (mode == RAID_MODE_CAUCHY_AES) { + SPEED_START { + raid_gen3_neon32_aes(nd, size, v); + } SPEED_STOP + } else { + SPEED_START { + raid_gen3_neon32_raid(nd, size, v); + } SPEED_STOP + } + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { @@ -539,6 +578,20 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + if (mode == RAID_MODE_CAUCHY_AES) { + SPEED_START { + raid_gen4_neon32_aes(nd, size, v); + } SPEED_STOP + } else { + SPEED_START { + raid_gen4_neon32_raid(nd, size, v); + } SPEED_STOP + } + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { @@ -666,6 +719,20 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + if (mode == RAID_MODE_CAUCHY_AES) { + SPEED_START { + raid_gen5_neon32_aes(nd, size, v); + } SPEED_STOP + } else { + SPEED_START { + raid_gen5_neon32_raid(nd, size, v); + } SPEED_STOP + } + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { @@ -791,6 +858,20 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + if (mode == RAID_MODE_CAUCHY_AES) { + SPEED_START { + raid_gen6_neon32_aes(nd, size, v); + } SPEED_STOP + } else { + SPEED_START { + raid_gen6_neon32_raid(nd, size, v); + } SPEED_STOP + } + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { @@ -908,6 +989,9 @@ void speed_genz(int nd, void **v, int size, int delta, int period) #ifdef CONFIG_NEON printf("%8s", "neon"); #endif +#ifdef CONFIG_NEON32 + printf("%8s", "neon32"); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { printf("%8s", "sse2"); @@ -959,6 +1043,14 @@ void speed_genz(int nd, void **v, int size, int delta, int period) printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + SPEED_START { + raid_genz_neon32_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { @@ -1026,6 +1118,9 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #ifdef CONFIG_NEON printf("%8s", "neon"); #endif +#ifdef CONFIG_NEON32 + printf("%8s", "neon32"); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) printf("%8s", "ssse3"); @@ -1064,6 +1159,15 @@ void speed_rec(int nd, void **v, int size, int delta, int period) printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + SPEED_START { + raid_gen_force(1, raid_gen1_neon32); + raid_rec1_neon32(1, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { @@ -1140,6 +1244,16 @@ void speed_rec(int nd, void **v, int size, int delta, int period) printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + SPEED_START { + raid_gen_force(1, raid_gen1_neon32); + /* +1 to avoid GEN1 optimized case */ + raid_rec1_neon32(1, id, ip + 1, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { @@ -1219,6 +1333,15 @@ void speed_rec(int nd, void **v, int size, int delta, int period) printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + SPEED_START { + raid_gen_force(2, raid_gen2_neon32_raid); + raid_rec2_neon32(2, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { @@ -1303,6 +1426,16 @@ void speed_rec(int nd, void **v, int size, int delta, int period) printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + SPEED_START { + raid_gen_force(2, raid_gen2_neon32_raid); + /* +1 to avoid GEN2 optimized case */ + raid_rec2_neon32(2, id, ip + 1, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { @@ -1390,6 +1523,15 @@ void speed_rec(int nd, void **v, int size, int delta, int period) printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + SPEED_START { + raid_gen_force(3, raid_gen3_neon32_raid); + raid_recX_neon32(3, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { @@ -1473,6 +1615,15 @@ void speed_rec(int nd, void **v, int size, int delta, int period) printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + SPEED_START { + raid_gen_force(4, raid_gen4_neon32_raid); + raid_recX_neon32(4, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { @@ -1556,6 +1707,15 @@ void speed_rec(int nd, void **v, int size, int delta, int period) printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + SPEED_START { + raid_gen_force(5, raid_gen5_neon32_raid); + raid_recX_neon32(5, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { @@ -1639,6 +1799,15 @@ void speed_rec(int nd, void **v, int size, int delta, int period) printf("%8" PRIu64, ds / dt); fflush(stdout); #endif +#ifdef CONFIG_NEON32 + SPEED_START { + raid_gen_force(6, raid_gen6_neon32_raid); + raid_recX_neon32(6, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { @@ -1761,6 +1930,9 @@ int main(void) #ifdef CONFIG_NEON printf("Including ARM NEON\n"); #endif +#ifdef CONFIG_NEON32 + printf("Including ARM NEON\n"); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) printf("Including x86 SSE2\n");