diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7d4476d2..8b311acc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,7 +16,8 @@ jobs: - run: brew install automake - run: autoreconf -i - run: ./configure --enable-warning-as-error - - run: make all dist check + - run: cd raid/test && make all test + - run: make all dist check build-linux: runs-on: ubuntu-latest steps: diff --git a/HISTORY b/HISTORY index 00340fb6..eca904db 100644 --- a/HISTORY +++ b/HISTORY @@ -22,6 +22,9 @@ SnapRAID HISTORY By scaling to 512-bit ZMM registers, the implementation now processes 64 bytes per iteration, effectively doubling the data width of the previous AVX2 routines. + * Implemented AArch64 (ARM64) NEON support for parity generation and + data recovery, significantly improving performance on ARM-based + platforms. * Improved the chances of a successful recovery after an interrupted sync. This is achieved by removing the sync optimization of not recomputing the parity if the input data matches the expected hash. diff --git a/Makefile.am b/Makefile.am index d5587087..c99e3872 100644 --- a/Makefile.am +++ b/Makefile.am @@ -27,6 +27,7 @@ snapraid_SOURCES = \ raid/int.c \ raid/x86.c \ raid/gfni.c \ + raid/neon.c \ raid/intz.c \ raid/x86z.c \ raid/helper.c \ diff --git a/cmdline/speed.c b/cmdline/speed.c index fed92e68..150b4d11 100644 --- a/cmdline/speed.c +++ b/cmdline/speed.c @@ -274,6 +274,9 @@ void speed(int period, int nd, int size) printf("%8s", "int8"); printf("%8s", "int32"); printf("%8s", "int64"); +#ifdef CONFIG_NEON + printf("%8s", "neon"); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { printf("%8s", "sse2"); @@ -325,6 +328,15 @@ void speed(int period, int nd, int size) printf("%8" PRIu64, ds / dt); fflush(stdout); +#ifdef CONFIG_NEON + SPEED_START { + raid_gen1_neon(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { SPEED_START { @@ -388,6 +400,15 @@ void speed(int period, int nd, int size) printf("%8" PRIu64, ds / dt); fflush(stdout); +#ifdef CONFIG_NEON + SPEED_START { + raid_gen2_neon(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { SPEED_START { @@ -473,6 +494,15 @@ void speed(int period, int nd, int size) printf("%8" PRIu64, ds / dt); fflush(stdout); +#ifdef CONFIG_NEON + SPEED_START { + raid_genz_neon(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { SPEED_START { @@ -526,6 +556,15 @@ void speed(int period, int nd, int size) printf("%8s", ""); printf("%8s", ""); +#ifdef CONFIG_NEON + SPEED_START { + raid_gen3_neon(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { printf("%8s", ""); @@ -606,6 +645,15 @@ void speed(int period, int nd, int size) printf("%8s", ""); printf("%8s", ""); +#ifdef CONFIG_NEON + SPEED_START { + raid_gen4_neon(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { printf("%8s", ""); @@ -686,6 +734,15 @@ void speed(int period, int nd, int size) printf("%8s", ""); printf("%8s", ""); +#ifdef CONFIG_NEON + SPEED_START { + raid_gen5_neon(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { printf("%8s", ""); @@ -765,6 +822,15 @@ void speed(int period, int nd, int size) printf("%8s", ""); printf("%8s", ""); +#ifdef CONFIG_NEON + SPEED_START { + raid_gen6_neon(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { printf("%8s", ""); @@ -835,6 +901,9 @@ void speed(int period, int nd, int size) printf("%8s", ""); printf("%8s", "best"); printf("%8s", "int8"); +#ifdef CONFIG_NEON + printf("%8s", "neon"); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) printf("%8s", "ssse3"); @@ -865,6 +934,18 @@ void speed(int period, int nd, int size) printf("%8" PRIu64, ds / dt); fflush(stdout); +#ifdef CONFIG_NEON + SPEED_START { + /* ensure to use same hardware in the delta step */ + raid_gen_force(1, raid_gen1_neon); + /* +1 to avoid GEN1 optimized case */ + raid_rec1_neon(1, id, ip + 1, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { SPEED_START { @@ -940,6 +1021,18 @@ void speed(int period, int nd, int size) printf("%8" PRIu64, ds / dt); fflush(stdout); +#ifdef CONFIG_NEON + SPEED_START { + /* ensure to use same hardware in the delta step */ + raid_gen_force(2, raid_gen2_neon); + /* +1 to avoid GEN2 optimized case */ + raid_rec2_neon(2, id, ip + 1, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { SPEED_START { @@ -1018,6 +1111,17 @@ void speed(int period, int nd, int size) printf("%8" PRIu64, ds / dt); fflush(stdout); +#ifdef CONFIG_NEON + SPEED_START { + /* ensure to use same hardware in the delta step */ + raid_gen_force(3, raid_gen3_neon); + raid_recX_neon(3, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { SPEED_START { @@ -1096,6 +1200,17 @@ void speed(int period, int nd, int size) printf("%8" PRIu64, ds / dt); fflush(stdout); +#ifdef CONFIG_NEON + SPEED_START { + /* ensure to use same hardware in the delta step */ + raid_gen_force(4, raid_gen4_neon); + raid_recX_neon(4, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { SPEED_START { @@ -1175,6 +1290,17 @@ void speed(int period, int nd, int size) printf("%8" PRIu64, ds / dt); fflush(stdout); +#ifdef CONFIG_NEON + SPEED_START { + /* ensure to use same hardware in the delta step */ + raid_gen_force(5, raid_gen5_neon); + raid_recX_neon(5, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { SPEED_START { @@ -1253,6 +1379,17 @@ void speed(int period, int nd, int size) printf("%8" PRIu64, ds / dt); fflush(stdout); +#ifdef CONFIG_NEON + SPEED_START { + /* ensure to use same hardware in the delta step */ + raid_gen_force(6, raid_gen6_neon); + raid_recX_neon(6, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { SPEED_START { diff --git a/raid/internal.h b/raid/internal.h index dea56e21..f383c470 100644 --- a/raid/internal.h +++ b/raid/internal.h @@ -23,7 +23,13 @@ #define CONFIG_X86 1 #define CONFIG_X86_64 1 #endif +#if defined(__arm__) || defined(__aarch64__) +#if defined(__ARM_NEON) || defined(__ARM_NEON__) +#define CONFIG_NEON 1 #endif +#endif +#endif + /* * Includes anything required for compatibility. @@ -144,6 +150,16 @@ void raid_recX_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv void raid_rec1_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec2_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_recX_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_gen1_neon(int nd, size_t size, void **vv); +void raid_gen2_neon(int nd, size_t size, void **vv); +void raid_genz_neon(int nd, size_t size, void **vv); +void raid_gen3_neon(int nd, size_t size, void **vv); +void raid_gen4_neon(int nd, size_t size, void **vv); +void raid_gen5_neon(int nd, size_t size, void **vv); +void raid_gen6_neon(int nd, size_t size, void **vv); +void raid_rec1_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec2_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv); /* * Functions for parity computation. @@ -348,5 +364,23 @@ static __always_inline void raid_avx_end(void) } #endif /* CONFIG_X86 */ +#ifdef CONFIG_NEON +static __always_inline void raid_neon_begin(void) +{ +} + +static __always_inline void raid_neon_end(void) +{ + asm volatile ("" : : : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7"); + asm volatile ("" : : : "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15"); + asm volatile ("" : : : "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23"); + asm volatile ("" : : : "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"); +} + +void raid_register_neon(void); #endif +#endif + + + diff --git a/raid/module.c b/raid/module.c index ab0f9fce..f02ba966 100644 --- a/raid/module.c +++ b/raid/module.c @@ -251,6 +251,9 @@ void raid_init(void) raid_register_avx2gfni(); raid_register_avx512gfni(); #endif +#ifdef CONFIG_NEON + raid_register_neon(); +#endif /* set the default mode */ raid_mode(RAID_MODE_CAUCHY); diff --git a/raid/neon.c b/raid/neon.c new file mode 100644 index 00000000..8c2002d1 --- /dev/null +++ b/raid/neon.c @@ -0,0 +1,1408 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +// Copyright (C) 2026 Andrea Mazzoleni + +#include "internal.h" +#include "gf.h" +#include "cpu.h" + +#ifdef CONFIG_NEON + +/* + * GEN1 (RAID5 with xor) NEON implementation + */ +void raid_gen1_neon(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + + raid_neon_begin(); + + for (i = 0; i < size; i += 64) { + asm volatile ( + "ldr q0, %0\n" + "ldr q1, %1\n" + "ldr q2, %2\n" + "ldr q3, %3\n" + : + : "m" (v[0][i]), "m" (v[0][i + 16]), "m" (v[0][i + 32]), "m" (v[0][i + 48]) + ); + + for (d = 1; d <= l; ++d) { + asm volatile ( + "ldr q4, %0\n" + "ldr q5, %1\n" + "ldr q6, %2\n" + "ldr q7, %3\n" + "eor v0.16b, v0.16b, v4.16b\n" + "eor v1.16b, v1.16b, v5.16b\n" + "eor v2.16b, v2.16b, v6.16b\n" + "eor v3.16b, v3.16b, v7.16b\n" + : + : "m" (v[d][i]), "m" (v[d][i + 16]), "m" (v[d][i + 32]), "m" (v[d][i + 48]) + ); + } + + asm volatile ( + "str q0, %0\n" + "str q1, %1\n" + "str q2, %2\n" + "str q3, %3\n" + : "=m" (p[i]), "=m" (p[i + 16]), "=m" (p[i + 32]), "=m" (p[i + 48]) + ); + } + + raid_neon_end(); +} + +static const struct gfconst16 { + uint8_t poly[16]; + uint8_t low4[16]; + uint8_t half[16]; +} gfconst16 __aligned(16) = { + { + RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, + RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY + }, + { + 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, + 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f + }, + { + RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, + RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE + } +}; + +/* + * GEN2 (RAID6 with powers of 2) NEON implementation + */ +void raid_gen2_neon(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + + raid_neon_begin(); + + asm volatile ( + "ldr q29, %0\n" + : + : "m" (gfconst16.poly[0]) + ); + + for (i = 0; i < size; i += 32) { + asm volatile ( + "ldr q0, %0\n" + "ldr q1, %1\n" + "mov v2.16b, v0.16b\n" + "mov v3.16b, v1.16b\n" + : + : "m" (v[l][i]), "m" (v[l][i + 16]) + ); + + for (d = l - 1; d >= 0; --d) { + asm volatile ( + /* double Q0-Q1 */ + "sshr v16.16b, v2.16b, #7\n" + "shl v2.16b, v2.16b, #1\n" + "and v16.16b, v16.16b, v29.16b\n" + "eor v2.16b, v2.16b, v16.16b\n" + + "sshr v16.16b, v3.16b, #7\n" + "shl v3.16b, v3.16b, #1\n" + "and v16.16b, v16.16b, v29.16b\n" + "eor v3.16b, v3.16b, v16.16b\n" + + /* load and XOR */ + "ldr q12, %0\n" + "ldr q13, %1\n" + "eor v0.16b, v0.16b, v12.16b\n" + "eor v1.16b, v1.16b, v13.16b\n" + "eor v2.16b, v2.16b, v12.16b\n" + "eor v3.16b, v3.16b, v13.16b\n" + : + : "m" (v[d][i]), "m" (v[d][i + 16]) + ); + } + + asm volatile ( + "str q0, %0\n" + "str q1, %1\n" + "str q2, %2\n" + "str q3, %3\n" + : "=m" (p[i]), "=m" (p[i + 16]), + "=m" (q[i]), "=m" (q[i + 16]) + ); + } + + raid_neon_end(); +} + +/* + * GENz (triple parity with powers of 2^-1) NEON implementation + */ +void raid_genz_neon(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + + raid_neon_begin(); + + asm volatile ( + "ldr q29, %0\n" + "ldr q30, %1\n" + : + : "m" (gfconst16.poly[0]), "m" (gfconst16.half[0]) + ); + + for (i = 0; i < size; i += 32) { + asm volatile ( + "ldr q0, %0\n" + "ldr q1, %1\n" + "mov v2.16b, v0.16b\n" + "mov v3.16b, v1.16b\n" + "mov v4.16b, v0.16b\n" + "mov v5.16b, v1.16b\n" + : + : "m" (v[l][i]), "m" (v[l][i + 16]) + ); + + for (d = l - 1; d >= 0; --d) { + asm volatile ( + /* double Q0-Q1 */ + "sshr v16.16b, v2.16b, #7\n" + "shl v2.16b, v2.16b, #1\n" + "and v16.16b, v16.16b, v29.16b\n" + "eor v2.16b, v2.16b, v16.16b\n" + + "sshr v16.16b, v3.16b, #7\n" + "shl v3.16b, v3.16b, #1\n" + "and v16.16b, v16.16b, v29.16b\n" + "eor v3.16b, v3.16b, v16.16b\n" + + /* halve R0-R1 */ + "shl v17.16b, v4.16b, #7\n" + "sshr v17.16b, v17.16b, #7\n" + "ushr v4.16b, v4.16b, #1\n" + "and v17.16b, v17.16b, v30.16b\n" + "eor v4.16b, v4.16b, v17.16b\n" + + "shl v17.16b, v5.16b, #7\n" + "sshr v17.16b, v17.16b, #7\n" + "ushr v5.16b, v5.16b, #1\n" + "and v17.16b, v17.16b, v30.16b\n" + "eor v5.16b, v5.16b, v17.16b\n" + + /* load and XOR */ + "ldr q12, %0\n" + "ldr q13, %1\n" + "eor v0.16b, v0.16b, v12.16b\n" + "eor v1.16b, v1.16b, v13.16b\n" + "eor v2.16b, v2.16b, v12.16b\n" + "eor v3.16b, v3.16b, v13.16b\n" + "eor v4.16b, v4.16b, v12.16b\n" + "eor v5.16b, v5.16b, v13.16b\n" + : + : "m" (v[d][i]), "m" (v[d][i + 16]) + ); + } + + asm volatile ( + "str q0, %0\n" + "str q1, %1\n" + "str q2, %2\n" + "str q3, %3\n" + "str q4, %4\n" + "str q5, %5\n" + : "=m" (p[i]), "=m" (p[i + 16]), + "=m" (q[i]), "=m" (q[i + 16]), + "=m" (r[i]), "=m" (r[i + 16]) + ); + } + + raid_neon_end(); +} + +/* + * GEN3 (triple parity with Cauchy matrix) NEON implementation + */ +void raid_gen3_neon(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + + /* special case with only one data disk */ + if (l == 0) { + for (i = 0; i < 3; ++i) + memcpy(v[1 + i], v[0], size); + return; + } + + raid_neon_begin(); + + /* generic case with at least two data disks */ + asm volatile ( + "ldr q29, %0\n" + "ldr q28, %1\n" + : + : "m" (gfconst16.poly[0]), "m" (gfconst16.low4[0]) + ); + + for (i = 0; i < size; i += 32) { + /* last disk without the multiplication by two */ + asm volatile ( + "ldr q0, %0\n" + "ldr q1, %1\n" + "mov v2.16b, v0.16b\n" + "mov v3.16b, v1.16b\n" + : + : "m" (v[l][i]), "m" (v[l][i + 16]) + ); + + asm volatile ( + "ldr q24, %0\n" + "ldr q25, %1\n" + /* chunk 0 */ + "ushr v17.16b, v0.16b, #4\n" + "and v16.16b, v0.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v8.16b, {v24.16b}, v16.16b\n" + "tbl v20.16b, {v25.16b}, v17.16b\n" + "eor v8.16b, v8.16b, v20.16b\n" + /* chunk 1 */ + "ushr v17.16b, v1.16b, #4\n" + "and v16.16b, v1.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v9.16b, {v24.16b}, v16.16b\n" + "tbl v20.16b, {v25.16b}, v17.16b\n" + "eor v9.16b, v9.16b, v20.16b\n" + : + : "m" (gfgenpshufb[l][1][0][0]), "m" (gfgenpshufb[l][1][1][0]) + ); + + /* intermediate disks */ + for (d = l - 1; d > 0; --d) { + asm volatile ( + /* double Q0-Q1 */ + "sshr v16.16b, v2.16b, #7\n" + "shl v2.16b, v2.16b, #1\n" + "and v16.16b, v16.16b, v29.16b\n" + "eor v2.16b, v2.16b, v16.16b\n" + + "sshr v16.16b, v3.16b, #7\n" + "shl v3.16b, v3.16b, #1\n" + "and v16.16b, v16.16b, v29.16b\n" + "eor v3.16b, v3.16b, v16.16b\n" + + /* load disk data and XOR into P and Q */ + "ldr q12, %0\n" + "ldr q13, %1\n" + "eor v0.16b, v0.16b, v12.16b\n" + "eor v1.16b, v1.16b, v13.16b\n" + "eor v2.16b, v2.16b, v12.16b\n" + "eor v3.16b, v3.16b, v13.16b\n" + + /* load R tables */ + "ldr q24, %2\n" + "ldr q25, %3\n" + + /* cauchy multiply v12-v13 and XOR into R */ + "ushr v17.16b, v12.16b, #4\n" + "and v16.16b, v12.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v20.16b, {v24.16b}, v16.16b\n" + "tbl v21.16b, {v25.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v8.16b, v8.16b, v20.16b\n" + + "ushr v17.16b, v13.16b, #4\n" + "and v16.16b, v13.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v20.16b, {v24.16b}, v16.16b\n" + "tbl v21.16b, {v25.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v9.16b, v9.16b, v20.16b\n" + : + : "m" (v[d][i]), "m" (v[d][i + 16]), + "m" (gfgenpshufb[d][1][0][0]), "m" (gfgenpshufb[d][1][1][0]) + ); + } + + /* first disk with all coefficients at 1 */ + asm volatile ( + /* double Q0-Q1 */ + "sshr v16.16b, v2.16b, #7\n" + "shl v2.16b, v2.16b, #1\n" + "and v16.16b, v16.16b, v29.16b\n" + "eor v2.16b, v2.16b, v16.16b\n" + + "sshr v16.16b, v3.16b, #7\n" + "shl v3.16b, v3.16b, #1\n" + "and v16.16b, v16.16b, v29.16b\n" + "eor v3.16b, v3.16b, v16.16b\n" + + /* load disk 0 data and XOR into P, Q, R */ + "ldr q12, %0\n" + "ldr q13, %1\n" + "eor v0.16b, v0.16b, v12.16b\n" + "eor v1.16b, v1.16b, v13.16b\n" + "eor v2.16b, v2.16b, v12.16b\n" + "eor v3.16b, v3.16b, v13.16b\n" + "eor v8.16b, v8.16b, v12.16b\n" + "eor v9.16b, v9.16b, v13.16b\n" + : + : "m" (v[0][i]), "m" (v[0][i + 16]) + ); + + asm volatile ( + "str q0, %0\n" + "str q1, %1\n" + "str q2, %2\n" + "str q3, %3\n" + "str q8, %4\n" + "str q9, %5\n" + : "=m" (p[i]), "=m" (p[i + 16]), + "=m" (q[i]), "=m" (q[i + 16]), + "=m" (r[i]), "=m" (r[i + 16]) + ); + } + + raid_neon_end(); +} + +/* + * GEN4 (quad parity with Cauchy matrix) NEON implementation + */ +void raid_gen4_neon(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + uint8_t *s; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + s = v[nd + 3]; + + /* special case with only one data disk */ + if (l == 0) { + for (i = 0; i < 4; ++i) + memcpy(v[1 + i], v[0], size); + return; + } + + raid_neon_begin(); + + /* generic case with at least two data disks */ + asm volatile ( + "ldr q29, %0\n" + "ldr q28, %1\n" + : + : "m" (gfconst16.poly[0]), "m" (gfconst16.low4[0]) + ); + + for (i = 0; i < size; i += 32) { + /* last disk without the multiplication by two */ + asm volatile ( + "ldr q0, %0\n" + "ldr q1, %1\n" + "mov v2.16b, v0.16b\n" + "mov v3.16b, v1.16b\n" + : + : "m" (v[l][i]), "m" (v[l][i + 16]) + ); + + asm volatile ( + "ldr q24, %0\n" + "ldr q25, %1\n" + "ldr q26, %2\n" + "ldr q27, %3\n" + /* chunk 0 */ + "ushr v17.16b, v0.16b, #4\n" + "and v16.16b, v0.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v4.16b, {v24.16b}, v16.16b\n" + "tbl v20.16b, {v25.16b}, v17.16b\n" + "eor v4.16b, v4.16b, v20.16b\n" + "tbl v6.16b, {v26.16b}, v16.16b\n" + "tbl v20.16b, {v27.16b}, v17.16b\n" + "eor v6.16b, v6.16b, v20.16b\n" + /* chunk 1 */ + "ushr v17.16b, v1.16b, #4\n" + "and v16.16b, v1.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v5.16b, {v24.16b}, v16.16b\n" + "tbl v20.16b, {v25.16b}, v17.16b\n" + "eor v5.16b, v5.16b, v20.16b\n" + "tbl v7.16b, {v26.16b}, v16.16b\n" + "tbl v20.16b, {v27.16b}, v17.16b\n" + "eor v7.16b, v7.16b, v20.16b\n" + : + : "m" (gfgenpshufb[l][1][0][0]), "m" (gfgenpshufb[l][1][1][0]), + "m" (gfgenpshufb[l][2][0][0]), "m" (gfgenpshufb[l][2][1][0]) + ); + + /* intermediate disks */ + for (d = l - 1; d > 0; --d) { + asm volatile ( + /* double Q0-Q1 */ + "sshr v16.16b, v2.16b, #7\n" + "shl v2.16b, v2.16b, #1\n" + "and v16.16b, v16.16b, v29.16b\n" + "eor v2.16b, v2.16b, v16.16b\n" + + "sshr v16.16b, v3.16b, #7\n" + "shl v3.16b, v3.16b, #1\n" + "and v16.16b, v16.16b, v29.16b\n" + "eor v3.16b, v3.16b, v16.16b\n" + + /* load data */ + "ldr q12, %0\n" + "ldr q13, %1\n" + "eor v0.16b, v0.16b, v12.16b\n" + "eor v1.16b, v1.16b, v13.16b\n" + "eor v2.16b, v2.16b, v12.16b\n" + "eor v3.16b, v3.16b, v13.16b\n" + + /* load tables for R and S */ + "ldr q24, %2\n" + "ldr q25, %3\n" + "ldr q26, %4\n" + "ldr q27, %5\n" + + /* multiply and XOR into R and S */ + /* chunk 0 */ + "ushr v17.16b, v12.16b, #4\n" + "and v16.16b, v12.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v20.16b, {v24.16b}, v16.16b\n" + "tbl v21.16b, {v25.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v4.16b, v4.16b, v20.16b\n" + "tbl v20.16b, {v26.16b}, v16.16b\n" + "tbl v21.16b, {v27.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v6.16b, v6.16b, v20.16b\n" + + /* chunk 1 */ + "ushr v17.16b, v13.16b, #4\n" + "and v16.16b, v13.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v20.16b, {v24.16b}, v16.16b\n" + "tbl v21.16b, {v25.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v5.16b, v5.16b, v20.16b\n" + "tbl v20.16b, {v26.16b}, v16.16b\n" + "tbl v21.16b, {v27.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v7.16b, v7.16b, v20.16b\n" + : + : "m" (v[d][i]), "m" (v[d][i + 16]), + "m" (gfgenpshufb[d][1][0][0]), "m" (gfgenpshufb[d][1][1][0]), + "m" (gfgenpshufb[d][2][0][0]), "m" (gfgenpshufb[d][2][1][0]) + ); + } + + /* first disk with all coefficients at 1 */ + asm volatile ( + /* double Q0-Q1 */ + "sshr v16.16b, v2.16b, #7\n" + "shl v2.16b, v2.16b, #1\n" + "and v16.16b, v16.16b, v29.16b\n" + "eor v2.16b, v2.16b, v16.16b\n" + + "sshr v16.16b, v3.16b, #7\n" + "shl v3.16b, v3.16b, #1\n" + "and v16.16b, v16.16b, v29.16b\n" + "eor v3.16b, v3.16b, v16.16b\n" + + /* load disk 0 data and XOR */ + "ldr q12, %0\n" + "ldr q13, %1\n" + "eor v0.16b, v0.16b, v12.16b\n" + "eor v1.16b, v1.16b, v13.16b\n" + "eor v2.16b, v2.16b, v12.16b\n" + "eor v3.16b, v3.16b, v13.16b\n" + "eor v4.16b, v4.16b, v12.16b\n" + "eor v5.16b, v5.16b, v13.16b\n" + "eor v6.16b, v6.16b, v12.16b\n" + "eor v7.16b, v7.16b, v13.16b\n" + : + : "m" (v[0][i]), "m" (v[0][i + 16]) + ); + + asm volatile ( + "str q0, %0\n" + "str q1, %1\n" + "str q2, %2\n" + "str q3, %3\n" + "str q4, %4\n" + "str q5, %5\n" + "str q6, %6\n" + "str q7, %7\n" + : "=m" (p[i]), "=m" (p[i + 16]), + "=m" (q[i]), "=m" (q[i + 16]), + "=m" (r[i]), "=m" (r[i + 16]), + "=m" (s[i]), "=m" (s[i + 16]) + ); + } + + raid_neon_end(); +} + +/* + * GEN5 (penta parity with Cauchy matrix) NEON implementation + */ +void raid_gen5_neon(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + uint8_t *s; + uint8_t *t; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + s = v[nd + 3]; + t = v[nd + 4]; + + /* special case with only one data disk */ + if (l == 0) { + for (i = 0; i < 5; ++i) + memcpy(v[1 + i], v[0], size); + return; + } + + raid_neon_begin(); + + /* generic case with at least two data disks */ + asm volatile ( + "ldr q29, %0\n" + "ldr q28, %1\n" + : + : "m" (gfconst16.poly[0]), "m" (gfconst16.low4[0]) + ); + + for (i = 0; i < size; i += 32) { + /* last disk without the multiplication by two */ + asm volatile ( + "ldr q0, %0\n" + "ldr q1, %1\n" + "mov v2.16b, v0.16b\n" + "mov v3.16b, v1.16b\n" + : + : "m" (v[l][i]), "m" (v[l][i + 16]) + ); + + asm volatile ( + /* initialize R, S, T */ + /* table load Q (actually index 1 in gfgenpshufb is R) */ + "ldr q22, %0\n" + "ldr q23, %1\n" + /* chunk 0 */ + "ushr v17.16b, v0.16b, #4\n" + "and v16.16b, v0.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v4.16b, {v22.16b}, v16.16b\n" + "tbl v20.16b, {v23.16b}, v17.16b\n" + "eor v4.16b, v4.16b, v20.16b\n" + /* chunk 1 */ + "ushr v17.16b, v1.16b, #4\n" + "and v16.16b, v1.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v5.16b, {v22.16b}, v16.16b\n" + "tbl v20.16b, {v23.16b}, v17.16b\n" + "eor v5.16b, v5.16b, v20.16b\n" + + /* table load S (index 2) */ + "ldr q22, %2\n" + "ldr q23, %3\n" + /* chunk 0 */ + "ushr v17.16b, v0.16b, #4\n" + "and v16.16b, v0.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v6.16b, {v22.16b}, v16.16b\n" + "tbl v20.16b, {v23.16b}, v17.16b\n" + "eor v6.16b, v6.16b, v20.16b\n" + /* chunk 1 */ + "ushr v17.16b, v1.16b, #4\n" + "and v16.16b, v1.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v7.16b, {v22.16b}, v16.16b\n" + "tbl v20.16b, {v23.16b}, v17.16b\n" + "eor v7.16b, v7.16b, v20.16b\n" + + /* table load T (index 3) */ + "ldr q22, %4\n" + "ldr q23, %5\n" + /* chunk 0 */ + "ushr v17.16b, v0.16b, #4\n" + "and v16.16b, v0.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v8.16b, {v22.16b}, v16.16b\n" + "tbl v20.16b, {v23.16b}, v17.16b\n" + "eor v8.16b, v8.16b, v20.16b\n" + /* chunk 1 */ + "ushr v17.16b, v1.16b, #4\n" + "and v16.16b, v1.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v9.16b, {v22.16b}, v16.16b\n" + "tbl v20.16b, {v23.16b}, v17.16b\n" + "eor v9.16b, v9.16b, v20.16b\n" + : + : "m" (gfgenpshufb[l][1][0][0]), "m" (gfgenpshufb[l][1][1][0]), + "m" (gfgenpshufb[l][2][0][0]), "m" (gfgenpshufb[l][2][1][0]), + "m" (gfgenpshufb[l][3][0][0]), "m" (gfgenpshufb[l][3][1][0]) + ); + + /* intermediate disks */ + for (d = l - 1; d > 0; --d) { + asm volatile ( + /* double Q0-Q1 */ + "sshr v16.16b, v2.16b, #7\n" + "shl v2.16b, v2.16b, #1\n" + "and v16.16b, v16.16b, v29.16b\n" + "eor v2.16b, v2.16b, v16.16b\n" + + "sshr v16.16b, v3.16b, #7\n" + "shl v3.16b, v3.16b, #1\n" + "and v16.16b, v16.16b, v29.16b\n" + "eor v3.16b, v3.16b, v16.16b\n" + + /* load data */ + "ldr q12, %0\n" + "ldr q13, %1\n" + "eor v0.16b, v0.16b, v12.16b\n" + "eor v1.16b, v1.16b, v13.16b\n" + "eor v2.16b, v2.16b, v12.16b\n" + "eor v3.16b, v3.16b, v13.16b\n" + : + : "m" (v[d][i]), "m" (v[d][i + 16]) + ); + + /* multiply and XOR into R, S, T */ + asm volatile ( + /* load tables for R */ + "ldr q24, %0\n" + "ldr q25, %1\n" + /* chunk 0 */ + "ushr v17.16b, v12.16b, #4\n" + "and v16.16b, v12.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v20.16b, {v24.16b}, v16.16b\n" + "tbl v21.16b, {v25.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v4.16b, v4.16b, v20.16b\n" + /* chunk 1 */ + "ushr v17.16b, v13.16b, #4\n" + "and v16.16b, v13.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v20.16b, {v24.16b}, v16.16b\n" + "tbl v21.16b, {v25.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v5.16b, v5.16b, v20.16b\n" + + /* load tables for S */ + "ldr q24, %2\n" + "ldr q25, %3\n" + /* chunk 0 */ + "ushr v17.16b, v12.16b, #4\n" + "and v16.16b, v12.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v20.16b, {v24.16b}, v16.16b\n" + "tbl v21.16b, {v25.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v6.16b, v6.16b, v20.16b\n" + /* chunk 1 */ + "ushr v17.16b, v13.16b, #4\n" + "and v16.16b, v13.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v20.16b, {v24.16b}, v16.16b\n" + "tbl v21.16b, {v25.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v7.16b, v7.16b, v20.16b\n" + + /* load tables for T */ + "ldr q24, %4\n" + "ldr q25, %5\n" + /* chunk 0 */ + "ushr v17.16b, v12.16b, #4\n" + "and v16.16b, v12.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v20.16b, {v24.16b}, v16.16b\n" + "tbl v21.16b, {v25.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v8.16b, v8.16b, v20.16b\n" + /* chunk 1 */ + "ushr v17.16b, v13.16b, #4\n" + "and v16.16b, v13.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v20.16b, {v24.16b}, v16.16b\n" + "tbl v21.16b, {v25.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v9.16b, v9.16b, v20.16b\n" + : + : "m" (gfgenpshufb[d][1][0][0]), "m" (gfgenpshufb[d][1][1][0]), + "m" (gfgenpshufb[d][2][0][0]), "m" (gfgenpshufb[d][2][1][0]), + "m" (gfgenpshufb[d][3][0][0]), "m" (gfgenpshufb[d][3][1][0]) + ); + } + + /* first disk with all coefficients at 1 */ + asm volatile ( + /* double Q0-Q1 */ + "sshr v16.16b, v2.16b, #7\n" + "shl v2.16b, v2.16b, #1\n" + "and v16.16b, v16.16b, v29.16b\n" + "eor v2.16b, v2.16b, v16.16b\n" + + "sshr v16.16b, v3.16b, #7\n" + "shl v3.16b, v3.16b, #1\n" + "and v16.16b, v16.16b, v29.16b\n" + "eor v3.16b, v3.16b, v16.16b\n" + + /* load disk 0 data and XOR into P, Q, R, S, T */ + "ldr q12, %0\n" + "ldr q13, %1\n" + "eor v0.16b, v0.16b, v12.16b\n" + "eor v1.16b, v1.16b, v13.16b\n" + "eor v2.16b, v2.16b, v12.16b\n" + "eor v3.16b, v3.16b, v13.16b\n" + "eor v4.16b, v4.16b, v12.16b\n" + "eor v5.16b, v5.16b, v13.16b\n" + "eor v6.16b, v6.16b, v12.16b\n" + "eor v7.16b, v7.16b, v13.16b\n" + "eor v8.16b, v8.16b, v12.16b\n" + "eor v9.16b, v9.16b, v13.16b\n" + : + : "m" (v[0][i]), "m" (v[0][i + 16]) + ); + + asm volatile ( + "str q0, %0\n" + "str q1, %1\n" + "str q2, %2\n" + "str q3, %3\n" + "str q4, %4\n" + "str q5, %5\n" + "str q6, %6\n" + "str q7, %7\n" + "str q8, %8\n" + "str q9, %9\n" + : "=m" (p[i]), "=m" (p[i + 16]), + "=m" (q[i]), "=m" (q[i + 16]), + "=m" (r[i]), "=m" (r[i + 16]), + "=m" (s[i]), "=m" (s[i + 16]), + "=m" (t[i]), "=m" (t[i + 16]) + ); + } + + raid_neon_end(); +} + +/* + * GEN6 (hexa parity with Cauchy matrix) NEON implementation + */ +void raid_gen6_neon(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + uint8_t *s; + uint8_t *t; + uint8_t *u; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + s = v[nd + 3]; + t = v[nd + 4]; + u = v[nd + 5]; + + /* special case with only one data disk */ + if (l == 0) { + for (i = 0; i < 6; ++i) + memcpy(v[1 + i], v[0], size); + return; + } + + raid_neon_begin(); + + /* generic case with at least two data disks */ + asm volatile ( + "ldr q29, %0\n" + "ldr q28, %1\n" + : + : "m" (gfconst16.poly[0]), "m" (gfconst16.low4[0]) + ); + + for (i = 0; i < size; i += 32) { + /* last disk without the multiplication by two */ + asm volatile ( + "ldr q0, %0\n" + "ldr q1, %1\n" + "mov v2.16b, v0.16b\n" + "mov v3.16b, v1.16b\n" + : + : "m" (v[l][i]), "m" (v[l][i + 16]) + ); + + asm volatile ( + /* initialize R, S, T, U */ + /* table load Q (actually index 1 in gfgenpshufb is R) */ + "ldr q22, %0\n" + "ldr q23, %1\n" + /* chunk 0 */ + "ushr v17.16b, v0.16b, #4\n" + "and v16.16b, v0.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v4.16b, {v22.16b}, v16.16b\n" + "tbl v20.16b, {v23.16b}, v17.16b\n" + "eor v4.16b, v4.16b, v20.16b\n" + /* chunk 1 */ + "ushr v17.16b, v1.16b, #4\n" + "and v16.16b, v1.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v5.16b, {v22.16b}, v16.16b\n" + "tbl v20.16b, {v23.16b}, v17.16b\n" + "eor v5.16b, v5.16b, v20.16b\n" + + /* table load S (index 2) */ + "ldr q22, %2\n" + "ldr q23, %3\n" + /* chunk 0 */ + "ushr v17.16b, v0.16b, #4\n" + "and v16.16b, v0.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v6.16b, {v22.16b}, v16.16b\n" + "tbl v20.16b, {v23.16b}, v17.16b\n" + "eor v6.16b, v6.16b, v20.16b\n" + /* chunk 1 */ + "ushr v17.16b, v1.16b, #4\n" + "and v16.16b, v1.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v7.16b, {v22.16b}, v16.16b\n" + "tbl v20.16b, {v23.16b}, v17.16b\n" + "eor v7.16b, v7.16b, v20.16b\n" + + /* table load T (index 3) */ + "ldr q22, %4\n" + "ldr q23, %5\n" + /* chunk 0 */ + "ushr v17.16b, v0.16b, #4\n" + "and v16.16b, v0.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v8.16b, {v22.16b}, v16.16b\n" + "tbl v20.16b, {v23.16b}, v17.16b\n" + "eor v8.16b, v8.16b, v20.16b\n" + /* chunk 1 */ + "ushr v17.16b, v1.16b, #4\n" + "and v16.16b, v1.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v9.16b, {v22.16b}, v16.16b\n" + "tbl v20.16b, {v23.16b}, v17.16b\n" + "eor v9.16b, v9.16b, v20.16b\n" + + /* table load U (index 4) */ + "ldr q22, %6\n" + "ldr q23, %7\n" + /* chunk 0 */ + "ushr v17.16b, v0.16b, #4\n" + "and v16.16b, v0.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v10.16b, {v22.16b}, v16.16b\n" + "tbl v20.16b, {v23.16b}, v17.16b\n" + "eor v10.16b, v10.16b, v20.16b\n" + /* chunk 1 */ + "ushr v17.16b, v1.16b, #4\n" + "and v16.16b, v1.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v11.16b, {v22.16b}, v16.16b\n" + "tbl v20.16b, {v23.16b}, v17.16b\n" + "eor v11.16b, v11.16b, v20.16b\n" + : + : "m" (gfgenpshufb[l][1][0][0]), "m" (gfgenpshufb[l][1][1][0]), + "m" (gfgenpshufb[l][2][0][0]), "m" (gfgenpshufb[l][2][1][0]), + "m" (gfgenpshufb[l][3][0][0]), "m" (gfgenpshufb[l][3][1][0]), + "m" (gfgenpshufb[l][4][0][0]), "m" (gfgenpshufb[l][4][1][0]) + ); + + /* intermediate disks */ + for (d = l - 1; d > 0; --d) { + asm volatile ( + /* double Q0-Q1 */ + "sshr v16.16b, v2.16b, #7\n" + "shl v2.16b, v2.16b, #1\n" + "and v16.16b, v16.16b, v29.16b\n" + "eor v2.16b, v2.16b, v16.16b\n" + + "sshr v16.16b, v3.16b, #7\n" + "shl v3.16b, v3.16b, #1\n" + "and v16.16b, v16.16b, v29.16b\n" + "eor v3.16b, v3.16b, v16.16b\n" + + /* load data */ + "ldr q12, %0\n" + "ldr q13, %1\n" + "eor v0.16b, v0.16b, v12.16b\n" + "eor v1.16b, v1.16b, v13.16b\n" + "eor v2.16b, v2.16b, v12.16b\n" + "eor v3.16b, v3.16b, v13.16b\n" + : + : "m" (v[d][i]), "m" (v[d][i + 16]) + ); + + /* multiply and XOR into R, S, T, U */ + asm volatile ( + /* load tables for R */ + "ldr q24, %0\n" + "ldr q25, %1\n" + /* chunk 0 */ + "ushr v17.16b, v12.16b, #4\n" + "and v16.16b, v12.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v20.16b, {v24.16b}, v16.16b\n" + "tbl v21.16b, {v25.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v4.16b, v4.16b, v20.16b\n" + /* chunk 1 */ + "ushr v17.16b, v13.16b, #4\n" + "and v16.16b, v13.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v20.16b, {v24.16b}, v16.16b\n" + "tbl v21.16b, {v25.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v5.16b, v5.16b, v20.16b\n" + + /* load tables for S */ + "ldr q24, %2\n" + "ldr q25, %3\n" + /* chunk 0 */ + "ushr v17.16b, v12.16b, #4\n" + "and v16.16b, v12.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v20.16b, {v24.16b}, v16.16b\n" + "tbl v21.16b, {v25.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v6.16b, v6.16b, v20.16b\n" + /* chunk 1 */ + "ushr v17.16b, v13.16b, #4\n" + "and v16.16b, v13.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v20.16b, {v24.16b}, v16.16b\n" + "tbl v21.16b, {v25.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v7.16b, v7.16b, v20.16b\n" + + /* load tables for T */ + "ldr q24, %4\n" + "ldr q25, %5\n" + /* chunk 0 */ + "ushr v17.16b, v12.16b, #4\n" + "and v16.16b, v12.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v20.16b, {v24.16b}, v16.16b\n" + "tbl v21.16b, {v25.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v8.16b, v8.16b, v20.16b\n" + /* chunk 1 */ + "ushr v17.16b, v13.16b, #4\n" + "and v16.16b, v13.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v20.16b, {v24.16b}, v16.16b\n" + "tbl v21.16b, {v25.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v9.16b, v9.16b, v20.16b\n" + + /* load tables for U */ + "ldr q24, %6\n" + "ldr q25, %7\n" + /* chunk 0 */ + "ushr v17.16b, v12.16b, #4\n" + "and v16.16b, v12.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v20.16b, {v24.16b}, v16.16b\n" + "tbl v21.16b, {v25.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v10.16b, v10.16b, v20.16b\n" + /* chunk 1 */ + "ushr v17.16b, v13.16b, #4\n" + "and v16.16b, v13.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v20.16b, {v24.16b}, v16.16b\n" + "tbl v21.16b, {v25.16b}, v17.16b\n" + "eor v20.16b, v20.16b, v21.16b\n" + "eor v11.16b, v11.16b, v20.16b\n" + : + : "m" (gfgenpshufb[d][1][0][0]), "m" (gfgenpshufb[d][1][1][0]), + "m" (gfgenpshufb[d][2][0][0]), "m" (gfgenpshufb[d][2][1][0]), + "m" (gfgenpshufb[d][3][0][0]), "m" (gfgenpshufb[d][3][1][0]), + "m" (gfgenpshufb[d][4][0][0]), "m" (gfgenpshufb[d][4][1][0]) + ); + } + + /* first disk with all coefficients at 1 */ + asm volatile ( + /* double Q0-Q1 */ + "sshr v16.16b, v2.16b, #7\n" + "shl v2.16b, v2.16b, #1\n" + "and v16.16b, v16.16b, v29.16b\n" + "eor v2.16b, v2.16b, v16.16b\n" + + "sshr v16.16b, v3.16b, #7\n" + "shl v3.16b, v3.16b, #1\n" + "and v16.16b, v16.16b, v29.16b\n" + "eor v3.16b, v3.16b, v16.16b\n" + + /* load disk 0 data and XOR into P, Q, R, S, T, U */ + "ldr q12, %0\n" + "ldr q13, %1\n" + "eor v0.16b, v0.16b, v12.16b\n" + "eor v1.16b, v1.16b, v13.16b\n" + "eor v2.16b, v2.16b, v12.16b\n" + "eor v3.16b, v3.16b, v13.16b\n" + "eor v4.16b, v4.16b, v12.16b\n" + "eor v5.16b, v5.16b, v13.16b\n" + "eor v6.16b, v6.16b, v12.16b\n" + "eor v7.16b, v7.16b, v13.16b\n" + "eor v8.16b, v8.16b, v12.16b\n" + "eor v9.16b, v9.16b, v13.16b\n" + "eor v10.16b, v10.16b, v12.16b\n" + "eor v11.16b, v11.16b, v13.16b\n" + : + : "m" (v[0][i]), "m" (v[0][i + 16]) + ); + + asm volatile ( + "str q0, %0\n" + "str q1, %1\n" + "str q2, %2\n" + "str q3, %3\n" + "str q4, %4\n" + "str q5, %5\n" + "str q6, %6\n" + "str q7, %7\n" + "str q8, %8\n" + "str q9, %9\n" + "str q10, %10\n" + "str q11, %11\n" + : "=m" (p[i]), "=m" (p[i + 16]), + "=m" (q[i]), "=m" (q[i + 16]), + "=m" (r[i]), "=m" (r[i + 16]), + "=m" (s[i]), "=m" (s[i + 16]), + "=m" (t[i]), "=m" (t[i + 16]), + "=m" (u[i]), "=m" (u[i + 16]) + ); + } + + raid_neon_end(); +} + +/* + * RAID recovering for one disk NEON implementation + */ +void raid_rec1_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *pa; + uint8_t G; + uint8_t V; + size_t i; + + (void)nr; /* unused, it's always 1 */ + + /* if it's RAID5 uses the faster function */ + if (ip[0] == 0) { + raid_rec1of1(id, nd, size, vv); + return; + } + + /* setup the coefficients matrix */ + G = A(ip[0], id[0]); + + /* invert it to solve the system of linear equations */ + V = inv(G); + + /* compute delta parity */ + raid_delta_gen(1, id, ip, nd, size, vv); + + p = v[nd + ip[0]]; + pa = v[id[0]]; + + raid_neon_begin(); + + for (i = 0; i < size; i += 16) { + asm volatile ( + "ldr q28, %3\n" /* low4 */ + "ldr q24, %4\n" /* v low table */ + "ldr q25, %5\n" /* v high table */ + "ldr q0, %1\n" + "ldr q1, %2\n" + "eor v0.16b, v0.16b, v1.16b\n" + "ushr v17.16b, v0.16b, #4\n" + "and v16.16b, v0.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v2.16b, {v24.16b}, v16.16b\n" + "tbl v3.16b, {v25.16b}, v17.16b\n" + "eor v2.16b, v2.16b, v3.16b\n" + "str q2, %0\n" + : "=m" (pa[i]) + : "m" (p[i]), "m" (pa[i]), "m" (gfconst16.low4[0]), + "m" (gfmulpshufb[V][0][0]), "m" (gfmulpshufb[V][1][0]) + ); + } + + raid_neon_end(); +} + +/* + * RAID recovering for two disks NEON implementation + */ +void raid_rec2_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + const int N = 2; + uint8_t *p[N]; + uint8_t *pa[N]; + uint8_t G[N * N]; + uint8_t V[N * N]; + size_t i; + int j, k; + + (void)nr; /* unused, it's always 2 */ + + /* setup the coefficients matrix */ + for (j = 0; j < N; ++j) + for (k = 0; k < N; ++k) + G[j * N + k] = A(ip[j], id[k]); + + /* invert it to solve the system of linear equations */ + raid_invert(G, V, N); + + /* compute delta parity */ + raid_delta_gen(N, id, ip, nd, size, vv); + + for (j = 0; j < N; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + raid_neon_begin(); + + for (i = 0; i < size; i += 16) { + asm volatile ( + "ldr q28, %2\n" /* low4 */ + + /* load delta 0 */ + "ldr q0, %3\n" + "ldr q2, %4\n" + "eor v0.16b, v0.16b, v2.16b\n" + + /* load delta 1 */ + "ldr q1, %5\n" + "ldr q3, %6\n" + "eor v1.16b, v1.16b, v3.16b\n" + + /* reconstruct pa[0] */ + "ldr q24, %7\n" /* v[0] low table */ + "ldr q25, %8\n" /* v[0] high table */ + "ushr v17.16b, v0.16b, #4\n" + "and v16.16b, v0.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v4.16b, {v24.16b}, v16.16b\n" + "tbl v5.16b, {v25.16b}, v17.16b\n" + "eor v4.16b, v4.16b, v5.16b\n" /* v4 has delta0 * V[0] */ + + "ldr q24, %9\n" /* v[1] low table */ + "ldr q25, %10\n" /* v[1] high table */ + "ushr v17.16b, v1.16b, #4\n" + "and v16.16b, v1.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v6.16b, {v24.16b}, v16.16b\n" + "tbl v7.16b, {v25.16b}, v17.16b\n" + "eor v6.16b, v6.16b, v7.16b\n" /* v6 has delta1 * V[1] */ + + "eor v4.16b, v4.16b, v6.16b\n" + "str q4, %0\n" + + /* reconstruct pa[1] */ + "ldr q24, %11\n" /* v[2] low table */ + "ldr q25, %12\n" /* v[2] high table */ + "ushr v17.16b, v0.16b, #4\n" + "and v16.16b, v0.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v4.16b, {v24.16b}, v16.16b\n" + "tbl v5.16b, {v25.16b}, v17.16b\n" + "eor v4.16b, v4.16b, v5.16b\n" /* v4 has delta0 * V[2] */ + + "ldr q24, %13\n" /* v[3] low table */ + "ldr q25, %14\n" /* v[3] high table */ + "ushr v17.16b, v1.16b, #4\n" + "and v16.16b, v1.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v6.16b, {v24.16b}, v16.16b\n" + "tbl v7.16b, {v25.16b}, v17.16b\n" + "eor v6.16b, v6.16b, v7.16b\n" /* v6 has delta1 * V[3] */ + + "eor v4.16b, v4.16b, v6.16b\n" + "str q4, %1\n" + : "=m" (pa[0][i]), "=m" (pa[1][i]) + : "m" (gfconst16.low4[0]), + "m" (p[0][i]), "m" (pa[0][i]), + "m" (p[1][i]), "m" (pa[1][i]), + "m" (gfmulpshufb[V[0]][0][0]), "m" (gfmulpshufb[V[0]][1][0]), + "m" (gfmulpshufb[V[1]][0][0]), "m" (gfmulpshufb[V[1]][1][0]), + "m" (gfmulpshufb[V[2]][0][0]), "m" (gfmulpshufb[V[2]][1][0]), + "m" (gfmulpshufb[V[3]][0][0]), "m" (gfmulpshufb[V[3]][1][0]) + ); + } + + raid_neon_end(); +} + +/* + * RAID recovering NEON implementation + */ +void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + int N = nr; + uint8_t *p[RAID_PARITY_MAX]; + uint8_t *pa[RAID_PARITY_MAX]; + uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t buffer[RAID_PARITY_MAX*16+16]; + uint8_t *pd = __align_ptr(buffer, 16); + size_t i; + int j, k; + + /* setup the coefficients matrix */ + for (j = 0; j < N; ++j) + for (k = 0; k < N; ++k) + G[j * N + k] = A(ip[j], id[k]); + + /* invert it to solve the system of linear equations */ + raid_invert(G, V, N); + + /* compute delta parity */ + raid_delta_gen(N, id, ip, nd, size, vv); + + for (j = 0; j < N; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + raid_neon_begin(); + + for (i = 0; i < size; i += 16) { + /* delta */ + for (j = 0; j < N; ++j) { + asm volatile ( + "ldr q0, %1\n" + "ldr q1, %2\n" + "eor v0.16b, v0.16b, v1.16b\n" + "str q0, %0\n" + : "=m" (pd[j*16]) + : "m" (p[j][i]), "m" (pa[j][i]) + ); + } + + /* reconstruct */ + for (j = 0; j < N; ++j) { + asm volatile ( + "eor v0.16b, v0.16b, v0.16b\n" + ); + + for (k = 0; k < N; ++k) { + uint8_t m = V[j * N + k]; + asm volatile ( + "ldr q28, %1\n" /* low4 */ + "ldr q24, %2\n" + "ldr q25, %3\n" + "ldr q4, %0\n" + "ushr v17.16b, v4.16b, #4\n" + "and v16.16b, v4.16b, v28.16b\n" + "and v17.16b, v17.16b, v28.16b\n" + "tbl v2.16b, {v24.16b}, v16.16b\n" + "tbl v3.16b, {v25.16b}, v17.16b\n" + "eor v2.16b, v2.16b, v3.16b\n" + "eor v0.16b, v0.16b, v2.16b\n" + : + : "m" (pd[k*16]), "m" (gfconst16.low4[0]), + "m" (gfmulpshufb[m][0][0]), "m" (gfmulpshufb[m][1][0]) + ); + } + + asm volatile ( + "str q0, %0\n" + : "=m" (pa[j][i]) + ); + } + } + + raid_neon_end(); +} + +void raid_register_neon(void) +{ + raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "neon", raid_gen1_neon); + raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "neon", raid_gen2_neon); + raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "neon", raid_genz_neon); + raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "neon", raid_gen3_neon); + raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "neon", raid_gen4_neon); + raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "neon", raid_gen5_neon); + raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "neon", raid_gen6_neon); + + raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "neon", raid_rec1_neon); + raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "neon", raid_rec2_neon); + raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "neon", raid_recX_neon); + raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "neon", raid_recX_neon); + raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "neon", raid_recX_neon); + raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "neon", raid_recX_neon); +} + +#endif /* CONFIG_NEON */ diff --git a/raid/tables.c b/raid/tables.c index 024e8f58..0b2f9684 100644 --- a/raid/tables.c +++ b/raid/tables.c @@ -9119,7 +9119,7 @@ const uint8_t __aligned(256) raid_gfcauchy[6][256] = }, }; -#ifdef CONFIG_X86 +#if defined(CONFIG_X86) || defined(CONFIG_NEON) /** * PSHUFB tables for the Cauchy matrix. * @@ -14653,7 +14653,7 @@ const uint8_t __aligned(256) raid_gfcauchypshufb[251][5][2][16] = }; #endif -#ifdef CONFIG_X86 +#if defined(CONFIG_X86) || defined(CONFIG_NEON) /** * PSHUFB tables for generic multiplication. * @@ -24805,7 +24805,7 @@ const uint8_t __aligned(256) raid_gfcauchy[6][256] = }, }; -#ifdef CONFIG_X86 +#if defined(CONFIG_X86) || defined(CONFIG_NEON) /** * PSHUFB tables for the Cauchy matrix. * @@ -30339,7 +30339,7 @@ const uint8_t __aligned(256) raid_gfcauchypshufb[251][5][2][16] = }; #endif -#ifdef CONFIG_X86 +#if defined(CONFIG_X86) || defined(CONFIG_NEON) /** * PSHUFB tables for generic multiplication. * diff --git a/raid/test.c b/raid/test.c index 2dbb819b..52c60310 100644 --- a/raid/test.c +++ b/raid/test.c @@ -216,6 +216,9 @@ int raid_test_rec(int mode, int nd, size_t size) test_setup(i) = raid_rec1_avx512gfni; #endif #endif +#endif +#ifdef CONFIG_NEON + test_setup(i) = raid_rec1_neon; #endif } else if (i == 2) { test_setup(i) = raid_rec2_int8; @@ -234,6 +237,9 @@ int raid_test_rec(int mode, int nd, size_t size) test_setup(i) = raid_rec2_avx512gfni; #endif #endif +#endif +#ifdef CONFIG_NEON + test_setup(i) = raid_rec2_neon; #endif } else { test_setup(i) = raid_recX_int8; @@ -252,6 +258,9 @@ int raid_test_rec(int mode, int nd, size_t size) test_setup(i) = raid_recX_avx512gfni; #endif #endif +#endif +#ifdef CONFIG_NEON + test_setup(i) = raid_recX_neon; #endif } } @@ -370,6 +379,11 @@ int raid_test_par(int mode, int nd, size_t size) test_setup(2) = raid_gen2_int32; test_setup(2) = raid_gen2_int64; +#ifdef CONFIG_NEON + test_setup(1) = raid_gen1_neon; + test_setup(2) = raid_gen2_neon; +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { test_setup(1) = raid_gen1_sse2; @@ -404,6 +418,13 @@ int raid_test_par(int mode, int nd, size_t size) test_setup(5) = raid_gen5_int8; test_setup(6) = raid_gen6_int8; +#ifdef CONFIG_NEON + test_setup(3) = raid_gen3_neon; + test_setup(4) = raid_gen4_neon; + test_setup(5) = raid_gen5_neon; + test_setup(6) = raid_gen6_neon; +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { test_setup(3) = raid_gen3_ssse3; @@ -450,6 +471,10 @@ int raid_test_par(int mode, int nd, size_t size) test_setup(3) = raid_genz_int32; test_setup(3) = raid_genz_int64; +#ifdef CONFIG_NEON + test_setup(3) = raid_genz_neon; +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { test_setup(3) = raid_genz_sse2; diff --git a/raid/test/Makefile b/raid/test/Makefile index 8be99c37..56dfe5dd 100644 --- a/raid/test/Makefile +++ b/raid/test/Makefile @@ -36,7 +36,7 @@ CFLAGS += -O2 -DUSE_RAID_AES else CFLAGS += -O0 --coverage -DCOVERAGE=1 -DNDEBUG=1 endif -OBJS = raid.o check.o int.o intz.o x86.o x86z.o gfni.o tables.o memory.o test.o helper.o module.o +OBJS = raid.o check.o int.o intz.o x86.o x86z.o gfni.o neon.o tables.o memory.o test.o helper.o module.o %.o: ../%.c $(CC) $(CFLAGS) -c -o $@ $< diff --git a/raid/test/fulltest.c b/raid/test/fulltest.c index 20613e0b..05ede87b 100644 --- a/raid/test/fulltest.c +++ b/raid/test/fulltest.c @@ -30,6 +30,9 @@ int main(void) raid_init(); +#ifdef CONFIG_NEON + printf("Including ARM NEON\n"); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) printf("Including x86 SSE2\n"); diff --git a/raid/test/speedtest.c b/raid/test/speedtest.c index 8aab4416..edea99c5 100644 --- a/raid/test/speedtest.c +++ b/raid/test/speedtest.c @@ -125,6 +125,9 @@ void speed(void) printf("%8s", "int8"); printf("%8s", "int32"); printf("%8s", "int64"); +#ifdef CONFIG_NEON + printf("%8s", "neon"); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { printf("%8s", "sse2"); @@ -179,6 +182,15 @@ void speed(void) printf("%8" PRIu64, ds / dt); fflush(stdout); +#ifdef CONFIG_NEON + SPEED_START { + raid_gen1_neon(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { SPEED_START { @@ -242,6 +254,15 @@ void speed(void) printf("%8" PRIu64, ds / dt); fflush(stdout); +#ifdef CONFIG_NEON + SPEED_START { + raid_gen2_neon(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { SPEED_START { @@ -327,6 +348,15 @@ void speed(void) printf("%8" PRIu64, ds / dt); fflush(stdout); +#ifdef CONFIG_NEON + SPEED_START { + raid_genz_neon(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { SPEED_START { @@ -381,6 +411,15 @@ void speed(void) printf("%8s", ""); printf("%8s", ""); +#ifdef CONFIG_NEON + SPEED_START { + raid_gen3_neon(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { printf("%8s", ""); @@ -459,6 +498,15 @@ void speed(void) printf("%8s", ""); printf("%8s", ""); +#ifdef CONFIG_NEON + SPEED_START { + raid_gen4_neon(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { printf("%8s", ""); @@ -537,6 +585,15 @@ void speed(void) printf("%8s", ""); printf("%8s", ""); +#ifdef CONFIG_NEON + SPEED_START { + raid_gen5_neon(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { printf("%8s", ""); @@ -615,6 +672,15 @@ void speed(void) printf("%8s", ""); printf("%8s", ""); +#ifdef CONFIG_NEON + SPEED_START { + raid_gen6_neon(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { printf("%8s", ""); @@ -684,6 +750,9 @@ void speed(void) printf("%8s", ""); printf("%8s", "best"); printf("%8s", "int8"); +#ifdef CONFIG_NEON + printf("%8s", "neon"); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) printf("%8s", "ssse3"); @@ -713,6 +782,17 @@ void speed(void) printf("%8" PRIu64, ds / dt); fflush(stdout); +#ifdef CONFIG_NEON + SPEED_START { + raid_gen_force(1, raid_gen1_neon); + /* +1 to avoid GEN1 optimized case */ + raid_rec1_neon(1, id, ip + 1, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { SPEED_START { @@ -782,6 +862,17 @@ void speed(void) printf("%8" PRIu64, ds / dt); fflush(stdout); +#ifdef CONFIG_NEON + SPEED_START { + raid_gen_force(2, raid_gen2_neon); + /* +1 to avoid GEN2 optimized case */ + raid_rec2_neon(2, id, ip + 1, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { SPEED_START { @@ -854,6 +945,16 @@ void speed(void) printf("%8" PRIu64, ds / dt); fflush(stdout); +#ifdef CONFIG_NEON + SPEED_START { + raid_gen_force(3, raid_gen3_neon); + raid_recX_neon(3, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { SPEED_START { @@ -921,6 +1022,16 @@ void speed(void) printf("%8" PRIu64, ds / dt); fflush(stdout); +#ifdef CONFIG_NEON + SPEED_START { + raid_gen_force(4, raid_gen4_neon); + raid_recX_neon(4, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { SPEED_START { @@ -992,6 +1103,16 @@ void speed(void) printf("%8" PRIu64, ds / dt); fflush(stdout); +#ifdef CONFIG_NEON + SPEED_START { + raid_gen_force(5, raid_gen5_neon); + raid_recX_neon(5, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { SPEED_START { @@ -1063,6 +1184,16 @@ void speed(void) printf("%8" PRIu64, ds / dt); fflush(stdout); +#ifdef CONFIG_NEON + SPEED_START { + raid_gen_force(6, raid_gen6_neon); + raid_recX_neon(6, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { SPEED_START { @@ -1133,6 +1264,9 @@ int main(void) raid_init(); +#ifdef CONFIG_NEON + printf("Including ARM NEON\n"); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) printf("Including x86 SSE2\n");