From b78542685fd1296f4fe915e31e29e767e2b489c7 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Thu, 4 Jun 2026 17:53:25 +0200 Subject: [PATCH] Add portable 128-bit vector parity routines for gen1/gen2/genz to optimize non-x86 CPUs --- cmdline/portable.h | 18 ++++++++-- cmdline/speed.c | 55 ++++++++++++++++++++++++++++++ raid/gf.h | 39 +++++++++++++++++++++ raid/int.c | 79 +++++++++++++++++++++++++++++++++++++++++++ raid/internal.h | 24 +++++++++++-- raid/intz.c | 58 +++++++++++++++++++++++++++++++ raid/test.c | 9 +++++ raid/test/Makefile | 4 +-- raid/test/fulltest.c | 3 ++ raid/test/speedtest.c | 53 +++++++++++++++++++++++++++++ 10 files changed, 336 insertions(+), 6 deletions(-) diff --git a/cmdline/portable.h b/cmdline/portable.h index f4f17266..5e168566 100644 --- a/cmdline/portable.h +++ b/cmdline/portable.h @@ -65,16 +65,30 @@ #define __noreturn __attribute__((noreturn)) #endif +/** + * Enable C-based SIMD-accelerated vector optimizations on: + * - x86/x64 (SSE2) + * - ARM (NEON) + * - RISC-V (Vector Extension) + */ +#if defined(__SSE2__) || defined(__ARM_NEON) || defined(__riscv_vector) +#if defined(__has_attribute) +#if __has_attribute(__vector_size__) +#define CONFIG_VEC128 1 +#endif +#endif +#endif /** - * Architecture for inline assembly. + * Enable Assembly-based SIMD-accelerated vector optimizations on: + * - x86_32 (32 bit) + * - x86_64 (64 bit) */ #if HAVE_ASSEMBLY #if defined(__i386__) #define CONFIG_X86 1 #define CONFIG_X86_32 1 #endif - #if defined(__x86_64__) #define CONFIG_X86 1 #define CONFIG_X86_64 1 diff --git a/cmdline/speed.c b/cmdline/speed.c index fed92e68..365bbe64 100644 --- a/cmdline/speed.c +++ b/cmdline/speed.c @@ -157,6 +157,11 @@ void speed(int period, int nd, int size) #else printf("128-bit integers are not supported\n"); #endif +#if CONFIG_VEC128 + printf("128-bit vectors are supported\n"); +#else + printf("128-bit vectors are not supported\n"); +#endif #if HAVE_FUTIMENS printf("Support nanosecond timestamps with futimens()\n"); @@ -274,6 +279,9 @@ void speed(int period, int nd, int size) printf("%8s", "int8"); printf("%8s", "int32"); printf("%8s", "int64"); +#if CONFIG_VEC128 + printf("%8s", "vec128"); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { printf("%8s", "sse2"); @@ -325,6 +333,15 @@ void speed(int period, int nd, int size) printf("%8" PRIu64, ds / dt); fflush(stdout); +#if CONFIG_VEC128 + SPEED_START { + raid_gen1_vec128(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { SPEED_START { @@ -388,6 +405,15 @@ void speed(int period, int nd, int size) printf("%8" PRIu64, ds / dt); fflush(stdout); +#if CONFIG_VEC128 + SPEED_START { + raid_gen2_vec128(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { SPEED_START { @@ -473,6 +499,15 @@ void speed(int period, int nd, int size) printf("%8" PRIu64, ds / dt); fflush(stdout); +#if CONFIG_VEC128 + SPEED_START { + raid_genz_vec128(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { SPEED_START { @@ -525,6 +560,9 @@ void speed(int period, int nd, int size) printf("%8s", ""); printf("%8s", ""); +#if CONFIG_VEC128 + printf("%8s", ""); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { @@ -605,6 +643,9 @@ void speed(int period, int nd, int size) printf("%8s", ""); printf("%8s", ""); +#if CONFIG_VEC128 + printf("%8s", ""); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { @@ -685,6 +726,9 @@ void speed(int period, int nd, int size) printf("%8s", ""); printf("%8s", ""); +#if CONFIG_VEC128 + printf("%8s", ""); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { @@ -764,6 +808,9 @@ void speed(int period, int nd, int size) printf("%8s", ""); printf("%8s", ""); +#if CONFIG_VEC128 + printf("%8s", ""); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { @@ -857,7 +904,11 @@ void speed(int period, int nd, int size) SPEED_START { /* ensure to use same hardware in the delta step */ +#if CONFIG_VEC128 + raid_gen_force(1, raid_gen1_vec128); +#else raid_gen_force(1, sizeof(void*) == 8 ? raid_gen1_int64 : raid_gen1_int32); +#endif /* +1 to avoid GEN1 optimized case */ raid_rec1_int8(1, id, ip + 1, nd, size, v); } SPEED_STOP @@ -932,7 +983,11 @@ void speed(int period, int nd, int size) SPEED_START { /* ensure to use same hardware in the delta step */ +#if CONFIG_VEC128 + raid_gen_force(2, raid_gen2_vec128); +#else raid_gen_force(2, sizeof(void*) == 8 ? raid_gen2_int64 : raid_gen2_int32); +#endif /* +1 to avoid GEN2 optimized case */ raid_rec2_int8(2, id, ip + 1, nd, size, v); } SPEED_STOP diff --git a/raid/gf.h b/raid/gf.h index 67e72b9c..fa516a0c 100644 --- a/raid/gf.h +++ b/raid/gf.h @@ -70,6 +70,14 @@ static __always_inline uint8_t A(int p, int d) */ #define v_64(p) (*(uint64_t *)&(p)) +#if CONFIG_VEC128 +/* + * Dereference as uvec128_t + */ +typedef uint64_t uvec128_t __attribute__ ((__vector_size__ (16))); +#define v_128(p) (*(uvec128_t *)&(p)) +#endif + /* * Polynomial-dependent XOR masks for GF(2^8) multiply and divide by 2. * @@ -154,4 +162,35 @@ static __always_inline uint64_t d2_64(uint64_t v) return v; } +#if CONFIG_VEC128 +#define RAID_POLY_128 ((uvec128_t){ RAID_POLY_64, RAID_POLY_64 }) +#define RAID_INV2_128 ((uvec128_t){ RAID_INV2_64, RAID_INV2_64 }) + +/* + * Multiply each byte of a uint128 by 2 in GF(2^8). + */ +static __always_inline uvec128_t x2_128(uvec128_t v) +{ + uvec128_t mask = v & ((uvec128_t){ 0x8080808080808080ULL, 0x8080808080808080ULL }); + + mask = (mask << 1) - (mask >> 7); + v = (v << 1) & ((uvec128_t){ 0xfefefefefefefefeULL, 0xfefefefefefefefeULL }); + v ^= mask & RAID_POLY_128; + return v; +} + +/* + * Divide each byte of a uint128 by 2 in GF(2^8). + */ +static __always_inline uvec128_t d2_128(uvec128_t v) +{ + uvec128_t mask = v & ((uvec128_t){ 0x0101010101010101ULL, 0x0101010101010101ULL }); + + mask = (mask << 8) - mask; + v = (v >> 1) & ((uvec128_t){ 0x7f7f7f7f7f7f7f7fULL, 0x7f7f7f7f7f7f7f7fULL }); + v ^= mask & RAID_INV2_128; + return v; +} +#endif + #endif diff --git a/raid/int.c b/raid/int.c index 5993a227..24707b78 100644 --- a/raid/int.c +++ b/raid/int.c @@ -60,6 +60,36 @@ void raid_gen1_int64(int nd, size_t size, void **vv) } } +#if CONFIG_VEC128 +/* + * GEN1 (RAID5 with xor) 128bit C implementation + */ +void raid_gen1_vec128(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + int d, l; + size_t i; + + uvec128_t p0; + uvec128_t p1; + + l = nd - 1; + p = v[nd]; + + for (i = 0; i < size; i += 32) { + p0 = v_128(v[l][i]); + p1 = v_128(v[l][i + 16]); + for (d = l - 1; d >= 0; --d) { + p0 ^= v_128(v[d][i]); + p1 ^= v_128(v[d][i + 16]); + } + v_128(p[i]) = p0; + v_128(p[i + 16]) = p1; + } +} +#endif + /* * GEN2 (RAID6 with powers of 2) 32bit C implementation */ @@ -142,6 +172,50 @@ void raid_gen2_int64(int nd, size_t size, void **vv) } } +#if CONFIG_VEC128 +/* + * GEN2 (RAID6 with powers of 2) 128bit C implementation + */ +void raid_gen2_vec128(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + int d, l; + size_t i; + + uvec128_t d0, d1; + uvec128_t p0, p1; + uvec128_t q0, q1; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + + for (i = 0; i < size; i += 32) { + p0 = q0 = v_128(v[l][i]); + p1 = q1 = v_128(v[l][i + 16]); + for (d = l - 1; d >= 0; --d) { + d0 = v_128(v[d][i]); + d1 = v_128(v[d][i + 16]); + + p0 ^= d0; + p1 ^= d1; + + q0 = x2_128(q0); + q1 = x2_128(q1); + + q0 ^= d0; + q1 ^= d1; + } + v_128(p[i]) = p0; + v_128(p[i + 16]) = p1; + v_128(q[i]) = q0; + v_128(q[i + 16]) = q1; + } +} +#endif + /* * GEN3 (triple parity with Cauchy matrix) 8bit C implementation * @@ -554,6 +628,11 @@ void raid_register_int(void) raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "int64", raid_gen2_int64); raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "int64", raid_genz_int64); } +#if CONFIG_VEC128 + raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "vec128", raid_gen1_vec128); + raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "vec128", raid_gen2_vec128); + raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "vec128", raid_genz_vec128); +#endif raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "int8", raid_gen3_int8); raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "int8", raid_gen4_int8); raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "int8", raid_gen5_int8); diff --git a/raid/internal.h b/raid/internal.h index dea56e21..b046c07b 100644 --- a/raid/internal.h +++ b/raid/internal.h @@ -12,9 +12,26 @@ #define HAVE_ASSEMBLY 1 #endif -/* If the platforms supports assembly */ +/** + * Enable C-base SIMD-accelerated vector optimizations on: + * - x86/x64 (SSE2) + * - ARM (NEON) + * - RISC-V (Vector Extension) + */ +#if defined(__SSE2__) || defined(__ARM_NEON) || defined(__riscv_vector) +#if defined(__has_attribute) +#if __has_attribute(__vector_size__) +#define CONFIG_VEC128 1 +#endif +#endif +#endif + +/** + * Enable Assembly-base SIMD-accelerated vector optimizations on: + * - x86_32 (32 bit) + * - x86_64 (64 bit) + */ #if HAVE_ASSEMBLY -/* Autodetect from the compiler */ #if defined(__i386__) #define CONFIG_X86 1 #define CONFIG_X86_32 1 @@ -82,11 +99,13 @@ void raid_rec1of1(int *id, int nd, size_t size, void **v); void raid_rec2of2_int8(int *id, int *ip, int nd, size_t size, void **vv); void raid_gen1_int32(int nd, size_t size, void **vv); void raid_gen1_int64(int nd, size_t size, void **vv); +void raid_gen1_vec128(int nd, size_t size, void **vv); void raid_gen1_sse2(int nd, size_t size, void **vv); void raid_gen1_avx2(int nd, size_t size, void **vv); void raid_gen1_avx512bw(int nd, size_t size, void **vv); void raid_gen2_int32(int nd, size_t size, void **vv); void raid_gen2_int64(int nd, size_t size, void **vv); +void raid_gen2_vec128(int nd, size_t size, void **vv); void raid_gen2_sse2(int nd, size_t size, void **vv); void raid_gen2_avx2(int nd, size_t size, void **vv); void raid_gen2_sse2ext(int nd, size_t size, void **vv); @@ -95,6 +114,7 @@ void raid_gen2_avx2gfni(int nd, size_t size, void **vv); void raid_gen2_avx512gfni(int nd, size_t size, void **vv); void raid_genz_int32(int nd, size_t size, void **vv); void raid_genz_int64(int nd, size_t size, void **vv); +void raid_genz_vec128(int nd, size_t size, void **vv); void raid_genz_sse2(int nd, size_t size, void **vv); void raid_genz_sse2ext(int nd, size_t size, void **vv); void raid_genz_avx2ext(int nd, size_t size, void **vv); diff --git a/raid/intz.c b/raid/intz.c index e3c222d4..35c76c24 100644 --- a/raid/intz.c +++ b/raid/intz.c @@ -106,3 +106,61 @@ void raid_genz_int64(int nd, size_t size, void **vv) } } +#if CONFIG_VEC128 +/* + * GENz (triple parity with powers of 2^-1) 128bit C implementation + */ +void raid_genz_vec128(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + int d, l; + size_t i; + + uvec128_t d0, d1; + uvec128_t p0, p1; + uvec128_t q0, q1; + uvec128_t r0, r1; + + l = nd - 1; + p = v[nd]; + q = v[nd + 1]; + r = v[nd + 2]; + + for (i = 0; i < size; i += 32) { + r0 = q0 = p0 = v_128(v[l][i]); + r1 = q1 = p1 = v_128(v[l][i + 16]); + for (d = l - 1; d >= 0; --d) { + d0 = v_128(v[d][i]); + d1 = v_128(v[d][i + 16]); + + p0 ^= d0; + p1 ^= d1; + + q0 = x2_128(q0); + q1 = x2_128(q1); + + q0 ^= d0; + q1 ^= d1; + + r0 = d2_128(r0); + r1 = d2_128(r1); + + r0 ^= d0; + r1 ^= d1; + } + v_128(p[i]) = p0; + v_128(p[i + 16]) = p1; + + v_128(q[i]) = q0; + v_128(q[i + 16]) = q1; + + v_128(r[i]) = r0; + v_128(r[i + 16]) = r1; + } +} +#endif + + diff --git a/raid/test.c b/raid/test.c index 2dbb819b..f717fb1e 100644 --- a/raid/test.c +++ b/raid/test.c @@ -367,8 +367,14 @@ int raid_test_par(int mode, int nd, size_t size) /* setup all the available functions */ test_setup(1) = raid_gen1_int32; test_setup(1) = raid_gen1_int64; +#if CONFIG_VEC128 + test_setup(1) = raid_gen1_vec128; +#endif test_setup(2) = raid_gen2_int32; test_setup(2) = raid_gen2_int64; +#if CONFIG_VEC128 + test_setup(2) = raid_gen2_vec128; +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { @@ -449,6 +455,9 @@ int raid_test_par(int mode, int nd, size_t size) } else { test_setup(3) = raid_genz_int32; test_setup(3) = raid_genz_int64; +#if CONFIG_VEC128 + test_setup(3) = raid_genz_vec128; +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { diff --git a/raid/test/Makefile b/raid/test/Makefile index 8be99c37..c1535f41 100644 --- a/raid/test/Makefile +++ b/raid/test/Makefile @@ -30,9 +30,9 @@ SDE = sde64 endif CC = gcc LD = ld -CFLAGS = -I.. -Wall -Wextra -g +CFLAGS = -I.. -Wall -Wextra -g -DUSE_RAID_AES ifeq ($(COVERAGE),) -CFLAGS += -O2 -DUSE_RAID_AES +CFLAGS += -O2 else CFLAGS += -O0 --coverage -DCOVERAGE=1 -DNDEBUG=1 endif diff --git a/raid/test/fulltest.c b/raid/test/fulltest.c index 20613e0b..84a2d5b0 100644 --- a/raid/test/fulltest.c +++ b/raid/test/fulltest.c @@ -30,6 +30,9 @@ int main(void) raid_init(); +#ifdef CONFIG_VEC128 + printf("Including 128-bit vectors\n"); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) printf("Including x86 SSE2\n"); diff --git a/raid/test/speedtest.c b/raid/test/speedtest.c index 8aab4416..743cc997 100644 --- a/raid/test/speedtest.c +++ b/raid/test/speedtest.c @@ -125,6 +125,9 @@ void speed(void) printf("%8s", "int8"); printf("%8s", "int32"); printf("%8s", "int64"); +#if CONFIG_VEC128 + printf("%8s", "vec128"); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { printf("%8s", "sse2"); @@ -179,6 +182,15 @@ void speed(void) printf("%8" PRIu64, ds / dt); fflush(stdout); +#if CONFIG_VEC128 + SPEED_START { + raid_gen1_vec128(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { SPEED_START { @@ -242,6 +254,15 @@ void speed(void) printf("%8" PRIu64, ds / dt); fflush(stdout); +#if CONFIG_VEC128 + SPEED_START { + raid_gen2_vec128(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { SPEED_START { @@ -327,6 +348,15 @@ void speed(void) printf("%8" PRIu64, ds / dt); fflush(stdout); +#if CONFIG_VEC128 + SPEED_START { + raid_genz_vec128(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); +#endif + #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { SPEED_START { @@ -380,6 +410,9 @@ void speed(void) printf("%8s", ""); printf("%8s", ""); +#if CONFIG_VEC128 + printf("%8s", ""); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { @@ -458,6 +491,9 @@ void speed(void) printf("%8s", ""); printf("%8s", ""); +#if CONFIG_VEC128 + printf("%8s", ""); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { @@ -536,6 +572,9 @@ void speed(void) printf("%8s", ""); printf("%8s", ""); +#if CONFIG_VEC128 + printf("%8s", ""); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { @@ -614,6 +653,9 @@ void speed(void) printf("%8s", ""); printf("%8s", ""); +#if CONFIG_VEC128 + printf("%8s", ""); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) { @@ -705,7 +747,11 @@ void speed(void) fflush(stdout); SPEED_START { +#if CONFIG_VEC128 + raid_gen_force(1, raid_gen1_vec128); +#else raid_gen_force(1, sizeof(void *) == 8 ? raid_gen1_int64 : raid_gen1_int32); +#endif /* +1 to avoid GEN1 optimized case */ raid_rec1_int8(1, id, ip + 1, nd, size, v); } SPEED_STOP @@ -774,7 +820,11 @@ void speed(void) fflush(stdout); SPEED_START { +#if CONFIG_VEC128 + raid_gen_force(2, raid_gen2_vec128); +#else raid_gen_force(2, sizeof(void *) == 8 ? raid_gen2_int64 : raid_gen2_int32); +#endif /* +1 to avoid GEN2 optimized case */ raid_rec2_int8(2, id, ip + 1, nd, size, v); } SPEED_STOP @@ -1133,6 +1183,9 @@ int main(void) raid_init(); +#ifdef CONFIG_VEC128 + printf("Including 128-bit vectors\n"); +#endif #ifdef CONFIG_X86 if (raid_cpu_has_sse2()) printf("Including x86 SSE2\n");