From fc9fde000f8520a6f56df2dfa293b2b910a8c8a9 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Wed, 2 Sep 2026 18:40:23 +0200 Subject: [PATCH] Harmonize function across simd architectures --- raid/avx2.c | 32 ++++++++--------- raid/avx512.c | 48 ++++++++++++------------- raid/neon.c | 20 +++++++---- raid/neon32.c | 20 +++++++---- raid/sse2.c | 98 ++++++++++++++++++++++++++------------------------- raid/ssse3.c | 32 ++++++++--------- 6 files changed, 134 insertions(+), 116 deletions(-) diff --git a/raid/avx2.c b/raid/avx2.c index 22c2739d..8c36b9a6 100644 --- a/raid/avx2.c +++ b/raid/avx2.c @@ -44,14 +44,6 @@ static __always_inline void raid_gen1_avx2_gen(int nd, size_t size, void **vv, i raid_avx_end(streaming); } -void raid_gen1_avx2(int nd, size_t size, void **vv, int streaming) -{ - if (streaming) - raid_gen1_avx2_gen(nd, size, vv, 1); - else - raid_gen1_avx2_gen(nd, size, vv, 0); -} - /* * Generate two parity blocks (RAID6 with Cauchy matrix) using AVX2 implementation. * @@ -335,14 +327,6 @@ static __always_inline void raid_genz_avx2ext_gen(int nd, size_t size, void **vv raid_avx_end(streaming); } -void raid_genz_avx2ext_raid(int nd, size_t size, void **vv, int streaming) -{ - if (streaming) - raid_genz_avx2ext_gen(nd, size, vv, 1); - else - raid_genz_avx2ext_gen(nd, size, vv, 0); -} - /* * Generate three parity blocks with Cauchy matrix using AVX2 extended implementation. * @@ -2706,6 +2690,14 @@ static __always_inline void raid_recX_avx2ext(int nr, int has_p, int *id, int *i } #endif +void raid_gen1_avx2(int nd, size_t size, void **vv, int streaming) +{ + if (streaming) + raid_gen1_avx2_gen(nd, size, vv, 1); + else + raid_gen1_avx2_gen(nd, size, vv, 0); +} + void raid_gen2_avx2_raid(int nd, size_t size, void **vv, int streaming) { if (streaming) @@ -2803,6 +2795,14 @@ void raid_gen6_avx2ext_aes(int nd, size_t size, void **vv, int streaming) raid_genX_avx2ext(nd, size, vv, 6, 3, 0); } +void raid_genz_avx2ext_raid(int nd, size_t size, void **vv, int streaming) +{ + if (streaming) + raid_genz_avx2ext_gen(nd, size, vv, 1); + else + raid_genz_avx2ext_gen(nd, size, vv, 0); +} + #endif void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) diff --git a/raid/avx512.c b/raid/avx512.c index 4d214568..54372507 100644 --- a/raid/avx512.c +++ b/raid/avx512.c @@ -52,14 +52,6 @@ static __always_inline void raid_gen1_avx512bw_gen(int nd, size_t size, void **v raid_avx_end(streaming); } -void raid_gen1_avx512bw(int nd, size_t size, void **vv, int streaming) -{ - if (streaming) - raid_gen1_avx512bw_gen(nd, size, vv, 1); - else - raid_gen1_avx512bw_gen(nd, size, vv, 0); -} - /* * Generate two parity blocks (RAID6 with Cauchy matrix) using AVX512BW implementation. * @@ -294,14 +286,6 @@ store: raid_avx_end(streaming); } -void raid_gen2_avx512bw(int nd, size_t size, void **vv, int streaming) -{ - if (streaming) - raid_gen2_avx512bw_gen(nd, size, vv, 1); - else - raid_gen2_avx512bw_gen(nd, size, vv, 0); -} - /* * Generate three parity blocks with Cauchy matrix using AVX512BW implementation. * @@ -482,14 +466,6 @@ store: raid_avx_end(streaming); } -void raid_gen3_avx512bw(int nd, size_t size, void **vv, int streaming) -{ - if (streaming) - raid_gen3_avx512bw_gen(nd, size, vv, 1); - else - raid_gen3_avx512bw_gen(nd, size, vv, 0); -} - /* * Generate N parity blocks with Cauchy matrix using AVX512BW implementation. */ @@ -940,6 +916,30 @@ static __always_inline void raid_recX_avx512bw(int nr, int has_p, int *id, int * raid_avx_end(0); } +void raid_gen1_avx512bw(int nd, size_t size, void **vv, int streaming) +{ + if (streaming) + raid_gen1_avx512bw_gen(nd, size, vv, 1); + else + raid_gen1_avx512bw_gen(nd, size, vv, 0); +} + +void raid_gen2_avx512bw(int nd, size_t size, void **vv, int streaming) +{ + if (streaming) + raid_gen2_avx512bw_gen(nd, size, vv, 1); + else + raid_gen2_avx512bw_gen(nd, size, vv, 0); +} + +void raid_gen3_avx512bw(int nd, size_t size, void **vv, int streaming) +{ + if (streaming) + raid_gen3_avx512bw_gen(nd, size, vv, 1); + else + raid_gen3_avx512bw_gen(nd, size, vv, 0); +} + void raid_gen4_avx512bw(int nd, size_t size, void **vv, int streaming) { if (streaming) diff --git a/raid/neon.c b/raid/neon.c index 3dc412ed..84f9d888 100644 --- a/raid/neon.c +++ b/raid/neon.c @@ -11,15 +11,13 @@ * * Uses 64-byte chunks across four 16-byte vectors. */ -void raid_gen1_neon(int nd, size_t size, void **vv, int streaming) +static __always_inline void raid_gen1_neon_gen(int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p; int d, l; size_t i; - (void)streaming; - l = nd - 1; p = v[nd]; @@ -150,7 +148,7 @@ static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, i /* * Generate three parity blocks with powers of 2^-1 using NEON implementation. */ -void raid_genz_neon_raid(int nd, size_t size, void **vv, int streaming) +static __always_inline void raid_genz_neon_gen(int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -159,8 +157,6 @@ void raid_genz_neon_raid(int nd, size_t size, void **vv, int streaming) int d, l; size_t i; - (void)streaming; - l = nd - 1; p = v[nd]; q = v[nd + 1]; @@ -1329,6 +1325,12 @@ static __always_inline void raid_recX_neon(int nr, int has_p, int *id, int *ip, raid_neon_end(); } +void raid_gen1_neon(int nd, size_t size, void **vv, int streaming) +{ + (void)streaming; + raid_gen1_neon_gen(nd, size, vv); +} + void raid_gen2_neon_raid(int nd, size_t size, void **vv, int streaming) { (void)streaming; @@ -1341,6 +1343,12 @@ void raid_gen2_neon_aes(int nd, size_t size, void **vv, int streaming) raid_gen2_neon_gen(nd, size, vv, 3); } +void raid_genz_neon_raid(int nd, size_t size, void **vv, int streaming) +{ + (void)streaming; + raid_genz_neon_gen(nd, size, vv); +} + void raid_gen3_neon_raid(int nd, size_t size, void **vv, int streaming) { (void)streaming; diff --git a/raid/neon32.c b/raid/neon32.c index d63df997..0e1d8a00 100644 --- a/raid/neon32.c +++ b/raid/neon32.c @@ -11,15 +11,13 @@ * * Uses 64-byte chunks across four 16-byte vectors. */ -void raid_gen1_neon32(int nd, size_t size, void **vv, int streaming) +static __always_inline void raid_gen1_neon32_gen(int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p; int d, l; size_t i; - (void)streaming; - l = nd - 1; p = v[nd]; @@ -150,7 +148,7 @@ static __always_inline void raid_gen2_neon32_gen(int nd, size_t size, /* * Generate three parity blocks with powers of 2^-1 using AArch32 NEON implementation. */ -void raid_genz_neon32_raid(int nd, size_t size, void **vv, int streaming) +static __always_inline void raid_genz_neon32_gen(int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -159,8 +157,6 @@ void raid_genz_neon32_raid(int nd, size_t size, void **vv, int streaming) int d, l; size_t i; - (void)streaming; - l = nd - 1; p = v[nd]; q = v[nd + 1]; @@ -1168,6 +1164,12 @@ static __always_inline void raid_recX_neon32(int nr, int has_p, int *id, int *ip raid_neon32_end(); } +void raid_gen1_neon32(int nd, size_t size, void **vv, int streaming) +{ + (void)streaming; + raid_gen1_neon32_gen(nd, size, vv); +} + void raid_gen2_neon32_raid(int nd, size_t size, void **vv, int streaming) { (void)streaming; @@ -1180,6 +1182,12 @@ void raid_gen2_neon32_aes(int nd, size_t size, void **vv, int streaming) raid_gen2_neon32_gen(nd, size, vv, 3); } +void raid_genz_neon32_raid(int nd, size_t size, void **vv, int streaming) +{ + (void)streaming; + raid_genz_neon32_gen(nd, size, vv); +} + void raid_gen3_neon32_raid(int nd, size_t size, void **vv, int streaming) { (void)streaming; diff --git a/raid/sse2.c b/raid/sse2.c index bb5809ee..15e4c78c 100644 --- a/raid/sse2.c +++ b/raid/sse2.c @@ -145,14 +145,6 @@ static __always_inline void raid_gen1_sse2_gen(int nd, size_t size, void **vv, i raid_sse_end(streaming); } -void raid_gen1_sse2(int nd, size_t size, void **vv, int streaming) -{ - if (streaming) - raid_gen1_sse2_gen(nd, size, vv, 1); - else - raid_gen1_sse2_gen(nd, size, vv, 0); -} - /* * Generate two parity blocks (RAID6 with Cauchy matrix) using SSE2 implementation. */ @@ -232,22 +224,6 @@ static __always_inline void raid_gen2_sse2_gen(int nd, size_t size, void **vv, i raid_sse_end(streaming); } -void raid_gen2_sse2_raid(int nd, size_t size, void **vv, int streaming) -{ - if (streaming) - raid_gen2_sse2_gen(nd, size, vv, 2, 1); - else - raid_gen2_sse2_gen(nd, size, vv, 2, 0); -} - -void raid_gen2_sse2_aes(int nd, size_t size, void **vv, int streaming) -{ - if (streaming) - raid_gen2_sse2_gen(nd, size, vv, 3, 1); - else - raid_gen2_sse2_gen(nd, size, vv, 3, 0); -} - #ifdef CONFIG_X86_64 /* * Generate two parity blocks (RAID6 with Cauchy matrix) using SSE2 extended implementation. @@ -489,22 +465,6 @@ static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv raid_sse_end(streaming); } - -void raid_gen2_sse2ext_raid(int nd, size_t size, void **vv, int streaming) -{ - if (streaming) - raid_gen2_sse2ext_gen(nd, size, vv, 2, 1); - else - raid_gen2_sse2ext_gen(nd, size, vv, 2, 0); -} - -void raid_gen2_sse2ext_aes(int nd, size_t size, void **vv, int streaming) -{ - if (streaming) - raid_gen2_sse2ext_gen(nd, size, vv, 3, 1); - else - raid_gen2_sse2ext_gen(nd, size, vv, 3, 0); -} #endif /* @@ -569,14 +529,6 @@ static __always_inline void raid_genz_sse2_gen(int nd, size_t size, void **vv, i raid_sse_end(streaming); } -void raid_genz_sse2_raid(int nd, size_t size, void **vv, int streaming) -{ - if (streaming) - raid_genz_sse2_gen(nd, size, vv, 1); - else - raid_genz_sse2_gen(nd, size, vv, 0); -} - #ifdef CONFIG_X86_64 /* * Generate three parity blocks with powers of 2^-1 using SSE2 extended implementation. @@ -666,6 +618,56 @@ static __always_inline void raid_genz_sse2ext_gen(int nd, size_t size, void **vv raid_sse_end(streaming); } +#endif + +void raid_gen1_sse2(int nd, size_t size, void **vv, int streaming) +{ + if (streaming) + raid_gen1_sse2_gen(nd, size, vv, 1); + else + raid_gen1_sse2_gen(nd, size, vv, 0); +} + +void raid_gen2_sse2_raid(int nd, size_t size, void **vv, int streaming) +{ + if (streaming) + raid_gen2_sse2_gen(nd, size, vv, 2, 1); + else + raid_gen2_sse2_gen(nd, size, vv, 2, 0); +} + +void raid_gen2_sse2_aes(int nd, size_t size, void **vv, int streaming) +{ + if (streaming) + raid_gen2_sse2_gen(nd, size, vv, 3, 1); + else + raid_gen2_sse2_gen(nd, size, vv, 3, 0); +} + +void raid_genz_sse2_raid(int nd, size_t size, void **vv, int streaming) +{ + if (streaming) + raid_genz_sse2_gen(nd, size, vv, 1); + else + raid_genz_sse2_gen(nd, size, vv, 0); +} + +#ifdef CONFIG_X86_64 +void raid_gen2_sse2ext_raid(int nd, size_t size, void **vv, int streaming) +{ + if (streaming) + raid_gen2_sse2ext_gen(nd, size, vv, 2, 1); + else + raid_gen2_sse2ext_gen(nd, size, vv, 2, 0); +} + +void raid_gen2_sse2ext_aes(int nd, size_t size, void **vv, int streaming) +{ + if (streaming) + raid_gen2_sse2ext_gen(nd, size, vv, 3, 1); + else + raid_gen2_sse2ext_gen(nd, size, vv, 3, 0); +} void raid_genz_sse2ext_raid(int nd, size_t size, void **vv, int streaming) { diff --git a/raid/ssse3.c b/raid/ssse3.c index de1fb159..516591c6 100644 --- a/raid/ssse3.c +++ b/raid/ssse3.c @@ -392,14 +392,6 @@ static __always_inline void raid_gen5_ssse3_gen(int nd, size_t size, void **vv, raid_sse_end(streaming); } -void raid_gen5_ssse3_raid(int nd, size_t size, void **vv, int streaming) -{ - if (streaming) - raid_gen5_ssse3_gen(nd, size, vv, 1); - else - raid_gen5_ssse3_gen(nd, size, vv, 0); -} - /* * Generate six parity blocks with Cauchy matrix using SSSE3 implementation. * @@ -568,14 +560,6 @@ static __always_inline void raid_gen6_ssse3_gen(int nd, size_t size, void **vv, raid_sse_end(streaming); } -void raid_gen6_ssse3_raid(int nd, size_t size, void **vv, int streaming) -{ - if (streaming) - raid_gen6_ssse3_gen(nd, size, vv, 1); - else - raid_gen6_ssse3_gen(nd, size, vv, 0); -} - #ifdef CONFIG_X86_64 /* * Generate three parity blocks with Cauchy matrix using SSSE3 extended implementation. @@ -4097,6 +4081,22 @@ void raid_gen4_ssse3_aes(int nd, size_t size, void **vv, int streaming) raid_gen4_ssse3_gen(nd, size, vv, 3, 0); } +void raid_gen5_ssse3_raid(int nd, size_t size, void **vv, int streaming) +{ + if (streaming) + raid_gen5_ssse3_gen(nd, size, vv, 1); + else + raid_gen5_ssse3_gen(nd, size, vv, 0); +} + +void raid_gen6_ssse3_raid(int nd, size_t size, void **vv, int streaming) +{ + if (streaming) + raid_gen6_ssse3_gen(nd, size, vv, 1); + else + raid_gen6_ssse3_gen(nd, size, vv, 0); +} + #ifdef CONFIG_X86_64 void raid_gen3_ssse3ext_raid(int nd, size_t size, void **vv, int streaming) {