From e573e9d4cf94fba6cf3d1594dcc316c28819370e Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sun, 23 Aug 2026 22:28:21 +0200 Subject: [PATCH] Optimize multi-parity recovery with fused syndrome generation --- cmdline/speed.c | 215 +++--- raid/avx2.c | 1416 +++++++++++++++++++++++++++------- raid/avx512.c | 664 ++++++++-------- raid/gfni.c | 1702 ++++++++++++++++++----------------------- raid/internal.h | 46 +- raid/neon.c | 835 ++++++++++---------- raid/neon32.c | 974 +++++++++++++++-------- raid/test.c | 550 +++++++++---- raid/test/speedtest.c | 169 ++-- 9 files changed, 3926 insertions(+), 2645 deletions(-) diff --git a/cmdline/speed.c b/cmdline/speed.c index 54b99846..169fe047 100644 --- a/cmdline/speed.c +++ b/cmdline/speed.c @@ -1228,8 +1228,12 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) printf("%8s", "ssse3"); - if (raid_cpu_has_avx2()) + if (raid_cpu_has_avx2()) { printf("%8s", "avx2"); +#ifdef CONFIG_X86_64 + printf("%8s", "avx2e"); +#endif + } #ifdef CONFIG_X86_64 if (raid_cpu_has_avx512bw()) printf("%8s", "avx512"); @@ -1257,7 +1261,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #ifdef CONFIG_NEON SPEED_START { - /* ensure to use same hardware in the delta step */ + /* raid_rec1of1() internally calls raid_gen() */ raid_gen_force(1, raid_gen1_neon); raid_rec1_neon(1, id, ip, nd, size, v); } SPEED_STOP @@ -1267,7 +1271,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #endif #ifdef CONFIG_NEON32 SPEED_START { - /* ensure to use same hardware in the delta step */ + /* raid_rec1of1() internally calls raid_gen() */ raid_gen_force(1, raid_gen1_neon32); raid_rec1_neon32(1, id, ip, nd, size, v); } SPEED_STOP @@ -1298,9 +1302,22 @@ void speed_rec(int nd, void** v, int size, int delta, int period) fflush(stdout); } #ifdef CONFIG_X86_64 + if (raid_cpu_has_avx2()) { + SPEED_START { + /* + * raid_rec1_avx2ext() uses raid_rec1of1() for P, + * which internally calls raid_gen(). + */ + raid_gen_force(1, raid_gen1_avx2); + raid_rec1_avx2ext(1, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } if (raid_cpu_has_avx512bw()) { SPEED_START { - /* ensure to use same hardware in the delta step */ + /* raid_rec1of1() internally calls raid_gen() */ raid_gen_force(1, raid_gen1_avx512bw); raid_rec1_avx512bw(1, id, ip, nd, size, v); } SPEED_STOP @@ -1310,7 +1327,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period) } if (raid_cpu_has_avx2gfni()) { SPEED_START { - /* ensure to use same hardware in the delta step */ + /* raid_rec1of1() internally calls raid_gen() */ raid_gen_force(1, raid_gen1_avx2); raid_rec1_avx2gfni_raid(1, id, ip, nd, size, v); } SPEED_STOP @@ -1320,7 +1337,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period) } if (raid_cpu_has_avx512gfni()) { SPEED_START { - /* ensure to use same hardware in the delta step */ + /* raid_rec1of1() internally calls raid_gen() */ raid_gen_force(1, raid_gen1_avx512bw); /* there is no raid_gen1_avx512gfni */ raid_rec1_avx512gfni_raid(1, id, ip, nd, size, v); } SPEED_STOP @@ -1349,8 +1366,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #ifdef CONFIG_NEON SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(1, raid_gen1_neon); /* +1 to avoid GEN1 optimized case */ raid_rec1_neon(1, id, ip + 1, nd, size, v); } SPEED_STOP @@ -1360,8 +1375,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #endif #ifdef CONFIG_NEON32 SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(1, raid_gen1_neon32); /* +1 to avoid GEN1 optimized case */ raid_rec1_neon32(1, id, ip + 1, nd, size, v); } SPEED_STOP @@ -1394,11 +1407,18 @@ void speed_rec(int nd, void** v, int size, int delta, int period) fflush(stdout); } #ifdef CONFIG_X86_64 + if (raid_cpu_has_avx2()) { + SPEED_START { + /* +1 to avoid RAID5 optimized case */ + raid_rec1_avx2ext(1, id, ip + 1, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } if (raid_cpu_has_avx512bw()) { SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(1, raid_gen1_avx512bw); - /* +1 to avoid GEN1 optimized case */ + /* +1 to avoid RAID5 optimized case */ raid_rec1_avx512bw(1, id, ip + 1, nd, size, v); } SPEED_STOP @@ -1407,9 +1427,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period) } if (raid_cpu_has_avx2gfni()) { SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(1, raid_gen1_avx2); - /* +1 to avoid GEN1 optimized case */ + /* +1 to avoid RAID5 optimized case */ raid_rec1_avx2gfni_raid(1, id, ip + 1, nd, size, v); } SPEED_STOP @@ -1418,9 +1436,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period) } if (raid_cpu_has_avx512gfni()) { SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(1, raid_gen1_avx512bw); /* there is no raid_gen1_avx512gfni */ - /* +1 to avoid GEN1 optimized case */ + /* +1 to avoid RAID5 optimized case */ raid_rec1_avx512gfni_raid(1, id, ip + 1, nd, size, v); } SPEED_STOP @@ -1447,8 +1463,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #ifdef CONFIG_NEON SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(2, raid_gen2_neon_raid); raid_rec2_neon(2, id, ip, nd, size, v); } SPEED_STOP @@ -1457,8 +1471,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #endif #ifdef CONFIG_NEON32 SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(2, raid_gen2_neon32_raid); raid_rec2_neon32(2, id, ip, nd, size, v); } SPEED_STOP @@ -1496,10 +1508,16 @@ void speed_rec(int nd, void** v, int size, int delta, int period) fflush(stdout); } #ifdef CONFIG_X86_64 + if (raid_cpu_has_avx2()) { + SPEED_START { + raid_rec2_avx2ext(2, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } if (raid_cpu_has_avx512bw()) { SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(2, raid_gen2_avx512bw); raid_rec2_avx512bw(2, id, ip, nd, size, v); } SPEED_STOP @@ -1508,8 +1526,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period) } if (raid_cpu_has_avx2gfni()) { SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(2, raid_gen2_avx2gfni_raid); raid_rec2_avx2gfni_raid(2, id, ip, nd, size, v); } SPEED_STOP @@ -1518,8 +1534,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period) } if (raid_cpu_has_avx512gfni()) { SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(2, raid_gen2_avx512gfni_raid); raid_rec2_avx512gfni_raid(2, id, ip, nd, size, v); } SPEED_STOP @@ -1547,8 +1561,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #ifdef CONFIG_NEON SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(2, raid_gen2_neon_raid); /* +1 to avoid GEN2 optimized case */ raid_rec2_neon(2, id, ip + 1, nd, size, v); } SPEED_STOP @@ -1558,8 +1570,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #endif #ifdef CONFIG_NEON32 SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(2, raid_gen2_neon32_raid); /* +1 to avoid GEN2 optimized case */ raid_rec2_neon32(2, id, ip + 1, nd, size, v); } SPEED_STOP @@ -1600,10 +1610,17 @@ void speed_rec(int nd, void** v, int size, int delta, int period) fflush(stdout); } #ifdef CONFIG_X86_64 + if (raid_cpu_has_avx2()) { + SPEED_START { + /* +1 to avoid GEN2 optimized case */ + raid_rec2_avx2ext(2, id, ip + 1, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } if (raid_cpu_has_avx512bw()) { SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(2, raid_gen2_avx512bw); /* +1 to avoid GEN2 optimized case */ raid_rec2_avx512bw(2, id, ip + 1, nd, size, v); } SPEED_STOP @@ -1613,8 +1630,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period) } if (raid_cpu_has_avx2gfni()) { SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(2, raid_gen2_avx2gfni_raid); /* +1 to avoid GEN2 optimized case */ raid_rec2_avx2gfni_raid(2, id, ip + 1, nd, size, v); } SPEED_STOP @@ -1624,8 +1639,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period) } if (raid_cpu_has_avx512gfni()) { SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(2, raid_gen2_avx512gfni_raid); /* +1 to avoid GEN2 optimized case */ raid_rec2_avx512gfni_raid(2, id, ip + 1, nd, size, v); } SPEED_STOP @@ -1652,9 +1665,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #ifdef CONFIG_NEON SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(3, raid_gen3_neon_raid); - raid_recX_neon(3, id, ip, nd, size, v); + raid_rec3_neon(3, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1662,9 +1673,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #endif #ifdef CONFIG_NEON32 SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(3, raid_gen3_neon32_raid); - raid_recX_neon32(3, id, ip, nd, size, v); + raid_rec3_neon32(3, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1694,19 +1703,24 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #else raid_gen_force(3, raid_gen3_ssse3_raid); #endif - raid_recX_avx2(3, id, ip, nd, size, v); + raid_rec3_avx2(3, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); fflush(stdout); } #ifdef CONFIG_X86_64 + if (raid_cpu_has_avx2()) { + SPEED_START { + raid_rec3_avx2ext(3, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } if (raid_cpu_has_avx512bw()) { SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(3, raid_gen3_avx512bw); - /* +1 to avoid GEN1 optimized case */ - raid_recX_avx512bw(3, id, ip, nd, size, v); + raid_rec3_avx512bw(3, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1714,9 +1728,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period) } if (raid_cpu_has_avx2gfni()) { SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(3, raid_gen3_avx2gfni_raid); - raid_recX_avx2gfni_raid(3, id, ip, nd, size, v); + raid_rec3_avx2gfni_raid(3, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1724,9 +1736,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period) } if (raid_cpu_has_avx512gfni()) { SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(3, raid_gen3_avx512gfni_raid); - raid_recX_avx512gfni_raid(3, id, ip, nd, size, v); + raid_rec3_avx512gfni_raid(3, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1751,9 +1761,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #ifdef CONFIG_NEON SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(4, raid_gen4_neon_raid); - raid_recX_neon(4, id, ip, nd, size, v); + raid_rec4_neon(4, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1761,9 +1769,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #endif #ifdef CONFIG_NEON32 SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(4, raid_gen4_neon32_raid); - raid_recX_neon32(4, id, ip, nd, size, v); + raid_rec4_neon32(4, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1794,19 +1800,24 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #else raid_gen_force(4, raid_gen4_ssse3_raid); #endif - raid_recX_avx2(4, id, ip, nd, size, v); + raid_rec4_avx2(4, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); fflush(stdout); } #ifdef CONFIG_X86_64 + if (raid_cpu_has_avx2()) { + SPEED_START { + raid_rec4_avx2ext(4, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } if (raid_cpu_has_avx512bw()) { SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(4, raid_gen4_avx512bw); - /* +1 to avoid GEN1 optimized case */ - raid_recX_avx512bw(4, id, ip, nd, size, v); + raid_rec4_avx512bw(4, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1814,9 +1825,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period) } if (raid_cpu_has_avx2gfni()) { SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(4, raid_gen4_avx2gfni_raid); - raid_recX_avx2gfni_raid(4, id, ip, nd, size, v); + raid_rec4_avx2gfni_raid(4, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1824,9 +1833,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period) } if (raid_cpu_has_avx512gfni()) { SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(4, raid_gen4_avx512gfni_raid); - raid_recX_avx512gfni_raid(4, id, ip, nd, size, v); + raid_rec4_avx512gfni_raid(4, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1851,9 +1858,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #ifdef CONFIG_NEON SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(5, raid_gen5_neon_raid); - raid_recX_neon(5, id, ip, nd, size, v); + raid_rec5_neon(5, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1861,9 +1866,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #endif #ifdef CONFIG_NEON32 SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(5, raid_gen5_neon32_raid); - raid_recX_neon32(5, id, ip, nd, size, v); + raid_rec5_neon32(5, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1893,19 +1896,24 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #else raid_gen_force(5, raid_gen5_ssse3_raid); #endif - raid_recX_avx2(5, id, ip, nd, size, v); + raid_rec5_avx2(5, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); fflush(stdout); } #ifdef CONFIG_X86_64 + if (raid_cpu_has_avx2()) { + SPEED_START { + raid_rec5_avx2ext(5, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } if (raid_cpu_has_avx512bw()) { SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(5, raid_gen5_avx512bw); - /* +1 to avoid GEN1 optimized case */ - raid_recX_avx512bw(5, id, ip, nd, size, v); + raid_rec5_avx512bw(5, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1913,9 +1921,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period) } if (raid_cpu_has_avx2gfni()) { SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(5, raid_gen5_avx2gfni_raid); - raid_recX_avx2gfni_raid(5, id, ip, nd, size, v); + raid_rec5_avx2gfni_raid(5, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1923,9 +1929,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period) } if (raid_cpu_has_avx512gfni()) { SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(5, raid_gen5_avx512gfni_raid); - raid_recX_avx512gfni_raid(5, id, ip, nd, size, v); + raid_rec5_avx512gfni_raid(5, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1950,9 +1954,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #ifdef CONFIG_NEON SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(6, raid_gen6_neon_raid); - raid_recX_neon(6, id, ip, nd, size, v); + raid_rec6_neon(6, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1960,9 +1962,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #endif #ifdef CONFIG_NEON32 SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(6, raid_gen6_neon32_raid); - raid_recX_neon32(6, id, ip, nd, size, v); + raid_rec6_neon32(6, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1992,19 +1992,24 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #else raid_gen_force(6, raid_gen6_ssse3_raid); #endif - raid_recX_avx2(6, id, ip, nd, size, v); + raid_rec6_avx2(6, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); fflush(stdout); } #ifdef CONFIG_X86_64 + if (raid_cpu_has_avx2()) { + SPEED_START { + raid_rec6_avx2ext(6, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } if (raid_cpu_has_avx512bw()) { SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(6, raid_gen6_avx512bw); - /* +1 to avoid GEN1 optimized case */ - raid_recX_avx512bw(6, id, ip, nd, size, v); + raid_rec6_avx512bw(6, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -2012,9 +2017,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period) } if (raid_cpu_has_avx2gfni()) { SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(6, raid_gen6_avx2gfni_raid); - raid_recX_avx2gfni_raid(6, id, ip, nd, size, v); + raid_rec6_avx2gfni_raid(6, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -2022,9 +2025,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period) } if (raid_cpu_has_avx512gfni()) { SPEED_START { - /* ensure to use same hardware in the delta step */ - raid_gen_force(6, raid_gen6_avx512gfni_raid); - raid_recX_avx512gfni_raid(6, id, ip, nd, size, v); + raid_rec6_avx512gfni_raid(6, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); diff --git a/raid/avx2.c b/raid/avx2.c index 3271724b..69a486da 100644 --- a/raid/avx2.c +++ b/raid/avx2.c @@ -125,8 +125,7 @@ void raid_gen2_avx2_aes(int nd, size_t size, void **vv) * * Note that it uses 16 registers, meaning that x64 is required. */ -static __always_inline void raid_gen2_avx2ext_gen(int nd, size_t size, - void **vv, int generator) +static __always_inline void raid_gen2_avx2ext_gen(int nd, size_t size, void **vv, int generator) { uint8_t **v = (uint8_t **)vv; uint8_t *p; @@ -1064,74 +1063,6 @@ void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) raid_avx_begin(); -#ifdef CONFIG_X86_64 - asm volatile ("vpbroadcastb %0,%%ymm6" : : "m" (gfconst16.low4[0])); - - /* the inverse matrix V[] is constant for the whole recovery. */ - asm volatile ("vbroadcasti128 %0,%%ymm8" : : "m" (raid_gfmulpshufb[V[0]][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm9" : : "m" (raid_gfmulpshufb[V[0]][1][0])); - asm volatile ("vbroadcasti128 %0,%%ymm10" : : "m" (raid_gfmulpshufb[V[1]][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm11" : : "m" (raid_gfmulpshufb[V[1]][1][0])); - asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfmulpshufb[V[2]][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfmulpshufb[V[2]][1][0])); - asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfmulpshufb[V[3]][0][0])); - asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfmulpshufb[V[3]][1][0])); - - for (i = 0; i < size; i += 32) { - /* d0 = p[0] ^ pa[0] */ - asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i])); - asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pa[0][i])); - asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); - - /* d1 = p[1] ^ pa[1] */ - asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[1][i])); - asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pa[1][i])); - asm volatile ("vpxor %ymm4,%ymm1,%ymm1"); - - /* - * Split both deltas into low/high nibbles once. - * - * ymm0 = d0 low - * ymm2 = d0 high - * ymm1 = d1 low - * ymm3 = d1 high - */ - asm volatile ("vpsrlw $4,%ymm0,%ymm2"); - asm volatile ("vpsrlw $4,%ymm1,%ymm3"); - asm volatile ("vpand %ymm6,%ymm0,%ymm0"); - asm volatile ("vpand %ymm6,%ymm2,%ymm2"); - asm volatile ("vpand %ymm6,%ymm1,%ymm1"); - asm volatile ("vpand %ymm6,%ymm3,%ymm3"); - - /* - * pa[0] = V[0] * d0 ^ V[1] * d1 - */ - asm volatile ("vpshufb %ymm0,%ymm8,%ymm4"); - asm volatile ("vpshufb %ymm2,%ymm9,%ymm5"); - asm volatile ("vpxor %ymm5,%ymm4,%ymm4"); - - asm volatile ("vpshufb %ymm1,%ymm10,%ymm5"); - asm volatile ("vpxor %ymm5,%ymm4,%ymm4"); - asm volatile ("vpshufb %ymm3,%ymm11,%ymm5"); - asm volatile ("vpxor %ymm5,%ymm4,%ymm4"); - - asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pa[0][i])); - - /* - * pa[1] = V[2] * d0 ^ V[3] * d1 - */ - asm volatile ("vpshufb %ymm0,%ymm12,%ymm4"); - asm volatile ("vpshufb %ymm2,%ymm13,%ymm5"); - asm volatile ("vpxor %ymm5,%ymm4,%ymm4"); - - asm volatile ("vpshufb %ymm1,%ymm14,%ymm5"); - asm volatile ("vpxor %ymm5,%ymm4,%ymm4"); - asm volatile ("vpshufb %ymm3,%ymm15,%ymm5"); - asm volatile ("vpxor %ymm5,%ymm4,%ymm4"); - - asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pa[1][i])); - } -#else /* CONFIG_X86_32 */ asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 32) { @@ -1190,256 +1121,1146 @@ void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) asm volatile ("vmovdqa %%ymm6,%0" : "=m" (pa[1][i])); } -#endif raid_avx_end(); } /* - * RAID recovering AVX2 implementation + * RAID recovering AVX2 implementation optimized for up to four failures. + * + * Compute all selected syndromes in one scan of the surviving data, then + * retain only their low/high nibbles while reconstructing the missing data. */ -void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) +static __always_inline void raid_recX_avx2_1234(int nr, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; - int N = nr; uint8_t *p[RAID_PARITY_MAX]; uint8_t *pa[RAID_PARITY_MAX]; + uint8_t *src[RAID_DATA_MAX]; uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; - size_t i; - int j, k; - - /* setup the coefficients matrix */ - for (j = 0; j < N; ++j) - for (k = 0; k < N; ++k) - G[j * N + k] = A(ip[j], id[k]); - - /* invert it to solve the system of linear equations */ - raid_invert(G, V, N); - - /* compute delta parity */ - raid_delta_gen(N, id, ip, nd, size, vv); - - for (j = 0; j < N; ++j) { - p[j] = v[nd + ip[j]]; - pa[j] = v[id[j]]; - } - - raid_avx_begin(); - -#ifdef CONFIG_X86_64 - const uint8_t *T[RAID_PARITY_MAX * RAID_PARITY_MAX]; - - /* precompute shuffle table pointers */ - for (j = 0; j < N * N; ++j) - T[j] = &raid_gfmulpshufb[V[j]][0][0]; - - asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0])); - - for (i = 0; i < size; i += 32) { - /* delta */ - asm volatile ( - "movq 0(%2), %%rax\n" - "movq 0(%3), %%rbx\n" - "vmovdqa (%%rax, %1), %%ymm12\n" - "vmovdqa (%%rbx, %1), %%ymm13\n" - "vpxor %%ymm13, %%ymm12, %%ymm12\n" - "vpsrlw $4, %%ymm12, %%ymm1\n" - "vpand %%ymm15, %%ymm12, %%ymm0\n" - "vpand %%ymm15, %%ymm1, %%ymm1\n" - "cmpq $1, %0\n" - "jbe 1f\n" - - "movq 8(%2), %%rax\n" - "movq 8(%3), %%rbx\n" - "vmovdqa (%%rax, %1), %%ymm12\n" - "vmovdqa (%%rbx, %1), %%ymm13\n" - "vpxor %%ymm13, %%ymm12, %%ymm12\n" - "vpsrlw $4, %%ymm12, %%ymm3\n" - "vpand %%ymm15, %%ymm12, %%ymm2\n" - "vpand %%ymm15, %%ymm3, %%ymm3\n" - "cmpq $2, %0\n" - "jbe 1f\n" - - "movq 16(%2), %%rax\n" - "movq 16(%3), %%rbx\n" - "vmovdqa (%%rax, %1), %%ymm12\n" - "vmovdqa (%%rbx, %1), %%ymm13\n" - "vpxor %%ymm13, %%ymm12, %%ymm12\n" - "vpsrlw $4, %%ymm12, %%ymm5\n" - "vpand %%ymm15, %%ymm12, %%ymm4\n" - "vpand %%ymm15, %%ymm5, %%ymm5\n" - "cmpq $3, %0\n" - "jbe 1f\n" - - "movq 24(%2), %%rax\n" - "movq 24(%3), %%rbx\n" - "vmovdqa (%%rax, %1), %%ymm12\n" - "vmovdqa (%%rbx, %1), %%ymm13\n" - "vpxor %%ymm13, %%ymm12, %%ymm12\n" - "vpsrlw $4, %%ymm12, %%ymm7\n" - "vpand %%ymm15, %%ymm12, %%ymm6\n" - "vpand %%ymm15, %%ymm7, %%ymm7\n" - "cmpq $4, %0\n" - "jbe 1f\n" - - "movq 32(%2), %%rax\n" - "movq 32(%3), %%rbx\n" - "vmovdqa (%%rax, %1), %%ymm12\n" - "vmovdqa (%%rbx, %1), %%ymm13\n" - "vpxor %%ymm13, %%ymm12, %%ymm12\n" - "vpsrlw $4, %%ymm12, %%ymm9\n" - "vpand %%ymm15, %%ymm12, %%ymm8\n" - "vpand %%ymm15, %%ymm9, %%ymm9\n" - "cmpq $5, %0\n" - "jbe 1f\n" - - "movq 40(%2), %%rax\n" - "movq 40(%3), %%rbx\n" - "vmovdqa (%%rax, %1), %%ymm12\n" - "vmovdqa (%%rbx, %1), %%ymm13\n" - "vpxor %%ymm13, %%ymm12, %%ymm12\n" - "vpsrlw $4, %%ymm12, %%ymm11\n" - "vpand %%ymm15, %%ymm12, %%ymm10\n" - "vpand %%ymm15, %%ymm11, %%ymm11\n" - - "1:\n" - : - : "r" ((uint64_t)N), "r" (i), "r" (p), "r" (pa) - : "rax", "rbx", "cc", "memory" - ); - - /* reconstruct */ - for (j = 0; j < N; ++j) { - asm volatile ( - "movq 0(%2), %%rcx\n" - "vbroadcasti128 0(%%rcx), %%ymm13\n" - "vbroadcasti128 16(%%rcx), %%ymm14\n" - "vpshufb %%ymm0, %%ymm13, %%ymm13\n" - "vpshufb %%ymm1, %%ymm14, %%ymm14\n" - "cmpq $1, %0\n" - "jbe 1f\n" - - "movq 8(%2), %%rcx\n" - "vbroadcasti128 0(%%rcx), %%ymm12\n" - "vpshufb %%ymm2, %%ymm12, %%ymm12\n" - "vpxor %%ymm12, %%ymm13, %%ymm13\n" - "vbroadcasti128 16(%%rcx), %%ymm12\n" - "vpshufb %%ymm3, %%ymm12, %%ymm12\n" - "vpxor %%ymm12, %%ymm14, %%ymm14\n" - "cmpq $2, %0\n" - "jbe 1f\n" - - "movq 16(%2), %%rcx\n" - "vbroadcasti128 0(%%rcx), %%ymm12\n" - "vpshufb %%ymm4, %%ymm12, %%ymm12\n" - "vpxor %%ymm12, %%ymm13, %%ymm13\n" - "vbroadcasti128 16(%%rcx), %%ymm12\n" - "vpshufb %%ymm5, %%ymm12, %%ymm12\n" - "vpxor %%ymm12, %%ymm14, %%ymm14\n" - "cmpq $3, %0\n" - "jbe 1f\n" - - "movq 24(%2), %%rcx\n" - "vbroadcasti128 0(%%rcx), %%ymm12\n" - "vpshufb %%ymm6, %%ymm12, %%ymm12\n" - "vpxor %%ymm12, %%ymm13, %%ymm13\n" - "vbroadcasti128 16(%%rcx), %%ymm12\n" - "vpshufb %%ymm7, %%ymm12, %%ymm12\n" - "vpxor %%ymm12, %%ymm14, %%ymm14\n" - "cmpq $4, %0\n" - "jbe 1f\n" - - "movq 32(%2), %%rcx\n" - "vbroadcasti128 0(%%rcx), %%ymm12\n" - "vpshufb %%ymm8, %%ymm12, %%ymm12\n" - "vpxor %%ymm12, %%ymm13, %%ymm13\n" - "vbroadcasti128 16(%%rcx), %%ymm12\n" - "vpshufb %%ymm9, %%ymm12, %%ymm12\n" - "vpxor %%ymm12, %%ymm14, %%ymm14\n" - "cmpq $5, %0\n" - "jbe 1f\n" - - "movq 40(%2), %%rcx\n" - "vbroadcasti128 0(%%rcx), %%ymm12\n" - "vpshufb %%ymm10, %%ymm12, %%ymm12\n" - "vpxor %%ymm12, %%ymm13, %%ymm13\n" - "vbroadcasti128 16(%%rcx), %%ymm12\n" - "vpshufb %%ymm11, %%ymm12, %%ymm12\n" - "vpxor %%ymm12, %%ymm14, %%ymm14\n" - - "1:\n" - "vpxor %%ymm14, %%ymm13, %%ymm13\n" - "movq %3, %%rax\n" - "vmovdqa %%ymm13, (%%rax, %1)\n" - : - : "r" ((uint64_t)N), "r" (i), "r" (&T[j * N]), "r" (pa[j]) - : "rax", "rcx", "cc", "memory" - ); - } - } -#else /* CONFIG_X86_32 */ + const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX]; + const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX]; uint8_t buffer_low[RAID_PARITY_MAX * 32 + 32]; uint8_t buffer_high[RAID_PARITY_MAX * 32 + 32]; uint8_t *pd_low = __align_ptr(buffer_low, 32); uint8_t *pd_high = __align_ptr(buffer_high, 32); + size_t i; + int d, j, k, s; + int ns; + int has_p; + + BUG_ON(nr < 1 || nr > 4); + + /* setup the coefficients matrix */ + for (j = 0; j < nr; ++j) + for (k = 0; k < nr; ++k) + G[j * nr + k] = A(ip[j], id[k]); + + /* invert it to solve the system of linear equations */ + raid_invert(G, V, nr); + + /* setup selected parity and destination pointers */ + for (j = 0; j < nr; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + /* ip[] is ordered. If P is available, it is always ip[0] */ + has_p = ip[0] == 0; + + /* build the compact surviving-data list and its syndrome tables */ + ns = 0; + k = 0; + for (d = 0; d < nd; ++d) { + if (k < nr && d == id[k]) { + ++k; + continue; + } + + src[ns] = v[d]; + for (j = 0; j < nr; ++j) + S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0]; + ++ns; + } + + BUG_ON(k != nr); + BUG_ON(ns != nd - nr); + + /* precompute inverse-matrix multiplication table pointers */ + for (j = 0; j < nr; ++j) + for (k = 0; k < nr; ++k) + R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0]; + + raid_avx_begin(); asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 32) { - /* delta */ - for (j = 0; j < N; ++j) { - asm volatile ( - "vmovdqa %2, %%ymm0\n" - "vmovdqa %3, %%ymm1\n" - "vpxor %%ymm1, %%ymm0, %%ymm0\n" - "vpsrlw $4, %%ymm0, %%ymm1\n" - "vpand %%ymm7, %%ymm0, %%ymm0\n" - "vpand %%ymm7, %%ymm1, %%ymm1\n" - "vmovdqa %%ymm0, %0\n" - "vmovdqa %%ymm1, %1\n" - : "=m" (pd_low[j * 32]), "=m" (pd_high[j * 32]) - : "m" (p[j][i]), "m" (pa[j][i]) - ); - } + /* ymm0..ymm3 are syndrome accumulators */ + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i])); + if (nr >= 2) + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[1][i])); + if (nr >= 3) + asm volatile ("vmovdqa %0,%%ymm2" : : "m" (p[2][i])); + if (nr >= 4) + asm volatile ("vmovdqa %0,%%ymm3" : : "m" (p[3][i])); - /* reconstruct */ - for (j = 0; j < N; ++j) { - asm volatile ( - "vpxor %%ymm0, %%ymm0, %%ymm0\n" - "vpxor %%ymm1, %%ymm1, %%ymm1\n" - ); + /* all selected syndromes are accumulated in this single source scan */ + for (s = 0; s < ns; ++s) { + const uint8_t **t = S[s]; - for (k = 0; k < N; ++k) { - uint8_t m = V[j * N + k]; + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (src[s][i])); + if (has_p) + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); - asm volatile ( - "vbroadcasti128 %0, %%ymm2\n" - "vbroadcasti128 %1, %%ymm3\n" - "vmovdqa %2, %%ymm4\n" - "vmovdqa %3, %%ymm5\n" - "vpshufb %%ymm4, %%ymm2, %%ymm2\n" - "vpshufb %%ymm5, %%ymm3, %%ymm3\n" - "vpxor %%ymm2, %%ymm0, %%ymm0\n" - "vpxor %%ymm3, %%ymm1, %%ymm1\n" - : - : "m" (raid_gfmulpshufb[m][0][0]), "m" (raid_gfmulpshufb[m][1][0]), - "m" (pd_low[k * 32]), "m" (pd_high[k * 32]) - ); + asm volatile ("vpsrlw $4,%ymm4,%ymm5"); + asm volatile ("vpand %ymm7,%ymm4,%ymm4"); + asm volatile ("vpand %ymm7,%ymm5,%ymm5"); + + if (!has_p) { + asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (t[0][0])); + asm volatile ("vpshufb %ymm4,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm6,%ymm0,%ymm0"); + asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (t[0][16])); + asm volatile ("vpshufb %ymm5,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm6,%ymm0,%ymm0"); } - asm volatile ( - "vpxor %%ymm1, %%ymm0, %%ymm0\n" - "vmovdqa %%ymm0, %0\n" - : "=m" (pa[j][i]) - ); + if (nr >= 2) { + asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (t[1][0])); + asm volatile ("vpshufb %ymm4,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm6,%ymm1,%ymm1"); + asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (t[1][16])); + asm volatile ("vpshufb %ymm5,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm6,%ymm1,%ymm1"); + } + + if (nr >= 3) { + asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (t[2][0])); + asm volatile ("vpshufb %ymm4,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm6,%ymm2,%ymm2"); + asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (t[2][16])); + asm volatile ("vpshufb %ymm5,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm6,%ymm2,%ymm2"); + } + + if (nr >= 4) { + asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (t[3][0])); + asm volatile ("vpshufb %ymm4,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm6,%ymm3,%ymm3"); + asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (t[3][16])); + asm volatile ("vpshufb %ymm5,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm6,%ymm3,%ymm3"); + } + } + + /* preserve only aligned low/high nibbles, never full delta blocks */ + asm volatile ("vpsrlw $4,%ymm0,%ymm4"); + asm volatile ("vpand %ymm7,%ymm0,%ymm0"); + asm volatile ("vpand %ymm7,%ymm4,%ymm4"); + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pd_low[0])); + asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pd_high[0])); + + if (nr >= 2) { + asm volatile ("vpsrlw $4,%ymm1,%ymm4"); + asm volatile ("vpand %ymm7,%ymm1,%ymm1"); + asm volatile ("vpand %ymm7,%ymm4,%ymm4"); + asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pd_low[32])); + asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pd_high[32])); + } + + if (nr >= 3) { + asm volatile ("vpsrlw $4,%ymm2,%ymm4"); + asm volatile ("vpand %ymm7,%ymm2,%ymm2"); + asm volatile ("vpand %ymm7,%ymm4,%ymm4"); + asm volatile ("vmovdqa %%ymm2,%0" : "=m" (pd_low[64])); + asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pd_high[64])); + } + + if (nr >= 4) { + asm volatile ("vpsrlw $4,%ymm3,%ymm4"); + asm volatile ("vpand %ymm7,%ymm3,%ymm3"); + asm volatile ("vpand %ymm7,%ymm4,%ymm4"); + asm volatile ("vmovdqa %%ymm3,%0" : "=m" (pd_low[96])); + asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pd_high[96])); + } + + /* reconstruct every missing block, initializing from coefficient zero */ + for (j = 0; j < nr; ++j) { + const uint8_t **t = R[j]; + + asm volatile ("vbroadcasti128 %0,%%ymm0" : : "m" (t[0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm1" : : "m" (t[0][16])); + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pd_low[0])); + asm volatile ("vmovdqa %0,%%ymm5" : : "m" (pd_high[0])); + asm volatile ("vpshufb %ymm4,%ymm0,%ymm0"); + asm volatile ("vpshufb %ymm5,%ymm1,%ymm1"); + + for (k = 1; k < nr; ++k) { + asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (t[k][0])); + asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (t[k][16])); + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pd_low[k * 32])); + asm volatile ("vmovdqa %0,%%ymm5" : : "m" (pd_high[k * 32])); + asm volatile ("vpshufb %ymm4,%ymm2,%ymm2"); + asm volatile ("vpshufb %ymm5,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm2,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm3,%ymm1,%ymm1"); + } + + asm volatile ("vpxor %ymm1,%ymm0,%ymm0"); + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[j][i])); } } -#endif raid_avx_end(); } +/* + * RAID recovering AVX2 implementation for every supported failure count. + * + * With only eight YMM registers, selected syndromes are fused in groups of + * four while their low/high nibbles are retained for reconstruction. + */ +static __always_inline void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p[RAID_PARITY_MAX]; + uint8_t *pa[RAID_PARITY_MAX]; + uint8_t *src[RAID_DATA_MAX]; + uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; + const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX]; + const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX]; + uint8_t buffer_low[RAID_PARITY_MAX * 32 + 32]; + uint8_t buffer_high[RAID_PARITY_MAX * 32 + 32]; + uint8_t *pd_low = __align_ptr(buffer_low, 32); + uint8_t *pd_high = __align_ptr(buffer_high, 32); + size_t i; + int d, j, k, s; + int ns; + int has_p; + + BUG_ON(nr < 1 || nr > 6); + + /* setup the coefficients matrix */ + for (j = 0; j < nr; ++j) + for (k = 0; k < nr; ++k) + G[j * nr + k] = A(ip[j], id[k]); + + /* invert it to solve the system of linear equations */ + raid_invert(G, V, nr); + + /* setup selected parity and destination pointers */ + for (j = 0; j < nr; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + /* ip[] is ordered. If P is available, it is always ip[0] */ + has_p = ip[0] == 0; + + /* build the compact surviving-data list and its syndrome tables */ + ns = 0; + k = 0; + + for (d = 0; d < nd; ++d) { + if (k < nr && d == id[k]) { + ++k; + continue; + } + + src[ns] = v[d]; + + for (j = 0; j < nr; ++j) + S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0]; + + ++ns; + } + + BUG_ON(k != nr); + BUG_ON(ns != nd - nr); + + /* precompute inverse-matrix multiplication table pointers */ + for (j = 0; j < nr; ++j) + for (k = 0; k < nr; ++k) + R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0]; + + raid_avx_begin(); + + /* + * During syndrome generation: + * + * ymm0..ymm3 syndrome accumulators + * ymm4 source / source low nibble + * ymm5 source high nibble + * ymm6 multiplication table / temporary + * ymm7 low-nibble mask + */ + asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.low4[0])); + + for (i = 0; i < size; i += 32) { + /* + * First pass: up to syndromes 0..3. + */ + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i])); + if (nr >= 2) + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[1][i])); + if (nr >= 3) + asm volatile ("vmovdqa %0,%%ymm2" : : "m" (p[2][i])); + if (nr >= 4) + asm volatile ("vmovdqa %0,%%ymm3" : : "m" (p[3][i])); + + for (s = 0; s < ns; ++s) { + const uint8_t **t = S[s]; + + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (src[s][i])); + + /* P has coefficient 1 for every data disk */ + if (has_p) + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); + + /* split source into low/high nibbles */ + asm volatile ("vpsrlw $4,%ymm4,%ymm5"); + asm volatile ("vpand %ymm7,%ymm4,%ymm4"); + asm volatile ("vpand %ymm7,%ymm5,%ymm5"); + + /* syndrome 0 */ + if (!has_p) { + asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (t[0][0])); + asm volatile ("vpshufb %ymm4,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm6,%ymm0,%ymm0"); + asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (t[0][16])); + asm volatile ("vpshufb %ymm5,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm6,%ymm0,%ymm0"); + } + + if (nr >= 2) { + /* syndrome 1 */ + asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (t[1][0])); + asm volatile ("vpshufb %ymm4,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm6,%ymm1,%ymm1"); + asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (t[1][16])); + asm volatile ("vpshufb %ymm5,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm6,%ymm1,%ymm1"); + } + + if (nr >= 3) { + /* syndrome 2 */ + asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (t[2][0])); + asm volatile ("vpshufb %ymm4,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm6,%ymm2,%ymm2"); + asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (t[2][16])); + asm volatile ("vpshufb %ymm5,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm6,%ymm2,%ymm2"); + } + + if (nr >= 4) { + /* syndrome 3 */ + asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (t[3][0])); + asm volatile ("vpshufb %ymm4,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm6,%ymm3,%ymm3"); + asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (t[3][16])); + asm volatile ("vpshufb %ymm5,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm6,%ymm3,%ymm3"); + } + } + + /* split and store syndrome 0 */ + asm volatile ("vpsrlw $4,%ymm0,%ymm4"); + asm volatile ("vpand %ymm7,%ymm0,%ymm0"); + asm volatile ("vpand %ymm7,%ymm4,%ymm4"); + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pd_low[0])); + asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pd_high[0])); + + if (nr >= 2) { + /* split and store syndrome 1 */ + asm volatile ("vpsrlw $4,%ymm1,%ymm4"); + asm volatile ("vpand %ymm7,%ymm1,%ymm1"); + asm volatile ("vpand %ymm7,%ymm4,%ymm4"); + asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pd_low[32])); + asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pd_high[32])); + } + + if (nr >= 3) { + /* split and store syndrome 2 */ + asm volatile ("vpsrlw $4,%ymm2,%ymm4"); + asm volatile ("vpand %ymm7,%ymm2,%ymm2"); + asm volatile ("vpand %ymm7,%ymm4,%ymm4"); + asm volatile ("vmovdqa %%ymm2,%0" : "=m" (pd_low[64])); + asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pd_high[64])); + } + + if (nr >= 4) { + /* split and store syndrome 3 */ + asm volatile ("vpsrlw $4,%ymm3,%ymm4"); + asm volatile ("vpand %ymm7,%ymm3,%ymm3"); + asm volatile ("vpand %ymm7,%ymm4,%ymm4"); + asm volatile ("vmovdqa %%ymm3,%0" : "=m" (pd_low[96])); + asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pd_high[96])); + } + + /* + * Second pass: syndrome 4 and, for nr == 6, syndrome 5. + * + * P cannot occur here because ip[] is ordered. + */ + if (nr >= 5) { + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[4][i])); + + if (nr >= 6) + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[5][i])); + + for (s = 0; s < ns; ++s) { + const uint8_t **t = S[s]; + + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (src[s][i])); + asm volatile ("vpsrlw $4,%ymm4,%ymm5"); + asm volatile ("vpand %ymm7,%ymm4,%ymm4"); + asm volatile ("vpand %ymm7,%ymm5,%ymm5"); + + /* syndrome 4 */ + asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (t[4][0])); + asm volatile ("vpshufb %ymm4,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm6,%ymm0,%ymm0"); + asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (t[4][16])); + asm volatile ("vpshufb %ymm5,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm6,%ymm0,%ymm0"); + + /* syndrome 5 */ + if (nr >= 6) { + asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (t[5][0])); + asm volatile ("vpshufb %ymm4,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm6,%ymm1,%ymm1"); + asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (t[5][16])); + asm volatile ("vpshufb %ymm5,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm6,%ymm1,%ymm1"); + } + } + + /* split and store syndrome 4 */ + asm volatile ("vpsrlw $4,%ymm0,%ymm4"); + asm volatile ("vpand %ymm7,%ymm0,%ymm0"); + asm volatile ("vpand %ymm7,%ymm4,%ymm4"); + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pd_low[128])); + asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pd_high[128])); + + /* split and store syndrome 5 */ + if (nr >= 6) { + asm volatile ("vpsrlw $4,%ymm1,%ymm4"); + asm volatile ("vpand %ymm7,%ymm1,%ymm1"); + asm volatile ("vpand %ymm7,%ymm4,%ymm4"); + asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pd_low[160])); + asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pd_high[160])); + } + } + + /* + * Reconstruct every missing block. + * + * Low and high GF multiplications use independent dependency + * chains to maximize instruction-level parallelism: + * + * ymm0 low accumulator + * ymm1 high accumulator + * ymm2 low multiplication table/result + * ymm3 high multiplication table/result + * ymm4 low syndrome nibble + * ymm5 high syndrome nibble + */ + for (j = 0; j < nr; ++j) { + const uint8_t **t = R[j]; + + /* first coefficient initializes the accumulators */ + asm volatile ("vbroadcasti128 %0,%%ymm0" : : "m" (t[0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm1" : : "m" (t[0][16])); + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pd_low[0])); + asm volatile ("vmovdqa %0,%%ymm5" : : "m" (pd_high[0])); + asm volatile ("vpshufb %ymm4,%ymm0,%ymm0"); + asm volatile ("vpshufb %ymm5,%ymm1,%ymm1"); + + for (k = 1; k < nr; ++k) { + asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (t[k][0])); + asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (t[k][16])); + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pd_low[k * 32])); + asm volatile ("vmovdqa %0,%%ymm5" : : "m" (pd_high[k * 32])); + asm volatile ("vpshufb %ymm4,%ymm2,%ymm2"); + asm volatile ("vpshufb %ymm5,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm2,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm3,%ymm1,%ymm1"); + } + + asm volatile ("vpxor %ymm1,%ymm0,%ymm0"); + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[j][i])); + } + } + + raid_avx_end(); +} + +void raid_rec3_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 3); + raid_recX_avx2_1234(3, id, ip, nd, size, vv); +} + +void raid_rec4_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 4); + raid_recX_avx2_1234(4, id, ip, nd, size, vv); +} + +void raid_rec5_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 5); + raid_recX_avx2(5, id, ip, nd, size, vv); +} + +void raid_rec6_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 6); + raid_recX_avx2(6, id, ip, nd, size, vv); +} + +#ifdef CONFIG_X86_64 +/* + * RAID recovering AVX2 implementation. + * + * Compute only the selected syndromes, keeping them in registers. + * This avoids raid_delta_gen(), temporary syndrome buffers, and the + * generation of unused parity rows. + */ +static __always_inline void raid_recX_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p[RAID_PARITY_MAX]; + uint8_t *pa[RAID_PARITY_MAX]; + uint8_t *src[RAID_DATA_MAX]; + uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; + const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX]; + const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX]; + size_t i; + int d, j, k, s; + int ns; + int has_p; + + /* setup the coefficients matrix */ + for (j = 0; j < nr; ++j) + for (k = 0; k < nr; ++k) + G[j * nr + k] = A(ip[j], id[k]); + + /* invert it to solve the system of linear equations */ + raid_invert(G, V, nr); + + /* setup selected parity and destination pointers */ + for (j = 0; j < nr; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + /* ip[] is ordered. If P is available, it is always ip[0] */ + has_p = ip[0] == 0; + + /* build the compact surviving-data list and its syndrome tables */ + ns = 0; + k = 0; + + for (d = 0; d < nd; ++d) { + if (k < nr && d == id[k]) { + ++k; + continue; + } + + src[ns] = v[d]; + + for (j = 0; j < nr; ++j) + S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0]; + + ++ns; + } + + BUG_ON(k != nr); + BUG_ON(ns != nd - nr); + + /* precompute inverse-matrix multiplication table pointers */ + for (j = 0; j < nr; ++j) + for (k = 0; k < nr; ++k) + R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0]; + + raid_avx_begin(); + + asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0])); + + for (i = 0; i < size; i += 32) { + /* + * Start each syndrome with the corresponding stored parity. + * + * During syndrome computation: + * + * ymm0 syndrome 0 + * ymm2 syndrome 1 + * ymm4 syndrome 2 + * ymm6 syndrome 3 + * ymm8 syndrome 4 + * ymm10 syndrome 5 + * + * ymm12 source / source low nibble + * ymm13 source high nibble + * ymm14 multiplication table / temporary + * ymm15 low-nibble mask + */ + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i])); + + if (nr >= 2) + asm volatile ("vmovdqa %0,%%ymm2" : : "m" (p[1][i])); + + if (nr >= 3) + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (p[2][i])); + + if (nr >= 4) + asm volatile ("vmovdqa %0,%%ymm6" : : "m" (p[3][i])); + + if (nr >= 5) + asm volatile ("vmovdqa %0,%%ymm8" : : "m" (p[4][i])); + + if (nr >= 6) + asm volatile ("vmovdqa %0,%%ymm10" : : "m" (p[5][i])); + + /* + * Add all surviving data contributions. + */ + for (s = 0; s < ns; ++s) { + const uint8_t **t = S[s]; + + asm volatile ("vmovdqa %0,%%ymm12" : : "m" (src[s][i])); + + /* + * P has coefficient 1 for every data disk. + * + * Keep the simple XOR fast path when P is the + * first selected parity. Otherwise syndrome 0 + * uses the normal vpshufb multiplication. + */ + if (has_p) { + asm volatile ("vpxor %ymm12,%ymm0,%ymm0"); + + asm volatile ("vpsrlw $4,%ymm12,%ymm13"); + asm volatile ("vpand %ymm15,%ymm12,%ymm12"); + asm volatile ("vpand %ymm15,%ymm13,%ymm13"); + } else { + asm volatile ("vpsrlw $4,%ymm12,%ymm13"); + asm volatile ("vpand %ymm15,%ymm12,%ymm12"); + asm volatile ("vpand %ymm15,%ymm13,%ymm13"); + + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[0][0])); + asm volatile ("vpshufb %ymm12,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm14,%ymm0,%ymm0"); + + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[0][16])); + asm volatile ("vpshufb %ymm13,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm14,%ymm0,%ymm0"); + } + + if (nr >= 2) { + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[1][0])); + asm volatile ("vpshufb %ymm12,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm14,%ymm2,%ymm2"); + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[1][16])); + asm volatile ("vpshufb %ymm13,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm14,%ymm2,%ymm2"); + } + + if (nr >= 3) { + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[2][0])); + asm volatile ("vpshufb %ymm12,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm14,%ymm4,%ymm4"); + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[2][16])); + asm volatile ("vpshufb %ymm13,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm14,%ymm4,%ymm4"); + } + + if (nr >= 4) { + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[3][0])); + asm volatile ("vpshufb %ymm12,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm14,%ymm6,%ymm6"); + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[3][16])); + asm volatile ("vpshufb %ymm13,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm14,%ymm6,%ymm6"); + } + + if (nr >= 5) { + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[4][0])); + asm volatile ("vpshufb %ymm12,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm14,%ymm8,%ymm8"); + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[4][16])); + asm volatile ("vpshufb %ymm13,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm14,%ymm8,%ymm8"); + } + + if (nr >= 6) { + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[5][0])); + asm volatile ("vpshufb %ymm12,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm14,%ymm10,%ymm10"); + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[5][16])); + asm volatile ("vpshufb %ymm13,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm14,%ymm10,%ymm10"); + } + } + + /* + * Split every completed syndrome once. + * + * After this: + * + * ymm0 /ymm1 delta 0 low/high + * ymm2 /ymm3 delta 1 low/high + * ymm4 /ymm5 delta 2 low/high + * ymm6 /ymm7 delta 3 low/high + * ymm8 /ymm9 delta 4 low/high + * ymm10/ymm11 delta 5 low/high + */ + asm volatile ("vpsrlw $4,%ymm0,%ymm1"); + asm volatile ("vpand %ymm15,%ymm0,%ymm0"); + asm volatile ("vpand %ymm15,%ymm1,%ymm1"); + + if (nr >= 2) { + asm volatile ("vpsrlw $4,%ymm2,%ymm3"); + asm volatile ("vpand %ymm15,%ymm2,%ymm2"); + asm volatile ("vpand %ymm15,%ymm3,%ymm3"); + } + + if (nr >= 3) { + asm volatile ("vpsrlw $4,%ymm4,%ymm5"); + asm volatile ("vpand %ymm15,%ymm4,%ymm4"); + asm volatile ("vpand %ymm15,%ymm5,%ymm5"); + } + + if (nr >= 4) { + asm volatile ("vpsrlw $4,%ymm6,%ymm7"); + asm volatile ("vpand %ymm15,%ymm6,%ymm6"); + asm volatile ("vpand %ymm15,%ymm7,%ymm7"); + } + + if (nr >= 5) { + asm volatile ("vpsrlw $4,%ymm8,%ymm9"); + asm volatile ("vpand %ymm15,%ymm8,%ymm8"); + asm volatile ("vpand %ymm15,%ymm9,%ymm9"); + } + + if (nr >= 6) { + asm volatile ("vpsrlw $4,%ymm10,%ymm11"); + asm volatile ("vpand %ymm15,%ymm10,%ymm10"); + asm volatile ("vpand %ymm15,%ymm11,%ymm11"); + } + + /* reconstruct every missing data block */ + for (j = 0; j < nr; ++j) { + const uint8_t **t = R[j]; + + /* first coefficient initializes the low/high accumulators */ + asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (t[0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (t[0][16])); + asm volatile ("vpshufb %ymm0,%ymm12,%ymm12"); + asm volatile ("vpshufb %ymm1,%ymm13,%ymm13"); + + if (nr >= 2) { + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[1][0])); + asm volatile ("vpshufb %ymm2,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm14,%ymm12,%ymm12"); + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[1][16])); + asm volatile ("vpshufb %ymm3,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm14,%ymm13,%ymm13"); + } + + if (nr >= 3) { + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[2][0])); + asm volatile ("vpshufb %ymm4,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm14,%ymm12,%ymm12"); + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[2][16])); + asm volatile ("vpshufb %ymm5,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm14,%ymm13,%ymm13"); + } + + if (nr >= 4) { + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[3][0])); + asm volatile ("vpshufb %ymm6,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm14,%ymm12,%ymm12"); + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[3][16])); + asm volatile ("vpshufb %ymm7,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm14,%ymm13,%ymm13"); + } + + if (nr >= 5) { + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[4][0])); + asm volatile ("vpshufb %ymm8,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm14,%ymm12,%ymm12"); + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[4][16])); + asm volatile ("vpshufb %ymm9,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm14,%ymm13,%ymm13"); + } + + if (nr >= 6) { + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[5][0])); + asm volatile ("vpshufb %ymm10,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm14,%ymm12,%ymm12"); + + asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[5][16])); + asm volatile ("vpshufb %ymm11,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm14,%ymm13,%ymm13"); + } + + asm volatile ("vpxor %ymm13,%ymm12,%ymm12"); + asm volatile ("vmovdqa %%ymm12,%0" : "=m" (pa[j][i])); + } + } + + raid_avx_end(); +} + +/* + * RAID recovering AVX2 implementation optimized for up to three failures. + * + * Process 64 bytes at a time as two independent 32-byte lanes. + * Multiplication tables are broadcast once and shared between both lanes. + * + * Note that it uses 16 registers, meaning that x64 is required. + */ +static __always_inline void raid_recX_avx2ext_123(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p[RAID_PARITY_MAX]; + uint8_t *pa[RAID_PARITY_MAX]; + uint8_t *src[RAID_DATA_MAX]; + uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; + const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX]; + const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX]; + size_t i; + int d, j, k, s; + int ns; + int has_p; + + BUG_ON(nr < 1 || nr > 3); + + /* setup the coefficients matrix */ + for (j = 0; j < nr; ++j) + for (k = 0; k < nr; ++k) + G[j * nr + k] = A(ip[j], id[k]); + + /* invert it to solve the system of linear equations */ + raid_invert(G, V, nr); + + /* setup selected parity and destination pointers */ + for (j = 0; j < nr; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + /* ip[] is ordered. If P is available, it is always ip[0] */ + has_p = ip[0] == 0; + + /* build the compact surviving-data list and its syndrome tables */ + ns = 0; + k = 0; + + for (d = 0; d < nd; ++d) { + if (k < nr && d == id[k]) { + ++k; + continue; + } + + src[ns] = v[d]; + + for (j = 0; j < nr; ++j) + S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0]; + + ++ns; + } + + BUG_ON(k != nr); + BUG_ON(ns != nd - nr); + + /* precompute inverse-matrix multiplication table pointers */ + for (j = 0; j < nr; ++j) + for (k = 0; k < nr; ++k) + R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0]; + + raid_avx_begin(); + + for (i = 0; i < size; i += 64) { + asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0])); + + /* + * Start each syndrome with the corresponding stored parity. + * + * During syndrome computation: + * + * ymm0/ymm1 syndrome 0, first/second 32 bytes + * ymm2/ymm3 syndrome 1, first/second 32 bytes + * ymm4/ymm5 syndrome 2, first/second 32 bytes + * + * ymm6/ymm7 source low, first/second 32 bytes + * ymm8/ymm9 source high, first/second 32 bytes + * ymm12/ymm13 multiplication table / temporary + * ymm15 low-nibble mask + */ + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i])); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[0][i + 32])); + + if (nr >= 2) { + asm volatile ("vmovdqa %0,%%ymm2" : : "m" (p[1][i])); + asm volatile ("vmovdqa %0,%%ymm3" : : "m" (p[1][i + 32])); + } + + if (nr >= 3) { + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (p[2][i])); + asm volatile ("vmovdqa %0,%%ymm5" : : "m" (p[2][i + 32])); + } + + /* + * Add all surviving data contributions. + */ + for (s = 0; s < ns; ++s) { + const uint8_t **t = S[s]; + + asm volatile ("vmovdqa %0,%%ymm6" : : "m" (src[s][i])); + asm volatile ("vmovdqa %0,%%ymm7" : : "m" (src[s][i + 32])); + + /* + * P has coefficient 1 for every data disk. + * Do the XOR before destroying the original source + * while splitting it into low/high nibbles. + */ + if (has_p) { + asm volatile ("vpxor %ymm6,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm7,%ymm1,%ymm1"); + } + + /* split both 32-byte source lanes */ + asm volatile ("vpsrlw $4,%ymm6,%ymm8"); + asm volatile ("vpsrlw $4,%ymm7,%ymm9"); + asm volatile ("vpand %ymm15,%ymm6,%ymm6"); + asm volatile ("vpand %ymm15,%ymm7,%ymm7"); + asm volatile ("vpand %ymm15,%ymm8,%ymm8"); + asm volatile ("vpand %ymm15,%ymm9,%ymm9"); + + /* + * Syndrome 0. + * + * Share each table broadcast between the two + * 32-byte lanes. + */ + if (!has_p) { + asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (t[0][0])); + asm volatile ("vmovdqa %ymm12,%ymm13"); + asm volatile ("vpshufb %ymm6,%ymm12,%ymm12"); + asm volatile ("vpshufb %ymm7,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm12,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm13,%ymm1,%ymm1"); + + asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (t[0][16])); + asm volatile ("vmovdqa %ymm12,%ymm13"); + asm volatile ("vpshufb %ymm8,%ymm12,%ymm12"); + asm volatile ("vpshufb %ymm9,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm12,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm13,%ymm1,%ymm1"); + } + + /* syndrome 1 */ + if (nr >= 2) { + asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (t[1][0])); + asm volatile ("vmovdqa %ymm12,%ymm13"); + asm volatile ("vpshufb %ymm6,%ymm12,%ymm12"); + asm volatile ("vpshufb %ymm7,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm12,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm13,%ymm3,%ymm3"); + + asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (t[1][16])); + asm volatile ("vmovdqa %ymm12,%ymm13"); + asm volatile ("vpshufb %ymm8,%ymm12,%ymm12"); + asm volatile ("vpshufb %ymm9,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm12,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm13,%ymm3,%ymm3"); + } + + /* syndrome 2 */ + if (nr >= 3) { + asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (t[2][0])); + asm volatile ("vmovdqa %ymm12,%ymm13"); + asm volatile ("vpshufb %ymm6,%ymm12,%ymm12"); + asm volatile ("vpshufb %ymm7,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm12,%ymm4,%ymm4"); + asm volatile ("vpxor %ymm13,%ymm5,%ymm5"); + + asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (t[2][16])); + asm volatile ("vmovdqa %ymm12,%ymm13"); + asm volatile ("vpshufb %ymm8,%ymm12,%ymm12"); + asm volatile ("vpshufb %ymm9,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm12,%ymm4,%ymm4"); + asm volatile ("vpxor %ymm13,%ymm5,%ymm5"); + } + } + + /* + * Split all completed syndromes. + * + * After this: + * + * syndrome 0: + * ymm0 / ymm6 first 32 bytes low/high + * ymm1 / ymm7 second 32 bytes low/high + * + * syndrome 1: + * ymm2 / ymm8 first 32 bytes low/high + * ymm3 / ymm9 second 32 bytes low/high + * + * syndrome 2: + * ymm4 / ymm10 first 32 bytes low/high + * ymm5 / ymm11 second 32 bytes low/high + */ + asm volatile ("vpsrlw $4,%ymm0,%ymm6"); + asm volatile ("vpsrlw $4,%ymm1,%ymm7"); + asm volatile ("vpand %ymm15,%ymm0,%ymm0"); + asm volatile ("vpand %ymm15,%ymm1,%ymm1"); + asm volatile ("vpand %ymm15,%ymm6,%ymm6"); + asm volatile ("vpand %ymm15,%ymm7,%ymm7"); + + if (nr >= 2) { + asm volatile ("vpsrlw $4,%ymm2,%ymm8"); + asm volatile ("vpsrlw $4,%ymm3,%ymm9"); + asm volatile ("vpand %ymm15,%ymm2,%ymm2"); + asm volatile ("vpand %ymm15,%ymm3,%ymm3"); + asm volatile ("vpand %ymm15,%ymm8,%ymm8"); + asm volatile ("vpand %ymm15,%ymm9,%ymm9"); + } + + if (nr >= 3) { + asm volatile ("vpsrlw $4,%ymm4,%ymm10"); + asm volatile ("vpsrlw $4,%ymm5,%ymm11"); + asm volatile ("vpand %ymm15,%ymm4,%ymm4"); + asm volatile ("vpand %ymm15,%ymm5,%ymm5"); + asm volatile ("vpand %ymm15,%ymm10,%ymm10"); + asm volatile ("vpand %ymm15,%ymm11,%ymm11"); + } + + /* + * Reconstruct every missing data block. + * + * ymm12/ymm13 are a shared multiplication table pair. + * ymm14/ymm15 accumulate the two 32-byte output lanes. + * + * At this point ymm15 is no longer needed as the nibble mask, + * so all 16 YMM registers are available. + */ + for (j = 0; j < nr; ++j) { + const uint8_t **t = R[j]; + + /* + * First coefficient. + * Initialize both output accumulators from the low + * nibble products. + */ + asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (t[0][0])); + asm volatile ("vmovdqa %ymm12,%ymm13"); + asm volatile ("vpshufb %ymm0,%ymm12,%ymm12"); + asm volatile ("vpshufb %ymm1,%ymm13,%ymm13"); + asm volatile ("vmovdqa %ymm12,%ymm14"); + asm volatile ("vmovdqa %ymm13,%ymm15"); + + asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (t[0][16])); + asm volatile ("vmovdqa %ymm12,%ymm13"); + asm volatile ("vpshufb %ymm6,%ymm12,%ymm12"); + asm volatile ("vpshufb %ymm7,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm12,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm13,%ymm15,%ymm15"); + + /* second coefficient */ + if (nr >= 2) { + asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (t[1][0])); + asm volatile ("vmovdqa %ymm12,%ymm13"); + asm volatile ("vpshufb %ymm2,%ymm12,%ymm12"); + asm volatile ("vpshufb %ymm3,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm12,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm13,%ymm15,%ymm15"); + + asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (t[1][16])); + asm volatile ("vmovdqa %ymm12,%ymm13"); + asm volatile ("vpshufb %ymm8,%ymm12,%ymm12"); + asm volatile ("vpshufb %ymm9,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm12,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm13,%ymm15,%ymm15"); + } + + /* third coefficient */ + if (nr >= 3) { + asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (t[2][0])); + asm volatile ("vmovdqa %ymm12,%ymm13"); + asm volatile ("vpshufb %ymm4,%ymm12,%ymm12"); + asm volatile ("vpshufb %ymm5,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm12,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm13,%ymm15,%ymm15"); + + asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (t[2][16])); + asm volatile ("vmovdqa %ymm12,%ymm13"); + asm volatile ("vpshufb %ymm10,%ymm12,%ymm12"); + asm volatile ("vpshufb %ymm11,%ymm13,%ymm13"); + asm volatile ("vpxor %ymm12,%ymm14,%ymm14"); + asm volatile ("vpxor %ymm13,%ymm15,%ymm15"); + } + + asm volatile ("vmovdqa %%ymm14,%0" : "=m" (pa[j][i])); + asm volatile ("vmovdqa %%ymm15,%0" : "=m" (pa[j][i + 32])); + } + } + + raid_avx_end(); +} + +void raid_rec1_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 1); + + /* if it's RAID5 uses the faster function */ + if (ip[0] == 0) { + raid_rec1of1(id, nd, size, vv); + return; + } + + raid_recX_avx2ext_123(1, id, ip, nd, size, vv); +} + +void raid_rec2_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 2); + raid_recX_avx2ext_123(2, id, ip, nd, size, vv); +} + +void raid_rec3_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 3); + raid_recX_avx2ext_123(3, id, ip, nd, size, vv); +} + +void raid_rec4_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 4); + raid_recX_avx2ext(4, id, ip, nd, size, vv); +} + +void raid_rec5_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 5); + raid_recX_avx2ext(5, id, ip, nd, size, vv); +} + +void raid_rec6_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 6); + raid_recX_avx2ext(6, id, ip, nd, size, vv); +} +#endif + void raid_register_avx2(void) { if (raid_cpu_has_avx2()) { @@ -1462,10 +2283,19 @@ void raid_register_avx2(void) raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "avx2", raid_rec1_avx2, RAID_POLY_ANY); raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "avx2", raid_rec2_avx2, RAID_POLY_ANY); - raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "avx2", raid_recX_avx2, RAID_POLY_ANY); - raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "avx2", raid_recX_avx2, RAID_POLY_ANY); - raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "avx2", raid_recX_avx2, RAID_POLY_ANY); - raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "avx2", raid_recX_avx2, RAID_POLY_ANY); + raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "avx2", raid_rec3_avx2, RAID_POLY_ANY); + raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "avx2", raid_rec4_avx2, RAID_POLY_ANY); + raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "avx2", raid_rec5_avx2, RAID_POLY_ANY); + raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "avx2", raid_rec6_avx2, RAID_POLY_ANY); +#ifdef CONFIG_X86_64 + raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "avx2e", raid_rec1_avx2ext, RAID_POLY_ANY); + raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "avx2e", raid_rec2_avx2ext, RAID_POLY_ANY); + raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "avx2e", raid_rec3_avx2ext, RAID_POLY_ANY); + raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "avx2e", raid_rec4_avx2ext, RAID_POLY_ANY); + raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "avx2e", raid_rec5_avx2ext, RAID_POLY_ANY); + raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "avx2e", raid_rec6_avx2ext, RAID_POLY_ANY); + +#endif } } #endif diff --git a/raid/avx512.c b/raid/avx512.c index ab7e5185..67d9c5c8 100644 --- a/raid/avx512.c +++ b/raid/avx512.c @@ -569,354 +569,356 @@ void raid_gen6_avx512bw(int nd, size_t size, void **vv) } /* - * RAID recovering for one disk AVX512BW implementation + * RAID recovering AVX512BW implementation. + * + * Compute only the selected syndromes directly from the stored parity + * and surviving data. This avoids raid_delta_gen(), temporary delta + * parity blocks, and generation of unused parity rows. + * + * AVX512 provides enough registers to keep all six supported syndromes + * in registers in a single surviving-data scan. */ -void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv) +static __always_inline void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; - uint8_t *p, *pa; - uint8_t G, V; + uint8_t *p[RAID_PARITY_MAX]; + uint8_t *pa[RAID_PARITY_MAX]; + uint8_t *src[RAID_DATA_MAX]; + uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; + const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX]; + const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX]; size_t i; + int d, j, k, s; + int ns; + int has_p; - (void)nr; + BUG_ON(nr < 1 || nr > RAID_PARITY_MAX); + /* setup the coefficients matrix */ + for (j = 0; j < nr; ++j) + for (k = 0; k < nr; ++k) + G[j * nr + k] = A(ip[j], id[k]); + + /* invert it to solve the system of linear equations */ + raid_invert(G, V, nr); + + /* setup selected parity and destination pointers */ + for (j = 0; j < nr; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + /* ip[] is ordered. If P is available, it is always ip[0] */ + has_p = ip[0] == 0; + + /* + * Build the compact list of surviving data blocks and precompute + * the multiplication-table pointers for each selected syndrome. + */ + ns = 0; + k = 0; + + for (d = 0; d < nd; ++d) { + if (k < nr && d == id[k]) { + ++k; + continue; + } + + src[ns] = v[d]; + + for (j = 0; j < nr; ++j) + S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0]; + + ++ns; + } + + BUG_ON(k != nr); + BUG_ON(ns != nd - nr); + + /* precompute inverse-matrix multiplication-table pointers */ + for (j = 0; j < nr; ++j) + for (k = 0; k < nr; ++k) + R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0]; + + raid_avx_begin(); + + /* + * Register allocation during syndrome generation: + * + * zmm0 syndrome 0 + * zmm2 syndrome 1 + * zmm4 syndrome 2 + * zmm6 syndrome 3 + * zmm8 syndrome 4 + * zmm10 syndrome 5 + * + * zmm12 source / source low nibble + * zmm13 source high nibble + * zmm14 low multiplication table / result + * zmm15 high multiplication table / result + * zmm31 low-nibble mask + * + * After syndrome splitting: + * + * zmm0 /zmm1 syndrome 0 low/high + * zmm2 /zmm3 syndrome 1 low/high + * zmm4 /zmm5 syndrome 2 low/high + * zmm6 /zmm7 syndrome 3 low/high + * zmm8 /zmm9 syndrome 4 low/high + * zmm10/zmm11 syndrome 5 low/high + */ + + asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gfconst16.low4[0])); + + for (i = 0; i < size; i += 64) { + /* + * Start every selected syndrome from its stored parity block. + */ + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[0][i])); + + if (nr >= 2) + asm volatile ("vmovdqa64 %0,%%zmm2" : : "m" (p[1][i])); + + if (nr >= 3) + asm volatile ("vmovdqa64 %0,%%zmm4" : : "m" (p[2][i])); + + if (nr >= 4) + asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (p[3][i])); + + if (nr >= 5) + asm volatile ("vmovdqa64 %0,%%zmm8" : : "m" (p[4][i])); + + if (nr >= 6) + asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (p[5][i])); + + /* + * Add all surviving data contributions in one source scan. + */ + for (s = 0; s < ns; ++s) { + const uint8_t **t = S[s]; + + asm volatile ("vmovdqa64 %0,%%zmm12" : : "m" (src[s][i])); + + /* + * P has coefficient 1 for every data disk. + * XOR the original source before splitting it. + */ + if (has_p) + asm volatile ("vpxorq %zmm12,%zmm0,%zmm0"); + + asm volatile ("vpsrlw $4,%zmm12,%zmm13"); + asm volatile ("vpandq %zmm31,%zmm12,%zmm12"); + asm volatile ("vpandq %zmm31,%zmm13,%zmm13"); + + /* syndrome 0 */ + if (!has_p) { + asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[0][16])); + asm volatile ("vpshufb %zmm12,%zmm14,%zmm14"); + asm volatile ("vpshufb %zmm13,%zmm15,%zmm15"); + asm volatile ("vpternlogq $0x96,%zmm14,%zmm15,%zmm0"); + } + + /* syndrome 1 */ + if (nr >= 2) { + asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[1][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[1][16])); + asm volatile ("vpshufb %zmm12,%zmm14,%zmm14"); + asm volatile ("vpshufb %zmm13,%zmm15,%zmm15"); + asm volatile ("vpternlogq $0x96,%zmm14,%zmm15,%zmm2"); + } + + /* syndrome 2 */ + if (nr >= 3) { + asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[2][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[2][16])); + asm volatile ("vpshufb %zmm12,%zmm14,%zmm14"); + asm volatile ("vpshufb %zmm13,%zmm15,%zmm15"); + asm volatile ("vpternlogq $0x96,%zmm14,%zmm15,%zmm4"); + } + + /* syndrome 3 */ + if (nr >= 4) { + asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[3][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[3][16])); + asm volatile ("vpshufb %zmm12,%zmm14,%zmm14"); + asm volatile ("vpshufb %zmm13,%zmm15,%zmm15"); + asm volatile ("vpternlogq $0x96,%zmm14,%zmm15,%zmm6"); + } + + /* syndrome 4 */ + if (nr >= 5) { + asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[4][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[4][16])); + asm volatile ("vpshufb %zmm12,%zmm14,%zmm14"); + asm volatile ("vpshufb %zmm13,%zmm15,%zmm15"); + asm volatile ("vpternlogq $0x96,%zmm14,%zmm15,%zmm8"); + } + + /* syndrome 5 */ + if (nr >= 6) { + asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[5][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[5][16])); + asm volatile ("vpshufb %zmm12,%zmm14,%zmm14"); + asm volatile ("vpshufb %zmm13,%zmm15,%zmm15"); + asm volatile ("vpternlogq $0x96,%zmm14,%zmm15,%zmm10"); + } + } + + /* + * Split every completed syndrome once into low/high nibbles. + */ + asm volatile ("vpsrlw $4,%zmm0,%zmm1"); + asm volatile ("vpandq %zmm31,%zmm0,%zmm0"); + asm volatile ("vpandq %zmm31,%zmm1,%zmm1"); + + if (nr >= 2) { + asm volatile ("vpsrlw $4,%zmm2,%zmm3"); + asm volatile ("vpandq %zmm31,%zmm2,%zmm2"); + asm volatile ("vpandq %zmm31,%zmm3,%zmm3"); + } + + if (nr >= 3) { + asm volatile ("vpsrlw $4,%zmm4,%zmm5"); + asm volatile ("vpandq %zmm31,%zmm4,%zmm4"); + asm volatile ("vpandq %zmm31,%zmm5,%zmm5"); + } + + if (nr >= 4) { + asm volatile ("vpsrlw $4,%zmm6,%zmm7"); + asm volatile ("vpandq %zmm31,%zmm6,%zmm6"); + asm volatile ("vpandq %zmm31,%zmm7,%zmm7"); + } + + if (nr >= 5) { + asm volatile ("vpsrlw $4,%zmm8,%zmm9"); + asm volatile ("vpandq %zmm31,%zmm8,%zmm8"); + asm volatile ("vpandq %zmm31,%zmm9,%zmm9"); + } + + if (nr >= 6) { + asm volatile ("vpsrlw $4,%zmm10,%zmm11"); + asm volatile ("vpandq %zmm31,%zmm10,%zmm10"); + asm volatile ("vpandq %zmm31,%zmm11,%zmm11"); + } + + /* + * Reconstruct every missing data block. + * + * Keep independent low/high dependency chains: + * + * zmm12 low accumulator + * zmm13 high accumulator + * zmm14 low multiplication table/result + * zmm15 high multiplication table/result + * + * This is intentional. Do not serialize low/high products + * through a single temporary register. + */ + for (j = 0; j < nr; ++j) { + const uint8_t **t = R[j]; + + /* coefficient 0 initializes both accumulators */ + asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (t[0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (t[0][16])); + asm volatile ("vpshufb %zmm0,%zmm12,%zmm12"); + asm volatile ("vpshufb %zmm1,%zmm13,%zmm13"); + + if (nr >= 2) { + asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[1][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[1][16])); + asm volatile ("vpshufb %zmm2,%zmm14,%zmm14"); + asm volatile ("vpshufb %zmm3,%zmm15,%zmm15"); + asm volatile ("vpxorq %zmm14,%zmm12,%zmm12"); + asm volatile ("vpxorq %zmm15,%zmm13,%zmm13"); + } + + if (nr >= 3) { + asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[2][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[2][16])); + asm volatile ("vpshufb %zmm4,%zmm14,%zmm14"); + asm volatile ("vpshufb %zmm5,%zmm15,%zmm15"); + asm volatile ("vpxorq %zmm14,%zmm12,%zmm12"); + asm volatile ("vpxorq %zmm15,%zmm13,%zmm13"); + } + + if (nr >= 4) { + asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[3][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[3][16])); + asm volatile ("vpshufb %zmm6,%zmm14,%zmm14"); + asm volatile ("vpshufb %zmm7,%zmm15,%zmm15"); + asm volatile ("vpxorq %zmm14,%zmm12,%zmm12"); + asm volatile ("vpxorq %zmm15,%zmm13,%zmm13"); + } + + if (nr >= 5) { + asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[4][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[4][16])); + asm volatile ("vpshufb %zmm8,%zmm14,%zmm14"); + asm volatile ("vpshufb %zmm9,%zmm15,%zmm15"); + asm volatile ("vpxorq %zmm14,%zmm12,%zmm12"); + asm volatile ("vpxorq %zmm15,%zmm13,%zmm13"); + } + + if (nr >= 6) { + asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[5][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[5][16])); + asm volatile ("vpshufb %zmm10,%zmm14,%zmm14"); + asm volatile ("vpshufb %zmm11,%zmm15,%zmm15"); + asm volatile ("vpxorq %zmm14,%zmm12,%zmm12"); + asm volatile ("vpxorq %zmm15,%zmm13,%zmm13"); + } + + asm volatile ("vpxorq %zmm13,%zmm12,%zmm12"); + asm volatile ("vmovdqa64 %%zmm12,%0" : "=m" (pa[j][i])); + } + } + + raid_avx_end(); +} + +void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 1); + + /* if it's RAID5 uses the faster function */ if (ip[0] == 0) { raid_rec1of1(id, nd, size, vv); return; } - G = A(ip[0], id[0]); - V = inv(G); - - raid_delta_gen(1, id, ip, nd, size, vv); - - p = v[nd + ip[0]]; - pa = v[id[0]]; - - raid_avx_begin(); - - asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfconst16.low4[0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm4" : : "m" (raid_gfmulpshufb[V][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm5" : : "m" (raid_gfmulpshufb[V][1][0])); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i])); - asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (pa[i])); - asm volatile ("vpxord %zmm1,%zmm0,%zmm0"); - asm volatile ("vpsrlw $4,%zmm0,%zmm1"); - asm volatile ("vpandd %zmm7,%zmm0,%zmm0"); - asm volatile ("vpandd %zmm7,%zmm1,%zmm1"); - asm volatile ("vpshufb %zmm0,%zmm4,%zmm2"); - asm volatile ("vpshufb %zmm1,%zmm5,%zmm3"); - asm volatile ("vpxord %zmm3,%zmm2,%zmm2"); - asm volatile ("vmovdqa64 %%zmm2, %0" : "=m" (pa[i])); - } - - raid_avx_end(); + raid_recX_avx512bw(1, id, ip, nd, size, vv); } -/* - * RAID recovering for two disks AVX512BW implementation - */ void raid_rec2_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv) { - uint8_t **v = (uint8_t **)vv; - const int N = 2; - uint8_t *p[N]; - uint8_t *pa[N]; - uint8_t G[N * N]; - uint8_t V[N * N]; - size_t i; - int j, k; - - (void)nr; /* unused, it's always 2 */ - - /* setup the coefficients matrix */ - for (j = 0; j < N; ++j) - for (k = 0; k < N; ++k) - G[j * N + k] = A(ip[j], id[k]); - - /* invert it to solve the system of linear equations */ - raid_invert(G, V, N); - - /* compute delta parity */ - raid_delta_gen(N, id, ip, nd, size, vv); - - for (j = 0; j < N; ++j) { - p[j] = v[nd + ip[j]]; - pa[j] = v[id[j]]; - } - - raid_avx_begin(); - - /* - * zmm7 = 0x0f nibble mask - * - * zmm16 = V[0] low table - * zmm17 = V[0] high table - * zmm18 = V[1] low table - * zmm19 = V[1] high table - * zmm20 = V[2] low table - * zmm21 = V[2] high table - * zmm22 = V[3] low table - * zmm23 = V[3] high table - */ - - asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfconst16.low4[0])); - - /* the inverse matrix V[] is constant for the whole recovery. */ - asm volatile ("vbroadcasti32x4 %0,%%zmm16" : : "m" (raid_gfmulpshufb[V[0]][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm17" : : "m" (raid_gfmulpshufb[V[0]][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm18" : : "m" (raid_gfmulpshufb[V[1]][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm19" : : "m" (raid_gfmulpshufb[V[1]][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm20" : : "m" (raid_gfmulpshufb[V[2]][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm21" : : "m" (raid_gfmulpshufb[V[2]][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm22" : : "m" (raid_gfmulpshufb[V[3]][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm23" : : "m" (raid_gfmulpshufb[V[3]][1][0])); - - for (i = 0; i < size; i += 64) { - /* d0 = p[0] ^ pa[0] */ - asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[0][i])); - asm volatile ("vmovdqa64 %0,%%zmm2" : : "m" (pa[0][i])); - asm volatile ("vpxord %zmm2,%zmm0,%zmm0"); - - /* d1 = p[1] ^ pa[1] */ - asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (p[1][i])); - asm volatile ("vmovdqa64 %0,%%zmm3" : : "m" (pa[1][i])); - asm volatile ("vpxord %zmm3,%zmm1,%zmm1"); - - /* - * Split both deltas into low/high nibbles once. - * - * zmm4 = d0 low - * zmm5 = d0 high - * zmm12 = d1 low - * zmm13 = d1 high - */ - asm volatile ("vpsrlw $4,%zmm0,%zmm5"); - asm volatile ("vpsrlw $4,%zmm1,%zmm13"); - asm volatile ("vpandd %zmm7,%zmm0,%zmm4"); - asm volatile ("vpandd %zmm7,%zmm5,%zmm5"); - asm volatile ("vpandd %zmm7,%zmm1,%zmm12"); - asm volatile ("vpandd %zmm7,%zmm13,%zmm13"); - - /* - * pa[0] = V[0] * d0 ^ V[1] * d1 - */ - asm volatile ("vpshufb %zmm4,%zmm16,%zmm2"); - asm volatile ("vpshufb %zmm5,%zmm17,%zmm3"); - asm volatile ("vpshufb %zmm12,%zmm18,%zmm10"); - asm volatile ("vpshufb %zmm13,%zmm19,%zmm11"); - - asm volatile ("vpxord %zmm3,%zmm2,%zmm2"); - asm volatile ("vpxord %zmm10,%zmm2,%zmm2"); - asm volatile ("vpxord %zmm11,%zmm2,%zmm2"); - - asm volatile ("vmovdqa64 %%zmm2,%0" : "=m" (pa[0][i])); - - /* - * pa[1] = V[2] * d0 ^ V[3] * d1 - * - * Reuse the already computed low/high nibbles. - */ - asm volatile ("vpshufb %zmm4,%zmm20,%zmm2"); - asm volatile ("vpshufb %zmm5,%zmm21,%zmm3"); - asm volatile ("vpshufb %zmm12,%zmm22,%zmm10"); - asm volatile ("vpshufb %zmm13,%zmm23,%zmm11"); - - asm volatile ("vpxord %zmm3,%zmm2,%zmm2"); - asm volatile ("vpxord %zmm10,%zmm2,%zmm2"); - asm volatile ("vpxord %zmm11,%zmm2,%zmm2"); - - asm volatile ("vmovdqa64 %%zmm2,%0" : "=m" (pa[1][i])); - } - - raid_avx_end(); + BUG_ON(nr != 2); + raid_recX_avx512bw(2, id, ip, nd, size, vv); } -/* - * RAID recovering AVX512BW implementation - */ -void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv) +void raid_rec3_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv) { - uint8_t **v = (uint8_t **)vv; - int N = nr; - uint8_t *p[RAID_PARITY_MAX]; - uint8_t *pa[RAID_PARITY_MAX]; - uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; - uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; - const uint8_t *T[RAID_PARITY_MAX * RAID_PARITY_MAX]; - size_t i; - int j, k; + BUG_ON(nr != 3); + raid_recX_avx512bw(3, id, ip, nd, size, vv); +} - /* setup the coefficients matrix */ - for (j = 0; j < N; ++j) - for (k = 0; k < N; ++k) - G[j * N + k] = A(ip[j], id[k]); +void raid_rec4_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 4); + raid_recX_avx512bw(4, id, ip, nd, size, vv); +} - /* invert it to solve the system of linear equations */ - raid_invert(G, V, N); +void raid_rec5_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 5); + raid_recX_avx512bw(5, id, ip, nd, size, vv); +} - /* precompute shuffle table pointers */ - for (j = 0; j < N * N; ++j) - T[j] = &raid_gfmulpshufb[V[j]][0][0]; - - /* compute delta parity */ - raid_delta_gen(N, id, ip, nd, size, vv); - - for (j = 0; j < N; ++j) { - p[j] = v[nd + ip[j]]; - pa[j] = v[id[j]]; - } - - raid_avx_begin(); - - asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gfconst16.low4[0])); - - for (i = 0; i < size; i += 64) { - /* delta */ - asm volatile ( - "movq 0(%2), %%rax\n" - "movq 0(%3), %%rbx\n" - "vmovdqa64 (%%rax, %1), %%zmm12\n" - "vmovdqa64 (%%rbx, %1), %%zmm13\n" - "vpxorq %%zmm13, %%zmm12, %%zmm12\n" - "vpsrlw $4, %%zmm12, %%zmm1\n" - "vpandq %%zmm31, %%zmm12, %%zmm0\n" - "vpandq %%zmm31, %%zmm1, %%zmm1\n" - "cmpq $1, %0\n" - "jbe 1f\n" - - "movq 8(%2), %%rax\n" - "movq 8(%3), %%rbx\n" - "vmovdqa64 (%%rax, %1), %%zmm12\n" - "vmovdqa64 (%%rbx, %1), %%zmm13\n" - "vpxorq %%zmm13, %%zmm12, %%zmm12\n" - "vpsrlw $4, %%zmm12, %%zmm3\n" - "vpandq %%zmm31, %%zmm12, %%zmm2\n" - "vpandq %%zmm31, %%zmm3, %%zmm3\n" - "cmpq $2, %0\n" - "jbe 1f\n" - - "movq 16(%2), %%rax\n" - "movq 16(%3), %%rbx\n" - "vmovdqa64 (%%rax, %1), %%zmm12\n" - "vmovdqa64 (%%rbx, %1), %%zmm13\n" - "vpxorq %%zmm13, %%zmm12, %%zmm12\n" - "vpsrlw $4, %%zmm12, %%zmm5\n" - "vpandq %%zmm31, %%zmm12, %%zmm4\n" - "vpandq %%zmm31, %%zmm5, %%zmm5\n" - "cmpq $3, %0\n" - "jbe 1f\n" - - "movq 24(%2), %%rax\n" - "movq 24(%3), %%rbx\n" - "vmovdqa64 (%%rax, %1), %%zmm12\n" - "vmovdqa64 (%%rbx, %1), %%zmm13\n" - "vpxorq %%zmm13, %%zmm12, %%zmm12\n" - "vpsrlw $4, %%zmm12, %%zmm7\n" - "vpandq %%zmm31, %%zmm12, %%zmm6\n" - "vpandq %%zmm31, %%zmm7, %%zmm7\n" - "cmpq $4, %0\n" - "jbe 1f\n" - - "movq 32(%2), %%rax\n" - "movq 32(%3), %%rbx\n" - "vmovdqa64 (%%rax, %1), %%zmm12\n" - "vmovdqa64 (%%rbx, %1), %%zmm13\n" - "vpxorq %%zmm13, %%zmm12, %%zmm12\n" - "vpsrlw $4, %%zmm12, %%zmm9\n" - "vpandq %%zmm31, %%zmm12, %%zmm8\n" - "vpandq %%zmm31, %%zmm9, %%zmm9\n" - "cmpq $5, %0\n" - "jbe 1f\n" - - "movq 40(%2), %%rax\n" - "movq 40(%3), %%rbx\n" - "vmovdqa64 (%%rax, %1), %%zmm12\n" - "vmovdqa64 (%%rbx, %1), %%zmm13\n" - "vpxorq %%zmm13, %%zmm12, %%zmm12\n" - "vpsrlw $4, %%zmm12, %%zmm11\n" - "vpandq %%zmm31, %%zmm12, %%zmm10\n" - "vpandq %%zmm31, %%zmm11, %%zmm11\n" - - "1:\n" - : - : "r" ((uint64_t)N), "r" (i), "r" (p), "r" (pa) - : "rax", "rbx", "cc", "memory" - ); - - /* reconstruct */ - for (j = 0; j < N; ++j) { - asm volatile ( - "movq 0(%2), %%rcx\n" - "vbroadcasti32x4 0(%%rcx), %%zmm14\n" - "vbroadcasti32x4 16(%%rcx), %%zmm15\n" - "vpshufb %%zmm0, %%zmm14, %%zmm14\n" - "vpshufb %%zmm1, %%zmm15, %%zmm15\n" - "cmpq $1, %0\n" - "jbe 1f\n" - - "movq 8(%2), %%rcx\n" - "vbroadcasti32x4 0(%%rcx), %%zmm12\n" - "vbroadcasti32x4 16(%%rcx), %%zmm13\n" - "vpshufb %%zmm2, %%zmm12, %%zmm12\n" - "vpshufb %%zmm3, %%zmm13, %%zmm13\n" - "vpxorq %%zmm12, %%zmm14, %%zmm14\n" - "vpxorq %%zmm13, %%zmm15, %%zmm15\n" - "cmpq $2, %0\n" - "jbe 1f\n" - - "movq 16(%2), %%rcx\n" - "vbroadcasti32x4 0(%%rcx), %%zmm12\n" - "vbroadcasti32x4 16(%%rcx), %%zmm13\n" - "vpshufb %%zmm4, %%zmm12, %%zmm12\n" - "vpshufb %%zmm5, %%zmm13, %%zmm13\n" - "vpxorq %%zmm12, %%zmm14, %%zmm14\n" - "vpxorq %%zmm13, %%zmm15, %%zmm15\n" - "cmpq $3, %0\n" - "jbe 1f\n" - - "movq 24(%2), %%rcx\n" - "vbroadcasti32x4 0(%%rcx), %%zmm12\n" - "vbroadcasti32x4 16(%%rcx), %%zmm13\n" - "vpshufb %%zmm6, %%zmm12, %%zmm12\n" - "vpshufb %%zmm7, %%zmm13, %%zmm13\n" - "vpxorq %%zmm12, %%zmm14, %%zmm14\n" - "vpxorq %%zmm13, %%zmm15, %%zmm15\n" - "cmpq $4, %0\n" - "jbe 1f\n" - - "movq 32(%2), %%rcx\n" - "vbroadcasti32x4 0(%%rcx), %%zmm12\n" - "vbroadcasti32x4 16(%%rcx), %%zmm13\n" - "vpshufb %%zmm8, %%zmm12, %%zmm12\n" - "vpshufb %%zmm9, %%zmm13, %%zmm13\n" - "vpxorq %%zmm12, %%zmm14, %%zmm14\n" - "vpxorq %%zmm13, %%zmm15, %%zmm15\n" - "cmpq $5, %0\n" - "jbe 1f\n" - - "movq 40(%2), %%rcx\n" - "vbroadcasti32x4 0(%%rcx), %%zmm12\n" - "vbroadcasti32x4 16(%%rcx), %%zmm13\n" - "vpshufb %%zmm10, %%zmm12, %%zmm12\n" - "vpshufb %%zmm11, %%zmm13, %%zmm13\n" - "vpxorq %%zmm12, %%zmm14, %%zmm14\n" - "vpxorq %%zmm13, %%zmm15, %%zmm15\n" - - "1:\n" - "vpxorq %%zmm15, %%zmm14, %%zmm14\n" - "movq %3, %%rax\n" - "vmovdqa64 %%zmm14, (%%rax, %1)\n" - : - : "r" ((uint64_t)N), "r" (i), "r" (&T[j * N]), "r" (pa[j]) - : "rax", "rcx", "cc", "memory" - ); - } - } - - raid_avx_end(); +void raid_rec6_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 6); + raid_recX_avx512bw(6, id, ip, nd, size, vv); } void raid_register_avx512(void) @@ -932,10 +934,10 @@ void raid_register_avx512(void) raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "avx512", raid_rec1_avx512bw, RAID_POLY_ANY); raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "avx512", raid_rec2_avx512bw, RAID_POLY_ANY); - raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "avx512", raid_recX_avx512bw, RAID_POLY_ANY); - raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "avx512", raid_recX_avx512bw, RAID_POLY_ANY); - raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "avx512", raid_recX_avx512bw, RAID_POLY_ANY); - raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "avx512", raid_recX_avx512bw, RAID_POLY_ANY); + raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "avx512", raid_rec3_avx512bw, RAID_POLY_ANY); + raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "avx512", raid_rec4_avx512bw, RAID_POLY_ANY); + raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "avx512", raid_rec5_avx512bw, RAID_POLY_ANY); + raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "avx512", raid_rec6_avx512bw, RAID_POLY_ANY); } } } diff --git a/raid/gfni.c b/raid/gfni.c index 18120646..6a4616bd 100644 --- a/raid/gfni.c +++ b/raid/gfni.c @@ -561,1078 +561,866 @@ void raid_gen6_avx512gfni_aes(int nd, size_t size, void **vv) raid_genX_avx512gfni_aes(nd, size, vv, 6); } -/* - * RAID recovering for one disk AVX2 GFNI implementation - */ +static __always_inline void raid_recX_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p[RAID_PARITY_MAX]; + uint8_t *pa[RAID_PARITY_MAX]; + uint8_t *src[RAID_DATA_MAX]; + uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; + const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX]; + const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX]; + size_t i; + int d, j, k, s; + int ns; + int has_p; + + BUG_ON(nr < 1 || nr > RAID_PARITY_MAX); + + for (j = 0; j < nr; ++j) + for (k = 0; k < nr; ++k) + G[j * nr + k] = A(ip[j], id[k]); + + raid_invert(G, V, nr); + + for (j = 0; j < nr; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + has_p = ip[0] == 0; + + ns = 0; + k = 0; + + for (d = 0; d < nd; ++d) { + if (k < nr && d == id[k]) { + ++k; + continue; + } + + src[ns] = v[d]; + + for (j = 0; j < nr; ++j) + S[ns][j] = raid_gfaffine_raid[A(ip[j], d)]; + + ++ns; + } + + BUG_ON(k != nr); + BUG_ON(ns != nd - nr); + + for (j = 0; j < nr; ++j) + for (k = 0; k < nr; ++k) + R[j][k] = raid_gfaffine_raid[V[j * nr + k]]; + + raid_avx_begin(); + + for (i = 0; i < size; i += 64) { + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i])); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[0][i + 32])); + + if (nr >= 2) { + asm volatile ("vmovdqa %0,%%ymm2" : : "m" (p[1][i])); + asm volatile ("vmovdqa %0,%%ymm3" : : "m" (p[1][i + 32])); + } + + if (nr >= 3) { + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (p[2][i])); + asm volatile ("vmovdqa %0,%%ymm5" : : "m" (p[2][i + 32])); + } + + if (nr >= 4) { + asm volatile ("vmovdqa %0,%%ymm6" : : "m" (p[3][i])); + asm volatile ("vmovdqa %0,%%ymm7" : : "m" (p[3][i + 32])); + } + + if (nr >= 5) { + asm volatile ("vmovdqa %0,%%ymm8" : : "m" (p[4][i])); + asm volatile ("vmovdqa %0,%%ymm9" : : "m" (p[4][i + 32])); + } + + if (nr >= 6) { + asm volatile ("vmovdqa %0,%%ymm10" : : "m" (p[5][i])); + asm volatile ("vmovdqa %0,%%ymm11" : : "m" (p[5][i + 32])); + } + + for (s = 0; s < ns; ++s) { + const uint8_t **t = S[s]; + + asm volatile ("vmovdqa %0,%%ymm12" : : "m" (src[s][i])); + asm volatile ("vmovdqa %0,%%ymm13" : : "m" (src[s][i + 32])); + + if (has_p) { + asm volatile ("vpxor %ymm12,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm13,%ymm1,%ymm1"); + } else { + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[0][0])); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm0,%ymm0"); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm1,%ymm1"); + } + + if (nr >= 2) { + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[1][0])); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm2,%ymm2"); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm3,%ymm3"); + } + + if (nr >= 3) { + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[2][0])); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm4,%ymm4"); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm5,%ymm5"); + } + + if (nr >= 4) { + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[3][0])); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm6,%ymm6"); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm7,%ymm7"); + } + + if (nr >= 5) { + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[4][0])); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm8,%ymm8"); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm9,%ymm9"); + } + + if (nr >= 6) { + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[5][0])); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm10,%ymm10"); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm11,%ymm11"); + } + } + + for (j = 0; j < nr; ++j) { + const uint8_t **t = R[j]; + + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[0][0])); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm0,%ymm12"); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm1,%ymm13"); + + if (nr >= 2) { + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[1][0])); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm2,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm12,%ymm12"); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm3,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm13,%ymm13"); + } + + if (nr >= 3) { + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[2][0])); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm4,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm12,%ymm12"); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm5,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm13,%ymm13"); + } + + if (nr >= 4) { + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[3][0])); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm6,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm12,%ymm12"); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm7,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm13,%ymm13"); + } + + if (nr >= 5) { + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[4][0])); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm8,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm12,%ymm12"); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm9,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm13,%ymm13"); + } + + if (nr >= 6) { + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[5][0])); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm10,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm12,%ymm12"); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm11,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm13,%ymm13"); + } + + asm volatile ("vmovdqa %%ymm12,%0" : "=m" (pa[j][i])); + asm volatile ("vmovdqa %%ymm13,%0" : "=m" (pa[j][i + 32])); + } + } + + raid_avx_end(); +} + void raid_rec1_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) { - uint8_t **v = (uint8_t **)vv; - uint8_t *p, *pa; - uint8_t G, V; - size_t i; - - (void)nr; + BUG_ON(nr != 1); + /* if it's RAID5 uses the faster function */ if (ip[0] == 0) { raid_rec1of1(id, nd, size, vv); return; } - G = A(ip[0], id[0]); - V = inv(G); - - raid_delta_gen(1, id, ip, nd, size, vv); - - p = v[nd + ip[0]]; - pa = v[id[0]]; - - raid_avx_begin(); - - asm volatile ("vpbroadcastq %0,%%ymm4" : : "m" (raid_gfaffine_raid[V][0])); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[i])); - asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[i + 32])); - - asm volatile ("vmovdqa %0,%%ymm2" : : "m" (pa[i])); - asm volatile ("vmovdqa %0,%%ymm3" : : "m" (pa[i + 32])); - - asm volatile ("vpxor %ymm2,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm3,%ymm1,%ymm1"); - - asm volatile ("vgf2p8affineqb $0,%ymm4,%ymm0,%ymm0"); - asm volatile ("vgf2p8affineqb $0,%ymm4,%ymm1,%ymm1"); - - asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[i])); - asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[i + 32])); - } - - raid_avx_end(); + raid_recX_avx2gfni_raid(1, id, ip, nd, size, vv); } -/* - * RAID recovering for one disk GFNI implementation - */ -void raid_rec1_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p, *pa; - uint8_t G, V; - size_t i; - - (void)nr; - - if (ip[0] == 0) { - raid_rec1of1(id, nd, size, vv); - return; - } - - G = A(ip[0], id[0]); - V = inv(G); - - raid_delta_gen(1, id, ip, nd, size, vv); - - p = v[nd + ip[0]]; - pa = v[id[0]]; - - raid_avx_begin(); - - asm volatile ("vpbroadcastq %0,%%zmm4" : : "m" (raid_gfaffine_raid[V][0])); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i])); - asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (pa[i])); - asm volatile ("vpxorq %zmm1,%zmm0,%zmm0"); - asm volatile ("vgf2p8affineqb $0,%zmm4,%zmm0,%zmm0"); - asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[i])); - } - - raid_avx_end(); -} - -/* - * RAID recovering for two disks AVX2 GFNI implementation - */ void raid_rec2_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 2); + raid_recX_avx2gfni_raid(2, id, ip, nd, size, vv); +} + +void raid_rec3_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 3); + raid_recX_avx2gfni_raid(3, id, ip, nd, size, vv); +} + +void raid_rec4_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 4); + raid_recX_avx2gfni_raid(4, id, ip, nd, size, vv); +} + +void raid_rec5_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 5); + raid_recX_avx2gfni_raid(5, id, ip, nd, size, vv); +} + +void raid_rec6_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 6); + raid_recX_avx2gfni_raid(6, id, ip, nd, size, vv); +} + +static __always_inline void raid_recX_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; - const int N = 2; - uint8_t *p[N], *pa[N]; - uint8_t G[N * N], V[N * N]; + uint8_t *p[RAID_PARITY_MAX]; + uint8_t *pa[RAID_PARITY_MAX]; + uint8_t *src[RAID_DATA_MAX]; + uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; + const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX]; + const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX]; size_t i; - int j, k; + int d, j, k, s; + int ns; + int has_p; - (void)nr; + BUG_ON(nr < 1 || nr > RAID_PARITY_MAX); - for (j = 0; j < N; ++j) - for (k = 0; k < N; ++k) - G[j * N + k] = A(ip[j], id[k]); + for (j = 0; j < nr; ++j) + for (k = 0; k < nr; ++k) + G[j * nr + k] = A(ip[j], id[k]); - raid_invert(G, V, N); - raid_delta_gen(N, id, ip, nd, size, vv); + raid_invert(G, V, nr); - for (j = 0; j < N; ++j) { + for (j = 0; j < nr; ++j) { p[j] = v[nd + ip[j]]; pa[j] = v[id[j]]; } + has_p = ip[0] == 0; + + ns = 0; + k = 0; + + for (d = 0; d < nd; ++d) { + if (k < nr && d == id[k]) { + ++k; + continue; + } + + src[ns] = v[d]; + + for (j = 0; j < nr; ++j) + S[ns][j] = raid_gfaffine_raid[A(ip[j], d)]; + + ++ns; + } + + BUG_ON(k != nr); + BUG_ON(ns != nd - nr); + + for (j = 0; j < nr; ++j) + for (k = 0; k < nr; ++k) + R[j][k] = raid_gfaffine_raid[V[j * nr + k]]; + raid_avx_begin(); - asm volatile ("vpbroadcastq %0,%%ymm10" : : "m" (raid_gfaffine_raid[V[0]][0])); - asm volatile ("vpbroadcastq %0,%%ymm11" : : "m" (raid_gfaffine_raid[V[1]][0])); - asm volatile ("vpbroadcastq %0,%%ymm12" : : "m" (raid_gfaffine_raid[V[2]][0])); - asm volatile ("vpbroadcastq %0,%%ymm13" : : "m" (raid_gfaffine_raid[V[3]][0])); for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i])); - asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[0][i + 32])); + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[0][i])); + if (nr >= 2) + asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (p[1][i])); + if (nr >= 3) + asm volatile ("vmovdqa64 %0,%%zmm2" : : "m" (p[2][i])); + if (nr >= 4) + asm volatile ("vmovdqa64 %0,%%zmm3" : : "m" (p[3][i])); + if (nr >= 5) + asm volatile ("vmovdqa64 %0,%%zmm4" : : "m" (p[4][i])); + if (nr >= 6) + asm volatile ("vmovdqa64 %0,%%zmm5" : : "m" (p[5][i])); - asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pa[0][i])); - asm volatile ("vmovdqa %0,%%ymm5" : : "m" (pa[0][i + 32])); + for (s = 0; s < ns; ++s) { + const uint8_t **t = S[s]; - asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); + asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (src[s][i])); - asm volatile ("vmovdqa %0,%%ymm2" : : "m" (p[1][i])); - asm volatile ("vmovdqa %0,%%ymm3" : : "m" (p[1][i + 32])); + if (has_p) { + asm volatile ("vpxorq %zmm6,%zmm0,%zmm0"); + } else { + asm volatile ("vpbroadcastq %0,%%zmm7" : : "m" (t[0][0])); + asm volatile ("vgf2p8affineqb $0,%zmm7,%zmm6,%zmm8"); + asm volatile ("vpxorq %zmm8,%zmm0,%zmm0"); + } - asm volatile ("vmovdqa %0,%%ymm6" : : "m" (pa[1][i])); - asm volatile ("vmovdqa %0,%%ymm7" : : "m" (pa[1][i + 32])); + if (nr >= 2) { + asm volatile ("vpbroadcastq %0,%%zmm7" : : "m" (t[1][0])); + asm volatile ("vgf2p8affineqb $0,%zmm7,%zmm6,%zmm8"); + asm volatile ("vpxorq %zmm8,%zmm1,%zmm1"); + } - asm volatile ("vpxor %ymm6,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm7,%ymm3,%ymm3"); + if (nr >= 3) { + asm volatile ("vpbroadcastq %0,%%zmm7" : : "m" (t[2][0])); + asm volatile ("vgf2p8affineqb $0,%zmm7,%zmm6,%zmm8"); + asm volatile ("vpxorq %zmm8,%zmm2,%zmm2"); + } - asm volatile ("vgf2p8affineqb $0,%ymm10,%ymm0,%ymm4"); - asm volatile ("vgf2p8affineqb $0,%ymm10,%ymm1,%ymm5"); + if (nr >= 4) { + asm volatile ("vpbroadcastq %0,%%zmm7" : : "m" (t[3][0])); + asm volatile ("vgf2p8affineqb $0,%zmm7,%zmm6,%zmm8"); + asm volatile ("vpxorq %zmm8,%zmm3,%zmm3"); + } - asm volatile ("vgf2p8affineqb $0,%ymm11,%ymm2,%ymm9"); - asm volatile ("vpxor %ymm9,%ymm4,%ymm4"); - asm volatile ("vgf2p8affineqb $0,%ymm11,%ymm3,%ymm9"); - asm volatile ("vpxor %ymm9,%ymm5,%ymm5"); + if (nr >= 5) { + asm volatile ("vpbroadcastq %0,%%zmm7" : : "m" (t[4][0])); + asm volatile ("vgf2p8affineqb $0,%zmm7,%zmm6,%zmm8"); + asm volatile ("vpxorq %zmm8,%zmm4,%zmm4"); + } - asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pa[0][i])); - asm volatile ("vmovdqa %%ymm5,%0" : "=m" (pa[0][i + 32])); + if (nr >= 6) { + asm volatile ("vpbroadcastq %0,%%zmm7" : : "m" (t[5][0])); + asm volatile ("vgf2p8affineqb $0,%zmm7,%zmm6,%zmm8"); + asm volatile ("vpxorq %zmm8,%zmm5,%zmm5"); + } + } - asm volatile ("vgf2p8affineqb $0,%ymm12,%ymm0,%ymm6"); - asm volatile ("vgf2p8affineqb $0,%ymm12,%ymm1,%ymm7"); + for (j = 0; j < nr; ++j) { + const uint8_t **t = R[j]; - asm volatile ("vgf2p8affineqb $0,%ymm13,%ymm2,%ymm9"); - asm volatile ("vpxor %ymm9,%ymm6,%ymm6"); - asm volatile ("vgf2p8affineqb $0,%ymm13,%ymm3,%ymm9"); - asm volatile ("vpxor %ymm9,%ymm7,%ymm7"); + asm volatile ("vpbroadcastq %0,%%zmm6" : : "m" (t[0][0])); + asm volatile ("vgf2p8affineqb $0,%zmm6,%zmm0,%zmm7"); - asm volatile ("vmovdqa %%ymm6,%0" : "=m" (pa[1][i])); - asm volatile ("vmovdqa %%ymm7,%0" : "=m" (pa[1][i + 32])); + if (nr >= 2) { + asm volatile ("vpbroadcastq %0,%%zmm6" : : "m" (t[1][0])); + asm volatile ("vgf2p8affineqb $0,%zmm6,%zmm1,%zmm8"); + asm volatile ("vpxorq %zmm8,%zmm7,%zmm7"); + } + + if (nr >= 3) { + asm volatile ("vpbroadcastq %0,%%zmm6" : : "m" (t[2][0])); + asm volatile ("vgf2p8affineqb $0,%zmm6,%zmm2,%zmm8"); + asm volatile ("vpxorq %zmm8,%zmm7,%zmm7"); + } + + if (nr >= 4) { + asm volatile ("vpbroadcastq %0,%%zmm6" : : "m" (t[3][0])); + asm volatile ("vgf2p8affineqb $0,%zmm6,%zmm3,%zmm8"); + asm volatile ("vpxorq %zmm8,%zmm7,%zmm7"); + } + + if (nr >= 5) { + asm volatile ("vpbroadcastq %0,%%zmm6" : : "m" (t[4][0])); + asm volatile ("vgf2p8affineqb $0,%zmm6,%zmm4,%zmm8"); + asm volatile ("vpxorq %zmm8,%zmm7,%zmm7"); + } + + if (nr >= 6) { + asm volatile ("vpbroadcastq %0,%%zmm6" : : "m" (t[5][0])); + asm volatile ("vgf2p8affineqb $0,%zmm6,%zmm5,%zmm8"); + asm volatile ("vpxorq %zmm8,%zmm7,%zmm7"); + } + + asm volatile ("vmovdqa64 %%zmm7,%0" : "=m" (pa[j][i])); + } } raid_avx_end(); } -/* - * RAID recovering for two disks GFNI implementation - */ +void raid_rec1_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 1); + + /* if it's RAID5 uses the faster function */ + if (ip[0] == 0) { + raid_rec1of1(id, nd, size, vv); + return; + } + + raid_recX_avx512gfni_raid(1, id, ip, nd, size, vv); +} + void raid_rec2_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) { - uint8_t **v = (uint8_t **)vv; - const int N = 2; - uint8_t *p[N], *pa[N]; - uint8_t G[N * N], V[N * N]; - size_t i; - int j, k; - - (void)nr; - - for (j = 0; j < N; ++j) - for (k = 0; k < N; ++k) - G[j * N + k] = A(ip[j], id[k]); - - raid_invert(G, V, N); - raid_delta_gen(N, id, ip, nd, size, vv); - - for (j = 0; j < N; ++j) { - p[j] = v[nd + ip[j]]; - pa[j] = v[id[j]]; - } - - raid_avx_begin(); - asm volatile ("vpbroadcastq %0,%%zmm8" : : "m" (raid_gfaffine_raid[V[0]][0])); - asm volatile ("vpbroadcastq %0,%%zmm9" : : "m" (raid_gfaffine_raid[V[1]][0])); - asm volatile ("vpbroadcastq %0,%%zmm10" : : "m" (raid_gfaffine_raid[V[2]][0])); - asm volatile ("vpbroadcastq %0,%%zmm11" : : "m" (raid_gfaffine_raid[V[3]][0])); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[0][i])); - asm volatile ("vmovdqa64 %0,%%zmm2" : : "m" (pa[0][i])); - asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (p[1][i])); - asm volatile ("vmovdqa64 %0,%%zmm3" : : "m" (pa[1][i])); - asm volatile ("vpxorq %zmm2,%zmm0,%zmm0"); - asm volatile ("vpxorq %zmm3,%zmm1,%zmm1"); - - asm volatile ("vpxorq %zmm6,%zmm6,%zmm6"); - - asm volatile ("vgf2p8affineqb $0,%zmm8,%zmm0,%zmm2"); - asm volatile ("vpxorq %zmm2,%zmm6,%zmm6"); - - asm volatile ("vgf2p8affineqb $0,%zmm9,%zmm1,%zmm3"); - asm volatile ("vpxorq %zmm3,%zmm6,%zmm6"); - - asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (pa[0][i])); - - asm volatile ("vpxorq %zmm6,%zmm6,%zmm6"); - - asm volatile ("vgf2p8affineqb $0,%zmm10,%zmm0,%zmm2"); - asm volatile ("vpxorq %zmm2,%zmm6,%zmm6"); - - asm volatile ("vgf2p8affineqb $0,%zmm11,%zmm1,%zmm3"); - asm volatile ("vpxorq %zmm3,%zmm6,%zmm6"); - - asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (pa[1][i])); - } - - raid_avx_end(); + BUG_ON(nr != 2); + raid_recX_avx512gfni_raid(2, id, ip, nd, size, vv); } -/* - * RAID recovering AVX2 GFNI implementation - */ -void raid_recX_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) +void raid_rec3_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 3); + raid_recX_avx512gfni_raid(3, id, ip, nd, size, vv); +} + +void raid_rec4_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 4); + raid_recX_avx512gfni_raid(4, id, ip, nd, size, vv); +} + +void raid_rec5_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 5); + raid_recX_avx512gfni_raid(5, id, ip, nd, size, vv); +} + +void raid_rec6_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 6); + raid_recX_avx512gfni_raid(6, id, ip, nd, size, vv); +} + +static __always_inline void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; - int N = nr; uint8_t *p[RAID_PARITY_MAX]; uint8_t *pa[RAID_PARITY_MAX]; + uint8_t *src[RAID_DATA_MAX]; uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; - uint8_t C[RAID_PARITY_MAX * RAID_PARITY_MAX][8]; + uint8_t S[RAID_DATA_MAX][RAID_PARITY_MAX]; size_t i; - int j, k; + int d, j, k, s; + int ns; + int has_p; - /* setup the coefficients matrix */ - for (j = 0; j < N; ++j) - for (k = 0; k < N; ++k) - G[j * N + k] = A(ip[j], id[k]); + BUG_ON(nr < 1 || nr > RAID_PARITY_MAX); - /* invert it to solve the system of linear equations */ - raid_invert(G, V, N); + for (j = 0; j < nr; ++j) + for (k = 0; k < nr; ++k) + G[j * nr + k] = A(ip[j], id[k]); - /* precompute affine representations of the coefficients */ - for (j = 0; j < N * N; ++j) - memcpy(C[j], raid_gfaffine_raid[V[j]], 8); + raid_invert(G, V, nr); - /* compute delta parity */ - raid_delta_gen(N, id, ip, nd, size, vv); - - for (j = 0; j < N; ++j) { + for (j = 0; j < nr; ++j) { p[j] = v[nd + ip[j]]; pa[j] = v[id[j]]; } - raid_avx_begin(); + has_p = ip[0] == 0; - for (i = 0; i < size; i += 64) { - /* delta */ - asm volatile ( - "movq 0(%2), %%rax\n" - "movq 0(%3), %%rbx\n" - "vmovdqa 0(%%rax, %1), %%ymm0\n" - "vmovdqa 32(%%rax, %1), %%ymm1\n" - "vmovdqa 0(%%rbx, %1), %%ymm14\n" - "vmovdqa 32(%%rbx, %1), %%ymm15\n" - "vpxor %%ymm14, %%ymm0, %%ymm0\n" - "vpxor %%ymm15, %%ymm1, %%ymm1\n" - "cmpq $1, %0\n" - "jbe 1f\n" + ns = 0; + k = 0; - "movq 8(%2), %%rax\n" - "movq 8(%3), %%rbx\n" - "vmovdqa 0(%%rax, %1), %%ymm2\n" - "vmovdqa 32(%%rax, %1), %%ymm3\n" - "vmovdqa 0(%%rbx, %1), %%ymm14\n" - "vmovdqa 32(%%rbx, %1), %%ymm15\n" - "vpxor %%ymm14, %%ymm2, %%ymm2\n" - "vpxor %%ymm15, %%ymm3, %%ymm3\n" - "cmpq $2, %0\n" - "jbe 1f\n" - - "movq 16(%2), %%rax\n" - "movq 16(%3), %%rbx\n" - "vmovdqa 0(%%rax, %1), %%ymm4\n" - "vmovdqa 32(%%rax, %1), %%ymm5\n" - "vmovdqa 0(%%rbx, %1), %%ymm14\n" - "vmovdqa 32(%%rbx, %1), %%ymm15\n" - "vpxor %%ymm14, %%ymm4, %%ymm4\n" - "vpxor %%ymm15, %%ymm5, %%ymm5\n" - "cmpq $3, %0\n" - "jbe 1f\n" - - "movq 24(%2), %%rax\n" - "movq 24(%3), %%rbx\n" - "vmovdqa 0(%%rax, %1), %%ymm6\n" - "vmovdqa 32(%%rax, %1), %%ymm7\n" - "vmovdqa 0(%%rbx, %1), %%ymm14\n" - "vmovdqa 32(%%rbx, %1), %%ymm15\n" - "vpxor %%ymm14, %%ymm6, %%ymm6\n" - "vpxor %%ymm15, %%ymm7, %%ymm7\n" - "cmpq $4, %0\n" - "jbe 1f\n" - - "movq 32(%2), %%rax\n" - "movq 32(%3), %%rbx\n" - "vmovdqa 0(%%rax, %1), %%ymm8\n" - "vmovdqa 32(%%rax, %1), %%ymm9\n" - "vmovdqa 0(%%rbx, %1), %%ymm14\n" - "vmovdqa 32(%%rbx, %1), %%ymm15\n" - "vpxor %%ymm14, %%ymm8, %%ymm8\n" - "vpxor %%ymm15, %%ymm9, %%ymm9\n" - "cmpq $5, %0\n" - "jbe 1f\n" - - "movq 40(%2), %%rax\n" - "movq 40(%3), %%rbx\n" - "vmovdqa 0(%%rax, %1), %%ymm10\n" - "vmovdqa 32(%%rax, %1), %%ymm11\n" - "vmovdqa 0(%%rbx, %1), %%ymm14\n" - "vmovdqa 32(%%rbx, %1), %%ymm15\n" - "vpxor %%ymm14, %%ymm10, %%ymm10\n" - "vpxor %%ymm15, %%ymm11, %%ymm11\n" - - "1:\n" - : - : "r" ((uint64_t)N), "r" (i), "r" (p), "r" (pa) - : "rax", "rbx", "cc", "memory" - ); - - /* reconstruct */ - for (j = 0; j < N; ++j) { - asm volatile ( - "vpbroadcastq 0(%2), %%ymm14\n" - "vgf2p8affineqb $0, %%ymm14, %%ymm0, %%ymm12\n" - "vgf2p8affineqb $0, %%ymm14, %%ymm1, %%ymm13\n" - "cmpq $1, %0\n" - "jbe 1f\n" - - "vpbroadcastq 8(%2), %%ymm14\n" - "vgf2p8affineqb $0, %%ymm14, %%ymm2, %%ymm15\n" - "vpxor %%ymm15, %%ymm12, %%ymm12\n" - "vgf2p8affineqb $0, %%ymm14, %%ymm3, %%ymm15\n" - "vpxor %%ymm15, %%ymm13, %%ymm13\n" - "cmpq $2, %0\n" - "jbe 1f\n" - - "vpbroadcastq 16(%2), %%ymm14\n" - "vgf2p8affineqb $0, %%ymm14, %%ymm4, %%ymm15\n" - "vpxor %%ymm15, %%ymm12, %%ymm12\n" - "vgf2p8affineqb $0, %%ymm14, %%ymm5, %%ymm15\n" - "vpxor %%ymm15, %%ymm13, %%ymm13\n" - "cmpq $3, %0\n" - "jbe 1f\n" - - "vpbroadcastq 24(%2), %%ymm14\n" - "vgf2p8affineqb $0, %%ymm14, %%ymm6, %%ymm15\n" - "vpxor %%ymm15, %%ymm12, %%ymm12\n" - "vgf2p8affineqb $0, %%ymm14, %%ymm7, %%ymm15\n" - "vpxor %%ymm15, %%ymm13, %%ymm13\n" - "cmpq $4, %0\n" - "jbe 1f\n" - - "vpbroadcastq 32(%2), %%ymm14\n" - "vgf2p8affineqb $0, %%ymm14, %%ymm8, %%ymm15\n" - "vpxor %%ymm15, %%ymm12, %%ymm12\n" - "vgf2p8affineqb $0, %%ymm14, %%ymm9, %%ymm15\n" - "vpxor %%ymm15, %%ymm13, %%ymm13\n" - "cmpq $5, %0\n" - "jbe 1f\n" - - "vpbroadcastq 40(%2), %%ymm14\n" - "vgf2p8affineqb $0, %%ymm14, %%ymm10, %%ymm15\n" - "vpxor %%ymm15, %%ymm12, %%ymm12\n" - "vgf2p8affineqb $0, %%ymm14, %%ymm11, %%ymm15\n" - "vpxor %%ymm15, %%ymm13, %%ymm13\n" - - "1:\n" - "movq %3, %%rax\n" - "vmovdqa %%ymm12, 0(%%rax, %1)\n" - "vmovdqa %%ymm13, 32(%%rax, %1)\n" - : - : "r" ((uint64_t)N), "r" (i), "r" (C[j * N]), "r" (pa[j]) - : "rax", "cc", "memory" - ); + for (d = 0; d < nd; ++d) { + if (k < nr && d == id[k]) { + ++k; + continue; } + + src[ns] = v[d]; + + for (j = 0; j < nr; ++j) + S[ns][j] = A(ip[j], d); + + ++ns; } - raid_avx_end(); -} - -/* - * RAID recovering GFNI implementation - */ -void raid_recX_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - int N = nr; - uint8_t *p[RAID_PARITY_MAX]; - uint8_t *pa[RAID_PARITY_MAX]; - uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; - uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; - uint8_t C[RAID_PARITY_MAX * RAID_PARITY_MAX][8]; - size_t i; - int j, k; - - /* setup the coefficients matrix */ - for (j = 0; j < N; ++j) - for (k = 0; k < N; ++k) - G[j * N + k] = A(ip[j], id[k]); - - /* invert it to solve the system of linear equations */ - raid_invert(G, V, N); - - /* precompute affine representations of the coefficients */ - for (j = 0; j < N * N; ++j) - memcpy(C[j], raid_gfaffine_raid[V[j]], 8); - - /* compute delta parity */ - raid_delta_gen(N, id, ip, nd, size, vv); - - for (j = 0; j < N; ++j) { - p[j] = v[nd + ip[j]]; - pa[j] = v[id[j]]; - } + BUG_ON(k != nr); + BUG_ON(ns != nd - nr); raid_avx_begin(); - for (i = 0; i < size; i += 64) { - /* delta */ - asm volatile ( - "movq 0(%2), %%rax\n" - "movq 0(%3), %%rbx\n" - "vmovdqa64 (%%rax, %1), %%zmm0\n" - "vmovdqa64 (%%rbx, %1), %%zmm6\n" - "vpxorq %%zmm6, %%zmm0, %%zmm0\n" - "cmpq $1, %0\n" - "jbe 1f\n" - - "movq 8(%2), %%rax\n" - "movq 8(%3), %%rbx\n" - "vmovdqa64 (%%rax, %1), %%zmm1\n" - "vmovdqa64 (%%rbx, %1), %%zmm6\n" - "vpxorq %%zmm6, %%zmm1, %%zmm1\n" - "cmpq $2, %0\n" - "jbe 1f\n" - - "movq 16(%2), %%rax\n" - "movq 16(%3), %%rbx\n" - "vmovdqa64 (%%rax, %1), %%zmm2\n" - "vmovdqa64 (%%rbx, %1), %%zmm6\n" - "vpxorq %%zmm6, %%zmm2, %%zmm2\n" - "cmpq $3, %0\n" - "jbe 1f\n" - - "movq 24(%2), %%rax\n" - "movq 24(%3), %%rbx\n" - "vmovdqa64 (%%rax, %1), %%zmm3\n" - "vmovdqa64 (%%rbx, %1), %%zmm6\n" - "vpxorq %%zmm6, %%zmm3, %%zmm3\n" - "cmpq $4, %0\n" - "jbe 1f\n" - - "movq 32(%2), %%rax\n" - "movq 32(%3), %%rbx\n" - "vmovdqa64 (%%rax, %1), %%zmm4\n" - "vmovdqa64 (%%rbx, %1), %%zmm6\n" - "vpxorq %%zmm6, %%zmm4, %%zmm4\n" - "cmpq $5, %0\n" - "jbe 1f\n" - - "movq 40(%2), %%rax\n" - "movq 40(%3), %%rbx\n" - "vmovdqa64 (%%rax, %1), %%zmm5\n" - "vmovdqa64 (%%rbx, %1), %%zmm6\n" - "vpxorq %%zmm6, %%zmm5, %%zmm5\n" - - "1:\n" - : - : "r" ((uint64_t)N), "r" (i), "r" (p), "r" (pa) - : "rax", "rbx", "cc", "memory" - ); - - /* reconstruct */ - for (j = 0; j < N; ++j) { - asm volatile ( - "vpbroadcastq 0(%2), %%zmm6\n" - "vgf2p8affineqb $0, %%zmm6, %%zmm0, %%zmm7\n" - "cmpq $1, %0\n" - "jbe 1f\n" - - "vpbroadcastq 8(%2), %%zmm6\n" - "vgf2p8affineqb $0, %%zmm6, %%zmm1, %%zmm6\n" - "vpxorq %%zmm6, %%zmm7, %%zmm7\n" - "cmpq $2, %0\n" - "jbe 1f\n" - - "vpbroadcastq 16(%2), %%zmm6\n" - "vgf2p8affineqb $0, %%zmm6, %%zmm2, %%zmm6\n" - "vpxorq %%zmm6, %%zmm7, %%zmm7\n" - "cmpq $3, %0\n" - "jbe 1f\n" - - "vpbroadcastq 24(%2), %%zmm6\n" - "vgf2p8affineqb $0, %%zmm6, %%zmm3, %%zmm6\n" - "vpxorq %%zmm6, %%zmm7, %%zmm7\n" - "cmpq $4, %0\n" - "jbe 1f\n" - - "vpbroadcastq 32(%2), %%zmm6\n" - "vgf2p8affineqb $0, %%zmm6, %%zmm4, %%zmm6\n" - "vpxorq %%zmm6, %%zmm7, %%zmm7\n" - "cmpq $5, %0\n" - "jbe 1f\n" - - "vpbroadcastq 40(%2), %%zmm6\n" - "vgf2p8affineqb $0, %%zmm6, %%zmm5, %%zmm6\n" - "vpxorq %%zmm6, %%zmm7, %%zmm7\n" - - "1:\n" - "movq %3, %%rax\n" - "vmovdqa64 %%zmm7, (%%rax, %1)\n" - : - : "r" ((uint64_t)N), "r" (i), "r" (C[j * N]), "r" (pa[j]) - : "rax", "cc", "memory" - ); - } - } - - raid_avx_end(); -} - -/* - * AES recovering for one disk AVX2 GFNI implementation - */ -void raid_rec1_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p, *pa; - uint8_t G, V; - size_t i; - - (void)nr; - - if (ip[0] == 0) { - raid_rec1of1(id, nd, size, vv); - return; - } - - G = A(ip[0], id[0]); - V = inv(G); - - raid_delta_gen(1, id, ip, nd, size, vv); - - p = v[nd + ip[0]]; - pa = v[id[0]]; - - raid_avx_begin(); - - asm volatile ("vpbroadcastb %0,%%ymm4" : : "m" (V)); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[i])); - asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[i + 32])); - - asm volatile ("vmovdqa %0,%%ymm2" : : "m" (pa[i])); - asm volatile ("vmovdqa %0,%%ymm3" : : "m" (pa[i + 32])); - - asm volatile ("vpxor %ymm2,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm3,%ymm1,%ymm1"); - - asm volatile ("vgf2p8mulb %ymm4,%ymm0,%ymm0"); - asm volatile ("vgf2p8mulb %ymm4,%ymm1,%ymm1"); - - asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[i])); - asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[i + 32])); - } - - raid_avx_end(); -} - -/* - * AES recovering for one disk GFNI implementation - */ -void raid_rec1_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p, *pa; - uint8_t G, V; - size_t i; - - (void)nr; - - if (ip[0] == 0) { - raid_rec1of1(id, nd, size, vv); - return; - } - - G = A(ip[0], id[0]); - V = inv(G); - - raid_delta_gen(1, id, ip, nd, size, vv); - - p = v[nd + ip[0]]; - pa = v[id[0]]; - - raid_avx_begin(); - - asm volatile ("vpbroadcastb %0,%%zmm4" : : "m" (V)); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i])); - asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (pa[i])); - asm volatile ("vpxorq %zmm1,%zmm0,%zmm0"); - asm volatile ("vgf2p8mulb %zmm4,%zmm0,%zmm0"); - asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[i])); - } - - raid_avx_end(); -} - -/* - * AES recovering for two disks AVX2 GFNI implementation - */ -void raid_rec2_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - const int N = 2; - uint8_t *p[N], *pa[N]; - uint8_t G[N * N], V[N * N]; - size_t i; - int j, k; - - (void)nr; - - for (j = 0; j < N; ++j) - for (k = 0; k < N; ++k) - G[j * N + k] = A(ip[j], id[k]); - - raid_invert(G, V, N); - raid_delta_gen(N, id, ip, nd, size, vv); - - for (j = 0; j < N; ++j) { - p[j] = v[nd + ip[j]]; - pa[j] = v[id[j]]; - } - - raid_avx_begin(); - asm volatile ("vpbroadcastb %0,%%ymm10" : : "m" (V[0])); - asm volatile ("vpbroadcastb %0,%%ymm11" : : "m" (V[1])); - asm volatile ("vpbroadcastb %0,%%ymm12" : : "m" (V[2])); - asm volatile ("vpbroadcastb %0,%%ymm13" : : "m" (V[3])); - for (i = 0; i < size; i += 64) { asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i])); asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[0][i + 32])); - asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pa[0][i])); - asm volatile ("vmovdqa %0,%%ymm5" : : "m" (pa[0][i + 32])); + if (nr >= 2) { + asm volatile ("vmovdqa %0,%%ymm2" : : "m" (p[1][i])); + asm volatile ("vmovdqa %0,%%ymm3" : : "m" (p[1][i + 32])); + } - asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); + if (nr >= 3) { + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (p[2][i])); + asm volatile ("vmovdqa %0,%%ymm5" : : "m" (p[2][i + 32])); + } - asm volatile ("vmovdqa %0,%%ymm2" : : "m" (p[1][i])); - asm volatile ("vmovdqa %0,%%ymm3" : : "m" (p[1][i + 32])); + if (nr >= 4) { + asm volatile ("vmovdqa %0,%%ymm6" : : "m" (p[3][i])); + asm volatile ("vmovdqa %0,%%ymm7" : : "m" (p[3][i + 32])); + } - asm volatile ("vmovdqa %0,%%ymm6" : : "m" (pa[1][i])); - asm volatile ("vmovdqa %0,%%ymm7" : : "m" (pa[1][i + 32])); + if (nr >= 5) { + asm volatile ("vmovdqa %0,%%ymm8" : : "m" (p[4][i])); + asm volatile ("vmovdqa %0,%%ymm9" : : "m" (p[4][i + 32])); + } - asm volatile ("vpxor %ymm6,%ymm2,%ymm2"); - asm volatile ("vpxor %ymm7,%ymm3,%ymm3"); + if (nr >= 6) { + asm volatile ("vmovdqa %0,%%ymm10" : : "m" (p[5][i])); + asm volatile ("vmovdqa %0,%%ymm11" : : "m" (p[5][i + 32])); + } - asm volatile ("vgf2p8mulb %ymm0,%ymm10,%ymm4"); - asm volatile ("vgf2p8mulb %ymm1,%ymm10,%ymm5"); + for (s = 0; s < ns; ++s) { + asm volatile ("vmovdqa %0,%%ymm12" : : "m" (src[s][i])); + asm volatile ("vmovdqa %0,%%ymm13" : : "m" (src[s][i + 32])); - asm volatile ("vgf2p8mulb %ymm2,%ymm11,%ymm9"); - asm volatile ("vpxor %ymm9,%ymm4,%ymm4"); - asm volatile ("vgf2p8mulb %ymm3,%ymm11,%ymm9"); - asm volatile ("vpxor %ymm9,%ymm5,%ymm5"); + if (has_p) { + asm volatile ("vpxor %ymm12,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm13,%ymm1,%ymm1"); + } else { + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (S[s][0])); + asm volatile ("vgf2p8mulb %ymm14,%ymm12,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm0,%ymm0"); + asm volatile ("vgf2p8mulb %ymm14,%ymm13,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm1,%ymm1"); + } - asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pa[0][i])); - asm volatile ("vmovdqa %%ymm5,%0" : "=m" (pa[0][i + 32])); + if (nr >= 2) { + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (S[s][1])); + asm volatile ("vgf2p8mulb %ymm14,%ymm12,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm2,%ymm2"); + asm volatile ("vgf2p8mulb %ymm14,%ymm13,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm3,%ymm3"); + } - asm volatile ("vgf2p8mulb %ymm0,%ymm12,%ymm6"); - asm volatile ("vgf2p8mulb %ymm1,%ymm12,%ymm7"); + if (nr >= 3) { + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (S[s][2])); + asm volatile ("vgf2p8mulb %ymm14,%ymm12,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm4,%ymm4"); + asm volatile ("vgf2p8mulb %ymm14,%ymm13,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm5,%ymm5"); + } - asm volatile ("vgf2p8mulb %ymm2,%ymm13,%ymm9"); - asm volatile ("vpxor %ymm9,%ymm6,%ymm6"); - asm volatile ("vgf2p8mulb %ymm3,%ymm13,%ymm9"); - asm volatile ("vpxor %ymm9,%ymm7,%ymm7"); + if (nr >= 4) { + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (S[s][3])); + asm volatile ("vgf2p8mulb %ymm14,%ymm12,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm6,%ymm6"); + asm volatile ("vgf2p8mulb %ymm14,%ymm13,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm7,%ymm7"); + } - asm volatile ("vmovdqa %%ymm6,%0" : "=m" (pa[1][i])); - asm volatile ("vmovdqa %%ymm7,%0" : "=m" (pa[1][i + 32])); + if (nr >= 5) { + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (S[s][4])); + asm volatile ("vgf2p8mulb %ymm14,%ymm12,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm8,%ymm8"); + asm volatile ("vgf2p8mulb %ymm14,%ymm13,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm9,%ymm9"); + } + + if (nr >= 6) { + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (S[s][5])); + asm volatile ("vgf2p8mulb %ymm14,%ymm12,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm10,%ymm10"); + asm volatile ("vgf2p8mulb %ymm14,%ymm13,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm11,%ymm11"); + } + } + + for (j = 0; j < nr; ++j) { + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr])); + asm volatile ("vgf2p8mulb %ymm14,%ymm0,%ymm12"); + asm volatile ("vgf2p8mulb %ymm14,%ymm1,%ymm13"); + + if (nr >= 2) { + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 1])); + asm volatile ("vgf2p8mulb %ymm14,%ymm2,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm12,%ymm12"); + asm volatile ("vgf2p8mulb %ymm14,%ymm3,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm13,%ymm13"); + } + + if (nr >= 3) { + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 2])); + asm volatile ("vgf2p8mulb %ymm14,%ymm4,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm12,%ymm12"); + asm volatile ("vgf2p8mulb %ymm14,%ymm5,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm13,%ymm13"); + } + + if (nr >= 4) { + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 3])); + asm volatile ("vgf2p8mulb %ymm14,%ymm6,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm12,%ymm12"); + asm volatile ("vgf2p8mulb %ymm14,%ymm7,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm13,%ymm13"); + } + + if (nr >= 5) { + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 4])); + asm volatile ("vgf2p8mulb %ymm14,%ymm8,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm12,%ymm12"); + asm volatile ("vgf2p8mulb %ymm14,%ymm9,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm13,%ymm13"); + } + + if (nr >= 6) { + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 5])); + asm volatile ("vgf2p8mulb %ymm14,%ymm10,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm12,%ymm12"); + asm volatile ("vgf2p8mulb %ymm14,%ymm11,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm13,%ymm13"); + } + + asm volatile ("vmovdqa %%ymm12,%0" : "=m" (pa[j][i])); + asm volatile ("vmovdqa %%ymm13,%0" : "=m" (pa[j][i + 32])); + } } raid_avx_end(); } -/* - * AES recovering for two disks GFNI implementation - */ -void raid_rec2_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) +void raid_rec1_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 1); + + /* if it's RAID5 uses the faster function */ + if (ip[0] == 0) { + raid_rec1of1(id, nd, size, vv); + return; + } + + raid_recX_avx2gfni_aes(1, id, ip, nd, size, vv); +} + +void raid_rec2_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 2); + raid_recX_avx2gfni_aes(2, id, ip, nd, size, vv); +} + +void raid_rec3_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 3); + raid_recX_avx2gfni_aes(3, id, ip, nd, size, vv); +} + +void raid_rec4_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 4); + raid_recX_avx2gfni_aes(4, id, ip, nd, size, vv); +} + +void raid_rec5_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 5); + raid_recX_avx2gfni_aes(5, id, ip, nd, size, vv); +} + +void raid_rec6_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 6); + raid_recX_avx2gfni_aes(6, id, ip, nd, size, vv); +} + +static __always_inline void raid_recX_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; - const int N = 2; - uint8_t *p[N], *pa[N]; - uint8_t G[N * N], V[N * N]; + uint8_t *p[RAID_PARITY_MAX]; + uint8_t *pa[RAID_PARITY_MAX]; + uint8_t *src[RAID_DATA_MAX]; + uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t S[RAID_DATA_MAX][RAID_PARITY_MAX]; size_t i; - int j, k; + int d, j, k, s; + int ns; + int has_p; - (void)nr; + BUG_ON(nr < 1 || nr > RAID_PARITY_MAX); - for (j = 0; j < N; ++j) - for (k = 0; k < N; ++k) - G[j * N + k] = A(ip[j], id[k]); + for (j = 0; j < nr; ++j) + for (k = 0; k < nr; ++k) + G[j * nr + k] = A(ip[j], id[k]); - raid_invert(G, V, N); - raid_delta_gen(N, id, ip, nd, size, vv); + raid_invert(G, V, nr); - for (j = 0; j < N; ++j) { + for (j = 0; j < nr; ++j) { p[j] = v[nd + ip[j]]; pa[j] = v[id[j]]; } + has_p = ip[0] == 0; + + ns = 0; + k = 0; + + for (d = 0; d < nd; ++d) { + if (k < nr && d == id[k]) { + ++k; + continue; + } + + src[ns] = v[d]; + + for (j = 0; j < nr; ++j) + S[ns][j] = A(ip[j], d); + + ++ns; + } + + BUG_ON(k != nr); + BUG_ON(ns != nd - nr); + raid_avx_begin(); - asm volatile ("vpbroadcastb %0,%%zmm8" : : "m" (V[0])); - asm volatile ("vpbroadcastb %0,%%zmm9" : : "m" (V[1])); - asm volatile ("vpbroadcastb %0,%%zmm10" : : "m" (V[2])); - asm volatile ("vpbroadcastb %0,%%zmm11" : : "m" (V[3])); for (i = 0; i < size; i += 64) { asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[0][i])); - asm volatile ("vmovdqa64 %0,%%zmm2" : : "m" (pa[0][i])); - asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (p[1][i])); - asm volatile ("vmovdqa64 %0,%%zmm3" : : "m" (pa[1][i])); - asm volatile ("vpxorq %zmm2,%zmm0,%zmm0"); - asm volatile ("vpxorq %zmm3,%zmm1,%zmm1"); + if (nr >= 2) + asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (p[1][i])); + if (nr >= 3) + asm volatile ("vmovdqa64 %0,%%zmm2" : : "m" (p[2][i])); + if (nr >= 4) + asm volatile ("vmovdqa64 %0,%%zmm3" : : "m" (p[3][i])); + if (nr >= 5) + asm volatile ("vmovdqa64 %0,%%zmm4" : : "m" (p[4][i])); + if (nr >= 6) + asm volatile ("vmovdqa64 %0,%%zmm5" : : "m" (p[5][i])); - asm volatile ("vpxorq %zmm6,%zmm6,%zmm6"); + for (s = 0; s < ns; ++s) { + asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (src[s][i])); - asm volatile ("vgf2p8mulb %zmm0,%zmm8,%zmm2"); - asm volatile ("vpxorq %zmm2,%zmm6,%zmm6"); + if (has_p) { + asm volatile ("vpxorq %zmm6,%zmm0,%zmm0"); + } else { + asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (S[s][0])); + asm volatile ("vgf2p8mulb %zmm7,%zmm6,%zmm8"); + asm volatile ("vpxorq %zmm8,%zmm0,%zmm0"); + } - asm volatile ("vgf2p8mulb %zmm1,%zmm9,%zmm3"); - asm volatile ("vpxorq %zmm3,%zmm6,%zmm6"); + if (nr >= 2) { + asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (S[s][1])); + asm volatile ("vgf2p8mulb %zmm7,%zmm6,%zmm8"); + asm volatile ("vpxorq %zmm8,%zmm1,%zmm1"); + } - asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (pa[0][i])); + if (nr >= 3) { + asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (S[s][2])); + asm volatile ("vgf2p8mulb %zmm7,%zmm6,%zmm8"); + asm volatile ("vpxorq %zmm8,%zmm2,%zmm2"); + } - asm volatile ("vpxorq %zmm6,%zmm6,%zmm6"); + if (nr >= 4) { + asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (S[s][3])); + asm volatile ("vgf2p8mulb %zmm7,%zmm6,%zmm8"); + asm volatile ("vpxorq %zmm8,%zmm3,%zmm3"); + } - asm volatile ("vgf2p8mulb %zmm0,%zmm10,%zmm2"); - asm volatile ("vpxorq %zmm2,%zmm6,%zmm6"); + if (nr >= 5) { + asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (S[s][4])); + asm volatile ("vgf2p8mulb %zmm7,%zmm6,%zmm8"); + asm volatile ("vpxorq %zmm8,%zmm4,%zmm4"); + } - asm volatile ("vgf2p8mulb %zmm1,%zmm11,%zmm3"); - asm volatile ("vpxorq %zmm3,%zmm6,%zmm6"); + if (nr >= 6) { + asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (S[s][5])); + asm volatile ("vgf2p8mulb %zmm7,%zmm6,%zmm8"); + asm volatile ("vpxorq %zmm8,%zmm5,%zmm5"); + } + } - asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (pa[1][i])); - } + for (j = 0; j < nr; ++j) { + asm volatile ("vpbroadcastb %0,%%zmm6" : : "m" (V[j * nr])); + asm volatile ("vgf2p8mulb %zmm6,%zmm0,%zmm7"); - raid_avx_end(); -} + if (nr >= 2) { + asm volatile ("vpbroadcastb %0,%%zmm6" : : "m" (V[j * nr + 1])); + asm volatile ("vgf2p8mulb %zmm6,%zmm1,%zmm8"); + asm volatile ("vpxorq %zmm8,%zmm7,%zmm7"); + } -/* - * AES recovering AVX2 GFNI implementation - */ -void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - int N = nr; - uint8_t *p[RAID_PARITY_MAX]; - uint8_t *pa[RAID_PARITY_MAX]; - uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; - uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; - size_t i; - int j, k; + if (nr >= 3) { + asm volatile ("vpbroadcastb %0,%%zmm6" : : "m" (V[j * nr + 2])); + asm volatile ("vgf2p8mulb %zmm6,%zmm2,%zmm8"); + asm volatile ("vpxorq %zmm8,%zmm7,%zmm7"); + } - /* setup the coefficients matrix */ - for (j = 0; j < N; ++j) - for (k = 0; k < N; ++k) - G[j * N + k] = A(ip[j], id[k]); + if (nr >= 4) { + asm volatile ("vpbroadcastb %0,%%zmm6" : : "m" (V[j * nr + 3])); + asm volatile ("vgf2p8mulb %zmm6,%zmm3,%zmm8"); + asm volatile ("vpxorq %zmm8,%zmm7,%zmm7"); + } - /* invert it to solve the system of linear equations */ - raid_invert(G, V, N); + if (nr >= 5) { + asm volatile ("vpbroadcastb %0,%%zmm6" : : "m" (V[j * nr + 4])); + asm volatile ("vgf2p8mulb %zmm6,%zmm4,%zmm8"); + asm volatile ("vpxorq %zmm8,%zmm7,%zmm7"); + } - /* compute delta parity */ - raid_delta_gen(N, id, ip, nd, size, vv); + if (nr >= 6) { + asm volatile ("vpbroadcastb %0,%%zmm6" : : "m" (V[j * nr + 5])); + asm volatile ("vgf2p8mulb %zmm6,%zmm5,%zmm8"); + asm volatile ("vpxorq %zmm8,%zmm7,%zmm7"); + } - for (j = 0; j < N; ++j) { - p[j] = v[nd + ip[j]]; - pa[j] = v[id[j]]; - } - - raid_avx_begin(); - - for (i = 0; i < size; i += 64) { - /* delta */ - asm volatile ( - "movq 0(%2), %%rax\n" - "movq 0(%3), %%rbx\n" - "vmovdqa 0(%%rax, %1), %%ymm0\n" - "vmovdqa 32(%%rax, %1), %%ymm1\n" - "vmovdqa 0(%%rbx, %1), %%ymm14\n" - "vmovdqa 32(%%rbx, %1), %%ymm15\n" - "vpxor %%ymm14, %%ymm0, %%ymm0\n" - "vpxor %%ymm15, %%ymm1, %%ymm1\n" - "cmpq $1, %0\n" - "jbe 1f\n" - - "movq 8(%2), %%rax\n" - "movq 8(%3), %%rbx\n" - "vmovdqa 0(%%rax, %1), %%ymm2\n" - "vmovdqa 32(%%rax, %1), %%ymm3\n" - "vmovdqa 0(%%rbx, %1), %%ymm14\n" - "vmovdqa 32(%%rbx, %1), %%ymm15\n" - "vpxor %%ymm14, %%ymm2, %%ymm2\n" - "vpxor %%ymm15, %%ymm3, %%ymm3\n" - "cmpq $2, %0\n" - "jbe 1f\n" - - "movq 16(%2), %%rax\n" - "movq 16(%3), %%rbx\n" - "vmovdqa 0(%%rax, %1), %%ymm4\n" - "vmovdqa 32(%%rax, %1), %%ymm5\n" - "vmovdqa 0(%%rbx, %1), %%ymm14\n" - "vmovdqa 32(%%rbx, %1), %%ymm15\n" - "vpxor %%ymm14, %%ymm4, %%ymm4\n" - "vpxor %%ymm15, %%ymm5, %%ymm5\n" - "cmpq $3, %0\n" - "jbe 1f\n" - - "movq 24(%2), %%rax\n" - "movq 24(%3), %%rbx\n" - "vmovdqa 0(%%rax, %1), %%ymm6\n" - "vmovdqa 32(%%rax, %1), %%ymm7\n" - "vmovdqa 0(%%rbx, %1), %%ymm14\n" - "vmovdqa 32(%%rbx, %1), %%ymm15\n" - "vpxor %%ymm14, %%ymm6, %%ymm6\n" - "vpxor %%ymm15, %%ymm7, %%ymm7\n" - "cmpq $4, %0\n" - "jbe 1f\n" - - "movq 32(%2), %%rax\n" - "movq 32(%3), %%rbx\n" - "vmovdqa 0(%%rax, %1), %%ymm8\n" - "vmovdqa 32(%%rax, %1), %%ymm9\n" - "vmovdqa 0(%%rbx, %1), %%ymm14\n" - "vmovdqa 32(%%rbx, %1), %%ymm15\n" - "vpxor %%ymm14, %%ymm8, %%ymm8\n" - "vpxor %%ymm15, %%ymm9, %%ymm9\n" - "cmpq $5, %0\n" - "jbe 1f\n" - - "movq 40(%2), %%rax\n" - "movq 40(%3), %%rbx\n" - "vmovdqa 0(%%rax, %1), %%ymm10\n" - "vmovdqa 32(%%rax, %1), %%ymm11\n" - "vmovdqa 0(%%rbx, %1), %%ymm14\n" - "vmovdqa 32(%%rbx, %1), %%ymm15\n" - "vpxor %%ymm14, %%ymm10, %%ymm10\n" - "vpxor %%ymm15, %%ymm11, %%ymm11\n" - - "1:\n" - : - : "r" ((uint64_t)N), "r" (i), "r" (p), "r" (pa) - : "rax", "rbx", "cc", "memory" - ); - - /* reconstruct */ - for (j = 0; j < N; ++j) { - asm volatile ( - "vpbroadcastb 0(%2), %%ymm14\n" - "vgf2p8mulb %%ymm0, %%ymm14, %%ymm12\n" - "vgf2p8mulb %%ymm1, %%ymm14, %%ymm13\n" - "cmpq $1, %0\n" - "jbe 1f\n" - - "vpbroadcastb 1(%2), %%ymm14\n" - "vgf2p8mulb %%ymm2, %%ymm14, %%ymm15\n" - "vpxor %%ymm15, %%ymm12, %%ymm12\n" - "vgf2p8mulb %%ymm3, %%ymm14, %%ymm15\n" - "vpxor %%ymm15, %%ymm13, %%ymm13\n" - "cmpq $2, %0\n" - "jbe 1f\n" - - "vpbroadcastb 2(%2), %%ymm14\n" - "vgf2p8mulb %%ymm4, %%ymm14, %%ymm15\n" - "vpxor %%ymm15, %%ymm12, %%ymm12\n" - "vgf2p8mulb %%ymm5, %%ymm14, %%ymm15\n" - "vpxor %%ymm15, %%ymm13, %%ymm13\n" - "cmpq $3, %0\n" - "jbe 1f\n" - - "vpbroadcastb 3(%2), %%ymm14\n" - "vgf2p8mulb %%ymm6, %%ymm14, %%ymm15\n" - "vpxor %%ymm15, %%ymm12, %%ymm12\n" - "vgf2p8mulb %%ymm7, %%ymm14, %%ymm15\n" - "vpxor %%ymm15, %%ymm13, %%ymm13\n" - "cmpq $4, %0\n" - "jbe 1f\n" - - "vpbroadcastb 4(%2), %%ymm14\n" - "vgf2p8mulb %%ymm8, %%ymm14, %%ymm15\n" - "vpxor %%ymm15, %%ymm12, %%ymm12\n" - "vgf2p8mulb %%ymm9, %%ymm14, %%ymm15\n" - "vpxor %%ymm15, %%ymm13, %%ymm13\n" - "cmpq $5, %0\n" - "jbe 1f\n" - - "vpbroadcastb 5(%2), %%ymm14\n" - "vgf2p8mulb %%ymm10, %%ymm14, %%ymm15\n" - "vpxor %%ymm15, %%ymm12, %%ymm12\n" - "vgf2p8mulb %%ymm11, %%ymm14, %%ymm15\n" - "vpxor %%ymm15, %%ymm13, %%ymm13\n" - - "1:\n" - "movq %3, %%rax\n" - "vmovdqa %%ymm12, 0(%%rax, %1)\n" - "vmovdqa %%ymm13, 32(%%rax, %1)\n" - : - : "r" ((uint64_t)N), "r" (i), "r" (&V[j * N]), "r" (pa[j]) - : "rax", "cc", "memory" - ); + asm volatile ("vmovdqa64 %%zmm7,%0" : "=m" (pa[j][i])); } } raid_avx_end(); } -/* - * AES recovering GFNI implementation - */ -void raid_recX_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) +void raid_rec1_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { - uint8_t **v = (uint8_t **)vv; - int N = nr; - uint8_t *p[RAID_PARITY_MAX]; - uint8_t *pa[RAID_PARITY_MAX]; - uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; - uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; - size_t i; - int j, k; + BUG_ON(nr != 1); - /* setup the coefficients matrix */ - for (j = 0; j < N; ++j) - for (k = 0; k < N; ++k) - G[j * N + k] = A(ip[j], id[k]); - - /* invert it to solve the system of linear equations */ - raid_invert(G, V, N); - - /* compute delta parity */ - raid_delta_gen(N, id, ip, nd, size, vv); - - for (j = 0; j < N; ++j) { - p[j] = v[nd + ip[j]]; - pa[j] = v[id[j]]; + /* if it's RAID5 uses the faster function */ + if (ip[0] == 0) { + raid_rec1of1(id, nd, size, vv); + return; } - raid_avx_begin(); + raid_recX_avx512gfni_aes(1, id, ip, nd, size, vv); +} - for (i = 0; i < size; i += 64) { - /* delta */ - asm volatile ( - "movq 0(%2), %%rax\n" - "movq 0(%3), %%rbx\n" - "vmovdqa64 (%%rax, %1), %%zmm0\n" - "vmovdqa64 (%%rbx, %1), %%zmm6\n" - "vpxorq %%zmm6, %%zmm0, %%zmm0\n" - "cmpq $1, %0\n" - "jbe 1f\n" +void raid_rec2_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 2); + raid_recX_avx512gfni_aes(2, id, ip, nd, size, vv); +} - "movq 8(%2), %%rax\n" - "movq 8(%3), %%rbx\n" - "vmovdqa64 (%%rax, %1), %%zmm1\n" - "vmovdqa64 (%%rbx, %1), %%zmm6\n" - "vpxorq %%zmm6, %%zmm1, %%zmm1\n" - "cmpq $2, %0\n" - "jbe 1f\n" +void raid_rec3_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 3); + raid_recX_avx512gfni_aes(3, id, ip, nd, size, vv); +} - "movq 16(%2), %%rax\n" - "movq 16(%3), %%rbx\n" - "vmovdqa64 (%%rax, %1), %%zmm2\n" - "vmovdqa64 (%%rbx, %1), %%zmm6\n" - "vpxorq %%zmm6, %%zmm2, %%zmm2\n" - "cmpq $3, %0\n" - "jbe 1f\n" +void raid_rec4_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 4); + raid_recX_avx512gfni_aes(4, id, ip, nd, size, vv); +} - "movq 24(%2), %%rax\n" - "movq 24(%3), %%rbx\n" - "vmovdqa64 (%%rax, %1), %%zmm3\n" - "vmovdqa64 (%%rbx, %1), %%zmm6\n" - "vpxorq %%zmm6, %%zmm3, %%zmm3\n" - "cmpq $4, %0\n" - "jbe 1f\n" +void raid_rec5_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 5); + raid_recX_avx512gfni_aes(5, id, ip, nd, size, vv); +} - "movq 32(%2), %%rax\n" - "movq 32(%3), %%rbx\n" - "vmovdqa64 (%%rax, %1), %%zmm4\n" - "vmovdqa64 (%%rbx, %1), %%zmm6\n" - "vpxorq %%zmm6, %%zmm4, %%zmm4\n" - "cmpq $5, %0\n" - "jbe 1f\n" - - "movq 40(%2), %%rax\n" - "movq 40(%3), %%rbx\n" - "vmovdqa64 (%%rax, %1), %%zmm5\n" - "vmovdqa64 (%%rbx, %1), %%zmm6\n" - "vpxorq %%zmm6, %%zmm5, %%zmm5\n" - - "1:\n" - : - : "r" ((uint64_t)N), "r" (i), "r" (p), "r" (pa) - : "rax", "rbx", "cc", "memory" - ); - - /* reconstruct */ - for (j = 0; j < N; ++j) { - asm volatile ( - "vpbroadcastb 0(%2), %%zmm6\n" - "vgf2p8mulb %%zmm0, %%zmm6, %%zmm7\n" - "cmpq $1, %0\n" - "jbe 1f\n" - - "vpbroadcastb 1(%2), %%zmm6\n" - "vgf2p8mulb %%zmm1, %%zmm6, %%zmm6\n" - "vpxorq %%zmm6, %%zmm7, %%zmm7\n" - "cmpq $2, %0\n" - "jbe 1f\n" - - "vpbroadcastb 2(%2), %%zmm6\n" - "vgf2p8mulb %%zmm2, %%zmm6, %%zmm6\n" - "vpxorq %%zmm6, %%zmm7, %%zmm7\n" - "cmpq $3, %0\n" - "jbe 1f\n" - - "vpbroadcastb 3(%2), %%zmm6\n" - "vgf2p8mulb %%zmm3, %%zmm6, %%zmm6\n" - "vpxorq %%zmm6, %%zmm7, %%zmm7\n" - "cmpq $4, %0\n" - "jbe 1f\n" - - "vpbroadcastb 4(%2), %%zmm6\n" - "vgf2p8mulb %%zmm4, %%zmm6, %%zmm6\n" - "vpxorq %%zmm6, %%zmm7, %%zmm7\n" - "cmpq $5, %0\n" - "jbe 1f\n" - - "vpbroadcastb 5(%2), %%zmm6\n" - "vgf2p8mulb %%zmm5, %%zmm6, %%zmm6\n" - "vpxorq %%zmm6, %%zmm7, %%zmm7\n" - - "1:\n" - "movq %3, %%rax\n" - "vmovdqa64 %%zmm7, (%%rax, %1)\n" - : - : "r" ((uint64_t)N), "r" (i), "r" (&V[j * N]), "r" (pa[j]) - : "rax", "cc", "memory" - ); - } - } - - raid_avx_end(); +void raid_rec6_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 6); + raid_recX_avx512gfni_aes(6, id, ip, nd, size, vv); } void raid_register_avx2gfni(void) @@ -1652,17 +1440,17 @@ void raid_register_avx2gfni(void) raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni", raid_rec1_avx2gfni_raid, RAID_POLY_RAID); raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_rec2_avx2gfni_raid, RAID_POLY_RAID); - raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_recX_avx2gfni_raid, RAID_POLY_RAID); - raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_recX_avx2gfni_raid, RAID_POLY_RAID); - raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_recX_avx2gfni_raid, RAID_POLY_RAID); - raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_recX_avx2gfni_raid, RAID_POLY_RAID); + raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_rec3_avx2gfni_raid, RAID_POLY_RAID); + raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_rec4_avx2gfni_raid, RAID_POLY_RAID); + raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_rec5_avx2gfni_raid, RAID_POLY_RAID); + raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_rec6_avx2gfni_raid, RAID_POLY_RAID); raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni", raid_rec1_avx2gfni_aes, RAID_POLY_AES); raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_rec2_avx2gfni_aes, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_recX_avx2gfni_aes, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_recX_avx2gfni_aes, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_recX_avx2gfni_aes, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_recX_avx2gfni_aes, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_rec3_avx2gfni_aes, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_rec4_avx2gfni_aes, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_rec5_avx2gfni_aes, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_rec6_avx2gfni_aes, RAID_POLY_AES); } } @@ -1684,17 +1472,17 @@ void raid_register_avx512gfni(void) raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni512", raid_rec1_avx512gfni_raid, RAID_POLY_RAID); raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_rec2_avx512gfni_raid, RAID_POLY_RAID); - raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_recX_avx512gfni_raid, RAID_POLY_RAID); - raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_recX_avx512gfni_raid, RAID_POLY_RAID); - raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_recX_avx512gfni_raid, RAID_POLY_RAID); - raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_recX_avx512gfni_raid, RAID_POLY_RAID); + raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_rec3_avx512gfni_raid, RAID_POLY_RAID); + raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_rec4_avx512gfni_raid, RAID_POLY_RAID); + raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_rec5_avx512gfni_raid, RAID_POLY_RAID); + raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_rec6_avx512gfni_raid, RAID_POLY_RAID); raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni512", raid_rec1_avx512gfni_aes, RAID_POLY_AES); raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_rec2_avx512gfni_aes, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_recX_avx512gfni_aes, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_recX_avx512gfni_aes, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_recX_avx512gfni_aes, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_recX_avx512gfni_aes, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_rec3_avx512gfni_aes, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_rec4_avx512gfni_aes, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_rec5_avx512gfni_aes, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_rec6_avx512gfni_aes, RAID_POLY_AES); } } } diff --git a/raid/internal.h b/raid/internal.h index 7b459d15..097b1788 100644 --- a/raid/internal.h +++ b/raid/internal.h @@ -172,22 +172,46 @@ void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv); -void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec3_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec4_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec5_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec6_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec1_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec2_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec3_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec4_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec5_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec6_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec2_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv); -void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec3_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec4_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec5_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec6_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec1_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec2_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv); -void raid_recX_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec3_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec4_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec5_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec6_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec1_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec2_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv); -void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec3_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec4_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec5_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec6_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec1_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec2_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv); -void raid_recX_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec3_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec4_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec5_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec6_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec1_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec2_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv); -void raid_recX_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec3_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec4_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec5_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec6_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_gen1_neon(int nd, size_t size, void **vv); void raid_gen2_neon_raid(int nd, size_t size, void **vv); void raid_gen2_neon_aes(int nd, size_t size, void **vv); @@ -202,7 +226,10 @@ void raid_gen6_neon_raid(int nd, size_t size, void **vv); void raid_gen6_neon_aes(int nd, size_t size, void **vv); void raid_rec1_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec2_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv); -void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec3_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec4_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec5_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec6_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_gen1_neon32(int nd, size_t size, void **vv); void raid_gen2_neon32_raid(int nd, size_t size, void **vv); void raid_gen2_neon32_aes(int nd, size_t size, void **vv); @@ -217,7 +244,10 @@ void raid_gen6_neon32_raid(int nd, size_t size, void **vv); void raid_gen6_neon32_aes(int nd, size_t size, void **vv); void raid_rec1_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec2_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv); -void raid_recX_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec3_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec4_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec5_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec6_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv); /* * Functions for parity computation. diff --git a/raid/neon.c b/raid/neon.c index 0e78269b..c59c2ef1 100644 --- a/raid/neon.c +++ b/raid/neon.c @@ -629,431 +629,448 @@ void raid_gen6_neon_aes(int nd, size_t size, void **vv) raid_genX_neon(nd, size, vv, 6, 3); } -/* - * RAID recovering for one disk NEON implementation - */ -void raid_rec1_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) +static __always_inline void raid_recX_neon(int nr, int *id, int *ip, + int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *pa; - uint8_t G; - uint8_t V; + uint8_t *p[RAID_PARITY_MAX]; + uint8_t *pa[RAID_PARITY_MAX]; + uint8_t *src[RAID_DATA_MAX]; + uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; + const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX]; + const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX]; size_t i; + int d, j, k, s; + int ns; + int has_p; - (void)nr; /* unused, it's always 1 */ + BUG_ON(nr < 1 || nr > RAID_PARITY_MAX); - /* if it's RAID5 uses the faster function */ + /* setup the coefficients matrix */ + for (j = 0; j < nr; ++j) + for (k = 0; k < nr; ++k) + G[j * nr + k] = A(ip[j], id[k]); + + /* invert it to solve the system of linear equations */ + raid_invert(G, V, nr); + + /* setup selected parity and destination pointers */ + for (j = 0; j < nr; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + /* ip[] is ordered. If P is selected it is always ip[0]. */ + has_p = ip[0] == 0; + + /* + * Build a compact survivor list and precompute the multiplication + * table pointers for every selected syndrome. + */ + ns = 0; + k = 0; + + for (d = 0; d < nd; ++d) { + if (k < nr && d == id[k]) { + ++k; + continue; + } + + src[ns] = v[d]; + + for (j = 0; j < nr; ++j) + S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0]; + + ++ns; + } + + BUG_ON(k != nr); + BUG_ON(ns != nd - nr); + + /* precompute inverse-matrix multiplication table pointers */ + for (j = 0; j < nr; ++j) + for (k = 0; k < nr; ++k) + R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0]; + + raid_neon_begin(); + + for (i = 0; i < size; i += 32) { + /* + * v31 is reused during reconstruction, so reload the nibble mask + * at the beginning of every iteration. + */ + asm volatile ("ldr q31, %0" : : "m" (gfconst16.low4[0])); + + /* start all selected syndromes from the stored parity */ + asm volatile ("ldr q0, %0" : : "m" (p[0][i])); + asm volatile ("ldr q1, %0" : : "m" (p[0][i + 16])); + + if (nr >= 2) { + asm volatile ("ldr q2, %0" : : "m" (p[1][i])); + asm volatile ("ldr q3, %0" : : "m" (p[1][i + 16])); + } + + if (nr >= 3) { + asm volatile ("ldr q4, %0" : : "m" (p[2][i])); + asm volatile ("ldr q5, %0" : : "m" (p[2][i + 16])); + } + + if (nr >= 4) { + asm volatile ("ldr q6, %0" : : "m" (p[3][i])); + asm volatile ("ldr q7, %0" : : "m" (p[3][i + 16])); + } + + if (nr >= 5) { + asm volatile ("ldr q8, %0" : : "m" (p[4][i])); + asm volatile ("ldr q9, %0" : : "m" (p[4][i + 16])); + } + + if (nr >= 6) { + asm volatile ("ldr q10, %0" : : "m" (p[5][i])); + asm volatile ("ldr q11, %0" : : "m" (p[5][i + 16])); + } + + /* + * Add all surviving data contributions. + * All source reads for this chunk occur before any destination + * write. + */ + for (s = 0; s < ns; ++s) { + const uint8_t **t = S[s]; + + asm volatile ("ldr q12, %0" : : "m" (src[s][i])); + asm volatile ("ldr q13, %0" : : "m" (src[s][i + 16])); + + /* + * P has coefficient 1. Use the original source before + * destructively splitting it into nibbles. + */ + if (has_p) { + asm volatile ("eor v0.16b, v0.16b, v12.16b"); + asm volatile ("eor v1.16b, v1.16b, v13.16b"); + } + + /* split the source lanes */ + asm volatile ("ushr v14.16b, v12.16b, #4"); + asm volatile ("ushr v15.16b, v13.16b, #4"); + asm volatile ("and v12.16b, v12.16b, v31.16b"); + asm volatile ("and v13.16b, v13.16b, v31.16b"); + asm volatile ("and v14.16b, v14.16b, v31.16b"); + asm volatile ("and v15.16b, v15.16b, v31.16b"); + + /* syndrome 0 */ + if (!has_p) { + asm volatile ("ldr q24, %0" : : "m" (t[0][0])); + asm volatile ("ldr q25, %0" : : "m" (t[0][16])); + + asm volatile ("tbl v26.16b, {v24.16b}, v12.16b"); + asm volatile ("tbl v27.16b, {v25.16b}, v14.16b"); + asm volatile ("eor v26.16b, v26.16b, v27.16b"); + asm volatile ("eor v0.16b, v0.16b, v26.16b"); + + asm volatile ("tbl v26.16b, {v24.16b}, v13.16b"); + asm volatile ("tbl v27.16b, {v25.16b}, v15.16b"); + asm volatile ("eor v26.16b, v26.16b, v27.16b"); + asm volatile ("eor v1.16b, v1.16b, v26.16b"); + } + + /* syndrome 1 */ + if (nr >= 2) { + asm volatile ("ldr q24, %0" : : "m" (t[1][0])); + asm volatile ("ldr q25, %0" : : "m" (t[1][16])); + + asm volatile ("tbl v26.16b, {v24.16b}, v12.16b"); + asm volatile ("tbl v27.16b, {v25.16b}, v14.16b"); + asm volatile ("eor v26.16b, v26.16b, v27.16b"); + asm volatile ("eor v2.16b, v2.16b, v26.16b"); + + asm volatile ("tbl v26.16b, {v24.16b}, v13.16b"); + asm volatile ("tbl v27.16b, {v25.16b}, v15.16b"); + asm volatile ("eor v26.16b, v26.16b, v27.16b"); + asm volatile ("eor v3.16b, v3.16b, v26.16b"); + } + + /* syndrome 2 */ + if (nr >= 3) { + asm volatile ("ldr q24, %0" : : "m" (t[2][0])); + asm volatile ("ldr q25, %0" : : "m" (t[2][16])); + + asm volatile ("tbl v26.16b, {v24.16b}, v12.16b"); + asm volatile ("tbl v27.16b, {v25.16b}, v14.16b"); + asm volatile ("eor v26.16b, v26.16b, v27.16b"); + asm volatile ("eor v4.16b, v4.16b, v26.16b"); + + asm volatile ("tbl v26.16b, {v24.16b}, v13.16b"); + asm volatile ("tbl v27.16b, {v25.16b}, v15.16b"); + asm volatile ("eor v26.16b, v26.16b, v27.16b"); + asm volatile ("eor v5.16b, v5.16b, v26.16b"); + } + + /* syndrome 3 */ + if (nr >= 4) { + asm volatile ("ldr q24, %0" : : "m" (t[3][0])); + asm volatile ("ldr q25, %0" : : "m" (t[3][16])); + + asm volatile ("tbl v26.16b, {v24.16b}, v12.16b"); + asm volatile ("tbl v27.16b, {v25.16b}, v14.16b"); + asm volatile ("eor v26.16b, v26.16b, v27.16b"); + asm volatile ("eor v6.16b, v6.16b, v26.16b"); + + asm volatile ("tbl v26.16b, {v24.16b}, v13.16b"); + asm volatile ("tbl v27.16b, {v25.16b}, v15.16b"); + asm volatile ("eor v26.16b, v26.16b, v27.16b"); + asm volatile ("eor v7.16b, v7.16b, v26.16b"); + } + + /* syndrome 4 */ + if (nr >= 5) { + asm volatile ("ldr q24, %0" : : "m" (t[4][0])); + asm volatile ("ldr q25, %0" : : "m" (t[4][16])); + + asm volatile ("tbl v26.16b, {v24.16b}, v12.16b"); + asm volatile ("tbl v27.16b, {v25.16b}, v14.16b"); + asm volatile ("eor v26.16b, v26.16b, v27.16b"); + asm volatile ("eor v8.16b, v8.16b, v26.16b"); + + asm volatile ("tbl v26.16b, {v24.16b}, v13.16b"); + asm volatile ("tbl v27.16b, {v25.16b}, v15.16b"); + asm volatile ("eor v26.16b, v26.16b, v27.16b"); + asm volatile ("eor v9.16b, v9.16b, v26.16b"); + } + + /* syndrome 5 */ + if (nr >= 6) { + asm volatile ("ldr q24, %0" : : "m" (t[5][0])); + asm volatile ("ldr q25, %0" : : "m" (t[5][16])); + + asm volatile ("tbl v26.16b, {v24.16b}, v12.16b"); + asm volatile ("tbl v27.16b, {v25.16b}, v14.16b"); + asm volatile ("eor v26.16b, v26.16b, v27.16b"); + asm volatile ("eor v10.16b, v10.16b, v26.16b"); + + asm volatile ("tbl v26.16b, {v24.16b}, v13.16b"); + asm volatile ("tbl v27.16b, {v25.16b}, v15.16b"); + asm volatile ("eor v26.16b, v26.16b, v27.16b"); + asm volatile ("eor v11.16b, v11.16b, v26.16b"); + } + } + + /* + * Expand raw syndromes backwards into: + * + * S0 = v0..v3 + * S1 = v4..v7 + * S2 = v8..v11 + * S3 = v12..v15 + * S4 = v16..v19 + * S5 = v20..v23 + */ + + if (nr >= 6) { + asm volatile ("ushr v21.16b, v10.16b, #4"); + asm volatile ("and v20.16b, v10.16b, v31.16b"); + asm volatile ("and v21.16b, v21.16b, v31.16b"); + asm volatile ("ushr v23.16b, v11.16b, #4"); + asm volatile ("and v22.16b, v11.16b, v31.16b"); + asm volatile ("and v23.16b, v23.16b, v31.16b"); + } + + if (nr >= 5) { + asm volatile ("ushr v17.16b, v8.16b, #4"); + asm volatile ("and v16.16b, v8.16b, v31.16b"); + asm volatile ("and v17.16b, v17.16b, v31.16b"); + asm volatile ("ushr v19.16b, v9.16b, #4"); + asm volatile ("and v18.16b, v9.16b, v31.16b"); + asm volatile ("and v19.16b, v19.16b, v31.16b"); + } + + if (nr >= 4) { + asm volatile ("ushr v13.16b, v6.16b, #4"); + asm volatile ("and v12.16b, v6.16b, v31.16b"); + asm volatile ("and v13.16b, v13.16b, v31.16b"); + asm volatile ("ushr v15.16b, v7.16b, #4"); + asm volatile ("and v14.16b, v7.16b, v31.16b"); + asm volatile ("and v15.16b, v15.16b, v31.16b"); + } + + if (nr >= 3) { + asm volatile ("ushr v9.16b, v4.16b, #4"); + asm volatile ("and v8.16b, v4.16b, v31.16b"); + asm volatile ("and v9.16b, v9.16b, v31.16b"); + asm volatile ("ushr v11.16b, v5.16b, #4"); + asm volatile ("and v10.16b, v5.16b, v31.16b"); + asm volatile ("and v11.16b, v11.16b, v31.16b"); + } + + if (nr >= 2) { + asm volatile ("ushr v5.16b, v2.16b, #4"); + asm volatile ("and v4.16b, v2.16b, v31.16b"); + asm volatile ("and v5.16b, v5.16b, v31.16b"); + asm volatile ("ushr v7.16b, v3.16b, #4"); + asm volatile ("and v6.16b, v3.16b, v31.16b"); + asm volatile ("and v7.16b, v7.16b, v31.16b"); + } + + /* + * S0 overlaps its original raw lane registers. + * Process raw lane 1 before overwriting v1. + */ + asm volatile ("ushr v3.16b, v1.16b, #4"); + asm volatile ("and v2.16b, v1.16b, v31.16b"); + asm volatile ("and v3.16b, v3.16b, v31.16b"); + asm volatile ("ushr v1.16b, v0.16b, #4"); + asm volatile ("and v0.16b, v0.16b, v31.16b"); + asm volatile ("and v1.16b, v1.16b, v31.16b"); + + /* + * Reconstruct all missing blocks. + * + * v24/v25 = low/high tables + * v26/v27 = lane-0 low/high accumulators + * v28/v29 = lane-1 low/high accumulators + * v30/v31 = low/high multiplication temporaries + */ + for (j = 0; j < nr; ++j) { + const uint8_t **t = R[j]; + + /* coefficient 0 initializes both lane accumulators */ + asm volatile ("ldr q24, %0" : : "m" (t[0][0])); + asm volatile ("ldr q25, %0" : : "m" (t[0][16])); + asm volatile ("tbl v26.16b, {v24.16b}, v0.16b"); + asm volatile ("tbl v27.16b, {v25.16b}, v1.16b"); + asm volatile ("tbl v28.16b, {v24.16b}, v2.16b"); + asm volatile ("tbl v29.16b, {v25.16b}, v3.16b"); + + if (nr >= 2) { + asm volatile ("ldr q24, %0" : : "m" (t[1][0])); + asm volatile ("ldr q25, %0" : : "m" (t[1][16])); + + asm volatile ("tbl v30.16b, {v24.16b}, v4.16b"); + asm volatile ("tbl v31.16b, {v25.16b}, v5.16b"); + asm volatile ("eor v26.16b, v26.16b, v30.16b"); + asm volatile ("eor v27.16b, v27.16b, v31.16b"); + + asm volatile ("tbl v30.16b, {v24.16b}, v6.16b"); + asm volatile ("tbl v31.16b, {v25.16b}, v7.16b"); + asm volatile ("eor v28.16b, v28.16b, v30.16b"); + asm volatile ("eor v29.16b, v29.16b, v31.16b"); + } + + if (nr >= 3) { + asm volatile ("ldr q24, %0" : : "m" (t[2][0])); + asm volatile ("ldr q25, %0" : : "m" (t[2][16])); + + asm volatile ("tbl v30.16b, {v24.16b}, v8.16b"); + asm volatile ("tbl v31.16b, {v25.16b}, v9.16b"); + asm volatile ("eor v26.16b, v26.16b, v30.16b"); + asm volatile ("eor v27.16b, v27.16b, v31.16b"); + + asm volatile ("tbl v30.16b, {v24.16b}, v10.16b"); + asm volatile ("tbl v31.16b, {v25.16b}, v11.16b"); + asm volatile ("eor v28.16b, v28.16b, v30.16b"); + asm volatile ("eor v29.16b, v29.16b, v31.16b"); + } + + if (nr >= 4) { + asm volatile ("ldr q24, %0" : : "m" (t[3][0])); + asm volatile ("ldr q25, %0" : : "m" (t[3][16])); + + asm volatile ("tbl v30.16b, {v24.16b}, v12.16b"); + asm volatile ("tbl v31.16b, {v25.16b}, v13.16b"); + asm volatile ("eor v26.16b, v26.16b, v30.16b"); + asm volatile ("eor v27.16b, v27.16b, v31.16b"); + + asm volatile ("tbl v30.16b, {v24.16b}, v14.16b"); + asm volatile ("tbl v31.16b, {v25.16b}, v15.16b"); + asm volatile ("eor v28.16b, v28.16b, v30.16b"); + asm volatile ("eor v29.16b, v29.16b, v31.16b"); + } + + if (nr >= 5) { + asm volatile ("ldr q24, %0" : : "m" (t[4][0])); + asm volatile ("ldr q25, %0" : : "m" (t[4][16])); + + asm volatile ("tbl v30.16b, {v24.16b}, v16.16b"); + asm volatile ("tbl v31.16b, {v25.16b}, v17.16b"); + asm volatile ("eor v26.16b, v26.16b, v30.16b"); + asm volatile ("eor v27.16b, v27.16b, v31.16b"); + + asm volatile ("tbl v30.16b, {v24.16b}, v18.16b"); + asm volatile ("tbl v31.16b, {v25.16b}, v19.16b"); + asm volatile ("eor v28.16b, v28.16b, v30.16b"); + asm volatile ("eor v29.16b, v29.16b, v31.16b"); + } + + if (nr >= 6) { + asm volatile ("ldr q24, %0" : : "m" (t[5][0])); + asm volatile ("ldr q25, %0" : : "m" (t[5][16])); + + asm volatile ("tbl v30.16b, {v24.16b}, v20.16b"); + asm volatile ("tbl v31.16b, {v25.16b}, v21.16b"); + asm volatile ("eor v26.16b, v26.16b, v30.16b"); + asm volatile ("eor v27.16b, v27.16b, v31.16b"); + + asm volatile ("tbl v30.16b, {v24.16b}, v22.16b"); + asm volatile ("tbl v31.16b, {v25.16b}, v23.16b"); + asm volatile ("eor v28.16b, v28.16b, v30.16b"); + asm volatile ("eor v29.16b, v29.16b, v31.16b"); + } + + asm volatile ("eor v26.16b, v26.16b, v27.16b"); + asm volatile ("eor v28.16b, v28.16b, v29.16b"); + asm volatile ("str q26, %0" : "=m" (pa[j][i])); + asm volatile ("str q28, %0" : "=m" (pa[j][i + 16])); + } + } + + raid_neon_end(); +} + +void raid_rec1_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 1); + + /* + * Preserve the RAID5 fast path. raid_rec1of1() internally calls + * raid_gen(). + */ if (ip[0] == 0) { raid_rec1of1(id, nd, size, vv); return; } - /* setup the coefficients matrix */ - G = A(ip[0], id[0]); - - /* invert it to solve the system of linear equations */ - V = inv(G); - - /* compute delta parity */ - raid_delta_gen(1, id, ip, nd, size, vv); - - p = v[nd + ip[0]]; - pa = v[id[0]]; - - raid_neon_begin(); - - /* preload tables */ - asm volatile ( - "ldr q28, %0\n" /* low4 */ - "ldr q24, %1\n" /* v low table */ - "ldr q25, %2\n" /* v high table */ - : - : "m" (gfconst16.low4[0]), - "m" (raid_gfmulpshufb[V][0][0]), "m" (raid_gfmulpshufb[V][1][0]) - ); - - for (i = 0; i < size; i += 64) { - asm volatile ( - "ldr q0, %4\n" - "ldr q1, %5\n" - "ldr q2, %6\n" - "ldr q3, %7\n" - - "ldr q8, %8\n" - "ldr q9, %9\n" - "ldr q10, %10\n" - "ldr q11, %11\n" - - "eor v0.16b, v0.16b, v8.16b\n" - "eor v1.16b, v1.16b, v9.16b\n" - "eor v2.16b, v2.16b, v10.16b\n" - "eor v3.16b, v3.16b, v11.16b\n" - - "ushr v8.16b, v0.16b, #4\n" - "ushr v9.16b, v1.16b, #4\n" - "ushr v10.16b, v2.16b, #4\n" - "ushr v11.16b, v3.16b, #4\n" - - "and v0.16b, v0.16b, v28.16b\n" - "and v1.16b, v1.16b, v28.16b\n" - "and v2.16b, v2.16b, v28.16b\n" - "and v3.16b, v3.16b, v28.16b\n" - - "and v8.16b, v8.16b, v28.16b\n" - "and v9.16b, v9.16b, v28.16b\n" - "and v10.16b, v10.16b, v28.16b\n" - "and v11.16b, v11.16b, v28.16b\n" - - "tbl v12.16b, {v24.16b}, v0.16b\n" - "tbl v13.16b, {v24.16b}, v1.16b\n" - "tbl v14.16b, {v24.16b}, v2.16b\n" - "tbl v15.16b, {v24.16b}, v3.16b\n" - - "tbl v8.16b, {v25.16b}, v8.16b\n" - "tbl v9.16b, {v25.16b}, v9.16b\n" - "tbl v10.16b, {v25.16b}, v10.16b\n" - "tbl v11.16b, {v25.16b}, v11.16b\n" - - "eor v0.16b, v12.16b, v8.16b\n" - "eor v1.16b, v13.16b, v9.16b\n" - "eor v2.16b, v14.16b, v10.16b\n" - "eor v3.16b, v15.16b, v11.16b\n" - - "str q0, %0\n" - "str q1, %1\n" - "str q2, %2\n" - "str q3, %3\n" - : "=m" (pa[i]), "=m" (pa[i + 16]), "=m" (pa[i + 32]), "=m" (pa[i + 48]) - : "m" (p[i]), "m" (p[i + 16]), "m" (p[i + 32]), "m" (p[i + 48]), - "m" (pa[i]), "m" (pa[i + 16]), "m" (pa[i + 32]), "m" (pa[i + 48]) - ); - } - - raid_neon_end(); + raid_recX_neon(1, id, ip, nd, size, vv); } -/* - * RAID recovering for two disks NEON implementation - */ void raid_rec2_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) { - uint8_t **v = (uint8_t **)vv; - const int N = 2; - uint8_t *p[2]; - uint8_t *pa[2]; - uint8_t G[2 * 2]; - uint8_t V[2 * 2]; - size_t i; - int j, k; - - (void)nr; /* unused, it's always 2 */ - - /* setup the coefficients matrix */ - for (j = 0; j < N; ++j) - for (k = 0; k < N; ++k) - G[j * N + k] = A(ip[j], id[k]); - - /* invert it to solve the system of linear equations */ - raid_invert(G, V, N); - - /* compute delta parity */ - raid_delta_gen(N, id, ip, nd, size, vv); - - for (j = 0; j < N; ++j) { - p[j] = v[nd + ip[j]]; - pa[j] = v[id[j]]; - } - - raid_neon_begin(); - - /* preload tables */ - asm volatile ( - "ldr q28, %0\n" /* low4 */ - "ldr q20, %1\n" /* v[0] low table */ - "ldr q21, %2\n" /* v[0] high table */ - "ldr q22, %3\n" /* v[1] low table */ - "ldr q23, %4\n" /* v[1] high table */ - "ldr q24, %5\n" /* v[2] low table */ - "ldr q25, %6\n" /* v[2] high table */ - "ldr q26, %7\n" /* v[3] low table */ - "ldr q27, %8\n" /* v[3] high table */ - : - : "m" (gfconst16.low4[0]), - "m" (raid_gfmulpshufb[V[0]][0][0]), "m" (raid_gfmulpshufb[V[0]][1][0]), - "m" (raid_gfmulpshufb[V[1]][0][0]), "m" (raid_gfmulpshufb[V[1]][1][0]), - "m" (raid_gfmulpshufb[V[2]][0][0]), "m" (raid_gfmulpshufb[V[2]][1][0]), - "m" (raid_gfmulpshufb[V[3]][0][0]), "m" (raid_gfmulpshufb[V[3]][1][0]) - ); - - for (i = 0; i < size; i += 32) { - asm volatile ( - /* load delta 0 block 0 & 1 */ - "ldr q0, %4\n" - "ldr q2, %5\n" - "ldr q8, %8\n" - "ldr q9, %9\n" - "eor v0.16b, v0.16b, v8.16b\n" - "eor v2.16b, v2.16b, v9.16b\n" - - /* load delta 1 block 0 & 1 */ - "ldr q4, %6\n" - "ldr q6, %7\n" - "ldr q10, %10\n" - "ldr q11, %11\n" - "eor v4.16b, v4.16b, v10.16b\n" - "eor v6.16b, v6.16b, v11.16b\n" - - /* split delta 0 (v0, v2 -> v0 low, v1 high, v2 low, v3 high) */ - "ushr v1.16b, v0.16b, #4\n" - "ushr v3.16b, v2.16b, #4\n" - "and v0.16b, v0.16b, v28.16b\n" - "and v1.16b, v1.16b, v28.16b\n" - "and v2.16b, v2.16b, v28.16b\n" - "and v3.16b, v3.16b, v28.16b\n" - - /* split delta 1 (v4, v6 -> v4 low, v5 high, v6 low, v7 high) */ - "ushr v5.16b, v4.16b, #4\n" - "ushr v7.16b, v6.16b, #4\n" - "and v4.16b, v4.16b, v28.16b\n" - "and v5.16b, v5.16b, v28.16b\n" - "and v6.16b, v6.16b, v28.16b\n" - "and v7.16b, v7.16b, v28.16b\n" - - /* reconstruct pa[0] (v8 = block 0, v9 = block 1) */ - /* V[0] * delta 0 */ - "tbl v8.16b, {v20.16b}, v0.16b\n" - "tbl v9.16b, {v20.16b}, v2.16b\n" - "tbl v12.16b, {v21.16b}, v1.16b\n" - "tbl v13.16b, {v21.16b}, v3.16b\n" - "eor v8.16b, v8.16b, v12.16b\n" - "eor v9.16b, v9.16b, v13.16b\n" - - /* V[1] * delta 1 */ - "tbl v12.16b, {v22.16b}, v4.16b\n" - "tbl v13.16b, {v22.16b}, v6.16b\n" - "tbl v14.16b, {v23.16b}, v5.16b\n" - "tbl v15.16b, {v23.16b}, v7.16b\n" - "eor v12.16b, v12.16b, v14.16b\n" - "eor v13.16b, v13.16b, v15.16b\n" - "eor v8.16b, v8.16b, v12.16b\n" - "eor v9.16b, v9.16b, v13.16b\n" - "str q8, %0\n" - "str q9, %1\n" - - /* reconstruct pa[1] (v10 = block 0, v11 = block 1) */ - /* V[2] * delta 0 */ - "tbl v10.16b, {v24.16b}, v0.16b\n" - "tbl v11.16b, {v24.16b}, v2.16b\n" - "tbl v12.16b, {v25.16b}, v1.16b\n" - "tbl v13.16b, {v25.16b}, v3.16b\n" - "eor v10.16b, v10.16b, v12.16b\n" - "eor v11.16b, v11.16b, v13.16b\n" - - /* V[3] * delta 1 */ - "tbl v12.16b, {v26.16b}, v4.16b\n" - "tbl v13.16b, {v26.16b}, v6.16b\n" - "tbl v14.16b, {v27.16b}, v5.16b\n" - "tbl v15.16b, {v27.16b}, v7.16b\n" - "eor v12.16b, v12.16b, v14.16b\n" - "eor v13.16b, v13.16b, v15.16b\n" - "eor v10.16b, v10.16b, v12.16b\n" - "eor v11.16b, v11.16b, v13.16b\n" - "str q10, %2\n" - "str q11, %3\n" - : "=m" (pa[0][i]), "=m" (pa[0][i + 16]), - "=m" (pa[1][i]), "=m" (pa[1][i + 16]) - : "m" (p[0][i]), "m" (p[0][i + 16]), - "m" (p[1][i]), "m" (p[1][i + 16]), - "m" (pa[0][i]), "m" (pa[0][i + 16]), - "m" (pa[1][i]), "m" (pa[1][i + 16]) - ); - } - - raid_neon_end(); + BUG_ON(nr != 2); + raid_recX_neon(2, id, ip, nd, size, vv); } -/* - * RAID recovering NEON implementation - */ -void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) +void raid_rec3_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) { - uint8_t **v = (uint8_t **)vv; - int N = nr; - uint8_t *p[RAID_PARITY_MAX]; - uint8_t *pa[RAID_PARITY_MAX]; - uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; - uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; - const uint8_t *T[RAID_PARITY_MAX * RAID_PARITY_MAX]; - size_t i; - int j, k; + BUG_ON(nr != 3); + raid_recX_neon(3, id, ip, nd, size, vv); +} - /* setup the coefficients matrix */ - for (j = 0; j < N; ++j) - for (k = 0; k < N; ++k) - G[j * N + k] = A(ip[j], id[k]); +void raid_rec4_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 4); + raid_recX_neon(4, id, ip, nd, size, vv); +} - /* invert it to solve the system of linear equations */ - raid_invert(G, V, N); +void raid_rec5_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 5); + raid_recX_neon(5, id, ip, nd, size, vv); +} - /* precompute shuffle table pointers */ - for (j = 0; j < N * N; ++j) - T[j] = &raid_gfmulpshufb[V[j]][0][0]; - - /* compute delta parity */ - raid_delta_gen(N, id, ip, nd, size, vv); - - for (j = 0; j < N; ++j) { - p[j] = v[nd + ip[j]]; - pa[j] = v[id[j]]; - } - - raid_neon_begin(); - - /* preload mask */ - asm volatile ( - "ldr q28, %0\n" /* low4 */ - : - : "m" (gfconst16.low4[0]) - ); - - for (i = 0; i < size; i += 16) { - /* delta */ - asm volatile ( - "ldr x4, [%2, #0]\n" - "ldr x5, [%3, #0]\n" - "ldr q0, [x4, %1]\n" - "ldr q14, [x5, %1]\n" - "eor v0.16b, v0.16b, v14.16b\n" - "ushr v1.16b, v0.16b, #4\n" - "and v0.16b, v0.16b, v28.16b\n" - "and v1.16b, v1.16b, v28.16b\n" - "cmp %0, #1\n" - "b.ls 1f\n" - - "ldr x4, [%2, #8]\n" - "ldr x5, [%3, #8]\n" - "ldr q2, [x4, %1]\n" - "ldr q14, [x5, %1]\n" - "eor v2.16b, v2.16b, v14.16b\n" - "ushr v3.16b, v2.16b, #4\n" - "and v2.16b, v2.16b, v28.16b\n" - "and v3.16b, v3.16b, v28.16b\n" - "cmp %0, #2\n" - "b.ls 1f\n" - - "ldr x4, [%2, #16]\n" - "ldr x5, [%3, #16]\n" - "ldr q4, [x4, %1]\n" - "ldr q14, [x5, %1]\n" - "eor v4.16b, v4.16b, v14.16b\n" - "ushr v5.16b, v4.16b, #4\n" - "and v4.16b, v4.16b, v28.16b\n" - "and v5.16b, v5.16b, v28.16b\n" - "cmp %0, #3\n" - "b.ls 1f\n" - - "ldr x4, [%2, #24]\n" - "ldr x5, [%3, #24]\n" - "ldr q6, [x4, %1]\n" - "ldr q14, [x5, %1]\n" - "eor v6.16b, v6.16b, v14.16b\n" - "ushr v7.16b, v6.16b, #4\n" - "and v6.16b, v6.16b, v28.16b\n" - "and v7.16b, v7.16b, v28.16b\n" - "cmp %0, #4\n" - "b.ls 1f\n" - - "ldr x4, [%2, #32]\n" - "ldr x5, [%3, #32]\n" - "ldr q8, [x4, %1]\n" - "ldr q14, [x5, %1]\n" - "eor v8.16b, v8.16b, v14.16b\n" - "ushr v9.16b, v8.16b, #4\n" - "and v8.16b, v8.16b, v28.16b\n" - "and v9.16b, v9.16b, v28.16b\n" - "cmp %0, #5\n" - "b.ls 1f\n" - - "ldr x4, [%2, #40]\n" - "ldr x5, [%3, #40]\n" - "ldr q10, [x4, %1]\n" - "ldr q14, [x5, %1]\n" - "eor v10.16b, v10.16b, v14.16b\n" - "ushr v11.16b, v10.16b, #4\n" - "and v10.16b, v10.16b, v28.16b\n" - "and v11.16b, v11.16b, v28.16b\n" - - "1:\n" - : - : "r" ((uint64_t)N), "r" (i), "r" (p), "r" (pa) - : "x4", "x5", "cc", "memory" - ); - - /* reconstruct */ - for (j = 0; j < N; ++j) { - asm volatile ( - "ldr x5, [%2, #0]\n" - "ldp q24, q25, [x5]\n" - "tbl v12.16b, {v24.16b}, v0.16b\n" - "tbl v13.16b, {v25.16b}, v1.16b\n" - "cmp %0, #1\n" - "b.ls 1f\n" - - "ldr x5, [%2, #8]\n" - "ldp q24, q25, [x5]\n" - "tbl v14.16b, {v24.16b}, v2.16b\n" - "tbl v15.16b, {v25.16b}, v3.16b\n" - "eor v12.16b, v12.16b, v14.16b\n" - "eor v13.16b, v13.16b, v15.16b\n" - "cmp %0, #2\n" - "b.ls 1f\n" - - "ldr x5, [%2, #16]\n" - "ldp q24, q25, [x5]\n" - "tbl v14.16b, {v24.16b}, v4.16b\n" - "tbl v15.16b, {v25.16b}, v5.16b\n" - "eor v12.16b, v12.16b, v14.16b\n" - "eor v13.16b, v13.16b, v15.16b\n" - "cmp %0, #3\n" - "b.ls 1f\n" - - "ldr x5, [%2, #24]\n" - "ldp q24, q25, [x5]\n" - "tbl v14.16b, {v24.16b}, v6.16b\n" - "tbl v15.16b, {v25.16b}, v7.16b\n" - "eor v12.16b, v12.16b, v14.16b\n" - "eor v13.16b, v13.16b, v15.16b\n" - "cmp %0, #4\n" - "b.ls 1f\n" - - "ldr x5, [%2, #32]\n" - "ldp q24, q25, [x5]\n" - "tbl v14.16b, {v24.16b}, v8.16b\n" - "tbl v15.16b, {v25.16b}, v9.16b\n" - "eor v12.16b, v12.16b, v14.16b\n" - "eor v13.16b, v13.16b, v15.16b\n" - "cmp %0, #5\n" - "b.ls 1f\n" - - "ldr x5, [%2, #40]\n" - "ldp q24, q25, [x5]\n" - "tbl v14.16b, {v24.16b}, v10.16b\n" - "tbl v15.16b, {v25.16b}, v11.16b\n" - "eor v12.16b, v12.16b, v14.16b\n" - "eor v13.16b, v13.16b, v15.16b\n" - - "1:\n" - "eor v12.16b, v12.16b, v13.16b\n" - "str q12, [%3, %1]\n" - : - : "r" ((uint64_t)N), "r" (i), "r" (&T[j * N]), "r" (pa[j]) - : "x4", "x5", "cc", "memory" - ); - } - } - - raid_neon_end(); +void raid_rec6_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + BUG_ON(nr != 6); + raid_recX_neon(6, id, ip, nd, size, vv); } void raid_register_neon(void) @@ -1073,9 +1090,9 @@ void raid_register_neon(void) raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "neon", raid_rec1_neon, RAID_POLY_ANY); raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "neon", raid_rec2_neon, RAID_POLY_ANY); - raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "neon", raid_recX_neon, RAID_POLY_ANY); - raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "neon", raid_recX_neon, RAID_POLY_ANY); - raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "neon", raid_recX_neon, RAID_POLY_ANY); - raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "neon", raid_recX_neon, RAID_POLY_ANY); + raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "neon", raid_rec3_neon, RAID_POLY_ANY); + raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "neon", raid_rec4_neon, RAID_POLY_ANY); + raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "neon", raid_rec5_neon, RAID_POLY_ANY); + raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "neon", raid_rec6_neon, RAID_POLY_ANY); } #endif diff --git a/raid/neon32.c b/raid/neon32.c index a959139f..cf472bde 100644 --- a/raid/neon32.c +++ b/raid/neon32.c @@ -580,333 +580,686 @@ void raid_gen6_neon32_aes(int nd, size_t size, void **vv) raid_genX_neon32(nd, size, vv, 6, 3); } -/* - * RAID recovering for one disk AArch32 NEON implementation - */ +static __always_inline void raid_recX_neon32_123(int nr, int *id, int *ip, + int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p[RAID_PARITY_MAX]; + uint8_t *pa[RAID_PARITY_MAX]; + uint8_t *src[RAID_DATA_MAX]; + uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; + const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX]; + const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX]; + size_t i; + int d, j, k, s; + int ns; + int has_p; + + BUG_ON(nr < 1 || nr > 3); + + for (j = 0; j < nr; ++j) + for (k = 0; k < nr; ++k) + G[j * nr + k] = A(ip[j], id[k]); + + raid_invert(G, V, nr); + + for (j = 0; j < nr; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + has_p = ip[0] == 0; + + ns = 0; + k = 0; + + for (d = 0; d < nd; ++d) { + if (k < nr && d == id[k]) { + ++k; + continue; + } + + src[ns] = v[d]; + + for (j = 0; j < nr; ++j) + S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0]; + + ++ns; + } + + BUG_ON(k != nr); + BUG_ON(ns != nd - nr); + + for (j = 0; j < nr; ++j) + for (k = 0; k < nr; ++k) + R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0]; + + raid_neon32_begin(); + + for (i = 0; i < size; i += 32) { + /* + * q15 is the low-nibble mask during generation/splitting. + * Reconstruction later reuses q15 as the second accumulator, + * therefore reload it on every iteration. + */ + asm volatile ("vld1.8 {q15}, %0" : : "Q" (gfconst16.low4[0])); + + /* selected stored parity */ + asm volatile ("vld1.8 {q0}, %0" : : "Q" (p[0][i])); + asm volatile ("vld1.8 {q1}, %0" : : "Q" (p[0][i + 16])); + + if (nr >= 2) { + asm volatile ("vld1.8 {q2}, %0" : : "Q" (p[1][i])); + asm volatile ("vld1.8 {q3}, %0" : : "Q" (p[1][i + 16])); + } + + if (nr >= 3) { + asm volatile ("vld1.8 {q4}, %0" : : "Q" (p[2][i])); + asm volatile ("vld1.8 {q5}, %0" : : "Q" (p[2][i + 16])); + } + + for (s = 0; s < ns; ++s) { + const uint8_t **t = S[s]; + + asm volatile ("vld1.8 {q6}, %0" : : "Q" (src[s][i])); + asm volatile ("vld1.8 {q7}, %0" : : "Q" (src[s][i + 16])); + + if (has_p) { + asm volatile ("veor q0, q0, q6"); + asm volatile ("veor q1, q1, q7"); + } + + asm volatile ("vshr.u8 q8, q6, #4"); + asm volatile ("vshr.u8 q9, q7, #4"); + asm volatile ("vand q6, q6, q15"); + asm volatile ("vand q7, q7, q15"); + asm volatile ("vand q8, q8, q15"); + asm volatile ("vand q9, q9, q15"); + + /* + * q10 = low table + * q11 = high table + * q12/q13 = multiplication temporaries + */ + + if (!has_p) { + asm volatile ("vld1.8 {q10}, %0" : : "Q" (t[0][0])); + asm volatile ("vld1.8 {q11}, %0" : : "Q" (t[0][16])); + + asm volatile ("vtbl.8 d24, {d20-d21}, d12"); + asm volatile ("vtbl.8 d25, {d20-d21}, d13"); + asm volatile ("vtbl.8 d26, {d22-d23}, d16"); + asm volatile ("vtbl.8 d27, {d22-d23}, d17"); + asm volatile ("veor q12, q12, q13"); + asm volatile ("veor q0, q0, q12"); + + asm volatile ("vtbl.8 d24, {d20-d21}, d14"); + asm volatile ("vtbl.8 d25, {d20-d21}, d15"); + asm volatile ("vtbl.8 d26, {d22-d23}, d18"); + asm volatile ("vtbl.8 d27, {d22-d23}, d19"); + asm volatile ("veor q12, q12, q13"); + asm volatile ("veor q1, q1, q12"); + } + + if (nr >= 2) { + asm volatile ("vld1.8 {q10}, %0" : : "Q" (t[1][0])); + asm volatile ("vld1.8 {q11}, %0" : : "Q" (t[1][16])); + + asm volatile ("vtbl.8 d24, {d20-d21}, d12"); + asm volatile ("vtbl.8 d25, {d20-d21}, d13"); + asm volatile ("vtbl.8 d26, {d22-d23}, d16"); + asm volatile ("vtbl.8 d27, {d22-d23}, d17"); + asm volatile ("veor q12, q12, q13"); + asm volatile ("veor q2, q2, q12"); + + asm volatile ("vtbl.8 d24, {d20-d21}, d14"); + asm volatile ("vtbl.8 d25, {d20-d21}, d15"); + asm volatile ("vtbl.8 d26, {d22-d23}, d18"); + asm volatile ("vtbl.8 d27, {d22-d23}, d19"); + asm volatile ("veor q12, q12, q13"); + asm volatile ("veor q3, q3, q12"); + } + + if (nr >= 3) { + asm volatile ("vld1.8 {q10}, %0" : : "Q" (t[2][0])); + asm volatile ("vld1.8 {q11}, %0" : : "Q" (t[2][16])); + + asm volatile ("vtbl.8 d24, {d20-d21}, d12"); + asm volatile ("vtbl.8 d25, {d20-d21}, d13"); + asm volatile ("vtbl.8 d26, {d22-d23}, d16"); + asm volatile ("vtbl.8 d27, {d22-d23}, d17"); + asm volatile ("veor q12, q12, q13"); + asm volatile ("veor q4, q4, q12"); + + asm volatile ("vtbl.8 d24, {d20-d21}, d14"); + asm volatile ("vtbl.8 d25, {d20-d21}, d15"); + asm volatile ("vtbl.8 d26, {d22-d23}, d18"); + asm volatile ("vtbl.8 d27, {d22-d23}, d19"); + asm volatile ("veor q12, q12, q13"); + asm volatile ("veor q5, q5, q12"); + } + } + + /* + * Expand raw syndromes backwards. + * + * Final layout: + * + * S0: + * q0 = lane0 low + * q1 = lane0 high + * q2 = lane1 low + * q3 = lane1 high + * + * S1: + * q4/q5 = lane0 low/high + * q6/q7 = lane1 low/high + * + * S2: + * q8/q9 = lane0 low/high + * q10/q11 = lane1 low/high + */ + + if (nr >= 3) { + asm volatile ("vshr.u8 q9, q4, #4"); + asm volatile ("vand q8, q4, q15"); + asm volatile ("vand q9, q9, q15"); + + asm volatile ("vshr.u8 q11, q5, #4"); + asm volatile ("vand q10, q5, q15"); + asm volatile ("vand q11, q11, q15"); + } + + if (nr >= 2) { + asm volatile ("vshr.u8 q5, q2, #4"); + asm volatile ("vand q4, q2, q15"); + asm volatile ("vand q5, q5, q15"); + + asm volatile ("vshr.u8 q7, q3, #4"); + asm volatile ("vand q6, q3, q15"); + asm volatile ("vand q7, q7, q15"); + } + + /* + * Raw S0 lane1 is q1, so process it before q1 is overwritten + * by the high nibble of lane0. + */ + asm volatile ("vshr.u8 q3, q1, #4"); + asm volatile ("vand q2, q1, q15"); + asm volatile ("vand q3, q3, q15"); + + asm volatile ("vshr.u8 q1, q0, #4"); + asm volatile ("vand q0, q0, q15"); + asm volatile ("vand q1, q1, q15"); + + /* + * Reconstruction. + * + * q12 = multiplication table + * q13 = temporary result + * q14 = lane0 accumulator + * q15 = lane1 accumulator + */ + for (j = 0; j < nr; ++j) { + const uint8_t **t = R[j]; + + /* + * coefficient 0 initializes both accumulators + */ + + asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[0][0])); + + asm volatile ("vtbl.8 d28, {d24-d25}, d0"); + asm volatile ("vtbl.8 d29, {d24-d25}, d1"); + asm volatile ("vtbl.8 d30, {d24-d25}, d4"); + asm volatile ("vtbl.8 d31, {d24-d25}, d5"); + + asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[0][16])); + + asm volatile ("vtbl.8 d26, {d24-d25}, d2"); + asm volatile ("vtbl.8 d27, {d24-d25}, d3"); + asm volatile ("veor q14, q14, q13"); + + asm volatile ("vtbl.8 d26, {d24-d25}, d6"); + asm volatile ("vtbl.8 d27, {d24-d25}, d7"); + asm volatile ("veor q15, q15, q13"); + + if (nr >= 2) { + asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[1][0])); + + asm volatile ("vtbl.8 d26, {d24-d25}, d8"); + asm volatile ("vtbl.8 d27, {d24-d25}, d9"); + asm volatile ("veor q14, q14, q13"); + + asm volatile ("vtbl.8 d26, {d24-d25}, d12"); + asm volatile ("vtbl.8 d27, {d24-d25}, d13"); + asm volatile ("veor q15, q15, q13"); + + asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[1][16])); + + asm volatile ("vtbl.8 d26, {d24-d25}, d10"); + asm volatile ("vtbl.8 d27, {d24-d25}, d11"); + asm volatile ("veor q14, q14, q13"); + + asm volatile ("vtbl.8 d26, {d24-d25}, d14"); + asm volatile ("vtbl.8 d27, {d24-d25}, d15"); + asm volatile ("veor q15, q15, q13"); + } + + if (nr >= 3) { + asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[2][0])); + + asm volatile ("vtbl.8 d26, {d24-d25}, d16"); + asm volatile ("vtbl.8 d27, {d24-d25}, d17"); + asm volatile ("veor q14, q14, q13"); + + asm volatile ("vtbl.8 d26, {d24-d25}, d20"); + asm volatile ("vtbl.8 d27, {d24-d25}, d21"); + asm volatile ("veor q15, q15, q13"); + + asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[2][16])); + + asm volatile ("vtbl.8 d26, {d24-d25}, d18"); + asm volatile ("vtbl.8 d27, {d24-d25}, d19"); + asm volatile ("veor q14, q14, q13"); + + asm volatile ("vtbl.8 d26, {d24-d25}, d22"); + asm volatile ("vtbl.8 d27, {d24-d25}, d23"); + asm volatile ("veor q15, q15, q13"); + } + + asm volatile ("vst1.8 {q14}, %0" : "=Q" (pa[j][i])); + asm volatile ("vst1.8 {q15}, %0" : "=Q" (pa[j][i + 16])); + } + } + + raid_neon32_end(); +} + +static __always_inline void raid_recX_neon32(int nr, int *id, int *ip, + int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p[RAID_PARITY_MAX]; + uint8_t *pa[RAID_PARITY_MAX]; + uint8_t *src[RAID_DATA_MAX]; + uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; + const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX]; + const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX]; + size_t i; + int d, j, k, s; + int ns; + int has_p; + + BUG_ON(nr < 1 || nr > RAID_PARITY_MAX); + + for (j = 0; j < nr; ++j) + for (k = 0; k < nr; ++k) + G[j * nr + k] = A(ip[j], id[k]); + + raid_invert(G, V, nr); + + for (j = 0; j < nr; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + has_p = ip[0] == 0; + + ns = 0; + k = 0; + + for (d = 0; d < nd; ++d) { + if (k < nr && d == id[k]) { + ++k; + continue; + } + + src[ns] = v[d]; + + for (j = 0; j < nr; ++j) + S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0]; + + ++ns; + } + + BUG_ON(k != nr); + BUG_ON(ns != nd - nr); + + for (j = 0; j < nr; ++j) + for (k = 0; k < nr; ++k) + R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0]; + + raid_neon32_begin(); + + for (i = 0; i < size; i += 16) { + /* + * q15 is the low-nibble mask during generation/splitting. + * Reconstruction later reuses q15 as a multiplication temporary. + */ + asm volatile ("vld1.8 {q15}, %0" : : "Q" (gfconst16.low4[0])); + + /* + * Raw syndrome registers: + * + * q0 = syndrome 0 + * q1 = syndrome 1 + * q2 = syndrome 2 + * q3 = syndrome 3 + * q4 = syndrome 4 + * q5 = syndrome 5 + */ + + asm volatile ("vld1.8 {q0}, %0" : : "Q" (p[0][i])); + + if (nr >= 2) + asm volatile ("vld1.8 {q1}, %0" : : "Q" (p[1][i])); + + if (nr >= 3) + asm volatile ("vld1.8 {q2}, %0" : : "Q" (p[2][i])); + + if (nr >= 4) + asm volatile ("vld1.8 {q3}, %0" : : "Q" (p[3][i])); + + if (nr >= 5) + asm volatile ("vld1.8 {q4}, %0" : : "Q" (p[4][i])); + + if (nr >= 6) + asm volatile ("vld1.8 {q5}, %0" : : "Q" (p[5][i])); + + /* + * During survivor generation: + * + * q6 = source low + * q7 = source high + * q8 = low table + * q9 = high table + * q10 = low result + * q11 = high result + */ + + for (s = 0; s < ns; ++s) { + const uint8_t **t = S[s]; + + asm volatile ("vld1.8 {q6}, %0" : : "Q" (src[s][i])); + + if (has_p) + asm volatile ("veor q0, q0, q6"); + + asm volatile ("vshr.u8 q7, q6, #4"); + asm volatile ("vand q6, q6, q15"); + asm volatile ("vand q7, q7, q15"); + + if (!has_p) { + asm volatile ("vld1.8 {q8}, %0" : : "Q" (t[0][0])); + asm volatile ("vld1.8 {q9}, %0" : : "Q" (t[0][16])); + + asm volatile ("vtbl.8 d20, {d16-d17}, d12"); + asm volatile ("vtbl.8 d21, {d16-d17}, d13"); + asm volatile ("vtbl.8 d22, {d18-d19}, d14"); + asm volatile ("vtbl.8 d23, {d18-d19}, d15"); + asm volatile ("veor q10, q10, q11"); + asm volatile ("veor q0, q0, q10"); + } + + if (nr >= 2) { + asm volatile ("vld1.8 {q8}, %0" : : "Q" (t[1][0])); + asm volatile ("vld1.8 {q9}, %0" : : "Q" (t[1][16])); + + asm volatile ("vtbl.8 d20, {d16-d17}, d12"); + asm volatile ("vtbl.8 d21, {d16-d17}, d13"); + asm volatile ("vtbl.8 d22, {d18-d19}, d14"); + asm volatile ("vtbl.8 d23, {d18-d19}, d15"); + asm volatile ("veor q10, q10, q11"); + asm volatile ("veor q1, q1, q10"); + } + + if (nr >= 3) { + asm volatile ("vld1.8 {q8}, %0" : : "Q" (t[2][0])); + asm volatile ("vld1.8 {q9}, %0" : : "Q" (t[2][16])); + + asm volatile ("vtbl.8 d20, {d16-d17}, d12"); + asm volatile ("vtbl.8 d21, {d16-d17}, d13"); + asm volatile ("vtbl.8 d22, {d18-d19}, d14"); + asm volatile ("vtbl.8 d23, {d18-d19}, d15"); + asm volatile ("veor q10, q10, q11"); + asm volatile ("veor q2, q2, q10"); + } + + if (nr >= 4) { + asm volatile ("vld1.8 {q8}, %0" : : "Q" (t[3][0])); + asm volatile ("vld1.8 {q9}, %0" : : "Q" (t[3][16])); + + asm volatile ("vtbl.8 d20, {d16-d17}, d12"); + asm volatile ("vtbl.8 d21, {d16-d17}, d13"); + asm volatile ("vtbl.8 d22, {d18-d19}, d14"); + asm volatile ("vtbl.8 d23, {d18-d19}, d15"); + asm volatile ("veor q10, q10, q11"); + asm volatile ("veor q3, q3, q10"); + } + + if (nr >= 5) { + asm volatile ("vld1.8 {q8}, %0" : : "Q" (t[4][0])); + asm volatile ("vld1.8 {q9}, %0" : : "Q" (t[4][16])); + + asm volatile ("vtbl.8 d20, {d16-d17}, d12"); + asm volatile ("vtbl.8 d21, {d16-d17}, d13"); + asm volatile ("vtbl.8 d22, {d18-d19}, d14"); + asm volatile ("vtbl.8 d23, {d18-d19}, d15"); + asm volatile ("veor q10, q10, q11"); + asm volatile ("veor q4, q4, q10"); + } + + if (nr >= 6) { + asm volatile ("vld1.8 {q8}, %0" : : "Q" (t[5][0])); + asm volatile ("vld1.8 {q9}, %0" : : "Q" (t[5][16])); + + asm volatile ("vtbl.8 d20, {d16-d17}, d12"); + asm volatile ("vtbl.8 d21, {d16-d17}, d13"); + asm volatile ("vtbl.8 d22, {d18-d19}, d14"); + asm volatile ("vtbl.8 d23, {d18-d19}, d15"); + asm volatile ("veor q10, q10, q11"); + asm volatile ("veor q5, q5, q10"); + } + } + + /* + * Expand raw syndromes backwards. + * + * Final layout: + * + * q0/q1 syndrome 0 low/high + * q2/q3 syndrome 1 low/high + * q4/q5 syndrome 2 low/high + * q6/q7 syndrome 3 low/high + * q8/q9 syndrome 4 low/high + * q10/q11 syndrome 5 low/high + */ + + if (nr >= 6) { + asm volatile ("vshr.u8 q11, q5, #4"); + asm volatile ("vand q10, q5, q15"); + asm volatile ("vand q11, q11, q15"); + } + + if (nr >= 5) { + asm volatile ("vshr.u8 q9, q4, #4"); + asm volatile ("vand q8, q4, q15"); + asm volatile ("vand q9, q9, q15"); + } + + if (nr >= 4) { + asm volatile ("vshr.u8 q7, q3, #4"); + asm volatile ("vand q6, q3, q15"); + asm volatile ("vand q7, q7, q15"); + } + + if (nr >= 3) { + asm volatile ("vshr.u8 q5, q2, #4"); + asm volatile ("vand q4, q2, q15"); + asm volatile ("vand q5, q5, q15"); + } + + if (nr >= 2) { + asm volatile ("vshr.u8 q3, q1, #4"); + asm volatile ("vand q2, q1, q15"); + asm volatile ("vand q3, q3, q15"); + } + + asm volatile ("vshr.u8 q1, q0, #4"); + asm volatile ("vand q0, q0, q15"); + asm volatile ("vand q1, q1, q15"); + + /* + * Reconstruction: + * + * q12 = low table + * q13 = high table + * q14 = output accumulator + * q15 = multiplication temporary + */ + + for (j = 0; j < nr; ++j) { + const uint8_t **t = R[j]; + + /* + * coefficient 0 initializes q14 + */ + asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[0][0])); + asm volatile ("vld1.8 {q13}, %0" : : "Q" (t[0][16])); + + asm volatile ("vtbl.8 d28, {d24-d25}, d0"); + asm volatile ("vtbl.8 d29, {d24-d25}, d1"); + + asm volatile ("vtbl.8 d30, {d26-d27}, d2"); + asm volatile ("vtbl.8 d31, {d26-d27}, d3"); + + asm volatile ("veor q14, q14, q15"); + + if (nr >= 2) { + asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[1][0])); + asm volatile ("vld1.8 {q13}, %0" : : "Q" (t[1][16])); + + asm volatile ("vtbl.8 d30, {d24-d25}, d4"); + asm volatile ("vtbl.8 d31, {d24-d25}, d5"); + asm volatile ("veor q14, q14, q15"); + + asm volatile ("vtbl.8 d30, {d26-d27}, d6"); + asm volatile ("vtbl.8 d31, {d26-d27}, d7"); + asm volatile ("veor q14, q14, q15"); + } + + if (nr >= 3) { + asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[2][0])); + asm volatile ("vld1.8 {q13}, %0" : : "Q" (t[2][16])); + + asm volatile ("vtbl.8 d30, {d24-d25}, d8"); + asm volatile ("vtbl.8 d31, {d24-d25}, d9"); + asm volatile ("veor q14, q14, q15"); + + asm volatile ("vtbl.8 d30, {d26-d27}, d10"); + asm volatile ("vtbl.8 d31, {d26-d27}, d11"); + asm volatile ("veor q14, q14, q15"); + } + + if (nr >= 4) { + asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[3][0])); + asm volatile ("vld1.8 {q13}, %0" : : "Q" (t[3][16])); + + asm volatile ("vtbl.8 d30, {d24-d25}, d12"); + asm volatile ("vtbl.8 d31, {d24-d25}, d13"); + asm volatile ("veor q14, q14, q15"); + + asm volatile ("vtbl.8 d30, {d26-d27}, d14"); + asm volatile ("vtbl.8 d31, {d26-d27}, d15"); + asm volatile ("veor q14, q14, q15"); + } + + if (nr >= 5) { + asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[4][0])); + asm volatile ("vld1.8 {q13}, %0" : : "Q" (t[4][16])); + + asm volatile ("vtbl.8 d30, {d24-d25}, d16"); + asm volatile ("vtbl.8 d31, {d24-d25}, d17"); + asm volatile ("veor q14, q14, q15"); + + asm volatile ("vtbl.8 d30, {d26-d27}, d18"); + asm volatile ("vtbl.8 d31, {d26-d27}, d19"); + asm volatile ("veor q14, q14, q15"); + } + + if (nr >= 6) { + asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[5][0])); + asm volatile ("vld1.8 {q13}, %0" : : "Q" (t[5][16])); + + asm volatile ("vtbl.8 d30, {d24-d25}, d20"); + asm volatile ("vtbl.8 d31, {d24-d25}, d21"); + asm volatile ("veor q14, q14, q15"); + + asm volatile ("vtbl.8 d30, {d26-d27}, d22"); + asm volatile ("vtbl.8 d31, {d26-d27}, d23"); + asm volatile ("veor q14, q14, q15"); + } + + asm volatile ("vst1.8 {q14}, %0" : "=Q" (pa[j][i])); + } + } + + raid_neon32_end(); +} + void raid_rec1_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv) { - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *pa; - uint8_t G; - uint8_t V; - size_t i; + BUG_ON(nr != 1); - (void)nr; /* unused, it's always 1 */ - - /* if it's RAID5 use the faster function */ + /* + * Keep the RAID5 fast path because raid_rec1of1() is faster and + * internally calls raid_gen(). + */ if (ip[0] == 0) { raid_rec1of1(id, nd, size, vv); return; } - /* setup the coefficients matrix */ - G = A(ip[0], id[0]); - - /* invert it to solve the system of linear equations */ - V = inv(G); - - /* compute delta parity */ - raid_delta_gen(1, id, ip, nd, size, vv); - - p = v[nd + ip[0]]; - pa = v[id[0]]; - - raid_neon32_begin(); - - /* - * Preload the mask and multiplication tables, just as neon.c does. - * - * q12 = low table - * q13 = high table - * q14 = low4 mask - */ - asm volatile ( - "vld1.8 {q14}, %0\n" - "vld1.8 {q12}, %1\n" - "vld1.8 {q13}, %2\n" - : - : "Q" (gfconst16.low4[0]), - "Q" (raid_gfmulpshufb[V][0][0]), - "Q" (raid_gfmulpshufb[V][1][0]) - ); - - for (i = 0; i < size; i += 32) { - asm volatile ( - "vld1.8 {q0}, %2\n" - "vld1.8 {q1}, %3\n" - "vld1.8 {q2}, %4\n" - "vld1.8 {q3}, %5\n" - - "veor q0, q0, q2\n" - "veor q1, q1, q3\n" - - "vshr.u8 q4, q0, #4\n" - "vshr.u8 q5, q1, #4\n" - - "vand q0, q0, q14\n" - "vand q1, q1, q14\n" - "vand q4, q4, q14\n" - "vand q5, q5, q14\n" - - /* low nibbles */ - "vtbl.8 d12, {d24-d25}, d0\n" - "vtbl.8 d13, {d24-d25}, d1\n" - "vtbl.8 d14, {d24-d25}, d2\n" - "vtbl.8 d15, {d24-d25}, d3\n" - - /* high nibbles */ - "vtbl.8 d16, {d26-d27}, d8\n" - "vtbl.8 d17, {d26-d27}, d9\n" - "vtbl.8 d18, {d26-d27}, d10\n" - "vtbl.8 d19, {d26-d27}, d11\n" - - "veor q6, q6, q8\n" - "veor q7, q7, q9\n" - - "vst1.8 {q6}, %0\n" - "vst1.8 {q7}, %1\n" - : "=Q" (pa[i]), "=Q" (pa[i + 16]) - : "Q" (p[i]), "Q" (p[i + 16]), - "Q" (pa[i]), "Q" (pa[i + 16]) - ); - } - - raid_neon32_end(); + raid_recX_neon32_123(1, id, ip, nd, size, vv); } -/* - * RAID recovering for two disks AArch32 NEON implementation - */ void raid_rec2_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv) { - uint8_t **v = (uint8_t **)vv; - const int N = 2; - uint8_t *p[2]; - uint8_t *pa[2]; - uint8_t G[2 * 2]; - uint8_t V[2 * 2]; - size_t i; - int j, k; - - (void)nr; /* unused, it's always 2 */ - - /* setup the coefficients matrix */ - for (j = 0; j < N; ++j) - for (k = 0; k < N; ++k) - G[j * N + k] = A(ip[j], id[k]); - - /* invert it to solve the system of linear equations */ - raid_invert(G, V, N); - - /* compute delta parity */ - raid_delta_gen(N, id, ip, nd, size, vv); - - for (j = 0; j < N; ++j) { - p[j] = v[nd + ip[j]]; - pa[j] = v[id[j]]; - } - - raid_neon32_begin(); - - /* - * q4-q11 contain the four low/high multiplication table pairs. - * q15 contains low4. - */ - asm volatile ( - "vld1.8 {q15}, %0\n" - - "vld1.8 {q4}, %1\n" - "vld1.8 {q5}, %2\n" - - "vld1.8 {q6}, %3\n" - "vld1.8 {q7}, %4\n" - - "vld1.8 {q8}, %5\n" - "vld1.8 {q9}, %6\n" - - "vld1.8 {q10}, %7\n" - "vld1.8 {q11}, %8\n" - : - : "Q" (gfconst16.low4[0]), - "Q" (raid_gfmulpshufb[V[0]][0][0]), - "Q" (raid_gfmulpshufb[V[0]][1][0]), - "Q" (raid_gfmulpshufb[V[1]][0][0]), - "Q" (raid_gfmulpshufb[V[1]][1][0]), - "Q" (raid_gfmulpshufb[V[2]][0][0]), - "Q" (raid_gfmulpshufb[V[2]][1][0]), - "Q" (raid_gfmulpshufb[V[3]][0][0]), - "Q" (raid_gfmulpshufb[V[3]][1][0]) - ); - - for (i = 0; i < size; i += 16) { - asm volatile ( - /* delta 0 */ - "vld1.8 {q0}, %2\n" - "vld1.8 {q12}, %4\n" - "veor q0, q0, q12\n" - "vshr.u8 q1, q0, #4\n" - "vand q0, q0, q15\n" - "vand q1, q1, q15\n" - - /* delta 1 */ - "vld1.8 {q2}, %3\n" - "vld1.8 {q12}, %5\n" - "veor q2, q2, q12\n" - "vshr.u8 q3, q2, #4\n" - "vand q2, q2, q15\n" - "vand q3, q3, q15\n" - - /* pa[0] = V[0] * delta0 */ - - "vtbl.8 d24, {d8-d9}, d0\n" - "vtbl.8 d25, {d8-d9}, d1\n" - "vtbl.8 d26, {d10-d11}, d2\n" - "vtbl.8 d27, {d10-d11}, d3\n" - "veor q14, q12, q13\n" - - /* ^ V[1] * delta1 */ - - "vtbl.8 d24, {d12-d13}, d4\n" - "vtbl.8 d25, {d12-d13}, d5\n" - "vtbl.8 d26, {d14-d15}, d6\n" - "vtbl.8 d27, {d14-d15}, d7\n" - "veor q12, q12, q13\n" - "veor q14, q14, q12\n" - - "vst1.8 {q14}, %0\n" - - /* pa[1] = V[2] * delta0 */ - - "vtbl.8 d24, {d16-d17}, d0\n" - "vtbl.8 d25, {d16-d17}, d1\n" - "vtbl.8 d26, {d18-d19}, d2\n" - "vtbl.8 d27, {d18-d19}, d3\n" - "veor q14, q12, q13\n" - - /* ^ V[3] * delta1 */ - - "vtbl.8 d24, {d20-d21}, d4\n" - "vtbl.8 d25, {d20-d21}, d5\n" - "vtbl.8 d26, {d22-d23}, d6\n" - "vtbl.8 d27, {d22-d23}, d7\n" - "veor q12, q12, q13\n" - "veor q14, q14, q12\n" - - "vst1.8 {q14}, %1\n" - : "=Q" (pa[0][i]), "=Q" (pa[1][i]) - : "Q" (p[0][i]), "Q" (p[1][i]), - "Q" (pa[0][i]), "Q" (pa[1][i]) - ); - } - - raid_neon32_end(); + BUG_ON(nr != 2); + raid_recX_neon32_123(2, id, ip, nd, size, vv); } -/* - * RAID recovering AArch32 NEON implementation - */ -void raid_recX_neon32(int nr, int *id, int *ip, int nd, +void raid_rec3_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv) { - uint8_t **v = (uint8_t **)vv; - int N = nr; - uint8_t *p[RAID_PARITY_MAX]; - uint8_t *pa[RAID_PARITY_MAX]; - uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; - uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; - const uint8_t *T[RAID_PARITY_MAX * RAID_PARITY_MAX]; - uint8_t D[RAID_PARITY_MAX][2][16] __aligned(16); - size_t i; - int j, k; + BUG_ON(nr != 3); + raid_recX_neon32_123(3, id, ip, nd, size, vv); +} - /* setup the coefficients matrix */ - for (j = 0; j < N; ++j) - for (k = 0; k < N; ++k) - G[j * N + k] = A(ip[j], id[k]); +void raid_rec4_neon32(int nr, int *id, int *ip, int nd, + size_t size, void **vv) +{ + BUG_ON(nr != 4); + raid_recX_neon32(4, id, ip, nd, size, vv); +} - /* invert it to solve the system of linear equations */ - raid_invert(G, V, N); +void raid_rec5_neon32(int nr, int *id, int *ip, int nd, + size_t size, void **vv) +{ + BUG_ON(nr != 5); + raid_recX_neon32(5, id, ip, nd, size, vv); +} - /* precompute shuffle table pointers */ - for (j = 0; j < N * N; ++j) - T[j] = &raid_gfmulpshufb[V[j]][0][0]; - - /* compute delta parity */ - raid_delta_gen(N, id, ip, nd, size, vv); - - for (j = 0; j < N; ++j) { - p[j] = v[nd + ip[j]]; - pa[j] = v[id[j]]; - } - - raid_neon32_begin(); - - /* preload mask */ - asm volatile ( - "vld1.8 {q15}, %0\n" - : - : "Q" (gfconst16.low4[0]) - ); - - for (i = 0; i < size; i += 16) { - for (k = 0; k < N; ++k) { - asm volatile ( - "vld1.8 {q0}, %2\n" - "vld1.8 {q1}, %3\n" - "veor q0, q0, q1\n" - - "vshr.u8 q1, q0, #4\n" - "vand q0, q0, q15\n" - "vand q1, q1, q15\n" - - "vst1.8 {q0}, %0\n" - "vst1.8 {q1}, %1\n" - : "=Q" (D[k][0][0]), "=Q" (D[k][1][0]) - : "Q" (p[k][i]), "Q" (pa[k][i]) - ); - } - - /* reconstruct */ - for (j = 0; j < N; ++j) { - asm volatile ( - "veor q0, q0, q0\n" - ); - - for (k = 0; k < N; ++k) { - asm volatile ( - "vld1.8 {q2}, %0\n" - "vld1.8 {q3}, %1\n" - - "vld1.8 {q4}, %2\n" - "vld1.8 {q5}, %3\n" - - "vtbl.8 d12, {d8-d9}, d4\n" - "vtbl.8 d13, {d8-d9}, d5\n" - - "vtbl.8 d14, {d10-d11}, d6\n" - "vtbl.8 d15, {d10-d11}, d7\n" - - "veor q6, q6, q7\n" - "veor q0, q0, q6\n" - : - : "Q" (D[k][0][0]), - "Q" (D[k][1][0]), - "Q" (T[j * N + k][0]), - "Q" (T[j * N + k][16]) - ); - } - - asm volatile ( - "vst1.8 {q0}, %0\n" - : "=Q" (pa[j][i]) - ); - } - } - - raid_neon32_end(); +void raid_rec6_neon32(int nr, int *id, int *ip, int nd, + size_t size, void **vv) +{ + BUG_ON(nr != 6); + raid_recX_neon32(6, id, ip, nd, size, vv); } void raid_register_neon32(void) @@ -932,10 +1285,9 @@ void raid_register_neon32(void) raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "neon32", raid_rec1_neon32, RAID_POLY_ANY); raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "neon32", raid_rec2_neon32, RAID_POLY_ANY); - raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "neon32", raid_recX_neon32, RAID_POLY_ANY); - raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "neon32", raid_recX_neon32, RAID_POLY_ANY); - raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "neon32", raid_recX_neon32, RAID_POLY_ANY); - raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "neon32", raid_recX_neon32, RAID_POLY_ANY); + raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "neon32", raid_rec3_neon32, RAID_POLY_ANY); + raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "neon32", raid_rec4_neon32, RAID_POLY_ANY); + raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "neon32", raid_rec5_neon32, RAID_POLY_ANY); + raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "neon32", raid_rec6_neon32, RAID_POLY_ANY); } - #endif diff --git a/raid/test.c b/raid/test.c index 89f2919f..7f897985 100644 --- a/raid/test.c +++ b/raid/test.c @@ -391,101 +391,214 @@ int raid_test_rec(int mode, int nd, size_t size) /* setup recov functions */ for (i = 1; i <= np; ++i) { - if (i == 1) { + switch (i) { + case 1: test_setup(i) = raid_rec1_int8; -#ifdef CONFIG_X86 - if (raid_cpu_has_ssse3()) - test_setup(i) = raid_rec1_ssse3; - if (raid_cpu_has_avx2()) - test_setup(i) = raid_rec1_avx2; -#ifdef CONFIG_X86_64 - if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) { - if (raid_cpu_has_avx2gfni()) { - if (mode == RAID_MODE_CAUCHY_AES) - test_setup(i) = raid_rec1_avx2gfni_aes; - else - test_setup(i) = raid_rec1_avx2gfni_raid; - } - if (raid_cpu_has_avx512gfni()) { - if (mode == RAID_MODE_CAUCHY_AES) - test_setup(i) = raid_rec1_avx512gfni_aes; - else - test_setup(i) = raid_rec1_avx512gfni_raid; - } - } -#endif -#endif -#ifdef CONFIG_NEON - test_setup(i) = raid_rec1_neon; -#endif -#ifdef CONFIG_NEON32 - test_setup(i) = raid_rec1_neon32; -#endif - } else if (i == 2) { + break; + case 2: test_setup(i) = raid_rec2_int8; -#ifdef CONFIG_X86 - if (raid_cpu_has_ssse3()) - test_setup(i) = raid_rec2_ssse3; - if (raid_cpu_has_avx2()) - test_setup(i) = raid_rec2_avx2; -#ifdef CONFIG_X86_64 - if (raid_cpu_has_avx512bw()) - test_setup(i) = raid_rec2_avx512bw; - if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) { - if (raid_cpu_has_avx2gfni()) { - if (mode == RAID_MODE_CAUCHY_AES) - test_setup(i) = raid_rec2_avx2gfni_aes; - else - test_setup(i) = raid_rec2_avx2gfni_raid; - } - if (raid_cpu_has_avx512gfni()) { - if (mode == RAID_MODE_CAUCHY_AES) - test_setup(i) = raid_rec2_avx512gfni_aes; - else - test_setup(i) = raid_rec2_avx512gfni_raid; - } - } -#endif -#endif -#ifdef CONFIG_NEON - test_setup(i) = raid_rec2_neon; -#endif -#ifdef CONFIG_NEON32 - test_setup(i) = raid_rec2_neon32; -#endif - } else { + break; + default: test_setup(i) = raid_recX_int8; + break; + } #ifdef CONFIG_X86 - if (raid_cpu_has_ssse3()) + if (raid_cpu_has_ssse3()) { + switch (i) { + case 1: + test_setup(i) = raid_rec1_ssse3; + break; + case 2: + test_setup(i) = raid_rec2_ssse3; + break; + default: test_setup(i) = raid_recX_ssse3; - if (raid_cpu_has_avx2()) - test_setup(i) = raid_recX_avx2; + break; + } + } + if (raid_cpu_has_avx2()) { + switch (i) { + case 1: + test_setup(i) = raid_rec1_avx2; + break; + case 2: + test_setup(i) = raid_rec2_avx2; + break; + case 3: + test_setup(i) = raid_rec3_avx2; + break; + case 4: + test_setup(i) = raid_rec4_avx2; + break; + case 5: + test_setup(i) = raid_rec5_avx2; + break; + case 6: + test_setup(i) = raid_rec6_avx2; + break; + } + } #ifdef CONFIG_X86_64 - if (raid_cpu_has_avx512bw()) - test_setup(i) = raid_recX_avx512bw; - if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) { - if (raid_cpu_has_avx2gfni()) { - if (mode == RAID_MODE_CAUCHY_AES) - test_setup(i) = raid_recX_avx2gfni_aes; - else - test_setup(i) = raid_recX_avx2gfni_raid; - } - if (raid_cpu_has_avx512gfni()) { - if (mode == RAID_MODE_CAUCHY_AES) - test_setup(i) = raid_recX_avx512gfni_aes; - else - test_setup(i) = raid_recX_avx512gfni_raid; + if (raid_cpu_has_avx512bw()) { + switch (i) { + case 1: + test_setup(i) = raid_rec1_avx512bw; + break; + case 2: + test_setup(i) = raid_rec2_avx512bw; + break; + case 3: + test_setup(i) = raid_rec3_avx512bw; + break; + case 4: + test_setup(i) = raid_rec4_avx512bw; + break; + case 5: + test_setup(i) = raid_rec5_avx512bw; + break; + case 6: + test_setup(i) = raid_rec6_avx512bw; + break; + } + } + if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) { + if (raid_cpu_has_avx2gfni()) { + if (mode == RAID_MODE_CAUCHY_AES) { + switch (i) { + case 1: + test_setup(i) = raid_rec1_avx2gfni_aes; + break; + case 2: + test_setup(i) = raid_rec2_avx2gfni_aes; + break; + case 3: + test_setup(i) = raid_rec3_avx2gfni_aes; + break; + case 4: + test_setup(i) = raid_rec4_avx2gfni_aes; + break; + case 5: + test_setup(i) = raid_rec5_avx2gfni_aes; + break; + case 6: + test_setup(i) = raid_rec6_avx2gfni_aes; + break; + } + } else { + switch (i) { + case 1: + test_setup(i) = raid_rec1_avx2gfni_raid; + break; + case 2: + test_setup(i) = raid_rec2_avx2gfni_raid; + break; + case 3: + test_setup(i) = raid_rec3_avx2gfni_raid; + break; + case 4: + test_setup(i) = raid_rec4_avx2gfni_raid; + break; + case 5: + test_setup(i) = raid_rec5_avx2gfni_raid; + break; + case 6: + test_setup(i) = raid_rec6_avx2gfni_raid; + break; + } } } + if (raid_cpu_has_avx512gfni()) { + if (mode == RAID_MODE_CAUCHY_AES) { + switch (i) { + case 1: + test_setup(i) = raid_rec1_avx512gfni_aes; + break; + case 2: + test_setup(i) = raid_rec2_avx512gfni_aes; + break; + case 3: + test_setup(i) = raid_rec3_avx512gfni_aes; + break; + case 4: + test_setup(i) = raid_rec4_avx512gfni_aes; + break; + case 5: + test_setup(i) = raid_rec5_avx512gfni_aes; + break; + case 6: + test_setup(i) = raid_rec6_avx512gfni_aes; + break; + } + } else { + switch (i) { + case 1: + test_setup(i) = raid_rec1_avx512gfni_raid; + break; + case 2: + test_setup(i) = raid_rec2_avx512gfni_raid; + break; + case 3: + test_setup(i) = raid_rec3_avx512gfni_raid; + break; + case 4: + test_setup(i) = raid_rec4_avx512gfni_raid; + break; + case 5: + test_setup(i) = raid_rec5_avx512gfni_raid; + break; + case 6: + test_setup(i) = raid_rec6_avx512gfni_raid; + break; + } + } + } + } #endif #endif #ifdef CONFIG_NEON - test_setup(i) = raid_recX_neon; + switch (i) { + case 1: + test_setup(i) = raid_rec1_neon; + break; + case 2: + test_setup(i) = raid_rec2_neon; + break; + case 3: + test_setup(i) = raid_rec3_neon; + break; + case 4: + test_setup(i) = raid_rec4_neon; + break; + case 5: + test_setup(i) = raid_rec5_neon; + break; + case 6: + test_setup(i) = raid_rec6_neon; + break; + } #endif #ifdef CONFIG_NEON32 - test_setup(i) = raid_recX_neon32; -#endif + switch (i) { + case 1: + test_setup(i) = raid_rec1_neon32; + break; + case 2: + test_setup(i) = raid_rec2_neon32; + break; + case 3: + test_setup(i) = raid_rec3_neon32; + break; + case 4: + test_setup(i) = raid_rec4_neon32; + break; + case 5: + test_setup(i) = raid_rec5_neon32; + break; + case 6: + test_setup(i) = raid_rec6_neon32; + break; } +#endif } /* compute the parity */ @@ -607,101 +720,214 @@ int raid_test_tail(int mode, int nd_max, size_t size) /* setup recov functions */ for (i = 1; i <= np; ++i) { - if (i == 1) { + switch (i) { + case 1: test_setup(i) = raid_rec1_int8; -#ifdef CONFIG_X86 - if (raid_cpu_has_ssse3()) - test_setup(i) = raid_rec1_ssse3; - if (raid_cpu_has_avx2()) - test_setup(i) = raid_rec1_avx2; -#ifdef CONFIG_X86_64 - if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) { - if (raid_cpu_has_avx2gfni()) { - if (mode == RAID_MODE_CAUCHY_AES) - test_setup(i) = raid_rec1_avx2gfni_aes; - else - test_setup(i) = raid_rec1_avx2gfni_raid; - } - if (raid_cpu_has_avx512gfni()) { - if (mode == RAID_MODE_CAUCHY_AES) - test_setup(i) = raid_rec1_avx512gfni_aes; - else - test_setup(i) = raid_rec1_avx512gfni_raid; - } - } -#endif -#endif -#ifdef CONFIG_NEON - test_setup(i) = raid_rec1_neon; -#endif -#ifdef CONFIG_NEON32 - test_setup(i) = raid_rec1_neon32; -#endif - } else if (i == 2) { + break; + case 2: test_setup(i) = raid_rec2_int8; -#ifdef CONFIG_X86 - if (raid_cpu_has_ssse3()) - test_setup(i) = raid_rec2_ssse3; - if (raid_cpu_has_avx2()) - test_setup(i) = raid_rec2_avx2; -#ifdef CONFIG_X86_64 - if (raid_cpu_has_avx512bw()) - test_setup(i) = raid_rec2_avx512bw; - if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) { - if (raid_cpu_has_avx2gfni()) { - if (mode == RAID_MODE_CAUCHY_AES) - test_setup(i) = raid_rec2_avx2gfni_aes; - else - test_setup(i) = raid_rec2_avx2gfni_raid; - } - if (raid_cpu_has_avx512gfni()) { - if (mode == RAID_MODE_CAUCHY_AES) - test_setup(i) = raid_rec2_avx512gfni_aes; - else - test_setup(i) = raid_rec2_avx512gfni_raid; - } - } -#endif -#endif -#ifdef CONFIG_NEON - test_setup(i) = raid_rec2_neon; -#endif -#ifdef CONFIG_NEON32 - test_setup(i) = raid_rec2_neon32; -#endif - } else { + break; + default: test_setup(i) = raid_recX_int8; + break; + } #ifdef CONFIG_X86 - if (raid_cpu_has_ssse3()) + if (raid_cpu_has_ssse3()) { + switch (i) { + case 1: + test_setup(i) = raid_rec1_ssse3; + break; + case 2: + test_setup(i) = raid_rec2_ssse3; + break; + default: test_setup(i) = raid_recX_ssse3; - if (raid_cpu_has_avx2()) - test_setup(i) = raid_recX_avx2; + break; + } + } + if (raid_cpu_has_avx2()) { + switch (i) { + case 1: + test_setup(i) = raid_rec1_avx2; + break; + case 2: + test_setup(i) = raid_rec2_avx2; + break; + case 3: + test_setup(i) = raid_rec3_avx2; + break; + case 4: + test_setup(i) = raid_rec4_avx2; + break; + case 5: + test_setup(i) = raid_rec5_avx2; + break; + case 6: + test_setup(i) = raid_rec6_avx2; + break; + } + } #ifdef CONFIG_X86_64 - if (raid_cpu_has_avx512bw()) - test_setup(i) = raid_recX_avx512bw; - if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) { - if (raid_cpu_has_avx2gfni()) { - if (mode == RAID_MODE_CAUCHY_AES) - test_setup(i) = raid_recX_avx2gfni_aes; - else - test_setup(i) = raid_recX_avx2gfni_raid; - } - if (raid_cpu_has_avx512gfni()) { - if (mode == RAID_MODE_CAUCHY_AES) - test_setup(i) = raid_recX_avx512gfni_aes; - else - test_setup(i) = raid_recX_avx512gfni_raid; + if (raid_cpu_has_avx512bw()) { + switch (i) { + case 1: + test_setup(i) = raid_rec1_avx512bw; + break; + case 2: + test_setup(i) = raid_rec2_avx512bw; + break; + case 3: + test_setup(i) = raid_rec3_avx512bw; + break; + case 4: + test_setup(i) = raid_rec4_avx512bw; + break; + case 5: + test_setup(i) = raid_rec5_avx512bw; + break; + case 6: + test_setup(i) = raid_rec6_avx512bw; + break; + } + } + if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) { + if (raid_cpu_has_avx2gfni()) { + if (mode == RAID_MODE_CAUCHY_AES) { + switch (i) { + case 1: + test_setup(i) = raid_rec1_avx2gfni_aes; + break; + case 2: + test_setup(i) = raid_rec2_avx2gfni_aes; + break; + case 3: + test_setup(i) = raid_rec3_avx2gfni_aes; + break; + case 4: + test_setup(i) = raid_rec4_avx2gfni_aes; + break; + case 5: + test_setup(i) = raid_rec5_avx2gfni_aes; + break; + case 6: + test_setup(i) = raid_rec6_avx2gfni_aes; + break; + } + } else { + switch (i) { + case 1: + test_setup(i) = raid_rec1_avx2gfni_raid; + break; + case 2: + test_setup(i) = raid_rec2_avx2gfni_raid; + break; + case 3: + test_setup(i) = raid_rec3_avx2gfni_raid; + break; + case 4: + test_setup(i) = raid_rec4_avx2gfni_raid; + break; + case 5: + test_setup(i) = raid_rec5_avx2gfni_raid; + break; + case 6: + test_setup(i) = raid_rec6_avx2gfni_raid; + break; + } } } + if (raid_cpu_has_avx512gfni()) { + if (mode == RAID_MODE_CAUCHY_AES) { + switch (i) { + case 1: + test_setup(i) = raid_rec1_avx512gfni_aes; + break; + case 2: + test_setup(i) = raid_rec2_avx512gfni_aes; + break; + case 3: + test_setup(i) = raid_rec3_avx512gfni_aes; + break; + case 4: + test_setup(i) = raid_rec4_avx512gfni_aes; + break; + case 5: + test_setup(i) = raid_rec5_avx512gfni_aes; + break; + case 6: + test_setup(i) = raid_rec6_avx512gfni_aes; + break; + } + } else { + switch (i) { + case 1: + test_setup(i) = raid_rec1_avx512gfni_raid; + break; + case 2: + test_setup(i) = raid_rec2_avx512gfni_raid; + break; + case 3: + test_setup(i) = raid_rec3_avx512gfni_raid; + break; + case 4: + test_setup(i) = raid_rec4_avx512gfni_raid; + break; + case 5: + test_setup(i) = raid_rec5_avx512gfni_raid; + break; + case 6: + test_setup(i) = raid_rec6_avx512gfni_raid; + break; + } + } + } + } #endif #endif #ifdef CONFIG_NEON - test_setup(i) = raid_recX_neon; + switch (i) { + case 1: + test_setup(i) = raid_rec1_neon; + break; + case 2: + test_setup(i) = raid_rec2_neon; + break; + case 3: + test_setup(i) = raid_rec3_neon; + break; + case 4: + test_setup(i) = raid_rec4_neon; + break; + case 5: + test_setup(i) = raid_rec5_neon; + break; + case 6: + test_setup(i) = raid_rec6_neon; + break; + } #endif #ifdef CONFIG_NEON32 - test_setup(i) = raid_recX_neon32; -#endif + switch (i) { + case 1: + test_setup(i) = raid_rec1_neon32; + break; + case 2: + test_setup(i) = raid_rec2_neon32; + break; + case 3: + test_setup(i) = raid_rec3_neon32; + break; + case 4: + test_setup(i) = raid_rec4_neon32; + break; + case 5: + test_setup(i) = raid_rec5_neon32; + break; + case 6: + test_setup(i) = raid_rec6_neon32; + break; } +#endif } /* test all disk counts from 1 to nd_max */ diff --git a/raid/test/speedtest.c b/raid/test/speedtest.c index 591ff39a..eb0a5190 100644 --- a/raid/test/speedtest.c +++ b/raid/test/speedtest.c @@ -1124,8 +1124,12 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) printf("%8s", "ssse3"); - if (raid_cpu_has_avx2()) + if (raid_cpu_has_avx2()) { printf("%8s", "avx2"); +#ifdef CONFIG_X86_64 + printf("%8s", "avx2e"); +#endif + } #ifdef CONFIG_X86_64 if (raid_cpu_has_avx512bw()) printf("%8s", "avx512"); @@ -1189,6 +1193,19 @@ void speed_rec(int nd, void **v, int size, int delta, int period) fflush(stdout); } #ifdef CONFIG_X86_64 + if (raid_cpu_has_avx2()) { + SPEED_START { + /* + * raid_rec1_avx2ext() uses raid_rec1of1() for P, + * which internally calls raid_gen(). + */ + raid_gen_force(1, raid_gen1_avx2); + raid_rec1_avx2ext(1, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } if (raid_cpu_has_avx512bw()) { SPEED_START { raid_gen_force(1, raid_gen1_avx512bw); @@ -1236,7 +1253,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #ifdef CONFIG_NEON SPEED_START { - raid_gen_force(1, raid_gen1_neon); /* +1 to avoid GEN1 optimized case */ raid_rec1_neon(1, id, ip + 1, nd, size, v); } SPEED_STOP @@ -1246,7 +1262,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #endif #ifdef CONFIG_NEON32 SPEED_START { - raid_gen_force(1, raid_gen1_neon32); /* +1 to avoid GEN1 optimized case */ raid_rec1_neon32(1, id, ip + 1, nd, size, v); } SPEED_STOP @@ -1277,10 +1292,18 @@ void speed_rec(int nd, void **v, int size, int delta, int period) fflush(stdout); } #ifdef CONFIG_X86_64 + if (raid_cpu_has_avx2()) { + SPEED_START { + /* +1 to avoid RAID5 optimized case */ + raid_rec1_avx2ext(1, id, ip + 1, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } if (raid_cpu_has_avx512bw()) { SPEED_START { - raid_gen_force(1, raid_gen1_avx512bw); - /* +1 to avoid GEN1 optimized case */ + /* +1 to avoid RAID5 optimized case */ raid_rec1_avx512bw(1, id, ip + 1, nd, size, v); } SPEED_STOP @@ -1289,8 +1312,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx2gfni()) { SPEED_START { - raid_gen_force(1, raid_gen1_avx2); - /* +1 to avoid GEN1 optimized case */ + /* +1 to avoid RAID5 optimized case */ raid_rec1_avx2gfni_raid(1, id, ip + 1, nd, size, v); } SPEED_STOP @@ -1299,8 +1321,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx512gfni()) { SPEED_START { - raid_gen_force(1, raid_gen1_avx512bw); /* there is no raid_gen1_avx512gfni */ - /* +1 to avoid GEN1 optimized case */ + /* +1 to avoid RAID5 optimized case */ raid_rec1_avx512gfni_raid(1, id, ip + 1, nd, size, v); } SPEED_STOP @@ -1326,7 +1347,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #ifdef CONFIG_NEON SPEED_START { - raid_gen_force(2, raid_gen2_neon_raid); raid_rec2_neon(2, id, ip, nd, size, v); } SPEED_STOP @@ -1335,7 +1355,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #endif #ifdef CONFIG_NEON32 SPEED_START { - raid_gen_force(2, raid_gen2_neon32_raid); raid_rec2_neon32(2, id, ip, nd, size, v); } SPEED_STOP @@ -1371,9 +1390,16 @@ void speed_rec(int nd, void **v, int size, int delta, int period) fflush(stdout); } #ifdef CONFIG_X86_64 + if (raid_cpu_has_avx2()) { + SPEED_START { + raid_rec2_avx2ext(2, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } if (raid_cpu_has_avx512bw()) { SPEED_START { - raid_gen_force(2, raid_gen2_avx512bw); raid_rec2_avx512bw(2, id, ip, nd, size, v); } SPEED_STOP @@ -1382,7 +1408,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx2gfni()) { SPEED_START { - raid_gen_force(2, raid_gen2_avx2gfni_raid); raid_rec2_avx2gfni_raid(2, id, ip, nd, size, v); } SPEED_STOP @@ -1391,7 +1416,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx512gfni()) { SPEED_START { - raid_gen_force(2, raid_gen2_avx512gfni_raid); raid_rec2_avx512gfni_raid(2, id, ip, nd, size, v); } SPEED_STOP @@ -1418,7 +1442,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #ifdef CONFIG_NEON SPEED_START { - raid_gen_force(2, raid_gen2_neon_raid); /* +1 to avoid GEN2 optimized case */ raid_rec2_neon(2, id, ip + 1, nd, size, v); } SPEED_STOP @@ -1428,7 +1451,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #endif #ifdef CONFIG_NEON32 SPEED_START { - raid_gen_force(2, raid_gen2_neon32_raid); /* +1 to avoid GEN2 optimized case */ raid_rec2_neon32(2, id, ip + 1, nd, size, v); } SPEED_STOP @@ -1467,9 +1489,16 @@ void speed_rec(int nd, void **v, int size, int delta, int period) fflush(stdout); } #ifdef CONFIG_X86_64 + if (raid_cpu_has_avx2()) { + SPEED_START { + raid_rec2_avx2ext(2, id, ip + 1, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } if (raid_cpu_has_avx512bw()) { SPEED_START { - raid_gen_force(2, raid_gen2_avx512bw); /* +1 to avoid GEN2 optimized case */ raid_rec2_avx512bw(2, id, ip + 1, nd, size, v); } SPEED_STOP @@ -1479,7 +1508,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx2gfni()) { SPEED_START { - raid_gen_force(2, raid_gen2_avx2gfni_raid); /* +1 to avoid GEN2 optimized case */ raid_rec2_avx2gfni_raid(2, id, ip + 1, nd, size, v); } SPEED_STOP @@ -1489,7 +1517,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx512gfni()) { SPEED_START { - raid_gen_force(2, raid_gen2_avx512gfni_raid); /* +1 to avoid GEN2 optimized case */ raid_rec2_avx512gfni_raid(2, id, ip + 1, nd, size, v); } SPEED_STOP @@ -1516,8 +1543,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #ifdef CONFIG_NEON SPEED_START { - raid_gen_force(3, raid_gen3_neon_raid); - raid_recX_neon(3, id, ip, nd, size, v); + raid_rec3_neon(3, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1525,8 +1551,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #endif #ifdef CONFIG_NEON32 SPEED_START { - raid_gen_force(3, raid_gen3_neon32_raid); - raid_recX_neon32(3, id, ip, nd, size, v); + raid_rec3_neon32(3, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1554,18 +1579,24 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #else raid_gen_force(3, raid_gen3_ssse3_raid); #endif - raid_recX_avx2(3, id, ip, nd, size, v); + raid_rec3_avx2(3, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); fflush(stdout); } #ifdef CONFIG_X86_64 + if (raid_cpu_has_avx2()) { + SPEED_START { + raid_rec3_avx2ext(3, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } if (raid_cpu_has_avx512bw()) { SPEED_START { - raid_gen_force(3, raid_gen3_avx512bw); - /* +1 to avoid GEN1 optimized case */ - raid_recX_avx512bw(3, id, ip, nd, size, v); + raid_rec3_avx512bw(3, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1573,8 +1604,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx2gfni()) { SPEED_START { - raid_gen_force(3, raid_gen3_avx2gfni_raid); - raid_recX_avx2gfni_raid(3, id, ip, nd, size, v); + raid_rec3_avx2gfni_raid(3, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1582,8 +1612,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx512gfni()) { SPEED_START { - raid_gen_force(3, raid_gen3_avx512gfni_raid); - raid_recX_avx512gfni_raid(3, id, ip, nd, size, v); + raid_rec3_avx512gfni_raid(3, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1608,8 +1637,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #ifdef CONFIG_NEON SPEED_START { - raid_gen_force(4, raid_gen4_neon_raid); - raid_recX_neon(4, id, ip, nd, size, v); + raid_rec4_neon(4, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1617,8 +1645,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #endif #ifdef CONFIG_NEON32 SPEED_START { - raid_gen_force(4, raid_gen4_neon32_raid); - raid_recX_neon32(4, id, ip, nd, size, v); + raid_rec4_neon32(4, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1646,18 +1673,24 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #else raid_gen_force(4, raid_gen4_ssse3_raid); #endif - raid_recX_avx2(4, id, ip, nd, size, v); + raid_rec4_avx2(4, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); fflush(stdout); } #ifdef CONFIG_X86_64 + if (raid_cpu_has_avx2()) { + SPEED_START { + raid_rec4_avx2ext(4, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } if (raid_cpu_has_avx512bw()) { SPEED_START { - raid_gen_force(4, raid_gen4_avx512bw); - /* +1 to avoid GEN1 optimized case */ - raid_recX_avx512bw(4, id, ip, nd, size, v); + raid_rec4_avx512bw(4, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1665,8 +1698,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx2gfni()) { SPEED_START { - raid_gen_force(4, raid_gen4_avx2gfni_raid); - raid_recX_avx2gfni_raid(4, id, ip, nd, size, v); + raid_rec4_avx2gfni_raid(4, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1674,8 +1706,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx512gfni()) { SPEED_START { - raid_gen_force(4, raid_gen4_avx512gfni_raid); - raid_recX_avx512gfni_raid(4, id, ip, nd, size, v); + raid_rec4_avx512gfni_raid(4, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1700,8 +1731,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #ifdef CONFIG_NEON SPEED_START { - raid_gen_force(5, raid_gen5_neon_raid); - raid_recX_neon(5, id, ip, nd, size, v); + raid_rec5_neon(5, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1709,8 +1739,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #endif #ifdef CONFIG_NEON32 SPEED_START { - raid_gen_force(5, raid_gen5_neon32_raid); - raid_recX_neon32(5, id, ip, nd, size, v); + raid_rec5_neon32(5, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1738,18 +1767,24 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #else raid_gen_force(5, raid_gen5_ssse3_raid); #endif - raid_recX_avx2(5, id, ip, nd, size, v); + raid_rec5_avx2(5, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); fflush(stdout); } #ifdef CONFIG_X86_64 + if (raid_cpu_has_avx2()) { + SPEED_START { + raid_rec5_avx2ext(5, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } if (raid_cpu_has_avx512bw()) { SPEED_START { - raid_gen_force(5, raid_gen5_avx512bw); - /* +1 to avoid GEN1 optimized case */ - raid_recX_avx512bw(5, id, ip, nd, size, v); + raid_rec5_avx512bw(5, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1757,8 +1792,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx2gfni()) { SPEED_START { - raid_gen_force(5, raid_gen5_avx2gfni_raid); - raid_recX_avx2gfni_raid(5, id, ip, nd, size, v); + raid_rec5_avx2gfni_raid(5, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1766,8 +1800,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx512gfni()) { SPEED_START { - raid_gen_force(5, raid_gen5_avx512gfni_raid); - raid_recX_avx512gfni_raid(5, id, ip, nd, size, v); + raid_rec5_avx512gfni_raid(5, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1792,8 +1825,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #ifdef CONFIG_NEON SPEED_START { - raid_gen_force(6, raid_gen6_neon_raid); - raid_recX_neon(6, id, ip, nd, size, v); + raid_rec6_neon(6, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1801,8 +1833,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #endif #ifdef CONFIG_NEON32 SPEED_START { - raid_gen_force(6, raid_gen6_neon32_raid); - raid_recX_neon32(6, id, ip, nd, size, v); + raid_rec6_neon32(6, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1830,18 +1861,24 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #else raid_gen_force(6, raid_gen6_ssse3_raid); #endif - raid_recX_avx2(6, id, ip, nd, size, v); + raid_rec6_avx2(6, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); fflush(stdout); } #ifdef CONFIG_X86_64 + if (raid_cpu_has_avx2()) { + SPEED_START { + raid_rec6_avx2ext(6, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } if (raid_cpu_has_avx512bw()) { SPEED_START { - raid_gen_force(6, raid_gen6_avx512bw); - /* +1 to avoid GEN1 optimized case */ - raid_recX_avx512bw(6, id, ip, nd, size, v); + raid_rec6_avx512bw(6, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1849,8 +1886,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx2gfni()) { SPEED_START { - raid_gen_force(6, raid_gen6_avx2gfni_raid); - raid_recX_avx2gfni_raid(6, id, ip, nd, size, v); + raid_rec6_avx2gfni_raid(6, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1858,8 +1894,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx512gfni()) { SPEED_START { - raid_gen_force(6, raid_gen6_avx512gfni_raid); - raid_recX_avx512gfni_raid(6, id, ip, nd, size, v); + raid_rec6_avx512gfni_raid(6, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt);