Optimize multi-parity recovery with fused syndrome generation
Some checks are pending
Build / build-mac (push) Waiting to run
Build / build-linux (push) Waiting to run
Build / build-arm32 (push) Waiting to run
Build / build-win (push) Waiting to run
CodeQL Advanced / Analyze (c-cpp) (push) Waiting to run
CodeQL Advanced / Analyze (actions) (push) Waiting to run

This commit is contained in:
Andrea Mazzoleni 2026-08-23 22:28:21 +02:00
parent 5662eab6cf
commit e573e9d4cf
9 changed files with 3926 additions and 2645 deletions

View file

@ -1228,8 +1228,12 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3())
printf("%8s", "ssse3");
if (raid_cpu_has_avx2())
if (raid_cpu_has_avx2()) {
printf("%8s", "avx2");
#ifdef CONFIG_X86_64
printf("%8s", "avx2e");
#endif
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx512bw())
printf("%8s", "avx512");
@ -1257,7 +1261,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
/* ensure to use same hardware in the delta step */
/* raid_rec1of1() internally calls raid_gen() */
raid_gen_force(1, raid_gen1_neon);
raid_rec1_neon(1, id, ip, nd, size, v);
} SPEED_STOP
@ -1267,7 +1271,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* ensure to use same hardware in the delta step */
/* raid_rec1of1() internally calls raid_gen() */
raid_gen_force(1, raid_gen1_neon32);
raid_rec1_neon32(1, id, ip, nd, size, v);
} SPEED_STOP
@ -1298,9 +1302,22 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
/*
* raid_rec1_avx2ext() uses raid_rec1of1() for P,
* which internally calls raid_gen().
*/
raid_gen_force(1, raid_gen1_avx2);
raid_rec1_avx2ext(1, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
/* raid_rec1of1() internally calls raid_gen() */
raid_gen_force(1, raid_gen1_avx512bw);
raid_rec1_avx512bw(1, id, ip, nd, size, v);
} SPEED_STOP
@ -1310,7 +1327,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
/* raid_rec1of1() internally calls raid_gen() */
raid_gen_force(1, raid_gen1_avx2);
raid_rec1_avx2gfni_raid(1, id, ip, nd, size, v);
} SPEED_STOP
@ -1320,7 +1337,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
/* raid_rec1of1() internally calls raid_gen() */
raid_gen_force(1, raid_gen1_avx512bw); /* there is no raid_gen1_avx512gfni */
raid_rec1_avx512gfni_raid(1, id, ip, nd, size, v);
} SPEED_STOP
@ -1349,8 +1366,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, raid_gen1_neon);
/* +1 to avoid GEN1 optimized case */
raid_rec1_neon(1, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1360,8 +1375,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, raid_gen1_neon32);
/* +1 to avoid GEN1 optimized case */
raid_rec1_neon32(1, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1394,11 +1407,18 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
/* +1 to avoid RAID5 optimized case */
raid_rec1_avx2ext(1, id, ip + 1, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, raid_gen1_avx512bw);
/* +1 to avoid GEN1 optimized case */
/* +1 to avoid RAID5 optimized case */
raid_rec1_avx512bw(1, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1407,9 +1427,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, raid_gen1_avx2);
/* +1 to avoid GEN1 optimized case */
/* +1 to avoid RAID5 optimized case */
raid_rec1_avx2gfni_raid(1, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1418,9 +1436,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, raid_gen1_avx512bw); /* there is no raid_gen1_avx512gfni */
/* +1 to avoid GEN1 optimized case */
/* +1 to avoid RAID5 optimized case */
raid_rec1_avx512gfni_raid(1, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1447,8 +1463,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_neon_raid);
raid_rec2_neon(2, id, ip, nd, size, v);
} SPEED_STOP
@ -1457,8 +1471,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_neon32_raid);
raid_rec2_neon32(2, id, ip, nd, size, v);
} SPEED_STOP
@ -1496,10 +1508,16 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
raid_rec2_avx2ext(2, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_avx512bw);
raid_rec2_avx512bw(2, id, ip, nd, size, v);
} SPEED_STOP
@ -1508,8 +1526,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_avx2gfni_raid);
raid_rec2_avx2gfni_raid(2, id, ip, nd, size, v);
} SPEED_STOP
@ -1518,8 +1534,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_avx512gfni_raid);
raid_rec2_avx512gfni_raid(2, id, ip, nd, size, v);
} SPEED_STOP
@ -1547,8 +1561,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_neon_raid);
/* +1 to avoid GEN2 optimized case */
raid_rec2_neon(2, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1558,8 +1570,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_neon32_raid);
/* +1 to avoid GEN2 optimized case */
raid_rec2_neon32(2, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1600,10 +1610,17 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
/* +1 to avoid GEN2 optimized case */
raid_rec2_avx2ext(2, id, ip + 1, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_avx512bw);
/* +1 to avoid GEN2 optimized case */
raid_rec2_avx512bw(2, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1613,8 +1630,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_avx2gfni_raid);
/* +1 to avoid GEN2 optimized case */
raid_rec2_avx2gfni_raid(2, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1624,8 +1639,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_avx512gfni_raid);
/* +1 to avoid GEN2 optimized case */
raid_rec2_avx512gfni_raid(2, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1652,9 +1665,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(3, raid_gen3_neon_raid);
raid_recX_neon(3, id, ip, nd, size, v);
raid_rec3_neon(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1662,9 +1673,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(3, raid_gen3_neon32_raid);
raid_recX_neon32(3, id, ip, nd, size, v);
raid_rec3_neon32(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1694,19 +1703,24 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#else
raid_gen_force(3, raid_gen3_ssse3_raid);
#endif
raid_recX_avx2(3, id, ip, nd, size, v);
raid_rec3_avx2(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
raid_rec3_avx2ext(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(3, raid_gen3_avx512bw);
/* +1 to avoid GEN1 optimized case */
raid_recX_avx512bw(3, id, ip, nd, size, v);
raid_rec3_avx512bw(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1714,9 +1728,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(3, raid_gen3_avx2gfni_raid);
raid_recX_avx2gfni_raid(3, id, ip, nd, size, v);
raid_rec3_avx2gfni_raid(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1724,9 +1736,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(3, raid_gen3_avx512gfni_raid);
raid_recX_avx512gfni_raid(3, id, ip, nd, size, v);
raid_rec3_avx512gfni_raid(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1751,9 +1761,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(4, raid_gen4_neon_raid);
raid_recX_neon(4, id, ip, nd, size, v);
raid_rec4_neon(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1761,9 +1769,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(4, raid_gen4_neon32_raid);
raid_recX_neon32(4, id, ip, nd, size, v);
raid_rec4_neon32(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1794,19 +1800,24 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#else
raid_gen_force(4, raid_gen4_ssse3_raid);
#endif
raid_recX_avx2(4, id, ip, nd, size, v);
raid_rec4_avx2(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
raid_rec4_avx2ext(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(4, raid_gen4_avx512bw);
/* +1 to avoid GEN1 optimized case */
raid_recX_avx512bw(4, id, ip, nd, size, v);
raid_rec4_avx512bw(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1814,9 +1825,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(4, raid_gen4_avx2gfni_raid);
raid_recX_avx2gfni_raid(4, id, ip, nd, size, v);
raid_rec4_avx2gfni_raid(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1824,9 +1833,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(4, raid_gen4_avx512gfni_raid);
raid_recX_avx512gfni_raid(4, id, ip, nd, size, v);
raid_rec4_avx512gfni_raid(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1851,9 +1858,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(5, raid_gen5_neon_raid);
raid_recX_neon(5, id, ip, nd, size, v);
raid_rec5_neon(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1861,9 +1866,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(5, raid_gen5_neon32_raid);
raid_recX_neon32(5, id, ip, nd, size, v);
raid_rec5_neon32(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1893,19 +1896,24 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#else
raid_gen_force(5, raid_gen5_ssse3_raid);
#endif
raid_recX_avx2(5, id, ip, nd, size, v);
raid_rec5_avx2(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
raid_rec5_avx2ext(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(5, raid_gen5_avx512bw);
/* +1 to avoid GEN1 optimized case */
raid_recX_avx512bw(5, id, ip, nd, size, v);
raid_rec5_avx512bw(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1913,9 +1921,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(5, raid_gen5_avx2gfni_raid);
raid_recX_avx2gfni_raid(5, id, ip, nd, size, v);
raid_rec5_avx2gfni_raid(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1923,9 +1929,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(5, raid_gen5_avx512gfni_raid);
raid_recX_avx512gfni_raid(5, id, ip, nd, size, v);
raid_rec5_avx512gfni_raid(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1950,9 +1954,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(6, raid_gen6_neon_raid);
raid_recX_neon(6, id, ip, nd, size, v);
raid_rec6_neon(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1960,9 +1962,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(6, raid_gen6_neon32_raid);
raid_recX_neon32(6, id, ip, nd, size, v);
raid_rec6_neon32(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1992,19 +1992,24 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#else
raid_gen_force(6, raid_gen6_ssse3_raid);
#endif
raid_recX_avx2(6, id, ip, nd, size, v);
raid_rec6_avx2(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
raid_rec6_avx2ext(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(6, raid_gen6_avx512bw);
/* +1 to avoid GEN1 optimized case */
raid_recX_avx512bw(6, id, ip, nd, size, v);
raid_rec6_avx512bw(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -2012,9 +2017,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(6, raid_gen6_avx2gfni_raid);
raid_recX_avx2gfni_raid(6, id, ip, nd, size, v);
raid_rec6_avx2gfni_raid(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -2022,9 +2025,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(6, raid_gen6_avx512gfni_raid);
raid_recX_avx512gfni_raid(6, id, ip, nd, size, v);
raid_rec6_avx512gfni_raid(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);