Optimize multi-parity recovery with fused syndrome generation
Some checks are pending
Build / build-mac (push) Waiting to run
Build / build-linux (push) Waiting to run
Build / build-arm32 (push) Waiting to run
Build / build-win (push) Waiting to run
CodeQL Advanced / Analyze (c-cpp) (push) Waiting to run
CodeQL Advanced / Analyze (actions) (push) Waiting to run

This commit is contained in:
Andrea Mazzoleni 2026-08-23 22:28:21 +02:00
parent 5662eab6cf
commit e573e9d4cf
9 changed files with 3926 additions and 2645 deletions

View file

@ -1124,8 +1124,12 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3())
printf("%8s", "ssse3");
if (raid_cpu_has_avx2())
if (raid_cpu_has_avx2()) {
printf("%8s", "avx2");
#ifdef CONFIG_X86_64
printf("%8s", "avx2e");
#endif
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx512bw())
printf("%8s", "avx512");
@ -1189,6 +1193,19 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
/*
* raid_rec1_avx2ext() uses raid_rec1of1() for P,
* which internally calls raid_gen().
*/
raid_gen_force(1, raid_gen1_avx2);
raid_rec1_avx2ext(1, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
raid_gen_force(1, raid_gen1_avx512bw);
@ -1236,7 +1253,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
raid_gen_force(1, raid_gen1_neon);
/* +1 to avoid GEN1 optimized case */
raid_rec1_neon(1, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1246,7 +1262,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen_force(1, raid_gen1_neon32);
/* +1 to avoid GEN1 optimized case */
raid_rec1_neon32(1, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1277,10 +1292,18 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
/* +1 to avoid RAID5 optimized case */
raid_rec1_avx2ext(1, id, ip + 1, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
raid_gen_force(1, raid_gen1_avx512bw);
/* +1 to avoid GEN1 optimized case */
/* +1 to avoid RAID5 optimized case */
raid_rec1_avx512bw(1, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1289,8 +1312,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
raid_gen_force(1, raid_gen1_avx2);
/* +1 to avoid GEN1 optimized case */
/* +1 to avoid RAID5 optimized case */
raid_rec1_avx2gfni_raid(1, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1299,8 +1321,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
raid_gen_force(1, raid_gen1_avx512bw); /* there is no raid_gen1_avx512gfni */
/* +1 to avoid GEN1 optimized case */
/* +1 to avoid RAID5 optimized case */
raid_rec1_avx512gfni_raid(1, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1326,7 +1347,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
raid_gen_force(2, raid_gen2_neon_raid);
raid_rec2_neon(2, id, ip, nd, size, v);
} SPEED_STOP
@ -1335,7 +1355,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen_force(2, raid_gen2_neon32_raid);
raid_rec2_neon32(2, id, ip, nd, size, v);
} SPEED_STOP
@ -1371,9 +1390,16 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
raid_rec2_avx2ext(2, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
raid_gen_force(2, raid_gen2_avx512bw);
raid_rec2_avx512bw(2, id, ip, nd, size, v);
} SPEED_STOP
@ -1382,7 +1408,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
raid_gen_force(2, raid_gen2_avx2gfni_raid);
raid_rec2_avx2gfni_raid(2, id, ip, nd, size, v);
} SPEED_STOP
@ -1391,7 +1416,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
raid_gen_force(2, raid_gen2_avx512gfni_raid);
raid_rec2_avx512gfni_raid(2, id, ip, nd, size, v);
} SPEED_STOP
@ -1418,7 +1442,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
raid_gen_force(2, raid_gen2_neon_raid);
/* +1 to avoid GEN2 optimized case */
raid_rec2_neon(2, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1428,7 +1451,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen_force(2, raid_gen2_neon32_raid);
/* +1 to avoid GEN2 optimized case */
raid_rec2_neon32(2, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1467,9 +1489,16 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
raid_rec2_avx2ext(2, id, ip + 1, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
raid_gen_force(2, raid_gen2_avx512bw);
/* +1 to avoid GEN2 optimized case */
raid_rec2_avx512bw(2, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1479,7 +1508,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
raid_gen_force(2, raid_gen2_avx2gfni_raid);
/* +1 to avoid GEN2 optimized case */
raid_rec2_avx2gfni_raid(2, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1489,7 +1517,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
raid_gen_force(2, raid_gen2_avx512gfni_raid);
/* +1 to avoid GEN2 optimized case */
raid_rec2_avx512gfni_raid(2, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1516,8 +1543,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
raid_gen_force(3, raid_gen3_neon_raid);
raid_recX_neon(3, id, ip, nd, size, v);
raid_rec3_neon(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1525,8 +1551,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen_force(3, raid_gen3_neon32_raid);
raid_recX_neon32(3, id, ip, nd, size, v);
raid_rec3_neon32(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1554,18 +1579,24 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#else
raid_gen_force(3, raid_gen3_ssse3_raid);
#endif
raid_recX_avx2(3, id, ip, nd, size, v);
raid_rec3_avx2(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
raid_rec3_avx2ext(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
raid_gen_force(3, raid_gen3_avx512bw);
/* +1 to avoid GEN1 optimized case */
raid_recX_avx512bw(3, id, ip, nd, size, v);
raid_rec3_avx512bw(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1573,8 +1604,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
raid_gen_force(3, raid_gen3_avx2gfni_raid);
raid_recX_avx2gfni_raid(3, id, ip, nd, size, v);
raid_rec3_avx2gfni_raid(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1582,8 +1612,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
raid_gen_force(3, raid_gen3_avx512gfni_raid);
raid_recX_avx512gfni_raid(3, id, ip, nd, size, v);
raid_rec3_avx512gfni_raid(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1608,8 +1637,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
raid_gen_force(4, raid_gen4_neon_raid);
raid_recX_neon(4, id, ip, nd, size, v);
raid_rec4_neon(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1617,8 +1645,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen_force(4, raid_gen4_neon32_raid);
raid_recX_neon32(4, id, ip, nd, size, v);
raid_rec4_neon32(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1646,18 +1673,24 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#else
raid_gen_force(4, raid_gen4_ssse3_raid);
#endif
raid_recX_avx2(4, id, ip, nd, size, v);
raid_rec4_avx2(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
raid_rec4_avx2ext(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
raid_gen_force(4, raid_gen4_avx512bw);
/* +1 to avoid GEN1 optimized case */
raid_recX_avx512bw(4, id, ip, nd, size, v);
raid_rec4_avx512bw(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1665,8 +1698,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
raid_gen_force(4, raid_gen4_avx2gfni_raid);
raid_recX_avx2gfni_raid(4, id, ip, nd, size, v);
raid_rec4_avx2gfni_raid(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1674,8 +1706,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
raid_gen_force(4, raid_gen4_avx512gfni_raid);
raid_recX_avx512gfni_raid(4, id, ip, nd, size, v);
raid_rec4_avx512gfni_raid(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1700,8 +1731,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
raid_gen_force(5, raid_gen5_neon_raid);
raid_recX_neon(5, id, ip, nd, size, v);
raid_rec5_neon(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1709,8 +1739,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen_force(5, raid_gen5_neon32_raid);
raid_recX_neon32(5, id, ip, nd, size, v);
raid_rec5_neon32(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1738,18 +1767,24 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#else
raid_gen_force(5, raid_gen5_ssse3_raid);
#endif
raid_recX_avx2(5, id, ip, nd, size, v);
raid_rec5_avx2(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
raid_rec5_avx2ext(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
raid_gen_force(5, raid_gen5_avx512bw);
/* +1 to avoid GEN1 optimized case */
raid_recX_avx512bw(5, id, ip, nd, size, v);
raid_rec5_avx512bw(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1757,8 +1792,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
raid_gen_force(5, raid_gen5_avx2gfni_raid);
raid_recX_avx2gfni_raid(5, id, ip, nd, size, v);
raid_rec5_avx2gfni_raid(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1766,8 +1800,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
raid_gen_force(5, raid_gen5_avx512gfni_raid);
raid_recX_avx512gfni_raid(5, id, ip, nd, size, v);
raid_rec5_avx512gfni_raid(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1792,8 +1825,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
raid_gen_force(6, raid_gen6_neon_raid);
raid_recX_neon(6, id, ip, nd, size, v);
raid_rec6_neon(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1801,8 +1833,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen_force(6, raid_gen6_neon32_raid);
raid_recX_neon32(6, id, ip, nd, size, v);
raid_rec6_neon32(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1830,18 +1861,24 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#else
raid_gen_force(6, raid_gen6_ssse3_raid);
#endif
raid_recX_avx2(6, id, ip, nd, size, v);
raid_rec6_avx2(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
raid_rec6_avx2ext(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
raid_gen_force(6, raid_gen6_avx512bw);
/* +1 to avoid GEN1 optimized case */
raid_recX_avx512bw(6, id, ip, nd, size, v);
raid_rec6_avx512bw(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1849,8 +1886,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
raid_gen_force(6, raid_gen6_avx2gfni_raid);
raid_recX_avx2gfni_raid(6, id, ip, nd, size, v);
raid_rec6_avx2gfni_raid(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1858,8 +1894,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
raid_gen_force(6, raid_gen6_avx512gfni_raid);
raid_recX_avx512gfni_raid(6, id, ip, nd, size, v);
raid_rec6_avx512gfni_raid(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);