rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 19:01:57 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Optimize multi-parity recovery with fused syndrome generation

Browse source
This commit is contained in:
Andrea Mazzoleni 2026-08-23 22:28:21 +02:00
parent 9d842b0dfb
commit ae703ce68c
12 changed files with 6607 additions and 3934 deletions
Download patch file Download diff file Expand all files Collapse all files

5
raid/test/Makefile
View file

@ -103,6 +103,11 @@ sdecovtest:
# Use this target to run a complete test on Ice Lake simulator
sdetest:
$(SDE) -p4p -- ./selftest
$(SDE) -mrm -- ./selftest
$(SDE) -nhm -- ./selftest
$(SDE) -hsw -- ./selftest
$(SDE) -skx -- ./selftest
$(SDE) -icl -- ./selftest
$(SDE) -icl -- ./fulltest

552
raid/test/speedtest.c
View file

@ -1122,10 +1122,18 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
printf("%8s", "neon32");
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3())
if (raid_cpu_has_ssse3()) {
printf("%8s", "ssse3");
if (raid_cpu_has_avx2())
#ifdef CONFIG_X86_64
printf("%8s", "ssse3e");
#endif
}
if (raid_cpu_has_avx2()) {
printf("%8s", "avx2");
#ifdef CONFIG_X86_64
printf("%8s", "avx2e");
#endif
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx512bw())
printf("%8s", "avx512");
@ -1172,12 +1180,24 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
SPEED_START {
/* raid_rec1of1() internally calls raid_gen() */
raid_gen_force(1, raid_gen1_sse2);
raid_rec1_ssse3(1, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
/* raid_rec1of1() internally calls raid_gen() */
raid_gen_force(1, raid_gen1_sse2);
raid_rec1_ssse3ext(1, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
@ -1189,6 +1209,19 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
/*
* raid_rec1_avx2ext() uses raid_rec1of1() for P,
* which internally calls raid_gen().
*/
raid_gen_force(1, raid_gen1_avx2);
raid_rec1_avx2ext(1, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
raid_gen_force(1, raid_gen1_avx512bw);
@ -1236,7 +1269,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
raid_gen_force(1, raid_gen1_neon);
/* +1 to avoid GEN1 optimized case */
raid_rec1_neon(1, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1246,7 +1278,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen_force(1, raid_gen1_neon32);
/* +1 to avoid GEN1 optimized case */
raid_rec1_neon32(1, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1258,6 +1289,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, raid_gen1_sse2);
/* +1 to avoid GEN1 optimized case */
raid_rec1_ssse3(1, id, ip + 1, nd, size, v);
@ -1265,6 +1297,18 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, raid_gen1_sse2);
/* +1 to avoid GEN1 optimized case */
raid_rec1_ssse3ext(1, id, ip + 1, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
@ -1277,10 +1321,20 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, raid_gen1_avx2);
/* +1 to avoid GEN1 optimized case */
raid_rec1_avx2ext(1, id, ip + 1, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
raid_gen_force(1, raid_gen1_avx512bw);
/* +1 to avoid GEN1 optimized case */
/* +1 to avoid RAID5 optimized case */
raid_rec1_avx512bw(1, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1289,8 +1343,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
raid_gen_force(1, raid_gen1_avx2);
/* +1 to avoid GEN1 optimized case */
/* +1 to avoid RAID5 optimized case */
raid_rec1_avx2gfni_raid(1, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1299,8 +1352,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
raid_gen_force(1, raid_gen1_avx512bw); /* there is no raid_gen1_avx512gfni */
/* +1 to avoid GEN1 optimized case */
/* +1 to avoid RAID5 optimized case */
raid_rec1_avx512gfni_raid(1, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1311,6 +1363,116 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#endif
printf("\n");
/* REC1OF3 */
printf("%8s", "rec1of3");
printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR1));
fflush(stdout);
SPEED_START {
raid_gen_force(1, sizeof(void *) == 8 ? raid_gen1_int64 : raid_gen1_int32);
/* +2 to select R instead of Q */
raid_rec1_int8(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_NEON
SPEED_START {
/* +2 to select R instead of Q */
raid_rec1_neon(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* +2 to select R instead of Q */
raid_rec1_neon32(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, raid_gen1_sse2);
/* +2 to select R instead of Q */
raid_rec1_ssse3(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, raid_gen1_sse2);
/* +2 to select R instead of Q */
raid_rec1_ssse3ext(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
raid_gen_force(1, raid_gen1_avx2);
/* +2 to select R instead of Q */
raid_rec1_avx2(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, raid_gen1_avx2);
/* +2 to select R instead of Q */
raid_rec1_avx2ext(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
/* +2 to select R instead of Q */
raid_rec1_avx512bw(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
/* +2 to select R instead of Q */
raid_rec1_avx2gfni_raid(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
/* +2 to select R instead of Q */
raid_rec1_avx512gfni_raid(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#endif
#endif
printf("\n");
/* REC2OF2 */
printf("%8s", "rec2of2");
printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR2));
@ -1326,7 +1488,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
raid_gen_force(2, raid_gen2_neon_raid);
raid_rec2_neon(2, id, ip, nd, size, v);
} SPEED_STOP
@ -1335,7 +1496,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen_force(2, raid_gen2_neon32_raid);
raid_rec2_neon32(2, id, ip, nd, size, v);
} SPEED_STOP
@ -1346,24 +1506,28 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
SPEED_START {
#ifdef CONFIG_X86_64
raid_gen_force(2, raid_gen2_sse2ext_raid);
#else
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_sse2_raid);
#endif
raid_rec2_ssse3(2, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_sse2ext_raid);
raid_rec2_ssse3ext(2, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
#ifdef CONFIG_X86_64
raid_gen_force(2, raid_gen2_avx2ext_raid);
#else
raid_gen_force(2, raid_gen2_avx2_raid);
#endif
raid_rec2_avx2(2, id, ip, nd, size, v);
} SPEED_STOP
@ -1371,9 +1535,18 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_avx2ext_raid);
raid_rec2_avx2ext(2, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
raid_gen_force(2, raid_gen2_avx512bw);
raid_rec2_avx512bw(2, id, ip, nd, size, v);
} SPEED_STOP
@ -1382,7 +1555,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
raid_gen_force(2, raid_gen2_avx2gfni_raid);
raid_rec2_avx2gfni_raid(2, id, ip, nd, size, v);
} SPEED_STOP
@ -1391,7 +1563,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
raid_gen_force(2, raid_gen2_avx512gfni_raid);
raid_rec2_avx512gfni_raid(2, id, ip, nd, size, v);
} SPEED_STOP
@ -1418,7 +1589,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
raid_gen_force(2, raid_gen2_neon_raid);
/* +1 to avoid GEN2 optimized case */
raid_rec2_neon(2, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1428,7 +1598,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen_force(2, raid_gen2_neon32_raid);
/* +1 to avoid GEN2 optimized case */
raid_rec2_neon32(2, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1440,25 +1609,30 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
SPEED_START {
#ifdef CONFIG_X86_64
raid_gen_force(2, raid_gen2_sse2ext_raid);
#else
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_sse2_raid);
#endif
/* +1 to avoid GEN2 optimized case */
raid_rec2_ssse3(2, id, ip + 1, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_sse2ext_raid);
/* +1 to avoid GEN2 optimized case */
raid_rec2_ssse3ext(2, id, ip + 1, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
#ifdef CONFIG_X86_64
raid_gen_force(2, raid_gen2_avx2ext_raid);
#else
raid_gen_force(2, raid_gen2_avx2_raid);
#endif
/* +1 to avoid GEN2 optimized case */
raid_rec2_avx2(2, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1467,9 +1641,19 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_avx2ext_raid);
/* +1 to avoid GEN2 optimized case */
raid_rec2_avx2ext(2, id, ip + 1, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
raid_gen_force(2, raid_gen2_avx512bw);
/* +1 to avoid GEN2 optimized case */
raid_rec2_avx512bw(2, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1479,7 +1663,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
raid_gen_force(2, raid_gen2_avx2gfni_raid);
/* +1 to avoid GEN2 optimized case */
raid_rec2_avx2gfni_raid(2, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1489,7 +1672,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
raid_gen_force(2, raid_gen2_avx512gfni_raid);
/* +1 to avoid GEN2 optimized case */
raid_rec2_avx512gfni_raid(2, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1501,6 +1683,116 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#endif
printf("\n");
/* REC2OF4 */
printf("%8s", "rec2of4");
printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR2));
fflush(stdout);
SPEED_START {
raid_gen_force(2, sizeof(void *) == 8 ? raid_gen2_int64_raid : raid_gen2_int32_raid);
/* +2 to select R,S instead of Q,R */
raid_rec2_int8(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_NEON
SPEED_START {
/* +2 to select R,S instead of Q,R */
raid_rec2_neon(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* +2 to select R,S instead of Q,R */
raid_rec2_neon32(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_sse2_raid);
/* +2 to select R,S instead of Q,R */
raid_rec2_ssse3(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_sse2ext_raid);
/* +2 to select R,S instead of Q,R */
raid_rec2_ssse3ext(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
raid_gen_force(2, raid_gen2_avx2_raid);
/* +2 to select R,S instead of Q,R */
raid_rec2_avx2(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_avx2ext_raid);
/* +2 to select R,S instead of Q,R */
raid_rec2_avx2ext(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
/* +2 to select R,S instead of Q,R */
raid_rec2_avx512bw(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
/* +2 to select R,S instead of Q,R */
raid_rec2_avx2gfni_raid(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
/* +2 to select R,S instead of Q,R */
raid_rec2_avx512gfni_raid(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#endif
#endif
printf("\n");
/* REC3OF3 */
printf("%8s", "rec3of3");
printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR3));
@ -1516,8 +1808,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
raid_gen_force(3, raid_gen3_neon_raid);
raid_recX_neon(3, id, ip, nd, size, v);
raid_rec3_neon(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1525,8 +1816,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen_force(3, raid_gen3_neon32_raid);
raid_recX_neon32(3, id, ip, nd, size, v);
raid_rec3_neon32(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1536,36 +1826,41 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
SPEED_START {
#ifdef CONFIG_X86_64
raid_gen_force(3, raid_gen3_ssse3ext_raid);
#else
raid_gen_force(3, raid_gen3_ssse3_raid);
#endif
raid_recX_ssse3(3, id, ip, nd, size, v);
raid_rec3_ssse3(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
raid_rec3_ssse3ext(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
#ifdef CONFIG_X86_64
raid_gen_force(3, raid_gen3_avx2ext_raid);
#else
raid_gen_force(3, raid_gen3_ssse3_raid);
#endif
raid_recX_avx2(3, id, ip, nd, size, v);
raid_rec3_avx2(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
raid_rec3_avx2ext(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
raid_gen_force(3, raid_gen3_avx512bw);
/* +1 to avoid GEN1 optimized case */
raid_recX_avx512bw(3, id, ip, nd, size, v);
raid_rec3_avx512bw(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1573,8 +1868,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
raid_gen_force(3, raid_gen3_avx2gfni_raid);
raid_recX_avx2gfni_raid(3, id, ip, nd, size, v);
raid_rec3_avx2gfni_raid(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1582,8 +1876,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
raid_gen_force(3, raid_gen3_avx512gfni_raid);
raid_recX_avx512gfni_raid(3, id, ip, nd, size, v);
raid_rec3_avx512gfni_raid(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1608,8 +1901,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
raid_gen_force(4, raid_gen4_neon_raid);
raid_recX_neon(4, id, ip, nd, size, v);
raid_rec4_neon(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1617,8 +1909,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen_force(4, raid_gen4_neon32_raid);
raid_recX_neon32(4, id, ip, nd, size, v);
raid_rec4_neon32(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1628,36 +1919,41 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
SPEED_START {
#ifdef CONFIG_X86_64
raid_gen_force(4, raid_gen4_ssse3ext_raid);
#else
raid_gen_force(4, raid_gen4_ssse3_raid);
#endif
raid_recX_ssse3(4, id, ip, nd, size, v);
raid_rec4_ssse3(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
raid_rec4_ssse3ext(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
#ifdef CONFIG_X86_64
raid_gen_force(4, raid_gen4_avx2ext_raid);
#else
raid_gen_force(4, raid_gen4_ssse3_raid);
#endif
raid_recX_avx2(4, id, ip, nd, size, v);
raid_rec4_avx2(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
raid_rec4_avx2ext(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
raid_gen_force(4, raid_gen4_avx512bw);
/* +1 to avoid GEN1 optimized case */
raid_recX_avx512bw(4, id, ip, nd, size, v);
raid_rec4_avx512bw(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1665,8 +1961,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
raid_gen_force(4, raid_gen4_avx2gfni_raid);
raid_recX_avx2gfni_raid(4, id, ip, nd, size, v);
raid_rec4_avx2gfni_raid(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1674,8 +1969,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
raid_gen_force(4, raid_gen4_avx512gfni_raid);
raid_recX_avx512gfni_raid(4, id, ip, nd, size, v);
raid_rec4_avx512gfni_raid(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1700,8 +1994,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
raid_gen_force(5, raid_gen5_neon_raid);
raid_recX_neon(5, id, ip, nd, size, v);
raid_rec5_neon(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1709,8 +2002,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen_force(5, raid_gen5_neon32_raid);
raid_recX_neon32(5, id, ip, nd, size, v);
raid_rec5_neon32(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1720,36 +2012,41 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
SPEED_START {
#ifdef CONFIG_X86_64
raid_gen_force(5, raid_gen5_ssse3ext_raid);
#else
raid_gen_force(5, raid_gen5_ssse3_raid);
#endif
raid_recX_ssse3(5, id, ip, nd, size, v);
raid_rec5_ssse3(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
raid_rec5_ssse3ext(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
#ifdef CONFIG_X86_64
raid_gen_force(5, raid_gen5_avx2ext_raid);
#else
raid_gen_force(5, raid_gen5_ssse3_raid);
#endif
raid_recX_avx2(5, id, ip, nd, size, v);
raid_rec5_avx2(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
raid_rec5_avx2ext(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
raid_gen_force(5, raid_gen5_avx512bw);
/* +1 to avoid GEN1 optimized case */
raid_recX_avx512bw(5, id, ip, nd, size, v);
raid_rec5_avx512bw(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1757,8 +2054,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
raid_gen_force(5, raid_gen5_avx2gfni_raid);
raid_recX_avx2gfni_raid(5, id, ip, nd, size, v);
raid_rec5_avx2gfni_raid(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1766,8 +2062,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
raid_gen_force(5, raid_gen5_avx512gfni_raid);
raid_recX_avx512gfni_raid(5, id, ip, nd, size, v);
raid_rec5_avx512gfni_raid(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1792,8 +2087,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
raid_gen_force(6, raid_gen6_neon_raid);
raid_recX_neon(6, id, ip, nd, size, v);
raid_rec6_neon(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1801,8 +2095,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen_force(6, raid_gen6_neon32_raid);
raid_recX_neon32(6, id, ip, nd, size, v);
raid_rec6_neon32(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1812,36 +2105,41 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
SPEED_START {
#ifdef CONFIG_X86_64
raid_gen_force(6, raid_gen6_ssse3ext_raid);
#else
raid_gen_force(6, raid_gen6_ssse3_raid);
#endif
raid_recX_ssse3(6, id, ip, nd, size, v);
raid_rec6_ssse3(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
raid_rec6_ssse3ext(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
#ifdef CONFIG_X86_64
raid_gen_force(6, raid_gen6_avx2ext_raid);
#else
raid_gen_force(6, raid_gen6_ssse3_raid);
#endif
raid_recX_avx2(6, id, ip, nd, size, v);
raid_rec6_avx2(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
raid_rec6_avx2ext(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
raid_gen_force(6, raid_gen6_avx512bw);
/* +1 to avoid GEN1 optimized case */
raid_recX_avx512bw(6, id, ip, nd, size, v);
raid_rec6_avx512bw(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1849,8 +2147,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
raid_gen_force(6, raid_gen6_avx2gfni_raid);
raid_recX_avx2gfni_raid(6, id, ip, nd, size, v);
raid_rec6_avx2gfni_raid(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1858,8 +2155,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
raid_gen_force(6, raid_gen6_avx512gfni_raid);
raid_recX_avx512gfni_raid(6, id, ip, nd, size, v);
raid_rec6_avx512gfni_raid(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);