rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 10:51:54 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Optimize multi-parity recovery with fused syndrome generation

Browse source
This commit is contained in:
Andrea Mazzoleni 2026-08-23 22:28:21 +02:00
parent 9d842b0dfb
commit ae703ce68c
12 changed files with 6607 additions and 3934 deletions
Download patch file Download diff file Expand all files Collapse all files

606
cmdline/speed.c
View file

@ -1226,10 +1226,18 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
printf("%8s", "neon32");
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3())
if (raid_cpu_has_ssse3()) {
printf("%8s", "ssse3");
if (raid_cpu_has_avx2())
#ifdef CONFIG_X86_64
printf("%8s", "ssse3e");
#endif
}
if (raid_cpu_has_avx2()) {
printf("%8s", "avx2");
#ifdef CONFIG_X86_64
printf("%8s", "avx2e");
#endif
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx512bw())
printf("%8s", "avx512");
@ -1257,7 +1265,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
/* ensure to use same hardware in the delta step */
/* raid_rec1of1() internally calls raid_gen() */
raid_gen_force(1, raid_gen1_neon);
raid_rec1_neon(1, id, ip, nd, size, v);
} SPEED_STOP
@ -1267,7 +1275,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* ensure to use same hardware in the delta step */
/* raid_rec1of1() internally calls raid_gen() */
raid_gen_force(1, raid_gen1_neon32);
raid_rec1_neon32(1, id, ip, nd, size, v);
} SPEED_STOP
@ -1279,13 +1287,24 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
/* raid_rec1of1() internally calls raid_gen() */
raid_gen_force(1, raid_gen1_sse2);
raid_rec1_ssse3(1, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
/* raid_rec1of1() internally calls raid_gen() */
raid_gen_force(1, raid_gen1_sse2);
raid_rec1_ssse3ext(1, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
@ -1298,9 +1317,22 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
/*
* raid_rec1_avx2ext() uses raid_rec1of1() for P,
* which internally calls raid_gen().
*/
raid_gen_force(1, raid_gen1_avx2);
raid_rec1_avx2ext(1, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
/* raid_rec1of1() internally calls raid_gen() */
raid_gen_force(1, raid_gen1_avx512bw);
raid_rec1_avx512bw(1, id, ip, nd, size, v);
} SPEED_STOP
@ -1310,7 +1342,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
/* raid_rec1of1() internally calls raid_gen() */
raid_gen_force(1, raid_gen1_avx2);
raid_rec1_avx2gfni_raid(1, id, ip, nd, size, v);
} SPEED_STOP
@ -1320,7 +1352,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
/* raid_rec1of1() internally calls raid_gen() */
raid_gen_force(1, raid_gen1_avx512bw); /* there is no raid_gen1_avx512gfni */
raid_rec1_avx512gfni_raid(1, id, ip, nd, size, v);
} SPEED_STOP
@ -1349,8 +1381,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, raid_gen1_neon);
/* +1 to avoid GEN1 optimized case */
raid_rec1_neon(1, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1360,8 +1390,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, raid_gen1_neon32);
/* +1 to avoid GEN1 optimized case */
raid_rec1_neon32(1, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1381,6 +1409,18 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, raid_gen1_sse2);
/* +1 to avoid GEN1 optimized case */
raid_rec1_ssse3ext(1, id, ip + 1, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
@ -1394,11 +1434,20 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx512bw()) {
if (raid_cpu_has_avx2()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, raid_gen1_avx512bw);
raid_gen_force(1, raid_gen1_avx2);
/* +1 to avoid GEN1 optimized case */
raid_rec1_avx2ext(1, id, ip + 1, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
/* +1 to avoid RAID5 optimized case */
raid_rec1_avx512bw(1, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1407,9 +1456,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, raid_gen1_avx2);
/* +1 to avoid GEN1 optimized case */
/* +1 to avoid RAID5 optimized case */
raid_rec1_avx2gfni_raid(1, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1418,9 +1465,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, raid_gen1_avx512bw); /* there is no raid_gen1_avx512gfni */
/* +1 to avoid GEN1 optimized case */
/* +1 to avoid RAID5 optimized case */
raid_rec1_avx512gfni_raid(1, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1431,6 +1476,118 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#endif
printf("\n");
/* REC1OF3 */
printf("%8s", "rec1of3");
printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR1));
fflush(stdout);
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, sizeof(void*) == 8 ? raid_gen1_int64 : raid_gen1_int32);
/* +2 to select R instead of Q */
raid_rec1_int8(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_NEON
SPEED_START {
/* +2 to select R instead of Q */
raid_rec1_neon(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* +2 to select R instead of Q */
raid_rec1_neon32(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, raid_gen1_sse2);
/* +2 to select R instead of Q */
raid_rec1_ssse3(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, raid_gen1_sse2);
/* +2 to select R instead of Q */
raid_rec1_ssse3ext(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, raid_gen1_avx2);
/* +2 to select R instead of Q */
raid_rec1_avx2(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, raid_gen1_avx2);
/* +2 to select R instead of Q */
raid_rec1_avx2ext(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
/* +2 to select R instead of Q */
raid_rec1_avx512bw(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
/* +2 to select R instead of Q */
raid_rec1_avx2gfni_raid(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
/* +2 to select R instead of Q */
raid_rec1_avx512gfni_raid(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#endif
#endif
printf("\n");
/* REC2OF2 */
printf("%8s", "rec2of2");
printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR2));
@ -1447,8 +1604,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_neon_raid);
raid_rec2_neon(2, id, ip, nd, size, v);
} SPEED_STOP
@ -1457,8 +1612,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_neon32_raid);
raid_rec2_neon32(2, id, ip, nd, size, v);
} SPEED_STOP
@ -1470,25 +1623,28 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
if (raid_cpu_has_ssse3()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
#ifdef CONFIG_X86_64
raid_gen_force(2, raid_gen2_sse2ext_raid);
#else
raid_gen_force(2, raid_gen2_sse2_raid);
#endif
raid_rec2_ssse3(2, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_sse2ext_raid);
raid_rec2_ssse3ext(2, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
#ifdef CONFIG_X86_64
raid_gen_force(2, raid_gen2_avx2ext_raid);
#else
raid_gen_force(2, raid_gen2_avx2_raid);
#endif
raid_rec2_avx2(2, id, ip, nd, size, v);
} SPEED_STOP
@ -1496,10 +1652,18 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx512bw()) {
if (raid_cpu_has_avx2()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_avx512bw);
raid_gen_force(2, raid_gen2_avx2ext_raid);
raid_rec2_avx2ext(2, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
raid_rec2_avx512bw(2, id, ip, nd, size, v);
} SPEED_STOP
@ -1508,8 +1672,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_avx2gfni_raid);
raid_rec2_avx2gfni_raid(2, id, ip, nd, size, v);
} SPEED_STOP
@ -1518,8 +1680,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_avx512gfni_raid);
raid_rec2_avx512gfni_raid(2, id, ip, nd, size, v);
} SPEED_STOP
@ -1547,8 +1707,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_neon_raid);
/* +1 to avoid GEN2 optimized case */
raid_rec2_neon(2, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1558,8 +1716,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_neon32_raid);
/* +1 to avoid GEN2 optimized case */
raid_rec2_neon32(2, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1572,26 +1728,30 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
if (raid_cpu_has_ssse3()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
#ifdef CONFIG_X86_64
raid_gen_force(2, raid_gen2_sse2ext_raid);
#else
raid_gen_force(2, raid_gen2_sse2_raid);
#endif
/* +1 to avoid GEN2 optimized case */
raid_rec2_ssse3(2, id, ip + 1, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_sse2ext_raid);
/* +1 to avoid GEN2 optimized case */
raid_rec2_ssse3ext(2, id, ip + 1, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
#ifdef CONFIG_X86_64
raid_gen_force(2, raid_gen2_avx2ext_raid);
#else
raid_gen_force(2, raid_gen2_avx2_raid);
#endif
/* +1 to avoid GEN2 optimized case */
raid_rec2_avx2(2, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1600,10 +1760,19 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx512bw()) {
if (raid_cpu_has_avx2()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_avx512bw);
raid_gen_force(2, raid_gen2_avx2ext_raid);
/* +1 to avoid GEN2 optimized case */
raid_rec2_avx2ext(2, id, ip + 1, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
/* +1 to avoid GEN2 optimized case */
raid_rec2_avx512bw(2, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1613,8 +1782,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_avx2gfni_raid);
/* +1 to avoid GEN2 optimized case */
raid_rec2_avx2gfni_raid(2, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1624,8 +1791,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_avx512gfni_raid);
/* +1 to avoid GEN2 optimized case */
raid_rec2_avx512gfni_raid(2, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1637,6 +1802,118 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#endif
printf("\n");
/* REC2OF4 */
printf("%8s", "rec2of4");
printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR2));
fflush(stdout);
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, sizeof(void*) == 8 ? raid_gen2_int64_raid : raid_gen2_int32_raid);
/* +2 to select R,S instead of Q,R */
raid_rec2_int8(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_NEON
SPEED_START {
/* +2 to select R,S instead of Q,R */
raid_rec2_neon(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* +2 to select R,S instead of Q,R */
raid_rec2_neon32(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_sse2_raid);
/* +2 to select R,S instead of Q,R */
raid_rec2_ssse3(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_sse2ext_raid);
/* +2 to select R,S instead of Q,R */
raid_rec2_ssse3ext(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_avx2_raid);
/* +2 to select R,S instead of Q,R */
raid_rec2_avx2(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_avx2ext_raid);
/* +2 to select R,S instead of Q,R */
raid_rec2_avx2ext(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
/* +2 to select R,S instead of Q,R */
raid_rec2_avx512bw(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
/* +2 to select R,S instead of Q,R */
raid_rec2_avx2gfni_raid(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
/* +2 to select R,S instead of Q,R */
raid_rec2_avx512gfni_raid(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#endif
#endif
printf("\n");
printf("%8s", "rec3of3");
printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR3));
fflush(stdout);
@ -1652,9 +1929,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(3, raid_gen3_neon_raid);
raid_recX_neon(3, id, ip, nd, size, v);
raid_rec3_neon(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1662,9 +1937,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(3, raid_gen3_neon32_raid);
raid_recX_neon32(3, id, ip, nd, size, v);
raid_rec3_neon32(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1674,39 +1947,41 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
#ifdef CONFIG_X86_64
raid_gen_force(3, raid_gen3_ssse3ext_raid);
#else
raid_gen_force(3, raid_gen3_ssse3_raid);
#endif
raid_recX_ssse3(3, id, ip, nd, size, v);
raid_rec3_ssse3(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
raid_rec3_ssse3ext(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
#ifdef CONFIG_X86_64
raid_gen_force(3, raid_gen3_avx2ext_raid);
#else
raid_gen_force(3, raid_gen3_ssse3_raid);
#endif
raid_recX_avx2(3, id, ip, nd, size, v);
raid_rec3_avx2(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
raid_rec3_avx2ext(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(3, raid_gen3_avx512bw);
/* +1 to avoid GEN1 optimized case */
raid_recX_avx512bw(3, id, ip, nd, size, v);
raid_rec3_avx512bw(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1714,9 +1989,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(3, raid_gen3_avx2gfni_raid);
raid_recX_avx2gfni_raid(3, id, ip, nd, size, v);
raid_rec3_avx2gfni_raid(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1724,9 +1997,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(3, raid_gen3_avx512gfni_raid);
raid_recX_avx512gfni_raid(3, id, ip, nd, size, v);
raid_rec3_avx512gfni_raid(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1751,9 +2022,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(4, raid_gen4_neon_raid);
raid_recX_neon(4, id, ip, nd, size, v);
raid_rec4_neon(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1761,9 +2030,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(4, raid_gen4_neon32_raid);
raid_recX_neon32(4, id, ip, nd, size, v);
raid_rec4_neon32(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1773,40 +2040,41 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
#ifdef CONFIG_X86_64
raid_gen_force(4, raid_gen4_ssse3ext_raid);
#else
raid_gen_force(4, raid_gen4_ssse3_raid);
#endif
raid_recX_ssse3(4, id, ip, nd, size, v);
raid_rec4_ssse3(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
raid_rec4_ssse3ext(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
#ifdef CONFIG_X86_64
raid_gen_force(4, raid_gen4_avx2ext_raid);
#else
raid_gen_force(4, raid_gen4_ssse3_raid);
#endif
raid_recX_avx2(4, id, ip, nd, size, v);
raid_rec4_avx2(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
raid_rec4_avx2ext(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(4, raid_gen4_avx512bw);
/* +1 to avoid GEN1 optimized case */
raid_recX_avx512bw(4, id, ip, nd, size, v);
raid_rec4_avx512bw(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1814,9 +2082,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(4, raid_gen4_avx2gfni_raid);
raid_recX_avx2gfni_raid(4, id, ip, nd, size, v);
raid_rec4_avx2gfni_raid(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1824,9 +2090,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(4, raid_gen4_avx512gfni_raid);
raid_recX_avx512gfni_raid(4, id, ip, nd, size, v);
raid_rec4_avx512gfni_raid(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1851,9 +2115,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(5, raid_gen5_neon_raid);
raid_recX_neon(5, id, ip, nd, size, v);
raid_rec5_neon(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1861,9 +2123,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(5, raid_gen5_neon32_raid);
raid_recX_neon32(5, id, ip, nd, size, v);
raid_rec5_neon32(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1873,39 +2133,41 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
#ifdef CONFIG_X86_64
raid_gen_force(5, raid_gen5_ssse3ext_raid);
#else
raid_gen_force(5, raid_gen5_ssse3_raid);
#endif
raid_recX_ssse3(5, id, ip, nd, size, v);
raid_rec5_ssse3(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
raid_rec5_ssse3ext(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
#ifdef CONFIG_X86_64
raid_gen_force(5, raid_gen5_avx2ext_raid);
#else
raid_gen_force(5, raid_gen5_ssse3_raid);
#endif
raid_recX_avx2(5, id, ip, nd, size, v);
raid_rec5_avx2(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
raid_rec5_avx2ext(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(5, raid_gen5_avx512bw);
/* +1 to avoid GEN1 optimized case */
raid_recX_avx512bw(5, id, ip, nd, size, v);
raid_rec5_avx512bw(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1913,9 +2175,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(5, raid_gen5_avx2gfni_raid);
raid_recX_avx2gfni_raid(5, id, ip, nd, size, v);
raid_rec5_avx2gfni_raid(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1923,9 +2183,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(5, raid_gen5_avx512gfni_raid);
raid_recX_avx512gfni_raid(5, id, ip, nd, size, v);
raid_rec5_avx512gfni_raid(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1950,9 +2208,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(6, raid_gen6_neon_raid);
raid_recX_neon(6, id, ip, nd, size, v);
raid_rec6_neon(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1960,9 +2216,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(6, raid_gen6_neon32_raid);
raid_recX_neon32(6, id, ip, nd, size, v);
raid_rec6_neon32(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1972,39 +2226,41 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
#ifdef CONFIG_X86_64
raid_gen_force(6, raid_gen6_ssse3ext_raid);
#else
raid_gen_force(6, raid_gen6_ssse3_raid);
#endif
raid_recX_ssse3(6, id, ip, nd, size, v);
raid_rec6_ssse3(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
raid_rec6_ssse3ext(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
#ifdef CONFIG_X86_64
raid_gen_force(6, raid_gen6_avx2ext_raid);
#else
raid_gen_force(6, raid_gen6_ssse3_raid);
#endif
raid_recX_avx2(6, id, ip, nd, size, v);
raid_rec6_avx2(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
raid_rec6_avx2ext(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(6, raid_gen6_avx512bw);
/* +1 to avoid GEN1 optimized case */
raid_recX_avx512bw(6, id, ip, nd, size, v);
raid_rec6_avx512bw(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -2012,9 +2268,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(6, raid_gen6_avx2gfni_raid);
raid_recX_avx2gfni_raid(6, id, ip, nd, size, v);
raid_rec6_avx2gfni_raid(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -2022,9 +2276,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(6, raid_gen6_avx512gfni_raid);
raid_recX_avx512gfni_raid(6, id, ip, nd, size, v);
raid_rec6_avx512gfni_raid(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);

1590
raid/avx2.c
View file

File diff suppressed because it is too large Load diff

671
raid/avx512.c
View file

@ -6,7 +6,6 @@
#include "cpu.h"
#ifdef CONFIG_X86_64
/*
* GEN1 (RAID5 with xor) AVX512BW implementation
*
@ -544,6 +543,313 @@ static __always_inline void raid_genX_avx512bw(int nd, size_t size, void **vv, i
raid_avx_end();
}
/*
* RAID recovering AVX512BW implementation.
*
* Compute only the selected syndromes directly from the stored parity
* and surviving data. This avoids raid_delta_gen(), temporary delta
* parity blocks, and generation of unused parity rows.
*
* AVX512 provides enough registers to keep all six supported syndromes
* in registers in a single surviving-data scan.
*/
static __always_inline void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p[RAID_PARITY_MAX];
uint8_t *pa[RAID_PARITY_MAX];
uint8_t *src[RAID_DATA_MAX];
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX];
const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
size_t i;
int d, j, k, s;
int ns;
int has_p;
BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
/* setup the coefficients matrix */
for (j = 0; j < nr; ++j)
for (k = 0; k < nr; ++k)
G[j * nr + k] = A(ip[j], id[k]);
/* invert it to solve the system of linear equations */
raid_invert(G, V, nr);
/* setup selected parity and destination pointers */
for (j = 0; j < nr; ++j) {
p[j] = v[nd + ip[j]];
pa[j] = v[id[j]];
}
/* ip[] is ordered. If P is available, it is always ip[0] */
has_p = ip[0] == 0;
/*
* Build the compact list of surviving data blocks and precompute
* the multiplication-table pointers for each selected syndrome.
*/
ns = 0;
k = 0;
for (d = 0; d < nd; ++d) {
if (k < nr && d == id[k]) {
++k;
continue;
}
src[ns] = v[d];
for (j = 0; j < nr; ++j)
S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0];
++ns;
}
BUG_ON(k != nr);
BUG_ON(ns != nd - nr);
/* precompute inverse-matrix multiplication table pointers */
for (j = 0; j < nr; ++j)
for (k = 0; k < nr; ++k)
R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
raid_avx_begin();
/*
* Register allocation during syndrome generation:
*
* zmm0 syndrome 0
* zmm2 syndrome 1
* zmm4 syndrome 2
* zmm6 syndrome 3
* zmm8 syndrome 4
* zmm10 syndrome 5
*
* zmm12 source / source low nibble
* zmm13 source high nibble
* zmm14 low multiplication table / result
* zmm15 high multiplication table / result
* zmm31 low-nibble mask
*
* After syndrome splitting:
*
* zmm0 /zmm1 syndrome 0 low/high
* zmm2 /zmm3 syndrome 1 low/high
* zmm4 /zmm5 syndrome 2 low/high
* zmm6 /zmm7 syndrome 3 low/high
* zmm8 /zmm9 syndrome 4 low/high
* zmm10/zmm11 syndrome 5 low/high
*/
asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gfconst16.low4[0]));
for (i = 0; i < size; i += 64) {
/* start every selected syndrome from its stored parity block */
asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[0][i]));
if (nr >= 2)
asm volatile ("vmovdqa64 %0,%%zmm2" : : "m" (p[1][i]));
if (nr >= 3)
asm volatile ("vmovdqa64 %0,%%zmm4" : : "m" (p[2][i]));
if (nr >= 4)
asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (p[3][i]));
if (nr >= 5)
asm volatile ("vmovdqa64 %0,%%zmm8" : : "m" (p[4][i]));
if (nr >= 6)
asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (p[5][i]));
/* add all surviving data contributions in one source scan */
for (s = 0; s < ns; ++s) {
const uint8_t **t = S[s];
asm volatile ("vmovdqa64 %0,%%zmm12" : : "m" (src[s][i]));
/*
* P has coefficient 1 for every data disk.
* XOR the original source before splitting it.
*/
if (has_p) {
asm volatile ("vpxorq %zmm12,%zmm0,%zmm0");
asm volatile ("vpsrlw $4,%zmm12,%zmm13");
asm volatile ("vpandq %zmm31,%zmm12,%zmm12");
asm volatile ("vpandq %zmm31,%zmm13,%zmm13");
} else {
asm volatile ("vpsrlw $4,%zmm12,%zmm13");
asm volatile ("vpandq %zmm31,%zmm12,%zmm12");
asm volatile ("vpandq %zmm31,%zmm13,%zmm13");
/* syndrome 0 */
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[0][16]));
asm volatile ("vpshufb %zmm12,%zmm14,%zmm14");
asm volatile ("vpshufb %zmm13,%zmm15,%zmm15");
asm volatile ("vpternlogq $0x96,%zmm14,%zmm15,%zmm0");
}
/* syndrome 1 */
if (nr >= 2) {
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[1][16]));
asm volatile ("vpshufb %zmm12,%zmm14,%zmm14");
asm volatile ("vpshufb %zmm13,%zmm15,%zmm15");
asm volatile ("vpternlogq $0x96,%zmm14,%zmm15,%zmm2");
}
/* syndrome 2 */
if (nr >= 3) {
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[2][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[2][16]));
asm volatile ("vpshufb %zmm12,%zmm14,%zmm14");
asm volatile ("vpshufb %zmm13,%zmm15,%zmm15");
asm volatile ("vpternlogq $0x96,%zmm14,%zmm15,%zmm4");
}
/* syndrome 3 */
if (nr >= 4) {
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[3][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[3][16]));
asm volatile ("vpshufb %zmm12,%zmm14,%zmm14");
asm volatile ("vpshufb %zmm13,%zmm15,%zmm15");
asm volatile ("vpternlogq $0x96,%zmm14,%zmm15,%zmm6");
}
/* syndrome 4 */
if (nr >= 5) {
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[4][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[4][16]));
asm volatile ("vpshufb %zmm12,%zmm14,%zmm14");
asm volatile ("vpshufb %zmm13,%zmm15,%zmm15");
asm volatile ("vpternlogq $0x96,%zmm14,%zmm15,%zmm8");
}
/* syndrome 5 */
if (nr >= 6) {
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[5][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[5][16]));
asm volatile ("vpshufb %zmm12,%zmm14,%zmm14");
asm volatile ("vpshufb %zmm13,%zmm15,%zmm15");
asm volatile ("vpternlogq $0x96,%zmm14,%zmm15,%zmm10");
}
}
/* split every completed syndrome once into low/high nibbles */
asm volatile ("vpsrlw $4,%zmm0,%zmm1");
asm volatile ("vpandq %zmm31,%zmm0,%zmm0");
asm volatile ("vpandq %zmm31,%zmm1,%zmm1");
if (nr >= 2) {
asm volatile ("vpsrlw $4,%zmm2,%zmm3");
asm volatile ("vpandq %zmm31,%zmm2,%zmm2");
asm volatile ("vpandq %zmm31,%zmm3,%zmm3");
}
if (nr >= 3) {
asm volatile ("vpsrlw $4,%zmm4,%zmm5");
asm volatile ("vpandq %zmm31,%zmm4,%zmm4");
asm volatile ("vpandq %zmm31,%zmm5,%zmm5");
}
if (nr >= 4) {
asm volatile ("vpsrlw $4,%zmm6,%zmm7");
asm volatile ("vpandq %zmm31,%zmm6,%zmm6");
asm volatile ("vpandq %zmm31,%zmm7,%zmm7");
}
if (nr >= 5) {
asm volatile ("vpsrlw $4,%zmm8,%zmm9");
asm volatile ("vpandq %zmm31,%zmm8,%zmm8");
asm volatile ("vpandq %zmm31,%zmm9,%zmm9");
}
if (nr >= 6) {
asm volatile ("vpsrlw $4,%zmm10,%zmm11");
asm volatile ("vpandq %zmm31,%zmm10,%zmm10");
asm volatile ("vpandq %zmm31,%zmm11,%zmm11");
}
/*
* Reconstruct every missing data block.
*
* Keep independent low/high dependency chains:
*
* zmm12 low accumulator
* zmm13 high accumulator
* zmm14 low multiplication table/result
* zmm15 high multiplication table/result
*
* This is intentional. Do not serialize low/high products
* through a single temporary register.
*/
for (j = 0; j < nr; ++j) {
const uint8_t **t = R[j];
/* coefficient 0 initializes both accumulators */
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (t[0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (t[0][16]));
asm volatile ("vpshufb %zmm0,%zmm12,%zmm12");
asm volatile ("vpshufb %zmm1,%zmm13,%zmm13");
if (nr >= 2) {
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[1][16]));
asm volatile ("vpshufb %zmm2,%zmm14,%zmm14");
asm volatile ("vpshufb %zmm3,%zmm15,%zmm15");
asm volatile ("vpxorq %zmm14,%zmm12,%zmm12");
asm volatile ("vpxorq %zmm15,%zmm13,%zmm13");
}
if (nr >= 3) {
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[2][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[2][16]));
asm volatile ("vpshufb %zmm4,%zmm14,%zmm14");
asm volatile ("vpshufb %zmm5,%zmm15,%zmm15");
asm volatile ("vpxorq %zmm14,%zmm12,%zmm12");
asm volatile ("vpxorq %zmm15,%zmm13,%zmm13");
}
if (nr >= 4) {
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[3][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[3][16]));
asm volatile ("vpshufb %zmm6,%zmm14,%zmm14");
asm volatile ("vpshufb %zmm7,%zmm15,%zmm15");
asm volatile ("vpxorq %zmm14,%zmm12,%zmm12");
asm volatile ("vpxorq %zmm15,%zmm13,%zmm13");
}
if (nr >= 5) {
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[4][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[4][16]));
asm volatile ("vpshufb %zmm8,%zmm14,%zmm14");
asm volatile ("vpshufb %zmm9,%zmm15,%zmm15");
asm volatile ("vpxorq %zmm14,%zmm12,%zmm12");
asm volatile ("vpxorq %zmm15,%zmm13,%zmm13");
}
if (nr >= 6) {
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[5][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[5][16]));
asm volatile ("vpshufb %zmm10,%zmm14,%zmm14");
asm volatile ("vpshufb %zmm11,%zmm15,%zmm15");
asm volatile ("vpxorq %zmm14,%zmm12,%zmm12");
asm volatile ("vpxorq %zmm15,%zmm13,%zmm13");
}
asm volatile ("vpxorq %zmm13,%zmm12,%zmm12");
asm volatile ("vmovdqa64 %%zmm12,%0" : "=m" (pa[j][i]));
}
}
raid_avx_end();
}
/*
* GEN4 (quad parity with Cauchy matrix) AVX512BW implementation
*/
@ -568,355 +874,47 @@ void raid_gen6_avx512bw(int nd, size_t size, void **vv)
raid_genX_avx512bw(nd, size, vv, 6);
}
/*
* RAID recovering for one disk AVX512BW implementation
*/
void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p, *pa;
uint8_t G, V;
size_t i;
(void)nr;
BUG_ON(nr != 1);
/* if recovering with P uses the delta function */
if (ip[0] == 0) {
raid_rec1of1(id, nd, size, vv);
return;
}
G = A(ip[0], id[0]);
V = inv(G);
raid_delta_gen(1, id, ip, nd, size, vv);
p = v[nd + ip[0]];
pa = v[id[0]];
raid_avx_begin();
asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfconst16.low4[0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm4" : : "m" (raid_gfmulpshufb[V][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm5" : : "m" (raid_gfmulpshufb[V][1][0]));
for (i = 0; i < size; i += 64) {
asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i]));
asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (pa[i]));
asm volatile ("vpxord %zmm1,%zmm0,%zmm0");
asm volatile ("vpsrlw $4,%zmm0,%zmm1");
asm volatile ("vpandd %zmm7,%zmm0,%zmm0");
asm volatile ("vpandd %zmm7,%zmm1,%zmm1");
asm volatile ("vpshufb %zmm0,%zmm4,%zmm2");
asm volatile ("vpshufb %zmm1,%zmm5,%zmm3");
asm volatile ("vpxord %zmm3,%zmm2,%zmm2");
asm volatile ("vmovdqa64 %%zmm2, %0" : "=m" (pa[i]));
}
raid_avx_end();
raid_recX_avx512bw(1, id, ip, nd, size, vv);
}
/*
* RAID recovering for two disks AVX512BW implementation
*/
void raid_rec2_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
const int N = 2;
uint8_t *p[N];
uint8_t *pa[N];
uint8_t G[N * N];
uint8_t V[N * N];
size_t i;
int j, k;
(void)nr; /* unused, it's always 2 */
/* setup the coefficients matrix */
for (j = 0; j < N; ++j)
for (k = 0; k < N; ++k)
G[j * N + k] = A(ip[j], id[k]);
/* invert it to solve the system of linear equations */
raid_invert(G, V, N);
/* compute delta parity */
raid_delta_gen(N, id, ip, nd, size, vv);
for (j = 0; j < N; ++j) {
p[j] = v[nd + ip[j]];
pa[j] = v[id[j]];
}
raid_avx_begin();
/*
* zmm7 = 0x0f nibble mask
*
* zmm16 = V[0] low table
* zmm17 = V[0] high table
* zmm18 = V[1] low table
* zmm19 = V[1] high table
* zmm20 = V[2] low table
* zmm21 = V[2] high table
* zmm22 = V[3] low table
* zmm23 = V[3] high table
*/
asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfconst16.low4[0]));
/* the inverse matrix V[] is constant for the whole recovery. */
asm volatile ("vbroadcasti32x4 %0,%%zmm16" : : "m" (raid_gfmulpshufb[V[0]][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm17" : : "m" (raid_gfmulpshufb[V[0]][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm18" : : "m" (raid_gfmulpshufb[V[1]][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm19" : : "m" (raid_gfmulpshufb[V[1]][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm20" : : "m" (raid_gfmulpshufb[V[2]][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm21" : : "m" (raid_gfmulpshufb[V[2]][1][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm22" : : "m" (raid_gfmulpshufb[V[3]][0][0]));
asm volatile ("vbroadcasti32x4 %0,%%zmm23" : : "m" (raid_gfmulpshufb[V[3]][1][0]));
for (i = 0; i < size; i += 64) {
/* d0 = p[0] ^ pa[0] */
asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[0][i]));
asm volatile ("vmovdqa64 %0,%%zmm2" : : "m" (pa[0][i]));
asm volatile ("vpxord %zmm2,%zmm0,%zmm0");
/* d1 = p[1] ^ pa[1] */
asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (p[1][i]));
asm volatile ("vmovdqa64 %0,%%zmm3" : : "m" (pa[1][i]));
asm volatile ("vpxord %zmm3,%zmm1,%zmm1");
/*
* Split both deltas into low/high nibbles once.
*
* zmm4 = d0 low
* zmm5 = d0 high
* zmm12 = d1 low
* zmm13 = d1 high
*/
asm volatile ("vpsrlw $4,%zmm0,%zmm5");
asm volatile ("vpsrlw $4,%zmm1,%zmm13");
asm volatile ("vpandd %zmm7,%zmm0,%zmm4");
asm volatile ("vpandd %zmm7,%zmm5,%zmm5");
asm volatile ("vpandd %zmm7,%zmm1,%zmm12");
asm volatile ("vpandd %zmm7,%zmm13,%zmm13");
/*
* pa[0] = V[0] * d0 ^ V[1] * d1
*/
asm volatile ("vpshufb %zmm4,%zmm16,%zmm2");
asm volatile ("vpshufb %zmm5,%zmm17,%zmm3");
asm volatile ("vpshufb %zmm12,%zmm18,%zmm10");
asm volatile ("vpshufb %zmm13,%zmm19,%zmm11");
asm volatile ("vpxord %zmm3,%zmm2,%zmm2");
asm volatile ("vpxord %zmm10,%zmm2,%zmm2");
asm volatile ("vpxord %zmm11,%zmm2,%zmm2");
asm volatile ("vmovdqa64 %%zmm2,%0" : "=m" (pa[0][i]));
/*
* pa[1] = V[2] * d0 ^ V[3] * d1
*
* Reuse the already computed low/high nibbles.
*/
asm volatile ("vpshufb %zmm4,%zmm20,%zmm2");
asm volatile ("vpshufb %zmm5,%zmm21,%zmm3");
asm volatile ("vpshufb %zmm12,%zmm22,%zmm10");
asm volatile ("vpshufb %zmm13,%zmm23,%zmm11");
asm volatile ("vpxord %zmm3,%zmm2,%zmm2");
asm volatile ("vpxord %zmm10,%zmm2,%zmm2");
asm volatile ("vpxord %zmm11,%zmm2,%zmm2");
asm volatile ("vmovdqa64 %%zmm2,%0" : "=m" (pa[1][i]));
}
raid_avx_end();
BUG_ON(nr != 2);
raid_recX_avx512bw(2, id, ip, nd, size, vv);
}
/*
* RAID recovering AVX512BW implementation
*/
void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
void raid_rec3_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
int N = nr;
uint8_t *p[RAID_PARITY_MAX];
uint8_t *pa[RAID_PARITY_MAX];
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
const uint8_t *T[RAID_PARITY_MAX * RAID_PARITY_MAX];
size_t i;
int j, k;
BUG_ON(nr != 3);
raid_recX_avx512bw(3, id, ip, nd, size, vv);
}
/* setup the coefficients matrix */
for (j = 0; j < N; ++j)
for (k = 0; k < N; ++k)
G[j * N + k] = A(ip[j], id[k]);
void raid_rec4_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 4);
raid_recX_avx512bw(4, id, ip, nd, size, vv);
}
/* invert it to solve the system of linear equations */
raid_invert(G, V, N);
void raid_rec5_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 5);
raid_recX_avx512bw(5, id, ip, nd, size, vv);
}
/* precompute shuffle table pointers */
for (j = 0; j < N * N; ++j)
T[j] = &raid_gfmulpshufb[V[j]][0][0];
/* compute delta parity */
raid_delta_gen(N, id, ip, nd, size, vv);
for (j = 0; j < N; ++j) {
p[j] = v[nd + ip[j]];
pa[j] = v[id[j]];
}
raid_avx_begin();
asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gfconst16.low4[0]));
for (i = 0; i < size; i += 64) {
/* delta */
asm volatile (
"movq 0(%2), %%rax\n"
"movq 0(%3), %%rbx\n"
"vmovdqa64 (%%rax, %1), %%zmm12\n"
"vmovdqa64 (%%rbx, %1), %%zmm13\n"
"vpxorq %%zmm13, %%zmm12, %%zmm12\n"
"vpsrlw $4, %%zmm12, %%zmm1\n"
"vpandq %%zmm31, %%zmm12, %%zmm0\n"
"vpandq %%zmm31, %%zmm1, %%zmm1\n"
"cmpq $1, %0\n"
"jbe 1f\n"
"movq 8(%2), %%rax\n"
"movq 8(%3), %%rbx\n"
"vmovdqa64 (%%rax, %1), %%zmm12\n"
"vmovdqa64 (%%rbx, %1), %%zmm13\n"
"vpxorq %%zmm13, %%zmm12, %%zmm12\n"
"vpsrlw $4, %%zmm12, %%zmm3\n"
"vpandq %%zmm31, %%zmm12, %%zmm2\n"
"vpandq %%zmm31, %%zmm3, %%zmm3\n"
"cmpq $2, %0\n"
"jbe 1f\n"
"movq 16(%2), %%rax\n"
"movq 16(%3), %%rbx\n"
"vmovdqa64 (%%rax, %1), %%zmm12\n"
"vmovdqa64 (%%rbx, %1), %%zmm13\n"
"vpxorq %%zmm13, %%zmm12, %%zmm12\n"
"vpsrlw $4, %%zmm12, %%zmm5\n"
"vpandq %%zmm31, %%zmm12, %%zmm4\n"
"vpandq %%zmm31, %%zmm5, %%zmm5\n"
"cmpq $3, %0\n"
"jbe 1f\n"
"movq 24(%2), %%rax\n"
"movq 24(%3), %%rbx\n"
"vmovdqa64 (%%rax, %1), %%zmm12\n"
"vmovdqa64 (%%rbx, %1), %%zmm13\n"
"vpxorq %%zmm13, %%zmm12, %%zmm12\n"
"vpsrlw $4, %%zmm12, %%zmm7\n"
"vpandq %%zmm31, %%zmm12, %%zmm6\n"
"vpandq %%zmm31, %%zmm7, %%zmm7\n"
"cmpq $4, %0\n"
"jbe 1f\n"
"movq 32(%2), %%rax\n"
"movq 32(%3), %%rbx\n"
"vmovdqa64 (%%rax, %1), %%zmm12\n"
"vmovdqa64 (%%rbx, %1), %%zmm13\n"
"vpxorq %%zmm13, %%zmm12, %%zmm12\n"
"vpsrlw $4, %%zmm12, %%zmm9\n"
"vpandq %%zmm31, %%zmm12, %%zmm8\n"
"vpandq %%zmm31, %%zmm9, %%zmm9\n"
"cmpq $5, %0\n"
"jbe 1f\n"
"movq 40(%2), %%rax\n"
"movq 40(%3), %%rbx\n"
"vmovdqa64 (%%rax, %1), %%zmm12\n"
"vmovdqa64 (%%rbx, %1), %%zmm13\n"
"vpxorq %%zmm13, %%zmm12, %%zmm12\n"
"vpsrlw $4, %%zmm12, %%zmm11\n"
"vpandq %%zmm31, %%zmm12, %%zmm10\n"
"vpandq %%zmm31, %%zmm11, %%zmm11\n"
"1:\n"
:
: "r" ((uint64_t)N), "r" (i), "r" (p), "r" (pa)
: "rax", "rbx", "cc", "memory"
);
/* reconstruct */
for (j = 0; j < N; ++j) {
asm volatile (
"movq 0(%2), %%rcx\n"
"vbroadcasti32x4 0(%%rcx), %%zmm14\n"
"vbroadcasti32x4 16(%%rcx), %%zmm15\n"
"vpshufb %%zmm0, %%zmm14, %%zmm14\n"
"vpshufb %%zmm1, %%zmm15, %%zmm15\n"
"cmpq $1, %0\n"
"jbe 1f\n"
"movq 8(%2), %%rcx\n"
"vbroadcasti32x4 0(%%rcx), %%zmm12\n"
"vbroadcasti32x4 16(%%rcx), %%zmm13\n"
"vpshufb %%zmm2, %%zmm12, %%zmm12\n"
"vpshufb %%zmm3, %%zmm13, %%zmm13\n"
"vpxorq %%zmm12, %%zmm14, %%zmm14\n"
"vpxorq %%zmm13, %%zmm15, %%zmm15\n"
"cmpq $2, %0\n"
"jbe 1f\n"
"movq 16(%2), %%rcx\n"
"vbroadcasti32x4 0(%%rcx), %%zmm12\n"
"vbroadcasti32x4 16(%%rcx), %%zmm13\n"
"vpshufb %%zmm4, %%zmm12, %%zmm12\n"
"vpshufb %%zmm5, %%zmm13, %%zmm13\n"
"vpxorq %%zmm12, %%zmm14, %%zmm14\n"
"vpxorq %%zmm13, %%zmm15, %%zmm15\n"
"cmpq $3, %0\n"
"jbe 1f\n"
"movq 24(%2), %%rcx\n"
"vbroadcasti32x4 0(%%rcx), %%zmm12\n"
"vbroadcasti32x4 16(%%rcx), %%zmm13\n"
"vpshufb %%zmm6, %%zmm12, %%zmm12\n"
"vpshufb %%zmm7, %%zmm13, %%zmm13\n"
"vpxorq %%zmm12, %%zmm14, %%zmm14\n"
"vpxorq %%zmm13, %%zmm15, %%zmm15\n"
"cmpq $4, %0\n"
"jbe 1f\n"
"movq 32(%2), %%rcx\n"
"vbroadcasti32x4 0(%%rcx), %%zmm12\n"
"vbroadcasti32x4 16(%%rcx), %%zmm13\n"
"vpshufb %%zmm8, %%zmm12, %%zmm12\n"
"vpshufb %%zmm9, %%zmm13, %%zmm13\n"
"vpxorq %%zmm12, %%zmm14, %%zmm14\n"
"vpxorq %%zmm13, %%zmm15, %%zmm15\n"
"cmpq $5, %0\n"
"jbe 1f\n"
"movq 40(%2), %%rcx\n"
"vbroadcasti32x4 0(%%rcx), %%zmm12\n"
"vbroadcasti32x4 16(%%rcx), %%zmm13\n"
"vpshufb %%zmm10, %%zmm12, %%zmm12\n"
"vpshufb %%zmm11, %%zmm13, %%zmm13\n"
"vpxorq %%zmm12, %%zmm14, %%zmm14\n"
"vpxorq %%zmm13, %%zmm15, %%zmm15\n"
"1:\n"
"vpxorq %%zmm15, %%zmm14, %%zmm14\n"
"movq %3, %%rax\n"
"vmovdqa64 %%zmm14, (%%rax, %1)\n"
:
: "r" ((uint64_t)N), "r" (i), "r" (&T[j * N]), "r" (pa[j])
: "rax", "rcx", "cc", "memory"
);
}
}
raid_avx_end();
void raid_rec6_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 6);
raid_recX_avx512bw(6, id, ip, nd, size, vv);
}
void raid_register_avx512(void)
@ -932,11 +930,12 @@ void raid_register_avx512(void)
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "avx512", raid_rec1_avx512bw, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "avx512", raid_rec2_avx512bw, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "avx512", raid_recX_avx512bw, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "avx512", raid_recX_avx512bw, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "avx512", raid_recX_avx512bw, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "avx512", raid_recX_avx512bw, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "avx512", raid_rec3_avx512bw, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "avx512", raid_rec4_avx512bw, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "avx512", raid_rec5_avx512bw, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "avx512", raid_rec6_avx512bw, RAID_POLY_ANY);
}
}
}
#endif

1869
raid/gfni.c
View file

File diff suppressed because it is too large Load diff

2
raid/int.c
View file

@ -533,7 +533,7 @@ void raid_rec1_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv)
(void)nr; /* unused, it's always 1 */
/* if it's RAID5, use the faster function */
/* if recovering with P uses the delta function */
if (ip[0] == 0) {
raid_rec1of1(id, nd, size, vv);
return;

57
raid/internal.h
View file

@ -169,25 +169,58 @@ void raid_rec2_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_recX_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec3_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec4_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec5_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec6_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec1_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec2_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec3_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec4_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec5_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec6_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec3_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec4_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec5_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec6_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec1_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec2_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec3_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec4_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec5_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec6_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec2_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec3_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec4_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec5_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec6_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec1_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec2_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_recX_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec3_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec4_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec5_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec6_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec1_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec2_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec3_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec4_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec5_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec6_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec1_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec2_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_recX_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec3_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec4_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec5_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec6_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec1_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec2_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_recX_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec3_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec4_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec5_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec6_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_gen1_neon(int nd, size_t size, void **vv);
void raid_gen2_neon_raid(int nd, size_t size, void **vv);
void raid_gen2_neon_aes(int nd, size_t size, void **vv);
@ -202,7 +235,10 @@ void raid_gen6_neon_raid(int nd, size_t size, void **vv);
void raid_gen6_neon_aes(int nd, size_t size, void **vv);
void raid_rec1_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec2_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec3_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec4_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec5_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec6_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_gen1_neon32(int nd, size_t size, void **vv);
void raid_gen2_neon32_raid(int nd, size_t size, void **vv);
void raid_gen2_neon32_aes(int nd, size_t size, void **vv);
@ -217,7 +253,10 @@ void raid_gen6_neon32_raid(int nd, size_t size, void **vv);
void raid_gen6_neon32_aes(int nd, size_t size, void **vv);
void raid_rec1_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec2_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_recX_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec3_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec4_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec5_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec6_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv);
/*
* Functions for parity computation.

866
raid/neon.c
View file

@ -6,7 +6,6 @@
#include "cpu.h"
#ifdef CONFIG_NEON
/*
* GEN1 (RAID5 with xor) NEON implementation
*/
@ -94,7 +93,7 @@ static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, i
for (d = l - 1; d >= 0; --d) {
if (generator == 3) {
/* v14-v15 are otherwise unused and preserve Q across the AES xtime. */
/* v14-v15 are otherwise unused and preserve Q across the AES xtime */
asm volatile (
"mov v14.16b, v2.16b\n"
"mov v15.16b, v3.16b\n"
@ -143,16 +142,6 @@ static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, i
raid_neon_end();
}
void raid_gen2_neon_raid(int nd, size_t size, void **vv)
{
raid_gen2_neon_gen(nd, size, vv, 2);
}
void raid_gen2_neon_aes(int nd, size_t size, void **vv)
{
raid_gen2_neon_gen(nd, size, vv, 3);
}
/*
* GENz (triple parity with powers of 2^-1) NEON implementation
*/
@ -577,6 +566,420 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
raid_neon_end();
}
static __always_inline void raid_recX_neon(int nr, int *id, int *ip,
int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p[RAID_PARITY_MAX];
uint8_t *pa[RAID_PARITY_MAX];
uint8_t *src[RAID_DATA_MAX];
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX];
const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
size_t i;
int d, j, k, s;
int ns;
int has_p;
BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
/* setup the coefficients matrix */
for (j = 0; j < nr; ++j)
for (k = 0; k < nr; ++k)
G[j * nr + k] = A(ip[j], id[k]);
/* invert it to solve the system of linear equations */
raid_invert(G, V, nr);
/* setup selected parity and destination pointers */
for (j = 0; j < nr; ++j) {
p[j] = v[nd + ip[j]];
pa[j] = v[id[j]];
}
/* ip[] is ordered. If P is available, it is always ip[0] */
has_p = ip[0] == 0;
/*
* Build the compact list of surviving data blocks and precompute
* the multiplication-table pointers for each selected syndrome.
*/
ns = 0;
k = 0;
for (d = 0; d < nd; ++d) {
if (k < nr && d == id[k]) {
++k;
continue;
}
src[ns] = v[d];
for (j = 0; j < nr; ++j)
S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0];
++ns;
}
BUG_ON(k != nr);
BUG_ON(ns != nd - nr);
/* precompute inverse-matrix multiplication table pointers */
for (j = 0; j < nr; ++j)
for (k = 0; k < nr; ++k)
R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
raid_neon_begin();
for (i = 0; i < size; i += 32) {
/*
* V31 is reused during reconstruction, so reload the nibble mask
* at the beginning of every iteration.
*/
asm volatile ("ldr q31, %0" : : "m" (gfconst16.low4[0]));
/* start all selected syndromes from the stored parity */
asm volatile ("ldr q0, %0" : : "m" (p[0][i]));
asm volatile ("ldr q1, %0" : : "m" (p[0][i + 16]));
if (nr >= 2) {
asm volatile ("ldr q2, %0" : : "m" (p[1][i]));
asm volatile ("ldr q3, %0" : : "m" (p[1][i + 16]));
}
if (nr >= 3) {
asm volatile ("ldr q4, %0" : : "m" (p[2][i]));
asm volatile ("ldr q5, %0" : : "m" (p[2][i + 16]));
}
if (nr >= 4) {
asm volatile ("ldr q6, %0" : : "m" (p[3][i]));
asm volatile ("ldr q7, %0" : : "m" (p[3][i + 16]));
}
if (nr >= 5) {
asm volatile ("ldr q8, %0" : : "m" (p[4][i]));
asm volatile ("ldr q9, %0" : : "m" (p[4][i + 16]));
}
if (nr >= 6) {
asm volatile ("ldr q10, %0" : : "m" (p[5][i]));
asm volatile ("ldr q11, %0" : : "m" (p[5][i + 16]));
}
/*
* Add all surviving data contributions.
* All source reads for this chunk occur before any destination
* write.
*/
for (s = 0; s < ns; ++s) {
const uint8_t **t = S[s];
asm volatile ("ldr q12, %0" : : "m" (src[s][i]));
asm volatile ("ldr q13, %0" : : "m" (src[s][i + 16]));
/*
* P has coefficient 1. Use the original source before
* destructively splitting it into nibbles.
*/
if (has_p) {
asm volatile ("eor v0.16b, v0.16b, v12.16b");
asm volatile ("eor v1.16b, v1.16b, v13.16b");
/* split the source lanes */
asm volatile ("ushr v14.16b, v12.16b, #4");
asm volatile ("ushr v15.16b, v13.16b, #4");
asm volatile ("and v12.16b, v12.16b, v31.16b");
asm volatile ("and v13.16b, v13.16b, v31.16b");
asm volatile ("and v14.16b, v14.16b, v31.16b");
asm volatile ("and v15.16b, v15.16b, v31.16b");
} else {
/* split the source lanes */
asm volatile ("ushr v14.16b, v12.16b, #4");
asm volatile ("ushr v15.16b, v13.16b, #4");
asm volatile ("and v12.16b, v12.16b, v31.16b");
asm volatile ("and v13.16b, v13.16b, v31.16b");
asm volatile ("and v14.16b, v14.16b, v31.16b");
asm volatile ("and v15.16b, v15.16b, v31.16b");
/* syndrome 0 */
asm volatile ("ldr q24, %0" : : "m" (t[0][0]));
asm volatile ("ldr q25, %0" : : "m" (t[0][16]));
asm volatile ("tbl v26.16b, {v24.16b}, v12.16b");
asm volatile ("tbl v27.16b, {v25.16b}, v14.16b");
asm volatile ("eor v26.16b, v26.16b, v27.16b");
asm volatile ("eor v0.16b, v0.16b, v26.16b");
asm volatile ("tbl v26.16b, {v24.16b}, v13.16b");
asm volatile ("tbl v27.16b, {v25.16b}, v15.16b");
asm volatile ("eor v26.16b, v26.16b, v27.16b");
asm volatile ("eor v1.16b, v1.16b, v26.16b");
}
/* syndrome 1 */
if (nr >= 2) {
asm volatile ("ldr q24, %0" : : "m" (t[1][0]));
asm volatile ("ldr q25, %0" : : "m" (t[1][16]));
asm volatile ("tbl v26.16b, {v24.16b}, v12.16b");
asm volatile ("tbl v27.16b, {v25.16b}, v14.16b");
asm volatile ("eor v26.16b, v26.16b, v27.16b");
asm volatile ("eor v2.16b, v2.16b, v26.16b");
asm volatile ("tbl v26.16b, {v24.16b}, v13.16b");
asm volatile ("tbl v27.16b, {v25.16b}, v15.16b");
asm volatile ("eor v26.16b, v26.16b, v27.16b");
asm volatile ("eor v3.16b, v3.16b, v26.16b");
}
/* syndrome 2 */
if (nr >= 3) {
asm volatile ("ldr q24, %0" : : "m" (t[2][0]));
asm volatile ("ldr q25, %0" : : "m" (t[2][16]));
asm volatile ("tbl v26.16b, {v24.16b}, v12.16b");
asm volatile ("tbl v27.16b, {v25.16b}, v14.16b");
asm volatile ("eor v26.16b, v26.16b, v27.16b");
asm volatile ("eor v4.16b, v4.16b, v26.16b");
asm volatile ("tbl v26.16b, {v24.16b}, v13.16b");
asm volatile ("tbl v27.16b, {v25.16b}, v15.16b");
asm volatile ("eor v26.16b, v26.16b, v27.16b");
asm volatile ("eor v5.16b, v5.16b, v26.16b");
}
/* syndrome 3 */
if (nr >= 4) {
asm volatile ("ldr q24, %0" : : "m" (t[3][0]));
asm volatile ("ldr q25, %0" : : "m" (t[3][16]));
asm volatile ("tbl v26.16b, {v24.16b}, v12.16b");
asm volatile ("tbl v27.16b, {v25.16b}, v14.16b");
asm volatile ("eor v26.16b, v26.16b, v27.16b");
asm volatile ("eor v6.16b, v6.16b, v26.16b");
asm volatile ("tbl v26.16b, {v24.16b}, v13.16b");
asm volatile ("tbl v27.16b, {v25.16b}, v15.16b");
asm volatile ("eor v26.16b, v26.16b, v27.16b");
asm volatile ("eor v7.16b, v7.16b, v26.16b");
}
/* syndrome 4 */
if (nr >= 5) {
asm volatile ("ldr q24, %0" : : "m" (t[4][0]));
asm volatile ("ldr q25, %0" : : "m" (t[4][16]));
asm volatile ("tbl v26.16b, {v24.16b}, v12.16b");
asm volatile ("tbl v27.16b, {v25.16b}, v14.16b");
asm volatile ("eor v26.16b, v26.16b, v27.16b");
asm volatile ("eor v8.16b, v8.16b, v26.16b");
asm volatile ("tbl v26.16b, {v24.16b}, v13.16b");
asm volatile ("tbl v27.16b, {v25.16b}, v15.16b");
asm volatile ("eor v26.16b, v26.16b, v27.16b");
asm volatile ("eor v9.16b, v9.16b, v26.16b");
}
/* syndrome 5 */
if (nr >= 6) {
asm volatile ("ldr q24, %0" : : "m" (t[5][0]));
asm volatile ("ldr q25, %0" : : "m" (t[5][16]));
asm volatile ("tbl v26.16b, {v24.16b}, v12.16b");
asm volatile ("tbl v27.16b, {v25.16b}, v14.16b");
asm volatile ("eor v26.16b, v26.16b, v27.16b");
asm volatile ("eor v10.16b, v10.16b, v26.16b");
asm volatile ("tbl v26.16b, {v24.16b}, v13.16b");
asm volatile ("tbl v27.16b, {v25.16b}, v15.16b");
asm volatile ("eor v26.16b, v26.16b, v27.16b");
asm volatile ("eor v11.16b, v11.16b, v26.16b");
}
}
/*
* Expand raw syndromes backwards into:
*
* S0 = v0..v3
* S1 = v4..v7
* S2 = v8..v11
* S3 = v12..v15
* S4 = v16..v19
* S5 = v20..v23
*/
if (nr >= 6) {
asm volatile ("ushr v21.16b, v10.16b, #4");
asm volatile ("and v20.16b, v10.16b, v31.16b");
asm volatile ("and v21.16b, v21.16b, v31.16b");
asm volatile ("ushr v23.16b, v11.16b, #4");
asm volatile ("and v22.16b, v11.16b, v31.16b");
asm volatile ("and v23.16b, v23.16b, v31.16b");
}
if (nr >= 5) {
asm volatile ("ushr v17.16b, v8.16b, #4");
asm volatile ("and v16.16b, v8.16b, v31.16b");
asm volatile ("and v17.16b, v17.16b, v31.16b");
asm volatile ("ushr v19.16b, v9.16b, #4");
asm volatile ("and v18.16b, v9.16b, v31.16b");
asm volatile ("and v19.16b, v19.16b, v31.16b");
}
if (nr >= 4) {
asm volatile ("ushr v13.16b, v6.16b, #4");
asm volatile ("and v12.16b, v6.16b, v31.16b");
asm volatile ("and v13.16b, v13.16b, v31.16b");
asm volatile ("ushr v15.16b, v7.16b, #4");
asm volatile ("and v14.16b, v7.16b, v31.16b");
asm volatile ("and v15.16b, v15.16b, v31.16b");
}
if (nr >= 3) {
asm volatile ("ushr v9.16b, v4.16b, #4");
asm volatile ("and v8.16b, v4.16b, v31.16b");
asm volatile ("and v9.16b, v9.16b, v31.16b");
asm volatile ("ushr v11.16b, v5.16b, #4");
asm volatile ("and v10.16b, v5.16b, v31.16b");
asm volatile ("and v11.16b, v11.16b, v31.16b");
}
if (nr >= 2) {
asm volatile ("ushr v5.16b, v2.16b, #4");
asm volatile ("and v4.16b, v2.16b, v31.16b");
asm volatile ("and v5.16b, v5.16b, v31.16b");
asm volatile ("ushr v7.16b, v3.16b, #4");
asm volatile ("and v6.16b, v3.16b, v31.16b");
asm volatile ("and v7.16b, v7.16b, v31.16b");
}
/*
* S0 overlaps its original raw lane registers.
* Process raw lane 1 before overwriting v1.
*/
asm volatile ("ushr v3.16b, v1.16b, #4");
asm volatile ("and v2.16b, v1.16b, v31.16b");
asm volatile ("and v3.16b, v3.16b, v31.16b");
asm volatile ("ushr v1.16b, v0.16b, #4");
asm volatile ("and v0.16b, v0.16b, v31.16b");
asm volatile ("and v1.16b, v1.16b, v31.16b");
/*
* Reconstruct all missing blocks.
*
* v24/v25 = low/high tables
* v26/v27 = lane-0 low/high accumulators
* v28/v29 = lane-1 low/high accumulators
* v30/v31 = low/high multiplication temporaries
*/
for (j = 0; j < nr; ++j) {
const uint8_t **t = R[j];
/* coefficient 0 initializes both lane accumulators */
asm volatile ("ldr q24, %0" : : "m" (t[0][0]));
asm volatile ("ldr q25, %0" : : "m" (t[0][16]));
asm volatile ("tbl v26.16b, {v24.16b}, v0.16b");
asm volatile ("tbl v27.16b, {v25.16b}, v1.16b");
asm volatile ("tbl v28.16b, {v24.16b}, v2.16b");
asm volatile ("tbl v29.16b, {v25.16b}, v3.16b");
if (nr >= 2) {
asm volatile ("ldr q24, %0" : : "m" (t[1][0]));
asm volatile ("ldr q25, %0" : : "m" (t[1][16]));
asm volatile ("tbl v30.16b, {v24.16b}, v4.16b");
asm volatile ("tbl v31.16b, {v25.16b}, v5.16b");
asm volatile ("eor v26.16b, v26.16b, v30.16b");
asm volatile ("eor v27.16b, v27.16b, v31.16b");
asm volatile ("tbl v30.16b, {v24.16b}, v6.16b");
asm volatile ("tbl v31.16b, {v25.16b}, v7.16b");
asm volatile ("eor v28.16b, v28.16b, v30.16b");
asm volatile ("eor v29.16b, v29.16b, v31.16b");
}
if (nr >= 3) {
asm volatile ("ldr q24, %0" : : "m" (t[2][0]));
asm volatile ("ldr q25, %0" : : "m" (t[2][16]));
asm volatile ("tbl v30.16b, {v24.16b}, v8.16b");
asm volatile ("tbl v31.16b, {v25.16b}, v9.16b");
asm volatile ("eor v26.16b, v26.16b, v30.16b");
asm volatile ("eor v27.16b, v27.16b, v31.16b");
asm volatile ("tbl v30.16b, {v24.16b}, v10.16b");
asm volatile ("tbl v31.16b, {v25.16b}, v11.16b");
asm volatile ("eor v28.16b, v28.16b, v30.16b");
asm volatile ("eor v29.16b, v29.16b, v31.16b");
}
if (nr >= 4) {
asm volatile ("ldr q24, %0" : : "m" (t[3][0]));
asm volatile ("ldr q25, %0" : : "m" (t[3][16]));
asm volatile ("tbl v30.16b, {v24.16b}, v12.16b");
asm volatile ("tbl v31.16b, {v25.16b}, v13.16b");
asm volatile ("eor v26.16b, v26.16b, v30.16b");
asm volatile ("eor v27.16b, v27.16b, v31.16b");
asm volatile ("tbl v30.16b, {v24.16b}, v14.16b");
asm volatile ("tbl v31.16b, {v25.16b}, v15.16b");
asm volatile ("eor v28.16b, v28.16b, v30.16b");
asm volatile ("eor v29.16b, v29.16b, v31.16b");
}
if (nr >= 5) {
asm volatile ("ldr q24, %0" : : "m" (t[4][0]));
asm volatile ("ldr q25, %0" : : "m" (t[4][16]));
asm volatile ("tbl v30.16b, {v24.16b}, v16.16b");
asm volatile ("tbl v31.16b, {v25.16b}, v17.16b");
asm volatile ("eor v26.16b, v26.16b, v30.16b");
asm volatile ("eor v27.16b, v27.16b, v31.16b");
asm volatile ("tbl v30.16b, {v24.16b}, v18.16b");
asm volatile ("tbl v31.16b, {v25.16b}, v19.16b");
asm volatile ("eor v28.16b, v28.16b, v30.16b");
asm volatile ("eor v29.16b, v29.16b, v31.16b");
}
if (nr >= 6) {
asm volatile ("ldr q24, %0" : : "m" (t[5][0]));
asm volatile ("ldr q25, %0" : : "m" (t[5][16]));
asm volatile ("tbl v30.16b, {v24.16b}, v20.16b");
asm volatile ("tbl v31.16b, {v25.16b}, v21.16b");
asm volatile ("eor v26.16b, v26.16b, v30.16b");
asm volatile ("eor v27.16b, v27.16b, v31.16b");
asm volatile ("tbl v30.16b, {v24.16b}, v22.16b");
asm volatile ("tbl v31.16b, {v25.16b}, v23.16b");
asm volatile ("eor v28.16b, v28.16b, v30.16b");
asm volatile ("eor v29.16b, v29.16b, v31.16b");
}
asm volatile ("eor v26.16b, v26.16b, v27.16b");
asm volatile ("eor v28.16b, v28.16b, v29.16b");
asm volatile ("str q26, %0" : "=m" (pa[j][i]));
asm volatile ("str q28, %0" : "=m" (pa[j][i + 16]));
}
}
raid_neon_end();
}
void raid_gen2_neon_raid(int nd, size_t size, void **vv)
{
raid_gen2_neon_gen(nd, size, vv, 2);
}
void raid_gen2_neon_aes(int nd, size_t size, void **vv)
{
raid_gen2_neon_gen(nd, size, vv, 3);
}
/*
* GEN3 (triple parity with Cauchy matrix) NEON implementation
*/
@ -629,431 +1032,47 @@ void raid_gen6_neon_aes(int nd, size_t size, void **vv)
raid_genX_neon(nd, size, vv, 6, 3);
}
/*
* RAID recovering for one disk NEON implementation
*/
void raid_rec1_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
uint8_t *pa;
uint8_t G;
uint8_t V;
size_t i;
BUG_ON(nr != 1);
(void)nr; /* unused, it's always 1 */
/* if it's RAID5 uses the faster function */
/* if recovering with P uses the delta function */
if (ip[0] == 0) {
raid_rec1of1(id, nd, size, vv);
return;
}
/* setup the coefficients matrix */
G = A(ip[0], id[0]);
/* invert it to solve the system of linear equations */
V = inv(G);
/* compute delta parity */
raid_delta_gen(1, id, ip, nd, size, vv);
p = v[nd + ip[0]];
pa = v[id[0]];
raid_neon_begin();
/* preload tables */
asm volatile (
"ldr q28, %0\n" /* low4 */
"ldr q24, %1\n" /* v low table */
"ldr q25, %2\n" /* v high table */
:
: "m" (gfconst16.low4[0]),
"m" (raid_gfmulpshufb[V][0][0]), "m" (raid_gfmulpshufb[V][1][0])
);
for (i = 0; i < size; i += 64) {
asm volatile (
"ldr q0, %4\n"
"ldr q1, %5\n"
"ldr q2, %6\n"
"ldr q3, %7\n"
"ldr q8, %8\n"
"ldr q9, %9\n"
"ldr q10, %10\n"
"ldr q11, %11\n"
"eor v0.16b, v0.16b, v8.16b\n"
"eor v1.16b, v1.16b, v9.16b\n"
"eor v2.16b, v2.16b, v10.16b\n"
"eor v3.16b, v3.16b, v11.16b\n"
"ushr v8.16b, v0.16b, #4\n"
"ushr v9.16b, v1.16b, #4\n"
"ushr v10.16b, v2.16b, #4\n"
"ushr v11.16b, v3.16b, #4\n"
"and v0.16b, v0.16b, v28.16b\n"
"and v1.16b, v1.16b, v28.16b\n"
"and v2.16b, v2.16b, v28.16b\n"
"and v3.16b, v3.16b, v28.16b\n"
"and v8.16b, v8.16b, v28.16b\n"
"and v9.16b, v9.16b, v28.16b\n"
"and v10.16b, v10.16b, v28.16b\n"
"and v11.16b, v11.16b, v28.16b\n"
"tbl v12.16b, {v24.16b}, v0.16b\n"
"tbl v13.16b, {v24.16b}, v1.16b\n"
"tbl v14.16b, {v24.16b}, v2.16b\n"
"tbl v15.16b, {v24.16b}, v3.16b\n"
"tbl v8.16b, {v25.16b}, v8.16b\n"
"tbl v9.16b, {v25.16b}, v9.16b\n"
"tbl v10.16b, {v25.16b}, v10.16b\n"
"tbl v11.16b, {v25.16b}, v11.16b\n"
"eor v0.16b, v12.16b, v8.16b\n"
"eor v1.16b, v13.16b, v9.16b\n"
"eor v2.16b, v14.16b, v10.16b\n"
"eor v3.16b, v15.16b, v11.16b\n"
"str q0, %0\n"
"str q1, %1\n"
"str q2, %2\n"
"str q3, %3\n"
: "=m" (pa[i]), "=m" (pa[i + 16]), "=m" (pa[i + 32]), "=m" (pa[i + 48])
: "m" (p[i]), "m" (p[i + 16]), "m" (p[i + 32]), "m" (p[i + 48]),
"m" (pa[i]), "m" (pa[i + 16]), "m" (pa[i + 32]), "m" (pa[i + 48])
);
}
raid_neon_end();
raid_recX_neon(1, id, ip, nd, size, vv);
}
/*
* RAID recovering for two disks NEON implementation
*/
void raid_rec2_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
const int N = 2;
uint8_t *p[2];
uint8_t *pa[2];
uint8_t G[2 * 2];
uint8_t V[2 * 2];
size_t i;
int j, k;
(void)nr; /* unused, it's always 2 */
/* setup the coefficients matrix */
for (j = 0; j < N; ++j)
for (k = 0; k < N; ++k)
G[j * N + k] = A(ip[j], id[k]);
/* invert it to solve the system of linear equations */
raid_invert(G, V, N);
/* compute delta parity */
raid_delta_gen(N, id, ip, nd, size, vv);
for (j = 0; j < N; ++j) {
p[j] = v[nd + ip[j]];
pa[j] = v[id[j]];
}
raid_neon_begin();
/* preload tables */
asm volatile (
"ldr q28, %0\n" /* low4 */
"ldr q20, %1\n" /* v[0] low table */
"ldr q21, %2\n" /* v[0] high table */
"ldr q22, %3\n" /* v[1] low table */
"ldr q23, %4\n" /* v[1] high table */
"ldr q24, %5\n" /* v[2] low table */
"ldr q25, %6\n" /* v[2] high table */
"ldr q26, %7\n" /* v[3] low table */
"ldr q27, %8\n" /* v[3] high table */
:
: "m" (gfconst16.low4[0]),
"m" (raid_gfmulpshufb[V[0]][0][0]), "m" (raid_gfmulpshufb[V[0]][1][0]),
"m" (raid_gfmulpshufb[V[1]][0][0]), "m" (raid_gfmulpshufb[V[1]][1][0]),
"m" (raid_gfmulpshufb[V[2]][0][0]), "m" (raid_gfmulpshufb[V[2]][1][0]),
"m" (raid_gfmulpshufb[V[3]][0][0]), "m" (raid_gfmulpshufb[V[3]][1][0])
);
for (i = 0; i < size; i += 32) {
asm volatile (
/* load delta 0 block 0 & 1 */
"ldr q0, %4\n"
"ldr q2, %5\n"
"ldr q8, %8\n"
"ldr q9, %9\n"
"eor v0.16b, v0.16b, v8.16b\n"
"eor v2.16b, v2.16b, v9.16b\n"
/* load delta 1 block 0 & 1 */
"ldr q4, %6\n"
"ldr q6, %7\n"
"ldr q10, %10\n"
"ldr q11, %11\n"
"eor v4.16b, v4.16b, v10.16b\n"
"eor v6.16b, v6.16b, v11.16b\n"
/* split delta 0 (v0, v2 -> v0 low, v1 high, v2 low, v3 high) */
"ushr v1.16b, v0.16b, #4\n"
"ushr v3.16b, v2.16b, #4\n"
"and v0.16b, v0.16b, v28.16b\n"
"and v1.16b, v1.16b, v28.16b\n"
"and v2.16b, v2.16b, v28.16b\n"
"and v3.16b, v3.16b, v28.16b\n"
/* split delta 1 (v4, v6 -> v4 low, v5 high, v6 low, v7 high) */
"ushr v5.16b, v4.16b, #4\n"
"ushr v7.16b, v6.16b, #4\n"
"and v4.16b, v4.16b, v28.16b\n"
"and v5.16b, v5.16b, v28.16b\n"
"and v6.16b, v6.16b, v28.16b\n"
"and v7.16b, v7.16b, v28.16b\n"
/* reconstruct pa[0] (v8 = block 0, v9 = block 1) */
/* V[0] * delta 0 */
"tbl v8.16b, {v20.16b}, v0.16b\n"
"tbl v9.16b, {v20.16b}, v2.16b\n"
"tbl v12.16b, {v21.16b}, v1.16b\n"
"tbl v13.16b, {v21.16b}, v3.16b\n"
"eor v8.16b, v8.16b, v12.16b\n"
"eor v9.16b, v9.16b, v13.16b\n"
/* V[1] * delta 1 */
"tbl v12.16b, {v22.16b}, v4.16b\n"
"tbl v13.16b, {v22.16b}, v6.16b\n"
"tbl v14.16b, {v23.16b}, v5.16b\n"
"tbl v15.16b, {v23.16b}, v7.16b\n"
"eor v12.16b, v12.16b, v14.16b\n"
"eor v13.16b, v13.16b, v15.16b\n"
"eor v8.16b, v8.16b, v12.16b\n"
"eor v9.16b, v9.16b, v13.16b\n"
"str q8, %0\n"
"str q9, %1\n"
/* reconstruct pa[1] (v10 = block 0, v11 = block 1) */
/* V[2] * delta 0 */
"tbl v10.16b, {v24.16b}, v0.16b\n"
"tbl v11.16b, {v24.16b}, v2.16b\n"
"tbl v12.16b, {v25.16b}, v1.16b\n"
"tbl v13.16b, {v25.16b}, v3.16b\n"
"eor v10.16b, v10.16b, v12.16b\n"
"eor v11.16b, v11.16b, v13.16b\n"
/* V[3] * delta 1 */
"tbl v12.16b, {v26.16b}, v4.16b\n"
"tbl v13.16b, {v26.16b}, v6.16b\n"
"tbl v14.16b, {v27.16b}, v5.16b\n"
"tbl v15.16b, {v27.16b}, v7.16b\n"
"eor v12.16b, v12.16b, v14.16b\n"
"eor v13.16b, v13.16b, v15.16b\n"
"eor v10.16b, v10.16b, v12.16b\n"
"eor v11.16b, v11.16b, v13.16b\n"
"str q10, %2\n"
"str q11, %3\n"
: "=m" (pa[0][i]), "=m" (pa[0][i + 16]),
"=m" (pa[1][i]), "=m" (pa[1][i + 16])
: "m" (p[0][i]), "m" (p[0][i + 16]),
"m" (p[1][i]), "m" (p[1][i + 16]),
"m" (pa[0][i]), "m" (pa[0][i + 16]),
"m" (pa[1][i]), "m" (pa[1][i + 16])
);
}
raid_neon_end();
BUG_ON(nr != 2);
raid_recX_neon(2, id, ip, nd, size, vv);
}
/*
* RAID recovering NEON implementation
*/
void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
void raid_rec3_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
int N = nr;
uint8_t *p[RAID_PARITY_MAX];
uint8_t *pa[RAID_PARITY_MAX];
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
const uint8_t *T[RAID_PARITY_MAX * RAID_PARITY_MAX];
size_t i;
int j, k;
BUG_ON(nr != 3);
raid_recX_neon(3, id, ip, nd, size, vv);
}
/* setup the coefficients matrix */
for (j = 0; j < N; ++j)
for (k = 0; k < N; ++k)
G[j * N + k] = A(ip[j], id[k]);
void raid_rec4_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 4);
raid_recX_neon(4, id, ip, nd, size, vv);
}
/* invert it to solve the system of linear equations */
raid_invert(G, V, N);
void raid_rec5_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 5);
raid_recX_neon(5, id, ip, nd, size, vv);
}
/* precompute shuffle table pointers */
for (j = 0; j < N * N; ++j)
T[j] = &raid_gfmulpshufb[V[j]][0][0];
/* compute delta parity */
raid_delta_gen(N, id, ip, nd, size, vv);
for (j = 0; j < N; ++j) {
p[j] = v[nd + ip[j]];
pa[j] = v[id[j]];
}
raid_neon_begin();
/* preload mask */
asm volatile (
"ldr q28, %0\n" /* low4 */
:
: "m" (gfconst16.low4[0])
);
for (i = 0; i < size; i += 16) {
/* delta */
asm volatile (
"ldr x4, [%2, #0]\n"
"ldr x5, [%3, #0]\n"
"ldr q0, [x4, %1]\n"
"ldr q14, [x5, %1]\n"
"eor v0.16b, v0.16b, v14.16b\n"
"ushr v1.16b, v0.16b, #4\n"
"and v0.16b, v0.16b, v28.16b\n"
"and v1.16b, v1.16b, v28.16b\n"
"cmp %0, #1\n"
"b.ls 1f\n"
"ldr x4, [%2, #8]\n"
"ldr x5, [%3, #8]\n"
"ldr q2, [x4, %1]\n"
"ldr q14, [x5, %1]\n"
"eor v2.16b, v2.16b, v14.16b\n"
"ushr v3.16b, v2.16b, #4\n"
"and v2.16b, v2.16b, v28.16b\n"
"and v3.16b, v3.16b, v28.16b\n"
"cmp %0, #2\n"
"b.ls 1f\n"
"ldr x4, [%2, #16]\n"
"ldr x5, [%3, #16]\n"
"ldr q4, [x4, %1]\n"
"ldr q14, [x5, %1]\n"
"eor v4.16b, v4.16b, v14.16b\n"
"ushr v5.16b, v4.16b, #4\n"
"and v4.16b, v4.16b, v28.16b\n"
"and v5.16b, v5.16b, v28.16b\n"
"cmp %0, #3\n"
"b.ls 1f\n"
"ldr x4, [%2, #24]\n"
"ldr x5, [%3, #24]\n"
"ldr q6, [x4, %1]\n"
"ldr q14, [x5, %1]\n"
"eor v6.16b, v6.16b, v14.16b\n"
"ushr v7.16b, v6.16b, #4\n"
"and v6.16b, v6.16b, v28.16b\n"
"and v7.16b, v7.16b, v28.16b\n"
"cmp %0, #4\n"
"b.ls 1f\n"
"ldr x4, [%2, #32]\n"
"ldr x5, [%3, #32]\n"
"ldr q8, [x4, %1]\n"
"ldr q14, [x5, %1]\n"
"eor v8.16b, v8.16b, v14.16b\n"
"ushr v9.16b, v8.16b, #4\n"
"and v8.16b, v8.16b, v28.16b\n"
"and v9.16b, v9.16b, v28.16b\n"
"cmp %0, #5\n"
"b.ls 1f\n"
"ldr x4, [%2, #40]\n"
"ldr x5, [%3, #40]\n"
"ldr q10, [x4, %1]\n"
"ldr q14, [x5, %1]\n"
"eor v10.16b, v10.16b, v14.16b\n"
"ushr v11.16b, v10.16b, #4\n"
"and v10.16b, v10.16b, v28.16b\n"
"and v11.16b, v11.16b, v28.16b\n"
"1:\n"
:
: "r" ((uint64_t)N), "r" (i), "r" (p), "r" (pa)
: "x4", "x5", "cc", "memory"
);
/* reconstruct */
for (j = 0; j < N; ++j) {
asm volatile (
"ldr x5, [%2, #0]\n"
"ldp q24, q25, [x5]\n"
"tbl v12.16b, {v24.16b}, v0.16b\n"
"tbl v13.16b, {v25.16b}, v1.16b\n"
"cmp %0, #1\n"
"b.ls 1f\n"
"ldr x5, [%2, #8]\n"
"ldp q24, q25, [x5]\n"
"tbl v14.16b, {v24.16b}, v2.16b\n"
"tbl v15.16b, {v25.16b}, v3.16b\n"
"eor v12.16b, v12.16b, v14.16b\n"
"eor v13.16b, v13.16b, v15.16b\n"
"cmp %0, #2\n"
"b.ls 1f\n"
"ldr x5, [%2, #16]\n"
"ldp q24, q25, [x5]\n"
"tbl v14.16b, {v24.16b}, v4.16b\n"
"tbl v15.16b, {v25.16b}, v5.16b\n"
"eor v12.16b, v12.16b, v14.16b\n"
"eor v13.16b, v13.16b, v15.16b\n"
"cmp %0, #3\n"
"b.ls 1f\n"
"ldr x5, [%2, #24]\n"
"ldp q24, q25, [x5]\n"
"tbl v14.16b, {v24.16b}, v6.16b\n"
"tbl v15.16b, {v25.16b}, v7.16b\n"
"eor v12.16b, v12.16b, v14.16b\n"
"eor v13.16b, v13.16b, v15.16b\n"
"cmp %0, #4\n"
"b.ls 1f\n"
"ldr x5, [%2, #32]\n"
"ldp q24, q25, [x5]\n"
"tbl v14.16b, {v24.16b}, v8.16b\n"
"tbl v15.16b, {v25.16b}, v9.16b\n"
"eor v12.16b, v12.16b, v14.16b\n"
"eor v13.16b, v13.16b, v15.16b\n"
"cmp %0, #5\n"
"b.ls 1f\n"
"ldr x5, [%2, #40]\n"
"ldp q24, q25, [x5]\n"
"tbl v14.16b, {v24.16b}, v10.16b\n"
"tbl v15.16b, {v25.16b}, v11.16b\n"
"eor v12.16b, v12.16b, v14.16b\n"
"eor v13.16b, v13.16b, v15.16b\n"
"1:\n"
"eor v12.16b, v12.16b, v13.16b\n"
"str q12, [%3, %1]\n"
:
: "r" ((uint64_t)N), "r" (i), "r" (&T[j * N]), "r" (pa[j])
: "x4", "x5", "cc", "memory"
);
}
}
raid_neon_end();
void raid_rec6_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 6);
raid_recX_neon(6, id, ip, nd, size, vv);
}
void raid_register_neon(void)
@ -1073,9 +1092,10 @@ void raid_register_neon(void)
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "neon", raid_rec1_neon, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "neon", raid_rec2_neon, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "neon", raid_recX_neon, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "neon", raid_recX_neon, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "neon", raid_recX_neon, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "neon", raid_recX_neon, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "neon", raid_rec3_neon, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "neon", raid_rec4_neon, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "neon", raid_rec5_neon, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "neon", raid_rec6_neon, RAID_POLY_ANY);
}
#endif

1004
raid/neon32.c
View file

File diff suppressed because it is too large Load diff

2666
raid/ssse3.c
View file

File diff suppressed because it is too large Load diff

653
raid/test.c
View file

@ -70,6 +70,19 @@ int raid_test_gfaffine(void)
if (raid_gfaffine_mul((uint8_t)c, (uint8_t)x) != raid_gfmul_raid[c][x])
return -1;
#ifdef CONFIG_X86_64
int d, p;
for (d = 0; d < RAID_DATA_MAX; ++d) {
for (p = 1; p < RAID_PARITY_MAX; ++p) {
if (memcmp(raid_gfcauchyaffine_raid[d][p - 1],
raid_gfaffine_raid[raid_gfcauchy_raid[p][d]],
8) != 0)
return -1;
}
}
#endif
return 0;
}
@ -391,101 +404,245 @@ int raid_test_rec(int mode, int nd, size_t size)
/* setup recov functions */
for (i = 1; i <= np; ++i) {
if (i == 1) {
switch (i) {
case 1:
test_setup(i) = raid_rec1_int8;
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3())
test_setup(i) = raid_rec1_ssse3;
if (raid_cpu_has_avx2())
test_setup(i) = raid_rec1_avx2;
#ifdef CONFIG_X86_64
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
if (raid_cpu_has_avx2gfni()) {
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(i) = raid_rec1_avx2gfni_aes;
else
test_setup(i) = raid_rec1_avx2gfni_raid;
}
if (raid_cpu_has_avx512gfni()) {
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(i) = raid_rec1_avx512gfni_aes;
else
test_setup(i) = raid_rec1_avx512gfni_raid;
}
}
#endif
#endif
#ifdef CONFIG_NEON
test_setup(i) = raid_rec1_neon;
#endif
#ifdef CONFIG_NEON32
test_setup(i) = raid_rec1_neon32;
#endif
} else if (i == 2) {
break;
case 2:
test_setup(i) = raid_rec2_int8;
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3())
test_setup(i) = raid_rec2_ssse3;
if (raid_cpu_has_avx2())
test_setup(i) = raid_rec2_avx2;
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx512bw())
test_setup(i) = raid_rec2_avx512bw;
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
if (raid_cpu_has_avx2gfni()) {
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(i) = raid_rec2_avx2gfni_aes;
else
test_setup(i) = raid_rec2_avx2gfni_raid;
}
if (raid_cpu_has_avx512gfni()) {
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(i) = raid_rec2_avx512gfni_aes;
else
test_setup(i) = raid_rec2_avx512gfni_raid;
}
}
#endif
#endif
#ifdef CONFIG_NEON
test_setup(i) = raid_rec2_neon;
#endif
#ifdef CONFIG_NEON32
test_setup(i) = raid_rec2_neon32;
#endif
} else {
break;
default:
test_setup(i) = raid_recX_int8;
break;
}
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3())
test_setup(i) = raid_recX_ssse3;
if (raid_cpu_has_avx2())
test_setup(i) = raid_recX_avx2;
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx512bw())
test_setup(i) = raid_recX_avx512bw;
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
if (raid_cpu_has_avx2gfni()) {
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(i) = raid_recX_avx2gfni_aes;
else
test_setup(i) = raid_recX_avx2gfni_raid;
}
if (raid_cpu_has_avx512gfni()) {
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(i) = raid_recX_avx512gfni_aes;
else
test_setup(i) = raid_recX_avx512gfni_raid;
}
if (raid_cpu_has_ssse3()) {
switch (i) {
case 1:
test_setup(i) = raid_rec1_ssse3;
break;
case 2:
test_setup(i) = raid_rec2_ssse3;
break;
case 3:
test_setup(i) = raid_rec3_ssse3;
break;
case 4:
test_setup(i) = raid_rec4_ssse3;
break;
case 5:
test_setup(i) = raid_rec5_ssse3;
break;
case 6:
test_setup(i) = raid_rec6_ssse3;
break;
}
#ifdef CONFIG_X86_64
switch (i) {
case 1:
test_setup(i) = raid_rec1_ssse3ext;
break;
case 2:
test_setup(i) = raid_rec2_ssse3ext;
break;
case 3:
test_setup(i) = raid_rec3_ssse3ext;
break;
case 4:
test_setup(i) = raid_rec4_ssse3ext;
break;
case 5:
test_setup(i) = raid_rec5_ssse3ext;
break;
case 6:
test_setup(i) = raid_rec6_ssse3ext;
break;
}
#endif
#endif
#ifdef CONFIG_NEON
test_setup(i) = raid_recX_neon;
#endif
#ifdef CONFIG_NEON32
test_setup(i) = raid_recX_neon32;
#endif
}
if (raid_cpu_has_avx2()) {
switch (i) {
case 1:
test_setup(i) = raid_rec1_avx2;
break;
case 2:
test_setup(i) = raid_rec2_avx2;
break;
case 3:
test_setup(i) = raid_rec3_avx2;
break;
case 4:
test_setup(i) = raid_rec4_avx2;
break;
case 5:
test_setup(i) = raid_rec5_avx2;
break;
case 6:
test_setup(i) = raid_rec6_avx2;
break;
}
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx512bw()) {
switch (i) {
case 1:
test_setup(i) = raid_rec1_avx512bw;
break;
case 2:
test_setup(i) = raid_rec2_avx512bw;
break;
case 3:
test_setup(i) = raid_rec3_avx512bw;
break;
case 4:
test_setup(i) = raid_rec4_avx512bw;
break;
case 5:
test_setup(i) = raid_rec5_avx512bw;
break;
case 6:
test_setup(i) = raid_rec6_avx512bw;
break;
}
}
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
if (raid_cpu_has_avx2gfni()) {
if (mode == RAID_MODE_CAUCHY_AES) {
switch (i) {
case 1:
test_setup(i) = raid_rec1_avx2gfni_aes;
break;
case 2:
test_setup(i) = raid_rec2_avx2gfni_aes;
break;
case 3:
test_setup(i) = raid_rec3_avx2gfni_aes;
break;
case 4:
test_setup(i) = raid_rec4_avx2gfni_aes;
break;
case 5:
test_setup(i) = raid_rec5_avx2gfni_aes;
break;
case 6:
test_setup(i) = raid_rec6_avx2gfni_aes;
break;
}
} else {
switch (i) {
case 1:
test_setup(i) = raid_rec1_avx2gfni_raid;
break;
case 2:
test_setup(i) = raid_rec2_avx2gfni_raid;
break;
case 3:
test_setup(i) = raid_rec3_avx2gfni_raid;
break;
case 4:
test_setup(i) = raid_rec4_avx2gfni_raid;
break;
case 5:
test_setup(i) = raid_rec5_avx2gfni_raid;
break;
case 6:
test_setup(i) = raid_rec6_avx2gfni_raid;
break;
}
}
}
if (raid_cpu_has_avx512gfni()) {
if (mode == RAID_MODE_CAUCHY_AES) {
switch (i) {
case 1:
test_setup(i) = raid_rec1_avx512gfni_aes;
break;
case 2:
test_setup(i) = raid_rec2_avx512gfni_aes;
break;
case 3:
test_setup(i) = raid_rec3_avx512gfni_aes;
break;
case 4:
test_setup(i) = raid_rec4_avx512gfni_aes;
break;
case 5:
test_setup(i) = raid_rec5_avx512gfni_aes;
break;
case 6:
test_setup(i) = raid_rec6_avx512gfni_aes;
break;
}
} else {
switch (i) {
case 1:
test_setup(i) = raid_rec1_avx512gfni_raid;
break;
case 2:
test_setup(i) = raid_rec2_avx512gfni_raid;
break;
case 3:
test_setup(i) = raid_rec3_avx512gfni_raid;
break;
case 4:
test_setup(i) = raid_rec4_avx512gfni_raid;
break;
case 5:
test_setup(i) = raid_rec5_avx512gfni_raid;
break;
case 6:
test_setup(i) = raid_rec6_avx512gfni_raid;
break;
}
}
}
}
#endif
#endif
#ifdef CONFIG_NEON
switch (i) {
case 1:
test_setup(i) = raid_rec1_neon;
break;
case 2:
test_setup(i) = raid_rec2_neon;
break;
case 3:
test_setup(i) = raid_rec3_neon;
break;
case 4:
test_setup(i) = raid_rec4_neon;
break;
case 5:
test_setup(i) = raid_rec5_neon;
break;
case 6:
test_setup(i) = raid_rec6_neon;
break;
}
#endif
#ifdef CONFIG_NEON32
switch (i) {
case 1:
test_setup(i) = raid_rec1_neon32;
break;
case 2:
test_setup(i) = raid_rec2_neon32;
break;
case 3:
test_setup(i) = raid_rec3_neon32;
break;
case 4:
test_setup(i) = raid_rec4_neon32;
break;
case 5:
test_setup(i) = raid_rec5_neon32;
break;
case 6:
test_setup(i) = raid_rec6_neon32;
break;
}
#endif
}
/* compute the parity */
@ -607,101 +764,245 @@ int raid_test_tail(int mode, int nd_max, size_t size)
/* setup recov functions */
for (i = 1; i <= np; ++i) {
if (i == 1) {
switch (i) {
case 1:
test_setup(i) = raid_rec1_int8;
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3())
test_setup(i) = raid_rec1_ssse3;
if (raid_cpu_has_avx2())
test_setup(i) = raid_rec1_avx2;
#ifdef CONFIG_X86_64
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
if (raid_cpu_has_avx2gfni()) {
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(i) = raid_rec1_avx2gfni_aes;
else
test_setup(i) = raid_rec1_avx2gfni_raid;
}
if (raid_cpu_has_avx512gfni()) {
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(i) = raid_rec1_avx512gfni_aes;
else
test_setup(i) = raid_rec1_avx512gfni_raid;
}
}
#endif
#endif
#ifdef CONFIG_NEON
test_setup(i) = raid_rec1_neon;
#endif
#ifdef CONFIG_NEON32
test_setup(i) = raid_rec1_neon32;
#endif
} else if (i == 2) {
break;
case 2:
test_setup(i) = raid_rec2_int8;
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3())
test_setup(i) = raid_rec2_ssse3;
if (raid_cpu_has_avx2())
test_setup(i) = raid_rec2_avx2;
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx512bw())
test_setup(i) = raid_rec2_avx512bw;
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
if (raid_cpu_has_avx2gfni()) {
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(i) = raid_rec2_avx2gfni_aes;
else
test_setup(i) = raid_rec2_avx2gfni_raid;
}
if (raid_cpu_has_avx512gfni()) {
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(i) = raid_rec2_avx512gfni_aes;
else
test_setup(i) = raid_rec2_avx512gfni_raid;
}
}
#endif
#endif
#ifdef CONFIG_NEON
test_setup(i) = raid_rec2_neon;
#endif
#ifdef CONFIG_NEON32
test_setup(i) = raid_rec2_neon32;
#endif
} else {
break;
default:
test_setup(i) = raid_recX_int8;
break;
}
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3())
test_setup(i) = raid_recX_ssse3;
if (raid_cpu_has_avx2())
test_setup(i) = raid_recX_avx2;
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx512bw())
test_setup(i) = raid_recX_avx512bw;
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
if (raid_cpu_has_avx2gfni()) {
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(i) = raid_recX_avx2gfni_aes;
else
test_setup(i) = raid_recX_avx2gfni_raid;
}
if (raid_cpu_has_avx512gfni()) {
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(i) = raid_recX_avx512gfni_aes;
else
test_setup(i) = raid_recX_avx512gfni_raid;
}
if (raid_cpu_has_ssse3()) {
switch (i) {
case 1:
test_setup(i) = raid_rec1_ssse3;
break;
case 2:
test_setup(i) = raid_rec2_ssse3;
break;
case 3:
test_setup(i) = raid_rec3_ssse3;
break;
case 4:
test_setup(i) = raid_rec4_ssse3;
break;
case 5:
test_setup(i) = raid_rec5_ssse3;
break;
case 6:
test_setup(i) = raid_rec6_ssse3;
break;
}
#ifdef CONFIG_X86_64
switch (i) {
case 1:
test_setup(i) = raid_rec1_ssse3ext;
break;
case 2:
test_setup(i) = raid_rec2_ssse3ext;
break;
case 3:
test_setup(i) = raid_rec3_ssse3ext;
break;
case 4:
test_setup(i) = raid_rec4_ssse3ext;
break;
case 5:
test_setup(i) = raid_rec5_ssse3ext;
break;
case 6:
test_setup(i) = raid_rec6_ssse3ext;
break;
}
#endif
#endif
#ifdef CONFIG_NEON
test_setup(i) = raid_recX_neon;
#endif
#ifdef CONFIG_NEON32
test_setup(i) = raid_recX_neon32;
#endif
}
if (raid_cpu_has_avx2()) {
switch (i) {
case 1:
test_setup(i) = raid_rec1_avx2;
break;
case 2:
test_setup(i) = raid_rec2_avx2;
break;
case 3:
test_setup(i) = raid_rec3_avx2;
break;
case 4:
test_setup(i) = raid_rec4_avx2;
break;
case 5:
test_setup(i) = raid_rec5_avx2;
break;
case 6:
test_setup(i) = raid_rec6_avx2;
break;
}
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx512bw()) {
switch (i) {
case 1:
test_setup(i) = raid_rec1_avx512bw;
break;
case 2:
test_setup(i) = raid_rec2_avx512bw;
break;
case 3:
test_setup(i) = raid_rec3_avx512bw;
break;
case 4:
test_setup(i) = raid_rec4_avx512bw;
break;
case 5:
test_setup(i) = raid_rec5_avx512bw;
break;
case 6:
test_setup(i) = raid_rec6_avx512bw;
break;
}
}
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
if (raid_cpu_has_avx2gfni()) {
if (mode == RAID_MODE_CAUCHY_AES) {
switch (i) {
case 1:
test_setup(i) = raid_rec1_avx2gfni_aes;
break;
case 2:
test_setup(i) = raid_rec2_avx2gfni_aes;
break;
case 3:
test_setup(i) = raid_rec3_avx2gfni_aes;
break;
case 4:
test_setup(i) = raid_rec4_avx2gfni_aes;
break;
case 5:
test_setup(i) = raid_rec5_avx2gfni_aes;
break;
case 6:
test_setup(i) = raid_rec6_avx2gfni_aes;
break;
}
} else {
switch (i) {
case 1:
test_setup(i) = raid_rec1_avx2gfni_raid;
break;
case 2:
test_setup(i) = raid_rec2_avx2gfni_raid;
break;
case 3:
test_setup(i) = raid_rec3_avx2gfni_raid;
break;
case 4:
test_setup(i) = raid_rec4_avx2gfni_raid;
break;
case 5:
test_setup(i) = raid_rec5_avx2gfni_raid;
break;
case 6:
test_setup(i) = raid_rec6_avx2gfni_raid;
break;
}
}
}
if (raid_cpu_has_avx512gfni()) {
if (mode == RAID_MODE_CAUCHY_AES) {
switch (i) {
case 1:
test_setup(i) = raid_rec1_avx512gfni_aes;
break;
case 2:
test_setup(i) = raid_rec2_avx512gfni_aes;
break;
case 3:
test_setup(i) = raid_rec3_avx512gfni_aes;
break;
case 4:
test_setup(i) = raid_rec4_avx512gfni_aes;
break;
case 5:
test_setup(i) = raid_rec5_avx512gfni_aes;
break;
case 6:
test_setup(i) = raid_rec6_avx512gfni_aes;
break;
}
} else {
switch (i) {
case 1:
test_setup(i) = raid_rec1_avx512gfni_raid;
break;
case 2:
test_setup(i) = raid_rec2_avx512gfni_raid;
break;
case 3:
test_setup(i) = raid_rec3_avx512gfni_raid;
break;
case 4:
test_setup(i) = raid_rec4_avx512gfni_raid;
break;
case 5:
test_setup(i) = raid_rec5_avx512gfni_raid;
break;
case 6:
test_setup(i) = raid_rec6_avx512gfni_raid;
break;
}
}
}
}
#endif
#endif
#ifdef CONFIG_NEON
switch (i) {
case 1:
test_setup(i) = raid_rec1_neon;
break;
case 2:
test_setup(i) = raid_rec2_neon;
break;
case 3:
test_setup(i) = raid_rec3_neon;
break;
case 4:
test_setup(i) = raid_rec4_neon;
break;
case 5:
test_setup(i) = raid_rec5_neon;
break;
case 6:
test_setup(i) = raid_rec6_neon;
break;
}
#endif
#ifdef CONFIG_NEON32
switch (i) {
case 1:
test_setup(i) = raid_rec1_neon32;
break;
case 2:
test_setup(i) = raid_rec2_neon32;
break;
case 3:
test_setup(i) = raid_rec3_neon32;
break;
case 4:
test_setup(i) = raid_rec4_neon32;
break;
case 5:
test_setup(i) = raid_rec5_neon32;
break;
case 6:
test_setup(i) = raid_rec6_neon32;
break;
}
#endif
}
/* test all disk counts from 1 to nd_max */

5
raid/test/Makefile
View file

@ -103,6 +103,11 @@ sdecovtest:
# Use this target to run a complete test on Ice Lake simulator
sdetest:
$(SDE) -p4p -- ./selftest
$(SDE) -mrm -- ./selftest
$(SDE) -nhm -- ./selftest
$(SDE) -hsw -- ./selftest
$(SDE) -skx -- ./selftest
$(SDE) -icl -- ./selftest
$(SDE) -icl -- ./fulltest

552
raid/test/speedtest.c
View file

@ -1122,10 +1122,18 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
printf("%8s", "neon32");
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3())
if (raid_cpu_has_ssse3()) {
printf("%8s", "ssse3");
if (raid_cpu_has_avx2())
#ifdef CONFIG_X86_64
printf("%8s", "ssse3e");
#endif
}
if (raid_cpu_has_avx2()) {
printf("%8s", "avx2");
#ifdef CONFIG_X86_64
printf("%8s", "avx2e");
#endif
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx512bw())
printf("%8s", "avx512");
@ -1172,12 +1180,24 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
SPEED_START {
/* raid_rec1of1() internally calls raid_gen() */
raid_gen_force(1, raid_gen1_sse2);
raid_rec1_ssse3(1, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
/* raid_rec1of1() internally calls raid_gen() */
raid_gen_force(1, raid_gen1_sse2);
raid_rec1_ssse3ext(1, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
@ -1189,6 +1209,19 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
/*
* raid_rec1_avx2ext() uses raid_rec1of1() for P,
* which internally calls raid_gen().
*/
raid_gen_force(1, raid_gen1_avx2);
raid_rec1_avx2ext(1, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
raid_gen_force(1, raid_gen1_avx512bw);
@ -1236,7 +1269,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
raid_gen_force(1, raid_gen1_neon);
/* +1 to avoid GEN1 optimized case */
raid_rec1_neon(1, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1246,7 +1278,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen_force(1, raid_gen1_neon32);
/* +1 to avoid GEN1 optimized case */
raid_rec1_neon32(1, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1258,6 +1289,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, raid_gen1_sse2);
/* +1 to avoid GEN1 optimized case */
raid_rec1_ssse3(1, id, ip + 1, nd, size, v);
@ -1265,6 +1297,18 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, raid_gen1_sse2);
/* +1 to avoid GEN1 optimized case */
raid_rec1_ssse3ext(1, id, ip + 1, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
@ -1277,10 +1321,20 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, raid_gen1_avx2);
/* +1 to avoid GEN1 optimized case */
raid_rec1_avx2ext(1, id, ip + 1, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
raid_gen_force(1, raid_gen1_avx512bw);
/* +1 to avoid GEN1 optimized case */
/* +1 to avoid RAID5 optimized case */
raid_rec1_avx512bw(1, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1289,8 +1343,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
raid_gen_force(1, raid_gen1_avx2);
/* +1 to avoid GEN1 optimized case */
/* +1 to avoid RAID5 optimized case */
raid_rec1_avx2gfni_raid(1, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1299,8 +1352,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
raid_gen_force(1, raid_gen1_avx512bw); /* there is no raid_gen1_avx512gfni */
/* +1 to avoid GEN1 optimized case */
/* +1 to avoid RAID5 optimized case */
raid_rec1_avx512gfni_raid(1, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1311,6 +1363,116 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#endif
printf("\n");
/* REC1OF3 */
printf("%8s", "rec1of3");
printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR1));
fflush(stdout);
SPEED_START {
raid_gen_force(1, sizeof(void *) == 8 ? raid_gen1_int64 : raid_gen1_int32);
/* +2 to select R instead of Q */
raid_rec1_int8(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_NEON
SPEED_START {
/* +2 to select R instead of Q */
raid_rec1_neon(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* +2 to select R instead of Q */
raid_rec1_neon32(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, raid_gen1_sse2);
/* +2 to select R instead of Q */
raid_rec1_ssse3(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, raid_gen1_sse2);
/* +2 to select R instead of Q */
raid_rec1_ssse3ext(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
raid_gen_force(1, raid_gen1_avx2);
/* +2 to select R instead of Q */
raid_rec1_avx2(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(1, raid_gen1_avx2);
/* +2 to select R instead of Q */
raid_rec1_avx2ext(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
/* +2 to select R instead of Q */
raid_rec1_avx512bw(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
/* +2 to select R instead of Q */
raid_rec1_avx2gfni_raid(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
/* +2 to select R instead of Q */
raid_rec1_avx512gfni_raid(1, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#endif
#endif
printf("\n");
/* REC2OF2 */
printf("%8s", "rec2of2");
printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR2));
@ -1326,7 +1488,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
raid_gen_force(2, raid_gen2_neon_raid);
raid_rec2_neon(2, id, ip, nd, size, v);
} SPEED_STOP
@ -1335,7 +1496,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen_force(2, raid_gen2_neon32_raid);
raid_rec2_neon32(2, id, ip, nd, size, v);
} SPEED_STOP
@ -1346,24 +1506,28 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
SPEED_START {
#ifdef CONFIG_X86_64
raid_gen_force(2, raid_gen2_sse2ext_raid);
#else
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_sse2_raid);
#endif
raid_rec2_ssse3(2, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_sse2ext_raid);
raid_rec2_ssse3ext(2, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
#ifdef CONFIG_X86_64
raid_gen_force(2, raid_gen2_avx2ext_raid);
#else
raid_gen_force(2, raid_gen2_avx2_raid);
#endif
raid_rec2_avx2(2, id, ip, nd, size, v);
} SPEED_STOP
@ -1371,9 +1535,18 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_avx2ext_raid);
raid_rec2_avx2ext(2, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
raid_gen_force(2, raid_gen2_avx512bw);
raid_rec2_avx512bw(2, id, ip, nd, size, v);
} SPEED_STOP
@ -1382,7 +1555,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
raid_gen_force(2, raid_gen2_avx2gfni_raid);
raid_rec2_avx2gfni_raid(2, id, ip, nd, size, v);
} SPEED_STOP
@ -1391,7 +1563,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
raid_gen_force(2, raid_gen2_avx512gfni_raid);
raid_rec2_avx512gfni_raid(2, id, ip, nd, size, v);
} SPEED_STOP
@ -1418,7 +1589,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
raid_gen_force(2, raid_gen2_neon_raid);
/* +1 to avoid GEN2 optimized case */
raid_rec2_neon(2, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1428,7 +1598,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen_force(2, raid_gen2_neon32_raid);
/* +1 to avoid GEN2 optimized case */
raid_rec2_neon32(2, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1440,25 +1609,30 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
SPEED_START {
#ifdef CONFIG_X86_64
raid_gen_force(2, raid_gen2_sse2ext_raid);
#else
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_sse2_raid);
#endif
/* +1 to avoid GEN2 optimized case */
raid_rec2_ssse3(2, id, ip + 1, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_sse2ext_raid);
/* +1 to avoid GEN2 optimized case */
raid_rec2_ssse3ext(2, id, ip + 1, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
#ifdef CONFIG_X86_64
raid_gen_force(2, raid_gen2_avx2ext_raid);
#else
raid_gen_force(2, raid_gen2_avx2_raid);
#endif
/* +1 to avoid GEN2 optimized case */
raid_rec2_avx2(2, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1467,9 +1641,19 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_avx2ext_raid);
/* +1 to avoid GEN2 optimized case */
raid_rec2_avx2ext(2, id, ip + 1, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
raid_gen_force(2, raid_gen2_avx512bw);
/* +1 to avoid GEN2 optimized case */
raid_rec2_avx512bw(2, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1479,7 +1663,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
raid_gen_force(2, raid_gen2_avx2gfni_raid);
/* +1 to avoid GEN2 optimized case */
raid_rec2_avx2gfni_raid(2, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1489,7 +1672,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
raid_gen_force(2, raid_gen2_avx512gfni_raid);
/* +1 to avoid GEN2 optimized case */
raid_rec2_avx512gfni_raid(2, id, ip + 1, nd, size, v);
} SPEED_STOP
@ -1501,6 +1683,116 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#endif
printf("\n");
/* REC2OF4 */
printf("%8s", "rec2of4");
printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR2));
fflush(stdout);
SPEED_START {
raid_gen_force(2, sizeof(void *) == 8 ? raid_gen2_int64_raid : raid_gen2_int32_raid);
/* +2 to select R,S instead of Q,R */
raid_rec2_int8(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_NEON
SPEED_START {
/* +2 to select R,S instead of Q,R */
raid_rec2_neon(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_NEON32
SPEED_START {
/* +2 to select R,S instead of Q,R */
raid_rec2_neon32(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_sse2_raid);
/* +2 to select R,S instead of Q,R */
raid_rec2_ssse3(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_sse2ext_raid);
/* +2 to select R,S instead of Q,R */
raid_rec2_ssse3ext(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
raid_gen_force(2, raid_gen2_avx2_raid);
/* +2 to select R,S instead of Q,R */
raid_rec2_avx2(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
/* ensure to use same hardware in the delta step */
raid_gen_force(2, raid_gen2_avx2ext_raid);
/* +2 to select R,S instead of Q,R */
raid_rec2_avx2ext(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
/* +2 to select R,S instead of Q,R */
raid_rec2_avx512bw(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
/* +2 to select R,S instead of Q,R */
raid_rec2_avx2gfni_raid(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
/* +2 to select R,S instead of Q,R */
raid_rec2_avx512gfni_raid(2, id, ip + 2, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#endif
#endif
printf("\n");
/* REC3OF3 */
printf("%8s", "rec3of3");
printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR3));
@ -1516,8 +1808,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
raid_gen_force(3, raid_gen3_neon_raid);
raid_recX_neon(3, id, ip, nd, size, v);
raid_rec3_neon(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1525,8 +1816,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen_force(3, raid_gen3_neon32_raid);
raid_recX_neon32(3, id, ip, nd, size, v);
raid_rec3_neon32(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1536,36 +1826,41 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
SPEED_START {
#ifdef CONFIG_X86_64
raid_gen_force(3, raid_gen3_ssse3ext_raid);
#else
raid_gen_force(3, raid_gen3_ssse3_raid);
#endif
raid_recX_ssse3(3, id, ip, nd, size, v);
raid_rec3_ssse3(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
raid_rec3_ssse3ext(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
#ifdef CONFIG_X86_64
raid_gen_force(3, raid_gen3_avx2ext_raid);
#else
raid_gen_force(3, raid_gen3_ssse3_raid);
#endif
raid_recX_avx2(3, id, ip, nd, size, v);
raid_rec3_avx2(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
raid_rec3_avx2ext(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
raid_gen_force(3, raid_gen3_avx512bw);
/* +1 to avoid GEN1 optimized case */
raid_recX_avx512bw(3, id, ip, nd, size, v);
raid_rec3_avx512bw(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1573,8 +1868,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
raid_gen_force(3, raid_gen3_avx2gfni_raid);
raid_recX_avx2gfni_raid(3, id, ip, nd, size, v);
raid_rec3_avx2gfni_raid(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1582,8 +1876,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
raid_gen_force(3, raid_gen3_avx512gfni_raid);
raid_recX_avx512gfni_raid(3, id, ip, nd, size, v);
raid_rec3_avx512gfni_raid(3, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1608,8 +1901,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
raid_gen_force(4, raid_gen4_neon_raid);
raid_recX_neon(4, id, ip, nd, size, v);
raid_rec4_neon(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1617,8 +1909,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen_force(4, raid_gen4_neon32_raid);
raid_recX_neon32(4, id, ip, nd, size, v);
raid_rec4_neon32(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1628,36 +1919,41 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
SPEED_START {
#ifdef CONFIG_X86_64
raid_gen_force(4, raid_gen4_ssse3ext_raid);
#else
raid_gen_force(4, raid_gen4_ssse3_raid);
#endif
raid_recX_ssse3(4, id, ip, nd, size, v);
raid_rec4_ssse3(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
raid_rec4_ssse3ext(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
#ifdef CONFIG_X86_64
raid_gen_force(4, raid_gen4_avx2ext_raid);
#else
raid_gen_force(4, raid_gen4_ssse3_raid);
#endif
raid_recX_avx2(4, id, ip, nd, size, v);
raid_rec4_avx2(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
raid_rec4_avx2ext(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
raid_gen_force(4, raid_gen4_avx512bw);
/* +1 to avoid GEN1 optimized case */
raid_recX_avx512bw(4, id, ip, nd, size, v);
raid_rec4_avx512bw(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1665,8 +1961,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
raid_gen_force(4, raid_gen4_avx2gfni_raid);
raid_recX_avx2gfni_raid(4, id, ip, nd, size, v);
raid_rec4_avx2gfni_raid(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1674,8 +1969,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
raid_gen_force(4, raid_gen4_avx512gfni_raid);
raid_recX_avx512gfni_raid(4, id, ip, nd, size, v);
raid_rec4_avx512gfni_raid(4, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1700,8 +1994,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
raid_gen_force(5, raid_gen5_neon_raid);
raid_recX_neon(5, id, ip, nd, size, v);
raid_rec5_neon(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1709,8 +2002,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen_force(5, raid_gen5_neon32_raid);
raid_recX_neon32(5, id, ip, nd, size, v);
raid_rec5_neon32(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1720,36 +2012,41 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
SPEED_START {
#ifdef CONFIG_X86_64
raid_gen_force(5, raid_gen5_ssse3ext_raid);
#else
raid_gen_force(5, raid_gen5_ssse3_raid);
#endif
raid_recX_ssse3(5, id, ip, nd, size, v);
raid_rec5_ssse3(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
raid_rec5_ssse3ext(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
#ifdef CONFIG_X86_64
raid_gen_force(5, raid_gen5_avx2ext_raid);
#else
raid_gen_force(5, raid_gen5_ssse3_raid);
#endif
raid_recX_avx2(5, id, ip, nd, size, v);
raid_rec5_avx2(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
raid_rec5_avx2ext(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
raid_gen_force(5, raid_gen5_avx512bw);
/* +1 to avoid GEN1 optimized case */
raid_recX_avx512bw(5, id, ip, nd, size, v);
raid_rec5_avx512bw(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1757,8 +2054,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
raid_gen_force(5, raid_gen5_avx2gfni_raid);
raid_recX_avx2gfni_raid(5, id, ip, nd, size, v);
raid_rec5_avx2gfni_raid(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1766,8 +2062,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
raid_gen_force(5, raid_gen5_avx512gfni_raid);
raid_recX_avx512gfni_raid(5, id, ip, nd, size, v);
raid_rec5_avx512gfni_raid(5, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1792,8 +2087,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_NEON
SPEED_START {
raid_gen_force(6, raid_gen6_neon_raid);
raid_recX_neon(6, id, ip, nd, size, v);
raid_rec6_neon(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1801,8 +2095,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#endif
#ifdef CONFIG_NEON32
SPEED_START {
raid_gen_force(6, raid_gen6_neon32_raid);
raid_recX_neon32(6, id, ip, nd, size, v);
raid_rec6_neon32(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1812,36 +2105,41 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
SPEED_START {
#ifdef CONFIG_X86_64
raid_gen_force(6, raid_gen6_ssse3ext_raid);
#else
raid_gen_force(6, raid_gen6_ssse3_raid);
#endif
raid_recX_ssse3(6, id, ip, nd, size, v);
raid_rec6_ssse3(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#ifdef CONFIG_X86_64
SPEED_START {
raid_rec6_ssse3ext(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
#endif
}
if (raid_cpu_has_avx2()) {
SPEED_START {
#ifdef CONFIG_X86_64
raid_gen_force(6, raid_gen6_avx2ext_raid);
#else
raid_gen_force(6, raid_gen6_ssse3_raid);
#endif
raid_recX_avx2(6, id, ip, nd, size, v);
raid_rec6_avx2(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
raid_rec6_avx2ext(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx512bw()) {
SPEED_START {
raid_gen_force(6, raid_gen6_avx512bw);
/* +1 to avoid GEN1 optimized case */
raid_recX_avx512bw(6, id, ip, nd, size, v);
raid_rec6_avx512bw(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1849,8 +2147,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx2gfni()) {
SPEED_START {
raid_gen_force(6, raid_gen6_avx2gfni_raid);
raid_recX_avx2gfni_raid(6, id, ip, nd, size, v);
raid_rec6_avx2gfni_raid(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
@ -1858,8 +2155,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx512gfni()) {
SPEED_START {
raid_gen_force(6, raid_gen6_avx512gfni_raid);
raid_recX_avx512gfni_raid(6, id, ip, nd, size, v);
raid_rec6_avx512gfni_raid(6, id, ip, nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);