Optimize multi-parity recovery with fused syndrome generation
Some checks are pending
Build / build-mac (push) Waiting to run
Build / build-linux (push) Waiting to run
Build / build-arm32 (push) Waiting to run
Build / build-win (push) Waiting to run
CodeQL Advanced / Analyze (c-cpp) (push) Waiting to run
CodeQL Advanced / Analyze (actions) (push) Waiting to run

This commit is contained in:
Andrea Mazzoleni 2026-08-23 22:28:21 +02:00
parent 5662eab6cf
commit e573e9d4cf
9 changed files with 3926 additions and 2645 deletions

View file

@ -391,101 +391,214 @@ int raid_test_rec(int mode, int nd, size_t size)
/* setup recov functions */
for (i = 1; i <= np; ++i) {
if (i == 1) {
switch (i) {
case 1:
test_setup(i) = raid_rec1_int8;
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3())
test_setup(i) = raid_rec1_ssse3;
if (raid_cpu_has_avx2())
test_setup(i) = raid_rec1_avx2;
#ifdef CONFIG_X86_64
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
if (raid_cpu_has_avx2gfni()) {
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(i) = raid_rec1_avx2gfni_aes;
else
test_setup(i) = raid_rec1_avx2gfni_raid;
}
if (raid_cpu_has_avx512gfni()) {
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(i) = raid_rec1_avx512gfni_aes;
else
test_setup(i) = raid_rec1_avx512gfni_raid;
}
}
#endif
#endif
#ifdef CONFIG_NEON
test_setup(i) = raid_rec1_neon;
#endif
#ifdef CONFIG_NEON32
test_setup(i) = raid_rec1_neon32;
#endif
} else if (i == 2) {
break;
case 2:
test_setup(i) = raid_rec2_int8;
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3())
test_setup(i) = raid_rec2_ssse3;
if (raid_cpu_has_avx2())
test_setup(i) = raid_rec2_avx2;
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx512bw())
test_setup(i) = raid_rec2_avx512bw;
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
if (raid_cpu_has_avx2gfni()) {
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(i) = raid_rec2_avx2gfni_aes;
else
test_setup(i) = raid_rec2_avx2gfni_raid;
}
if (raid_cpu_has_avx512gfni()) {
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(i) = raid_rec2_avx512gfni_aes;
else
test_setup(i) = raid_rec2_avx512gfni_raid;
}
}
#endif
#endif
#ifdef CONFIG_NEON
test_setup(i) = raid_rec2_neon;
#endif
#ifdef CONFIG_NEON32
test_setup(i) = raid_rec2_neon32;
#endif
} else {
break;
default:
test_setup(i) = raid_recX_int8;
break;
}
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3())
if (raid_cpu_has_ssse3()) {
switch (i) {
case 1:
test_setup(i) = raid_rec1_ssse3;
break;
case 2:
test_setup(i) = raid_rec2_ssse3;
break;
default:
test_setup(i) = raid_recX_ssse3;
if (raid_cpu_has_avx2())
test_setup(i) = raid_recX_avx2;
break;
}
}
if (raid_cpu_has_avx2()) {
switch (i) {
case 1:
test_setup(i) = raid_rec1_avx2;
break;
case 2:
test_setup(i) = raid_rec2_avx2;
break;
case 3:
test_setup(i) = raid_rec3_avx2;
break;
case 4:
test_setup(i) = raid_rec4_avx2;
break;
case 5:
test_setup(i) = raid_rec5_avx2;
break;
case 6:
test_setup(i) = raid_rec6_avx2;
break;
}
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx512bw())
test_setup(i) = raid_recX_avx512bw;
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
if (raid_cpu_has_avx2gfni()) {
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(i) = raid_recX_avx2gfni_aes;
else
test_setup(i) = raid_recX_avx2gfni_raid;
}
if (raid_cpu_has_avx512gfni()) {
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(i) = raid_recX_avx512gfni_aes;
else
test_setup(i) = raid_recX_avx512gfni_raid;
if (raid_cpu_has_avx512bw()) {
switch (i) {
case 1:
test_setup(i) = raid_rec1_avx512bw;
break;
case 2:
test_setup(i) = raid_rec2_avx512bw;
break;
case 3:
test_setup(i) = raid_rec3_avx512bw;
break;
case 4:
test_setup(i) = raid_rec4_avx512bw;
break;
case 5:
test_setup(i) = raid_rec5_avx512bw;
break;
case 6:
test_setup(i) = raid_rec6_avx512bw;
break;
}
}
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
if (raid_cpu_has_avx2gfni()) {
if (mode == RAID_MODE_CAUCHY_AES) {
switch (i) {
case 1:
test_setup(i) = raid_rec1_avx2gfni_aes;
break;
case 2:
test_setup(i) = raid_rec2_avx2gfni_aes;
break;
case 3:
test_setup(i) = raid_rec3_avx2gfni_aes;
break;
case 4:
test_setup(i) = raid_rec4_avx2gfni_aes;
break;
case 5:
test_setup(i) = raid_rec5_avx2gfni_aes;
break;
case 6:
test_setup(i) = raid_rec6_avx2gfni_aes;
break;
}
} else {
switch (i) {
case 1:
test_setup(i) = raid_rec1_avx2gfni_raid;
break;
case 2:
test_setup(i) = raid_rec2_avx2gfni_raid;
break;
case 3:
test_setup(i) = raid_rec3_avx2gfni_raid;
break;
case 4:
test_setup(i) = raid_rec4_avx2gfni_raid;
break;
case 5:
test_setup(i) = raid_rec5_avx2gfni_raid;
break;
case 6:
test_setup(i) = raid_rec6_avx2gfni_raid;
break;
}
}
}
if (raid_cpu_has_avx512gfni()) {
if (mode == RAID_MODE_CAUCHY_AES) {
switch (i) {
case 1:
test_setup(i) = raid_rec1_avx512gfni_aes;
break;
case 2:
test_setup(i) = raid_rec2_avx512gfni_aes;
break;
case 3:
test_setup(i) = raid_rec3_avx512gfni_aes;
break;
case 4:
test_setup(i) = raid_rec4_avx512gfni_aes;
break;
case 5:
test_setup(i) = raid_rec5_avx512gfni_aes;
break;
case 6:
test_setup(i) = raid_rec6_avx512gfni_aes;
break;
}
} else {
switch (i) {
case 1:
test_setup(i) = raid_rec1_avx512gfni_raid;
break;
case 2:
test_setup(i) = raid_rec2_avx512gfni_raid;
break;
case 3:
test_setup(i) = raid_rec3_avx512gfni_raid;
break;
case 4:
test_setup(i) = raid_rec4_avx512gfni_raid;
break;
case 5:
test_setup(i) = raid_rec5_avx512gfni_raid;
break;
case 6:
test_setup(i) = raid_rec6_avx512gfni_raid;
break;
}
}
}
}
#endif
#endif
#ifdef CONFIG_NEON
test_setup(i) = raid_recX_neon;
switch (i) {
case 1:
test_setup(i) = raid_rec1_neon;
break;
case 2:
test_setup(i) = raid_rec2_neon;
break;
case 3:
test_setup(i) = raid_rec3_neon;
break;
case 4:
test_setup(i) = raid_rec4_neon;
break;
case 5:
test_setup(i) = raid_rec5_neon;
break;
case 6:
test_setup(i) = raid_rec6_neon;
break;
}
#endif
#ifdef CONFIG_NEON32
test_setup(i) = raid_recX_neon32;
#endif
switch (i) {
case 1:
test_setup(i) = raid_rec1_neon32;
break;
case 2:
test_setup(i) = raid_rec2_neon32;
break;
case 3:
test_setup(i) = raid_rec3_neon32;
break;
case 4:
test_setup(i) = raid_rec4_neon32;
break;
case 5:
test_setup(i) = raid_rec5_neon32;
break;
case 6:
test_setup(i) = raid_rec6_neon32;
break;
}
#endif
}
/* compute the parity */
@ -607,101 +720,214 @@ int raid_test_tail(int mode, int nd_max, size_t size)
/* setup recov functions */
for (i = 1; i <= np; ++i) {
if (i == 1) {
switch (i) {
case 1:
test_setup(i) = raid_rec1_int8;
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3())
test_setup(i) = raid_rec1_ssse3;
if (raid_cpu_has_avx2())
test_setup(i) = raid_rec1_avx2;
#ifdef CONFIG_X86_64
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
if (raid_cpu_has_avx2gfni()) {
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(i) = raid_rec1_avx2gfni_aes;
else
test_setup(i) = raid_rec1_avx2gfni_raid;
}
if (raid_cpu_has_avx512gfni()) {
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(i) = raid_rec1_avx512gfni_aes;
else
test_setup(i) = raid_rec1_avx512gfni_raid;
}
}
#endif
#endif
#ifdef CONFIG_NEON
test_setup(i) = raid_rec1_neon;
#endif
#ifdef CONFIG_NEON32
test_setup(i) = raid_rec1_neon32;
#endif
} else if (i == 2) {
break;
case 2:
test_setup(i) = raid_rec2_int8;
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3())
test_setup(i) = raid_rec2_ssse3;
if (raid_cpu_has_avx2())
test_setup(i) = raid_rec2_avx2;
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx512bw())
test_setup(i) = raid_rec2_avx512bw;
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
if (raid_cpu_has_avx2gfni()) {
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(i) = raid_rec2_avx2gfni_aes;
else
test_setup(i) = raid_rec2_avx2gfni_raid;
}
if (raid_cpu_has_avx512gfni()) {
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(i) = raid_rec2_avx512gfni_aes;
else
test_setup(i) = raid_rec2_avx512gfni_raid;
}
}
#endif
#endif
#ifdef CONFIG_NEON
test_setup(i) = raid_rec2_neon;
#endif
#ifdef CONFIG_NEON32
test_setup(i) = raid_rec2_neon32;
#endif
} else {
break;
default:
test_setup(i) = raid_recX_int8;
break;
}
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3())
if (raid_cpu_has_ssse3()) {
switch (i) {
case 1:
test_setup(i) = raid_rec1_ssse3;
break;
case 2:
test_setup(i) = raid_rec2_ssse3;
break;
default:
test_setup(i) = raid_recX_ssse3;
if (raid_cpu_has_avx2())
test_setup(i) = raid_recX_avx2;
break;
}
}
if (raid_cpu_has_avx2()) {
switch (i) {
case 1:
test_setup(i) = raid_rec1_avx2;
break;
case 2:
test_setup(i) = raid_rec2_avx2;
break;
case 3:
test_setup(i) = raid_rec3_avx2;
break;
case 4:
test_setup(i) = raid_rec4_avx2;
break;
case 5:
test_setup(i) = raid_rec5_avx2;
break;
case 6:
test_setup(i) = raid_rec6_avx2;
break;
}
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx512bw())
test_setup(i) = raid_recX_avx512bw;
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
if (raid_cpu_has_avx2gfni()) {
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(i) = raid_recX_avx2gfni_aes;
else
test_setup(i) = raid_recX_avx2gfni_raid;
}
if (raid_cpu_has_avx512gfni()) {
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(i) = raid_recX_avx512gfni_aes;
else
test_setup(i) = raid_recX_avx512gfni_raid;
if (raid_cpu_has_avx512bw()) {
switch (i) {
case 1:
test_setup(i) = raid_rec1_avx512bw;
break;
case 2:
test_setup(i) = raid_rec2_avx512bw;
break;
case 3:
test_setup(i) = raid_rec3_avx512bw;
break;
case 4:
test_setup(i) = raid_rec4_avx512bw;
break;
case 5:
test_setup(i) = raid_rec5_avx512bw;
break;
case 6:
test_setup(i) = raid_rec6_avx512bw;
break;
}
}
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
if (raid_cpu_has_avx2gfni()) {
if (mode == RAID_MODE_CAUCHY_AES) {
switch (i) {
case 1:
test_setup(i) = raid_rec1_avx2gfni_aes;
break;
case 2:
test_setup(i) = raid_rec2_avx2gfni_aes;
break;
case 3:
test_setup(i) = raid_rec3_avx2gfni_aes;
break;
case 4:
test_setup(i) = raid_rec4_avx2gfni_aes;
break;
case 5:
test_setup(i) = raid_rec5_avx2gfni_aes;
break;
case 6:
test_setup(i) = raid_rec6_avx2gfni_aes;
break;
}
} else {
switch (i) {
case 1:
test_setup(i) = raid_rec1_avx2gfni_raid;
break;
case 2:
test_setup(i) = raid_rec2_avx2gfni_raid;
break;
case 3:
test_setup(i) = raid_rec3_avx2gfni_raid;
break;
case 4:
test_setup(i) = raid_rec4_avx2gfni_raid;
break;
case 5:
test_setup(i) = raid_rec5_avx2gfni_raid;
break;
case 6:
test_setup(i) = raid_rec6_avx2gfni_raid;
break;
}
}
}
if (raid_cpu_has_avx512gfni()) {
if (mode == RAID_MODE_CAUCHY_AES) {
switch (i) {
case 1:
test_setup(i) = raid_rec1_avx512gfni_aes;
break;
case 2:
test_setup(i) = raid_rec2_avx512gfni_aes;
break;
case 3:
test_setup(i) = raid_rec3_avx512gfni_aes;
break;
case 4:
test_setup(i) = raid_rec4_avx512gfni_aes;
break;
case 5:
test_setup(i) = raid_rec5_avx512gfni_aes;
break;
case 6:
test_setup(i) = raid_rec6_avx512gfni_aes;
break;
}
} else {
switch (i) {
case 1:
test_setup(i) = raid_rec1_avx512gfni_raid;
break;
case 2:
test_setup(i) = raid_rec2_avx512gfni_raid;
break;
case 3:
test_setup(i) = raid_rec3_avx512gfni_raid;
break;
case 4:
test_setup(i) = raid_rec4_avx512gfni_raid;
break;
case 5:
test_setup(i) = raid_rec5_avx512gfni_raid;
break;
case 6:
test_setup(i) = raid_rec6_avx512gfni_raid;
break;
}
}
}
}
#endif
#endif
#ifdef CONFIG_NEON
test_setup(i) = raid_recX_neon;
switch (i) {
case 1:
test_setup(i) = raid_rec1_neon;
break;
case 2:
test_setup(i) = raid_rec2_neon;
break;
case 3:
test_setup(i) = raid_rec3_neon;
break;
case 4:
test_setup(i) = raid_rec4_neon;
break;
case 5:
test_setup(i) = raid_rec5_neon;
break;
case 6:
test_setup(i) = raid_rec6_neon;
break;
}
#endif
#ifdef CONFIG_NEON32
test_setup(i) = raid_recX_neon32;
#endif
switch (i) {
case 1:
test_setup(i) = raid_rec1_neon32;
break;
case 2:
test_setup(i) = raid_rec2_neon32;
break;
case 3:
test_setup(i) = raid_rec3_neon32;
break;
case 4:
test_setup(i) = raid_rec4_neon32;
break;
case 5:
test_setup(i) = raid_rec5_neon32;
break;
case 6:
test_setup(i) = raid_rec6_neon32;
break;
}
#endif
}
/* test all disk counts from 1 to nd_max */