From 557dcda9cdba791460a71b02b8ffaab0567024c4 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Wed, 27 May 2026 14:39:08 +0200 Subject: [PATCH] Disable AVX512 on CPUs with narrow shuffle ports or double-pumped 256-bit datapath --- cmdline/speed.c | 9 ++-- raid/cpu.h | 111 +++++++++++++++++++++++++++++++++++++----- raid/gfni.c | 24 ++++----- raid/test/Makefile | 2 + raid/test/speedtest.c | 4 +- raid/x86.c | 28 ++++++----- 6 files changed, 136 insertions(+), 42 deletions(-) diff --git a/cmdline/speed.c b/cmdline/speed.c index e5f0e4aa..2ec15222 100644 --- a/cmdline/speed.c +++ b/cmdline/speed.c @@ -119,7 +119,7 @@ void speed(int period, int nd, int size) raid_cpu_info(vendor, &family, &model); - printf("CPU %s, family %u, model %u, flags%s%s%s%s%s%s%s%s%s\n", vendor, family, model, + printf("CPU %s, family %u, model %u (%xh), flags%s%s%s%s%s%s%s%s%s%s\n", vendor, family, model, model, raid_cpu_has_sse2() ? " sse2" : "", raid_cpu_has_ssse3() ? " ssse3" : "", raid_cpu_has_crc32() ? " crc32" : "", @@ -128,7 +128,8 @@ void speed(int period, int nd, int size) raid_cpu_has_avx512bw() ? " avx512bw" : "", raid_cpu_has_avx512gfni() ? " avx512gfni" : "", raid_cpu_has_slowmult() ? " slowmult" : "", - raid_cpu_has_slowextendedreg() ? " slowext" : "" + raid_cpu_has_slow_extendedreg() ? " slowext" : "", + raid_cpu_has_slow_avx512() ? " slowavx512" : "" ); } #elif defined(__aarch64__) || defined(_M_ARM64) @@ -286,7 +287,7 @@ void speed(int period, int nd, int size) #ifdef CONFIG_X86_64 printf("%8s", "avx2e"); printf("%8s", "avx512"); - printf("%8s", "gfni"); + printf("%8s", "gfni512"); #endif #endif printf("\n"); @@ -851,7 +852,7 @@ void speed(int period, int nd, int size) printf("%8s", "avx2"); #ifdef CONFIG_X86_64 printf("%8s", "avx512"); - printf("%8s", "gfni"); + printf("%8s", "gfni512"); #endif #endif printf("\n"); diff --git a/raid/cpu.h b/raid/cpu.h index 702b180e..0062ac9a 100644 --- a/raid/cpu.h +++ b/raid/cpu.h @@ -10,7 +10,7 @@ static inline void raid_cpuid(uint32_t func_eax, uint32_t sub_ecx, uint32_t *reg { asm volatile ( #if defined(__i386__) && defined(__PIC__) - /* allow compilation in PIC mode saving ebx */ + /* allow compilation in PIC mode saving ebx */ "xchgl %%ebx, %1\n" "cpuid\n" "xchgl %%ebx, %1\n" @@ -24,15 +24,15 @@ static inline void raid_cpuid(uint32_t func_eax, uint32_t sub_ecx, uint32_t *reg ); } -static inline void raid_xgetbv(uint32_t* reg) +static inline void raid_xgetbv(uint32_t *reg) { /* get the value of the Extended Control Register ecx=0 */ asm volatile ( - /* - * Uses a direct encoding of the XGETBV instruction as only recent - * assemblers support it. - * the next line is equivalent to: "xgetbv\n" - */ + /* + * Uses a direct encoding of the XGETBV instruction as only recent + * assemblers support it. + * the next line is equivalent to: "xgetbv\n" + */ ".byte 0x0f, 0x01, 0xd0\n" : "=a" (reg[0]), "=d" (reg[3]) : "c" (0) @@ -48,9 +48,9 @@ static inline void raid_cpu_info(char *vendor, unsigned *family, unsigned *model raid_cpuid(0, 0, reg); - ((uint32_t*)vendor)[0] = reg[1]; - ((uint32_t*)vendor)[1] = reg[3]; - ((uint32_t*)vendor)[2] = reg[2]; + ((uint32_t *)vendor)[0] = reg[1]; + ((uint32_t *)vendor)[1] = reg[3]; + ((uint32_t *)vendor)[2] = reg[2]; vendor[12] = 0; raid_cpuid(1, 0, reg); @@ -303,7 +303,7 @@ static inline int raid_cpu_has_slowmult(void) * Check if the processor has a slow extended set of SSE registers. * If yes, it's better to limit the unrolling to the first 8 registers. */ -static inline int raid_cpu_has_slowextendedreg(void) +static inline int raid_cpu_has_slow_extendedreg(void) { char vendor[CPU_VENDOR_MAX]; unsigned family; @@ -349,7 +349,94 @@ static inline int raid_cpu_has_slowextendedreg(void) return 0; } + +/* + * Check if the processor has a "slow" AVX-512 implementation where a highly + * optimized AVX-2 path will outperform the native AVX-512 path for this + * specific parity kernel, which is dominated by vpshufb (AVX-512BW). + * + * Three distinct hardware conditions cause this: + * + * 1. Narrow Shuffle Port + Frequency Offset (Intel Skylake-X / Cannon Lake): + * The shuffle execution port (port 5) on these cores is only 256-bit wide. + * vpshufb zmm therefore decomposes into 2 µops through the same port, + * delivering identical bytes-per-cycle throughput as vpshufb ymm, there + * is zero computational gain from the wider register. On top of this, any + * active 512-bit instruction triggers a core frequency downclocking license + * (100-400 MHz under sustained load). The result is measurably worse + * throughput than AVX-2 for this workload. + * Note: Ice Lake, Tiger Lake, and Rocket Lake are intentionally excluded, + * their port 5 is natively 512-bit wide (1 µop, genuine 2× vpshufb + * throughput), which outweighs their smaller frequency offsets. + * + * 2. Double-Pumped 256-bit Datapath (AMD Zen 4, Zen 5 mobile): + * These processors execute all AVX-512 instructions, including vpshufb zmm, + * internally as two sequential 256-bit operations. This doubles the µop + * count and execution cycles for no gain in output bytes, making the + * AVX-512 path strictly slower than AVX-2. + */ +static inline int raid_cpu_has_slow_avx512(void) +{ + char vendor[CPU_VENDOR_MAX]; + unsigned family; + unsigned model; + + raid_cpu_info(vendor, &family, &model); + + if (strcmp(vendor, "AuthenticAMD") == 0) { + /* + * Zen 4 (Family 0x19): ALL models use a double-pumped 256-bit + * datapath for AVX-512, desktop, mobile, and server alike. + */ + if (family == 25) + return 1; + + /* + * Zen 5 (Family 0x1A): + * True 512-bit: Granite Ridge desktop (0x44), Turin server (0x00–0x0F) + * 256-bit mobile: Strix Point / Strix Halo (0x20–0x2F), + * 256-bit mobile: Krackan Point and variants (0x60–0x6F) + * + * CAUTION: if AMD places a future desktop/server SKU in 0x20–0x2F, + * this becomes a false positive. Re-validate against AMD's PPR on + * each new family 0x1A product. + */ + if (family == 26) { + if ((model >= 0x20 && model <= 0x2F) /* Strix Point / Strix Halo */ + || (model >= 0x60 && model <= 0x6F)) /* Krackan Point and variants */ + return 1; + } + } + + if (strcmp(vendor, "GenuineIntel") == 0) { + /* + * Skylake-X / Cascade Lake (0x55) and Cannon Lake (0x66). + * + * These have TWO compounding problems for vpshufb-heavy parity: + * + * 1. Port 5 (the shuffle port) is only 256-bit wide on these cores. + * vpshufb zmm decomposes into 2 µops through the same port, + * yielding identical bytes-per-cycle as vpshufb ymm. + * There is zero throughput gain from the wider register. + * + * 2. Executing AVX-512 instructions triggers a frequency downclocking + * "license" (L0/L1 levels), reducing core frequency by 100-400 MHz + * under sustained 512-bit load. + * + * Ice Lake (0x7E, 0x6A, 0x6C), Tiger Lake (0x8C, 0x8D), and + * Rocket Lake (0xA7) are intentionally NOT listed: their port 5 + * is natively 512-bit wide (1 µop, 2× throughput for vpshufb zmm), + * which outweighs their smaller frequency offsets. + */ + if (family == 6) { + if (model == 0x55 /* Skylake-X, Cascade Lake */ + || model == 0x66) /* Cannon Lake (rare) */ + return 1; + } + } + + return 0; +} #endif #endif - diff --git a/raid/gfni.c b/raid/gfni.c index 21273ce5..c6e951ce 100644 --- a/raid/gfni.c +++ b/raid/gfni.c @@ -514,18 +514,20 @@ void raid_register_avx512gfni(void) { #if defined(USE_RAID_AES) if (raid_cpu_has_avx512gfni()) { - raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_gen2_avx512gfni); - raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_gen3_avx512gfni); - raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_gen4_avx512gfni); - raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_gen5_avx512gfni); - raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_gen6_avx512gfni); + if (!raid_cpu_has_slow_avx512()) { + raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_gen2_avx512gfni); + raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_gen3_avx512gfni); + raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_gen4_avx512gfni); + raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_gen5_avx512gfni); + raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_gen6_avx512gfni); - raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni", raid_rec1_avx512gfni); - raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_rec2_avx512gfni); - raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_recX_avx512gfni); - raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_recX_avx512gfni); - raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_recX_avx512gfni); - raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_recX_avx512gfni); + raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni512", raid_rec1_avx512gfni); + raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_rec2_avx512gfni); + raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_recX_avx512gfni); + raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_recX_avx512gfni); + raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_recX_avx512gfni); + raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_recX_avx512gfni); + } } #endif } diff --git a/raid/test/Makefile b/raid/test/Makefile index 0ae85be7..0e57e11f 100644 --- a/raid/test/Makefile +++ b/raid/test/Makefile @@ -92,6 +92,7 @@ sdecovtest: $(SDE) -nhm -- ./selftest $(SDE) -hsw -- ./selftest $(SDE) -skx -- ./selftest + $(SDE) -clx -- ./selftest # For slow AVX-512 detection $(SDE) -icl -- ./selftest $(SDE) -icl -- ./speedtest $(MAKE) lcov_capture @@ -100,6 +101,7 @@ sdecovtest: # Use this target to run a complete test on Ice Lake simulator sde: $(SDE) -icl -- ./selftest + $(SDE) -clx -- ./speedtest $(SDE) -icl -- ./speedtest $(SDE) -icl -- ./fulltest diff --git a/raid/test/speedtest.c b/raid/test/speedtest.c index 1b7e51f7..ebcde8a2 100644 --- a/raid/test/speedtest.c +++ b/raid/test/speedtest.c @@ -138,7 +138,7 @@ void speed(void) #ifdef CONFIG_X86_64 printf("%8s", "avx2e"); printf("%8s", "avx512"); - printf("%8s", "gfni"); + printf("%8s", "gfni512"); #endif #endif printf("\n"); @@ -703,7 +703,7 @@ void speed(void) printf("%8s", "avx2"); #ifdef CONFIG_X86_64 printf("%8s", "avx512"); - printf("%8s", "gfni"); + printf("%8s", "gfni512"); #endif #endif printf("\n"); diff --git a/raid/x86.c b/raid/x86.c index e3295178..5c13b3ea 100644 --- a/raid/x86.c +++ b/raid/x86.c @@ -3353,9 +3353,9 @@ void raid_register_x86(void) raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "sse2", raid_gen1_sse2); raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "sse2", raid_gen2_sse2); #ifdef CONFIG_X86_64 - if (!raid_cpu_has_slowextendedreg()) + if (!raid_cpu_has_slow_extendedreg()) raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "sse2e", raid_gen2_sse2ext); - /* note that raid_cpu_has_slowextendedreg() doesn't affect vandermonde */ + /* note that raid_cpu_has_slow_extendedreg() doesn't affect vandermonde */ raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "sse2e", raid_genz_sse2ext); #else raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "sse2", raid_genz_sse2); @@ -3370,7 +3370,7 @@ void raid_register_x86(void) raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3", raid_gen5_ssse3); raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3", raid_gen6_ssse3); #ifdef CONFIG_X86_64 - if (!raid_cpu_has_slowextendedreg()) { + if (!raid_cpu_has_slow_extendedreg()) { raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3e", raid_gen3_ssse3ext); raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3e", raid_gen4_ssse3ext); raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3e", raid_gen5_ssse3ext); @@ -3413,17 +3413,19 @@ void raid_register_x86(void) if (raid_cpu_has_avx512bw()) { raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "avx512", raid_gen1_avx512bw); raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "avx512", raid_gen2_avx512bw); - raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "avx512", raid_gen3_avx512bw); - raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "avx512", raid_gen4_avx512bw); - raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "avx512", raid_gen5_avx512bw); - raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "avx512", raid_gen6_avx512bw); + if (!raid_cpu_has_slow_avx512()) { + raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "avx512", raid_gen3_avx512bw); + raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "avx512", raid_gen4_avx512bw); + raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "avx512", raid_gen5_avx512bw); + raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "avx512", raid_gen6_avx512bw); - raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "avx512", raid_rec1_avx512bw); - raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "avx512", raid_rec2_avx512bw); - raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "avx512", raid_recX_avx512bw); - raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "avx512", raid_recX_avx512bw); - raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "avx512", raid_recX_avx512bw); - raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "avx512", raid_recX_avx512bw); + raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "avx512", raid_rec1_avx512bw); + raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "avx512", raid_rec2_avx512bw); + raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "avx512", raid_recX_avx512bw); + raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "avx512", raid_recX_avx512bw); + raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "avx512", raid_recX_avx512bw); + raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "avx512", raid_recX_avx512bw); + } } #endif #endif