mirror of
https://github.com/amadvance/snapraid.git
synced 2026-10-26 10:51:54 -04:00
442 lines
13 KiB
C
442 lines
13 KiB
C
// SPDX-License-Identifier: GPL-2.0-or-later
|
||
// Copyright (C) 2013 Andrea Mazzoleni
|
||
|
||
#ifndef __RAID_CPU_H
|
||
#define __RAID_CPU_H
|
||
|
||
#ifdef CONFIG_X86
|
||
|
||
static inline void raid_cpuid(uint32_t func_eax, uint32_t sub_ecx, uint32_t *reg)
|
||
{
|
||
asm volatile (
|
||
#if defined(__i386__) && defined(__PIC__)
|
||
/* allow compilation in PIC mode saving ebx */
|
||
"xchgl %%ebx, %1\n"
|
||
"cpuid\n"
|
||
"xchgl %%ebx, %1\n"
|
||
: "=a" (reg[0]), "=r" (reg[1]), "=c" (reg[2]), "=d" (reg[3])
|
||
: "0" (func_eax), "2" (sub_ecx)
|
||
#else
|
||
"cpuid\n"
|
||
: "=a" (reg[0]), "=b" (reg[1]), "=c" (reg[2]), "=d" (reg[3])
|
||
: "0" (func_eax), "2" (sub_ecx)
|
||
#endif
|
||
);
|
||
}
|
||
|
||
static inline void raid_xgetbv(uint32_t *reg)
|
||
{
|
||
/* get the value of the Extended Control Register ecx=0 */
|
||
asm volatile (
|
||
/*
|
||
* Uses a direct encoding of the XGETBV instruction as only recent
|
||
* assemblers support it.
|
||
* the next line is equivalent to: "xgetbv\n"
|
||
*/
|
||
".byte 0x0f, 0x01, 0xd0\n"
|
||
: "=a" (reg[0]), "=d" (reg[3])
|
||
: "c" (0)
|
||
);
|
||
}
|
||
|
||
#define CPU_VENDOR_MAX 13
|
||
|
||
static inline void raid_cpu_info(char *vendor, unsigned *family, unsigned *model)
|
||
{
|
||
uint32_t reg[4];
|
||
unsigned f, ef, m, em;
|
||
|
||
raid_cpuid(0, 0, reg);
|
||
|
||
((uint32_t *)vendor)[0] = reg[1];
|
||
((uint32_t *)vendor)[1] = reg[3];
|
||
((uint32_t *)vendor)[2] = reg[2];
|
||
vendor[12] = 0;
|
||
|
||
raid_cpuid(1, 0, reg);
|
||
|
||
f = (reg[0] >> 8) & 0xF;
|
||
ef = (reg[0] >> 20) & 0xFF;
|
||
m = (reg[0] >> 4) & 0xF;
|
||
em = (reg[0] >> 16) & 0xF;
|
||
|
||
if (strcmp(vendor, "AuthenticAMD") == 0) {
|
||
if (f < 15) {
|
||
*family = f;
|
||
*model = m;
|
||
} else {
|
||
*family = f + ef;
|
||
*model = m + (em << 4);
|
||
}
|
||
} else {
|
||
*family = f + ef;
|
||
*model = m + (em << 4);
|
||
}
|
||
}
|
||
|
||
static inline int raid_cpu_match_sse(uint32_t cpuid_1_ecx, uint32_t cpuid_1_edx)
|
||
{
|
||
uint32_t reg[4];
|
||
|
||
raid_cpuid(1, 0, reg);
|
||
if ((reg[2] & cpuid_1_ecx) != cpuid_1_ecx)
|
||
return 0;
|
||
if ((reg[3] & cpuid_1_edx) != cpuid_1_edx)
|
||
return 0;
|
||
|
||
return 1;
|
||
}
|
||
|
||
static inline int raid_cpu_match_avx(uint32_t cpuid_1_ecx, uint32_t cpuid_7_ebx, uint32_t cpuid_7_ecx, uint32_t xcr0)
|
||
{
|
||
uint32_t reg[4];
|
||
|
||
raid_cpuid(1, 0, reg);
|
||
if ((reg[2] & cpuid_1_ecx) != cpuid_1_ecx)
|
||
return 0;
|
||
|
||
raid_xgetbv(reg);
|
||
if ((reg[0] & xcr0) != xcr0)
|
||
return 0;
|
||
|
||
raid_cpuid(7, 0, reg);
|
||
if ((reg[1] & cpuid_7_ebx) != cpuid_7_ebx)
|
||
return 0;
|
||
|
||
if ((reg[2] & cpuid_7_ecx) != cpuid_7_ecx)
|
||
return 0;
|
||
|
||
return 1;
|
||
}
|
||
|
||
static inline int raid_cpu_has_sse2(void)
|
||
{
|
||
/*
|
||
* Intel 64 and IA-32 Architectures Software Developer's Manual
|
||
* 325462-048US September 2013
|
||
*
|
||
* 11.6.2 Checking for SSE/SSE2 Support
|
||
* Before an application attempts to use the SSE and/or SSE2 extensions, it should check
|
||
* that they are present on the processor:
|
||
* 1. Check that the processor supports the CPUID instruction. Bit 21 of the EFLAGS
|
||
* register can be used to check if the processor supports the CPUID instruction.
|
||
* 2. Check that the processor supports the SSE and/or SSE2 extensions (true if
|
||
* CPUID.01H:EDX.SSE[bit 25] = 1 and/or CPUID.01H:EDX.SSE2[bit 26] = 1).
|
||
*/
|
||
return raid_cpu_match_sse(
|
||
0,
|
||
1 << 26); /* SSE2 */
|
||
}
|
||
|
||
static inline int raid_cpu_has_ssse3(void)
|
||
{
|
||
/*
|
||
* Intel 64 and IA-32 Architectures Software Developer's Manual
|
||
* 325462-048US September 2013
|
||
*
|
||
* 12.7.2 Checking for SSSE3 Support
|
||
* Before an application attempts to use the SSSE3 extensions, the application should
|
||
* follow the steps illustrated in Section 11.6.2, "Checking for SSE/SSE2 Support."
|
||
* Next, use the additional step provided below:
|
||
* Check that the processor supports SSSE3 (if CPUID.01H:ECX.SSSE3[bit 9] = 1).
|
||
*/
|
||
return raid_cpu_match_sse(
|
||
1 << 9, /* SSSE3 */
|
||
1 << 26); /* SSE2 */
|
||
}
|
||
|
||
static inline int raid_cpu_has_crc32(void)
|
||
{
|
||
/*
|
||
* Intel 64 and IA-32 Architectures Software Developer's Manual
|
||
* 325462-048US September 2013
|
||
*
|
||
* 12.12.3 Checking for SSE4.2 Support
|
||
* ...
|
||
* Before an application attempts to use the CRC32 instruction, it must check
|
||
* that the processor supports SSE4.2 (if CPUID.01H:ECX.SSE4_2[bit 20] = 1).
|
||
*/
|
||
return raid_cpu_match_sse(
|
||
1 << 20, /* CRC32 */
|
||
0);
|
||
}
|
||
|
||
static inline int raid_cpu_has_avx2(void)
|
||
{
|
||
/*
|
||
* Intel Architecture Instruction Set Extensions Programming Reference
|
||
* 319433-022 October 2014
|
||
*
|
||
* 14.3 Detection of AVX instructions
|
||
* 1) Detect CPUID.1:ECX.OSXSAVE[bit 27] = 1 (XGETBV enabled for application use1)
|
||
* 2) Issue XGETBV and verify that XCR0[2:1] = `11b' (XMM state and YMM state are enabled by OS).
|
||
* 3) detect CPUID.1:ECX.AVX[bit 28] = 1 (AVX instructions supported).
|
||
* (Step 3 can be done in any order relative to 1 and 2)
|
||
*
|
||
* 14.7.1 Detection of AVX2
|
||
* Hardware support for AVX2 is indicated by CPUID.(EAX=07H, ECX=0H):EBX.AVX2[bit 5]=1.
|
||
* Application Software must identify that hardware supports AVX, after that it must
|
||
* also detect support for AVX2 by checking CPUID.(EAX=07H, ECX=0H):EBX.AVX2[bit 5].
|
||
*/
|
||
return raid_cpu_match_avx(
|
||
(1 << 27) | (1 << 28), /* Leaf 1, ECX: XSAVE and AVX */
|
||
1 << 5, /* Leaf 7, EBX: AVX2 */
|
||
0, /* Leaf 7, ECX: */
|
||
3 << 1); /* XCR0: OS saves XMM and YMM registers */
|
||
}
|
||
|
||
static inline int raid_cpu_has_avx2gfni(void)
|
||
{
|
||
/*
|
||
* Detection of AVX2 + GFNI:
|
||
* 1) Verify OSXSAVE and XGETBV (CPUID.1:ECX[27])
|
||
* 2) Verify XCR0[2:1] = '11b' (XMM and YMM state enabled)
|
||
* 3) Verify AVX2 support (CPUID.7.0:EBX[5])
|
||
* 4) Verify GFNI support (CPUID.7.0:ECX[8])
|
||
*/
|
||
return raid_cpu_match_avx(
|
||
(1 << 27) | (1 << 28), /* Leaf 1, ECX: XSAVE and AVX */
|
||
1 << 5, /* Leaf 7, EBX: AVX2 */
|
||
1 << 8, /* Leaf 7, ECX: GFNI */
|
||
3 << 1); /* XCR0: OS saves XMM and YMM registers */
|
||
}
|
||
|
||
static inline int raid_cpu_has_avx512bw(void)
|
||
{
|
||
/*
|
||
* Intel Architecture Instruction Set Extensions Programming Reference
|
||
* 319433-022 October 2014
|
||
*
|
||
* 2.2 Detection of 512-bit Instruction Groups of Intel AVX-512 Family
|
||
* 1) Detect CPUID.1:ECX.OSXSAVE[bit 27] = 1 (XGETBV enabled for application use)
|
||
* 2) Execute XGETBV and verify that XCR0[7:5] = `111b' (OPMASK state, upper 256-bit of
|
||
* ZMM0-ZMM15 and ZMM16-ZMM31 state are enabled by OS) and that XCR0[2:1] = `11b'
|
||
* (XMM state and YMM state are enabled by OS).
|
||
* 3) Verify both CPUID.0x7.0:EBX.AVX512F[bit 16] = 1, CPUID.0x7.0:EBX.AVX512BW[bit 30] = 1.
|
||
*/
|
||
|
||
/*
|
||
* Note that intentionally we don't check for AVX and AVX2
|
||
* because the documentation doesn't require that
|
||
*/
|
||
return raid_cpu_match_avx(
|
||
1 << 27, /* Leaf 1, ECX: XSAVE/XGETBV */
|
||
(1 << 16) | (1 << 30), /* Leaf 7, EBX: AVX512F and AVX512BW */
|
||
0, /* Leaf 7, ECX: */
|
||
(3 << 1) | (7 << 5)); /* XCR0: OS saves XMM, YMM and ZMM registers */
|
||
}
|
||
|
||
static inline int raid_cpu_has_avx512gfni(void)
|
||
{
|
||
return raid_cpu_match_avx(
|
||
1 << 27, /* Leaf 1, ECX: XSAVE/XGETBV */
|
||
1 << 16, /* Leaf 7, EBX: AVX512F (Foundation) */
|
||
1 << 8, /* Leaf 7, ECX: GFNI */
|
||
(3 << 1) | (7 << 5) /* XCR0: OS saves XMM, YMM and ZMM registers */
|
||
);
|
||
}
|
||
|
||
/**
|
||
* Check if it's an Intel Atom CPU.
|
||
*/
|
||
static inline int raid_cpu_is_atom(unsigned family, unsigned model)
|
||
{
|
||
if (family != 6)
|
||
return 0;
|
||
|
||
/*
|
||
* x86 Architecture CPUID
|
||
* http://www.sandpile.org/x86/cpuid.htm
|
||
*
|
||
* Intel Atom
|
||
* 1C (28) Atom (45 nm) with 512 KB on-die L2
|
||
* 26 (38) Atom (45 nm) with 512 KB on-die L2
|
||
* 36 (54) Atom (32 nm) with 512 KB on-die L2
|
||
* 27 (39) Atom (32 nm) with 512 KB on-die L2
|
||
* 35 (53) Atom (?? nm) with ??? KB on-die L2
|
||
* 4A (74) Atom 2C (22 nm) 1 MB L2 + PowerVR (TGR)
|
||
* 5A (90) Atom 4C (22 nm) 2 MB L2 + PowerVR (ANN)
|
||
* 37 (55) Atom 4C (22 nm) 2 MB L2 + Intel Gen7 (BYT)
|
||
* 4C (76) Atom 4C (14 nm) 2 MB L2 + Intel Gen8 (BSW)
|
||
* 5D (93) Atom 4C (28 nm TSMC) 1 MB L2 + Mali (SoFIA)
|
||
* 4D (77) Atom 8C (22 nm) 4 MB L2 (AVN)
|
||
* ?? Atom ?C (14 nm) ? MB L2 (DVN)
|
||
*/
|
||
return model == 28 || model == 38 || model == 54
|
||
|| model == 39 || model == 53 || model == 74
|
||
|| model == 90 || model == 55 || model == 76
|
||
|| model == 93 || model == 77;
|
||
}
|
||
|
||
/**
|
||
* Check if the processor has a slow MULT implementation.
|
||
* If yes, it's better to use a hash not based on multiplication.
|
||
*/
|
||
static inline int raid_cpu_has_slowmult(void)
|
||
{
|
||
char vendor[CPU_VENDOR_MAX];
|
||
unsigned family;
|
||
unsigned model;
|
||
|
||
/*
|
||
* In some cases Murmur3 based on MUL instruction,
|
||
* is a LOT slower than Spooky2 based on SHIFTs.
|
||
*/
|
||
raid_cpu_info(vendor, &family, &model);
|
||
|
||
if (strcmp(vendor, "GenuineIntel") == 0) {
|
||
/*
|
||
* Intel Atom (Model 28)
|
||
* murmur3:378 MB/s, spooky2:3413 MB/s (x86)
|
||
*
|
||
* Intel Atom (Model 77)
|
||
* murmur3:1311 MB/s, spooky2:4056 MB/s (x64)
|
||
*/
|
||
if (raid_cpu_is_atom(family, model))
|
||
return 1;
|
||
}
|
||
|
||
return 0;
|
||
}
|
||
|
||
/**
|
||
* Check if the processor has a slow extended set of SSE registers.
|
||
* If yes, it's better to limit the unrolling to the first 8 registers.
|
||
*/
|
||
static inline int raid_cpu_has_slow_extendedreg(void)
|
||
{
|
||
char vendor[CPU_VENDOR_MAX];
|
||
unsigned family;
|
||
unsigned model;
|
||
|
||
/*
|
||
* In some cases the PAR2 implementation using 16 SSE registers
|
||
* is a LITTLE slower than the one using only the first 8 registers.
|
||
* This doesn't happen for PARZ.
|
||
*/
|
||
raid_cpu_info(vendor, &family, &model);
|
||
|
||
if (strcmp(vendor, "AuthenticAMD") == 0) {
|
||
/*
|
||
* AMD Bulldozer
|
||
* par2_sse2:4922 MB/s, par2_sse2e:4465 MB/s
|
||
*/
|
||
if (family == 21)
|
||
return 1;
|
||
}
|
||
|
||
if (strcmp(vendor, "GenuineIntel") == 0) {
|
||
/*
|
||
* Intel Atom (Model 77)
|
||
* par2_sse2:5686 MB/s, par2_sse2e:5250 MB/s
|
||
* parz_sse2:3100 MB/s, parz_sse2e:3400 MB/s
|
||
* par3_sse3:1921 MB/s, par3_sse3e:1813 MB/s
|
||
* par4_sse3:1175 MB/s, par4_sse3e:1113 MB/s
|
||
* par5_sse3:876 MB/s, par5_sse3e:675 MB/s
|
||
* par6_sse3:705 MB/s, par6_sse3e:529 MB/s
|
||
*
|
||
* Intel Atom (Model 77) "Avoton C2750"
|
||
* par2_sse2:5661 MB/s, par2_sse2e:5382 MB/s
|
||
* parz_sse2:3110 MB/s, parz_sse2e:3450 MB/s
|
||
* par3_sse3:1769 MB/s, par3_sse3e:1856 MB/s
|
||
* par4_sse3:1221 MB/s, par4_sse3e:1141 MB/s
|
||
* par5_sse3:910 MB/s, par5_sse3e:675 MB/s
|
||
* par6_sse3:720 MB/s, par6_sse3e:534 MB/s
|
||
*/
|
||
if (raid_cpu_is_atom(family, model))
|
||
return 1;
|
||
}
|
||
|
||
return 0;
|
||
}
|
||
|
||
/*
|
||
* Check if the processor has a "slow" AVX-512 implementation where a highly
|
||
* optimized AVX-2 path will outperform the native AVX-512 path for this
|
||
* specific parity kernel, which is dominated by vpshufb (AVX-512BW).
|
||
*
|
||
* Three distinct hardware conditions cause this:
|
||
*
|
||
* 1. Narrow Shuffle Port + Frequency Offset (Intel Skylake-X / Cannon Lake):
|
||
* The shuffle execution port (port 5) on these cores is only 256-bit wide.
|
||
* vpshufb zmm therefore decomposes into 2 µops through the same port,
|
||
* delivering identical bytes-per-cycle throughput as vpshufb ymm, there
|
||
* is zero computational gain from the wider register. On top of this, any
|
||
* active 512-bit instruction triggers a core frequency downclocking license
|
||
* (100-400 MHz under sustained load). The result is measurably worse
|
||
* throughput than AVX-2 for this workload.
|
||
* Note: Ice Lake, Tiger Lake, and Rocket Lake are intentionally excluded,
|
||
* their port 5 is natively 512-bit wide (1 µop, genuine 2× vpshufb
|
||
* throughput), which outweighs their smaller frequency offsets.
|
||
*
|
||
* 2. Double-Pumped 256-bit Datapath (AMD Zen 4, Zen 5 mobile):
|
||
* These processors execute all AVX-512 instructions, including vpshufb zmm,
|
||
* internally as two sequential 256-bit operations. This doubles the µop
|
||
* count and execution cycles for no gain in output bytes, making the
|
||
* AVX-512 path strictly slower than AVX-2.
|
||
*/
|
||
static inline int raid_cpu_has_slow_avx512(void)
|
||
{
|
||
char vendor[CPU_VENDOR_MAX];
|
||
unsigned family;
|
||
unsigned model;
|
||
|
||
raid_cpu_info(vendor, &family, &model);
|
||
|
||
if (strcmp(vendor, "AuthenticAMD") == 0) {
|
||
/*
|
||
* Zen 4 (Family 0x19): ALL models use a double-pumped 256-bit
|
||
* datapath for AVX-512, desktop, mobile, and server alike.
|
||
*/
|
||
if (family == 25)
|
||
return 1;
|
||
|
||
/*
|
||
* Zen 5 (Family 0x1A):
|
||
* True 512-bit: Granite Ridge desktop (0x44), Turin server (0x00–0x0F)
|
||
* 256-bit mobile: Strix Point / Strix Halo (0x20–0x2F),
|
||
* 256-bit mobile: Krackan Point and variants (0x60–0x6F)
|
||
*
|
||
* CAUTION: if AMD places a future desktop/server SKU in 0x20–0x2F,
|
||
* this becomes a false positive. Re-validate against AMD's PPR on
|
||
* each new family 0x1A product.
|
||
*/
|
||
if (family == 26) {
|
||
if ((model >= 0x20 && model <= 0x2F) /* Strix Point / Strix Halo */
|
||
|| (model >= 0x60 && model <= 0x6F)) /* Krackan Point and variants */
|
||
return 1;
|
||
}
|
||
}
|
||
|
||
if (strcmp(vendor, "GenuineIntel") == 0) {
|
||
/*
|
||
* Skylake-X / Cascade Lake (0x55) and Cannon Lake (0x66).
|
||
*
|
||
* These have TWO compounding problems for vpshufb-heavy parity:
|
||
*
|
||
* 1. Port 5 (the shuffle port) is only 256-bit wide on these cores.
|
||
* vpshufb zmm decomposes into 2 µops through the same port,
|
||
* yielding identical bytes-per-cycle as vpshufb ymm.
|
||
* There is zero throughput gain from the wider register.
|
||
*
|
||
* 2. Executing AVX-512 instructions triggers a frequency downclocking
|
||
* "license" (L0/L1 levels), reducing core frequency by 100-400 MHz
|
||
* under sustained 512-bit load.
|
||
*
|
||
* Ice Lake (0x7E, 0x6A, 0x6C), Tiger Lake (0x8C, 0x8D), and
|
||
* Rocket Lake (0xA7) are intentionally NOT listed: their port 5
|
||
* is natively 512-bit wide (1 µop, 2× throughput for vpshufb zmm),
|
||
* which outweighs their smaller frequency offsets.
|
||
*/
|
||
if (family == 6) {
|
||
if (model == 0x55 /* Skylake-X, Cascade Lake */
|
||
|| model == 0x66) /* Cannon Lake (rare) */
|
||
return 1;
|
||
}
|
||
}
|
||
|
||
return 0;
|
||
}
|
||
#endif
|
||
|
||
#endif
|