snapraid/raid/cpu.h

// SPDX-License-Identifier: GPL-2.0-or-later
// Copyright (C) 2013 Andrea Mazzoleni

#ifndef RAID_CPU_H
#define RAID_CPU_H

#ifdef CONFIG_X86

static inline void raid_cpuid(uint32_t func_eax, uint32_t sub_ecx, uint32_t *reg)
{
	asm volatile (
#if defined(__i386__) && defined(__PIC__)
		/* allow compilation in PIC mode saving ebx */
		"xchgl %%ebx, %1\n"
		"cpuid\n"
		"xchgl %%ebx, %1\n"
		: "=a" (reg[0]), "=r" (reg[1]), "=c" (reg[2]), "=d" (reg[3])
		: "0" (func_eax), "2" (sub_ecx)
#else
		"cpuid\n"
		: "=a" (reg[0]), "=b" (reg[1]), "=c" (reg[2]), "=d" (reg[3])
		: "0" (func_eax), "2" (sub_ecx)
#endif
	);
}

static inline void raid_xgetbv(uint32_t *reg)
{
	/* get the value of the Extended Control Register ecx=0 */
	asm volatile (
		/*
		 * Uses a direct encoding of the XGETBV instruction as only recent
		 * assemblers support it.
		 * the next line is equivalent to: "xgetbv\n"
		 */
		".byte 0x0f, 0x01, 0xd0\n"
		: "=a" (reg[0]), "=d" (reg[3])
		: "c" (0)
	);
}

#define CPU_VENDOR_MAX 13

static inline void raid_cpu_info(char *vendor, unsigned *family, unsigned *model)
{
	uint32_t reg[4];
	unsigned f, ef, m, em;

	raid_cpuid(0, 0, reg);

	memcpy(vendor + 0, &reg[1], 4);
	memcpy(vendor + 4, &reg[3], 4);
	memcpy(vendor + 8, &reg[2], 4);
	vendor[12] = 0;

	raid_cpuid(1, 0, reg);

	f = (reg[0] >> 8) & 0xF;
	ef = (reg[0] >> 20) & 0xFF;
	m = (reg[0] >> 4) & 0xF;
	em = (reg[0] >> 16) & 0xF;

	if (strcmp(vendor, "AuthenticAMD") == 0) {
		if (f < 15) {
			*family = f;
			*model = m;
		} else {
			*family = f + ef;
			*model = m + (em << 4);
		}
	} else {
		*family = f + ef;
		*model = m + (em << 4);
	}
}

static inline int raid_cpu_match_sse(uint32_t leaf_1_ecx, uint32_t leaf_1_edx)
{
	uint32_t reg[4];

	raid_cpuid(1, 0, reg);
	if ((reg[2] & leaf_1_ecx) != leaf_1_ecx)
		return 0;
	if ((reg[3] & leaf_1_edx) != leaf_1_edx)
		return 0;

	return 1;
}

static inline int raid_cpu_match_avx(uint32_t leaf_1_ecx, uint32_t leaf_7_ebx, uint32_t leaf_7_ecx, uint32_t xcr0)
{
	uint32_t reg[4];

	/* check that leaf 7 is supported */
	raid_cpuid(0, 0, reg);
	if (reg[0] < 7)
		return 0;

	raid_cpuid(1, 0, reg);
	if ((reg[2] & leaf_1_ecx) != leaf_1_ecx)
		return 0;

	raid_xgetbv(reg);
	if ((reg[0] & xcr0) != xcr0)
		return 0;

	raid_cpuid(7, 0, reg);
	if ((reg[1] & leaf_7_ebx) != leaf_7_ebx)
		return 0;

	if ((reg[2] & leaf_7_ecx) != leaf_7_ecx)
		return 0;

	return 1;
}

static inline int raid_cpu_has_sse2(void)
{
	/*
	 * Intel 64 and IA-32 Architectures Software Developer's Manual
	 * 325462-048US September 2013
	 *
	 * 11.6.2 Checking for SSE/SSE2 Support
	 * Before an application attempts to use the SSE and/or SSE2 extensions, it should check
	 * that they are present on the processor:
	 * 1. Check that the processor supports the CPUID instruction. Bit 21 of the EFLAGS
	 * register can be used to check if the processor supports the CPUID instruction.
	 * 2. Check that the processor supports the SSE and/or SSE2 extensions (true if
	 * CPUID.01H:EDX.SSE[bit 25] = 1 and/or CPUID.01H:EDX.SSE2[bit 26] = 1).
	 */
	uint32_t leaf_1_edx = 1 << 26; /* SSE2 */

	return raid_cpu_match_sse(0, leaf_1_edx);
}

static inline int raid_cpu_has_clflush(void)
{
	/*
	 * Intel 64 and IA-32 Architectures Software Developer's Manual
	 * CPUID - CPU Identification
	 *
	 * Check that the processor supports CLFLUSH (true if CPUID.01H:EDX.CLFSH[bit 19] = 1).
	 */
	uint32_t leaf_1_edx = 1 << 19; /* CLFLUSH */

	return raid_cpu_match_sse(0, leaf_1_edx);
}

static inline int raid_cpu_has_clflushopt(void)
{
	/*
	 * Intel 64 and IA-32 Architectures Software Developer's Manual
	 * CPUID - CPU Identification
	 *
	 * Check that the processor supports CLFLUSHOPT (true if CPUID.07H.00H:EBX.CLFLUSHOPT[bit 23] = 1).
	 */
	uint32_t reg[4];
	uint32_t leaf_7_ebx = 1 << 23; /* CLFLUSHOPT */

	/* check that leaf 7 is supported */
	raid_cpuid(0, 0, reg);
	if (reg[0] < 7)
		return 0;

	raid_cpuid(7, 0, reg);
	if ((reg[1] & leaf_7_ebx) != leaf_7_ebx)
		return 0;

	return 1;
}

static inline unsigned raid_cpu_clflush_size(void)
{
	/*
	 * Intel 64 and IA-32 Architectures Software Developer's Manual
	 * CPUID - CPU Identification
	 *
	 * CPUID.01H:EBX.CLFLUSH[bits 15:8] reports the CLFLUSH cache-line
	 * size in units of 8 bytes.
	 */
	uint32_t reg[4];

	raid_cpuid(1, 0, reg);

	return ((reg[1] >> 8) & 0xff) * 8;
}

static inline int raid_cpu_has_ssse3(void)
{
	/*
	 * Intel 64 and IA-32 Architectures Software Developer's Manual
	 * 325462-048US September 2013
	 *
	 * 12.7.2 Checking for SSSE3 Support
	 * Before an application attempts to use the SSSE3 extensions, the application should
	 * follow the steps illustrated in Section 11.6.2, "Checking for SSE/SSE2 Support."
	 * Next, use the additional step provided below:
	 * Check that the processor supports SSSE3 (if CPUID.01H:ECX.SSSE3[bit 9] = 1).
	 */
	uint32_t leaf_1_ecx = 1 << 9; /* SSSE3 */
	uint32_t leaf_1_edx = 1 << 26; /* SSE2 */

	return raid_cpu_match_sse(leaf_1_ecx, leaf_1_edx);
}

static inline int raid_cpu_has_crc32(void)
{
	/*
	 * Intel 64 and IA-32 Architectures Software Developer's Manual
	 * 325462-048US September 2013
	 *
	 * 12.12.3 Checking for SSE4.2 Support
	 * ...
	 * Before an application attempts to use the CRC32 instruction, it must check
	 * that the processor supports SSE4.2 (if CPUID.01H:ECX.SSE4_2[bit 20] = 1).
	 */
	uint32_t leaf_1_ecx = 1 << 20; /* CRC32 */

	return raid_cpu_match_sse(leaf_1_ecx, 0);
}

static inline int raid_cpu_has_avx2(void)
{
	/*
	 * Intel Architecture Instruction Set Extensions Programming Reference
	 * 319433-022 October 2014
	 *
	 * 14.3 Detection of AVX instructions
	 * 1) Detect CPUID.1:ECX.OSXSAVE[bit 27] = 1 (XGETBV enabled for application use1)
	 * 2) Issue XGETBV and verify that XCR0[2:1] = `11b' (XMM state and YMM state are enabled by OS).
	 * 3) detect CPUID.1:ECX.AVX[bit 28] = 1 (AVX instructions supported).
	 * (Step 3 can be done in any order relative to 1 and 2)
	 *
	 * 14.7.1 Detection of AVX2
	 * Hardware support for AVX2 is indicated by CPUID.(EAX=07H, ECX=0H):EBX.AVX2[bit 5]=1.
	 * Application Software must identify that hardware supports AVX, after that it must
	 * also detect support for AVX2 by checking CPUID.(EAX=07H, ECX=0H):EBX.AVX2[bit 5].
	 */
	uint32_t leaf_1_ecx = (1 << 27) | (1 << 28); /* XSAVE and AVX */
	uint32_t leaf_7_ebx = 1 << 5; /* AVX2 */
	uint32_t xcr0 = 3 << 1; /* OS saves XMM and YMM registers */

	return raid_cpu_match_avx(leaf_1_ecx, leaf_7_ebx, 0, xcr0);
}

static inline int raid_cpu_has_avx2gfni(void)
{
	/*
	 * Detection of AVX2 + GFNI:
	 * 1) Verify OSXSAVE and XGETBV (CPUID.1:ECX[27])
	 * 2) Verify XCR0[2:1] = '11b' (XMM and YMM state enabled)
	 * 3) Verify AVX2 support (CPUID.7.0:EBX[5])
	 * 4) Verify GFNI support (CPUID.7.0:ECX[8])
	 */
	uint32_t leaf_1_ecx = (1 << 27) | (1 << 28); /* XSAVE and AVX */
	uint32_t leaf_7_ebx = 1 << 5; /* AVX2 */
	uint32_t leaf_7_ecx = 1 << 8; /* GFNI */
	uint32_t xcr0 = 3 << 1; /* OS saves XMM and YMM registers */

	return raid_cpu_match_avx(leaf_1_ecx, leaf_7_ebx, leaf_7_ecx, xcr0);
}

static inline int raid_cpu_has_avx512bw(void)
{
	/*
	 * Intel Architecture Instruction Set Extensions Programming Reference
	 * 319433-022 October 2014
	 *
	 * 2.2 Detection of 512-bit Instruction Groups of Intel AVX-512 Family
	 * 1) Detect CPUID.1:ECX.OSXSAVE[bit 27] = 1 (XGETBV enabled for application use)
	 * 2) Execute XGETBV and verify that XCR0[7:5] = `111b' (OPMASK state, upper 256-bit of
	 * ZMM0-ZMM15 and ZMM16-ZMM31 state are enabled by OS) and that XCR0[2:1] = `11b'
	 * (XMM state and YMM state are enabled by OS).
	 * 3) Verify both CPUID.0x7.0:EBX.AVX512F[bit 16] = 1, CPUID.0x7.0:EBX.AVX512BW[bit 30] = 1.
	 */

	/*
	 * Note that intentionally we don't check for AVX and AVX2
	 * because the documentation doesn't require that
	 */
	uint32_t leaf_1_ecx = 1 << 27; /* XSAVE/XGETBV */
	uint32_t leaf_7_ebx = (1 << 16) | (1 << 30); /* AVX512F and AVX512BW */
	uint32_t xcr0 = (3 << 1) | (7 << 5); /* OS saves XMM, YMM and ZMM registers */

	return raid_cpu_match_avx(leaf_1_ecx, leaf_7_ebx, 0, xcr0);
}

static inline int raid_cpu_has_avx512gfni(void)
{
	uint32_t leaf_1_ecx = 1 << 27; /* XSAVE/XGETBV */
	uint32_t leaf_7_ebx = (1 << 16) | (1 << 30); /* AVX512F and AVX512BW */
	uint32_t leaf_7_ecx = 1 << 8; /* GFNI */
	uint32_t xcr0 = (3 << 1) | (7 << 5); /* OS saves XMM, YMM and ZMM registers */

	return raid_cpu_match_avx(leaf_1_ecx, leaf_7_ebx, leaf_7_ecx, xcr0);
}

/**
 * Check if it's an Intel Atom CPU.
 */
static inline int raid_cpu_is_atom(unsigned family, unsigned model)
{
	if (family != 6)
		return 0;

	/*
	 * x86 Architecture CPUID
	 * http://www.sandpile.org/x86/cpuid.htm
	 */
	switch (model) {
	case 28: /* 1C Atom (45 nm) */
	case 38: /* 26 Atom (45 nm) */
	case 39: /* 27 Atom (32 nm) */
	case 53: /* 35 Atom */
	case 54: /* 36 Atom (32 nm) */
	case 55: /* 37 Atom 4C (22 nm BYT) */
	case 74: /* 4A Atom 2C (22 nm TGR) */
	case 76: /* 4C Atom 4C (14 nm BSW) */
	case 77: /* 4D Atom 8C (22 nm AVN) */
	case 90: /* 5A Atom 4C (22 nm ANN) */
	case 93: /* 5D Atom 4C (28 nm SoFIA) */
		return 1;
	default:
		return 0;
	}
}

/**
 * Check if the processor has a slow MULT implementation.
 * If yes, it's better to use a hash not based on multiplication.
 */
static inline int raid_cpu_has_slowmult(void)
{
	char vendor[CPU_VENDOR_MAX];
	unsigned family;
	unsigned model;

	/*
	 * In some cases Murmur3 based on MUL instruction,
	 * is a LOT slower than Spooky2 based on SHIFTs.
	 */
	raid_cpu_info(vendor, &family, &model);

	if (strcmp(vendor, "GenuineIntel") == 0) {
		/*
		 * Intel Atom (Model 28)
		 * murmur3:378 MB/s, spooky2:3413 MB/s (x86)
		 *
		 * Intel Atom (Model 77)
		 * murmur3:1311 MB/s, spooky2:4056 MB/s (x64)
		 */
		if (raid_cpu_is_atom(family, model))
			return 1;
	}

	return 0;
}

/**
 * Check if the processor has a slow extended set of SSE registers.
 * If yes, it's better to limit the unrolling to the first 8 registers.
 */
static inline int raid_cpu_has_slow_extendedreg(void)
{
	char vendor[CPU_VENDOR_MAX];
	unsigned family;
	unsigned model;

	/*
	 * In some cases the PAR2 implementation using 16 SSE registers
	 * is a LITTLE slower than the one using only the first 8 registers.
	 * This doesn't happen for PARZ.
	 */
	raid_cpu_info(vendor, &family, &model);

	if (strcmp(vendor, "AuthenticAMD") == 0) {
		/*
		 * AMD Bulldozer
		 * par2_sse2:4922 MB/s, par2_sse2e:4465 MB/s
		 */
		if (family == 21)
			return 1;
	}

	if (strcmp(vendor, "GenuineIntel") == 0) {
		/*
		 * Intel Atom (Model 77)
		 * par2_sse2:5686 MB/s, par2_sse2e:5250 MB/s
		 * parz_sse2:3100 MB/s, parz_sse2e:3400 MB/s
		 * par3_sse3:1921 MB/s, par3_sse3e:1813 MB/s
		 * par4_sse3:1175 MB/s, par4_sse3e:1113 MB/s
		 * par5_sse3:876 MB/s, par5_sse3e:675 MB/s
		 * par6_sse3:705 MB/s, par6_sse3e:529 MB/s
		 *
		 * Intel Atom (Model 77) "Avoton C2750"
		 * par2_sse2:5661 MB/s, par2_sse2e:5382 MB/s
		 * parz_sse2:3110 MB/s, parz_sse2e:3450 MB/s
		 * par3_sse3:1769 MB/s, par3_sse3e:1856 MB/s
		 * par4_sse3:1221 MB/s, par4_sse3e:1141 MB/s
		 * par5_sse3:910 MB/s, par5_sse3e:675 MB/s
		 * par6_sse3:720 MB/s, par6_sse3e:534 MB/s
		 */
		if (raid_cpu_is_atom(family, model))
			return 1;
	}

	return 0;
}

/*
 * Check if the processor has a "slow" AVX-512 implementation where a highly
 * optimized AVX-2 path will outperform the native AVX-512 path for this
 * specific parity kernel, which is dominated by vpshufb (AVX-512BW).
 *
 * Hardware condition causing this:
 *
 * Narrow Shuffle Port + Frequency Offset (Intel Skylake-X / Cannon Lake):
 * The shuffle execution port (port 5) on these cores is only 256-bit wide.
 * vpshufb zmm therefore decomposes into 2 µops through the same port,
 * delivering identical bytes-per-cycle throughput as vpshufb ymm, there
 * is zero computational gain from the wider register. On top of this, any
 * active 512-bit instruction triggers a core frequency downclocking license
 * (100-400 MHz under sustained load). The result is measurably worse
 * throughput than AVX-2 for this workload.
 * Note: Ice Lake, Tiger Lake, and Rocket Lake are intentionally excluded,
 * their port 5 is natively 512-bit wide (1 µop, genuine 2× vpshufb
 * throughput), which outweighs their smaller frequency offsets.
 */
static inline int raid_cpu_has_slow_avx512(void)
{
	char vendor[CPU_VENDOR_MAX];
	unsigned family;
	unsigned model;

	raid_cpu_info(vendor, &family, &model);

	if (strcmp(vendor, "GenuineIntel") == 0) {
		/*
		 * Skylake-X / Cascade Lake (0x55) and Cannon Lake (0x66).
		 *
		 * These have TWO compounding problems for vpshufb-heavy parity:
		 *
		 * 1. Port 5 (the shuffle port) is only 256-bit wide on these cores.
		 *    vpshufb zmm decomposes into 2 µops through the same port,
		 *    yielding identical bytes-per-cycle as vpshufb ymm.
		 *    There is zero throughput gain from the wider register.
		 *
		 * 2. Executing AVX-512 instructions triggers a frequency downclocking
		 *    "license" (L0/L1 levels), reducing core frequency by 100-400 MHz
		 *    under sustained 512-bit load.
		 *
		 * Ice Lake (0x7E, 0x6A, 0x6C), Tiger Lake (0x8C, 0x8D), and
		 * Rocket Lake (0xA7) are intentionally NOT listed: their port 5
		 * is natively 512-bit wide (1 µop, 2× throughput for vpshufb zmm),
		 * which outweighs their smaller frequency offsets.
		 */
		if (family == 6) {
			if (model == 0x55 /* Skylake-X, Cascade Lake  */
				|| model == 0x66) /* Cannon Lake (rare) */
				return 1;
		}
	}

	return 0;
}
#endif

#endif