Line data Source code
1 : /*
2 : * Copyright (C) 2013 Andrea Mazzoleni
3 : *
4 : * This program is free software: you can redistribute it and/or modify
5 : * it under the terms of the GNU General Public License as published by
6 : * the Free Software Foundation, either version 2 of the License, or
7 : * (at your option) any later version.
8 : *
9 : * This program is distributed in the hope that it will be useful,
10 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : */
14 :
15 : #ifndef __RAID_CPU_H
16 : #define __RAID_CPU_H
17 :
18 : #ifdef CONFIG_X86
19 :
20 1879 : static inline void raid_cpuid(uint32_t func_eax, uint32_t sub_ecx, uint32_t *reg)
21 : {
22 1879 : asm volatile (
23 : #if defined(__i386__) && defined(__PIC__)
24 : /* allow compilation in PIC mode saving ebx */
25 : "xchgl %%ebx, %1\n"
26 : "cpuid\n"
27 : "xchgl %%ebx, %1\n"
28 : : "=a" (reg[0]), "=r" (reg[1]), "=c" (reg[2]), "=d" (reg[3])
29 : : "0" (func_eax), "2" (sub_ecx)
30 : #else
31 : "cpuid\n"
32 1879 : : "=a" (reg[0]), "=b" (reg[1]), "=c" (reg[2]), "=d" (reg[3])
33 : : "0" (func_eax), "2" (sub_ecx)
34 : #endif
35 : );
36 1879 : }
37 :
38 23 : static inline void raid_xgetbv(uint32_t* reg)
39 : {
40 : /* get the value of the Extended Control Register ecx=0 */
41 23 : asm volatile (
42 : /* uses a direct encoding of the XGETBV instruction as only recent */
43 : /* assemblers support it. */
44 : /* the next line is equivalent at: "xgetbv\n" */
45 : ".byte 0x0f, 0x01, 0xd0\n"
46 23 : : "=a" (reg[0]), "=d" (reg[3])
47 : : "c" (0)
48 : );
49 23 : }
50 :
51 : #define CPU_VENDOR_MAX 13
52 :
53 249 : static inline void raid_cpu_info(char *vendor, unsigned *family, unsigned *model)
54 : {
55 : uint32_t reg[4];
56 : unsigned f, ef, m, em;
57 :
58 249 : raid_cpuid(0, 0, reg);
59 :
60 249 : ((uint32_t*)vendor)[0] = reg[1];
61 249 : ((uint32_t*)vendor)[1] = reg[3];
62 249 : ((uint32_t*)vendor)[2] = reg[2];
63 249 : vendor[12] = 0;
64 :
65 249 : raid_cpuid(1, 0, reg);
66 :
67 249 : f = (reg[0] >> 8) & 0xF;
68 249 : ef = (reg[0] >> 20) & 0xFF;
69 249 : m = (reg[0] >> 4) & 0xF;
70 249 : em = (reg[0] >> 16) & 0xF;
71 :
72 249 : if (strcmp(vendor, "AuthenticAMD") == 0) {
73 0 : if (f < 15) {
74 0 : *family = f;
75 0 : *model = m;
76 : } else {
77 0 : *family = f + ef;
78 0 : *model = m + (em << 4);
79 : }
80 : } else {
81 249 : *family = f + ef;
82 249 : *model = m + (em << 4);
83 : }
84 249 : }
85 :
86 1070 : static inline int raid_cpu_match_sse(uint32_t cpuid_1_ecx, uint32_t cpuid_1_edx)
87 : {
88 : uint32_t reg[4];
89 :
90 1070 : raid_cpuid(1, 0, reg);
91 1070 : if ((reg[2] & cpuid_1_ecx) != cpuid_1_ecx)
92 18 : return 0;
93 1052 : if ((reg[3] & cpuid_1_edx) != cpuid_1_edx)
94 0 : return 0;
95 :
96 1052 : return 1;
97 : }
98 :
99 288 : static inline int raid_cpu_match_avx(uint32_t cpuid_1_ecx, uint32_t cpuid_7_ebx, uint32_t xcr0)
100 : {
101 : uint32_t reg[4];
102 :
103 288 : raid_cpuid(1, 0, reg);
104 288 : if ((reg[2] & cpuid_1_ecx) != cpuid_1_ecx)
105 265 : return 0;
106 :
107 23 : raid_xgetbv(reg);
108 23 : if ((reg[0] & xcr0) != xcr0)
109 0 : return 0;
110 :
111 23 : raid_cpuid(7, 0, reg);
112 23 : if ((reg[1] & cpuid_7_ebx) != cpuid_7_ebx)
113 0 : return 0;
114 :
115 23 : return 1;
116 : }
117 :
118 276 : static inline int raid_cpu_has_sse2(void)
119 : {
120 : /*
121 : * Intel® 64 and IA-32 Architectures Software Developer's Manual
122 : * 325462-048US September 2013
123 : *
124 : * 11.6.2 Checking for SSE/SSE2 Support
125 : * Before an application attempts to use the SSE and/or SSE2 extensions, it should check
126 : * that they are present on the processor:
127 : * 1. Check that the processor supports the CPUID instruction. Bit 21 of the EFLAGS
128 : * register can be used to check processor's support the CPUID instruction.
129 : * 2. Check that the processor supports the SSE and/or SSE2 extensions (true if
130 : * CPUID.01H:EDX.SSE[bit 25] = 1 and/or CPUID.01H:EDX.SSE2[bit 26] = 1).
131 : */
132 276 : return raid_cpu_match_sse(
133 : 0,
134 : 1 << 26); /* SSE2 */
135 : }
136 :
137 530 : static inline int raid_cpu_has_ssse3(void)
138 : {
139 : /*
140 : * Intel® 64 and IA-32 Architectures Software Developer's Manual
141 : * 325462-048US September 2013
142 : *
143 : * 12.7.2 Checking for SSSE3 Support
144 : * Before an application attempts to use the SSSE3 extensions, the application should
145 : * follow the steps illustrated in Section 11.6.2, "Checking for SSE/SSE2 Support."
146 : * Next, use the additional step provided below:
147 : * Check that the processor supports SSSE3 (if CPUID.01H:ECX.SSSE3[bit 9] = 1).
148 : */
149 530 : return raid_cpu_match_sse(
150 : 1 << 9, /* SSSE3 */
151 : 1 << 26); /* SSE2 */
152 : }
153 :
154 264 : static inline int raid_cpu_has_crc32(void)
155 : {
156 : /*
157 : * Intel® 64 and IA-32 Architectures Software Developer's Manual
158 : * 325462-048US September 2013
159 : *
160 : * 12.12.3 Checking for SSE4.2 Support
161 : * ...
162 : * Before an application attempts to use the CRC32 instruction, it must check
163 : * that the processor supports SSE4.2 (if CPUID.01H:ECX.SSE4_2[bit 20] = 1).
164 : */
165 264 : return raid_cpu_match_sse(
166 : 1 << 20, /* CRC32 */
167 : 0);
168 : }
169 :
170 288 : static inline int raid_cpu_has_avx2(void)
171 : {
172 : /*
173 : * Intel Architecture Instruction Set Extensions Programming Reference
174 : * 319433-022 October 2014
175 : *
176 : * 14.3 Detection of AVX instructions
177 : * 1) Detect CPUID.1:ECX.OSXSAVE[bit 27] = 1 (XGETBV enabled for application use1)
178 : * 2) Issue XGETBV and verify that XCR0[2:1] = `11b' (XMM state and YMM state are enabled by OS).
179 : * 3) detect CPUID.1:ECX.AVX[bit 28] = 1 (AVX instructions supported).
180 : * (Step 3 can be done in any order relative to 1 and 2)
181 : *
182 : * 14.7.1 Detection of AVX2
183 : * Hardware support for AVX2 is indicated by CPUID.(EAX=07H, ECX=0H):EBX.AVX2[bit 5]=1.
184 : * Application Software must identify that hardware supports AVX, after that it must
185 : * also detect support for AVX2 by checking CPUID.(EAX=07H, ECX=0H):EBX.AVX2[bit 5].
186 : */
187 288 : return raid_cpu_match_avx(
188 : (1 << 27) | (1 << 28), /* OSXSAVE and AVX */
189 : 1 << 5, /* AVX2 */
190 : 3 << 1); /* OS saves XMM and YMM registers */
191 : }
192 :
193 : static inline int raid_cpu_has_avx512bw(void)
194 : {
195 : /*
196 : * Intel Architecture Instruction Set Extensions Programming Reference
197 : * 319433-022 October 2014
198 : *
199 : * 2.2 Detection of 512-bit Instruction Groups of Intel AVX-512 Family
200 : * 1) Detect CPUID.1:ECX.OSXSAVE[bit 27] = 1 (XGETBV enabled for application use)
201 : * 2) Execute XGETBV and verify that XCR0[7:5] = `111b' (OPMASK state, upper 256-bit of
202 : * ZMM0-ZMM15 and ZMM16-ZMM31 state are enabled by OS) and that XCR0[2:1] = `11b'
203 : * (XMM state and YMM state are enabled by OS).
204 : * 3) Verify both CPUID.0x7.0:EBX.AVX512F[bit 16] = 1, CPUID.0x7.0:EBX.AVX512BW[bit 30] = 1.
205 : */
206 :
207 : /* note that intentionally we don't check for AVX and AVX2 */
208 : /* because the documentation doesn't require that */
209 : return raid_cpu_match_avx(
210 : 1 << 27, /* XSAVE/XGETBV */
211 : (1 << 16) | (1 << 30), /* AVX512F and AVX512BW */
212 : (3 << 1) | (7 << 5)); /* OS saves XMM, YMM and ZMM registers */
213 : }
214 :
215 : /**
216 : * Check if the processor has a slow MULT implementation.
217 : * If yes, it's better to use a hash not based on multiplication.
218 : */
219 241 : static inline int raid_cpu_has_slowmult(void)
220 : {
221 : char vendor[CPU_VENDOR_MAX];
222 : unsigned family;
223 : unsigned model;
224 :
225 241 : raid_cpu_info(vendor, &family, &model);
226 :
227 241 : if (strcmp(vendor, "GenuineIntel") == 0) {
228 : /*
229 : * Intel Atom
230 : * Murmur3 based on MUL instruction, is a lot slower
231 : * than Spooky2 based on SHIFTs.
232 : * Like: 378 MB/s vs 3413 MB/s
233 : */
234 241 : if (family == 6 && model == 28)
235 0 : return 1;
236 : }
237 :
238 241 : return 0;
239 : }
240 :
241 : /**
242 : * Check if the processor has a slow extended set of SSE registers.
243 : * If yes, it's better to limit the unroll to the firsrt 8 registers.
244 : */
245 4 : static inline int raid_cpu_has_slowextendedreg(void)
246 : {
247 : char vendor[CPU_VENDOR_MAX];
248 : unsigned family;
249 : unsigned model;
250 :
251 4 : raid_cpu_info(vendor, &family, &model);
252 :
253 4 : if (strcmp(vendor, "AuthenticAMD") == 0) {
254 : /*
255 : * AMD Bulldozer
256 : * PAR1/PAR2 implementations using 16 SSE registers are slower
257 : * than ones using only the first 8 registers.
258 : * Like: 4465 MB/s vs 4922 MB/s.
259 : */
260 0 : if (family == 21)
261 0 : return 1;
262 : }
263 :
264 4 : return 0;
265 : }
266 : #endif
267 :
268 : #endif
269 :
|