]>
Commit | Line | Data |
---|---|---|
905947c3 L |
1 | /* x86 CPU feature tuning. |
2 | This file is part of the GNU C Library. | |
6d7e8eda | 3 | Copyright (C) 2017-2023 Free Software Foundation, Inc. |
905947c3 L |
4 | |
5 | The GNU C Library is free software; you can redistribute it and/or | |
6 | modify it under the terms of the GNU Lesser General Public | |
7 | License as published by the Free Software Foundation; either | |
8 | version 2.1 of the License, or (at your option) any later version. | |
9 | ||
10 | The GNU C Library is distributed in the hope that it will be useful, | |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 | Lesser General Public License for more details. | |
14 | ||
15 | You should have received a copy of the GNU Lesser General Public | |
16 | License along with the GNU C Library; if not, see | |
5a82c748 | 17 | <https://www.gnu.org/licenses/>. */ |
905947c3 | 18 | |
33237fe8 AZN |
19 | #define TUNABLE_NAMESPACE cpu |
20 | #include <stdbool.h> | |
21 | #include <stdint.h> | |
22 | #include <unistd.h> /* Get STDOUT_FILENO for _dl_printf. */ | |
23 | #include <elf/dl-tunables.h> | |
24 | #include <string.h> | |
25 | #include <cpu-features.h> | |
26 | #include <ldsodefs.h> | |
905947c3 L |
27 | |
28 | /* We can't use IFUNC memcmp nor strlen in init_cpu_features from libc.a | |
29 | since IFUNC must be set up by init_cpu_features. */ | |
33237fe8 AZN |
30 | #if defined USE_MULTIARCH && !defined SHARED |
31 | # ifdef __x86_64__ | |
ae308947 | 32 | /* DEFAULT_MEMCMP by sysdeps/x86_64/memcmp-isa-default-impl.h. */ |
33237fe8 | 33 | # include <sysdeps/x86_64/memcmp-isa-default-impl.h> |
905947c3 | 34 | # else |
33237fe8 | 35 | # define DEFAULT_MEMCMP __memcmp_ia32 |
905947c3 | 36 | # endif |
33237fe8 AZN |
37 | extern __typeof (memcmp) DEFAULT_MEMCMP; |
38 | #else | |
39 | # define DEFAULT_MEMCMP memcmp | |
40 | #endif | |
905947c3 | 41 | |
33237fe8 | 42 | #define CHECK_GLIBC_IFUNC_CPU_OFF(f, cpu_features, name, len) \ |
905947c3 L |
43 | _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \ |
44 | if (!DEFAULT_MEMCMP (f, #name, len)) \ | |
45 | { \ | |
107e6a3c | 46 | CPU_FEATURE_UNSET (cpu_features, name) \ |
905947c3 L |
47 | break; \ |
48 | } | |
49 | ||
107e6a3c L |
50 | /* Disable a preferred feature NAME. We don't enable a preferred feature |
51 | which isn't available. */ | |
33237fe8 | 52 | #define CHECK_GLIBC_IFUNC_PREFERRED_OFF(f, cpu_features, name, len) \ |
905947c3 L |
53 | _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \ |
54 | if (!DEFAULT_MEMCMP (f, #name, len)) \ | |
55 | { \ | |
107e6a3c | 56 | cpu_features->preferred[index_arch_##name] \ |
905947c3 L |
57 | &= ~bit_arch_##name; \ |
58 | break; \ | |
59 | } | |
60 | ||
107e6a3c | 61 | /* Enable/disable a preferred feature NAME. */ |
33237fe8 | 62 | #define CHECK_GLIBC_IFUNC_PREFERRED_BOTH(f, cpu_features, name, \ |
107e6a3c | 63 | disable, len) \ |
905947c3 L |
64 | _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \ |
65 | if (!DEFAULT_MEMCMP (f, #name, len)) \ | |
66 | { \ | |
67 | if (disable) \ | |
107e6a3c | 68 | cpu_features->preferred[index_arch_##name] &= ~bit_arch_##name; \ |
905947c3 | 69 | else \ |
107e6a3c | 70 | cpu_features->preferred[index_arch_##name] |= bit_arch_##name; \ |
905947c3 L |
71 | break; \ |
72 | } | |
73 | ||
107e6a3c L |
74 | /* Enable/disable a preferred feature NAME. Enable a preferred feature |
75 | only if the feature NEED is usable. */ | |
33237fe8 | 76 | #define CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH(f, cpu_features, name, \ |
905947c3 L |
77 | need, disable, len) \ |
78 | _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \ | |
79 | if (!DEFAULT_MEMCMP (f, #name, len)) \ | |
80 | { \ | |
81 | if (disable) \ | |
107e6a3c L |
82 | cpu_features->preferred[index_arch_##name] &= ~bit_arch_##name; \ |
83 | else if (CPU_FEATURE_USABLE_P (cpu_features, need)) \ | |
84 | cpu_features->preferred[index_arch_##name] |= bit_arch_##name; \ | |
905947c3 L |
85 | break; \ |
86 | } | |
87 | ||
88 | attribute_hidden | |
89 | void | |
03feacb5 | 90 | TUNABLE_CALLBACK (set_hwcaps) (tunable_val_t *valp) |
905947c3 L |
91 | { |
92 | /* The current IFUNC selection is based on microbenchmarks in glibc. | |
93 | It should give the best performance for most workloads. But other | |
94 | choices may have better performance for a particular workload or on | |
95 | the hardware which wasn't available when the selection was made. | |
03feacb5 | 96 | The environment variable: |
905947c3 | 97 | |
dce452dc | 98 | GLIBC_TUNABLES=glibc.cpu.hwcaps=-xxx,yyy,-zzz,.... |
03feacb5 L |
99 | |
100 | can be used to enable CPU/ARCH feature yyy, disable CPU/ARCH feature | |
101 | yyy and zzz, where the feature name is case-sensitive and has to | |
102 | match the ones in cpu-features.h. It can be used by glibc developers | |
103 | to tune for a new processor or override the IFUNC selection to | |
104 | improve performance for a particular workload. | |
905947c3 L |
105 | |
106 | NOTE: the IFUNC selection may change over time. Please check all | |
107 | multiarch implementations when experimenting. */ | |
108 | ||
4c721f24 | 109 | const char *p = valp->strval, *c; |
905947c3 | 110 | struct cpu_features *cpu_features = &GLRO(dl_x86_cpu_features); |
905947c3 L |
111 | size_t len; |
112 | ||
113 | do | |
114 | { | |
4c721f24 | 115 | const char *n; |
905947c3 L |
116 | bool disable; |
117 | size_t nl; | |
118 | ||
119 | for (c = p; *c != ','; c++) | |
03feacb5 | 120 | if (*c == '\0') |
905947c3 L |
121 | break; |
122 | ||
123 | len = c - p; | |
124 | disable = *p == '-'; | |
125 | if (disable) | |
126 | { | |
127 | n = p + 1; | |
128 | nl = len - 1; | |
129 | } | |
130 | else | |
131 | { | |
132 | n = p; | |
133 | nl = len; | |
134 | } | |
135 | switch (nl) | |
136 | { | |
137 | default: | |
138 | break; | |
139 | case 3: | |
140 | if (disable) | |
141 | { | |
142 | CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX, 3); | |
143 | CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, CX8, 3); | |
144 | CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, FMA, 3); | |
145 | CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, HTT, 3); | |
73322d5f | 146 | CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, IBT, 3); |
905947c3 L |
147 | CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, RTM, 3); |
148 | } | |
149 | break; | |
150 | case 4: | |
151 | if (disable) | |
152 | { | |
153 | CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX2, 4); | |
154 | CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, BMI1, 4); | |
155 | CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, BMI2, 4); | |
156 | CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, CMOV, 4); | |
157 | CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, ERMS, 4); | |
158 | CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, FMA4, 4); | |
159 | CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE2, 4); | |
107e6a3c L |
160 | CHECK_GLIBC_IFUNC_PREFERRED_OFF (n, cpu_features, I586, 4); |
161 | CHECK_GLIBC_IFUNC_PREFERRED_OFF (n, cpu_features, I686, 4); | |
905947c3 L |
162 | } |
163 | break; | |
164 | case 5: | |
165 | if (disable) | |
166 | { | |
167 | CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, LZCNT, 5); | |
168 | CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, MOVBE, 5); | |
73322d5f | 169 | CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SHSTK, 5); |
905947c3 | 170 | CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSSE3, 5); |
27f74636 | 171 | CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, XSAVE, 5); |
905947c3 L |
172 | } |
173 | break; | |
174 | case 6: | |
175 | if (disable) | |
176 | { | |
177 | CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, POPCNT, 6); | |
178 | CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE4_1, 6); | |
179 | CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE4_2, 6); | |
107e6a3c L |
180 | if (!DEFAULT_MEMCMP (n, "XSAVEC", 6)) |
181 | { | |
182 | /* Update xsave_state_size to XSAVE state size. */ | |
183 | cpu_features->xsave_state_size | |
184 | = cpu_features->xsave_state_full_size; | |
185 | CPU_FEATURE_UNSET (cpu_features, XSAVEC); | |
186 | } | |
905947c3 L |
187 | } |
188 | break; | |
189 | case 7: | |
190 | if (disable) | |
191 | { | |
192 | CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512F, 7); | |
193 | CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, OSXSAVE, 7); | |
194 | } | |
195 | break; | |
196 | case 8: | |
197 | if (disable) | |
198 | { | |
199 | CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512CD, 8); | |
200 | CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512BW, 8); | |
201 | CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512DQ, 8); | |
202 | CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512ER, 8); | |
203 | CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512PF, 8); | |
204 | CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512VL, 8); | |
205 | } | |
107e6a3c L |
206 | CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features, Slow_BSF, |
207 | disable, 8); | |
905947c3 L |
208 | break; |
209 | case 11: | |
905947c3 | 210 | { |
107e6a3c L |
211 | CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features, |
212 | Prefer_ERMS, | |
905947c3 | 213 | disable, 11); |
107e6a3c L |
214 | CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features, |
215 | Prefer_FSRM, | |
216 | disable, 11); | |
217 | CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH (n, cpu_features, | |
218 | Slow_SSE4_2, | |
219 | SSE4_2, | |
220 | disable, 11); | |
905947c3 L |
221 | } |
222 | break; | |
223 | case 15: | |
905947c3 | 224 | { |
107e6a3c L |
225 | CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features, |
226 | Fast_Rep_String, | |
227 | disable, 15); | |
905947c3 | 228 | } |
905947c3 L |
229 | break; |
230 | case 16: | |
231 | { | |
107e6a3c L |
232 | CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH |
233 | (n, cpu_features, Prefer_No_AVX512, AVX512F, | |
905947c3 L |
234 | disable, 16); |
235 | } | |
236 | break; | |
237 | case 18: | |
238 | { | |
107e6a3c L |
239 | CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features, |
240 | Fast_Copy_Backward, | |
241 | disable, 18); | |
905947c3 L |
242 | } |
243 | break; | |
244 | case 19: | |
245 | { | |
107e6a3c L |
246 | CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features, |
247 | Fast_Unaligned_Load, | |
248 | disable, 19); | |
249 | CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features, | |
250 | Fast_Unaligned_Copy, | |
251 | disable, 19); | |
905947c3 L |
252 | } |
253 | break; | |
254 | case 20: | |
255 | { | |
107e6a3c L |
256 | CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH |
257 | (n, cpu_features, Prefer_No_VZEROUPPER, AVX, disable, | |
258 | 20); | |
905947c3 L |
259 | } |
260 | break; | |
905947c3 L |
261 | case 23: |
262 | { | |
107e6a3c L |
263 | CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH |
264 | (n, cpu_features, AVX_Fast_Unaligned_Load, AVX, | |
905947c3 L |
265 | disable, 23); |
266 | } | |
267 | break; | |
ef8adeb0 L |
268 | case 24: |
269 | { | |
107e6a3c L |
270 | CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH |
271 | (n, cpu_features, MathVec_Prefer_No_AVX512, AVX512F, | |
272 | disable, 24); | |
ef8adeb0 L |
273 | } |
274 | break; | |
905947c3 L |
275 | case 26: |
276 | { | |
107e6a3c | 277 | CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH |
905947c3 L |
278 | (n, cpu_features, Prefer_PMINUB_for_stringop, SSE2, |
279 | disable, 26); | |
280 | } | |
281 | break; | |
905947c3 L |
282 | } |
283 | p += len + 1; | |
284 | } | |
4c721f24 | 285 | while (*c != '\0'); |
905947c3 | 286 | } |
f753fa7d | 287 | |
33237fe8 | 288 | #if CET_ENABLED |
f753fa7d L |
289 | attribute_hidden |
290 | void | |
291 | TUNABLE_CALLBACK (set_x86_ibt) (tunable_val_t *valp) | |
292 | { | |
293 | if (DEFAULT_MEMCMP (valp->strval, "on", sizeof ("on")) == 0) | |
674ea882 | 294 | GL(dl_x86_feature_control).ibt = cet_always_on; |
f753fa7d | 295 | else if (DEFAULT_MEMCMP (valp->strval, "off", sizeof ("off")) == 0) |
674ea882 | 296 | GL(dl_x86_feature_control).ibt = cet_always_off; |
f753fa7d L |
297 | else if (DEFAULT_MEMCMP (valp->strval, "permissive", |
298 | sizeof ("permissive")) == 0) | |
674ea882 | 299 | GL(dl_x86_feature_control).ibt = cet_permissive; |
f753fa7d L |
300 | } |
301 | ||
302 | attribute_hidden | |
303 | void | |
304 | TUNABLE_CALLBACK (set_x86_shstk) (tunable_val_t *valp) | |
305 | { | |
306 | if (DEFAULT_MEMCMP (valp->strval, "on", sizeof ("on")) == 0) | |
674ea882 | 307 | GL(dl_x86_feature_control).shstk = cet_always_on; |
f753fa7d | 308 | else if (DEFAULT_MEMCMP (valp->strval, "off", sizeof ("off")) == 0) |
674ea882 | 309 | GL(dl_x86_feature_control).shstk = cet_always_off; |
f753fa7d L |
310 | else if (DEFAULT_MEMCMP (valp->strval, "permissive", |
311 | sizeof ("permissive")) == 0) | |
674ea882 | 312 | GL(dl_x86_feature_control).shstk = cet_permissive; |
f753fa7d | 313 | } |
905947c3 | 314 | #endif |