1 /* x86 CPU feature tuning.
2 This file is part of the GNU C Library.
3 Copyright (C) 2017-2021 Free Software Foundation, Inc.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
20 # define TUNABLE_NAMESPACE cpu
23 # include <unistd.h> /* Get STDOUT_FILENO for _dl_printf. */
24 # include <elf/dl-tunables.h>
26 # include <cpu-features.h>
27 # include <ldsodefs.h>
29 /* We can't use IFUNC memcmp nor strlen in init_cpu_features from libc.a
30 since IFUNC must be set up by init_cpu_features. */
31 # if defined USE_MULTIARCH && !defined SHARED
33 # define DEFAULT_MEMCMP __memcmp_sse2
35 # define DEFAULT_MEMCMP __memcmp_ia32
37 extern __typeof (memcmp
) DEFAULT_MEMCMP
;
39 # define DEFAULT_MEMCMP memcmp
42 # define CHECK_GLIBC_IFUNC_CPU_OFF(f, cpu_features, name, len) \
43 _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \
44 if (!DEFAULT_MEMCMP (f, #name, len)) \
46 CPU_FEATURE_UNSET (cpu_features, name) \
50 /* Disable a preferred feature NAME. We don't enable a preferred feature
51 which isn't available. */
52 # define CHECK_GLIBC_IFUNC_PREFERRED_OFF(f, cpu_features, name, len) \
53 _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \
54 if (!DEFAULT_MEMCMP (f, #name, len)) \
56 cpu_features->preferred[index_arch_##name] \
57 &= ~bit_arch_##name; \
61 /* Enable/disable a preferred feature NAME. */
62 # define CHECK_GLIBC_IFUNC_PREFERRED_BOTH(f, cpu_features, name, \
64 _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \
65 if (!DEFAULT_MEMCMP (f, #name, len)) \
68 cpu_features->preferred[index_arch_##name] &= ~bit_arch_##name; \
70 cpu_features->preferred[index_arch_##name] |= bit_arch_##name; \
74 /* Enable/disable a preferred feature NAME. Enable a preferred feature
75 only if the feature NEED is usable. */
76 # define CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH(f, cpu_features, name, \
78 _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \
79 if (!DEFAULT_MEMCMP (f, #name, len)) \
82 cpu_features->preferred[index_arch_##name] &= ~bit_arch_##name; \
83 else if (CPU_FEATURE_USABLE_P (cpu_features, need)) \
84 cpu_features->preferred[index_arch_##name] |= bit_arch_##name; \
90 TUNABLE_CALLBACK (set_hwcaps
) (tunable_val_t
*valp
)
92 /* The current IFUNC selection is based on microbenchmarks in glibc.
93 It should give the best performance for most workloads. But other
94 choices may have better performance for a particular workload or on
95 the hardware which wasn't available when the selection was made.
96 The environment variable:
98 GLIBC_TUNABLES=glibc.cpu.hwcaps=-xxx,yyy,-zzz,....
100 can be used to enable CPU/ARCH feature yyy, disable CPU/ARCH feature
101 yyy and zzz, where the feature name is case-sensitive and has to
102 match the ones in cpu-features.h. It can be used by glibc developers
103 to tune for a new processor or override the IFUNC selection to
104 improve performance for a particular workload.
106 NOTE: the IFUNC selection may change over time. Please check all
107 multiarch implementations when experimenting. */
109 const char *p
= valp
->strval
;
110 struct cpu_features
*cpu_features
= &GLRO(dl_x86_cpu_features
);
119 for (c
= p
; *c
!= ','; c
++)
142 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, AVX
, 3);
143 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, CX8
, 3);
144 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, FMA
, 3);
145 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, HTT
, 3);
146 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, IBT
, 3);
147 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, RTM
, 3);
153 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, AVX2
, 4);
154 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, BMI1
, 4);
155 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, BMI2
, 4);
156 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, CMOV
, 4);
157 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, ERMS
, 4);
158 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, FMA4
, 4);
159 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, SSE2
, 4);
160 CHECK_GLIBC_IFUNC_PREFERRED_OFF (n
, cpu_features
, I586
, 4);
161 CHECK_GLIBC_IFUNC_PREFERRED_OFF (n
, cpu_features
, I686
, 4);
167 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, LZCNT
, 5);
168 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, MOVBE
, 5);
169 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, SHSTK
, 5);
170 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, SSSE3
, 5);
176 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, POPCNT
, 6);
177 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, SSE4_1
, 6);
178 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, SSE4_2
, 6);
179 if (!DEFAULT_MEMCMP (n
, "XSAVEC", 6))
181 /* Update xsave_state_size to XSAVE state size. */
182 cpu_features
->xsave_state_size
183 = cpu_features
->xsave_state_full_size
;
184 CPU_FEATURE_UNSET (cpu_features
, XSAVEC
);
191 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, AVX512F
, 7);
192 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, OSXSAVE
, 7);
198 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, AVX512CD
, 8);
199 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, AVX512BW
, 8);
200 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, AVX512DQ
, 8);
201 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, AVX512ER
, 8);
202 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, AVX512PF
, 8);
203 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, AVX512VL
, 8);
205 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n
, cpu_features
, Slow_BSF
,
210 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n
, cpu_features
,
213 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n
, cpu_features
,
216 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH (n
, cpu_features
,
224 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n
, cpu_features
,
231 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
232 (n
, cpu_features
, Prefer_No_AVX512
, AVX512F
,
238 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n
, cpu_features
,
245 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n
, cpu_features
,
248 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n
, cpu_features
,
255 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
256 (n
, cpu_features
, Prefer_No_VZEROUPPER
, AVX
, disable
,
262 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n
, cpu_features
,
263 Prefer_MAP_32BIT_EXEC
,
269 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
270 (n
, cpu_features
, AVX_Fast_Unaligned_Load
, AVX
,
276 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
277 (n
, cpu_features
, MathVec_Prefer_No_AVX512
, AVX512F
,
283 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
284 (n
, cpu_features
, Prefer_PMINUB_for_stringop
, SSE2
,
298 TUNABLE_CALLBACK (set_x86_ibt
) (tunable_val_t
*valp
)
300 if (DEFAULT_MEMCMP (valp
->strval
, "on", sizeof ("on")) == 0)
301 GL(dl_x86_feature_control
).ibt
= cet_always_on
;
302 else if (DEFAULT_MEMCMP (valp
->strval
, "off", sizeof ("off")) == 0)
303 GL(dl_x86_feature_control
).ibt
= cet_always_off
;
304 else if (DEFAULT_MEMCMP (valp
->strval
, "permissive",
305 sizeof ("permissive")) == 0)
306 GL(dl_x86_feature_control
).ibt
= cet_permissive
;
311 TUNABLE_CALLBACK (set_x86_shstk
) (tunable_val_t
*valp
)
313 if (DEFAULT_MEMCMP (valp
->strval
, "on", sizeof ("on")) == 0)
314 GL(dl_x86_feature_control
).shstk
= cet_always_on
;
315 else if (DEFAULT_MEMCMP (valp
->strval
, "off", sizeof ("off")) == 0)
316 GL(dl_x86_feature_control
).shstk
= cet_always_off
;
317 else if (DEFAULT_MEMCMP (valp
->strval
, "permissive",
318 sizeof ("permissive")) == 0)
319 GL(dl_x86_feature_control
).shstk
= cet_permissive
;