sysdeps/x86/cpu-features.c

   1 /* Initialize CPU feature data.
   2    This file is part of the GNU C Library.
   3    Copyright (C) 2008-2018 Free Software Foundation, Inc.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <http://www.gnu.org/licenses/>.  */
  18
  19 #include <cpuid.h>
  20 #include <cpu-features.h>
  21 #include <dl-hwcap.h>
  22 #include <libc-pointer-arith.h>
  23
  24 #if HAVE_TUNABLES
  25 # define TUNABLE_NAMESPACE tune
  26 # include <unistd.h>            /* Get STDOUT_FILENO for _dl_printf.  */
  27 # include <elf/dl-tunables.h>
  28
  29 extern void TUNABLE_CALLBACK (set_hwcaps) (tunable_val_t *)
  30   attribute_hidden;
  31 #endif
  32
  33 static void
  34 get_common_indeces (struct cpu_features *cpu_features,
  35                     unsigned int *family, unsigned int *model,
  36                     unsigned int *extended_model, unsigned int *stepping)
  37 {
  38   if (family)
  39     {
  40       unsigned int eax;
  41       __cpuid (1, eax, cpu_features->cpuid[COMMON_CPUID_INDEX_1].ebx,
  42                cpu_features->cpuid[COMMON_CPUID_INDEX_1].ecx,
  43                cpu_features->cpuid[COMMON_CPUID_INDEX_1].edx);
  44       cpu_features->cpuid[COMMON_CPUID_INDEX_1].eax = eax;
  45       *family = (eax >> 8) & 0x0f;
  46       *model = (eax >> 4) & 0x0f;
  47       *extended_model = (eax >> 12) & 0xf0;
  48       *stepping = eax & 0x0f;
  49       if (*family == 0x0f)
  50         {
  51           *family += (eax >> 20) & 0xff;
  52           *model += *extended_model;
  53         }
  54     }
  55
  56   if (cpu_features->max_cpuid >= 7)
  57     __cpuid_count (7, 0,
  58                    cpu_features->cpuid[COMMON_CPUID_INDEX_7].eax,
  59                    cpu_features->cpuid[COMMON_CPUID_INDEX_7].ebx,
  60                    cpu_features->cpuid[COMMON_CPUID_INDEX_7].ecx,
  61                    cpu_features->cpuid[COMMON_CPUID_INDEX_7].edx);
  62
  63   /* Can we call xgetbv?  */
  64   if (CPU_FEATURES_CPU_P (cpu_features, OSXSAVE))
  65     {
  66       unsigned int xcrlow;
  67       unsigned int xcrhigh;
  68       asm ("xgetbv" : "=a" (xcrlow), "=d" (xcrhigh) : "c" (0));
  69       /* Is YMM and XMM state usable?  */
  70       if ((xcrlow & (bit_YMM_state | bit_XMM_state)) ==
  71           (bit_YMM_state | bit_XMM_state))
  72         {
  73           /* Determine if AVX is usable.  */
  74           if (CPU_FEATURES_CPU_P (cpu_features, AVX))
  75             {
  76               cpu_features->feature[index_arch_AVX_Usable]
  77                 |= bit_arch_AVX_Usable;
  78               /* The following features depend on AVX being usable.  */
  79               /* Determine if AVX2 is usable.  */
  80               if (CPU_FEATURES_CPU_P (cpu_features, AVX2))
  81                 cpu_features->feature[index_arch_AVX2_Usable]
  82                   |= bit_arch_AVX2_Usable;
  83               /* Determine if FMA is usable.  */
  84               if (CPU_FEATURES_CPU_P (cpu_features, FMA))
  85                 cpu_features->feature[index_arch_FMA_Usable]
  86                   |= bit_arch_FMA_Usable;
  87             }
  88
  89           /* Check if OPMASK state, upper 256-bit of ZMM0-ZMM15 and
  90              ZMM16-ZMM31 state are enabled.  */
  91           if ((xcrlow & (bit_Opmask_state | bit_ZMM0_15_state
  92                          | bit_ZMM16_31_state)) ==
  93               (bit_Opmask_state | bit_ZMM0_15_state | bit_ZMM16_31_state))
  94             {
  95               /* Determine if AVX512F is usable.  */
  96               if (CPU_FEATURES_CPU_P (cpu_features, AVX512F))
  97                 {
  98                   cpu_features->feature[index_arch_AVX512F_Usable]
  99                     |= bit_arch_AVX512F_Usable;
 100                   /* Determine if AVX512DQ is usable.  */
 101                   if (CPU_FEATURES_CPU_P (cpu_features, AVX512DQ))
 102                     cpu_features->feature[index_arch_AVX512DQ_Usable]
 103                       |= bit_arch_AVX512DQ_Usable;
 104                 }
 105             }
 106         }
 107
 108       /* For _dl_runtime_resolve, set xsave_state_size to xsave area
 109          size + integer register save size and align it to 64 bytes.  */
 110       if (cpu_features->max_cpuid >= 0xd)
 111         {
 112           unsigned int eax, ebx, ecx, edx;
 113
 114           __cpuid_count (0xd, 0, eax, ebx, ecx, edx);
 115           if (ebx != 0)
 116             {
 117               unsigned int xsave_state_full_size
 118                 = ALIGN_UP (ebx + STATE_SAVE_OFFSET, 64);
 119
 120               cpu_features->xsave_state_size
 121                 = xsave_state_full_size;
 122               cpu_features->xsave_state_full_size
 123                 = xsave_state_full_size;
 124
 125               __cpuid_count (0xd, 1, eax, ebx, ecx, edx);
 126
 127               /* Check if XSAVEC is available.  */
 128               if ((eax & (1 << 1)) != 0)
 129                 {
 130                   unsigned int xstate_comp_offsets[32];
 131                   unsigned int xstate_comp_sizes[32];
 132                   unsigned int i;
 133
 134                   xstate_comp_offsets[0] = 0;
 135                   xstate_comp_offsets[1] = 160;
 136                   xstate_comp_offsets[2] = 576;
 137                   xstate_comp_sizes[0] = 160;
 138                   xstate_comp_sizes[1] = 256;
 139
 140                   for (i = 2; i < 32; i++)
 141                     {
 142                       if ((STATE_SAVE_MASK & (1 << i)) != 0)
 143                         {
 144                           __cpuid_count (0xd, i, eax, ebx, ecx, edx);
 145                           xstate_comp_sizes[i] = eax;
 146                         }
 147                       else
 148                         {
 149                           ecx = 0;
 150                           xstate_comp_sizes[i] = 0;
 151                         }
 152
 153                       if (i > 2)
 154                         {
 155                           xstate_comp_offsets[i]
 156                             = (xstate_comp_offsets[i - 1]
 157                                + xstate_comp_sizes[i -1]);
 158                           if ((ecx & (1 << 1)) != 0)
 159                             xstate_comp_offsets[i]
 160                               = ALIGN_UP (xstate_comp_offsets[i], 64);
 161                         }
 162                     }
 163
 164                   /* Use XSAVEC.  */
 165                   unsigned int size
 166                     = xstate_comp_offsets[31] + xstate_comp_sizes[31];
 167                   if (size)
 168                     {
 169                       cpu_features->xsave_state_size
 170                         = ALIGN_UP (size + STATE_SAVE_OFFSET, 64);
 171                       cpu_features->feature[index_arch_XSAVEC_Usable]
 172                         |= bit_arch_XSAVEC_Usable;
 173                     }
 174                 }
 175             }
 176         }
 177     }
 178 }
 179
 180 static inline void
 181 init_cpu_features (struct cpu_features *cpu_features)
 182 {
 183   unsigned int ebx, ecx, edx;
 184   unsigned int family = 0;
 185   unsigned int model = 0;
 186   enum cpu_features_kind kind;
 187
 188 #if !HAS_CPUID
 189   if (__get_cpuid_max (0, 0) == 0)
 190     {
 191       kind = arch_kind_other;
 192       goto no_cpuid;
 193     }
 194 #endif
 195
 196   __cpuid (0, cpu_features->max_cpuid, ebx, ecx, edx);
 197
 198   /* This spells out "GenuineIntel".  */
 199   if (ebx == 0x756e6547 && ecx == 0x6c65746e && edx == 0x49656e69)
 200     {
 201       unsigned int extended_model, stepping;
 202
 203       kind = arch_kind_intel;
 204
 205       get_common_indeces (cpu_features, &family, &model, &extended_model,
 206                           &stepping);
 207
 208       if (family == 0x06)
 209         {
 210           model += extended_model;
 211           switch (model)
 212             {
 213             case 0x1c:
 214             case 0x26:
 215               /* BSF is slow on Atom.  */
 216               cpu_features->feature[index_arch_Slow_BSF]
 217                 |= bit_arch_Slow_BSF;
 218               break;
 219
 220             case 0x57:
 221               /* Knights Landing.  Enable Silvermont optimizations.  */
 222
 223             case 0x5c:
 224             case 0x5f:
 225               /* Unaligned load versions are faster than SSSE3
 226                  on Goldmont.  */
 227
 228             case 0x4c:
 229               /* Airmont is a die shrink of Silvermont.  */
 230
 231             case 0x37:
 232             case 0x4a:
 233             case 0x4d:
 234             case 0x5a:
 235             case 0x5d:
 236               /* Unaligned load versions are faster than SSSE3
 237                  on Silvermont.  */
 238 #if index_arch_Fast_Unaligned_Load != index_arch_Prefer_PMINUB_for_stringop
 239 # error index_arch_Fast_Unaligned_Load != index_arch_Prefer_PMINUB_for_stringop
 240 #endif
 241 #if index_arch_Fast_Unaligned_Load != index_arch_Slow_SSE4_2
 242 # error index_arch_Fast_Unaligned_Load != index_arch_Slow_SSE4_2
 243 #endif
 244 #if index_arch_Fast_Unaligned_Load != index_arch_Fast_Unaligned_Copy
 245 # error index_arch_Fast_Unaligned_Load != index_arch_Fast_Unaligned_Copy
 246 #endif
 247               cpu_features->feature[index_arch_Fast_Unaligned_Load]
 248                 |= (bit_arch_Fast_Unaligned_Load
 249                     | bit_arch_Fast_Unaligned_Copy
 250                     | bit_arch_Prefer_PMINUB_for_stringop
 251                     | bit_arch_Slow_SSE4_2);
 252               break;
 253
 254             default:
 255               /* Unknown family 0x06 processors.  Assuming this is one
 256                  of Core i3/i5/i7 processors if AVX is available.  */
 257               if (!CPU_FEATURES_CPU_P (cpu_features, AVX))
 258                 break;
 259
 260             case 0x1a:
 261             case 0x1e:
 262             case 0x1f:
 263             case 0x25:
 264             case 0x2c:
 265             case 0x2e:
 266             case 0x2f:
 267               /* Rep string instructions, unaligned load, unaligned copy,
 268                  and pminub are fast on Intel Core i3, i5 and i7.  */
 269 #if index_arch_Fast_Rep_String != index_arch_Fast_Unaligned_Load
 270 # error index_arch_Fast_Rep_String != index_arch_Fast_Unaligned_Load
 271 #endif
 272 #if index_arch_Fast_Rep_String != index_arch_Prefer_PMINUB_for_stringop
 273 # error index_arch_Fast_Rep_String != index_arch_Prefer_PMINUB_for_stringop
 274 #endif
 275 #if index_arch_Fast_Rep_String != index_arch_Fast_Unaligned_Copy
 276 # error index_arch_Fast_Rep_String != index_arch_Fast_Unaligned_Copy
 277 #endif
 278               cpu_features->feature[index_arch_Fast_Rep_String]
 279                 |= (bit_arch_Fast_Rep_String
 280                     | bit_arch_Fast_Unaligned_Load
 281                     | bit_arch_Fast_Unaligned_Copy
 282                     | bit_arch_Prefer_PMINUB_for_stringop);
 283               break;
 284
 285             case 0x3f:
 286               /* Xeon E7 v3 with stepping >= 4 has working TSX.  */
 287               if (stepping >= 4)
 288                 break;
 289             case 0x3c:
 290             case 0x45:
 291             case 0x46:
 292               /* Disable Intel TSX on Haswell processors (except Xeon E7 v3
 293                  with stepping >= 4) to avoid TSX on kernels that weren't
 294                  updated with the latest microcode package (which disables
 295                  broken feature by default).  */
 296               cpu_features->cpuid[index_cpu_RTM].reg_RTM &= ~bit_cpu_RTM;
 297               break;
 298             }
 299         }
 300
 301       /* Unaligned load with 256-bit AVX registers are faster on
 302          Intel processors with AVX2.  */
 303       if (CPU_FEATURES_ARCH_P (cpu_features, AVX2_Usable))
 304         cpu_features->feature[index_arch_AVX_Fast_Unaligned_Load]
 305           |= bit_arch_AVX_Fast_Unaligned_Load;
 306
 307       /* Since AVX512ER is unique to Xeon Phi, set Prefer_No_VZEROUPPER
 308          if AVX512ER is available.  Don't use AVX512 to avoid lower CPU
 309          frequency if AVX512ER isn't available.  */
 310       if (CPU_FEATURES_CPU_P (cpu_features, AVX512ER))
 311         cpu_features->feature[index_arch_Prefer_No_VZEROUPPER]
 312           |= bit_arch_Prefer_No_VZEROUPPER;
 313       else
 314         cpu_features->feature[index_arch_Prefer_No_AVX512]
 315           |= bit_arch_Prefer_No_AVX512;
 316     }
 317   /* This spells out "AuthenticAMD".  */
 318   else if (ebx == 0x68747541 && ecx == 0x444d4163 && edx == 0x69746e65)
 319     {
 320       unsigned int extended_model, stepping;
 321
 322       kind = arch_kind_amd;
 323
 324       get_common_indeces (cpu_features, &family, &model, &extended_model,
 325                           &stepping);
 326
 327       ecx = cpu_features->cpuid[COMMON_CPUID_INDEX_1].ecx;
 328
 329       unsigned int eax;
 330       __cpuid (0x80000000, eax, ebx, ecx, edx);
 331       if (eax >= 0x80000001)
 332         __cpuid (0x80000001,
 333                  cpu_features->cpuid[COMMON_CPUID_INDEX_80000001].eax,
 334                  cpu_features->cpuid[COMMON_CPUID_INDEX_80000001].ebx,
 335                  cpu_features->cpuid[COMMON_CPUID_INDEX_80000001].ecx,
 336                  cpu_features->cpuid[COMMON_CPUID_INDEX_80000001].edx);
 337
 338       if (HAS_ARCH_FEATURE (AVX_Usable))
 339         {
 340           /* Since the FMA4 bit is in COMMON_CPUID_INDEX_80000001 and
 341              FMA4 requires AVX, determine if FMA4 is usable here.  */
 342           if (CPU_FEATURES_CPU_P (cpu_features, FMA4))
 343             cpu_features->feature[index_arch_FMA4_Usable]
 344               |= bit_arch_FMA4_Usable;
 345         }
 346
 347       if (family == 0x15)
 348         {
 349 #if index_arch_Fast_Unaligned_Load != index_arch_Fast_Copy_Backward
 350 # error index_arch_Fast_Unaligned_Load != index_arch_Fast_Copy_Backward
 351 #endif
 352           /* "Excavator"   */
 353           if (model >= 0x60 && model <= 0x7f)
 354             cpu_features->feature[index_arch_Fast_Unaligned_Load]
 355               |= (bit_arch_Fast_Unaligned_Load
 356                   | bit_arch_Fast_Copy_Backward);
 357         }
 358     }
 359   else
 360     {
 361       kind = arch_kind_other;
 362       get_common_indeces (cpu_features, NULL, NULL, NULL, NULL);
 363     }
 364
 365   /* Support i586 if CX8 is available.  */
 366   if (CPU_FEATURES_CPU_P (cpu_features, CX8))
 367     cpu_features->feature[index_arch_I586] |= bit_arch_I586;
 368
 369   /* Support i686 if CMOV is available.  */
 370   if (CPU_FEATURES_CPU_P (cpu_features, CMOV))
 371     cpu_features->feature[index_arch_I686] |= bit_arch_I686;
 372
 373 #if !HAS_CPUID
 374 no_cpuid:
 375 #endif
 376
 377   cpu_features->family = family;
 378   cpu_features->model = model;
 379   cpu_features->kind = kind;
 380
 381 #if HAVE_TUNABLES
 382   TUNABLE_GET (hwcaps, tunable_val_t *, TUNABLE_CALLBACK (set_hwcaps));
 383   cpu_features->non_temporal_threshold
 384     = TUNABLE_GET (x86_non_temporal_threshold, long int, NULL);
 385   cpu_features->data_cache_size
 386     = TUNABLE_GET (x86_data_cache_size, long int, NULL);
 387   cpu_features->shared_cache_size
 388     = TUNABLE_GET (x86_shared_cache_size, long int, NULL);
 389 #endif
 390
 391   /* Reuse dl_platform, dl_hwcap and dl_hwcap_mask for x86.  */
 392 #if !HAVE_TUNABLES && defined SHARED
 393   /* The glibc.tune.hwcap_mask tunable is initialized already, so no need to do
 394      this.  */
 395   GLRO(dl_hwcap_mask) = HWCAP_IMPORTANT;
 396 #endif
 397
 398 #ifdef __x86_64__
 399   GLRO(dl_hwcap) = HWCAP_X86_64;
 400   if (cpu_features->kind == arch_kind_intel)
 401     {
 402       const char *platform = NULL;
 403
 404       if (CPU_FEATURES_ARCH_P (cpu_features, AVX512F_Usable)
 405           && CPU_FEATURES_CPU_P (cpu_features, AVX512CD))
 406         {
 407           if (CPU_FEATURES_CPU_P (cpu_features, AVX512ER))
 408             {
 409               if (CPU_FEATURES_CPU_P (cpu_features, AVX512PF))
 410                 platform = "xeon_phi";
 411             }
 412           else
 413             {
 414               if (CPU_FEATURES_CPU_P (cpu_features, AVX512BW)
 415                   && CPU_FEATURES_CPU_P (cpu_features, AVX512DQ)
 416                   && CPU_FEATURES_CPU_P (cpu_features, AVX512VL))
 417                 GLRO(dl_hwcap) |= HWCAP_X86_AVX512_1;
 418             }
 419         }
 420
 421       if (platform == NULL
 422           && CPU_FEATURES_ARCH_P (cpu_features, AVX2_Usable)
 423           && CPU_FEATURES_ARCH_P (cpu_features, FMA_Usable)
 424           && CPU_FEATURES_CPU_P (cpu_features, BMI1)
 425           && CPU_FEATURES_CPU_P (cpu_features, BMI2)
 426           && CPU_FEATURES_CPU_P (cpu_features, LZCNT)
 427           && CPU_FEATURES_CPU_P (cpu_features, MOVBE)
 428           && CPU_FEATURES_CPU_P (cpu_features, POPCNT))
 429         platform = "haswell";
 430
 431       if (platform != NULL)
 432         GLRO(dl_platform) = platform;
 433     }
 434 #else
 435   GLRO(dl_hwcap) = 0;
 436   if (CPU_FEATURES_CPU_P (cpu_features, SSE2))
 437     GLRO(dl_hwcap) |= HWCAP_X86_SSE2;
 438
 439   if (CPU_FEATURES_ARCH_P (cpu_features, I686))
 440     GLRO(dl_platform) = "i686";
 441   else if (CPU_FEATURES_ARCH_P (cpu_features, I586))
 442     GLRO(dl_platform) = "i586";
 443 #endif
 444 }