sysdeps/x86_64/multiarch/ifunc-impl-list.c

   1 /* Enumerate available IFUNC implementations of a function.  x86-64 version.
   2    Copyright (C) 2012-2021 Free Software Foundation, Inc.
   3    This file is part of the GNU C Library.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <https://www.gnu.org/licenses/>.  */
  18
  19 #include <assert.h>
  20 #include <string.h>
  21 #include <wchar.h>
  22 #include <ifunc-impl-list.h>
  23 #include <sysdep.h>
  24 #include "init-arch.h"
  25
  26 /* Maximum number of IFUNC implementations.  */
  27 #define MAX_IFUNC       5
  28
  29 /* Fill ARRAY of MAX elements with IFUNC implementations for function
  30    NAME supported on target machine and return the number of valid
  31    entries.  */
  32
  33 size_t
  34 __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
  35                         size_t max)
  36 {
  37   assert (max >= MAX_IFUNC);
  38
  39   size_t i = 0;
  40
  41   /* Support sysdeps/x86_64/multiarch/memchr.c.  */
  42   IFUNC_IMPL (i, name, memchr,
  43               IFUNC_IMPL_ADD (array, i, memchr,
  44                               CPU_FEATURE_USABLE (AVX2),
  45                               __memchr_avx2)
  46               IFUNC_IMPL_ADD (array, i, memchr,
  47                               (CPU_FEATURE_USABLE (AVX512VL)
  48                                && CPU_FEATURE_USABLE (AVX512BW)
  49                                && CPU_FEATURE_USABLE (BMI2)),
  50                               __memchr_evex)
  51               IFUNC_IMPL_ADD (array, i, memchr, 1, __memchr_sse2))
  52
  53   /* Support sysdeps/x86_64/multiarch/memcmp.c.  */
  54   IFUNC_IMPL (i, name, memcmp,
  55               IFUNC_IMPL_ADD (array, i, memcmp,
  56                               (CPU_FEATURE_USABLE (AVX2)
  57                                && CPU_FEATURE_USABLE (MOVBE)),
  58                               __memcmp_avx2_movbe)
  59               IFUNC_IMPL_ADD (array, i, memcmp, CPU_FEATURE_USABLE (SSE4_1),
  60                               __memcmp_sse4_1)
  61               IFUNC_IMPL_ADD (array, i, memcmp, CPU_FEATURE_USABLE (SSSE3),
  62                               __memcmp_ssse3)
  63               IFUNC_IMPL_ADD (array, i, memcmp, 1, __memcmp_sse2))
  64
  65 #ifdef SHARED
  66   /* Support sysdeps/x86_64/multiarch/memmove_chk.c.  */
  67   IFUNC_IMPL (i, name, __memmove_chk,
  68               IFUNC_IMPL_ADD (array, i, __memmove_chk,
  69                               CPU_FEATURE_USABLE (AVX512F),
  70                               __memmove_chk_avx512_no_vzeroupper)
  71               IFUNC_IMPL_ADD (array, i, __memmove_chk,
  72                               CPU_FEATURE_USABLE (AVX512F),
  73                               __memmove_chk_avx512_unaligned)
  74               IFUNC_IMPL_ADD (array, i, __memmove_chk,
  75                               CPU_FEATURE_USABLE (AVX512F),
  76                               __memmove_chk_avx512_unaligned_erms)
  77               IFUNC_IMPL_ADD (array, i, __memmove_chk,
  78                               CPU_FEATURE_USABLE (AVX),
  79                               __memmove_chk_avx_unaligned)
  80               IFUNC_IMPL_ADD (array, i, __memmove_chk,
  81                               CPU_FEATURE_USABLE (AVX),
  82                               __memmove_chk_avx_unaligned_erms)
  83               IFUNC_IMPL_ADD (array, i, __memmove_chk,
  84                               CPU_FEATURE_USABLE (AVX512VL),
  85                               __memmove_chk_evex_unaligned)
  86               IFUNC_IMPL_ADD (array, i, __memmove_chk,
  87                               CPU_FEATURE_USABLE (AVX512VL),
  88                               __memmove_chk_evex_unaligned_erms)
  89               IFUNC_IMPL_ADD (array, i, __memmove_chk,
  90                               CPU_FEATURE_USABLE (SSSE3),
  91                               __memmove_chk_ssse3_back)
  92               IFUNC_IMPL_ADD (array, i, __memmove_chk,
  93                               CPU_FEATURE_USABLE (SSSE3),
  94                               __memmove_chk_ssse3)
  95               IFUNC_IMPL_ADD (array, i, __memmove_chk, 1,
  96                               __memmove_chk_sse2_unaligned)
  97               IFUNC_IMPL_ADD (array, i, __memmove_chk, 1,
  98                               __memmove_chk_sse2_unaligned_erms)
  99               IFUNC_IMPL_ADD (array, i, __memmove_chk, 1,
 100                               __memmove_chk_erms))
 101 #endif
 102
 103   /* Support sysdeps/x86_64/multiarch/memmove.c.  */
 104   IFUNC_IMPL (i, name, memmove,
 105               IFUNC_IMPL_ADD (array, i, memmove,
 106                               CPU_FEATURE_USABLE (AVX),
 107                               __memmove_avx_unaligned)
 108               IFUNC_IMPL_ADD (array, i, memmove,
 109                               CPU_FEATURE_USABLE (AVX),
 110                               __memmove_avx_unaligned_erms)
 111               IFUNC_IMPL_ADD (array, i, memmove,
 112                               CPU_FEATURE_USABLE (AVX512VL),
 113                               __memmove_evex_unaligned)
 114               IFUNC_IMPL_ADD (array, i, memmove,
 115                               CPU_FEATURE_USABLE (AVX512VL),
 116                               __memmove_evex_unaligned_erms)
 117               IFUNC_IMPL_ADD (array, i, memmove,
 118                               CPU_FEATURE_USABLE (AVX512F),
 119                               __memmove_avx512_no_vzeroupper)
 120               IFUNC_IMPL_ADD (array, i, memmove,
 121                               CPU_FEATURE_USABLE (AVX512F),
 122                               __memmove_avx512_unaligned)
 123               IFUNC_IMPL_ADD (array, i, memmove,
 124                               CPU_FEATURE_USABLE (AVX512F),
 125                               __memmove_avx512_unaligned_erms)
 126               IFUNC_IMPL_ADD (array, i, memmove, CPU_FEATURE_USABLE (SSSE3),
 127                               __memmove_ssse3_back)
 128               IFUNC_IMPL_ADD (array, i, memmove, CPU_FEATURE_USABLE (SSSE3),
 129                               __memmove_ssse3)
 130               IFUNC_IMPL_ADD (array, i, memmove, 1, __memmove_erms)
 131               IFUNC_IMPL_ADD (array, i, memmove, 1,
 132                               __memmove_sse2_unaligned)
 133               IFUNC_IMPL_ADD (array, i, memmove, 1,
 134                               __memmove_sse2_unaligned_erms))
 135
 136   /* Support sysdeps/x86_64/multiarch/memrchr.c.  */
 137   IFUNC_IMPL (i, name, memrchr,
 138               IFUNC_IMPL_ADD (array, i, memrchr,
 139                               CPU_FEATURE_USABLE (AVX2),
 140                               __memrchr_avx2)
 141               IFUNC_IMPL_ADD (array, i, memrchr,
 142                               (CPU_FEATURE_USABLE (AVX512VL)
 143                                && CPU_FEATURE_USABLE (AVX512BW)),
 144                               __memrchr_evex)
 145
 146               IFUNC_IMPL_ADD (array, i, memrchr, 1, __memrchr_sse2))
 147
 148 #ifdef SHARED
 149   /* Support sysdeps/x86_64/multiarch/memset_chk.c.  */
 150   IFUNC_IMPL (i, name, __memset_chk,
 151               IFUNC_IMPL_ADD (array, i, __memset_chk, 1,
 152                               __memset_chk_erms)
 153               IFUNC_IMPL_ADD (array, i, __memset_chk, 1,
 154                               __memset_chk_sse2_unaligned)
 155               IFUNC_IMPL_ADD (array, i, __memset_chk, 1,
 156                               __memset_chk_sse2_unaligned_erms)
 157               IFUNC_IMPL_ADD (array, i, __memset_chk,
 158                               CPU_FEATURE_USABLE (AVX2),
 159                               __memset_chk_avx2_unaligned)
 160               IFUNC_IMPL_ADD (array, i, __memset_chk,
 161                               CPU_FEATURE_USABLE (AVX2),
 162                               __memset_chk_avx2_unaligned_erms)
 163               IFUNC_IMPL_ADD (array, i, __memset_chk,
 164                               (CPU_FEATURE_USABLE (AVX512VL)
 165                                && CPU_FEATURE_USABLE (AVX512BW)),
 166                               __memset_chk_evex_unaligned)
 167               IFUNC_IMPL_ADD (array, i, __memset_chk,
 168                               (CPU_FEATURE_USABLE (AVX512VL)
 169                                && CPU_FEATURE_USABLE (AVX512BW)),
 170                               __memset_chk_evex_unaligned_erms)
 171               IFUNC_IMPL_ADD (array, i, __memset_chk,
 172                               CPU_FEATURE_USABLE (AVX512F),
 173                               __memset_chk_avx512_unaligned_erms)
 174               IFUNC_IMPL_ADD (array, i, __memset_chk,
 175                               CPU_FEATURE_USABLE (AVX512F),
 176                               __memset_chk_avx512_unaligned)
 177               IFUNC_IMPL_ADD (array, i, __memset_chk,
 178                               CPU_FEATURE_USABLE (AVX512F),
 179                               __memset_chk_avx512_no_vzeroupper)
 180               )
 181 #endif
 182
 183   /* Support sysdeps/x86_64/multiarch/memset.c.  */
 184   IFUNC_IMPL (i, name, memset,
 185               IFUNC_IMPL_ADD (array, i, memset, 1,
 186                               __memset_sse2_unaligned)
 187               IFUNC_IMPL_ADD (array, i, memset, 1,
 188                               __memset_sse2_unaligned_erms)
 189               IFUNC_IMPL_ADD (array, i, memset, 1, __memset_erms)
 190               IFUNC_IMPL_ADD (array, i, memset,
 191                               CPU_FEATURE_USABLE (AVX2),
 192                               __memset_avx2_unaligned)
 193               IFUNC_IMPL_ADD (array, i, memset,
 194                               CPU_FEATURE_USABLE (AVX2),
 195                               __memset_avx2_unaligned_erms)
 196               IFUNC_IMPL_ADD (array, i, memset,
 197                               (CPU_FEATURE_USABLE (AVX512VL)
 198                                && CPU_FEATURE_USABLE (AVX512BW)),
 199                               __memset_evex_unaligned)
 200               IFUNC_IMPL_ADD (array, i, memset,
 201                               (CPU_FEATURE_USABLE (AVX512VL)
 202                                && CPU_FEATURE_USABLE (AVX512BW)),
 203                               __memset_evex_unaligned_erms)
 204               IFUNC_IMPL_ADD (array, i, memset,
 205                               CPU_FEATURE_USABLE (AVX512F),
 206                               __memset_avx512_unaligned_erms)
 207               IFUNC_IMPL_ADD (array, i, memset,
 208                               CPU_FEATURE_USABLE (AVX512F),
 209                               __memset_avx512_unaligned)
 210               IFUNC_IMPL_ADD (array, i, memset,
 211                               CPU_FEATURE_USABLE (AVX512F),
 212                               __memset_avx512_no_vzeroupper)
 213              )
 214
 215   /* Support sysdeps/x86_64/multiarch/rawmemchr.c.  */
 216   IFUNC_IMPL (i, name, rawmemchr,
 217               IFUNC_IMPL_ADD (array, i, rawmemchr,
 218                               CPU_FEATURE_USABLE (AVX2),
 219                               __rawmemchr_avx2)
 220               IFUNC_IMPL_ADD (array, i, rawmemchr,
 221                               (CPU_FEATURE_USABLE (AVX512VL)
 222                                && CPU_FEATURE_USABLE (AVX512BW)
 223                                && CPU_FEATURE_USABLE (BMI2)),
 224                               __rawmemchr_evex)
 225               IFUNC_IMPL_ADD (array, i, rawmemchr, 1, __rawmemchr_sse2))
 226
 227   /* Support sysdeps/x86_64/multiarch/strlen.c.  */
 228   IFUNC_IMPL (i, name, strlen,
 229               IFUNC_IMPL_ADD (array, i, strlen,
 230                               CPU_FEATURE_USABLE (AVX2),
 231                               __strlen_avx2)
 232               IFUNC_IMPL_ADD (array, i, strlen,
 233                               (CPU_FEATURE_USABLE (AVX512VL)
 234                                && CPU_FEATURE_USABLE (AVX512BW)),
 235                               __strlen_evex)
 236               IFUNC_IMPL_ADD (array, i, strlen, 1, __strlen_sse2))
 237
 238   /* Support sysdeps/x86_64/multiarch/strnlen.c.  */
 239   IFUNC_IMPL (i, name, strnlen,
 240               IFUNC_IMPL_ADD (array, i, strnlen,
 241                               CPU_FEATURE_USABLE (AVX2),
 242                               __strnlen_avx2)
 243               IFUNC_IMPL_ADD (array, i, strnlen,
 244                               (CPU_FEATURE_USABLE (AVX512VL)
 245                                && CPU_FEATURE_USABLE (AVX512BW)),
 246                               __strnlen_evex)
 247               IFUNC_IMPL_ADD (array, i, strnlen, 1, __strnlen_sse2))
 248
 249   /* Support sysdeps/x86_64/multiarch/stpncpy.c.  */
 250   IFUNC_IMPL (i, name, stpncpy,
 251               IFUNC_IMPL_ADD (array, i, stpncpy, CPU_FEATURE_USABLE (SSSE3),
 252                               __stpncpy_ssse3)
 253               IFUNC_IMPL_ADD (array, i, stpncpy, CPU_FEATURE_USABLE (AVX2),
 254                               __stpncpy_avx2)
 255               IFUNC_IMPL_ADD (array, i, stpncpy,
 256                               (CPU_FEATURE_USABLE (AVX512VL)
 257                                && CPU_FEATURE_USABLE (AVX512BW)),
 258                               __stpncpy_evex)
 259               IFUNC_IMPL_ADD (array, i, stpncpy, 1,
 260                               __stpncpy_sse2_unaligned)
 261               IFUNC_IMPL_ADD (array, i, stpncpy, 1, __stpncpy_sse2))
 262
 263   /* Support sysdeps/x86_64/multiarch/stpcpy.c.  */
 264   IFUNC_IMPL (i, name, stpcpy,
 265               IFUNC_IMPL_ADD (array, i, stpcpy, CPU_FEATURE_USABLE (SSSE3),
 266                               __stpcpy_ssse3)
 267               IFUNC_IMPL_ADD (array, i, stpcpy, CPU_FEATURE_USABLE (AVX2),
 268                               __stpcpy_avx2)
 269               IFUNC_IMPL_ADD (array, i, stpcpy,
 270                               (CPU_FEATURE_USABLE (AVX512VL)
 271                                && CPU_FEATURE_USABLE (AVX512BW)),
 272                               __stpcpy_evex)
 273               IFUNC_IMPL_ADD (array, i, stpcpy, 1, __stpcpy_sse2_unaligned)
 274               IFUNC_IMPL_ADD (array, i, stpcpy, 1, __stpcpy_sse2))
 275
 276   /* Support sysdeps/x86_64/multiarch/strcasecmp_l.c.  */
 277   IFUNC_IMPL (i, name, strcasecmp,
 278               IFUNC_IMPL_ADD (array, i, strcasecmp,
 279                               CPU_FEATURE_USABLE (AVX),
 280                               __strcasecmp_avx)
 281               IFUNC_IMPL_ADD (array, i, strcasecmp,
 282                               CPU_FEATURE_USABLE (SSE4_2),
 283                               __strcasecmp_sse42)
 284               IFUNC_IMPL_ADD (array, i, strcasecmp,
 285                               CPU_FEATURE_USABLE (SSSE3),
 286                               __strcasecmp_ssse3)
 287               IFUNC_IMPL_ADD (array, i, strcasecmp, 1, __strcasecmp_sse2))
 288
 289   /* Support sysdeps/x86_64/multiarch/strcasecmp_l.c.  */
 290   IFUNC_IMPL (i, name, strcasecmp_l,
 291               IFUNC_IMPL_ADD (array, i, strcasecmp_l,
 292                               CPU_FEATURE_USABLE (AVX),
 293                               __strcasecmp_l_avx)
 294               IFUNC_IMPL_ADD (array, i, strcasecmp_l,
 295                               CPU_FEATURE_USABLE (SSE4_2),
 296                               __strcasecmp_l_sse42)
 297               IFUNC_IMPL_ADD (array, i, strcasecmp_l,
 298                               CPU_FEATURE_USABLE (SSSE3),
 299                               __strcasecmp_l_ssse3)
 300               IFUNC_IMPL_ADD (array, i, strcasecmp_l, 1,
 301                               __strcasecmp_l_sse2))
 302
 303   /* Support sysdeps/x86_64/multiarch/strcat.c.  */
 304   IFUNC_IMPL (i, name, strcat,
 305               IFUNC_IMPL_ADD (array, i, strcat, CPU_FEATURE_USABLE (AVX2),
 306                               __strcat_avx2)
 307               IFUNC_IMPL_ADD (array, i, strcat,
 308                               (CPU_FEATURE_USABLE (AVX512VL)
 309                                && CPU_FEATURE_USABLE (AVX512BW)),
 310                               __strcat_evex)
 311               IFUNC_IMPL_ADD (array, i, strcat, CPU_FEATURE_USABLE (SSSE3),
 312                               __strcat_ssse3)
 313               IFUNC_IMPL_ADD (array, i, strcat, 1, __strcat_sse2_unaligned)
 314               IFUNC_IMPL_ADD (array, i, strcat, 1, __strcat_sse2))
 315
 316   /* Support sysdeps/x86_64/multiarch/strchr.c.  */
 317   IFUNC_IMPL (i, name, strchr,
 318               IFUNC_IMPL_ADD (array, i, strchr,
 319                               CPU_FEATURE_USABLE (AVX2),
 320                               __strchr_avx2)
 321               IFUNC_IMPL_ADD (array, i, strchr,
 322                               (CPU_FEATURE_USABLE (AVX512VL)
 323                                && CPU_FEATURE_USABLE (AVX512BW)
 324                                && CPU_FEATURE_USABLE (BMI2)),
 325                               __strchr_evex)
 326               IFUNC_IMPL_ADD (array, i, strchr, 1, __strchr_sse2_no_bsf)
 327               IFUNC_IMPL_ADD (array, i, strchr, 1, __strchr_sse2))
 328
 329   /* Support sysdeps/x86_64/multiarch/strchrnul.c.  */
 330   IFUNC_IMPL (i, name, strchrnul,
 331               IFUNC_IMPL_ADD (array, i, strchrnul,
 332                               CPU_FEATURE_USABLE (AVX2),
 333                               __strchrnul_avx2)
 334               IFUNC_IMPL_ADD (array, i, strchrnul,
 335                               (CPU_FEATURE_USABLE (AVX512VL)
 336                                && CPU_FEATURE_USABLE (AVX512BW)
 337                                && CPU_FEATURE_USABLE (BMI2)),
 338                               __strchrnul_evex)
 339               IFUNC_IMPL_ADD (array, i, strchrnul, 1, __strchrnul_sse2))
 340
 341   /* Support sysdeps/x86_64/multiarch/strrchr.c.  */
 342   IFUNC_IMPL (i, name, strrchr,
 343               IFUNC_IMPL_ADD (array, i, strrchr,
 344                               CPU_FEATURE_USABLE (AVX2),
 345                               __strrchr_avx2)
 346               IFUNC_IMPL_ADD (array, i, strrchr,
 347                               (CPU_FEATURE_USABLE (AVX512VL)
 348                                && CPU_FEATURE_USABLE (AVX512BW)),
 349                               __strrchr_evex)
 350               IFUNC_IMPL_ADD (array, i, strrchr, 1, __strrchr_sse2))
 351
 352   /* Support sysdeps/x86_64/multiarch/strcmp.c.  */
 353   IFUNC_IMPL (i, name, strcmp,
 354               IFUNC_IMPL_ADD (array, i, strcmp,
 355                               CPU_FEATURE_USABLE (AVX2),
 356                               __strcmp_avx2)
 357               IFUNC_IMPL_ADD (array, i, strcmp,
 358                               (CPU_FEATURE_USABLE (AVX512VL)
 359                                && CPU_FEATURE_USABLE (AVX512BW)
 360                                && CPU_FEATURE_USABLE (BMI2)),
 361                               __strcmp_evex)
 362               IFUNC_IMPL_ADD (array, i, strcmp, CPU_FEATURE_USABLE (SSE4_2),
 363                               __strcmp_sse42)
 364               IFUNC_IMPL_ADD (array, i, strcmp, CPU_FEATURE_USABLE (SSSE3),
 365                               __strcmp_ssse3)
 366               IFUNC_IMPL_ADD (array, i, strcmp, 1, __strcmp_sse2_unaligned)
 367               IFUNC_IMPL_ADD (array, i, strcmp, 1, __strcmp_sse2))
 368
 369   /* Support sysdeps/x86_64/multiarch/strcpy.c.  */
 370   IFUNC_IMPL (i, name, strcpy,
 371               IFUNC_IMPL_ADD (array, i, strcpy, CPU_FEATURE_USABLE (AVX2),
 372                               __strcpy_avx2)
 373               IFUNC_IMPL_ADD (array, i, strcpy,
 374                               (CPU_FEATURE_USABLE (AVX512VL)
 375                                && CPU_FEATURE_USABLE (AVX512BW)),
 376                               __strcpy_evex)
 377               IFUNC_IMPL_ADD (array, i, strcpy, CPU_FEATURE_USABLE (SSSE3),
 378                               __strcpy_ssse3)
 379               IFUNC_IMPL_ADD (array, i, strcpy, 1, __strcpy_sse2_unaligned)
 380               IFUNC_IMPL_ADD (array, i, strcpy, 1, __strcpy_sse2))
 381
 382   /* Support sysdeps/x86_64/multiarch/strcspn.c.  */
 383   IFUNC_IMPL (i, name, strcspn,
 384               IFUNC_IMPL_ADD (array, i, strcspn, CPU_FEATURE_USABLE (SSE4_2),
 385                               __strcspn_sse42)
 386               IFUNC_IMPL_ADD (array, i, strcspn, 1, __strcspn_sse2))
 387
 388   /* Support sysdeps/x86_64/multiarch/strncase_l.c.  */
 389   IFUNC_IMPL (i, name, strncasecmp,
 390               IFUNC_IMPL_ADD (array, i, strncasecmp,
 391                               CPU_FEATURE_USABLE (AVX),
 392                               __strncasecmp_avx)
 393               IFUNC_IMPL_ADD (array, i, strncasecmp,
 394                               CPU_FEATURE_USABLE (SSE4_2),
 395                               __strncasecmp_sse42)
 396               IFUNC_IMPL_ADD (array, i, strncasecmp,
 397                               CPU_FEATURE_USABLE (SSSE3),
 398                               __strncasecmp_ssse3)
 399               IFUNC_IMPL_ADD (array, i, strncasecmp, 1,
 400                               __strncasecmp_sse2))
 401
 402   /* Support sysdeps/x86_64/multiarch/strncase_l.c.  */
 403   IFUNC_IMPL (i, name, strncasecmp_l,
 404               IFUNC_IMPL_ADD (array, i, strncasecmp_l,
 405                               CPU_FEATURE_USABLE (AVX),
 406                               __strncasecmp_l_avx)
 407               IFUNC_IMPL_ADD (array, i, strncasecmp_l,
 408                               CPU_FEATURE_USABLE (SSE4_2),
 409                               __strncasecmp_l_sse42)
 410               IFUNC_IMPL_ADD (array, i, strncasecmp_l,
 411                               CPU_FEATURE_USABLE (SSSE3),
 412                               __strncasecmp_l_ssse3)
 413               IFUNC_IMPL_ADD (array, i, strncasecmp_l, 1,
 414                               __strncasecmp_l_sse2))
 415
 416   /* Support sysdeps/x86_64/multiarch/strncat.c.  */
 417   IFUNC_IMPL (i, name, strncat,
 418               IFUNC_IMPL_ADD (array, i, strncat, CPU_FEATURE_USABLE (AVX2),
 419                               __strncat_avx2)
 420               IFUNC_IMPL_ADD (array, i, strncat,
 421                               (CPU_FEATURE_USABLE (AVX512VL)
 422                                && CPU_FEATURE_USABLE (AVX512BW)),
 423                               __strncat_evex)
 424               IFUNC_IMPL_ADD (array, i, strncat, CPU_FEATURE_USABLE (SSSE3),
 425                               __strncat_ssse3)
 426               IFUNC_IMPL_ADD (array, i, strncat, 1,
 427                               __strncat_sse2_unaligned)
 428               IFUNC_IMPL_ADD (array, i, strncat, 1, __strncat_sse2))
 429
 430   /* Support sysdeps/x86_64/multiarch/strncpy.c.  */
 431   IFUNC_IMPL (i, name, strncpy,
 432               IFUNC_IMPL_ADD (array, i, strncpy, CPU_FEATURE_USABLE (AVX2),
 433                               __strncpy_avx2)
 434               IFUNC_IMPL_ADD (array, i, strncpy,
 435                               (CPU_FEATURE_USABLE (AVX512VL)
 436                                && CPU_FEATURE_USABLE (AVX512BW)),
 437                               __strncpy_evex)
 438               IFUNC_IMPL_ADD (array, i, strncpy, CPU_FEATURE_USABLE (SSSE3),
 439                               __strncpy_ssse3)
 440               IFUNC_IMPL_ADD (array, i, strncpy, 1,
 441                               __strncpy_sse2_unaligned)
 442               IFUNC_IMPL_ADD (array, i, strncpy, 1, __strncpy_sse2))
 443
 444   /* Support sysdeps/x86_64/multiarch/strpbrk.c.  */
 445   IFUNC_IMPL (i, name, strpbrk,
 446               IFUNC_IMPL_ADD (array, i, strpbrk, CPU_FEATURE_USABLE (SSE4_2),
 447                               __strpbrk_sse42)
 448               IFUNC_IMPL_ADD (array, i, strpbrk, 1, __strpbrk_sse2))
 449
 450
 451   /* Support sysdeps/x86_64/multiarch/strspn.c.  */
 452   IFUNC_IMPL (i, name, strspn,
 453               IFUNC_IMPL_ADD (array, i, strspn, CPU_FEATURE_USABLE (SSE4_2),
 454                               __strspn_sse42)
 455               IFUNC_IMPL_ADD (array, i, strspn, 1, __strspn_sse2))
 456
 457   /* Support sysdeps/x86_64/multiarch/strstr.c.  */
 458   IFUNC_IMPL (i, name, strstr,
 459               IFUNC_IMPL_ADD (array, i, strstr, 1, __strstr_sse2_unaligned)
 460               IFUNC_IMPL_ADD (array, i, strstr, 1, __strstr_sse2))
 461
 462   /* Support sysdeps/x86_64/multiarch/wcschr.c.  */
 463   IFUNC_IMPL (i, name, wcschr,
 464               IFUNC_IMPL_ADD (array, i, wcschr,
 465                               CPU_FEATURE_USABLE (AVX2),
 466                               __wcschr_avx2)
 467               IFUNC_IMPL_ADD (array, i, wcschr,
 468                               (CPU_FEATURE_USABLE (AVX512VL)
 469                                && CPU_FEATURE_USABLE (AVX512BW)
 470                                && CPU_FEATURE_USABLE (BMI2)),
 471                               __wcschr_evex)
 472               IFUNC_IMPL_ADD (array, i, wcschr, 1, __wcschr_sse2))
 473
 474   /* Support sysdeps/x86_64/multiarch/wcsrchr.c.  */
 475   IFUNC_IMPL (i, name, wcsrchr,
 476               IFUNC_IMPL_ADD (array, i, wcsrchr,
 477                               CPU_FEATURE_USABLE (AVX2),
 478                               __wcsrchr_avx2)
 479               IFUNC_IMPL_ADD (array, i, wcsrchr,
 480                               (CPU_FEATURE_USABLE (AVX512VL)
 481                                && CPU_FEATURE_USABLE (AVX512BW)
 482                                && CPU_FEATURE_USABLE (BMI2)),
 483                               __wcsrchr_evex)
 484               IFUNC_IMPL_ADD (array, i, wcsrchr, 1, __wcsrchr_sse2))
 485
 486   /* Support sysdeps/x86_64/multiarch/wcscmp.c.  */
 487   IFUNC_IMPL (i, name, wcscmp,
 488               IFUNC_IMPL_ADD (array, i, wcscmp,
 489                               CPU_FEATURE_USABLE (AVX2),
 490                               __wcscmp_avx2)
 491               IFUNC_IMPL_ADD (array, i, wcscmp,
 492                               (CPU_FEATURE_USABLE (AVX512VL)
 493                                && CPU_FEATURE_USABLE (AVX512BW)
 494                                && CPU_FEATURE_USABLE (BMI2)),
 495                               __wcscmp_evex)
 496               IFUNC_IMPL_ADD (array, i, wcscmp, 1, __wcscmp_sse2))
 497
 498   /* Support sysdeps/x86_64/multiarch/wcsncmp.c.  */
 499   IFUNC_IMPL (i, name, wcsncmp,
 500               IFUNC_IMPL_ADD (array, i, wcsncmp,
 501                               CPU_FEATURE_USABLE (AVX2),
 502                               __wcsncmp_avx2)
 503               IFUNC_IMPL_ADD (array, i, wcsncmp,
 504                               (CPU_FEATURE_USABLE (AVX512VL)
 505                                && CPU_FEATURE_USABLE (AVX512BW)
 506                                && CPU_FEATURE_USABLE (BMI2)),
 507                               __wcsncmp_evex)
 508               IFUNC_IMPL_ADD (array, i, wcsncmp, 1, __wcsncmp_sse2))
 509
 510   /* Support sysdeps/x86_64/multiarch/wcscpy.c.  */
 511   IFUNC_IMPL (i, name, wcscpy,
 512               IFUNC_IMPL_ADD (array, i, wcscpy, CPU_FEATURE_USABLE (SSSE3),
 513                               __wcscpy_ssse3)
 514               IFUNC_IMPL_ADD (array, i, wcscpy, 1, __wcscpy_sse2))
 515
 516   /* Support sysdeps/x86_64/multiarch/wcslen.c.  */
 517   IFUNC_IMPL (i, name, wcslen,
 518               IFUNC_IMPL_ADD (array, i, wcslen,
 519                               CPU_FEATURE_USABLE (AVX2),
 520                               __wcslen_avx2)
 521               IFUNC_IMPL_ADD (array, i, wcslen,
 522                               (CPU_FEATURE_USABLE (AVX512VL)
 523                                && CPU_FEATURE_USABLE (AVX512BW)
 524                                && CPU_FEATURE_USABLE (BMI2)),
 525                               __wcslen_evex)
 526               IFUNC_IMPL_ADD (array, i, wcslen, 1, __wcslen_sse2))
 527
 528   /* Support sysdeps/x86_64/multiarch/wcsnlen.c.  */
 529   IFUNC_IMPL (i, name, wcsnlen,
 530               IFUNC_IMPL_ADD (array, i, wcsnlen,
 531                               CPU_FEATURE_USABLE (AVX2),
 532                               __wcsnlen_avx2)
 533               IFUNC_IMPL_ADD (array, i, wcsnlen,
 534                               (CPU_FEATURE_USABLE (AVX512VL)
 535                                && CPU_FEATURE_USABLE (AVX512BW)
 536                                && CPU_FEATURE_USABLE (BMI2)),
 537                               __wcsnlen_evex)
 538               IFUNC_IMPL_ADD (array, i, wcsnlen,
 539                               CPU_FEATURE_USABLE (SSE4_1),
 540                               __wcsnlen_sse4_1)
 541               IFUNC_IMPL_ADD (array, i, wcsnlen, 1, __wcsnlen_sse2))
 542
 543   /* Support sysdeps/x86_64/multiarch/wmemchr.c.  */
 544   IFUNC_IMPL (i, name, wmemchr,
 545               IFUNC_IMPL_ADD (array, i, wmemchr,
 546                               CPU_FEATURE_USABLE (AVX2),
 547                               __wmemchr_avx2)
 548               IFUNC_IMPL_ADD (array, i, wmemchr,
 549                               (CPU_FEATURE_USABLE (AVX512VL)
 550                                && CPU_FEATURE_USABLE (AVX512BW)
 551                                && CPU_FEATURE_USABLE (BMI2)),
 552                               __wmemchr_evex)
 553               IFUNC_IMPL_ADD (array, i, wmemchr, 1, __wmemchr_sse2))
 554
 555   /* Support sysdeps/x86_64/multiarch/wmemcmp.c.  */
 556   IFUNC_IMPL (i, name, wmemcmp,
 557               IFUNC_IMPL_ADD (array, i, wmemcmp,
 558                               (CPU_FEATURE_USABLE (AVX2)
 559                                && CPU_FEATURE_USABLE (MOVBE)),
 560                               __wmemcmp_avx2_movbe)
 561               IFUNC_IMPL_ADD (array, i, wmemcmp, CPU_FEATURE_USABLE (SSE4_1),
 562                               __wmemcmp_sse4_1)
 563               IFUNC_IMPL_ADD (array, i, wmemcmp, CPU_FEATURE_USABLE (SSSE3),
 564                               __wmemcmp_ssse3)
 565               IFUNC_IMPL_ADD (array, i, wmemcmp, 1, __wmemcmp_sse2))
 566
 567   /* Support sysdeps/x86_64/multiarch/wmemset.c.  */
 568   IFUNC_IMPL (i, name, wmemset,
 569               IFUNC_IMPL_ADD (array, i, wmemset, 1,
 570                               __wmemset_sse2_unaligned)
 571               IFUNC_IMPL_ADD (array, i, wmemset,
 572                               CPU_FEATURE_USABLE (AVX2),
 573                               __wmemset_avx2_unaligned)
 574               IFUNC_IMPL_ADD (array, i, wmemset,
 575                               CPU_FEATURE_USABLE (AVX512VL),
 576                               __wmemset_evex_unaligned)
 577               IFUNC_IMPL_ADD (array, i, wmemset,
 578                               CPU_FEATURE_USABLE (AVX512F),
 579                               __wmemset_avx512_unaligned))
 580
 581 #ifdef SHARED
 582   /* Support sysdeps/x86_64/multiarch/memcpy_chk.c.  */
 583   IFUNC_IMPL (i, name, __memcpy_chk,
 584               IFUNC_IMPL_ADD (array, i, __memcpy_chk,
 585                               CPU_FEATURE_USABLE (AVX512F),
 586                               __memcpy_chk_avx512_no_vzeroupper)
 587               IFUNC_IMPL_ADD (array, i, __memcpy_chk,
 588                               CPU_FEATURE_USABLE (AVX512F),
 589                               __memcpy_chk_avx512_unaligned)
 590               IFUNC_IMPL_ADD (array, i, __memcpy_chk,
 591                               CPU_FEATURE_USABLE (AVX512F),
 592                               __memcpy_chk_avx512_unaligned_erms)
 593               IFUNC_IMPL_ADD (array, i, __memcpy_chk,
 594                               CPU_FEATURE_USABLE (AVX),
 595                               __memcpy_chk_avx_unaligned)
 596               IFUNC_IMPL_ADD (array, i, __memcpy_chk,
 597                               CPU_FEATURE_USABLE (AVX),
 598                               __memcpy_chk_avx_unaligned_erms)
 599               IFUNC_IMPL_ADD (array, i, __memcpy_chk,
 600                               CPU_FEATURE_USABLE (AVX512VL),
 601                               __memcpy_chk_evex_unaligned)
 602               IFUNC_IMPL_ADD (array, i, __memcpy_chk,
 603                               CPU_FEATURE_USABLE (AVX512VL),
 604                               __memcpy_chk_evex_unaligned_erms)
 605               IFUNC_IMPL_ADD (array, i, __memcpy_chk,
 606                               CPU_FEATURE_USABLE (SSSE3),
 607                               __memcpy_chk_ssse3_back)
 608               IFUNC_IMPL_ADD (array, i, __memcpy_chk,
 609                               CPU_FEATURE_USABLE (SSSE3),
 610                               __memcpy_chk_ssse3)
 611               IFUNC_IMPL_ADD (array, i, __memcpy_chk, 1,
 612                               __memcpy_chk_sse2_unaligned)
 613               IFUNC_IMPL_ADD (array, i, __memcpy_chk, 1,
 614                               __memcpy_chk_sse2_unaligned_erms)
 615               IFUNC_IMPL_ADD (array, i, __memcpy_chk, 1,
 616                               __memcpy_chk_erms))
 617 #endif
 618
 619   /* Support sysdeps/x86_64/multiarch/memcpy.c.  */
 620   IFUNC_IMPL (i, name, memcpy,
 621               IFUNC_IMPL_ADD (array, i, memcpy,
 622                               CPU_FEATURE_USABLE (AVX),
 623                               __memcpy_avx_unaligned)
 624               IFUNC_IMPL_ADD (array, i, memcpy,
 625                               CPU_FEATURE_USABLE (AVX),
 626                               __memcpy_avx_unaligned_erms)
 627               IFUNC_IMPL_ADD (array, i, memcpy,
 628                               CPU_FEATURE_USABLE (AVX512VL),
 629                               __memcpy_evex_unaligned)
 630               IFUNC_IMPL_ADD (array, i, memcpy,
 631                               CPU_FEATURE_USABLE (AVX512VL),
 632                               __memcpy_evex_unaligned_erms)
 633               IFUNC_IMPL_ADD (array, i, memcpy, CPU_FEATURE_USABLE (SSSE3),
 634                               __memcpy_ssse3_back)
 635               IFUNC_IMPL_ADD (array, i, memcpy, CPU_FEATURE_USABLE (SSSE3),
 636                               __memcpy_ssse3)
 637               IFUNC_IMPL_ADD (array, i, memcpy,
 638                               CPU_FEATURE_USABLE (AVX512F),
 639                               __memcpy_avx512_no_vzeroupper)
 640               IFUNC_IMPL_ADD (array, i, memcpy,
 641                               CPU_FEATURE_USABLE (AVX512F),
 642                               __memcpy_avx512_unaligned)
 643               IFUNC_IMPL_ADD (array, i, memcpy,
 644                               CPU_FEATURE_USABLE (AVX512F),
 645                               __memcpy_avx512_unaligned_erms)
 646               IFUNC_IMPL_ADD (array, i, memcpy, 1, __memcpy_sse2_unaligned)
 647               IFUNC_IMPL_ADD (array, i, memcpy, 1,
 648                               __memcpy_sse2_unaligned_erms)
 649               IFUNC_IMPL_ADD (array, i, memcpy, 1, __memcpy_erms))
 650
 651 #ifdef SHARED
 652   /* Support sysdeps/x86_64/multiarch/mempcpy_chk.c.  */
 653   IFUNC_IMPL (i, name, __mempcpy_chk,
 654               IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
 655                               CPU_FEATURE_USABLE (AVX512F),
 656                               __mempcpy_chk_avx512_no_vzeroupper)
 657               IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
 658                               CPU_FEATURE_USABLE (AVX512F),
 659                               __mempcpy_chk_avx512_unaligned)
 660               IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
 661                               CPU_FEATURE_USABLE (AVX512F),
 662                               __mempcpy_chk_avx512_unaligned_erms)
 663               IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
 664                               CPU_FEATURE_USABLE (AVX),
 665                               __mempcpy_chk_avx_unaligned)
 666               IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
 667                               CPU_FEATURE_USABLE (AVX),
 668                               __mempcpy_chk_avx_unaligned_erms)
 669               IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
 670                               CPU_FEATURE_USABLE (AVX512VL),
 671                               __mempcpy_chk_evex_unaligned)
 672               IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
 673                               CPU_FEATURE_USABLE (AVX512VL),
 674                               __mempcpy_chk_evex_unaligned_erms)
 675               IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
 676                               CPU_FEATURE_USABLE (SSSE3),
 677                               __mempcpy_chk_ssse3_back)
 678               IFUNC_IMPL_ADD (array, i, __mempcpy_chk,
 679                               CPU_FEATURE_USABLE (SSSE3),
 680                               __mempcpy_chk_ssse3)
 681               IFUNC_IMPL_ADD (array, i, __mempcpy_chk, 1,
 682                               __mempcpy_chk_sse2_unaligned)
 683               IFUNC_IMPL_ADD (array, i, __mempcpy_chk, 1,
 684                               __mempcpy_chk_sse2_unaligned_erms)
 685               IFUNC_IMPL_ADD (array, i, __mempcpy_chk, 1,
 686                               __mempcpy_chk_erms))
 687 #endif
 688
 689   /* Support sysdeps/x86_64/multiarch/mempcpy.c.  */
 690   IFUNC_IMPL (i, name, mempcpy,
 691               IFUNC_IMPL_ADD (array, i, mempcpy,
 692                               CPU_FEATURE_USABLE (AVX512F),
 693                               __mempcpy_avx512_no_vzeroupper)
 694               IFUNC_IMPL_ADD (array, i, mempcpy,
 695                               CPU_FEATURE_USABLE (AVX512F),
 696                               __mempcpy_avx512_unaligned)
 697               IFUNC_IMPL_ADD (array, i, mempcpy,
 698                               CPU_FEATURE_USABLE (AVX512F),
 699                               __mempcpy_avx512_unaligned_erms)
 700               IFUNC_IMPL_ADD (array, i, mempcpy,
 701                               CPU_FEATURE_USABLE (AVX),
 702                               __mempcpy_avx_unaligned)
 703               IFUNC_IMPL_ADD (array, i, mempcpy,
 704                               CPU_FEATURE_USABLE (AVX),
 705                               __mempcpy_avx_unaligned_erms)
 706               IFUNC_IMPL_ADD (array, i, mempcpy,
 707                               CPU_FEATURE_USABLE (AVX512VL),
 708                               __mempcpy_evex_unaligned)
 709               IFUNC_IMPL_ADD (array, i, mempcpy,
 710                               CPU_FEATURE_USABLE (AVX512VL),
 711                               __mempcpy_evex_unaligned_erms)
 712               IFUNC_IMPL_ADD (array, i, mempcpy, CPU_FEATURE_USABLE (SSSE3),
 713                               __mempcpy_ssse3_back)
 714               IFUNC_IMPL_ADD (array, i, mempcpy, CPU_FEATURE_USABLE (SSSE3),
 715                               __mempcpy_ssse3)
 716               IFUNC_IMPL_ADD (array, i, mempcpy, 1,
 717                               __mempcpy_sse2_unaligned)
 718               IFUNC_IMPL_ADD (array, i, mempcpy, 1,
 719                               __mempcpy_sse2_unaligned_erms)
 720               IFUNC_IMPL_ADD (array, i, mempcpy, 1, __mempcpy_erms))
 721
 722   /* Support sysdeps/x86_64/multiarch/strncmp.c.  */
 723   IFUNC_IMPL (i, name, strncmp,
 724               IFUNC_IMPL_ADD (array, i, strncmp,
 725                               CPU_FEATURE_USABLE (AVX2),
 726                               __strncmp_avx2)
 727               IFUNC_IMPL_ADD (array, i, strncmp,
 728                               (CPU_FEATURE_USABLE (AVX512VL)
 729                                && CPU_FEATURE_USABLE (AVX512BW)),
 730                               __strncmp_evex)
 731               IFUNC_IMPL_ADD (array, i, strncmp, CPU_FEATURE_USABLE (SSE4_2),
 732                               __strncmp_sse42)
 733               IFUNC_IMPL_ADD (array, i, strncmp, CPU_FEATURE_USABLE (SSSE3),
 734                               __strncmp_ssse3)
 735               IFUNC_IMPL_ADD (array, i, strncmp, 1, __strncmp_sse2))
 736
 737 #ifdef SHARED
 738   /* Support sysdeps/x86_64/multiarch/wmemset_chk.c.  */
 739   IFUNC_IMPL (i, name, __wmemset_chk,
 740               IFUNC_IMPL_ADD (array, i, __wmemset_chk, 1,
 741                               __wmemset_chk_sse2_unaligned)
 742               IFUNC_IMPL_ADD (array, i, __wmemset_chk,
 743                               CPU_FEATURE_USABLE (AVX2),
 744                               __wmemset_chk_avx2_unaligned)
 745               IFUNC_IMPL_ADD (array, i, __wmemset_chk,
 746                               CPU_FEATURE_USABLE (AVX512VL),
 747                               __wmemset_chk_evex_unaligned)
 748               IFUNC_IMPL_ADD (array, i, __wmemset_chk,
 749                               CPU_FEATURE_USABLE (AVX512F),
 750                               __wmemset_chk_avx512_unaligned))
 751 #endif
 752
 753   return i;
 754 }