[thirdparty/gcc.git] / libgcc / config / aarch64 / cpuinfo.c

/* CPU feature detection for AArch64 architecture.
   Copyright (C) 2023-2024 Free Software Foundation, Inc.

   This file is part of GCC.

   This file is free software; you can redistribute it and/or modify it
   under the terms of the GNU General Public License as published by the
   Free Software Foundation; either version 3, or (at your option) any
   later version.

   This file is distributed in the hope that it will be useful, but
   WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   General Public License for more details.

   Under Section 7 of GPL version 3, you are granted additional
   permissions described in the GCC Runtime Library Exception, version
   3.1, as published by the Free Software Foundation.
  
   You should have received a copy of the GNU General Public License and
   a copy of the GCC Runtime Library Exception along with this program;
   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
   <http://www.gnu.org/licenses/>.  */

#include "common/config/aarch64/cpuinfo.h"

#if __has_include(<sys/auxv.h>)
#include <sys/auxv.h>

#if __has_include(<sys/ifunc.h>)
#include <sys/ifunc.h>
#else
typedef struct __ifunc_arg_t {
  unsigned long _size;
  unsigned long _hwcap;
  unsigned long _hwcap2;
} __ifunc_arg_t;
#endif

#if __has_include(<asm/hwcap.h>)
#include <asm/hwcap.h>

/* Architecture features used in Function Multi Versioning.  */
struct {
  unsigned long long features;
  /* As features grows new fields could be added.  */
} __aarch64_cpu_features __attribute__((visibility("hidden"), nocommon));

#ifndef _IFUNC_ARG_HWCAP
#define _IFUNC_ARG_HWCAP (1ULL << 62)
#endif
#ifndef AT_HWCAP
#define AT_HWCAP 16
#endif
#ifndef HWCAP_CPUID
#define HWCAP_CPUID (1 << 11)
#endif
#ifndef HWCAP_FP
#define HWCAP_FP (1 << 0)
#endif
#ifndef HWCAP_ASIMD
#define HWCAP_ASIMD (1 << 1)
#endif
#ifndef HWCAP_AES
#define HWCAP_AES (1 << 3)
#endif
#ifndef HWCAP_PMULL
#define HWCAP_PMULL (1 << 4)
#endif
#ifndef HWCAP_SHA1
#define HWCAP_SHA1 (1 << 5)
#endif
#ifndef HWCAP_SHA2
#define HWCAP_SHA2 (1 << 6)
#endif
#ifndef HWCAP_ATOMICS
#define HWCAP_ATOMICS (1 << 8)
#endif
#ifndef HWCAP_FPHP
#define HWCAP_FPHP (1 << 9)
#endif
#ifndef HWCAP_ASIMDHP
#define HWCAP_ASIMDHP (1 << 10)
#endif
#ifndef HWCAP_ASIMDRDM
#define HWCAP_ASIMDRDM (1 << 12)
#endif
#ifndef HWCAP_JSCVT
#define HWCAP_JSCVT (1 << 13)
#endif
#ifndef HWCAP_FCMA
#define HWCAP_FCMA (1 << 14)
#endif
#ifndef HWCAP_LRCPC
#define HWCAP_LRCPC (1 << 15)
#endif
#ifndef HWCAP_DCPOP
#define HWCAP_DCPOP (1 << 16)
#endif
#ifndef HWCAP_SHA3
#define HWCAP_SHA3 (1 << 17)
#endif
#ifndef HWCAP_SM3
#define HWCAP_SM3 (1 << 18)
#endif
#ifndef HWCAP_SM4
#define HWCAP_SM4 (1 << 19)
#endif
#ifndef HWCAP_ASIMDDP
#define HWCAP_ASIMDDP (1 << 20)
#endif
#ifndef HWCAP_SHA512
#define HWCAP_SHA512 (1 << 21)
#endif
#ifndef HWCAP_SVE
#define HWCAP_SVE (1 << 22)
#endif
#ifndef HWCAP_ASIMDFHM
#define HWCAP_ASIMDFHM (1 << 23)
#endif
#ifndef HWCAP_DIT
#define HWCAP_DIT (1 << 24)
#endif
#ifndef HWCAP_ILRCPC
#define HWCAP_ILRCPC (1 << 26)
#endif
#ifndef HWCAP_FLAGM
#define HWCAP_FLAGM (1 << 27)
#endif
#ifndef HWCAP_SSBS
#define HWCAP_SSBS (1 << 28)
#endif
#ifndef HWCAP_SB
#define HWCAP_SB (1 << 29)
#endif

#ifndef HWCAP2_DCPODP
#define HWCAP2_DCPODP (1 << 0)
#endif
#ifndef HWCAP2_SVE2
#define HWCAP2_SVE2 (1 << 1)
#endif
#ifndef HWCAP2_SVEAES
#define HWCAP2_SVEAES (1 << 2)
#endif
#ifndef HWCAP2_SVEPMULL
#define HWCAP2_SVEPMULL (1 << 3)
#endif
#ifndef HWCAP2_SVEBITPERM
#define HWCAP2_SVEBITPERM (1 << 4)
#endif
#ifndef HWCAP2_SVESHA3
#define HWCAP2_SVESHA3 (1 << 5)
#endif
#ifndef HWCAP2_SVESM4
#define HWCAP2_SVESM4 (1 << 6)
#endif
#ifndef HWCAP2_FLAGM2
#define HWCAP2_FLAGM2 (1 << 7)
#endif
#ifndef HWCAP2_FRINT
#define HWCAP2_FRINT (1 << 8)
#endif
#ifndef HWCAP2_SVEI8MM
#define HWCAP2_SVEI8MM (1 << 9)
#endif
#ifndef HWCAP2_SVEF32MM
#define HWCAP2_SVEF32MM (1 << 10)
#endif
#ifndef HWCAP2_SVEF64MM
#define HWCAP2_SVEF64MM (1 << 11)
#endif
#ifndef HWCAP2_SVEBF16
#define HWCAP2_SVEBF16 (1 << 12)
#endif
#ifndef HWCAP2_I8MM
#define HWCAP2_I8MM (1 << 13)
#endif
#ifndef HWCAP2_BF16
#define HWCAP2_BF16 (1 << 14)
#endif
#ifndef HWCAP2_DGH
#define HWCAP2_DGH (1 << 15)
#endif
#ifndef HWCAP2_RNG
#define HWCAP2_RNG (1 << 16)
#endif
#ifndef HWCAP2_BTI
#define HWCAP2_BTI (1 << 17)
#endif
#ifndef HWCAP2_MTE
#define HWCAP2_MTE (1 << 18)
#endif
#ifndef HWCAP2_RPRES
#define HWCAP2_RPRES (1 << 21)
#endif
#ifndef HWCAP2_MTE3
#define HWCAP2_MTE3 (1 << 22)
#endif
#ifndef HWCAP2_SME
#define HWCAP2_SME (1 << 23)
#endif
#ifndef HWCAP2_SME_I16I64
#define HWCAP2_SME_I16I64 (1 << 24)
#endif
#ifndef HWCAP2_SME_F64F64
#define HWCAP2_SME_F64F64 (1 << 25)
#endif
#ifndef HWCAP2_WFXT
#define HWCAP2_WFXT (1UL << 31)
#endif
#ifndef HWCAP2_EBF16
#define HWCAP2_EBF16 (1UL << 32)
#endif
#ifndef HWCAP2_SVE_EBF16
#define HWCAP2_SVE_EBF16 (1UL << 33)
#endif

static void
__init_cpu_features_constructor(unsigned long hwcap,
				const __ifunc_arg_t *arg) {
#define setCPUFeature(F) __aarch64_cpu_features.features |= 1ULL << F
#define getCPUFeature(id, ftr) __asm__("mrs %0, " #id : "=r"(ftr))
#define extractBits(val, start, number) \
  (val & ((1ULL << number) - 1ULL) << start) >> start
  unsigned long hwcap2 = 0;
  if (hwcap & _IFUNC_ARG_HWCAP)
    hwcap2 = arg->_hwcap2;
  if (hwcap & HWCAP_CRC32)
    setCPUFeature(FEAT_CRC);
  if (hwcap & HWCAP_PMULL)
    setCPUFeature(FEAT_PMULL);
  if (hwcap & HWCAP_FLAGM)
    setCPUFeature(FEAT_FLAGM);
  if (hwcap2 & HWCAP2_FLAGM2) {
    setCPUFeature(FEAT_FLAGM);
    setCPUFeature(FEAT_FLAGM2);
  }
  if (hwcap & HWCAP_SM3 && hwcap & HWCAP_SM4)
    setCPUFeature(FEAT_SM4);
  if (hwcap & HWCAP_ASIMDDP)
    setCPUFeature(FEAT_DOTPROD);
  if (hwcap & HWCAP_ASIMDFHM)
    setCPUFeature(FEAT_FP16FML);
  if (hwcap & HWCAP_FPHP) {
    setCPUFeature(FEAT_FP16);
    setCPUFeature(FEAT_FP);
  }
  if (hwcap & HWCAP_DIT)
    setCPUFeature(FEAT_DIT);
  if (hwcap & HWCAP_ASIMDRDM)
    setCPUFeature(FEAT_RDM);
  if (hwcap & HWCAP_ILRCPC)
    setCPUFeature(FEAT_RCPC2);
  if (hwcap & HWCAP_AES)
    setCPUFeature(FEAT_AES);
  if (hwcap & HWCAP_SHA1)
    setCPUFeature(FEAT_SHA1);
  if (hwcap & HWCAP_SHA2)
    setCPUFeature(FEAT_SHA2);
  if (hwcap & HWCAP_JSCVT)
    setCPUFeature(FEAT_JSCVT);
  if (hwcap & HWCAP_FCMA)
    setCPUFeature(FEAT_FCMA);
  if (hwcap & HWCAP_SB)
    setCPUFeature(FEAT_SB);
  if (hwcap & HWCAP_SSBS)
    setCPUFeature(FEAT_SSBS2);
  if (hwcap2 & HWCAP2_MTE) {
    setCPUFeature(FEAT_MEMTAG);
    setCPUFeature(FEAT_MEMTAG2);
  }
  if (hwcap2 & HWCAP2_MTE3) {
    setCPUFeature(FEAT_MEMTAG);
    setCPUFeature(FEAT_MEMTAG2);
    setCPUFeature(FEAT_MEMTAG3);
  }
  if (hwcap2 & HWCAP2_SVEAES)
    setCPUFeature(FEAT_SVE_AES);
  if (hwcap2 & HWCAP2_SVEPMULL) {
    setCPUFeature(FEAT_SVE_AES);
    setCPUFeature(FEAT_SVE_PMULL128);
  }
  if (hwcap2 & HWCAP2_SVEBITPERM)
    setCPUFeature(FEAT_SVE_BITPERM);
  if (hwcap2 & HWCAP2_SVESHA3)
    setCPUFeature(FEAT_SVE_SHA3);
  if (hwcap2 & HWCAP2_SVESM4)
    setCPUFeature(FEAT_SVE_SM4);
  if (hwcap2 & HWCAP2_DCPODP)
    setCPUFeature(FEAT_DPB2);
  if (hwcap & HWCAP_ATOMICS)
    setCPUFeature(FEAT_LSE);
  if (hwcap2 & HWCAP2_RNG)
    setCPUFeature(FEAT_RNG);
  if (hwcap2 & HWCAP2_I8MM)
    setCPUFeature(FEAT_I8MM);
  if (hwcap2 & HWCAP2_EBF16)
    setCPUFeature(FEAT_EBF16);
  if (hwcap2 & HWCAP2_SVE_EBF16)
    setCPUFeature(FEAT_SVE_EBF16);
  if (hwcap2 & HWCAP2_DGH)
    setCPUFeature(FEAT_DGH);
  if (hwcap2 & HWCAP2_FRINT)
    setCPUFeature(FEAT_FRINTTS);
  if (hwcap2 & HWCAP2_SVEI8MM)
    setCPUFeature(FEAT_SVE_I8MM);
  if (hwcap2 & HWCAP2_SVEF32MM)
    setCPUFeature(FEAT_SVE_F32MM);
  if (hwcap2 & HWCAP2_SVEF64MM)
    setCPUFeature(FEAT_SVE_F64MM);
  if (hwcap2 & HWCAP2_BTI)
    setCPUFeature(FEAT_BTI);
  if (hwcap2 & HWCAP2_RPRES)
    setCPUFeature(FEAT_RPRES);
  if (hwcap2 & HWCAP2_WFXT)
    setCPUFeature(FEAT_WFXT);
  if (hwcap2 & HWCAP2_SME)
    setCPUFeature(FEAT_SME);
  if (hwcap2 & HWCAP2_SME_I16I64)
    setCPUFeature(FEAT_SME_I64);
  if (hwcap2 & HWCAP2_SME_F64F64)
    setCPUFeature(FEAT_SME_F64);
  if (hwcap & HWCAP_CPUID) {
    unsigned long ftr;
    getCPUFeature(ID_AA64PFR1_EL1, ftr);
    /* ID_AA64PFR1_EL1.MTE >= 0b0001  */
    if (extractBits(ftr, 8, 4) >= 0x1)
      setCPUFeature(FEAT_MEMTAG);
    /* ID_AA64PFR1_EL1.SSBS == 0b0001  */
    if (extractBits(ftr, 4, 4) == 0x1)
      setCPUFeature(FEAT_SSBS);
    /* ID_AA64PFR1_EL1.SME == 0b0010  */
    if (extractBits(ftr, 24, 4) == 0x2)
      setCPUFeature(FEAT_SME2);
    getCPUFeature(ID_AA64PFR0_EL1, ftr);
    /* ID_AA64PFR0_EL1.FP != 0b1111  */
    if (extractBits(ftr, 16, 4) != 0xF) {
      setCPUFeature(FEAT_FP);
      /* ID_AA64PFR0_EL1.AdvSIMD has the same value as ID_AA64PFR0_EL1.FP  */
      setCPUFeature(FEAT_SIMD);
    }
    /* ID_AA64PFR0_EL1.SVE != 0b0000  */
    if (extractBits(ftr, 32, 4) != 0x0) {
      /* get ID_AA64ZFR0_EL1, that name supported if sve enabled only  */
      getCPUFeature(S3_0_C0_C4_4, ftr);
      /* ID_AA64ZFR0_EL1.SVEver == 0b0000  */
      if (extractBits(ftr, 0, 4) == 0x0)
	setCPUFeature(FEAT_SVE);
      /* ID_AA64ZFR0_EL1.SVEver == 0b0001  */
      if (extractBits(ftr, 0, 4) == 0x1)
	setCPUFeature(FEAT_SVE2);
      /* ID_AA64ZFR0_EL1.BF16 != 0b0000  */
      if (extractBits(ftr, 20, 4) != 0x0)
	setCPUFeature(FEAT_SVE_BF16);
    }
    getCPUFeature(ID_AA64ISAR0_EL1, ftr);
    /* ID_AA64ISAR0_EL1.SHA3 != 0b0000  */
    if (extractBits(ftr, 32, 4) != 0x0)
      setCPUFeature(FEAT_SHA3);
    getCPUFeature(ID_AA64ISAR1_EL1, ftr);
    /* ID_AA64ISAR1_EL1.DPB >= 0b0001  */
    if (extractBits(ftr, 0, 4) >= 0x1)
      setCPUFeature(FEAT_DPB);
    /* ID_AA64ISAR1_EL1.LRCPC != 0b0000  */
    if (extractBits(ftr, 20, 4) != 0x0)
      setCPUFeature(FEAT_RCPC);
    /* ID_AA64ISAR1_EL1.LRCPC == 0b0011  */
    if (extractBits(ftr, 20, 4) == 0x3)
      setCPUFeature(FEAT_RCPC3);
    /* ID_AA64ISAR1_EL1.SPECRES == 0b0001  */
    if (extractBits(ftr, 40, 4) == 0x2)
      setCPUFeature(FEAT_PREDRES);
    /* ID_AA64ISAR1_EL1.BF16 != 0b0000  */
    if (extractBits(ftr, 44, 4) != 0x0)
      setCPUFeature(FEAT_BF16);
    /* ID_AA64ISAR1_EL1.LS64 >= 0b0001  */
    if (extractBits(ftr, 60, 4) >= 0x1)
      setCPUFeature(FEAT_LS64);
    /* ID_AA64ISAR1_EL1.LS64 >= 0b0010  */
    if (extractBits(ftr, 60, 4) >= 0x2)
      setCPUFeature(FEAT_LS64_V);
    /* ID_AA64ISAR1_EL1.LS64 >= 0b0011  */
    if (extractBits(ftr, 60, 4) >= 0x3)
      setCPUFeature(FEAT_LS64_ACCDATA);
  } else {
    /* Set some features in case of no CPUID support.  */
    if (hwcap & (HWCAP_FP | HWCAP_FPHP)) {
      setCPUFeature(FEAT_FP);
      /* FP and AdvSIMD fields have the same value.  */
      setCPUFeature(FEAT_SIMD);
    }
    if (hwcap & HWCAP_DCPOP || hwcap2 & HWCAP2_DCPODP)
      setCPUFeature(FEAT_DPB);
    if (hwcap & HWCAP_LRCPC || hwcap & HWCAP_ILRCPC)
      setCPUFeature(FEAT_RCPC);
    if (hwcap2 & HWCAP2_BF16 || hwcap2 & HWCAP2_EBF16)
      setCPUFeature(FEAT_BF16);
    if (hwcap2 & HWCAP2_SVEBF16)
      setCPUFeature(FEAT_SVE_BF16);
    if (hwcap2 & HWCAP2_SVE2 && hwcap & HWCAP_SVE)
      setCPUFeature(FEAT_SVE2);
    if (hwcap & HWCAP_SHA3)
      setCPUFeature(FEAT_SHA3);
  }
  setCPUFeature(FEAT_INIT);
}

void
__init_cpu_features_resolver(unsigned long hwcap, const __ifunc_arg_t *arg) {
  if (__aarch64_cpu_features.features)
    return;
  __init_cpu_features_constructor(hwcap, arg);
}

void __attribute__ ((constructor))
__init_cpu_features(void) {
  unsigned long hwcap;
  unsigned long hwcap2;
  /* CPU features already initialized.  */
  if (__aarch64_cpu_features.features)
    return;
  hwcap = getauxval(AT_HWCAP);
  hwcap2 = getauxval(AT_HWCAP2);
  __ifunc_arg_t arg;
  arg._size = sizeof(__ifunc_arg_t);
  arg._hwcap = hwcap;
  arg._hwcap2 = hwcap2;
  __init_cpu_features_constructor(hwcap | _IFUNC_ARG_HWCAP, &arg);
#undef extractBits
#undef getCPUFeature
#undef setCPUFeature
}
#endif /* __has_include(<asm/hwcap.h>)  */
#endif /* __has_include(<sys/auxv.h>)  */
Commit	Line	Data
2c200a12	1	/* CPU feature detection for AArch64 architecture.
a945c346	2	Copyright (C) 2023-2024 Free Software Foundation, Inc.
2c200a12 AC	3
	4	This file is part of GCC.
	5
	6	This file is free software; you can redistribute it and/or modify it
	7	under the terms of the GNU General Public License as published by the
	8	Free Software Foundation; either version 3, or (at your option) any
	9	later version.
	10
	11	This file is distributed in the hope that it will be useful, but
	12	WITHOUT ANY WARRANTY; without even the implied warranty of
	13	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	General Public License for more details.
	15
	16	Under Section 7 of GPL version 3, you are granted additional
	17	permissions described in the GCC Runtime Library Exception, version
	18	3.1, as published by the Free Software Foundation.
	19
	20	You should have received a copy of the GNU General Public License and
	21	a copy of the GCC Runtime Library Exception along with this program;
	22	see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
	23	<http://www.gnu.org/licenses/>. */
	24
0cfde688 AC	25	#include "common/config/aarch64/cpuinfo.h"
0cfde688 AC	26
2c200a12 AC	27	#if __has_include(<sys/auxv.h>)
	28	#include <sys/auxv.h>
	29
	30	#if __has_include(<sys/ifunc.h>)
	31	#include <sys/ifunc.h>
	32	#else
	33	typedef struct __ifunc_arg_t {
	34	unsigned long _size;
	35	unsigned long _hwcap;
	36	unsigned long _hwcap2;
	37	} __ifunc_arg_t;
	38	#endif
	39
	40	#if __has_include(<asm/hwcap.h>)
	41	#include <asm/hwcap.h>
	42
2c200a12 AC	43	/* Architecture features used in Function Multi Versioning. */
	44	struct {
	45	unsigned long long features;
	46	/* As features grows new fields could be added. */
	47	} __aarch64_cpu_features __attribute__((visibility("hidden"), nocommon));
	48
	49	#ifndef _IFUNC_ARG_HWCAP
	50	#define _IFUNC_ARG_HWCAP (1ULL << 62)
	51	#endif
	52	#ifndef AT_HWCAP
	53	#define AT_HWCAP 16
	54	#endif
	55	#ifndef HWCAP_CPUID
	56	#define HWCAP_CPUID (1 << 11)
	57	#endif
	58	#ifndef HWCAP_FP
	59	#define HWCAP_FP (1 << 0)
	60	#endif
	61	#ifndef HWCAP_ASIMD
	62	#define HWCAP_ASIMD (1 << 1)
	63	#endif
	64	#ifndef HWCAP_AES
	65	#define HWCAP_AES (1 << 3)
	66	#endif
	67	#ifndef HWCAP_PMULL
	68	#define HWCAP_PMULL (1 << 4)
	69	#endif
	70	#ifndef HWCAP_SHA1
	71	#define HWCAP_SHA1 (1 << 5)
	72	#endif
	73	#ifndef HWCAP_SHA2
	74	#define HWCAP_SHA2 (1 << 6)
	75	#endif
	76	#ifndef HWCAP_ATOMICS
	77	#define HWCAP_ATOMICS (1 << 8)
	78	#endif
	79	#ifndef HWCAP_FPHP
	80	#define HWCAP_FPHP (1 << 9)
	81	#endif
	82	#ifndef HWCAP_ASIMDHP
	83	#define HWCAP_ASIMDHP (1 << 10)
	84	#endif
	85	#ifndef HWCAP_ASIMDRDM
	86	#define HWCAP_ASIMDRDM (1 << 12)
	87	#endif
	88	#ifndef HWCAP_JSCVT
	89	#define HWCAP_JSCVT (1 << 13)
	90	#endif
	91	#ifndef HWCAP_FCMA
	92	#define HWCAP_FCMA (1 << 14)
	93	#endif
	94	#ifndef HWCAP_LRCPC
	95	#define HWCAP_LRCPC (1 << 15)
	96	#endif
	97	#ifndef HWCAP_DCPOP
	98	#define HWCAP_DCPOP (1 << 16)
	99	#endif
	100	#ifndef HWCAP_SHA3
	101	#define HWCAP_SHA3 (1 << 17)
	102	#endif
	103	#ifndef HWCAP_SM3
	104	#define HWCAP_SM3 (1 << 18)
	105	#endif
	106	#ifndef HWCAP_SM4
107	#define HWCAP_SM4 (1 << 19)
108	#endif
109	#ifndef HWCAP_ASIMDDP
110	#define HWCAP_ASIMDDP (1 << 20)
111	#endif
112	#ifndef HWCAP_SHA512
113	#define HWCAP_SHA512 (1 << 21)
114	#endif
115	#ifndef HWCAP_SVE
116	#define HWCAP_SVE (1 << 22)
117	#endif
118	#ifndef HWCAP_ASIMDFHM
119	#define HWCAP_ASIMDFHM (1 << 23)
120	#endif
121	#ifndef HWCAP_DIT
122	#define HWCAP_DIT (1 << 24)
123	#endif
124	#ifndef HWCAP_ILRCPC
125	#define HWCAP_ILRCPC (1 << 26)
126	#endif
127	#ifndef HWCAP_FLAGM
128	#define HWCAP_FLAGM (1 << 27)
129	#endif
130	#ifndef HWCAP_SSBS
131	#define HWCAP_SSBS (1 << 28)
132	#endif
133	#ifndef HWCAP_SB
134	#define HWCAP_SB (1 << 29)
135	#endif
136
137	#ifndef HWCAP2_DCPODP
138	#define HWCAP2_DCPODP (1 << 0)
139	#endif
140	#ifndef HWCAP2_SVE2
141	#define HWCAP2_SVE2 (1 << 1)
142	#endif
143	#ifndef HWCAP2_SVEAES
144	#define HWCAP2_SVEAES (1 << 2)
145	#endif
146	#ifndef HWCAP2_SVEPMULL
147	#define HWCAP2_SVEPMULL (1 << 3)
148	#endif
149	#ifndef HWCAP2_SVEBITPERM
150	#define HWCAP2_SVEBITPERM (1 << 4)
151	#endif
152	#ifndef HWCAP2_SVESHA3
153	#define HWCAP2_SVESHA3 (1 << 5)
154	#endif
155	#ifndef HWCAP2_SVESM4
156	#define HWCAP2_SVESM4 (1 << 6)
157	#endif
158	#ifndef HWCAP2_FLAGM2
159	#define HWCAP2_FLAGM2 (1 << 7)
160	#endif
161	#ifndef HWCAP2_FRINT
162	#define HWCAP2_FRINT (1 << 8)
163	#endif
164	#ifndef HWCAP2_SVEI8MM
165	#define HWCAP2_SVEI8MM (1 << 9)
166	#endif
167	#ifndef HWCAP2_SVEF32MM
168	#define HWCAP2_SVEF32MM (1 << 10)
169	#endif
170	#ifndef HWCAP2_SVEF64MM
171	#define HWCAP2_SVEF64MM (1 << 11)
172	#endif
173	#ifndef HWCAP2_SVEBF16
174	#define HWCAP2_SVEBF16 (1 << 12)
175	#endif
176	#ifndef HWCAP2_I8MM
177	#define HWCAP2_I8MM (1 << 13)
178	#endif
179	#ifndef HWCAP2_BF16
180	#define HWCAP2_BF16 (1 << 14)
181	#endif
182	#ifndef HWCAP2_DGH
183	#define HWCAP2_DGH (1 << 15)
184	#endif
185	#ifndef HWCAP2_RNG
186	#define HWCAP2_RNG (1 << 16)
187	#endif
188	#ifndef HWCAP2_BTI
189	#define HWCAP2_BTI (1 << 17)
190	#endif
191	#ifndef HWCAP2_MTE
192	#define HWCAP2_MTE (1 << 18)
193	#endif
194	#ifndef HWCAP2_RPRES
195	#define HWCAP2_RPRES (1 << 21)
196	#endif
197	#ifndef HWCAP2_MTE3
198	#define HWCAP2_MTE3 (1 << 22)
199	#endif
200	#ifndef HWCAP2_SME
201	#define HWCAP2_SME (1 << 23)
202	#endif
203	#ifndef HWCAP2_SME_I16I64
204	#define HWCAP2_SME_I16I64 (1 << 24)
205	#endif
206	#ifndef HWCAP2_SME_F64F64
207	#define HWCAP2_SME_F64F64 (1 << 25)
208	#endif
209	#ifndef HWCAP2_WFXT
210	#define HWCAP2_WFXT (1UL << 31)
211	#endif
212	#ifndef HWCAP2_EBF16
213	#define HWCAP2_EBF16 (1UL << 32)
214	#endif
215	#ifndef HWCAP2_SVE_EBF16
216	#define HWCAP2_SVE_EBF16 (1UL << 33)
217	#endif
218
219	static void
220	__init_cpu_features_constructor(unsigned long hwcap,
221	const __ifunc_arg_t *arg) {
222	#define setCPUFeature(F) __aarch64_cpu_features.features \|= 1ULL << F
223	#define getCPUFeature(id, ftr) __asm__("mrs %0, " #id : "=r"(ftr))
224	#define extractBits(val, start, number) \
225	(val & ((1ULL << number) - 1ULL) << start) >> start
226	unsigned long hwcap2 = 0;
227	if (hwcap & _IFUNC_ARG_HWCAP)
228	hwcap2 = arg->_hwcap2;
229	if (hwcap & HWCAP_CRC32)
230	setCPUFeature(FEAT_CRC);
231	if (hwcap & HWCAP_PMULL)
232	setCPUFeature(FEAT_PMULL);
233	if (hwcap & HWCAP_FLAGM)
234	setCPUFeature(FEAT_FLAGM);
235	if (hwcap2 & HWCAP2_FLAGM2) {
236	setCPUFeature(FEAT_FLAGM);
237	setCPUFeature(FEAT_FLAGM2);
238	}
239	if (hwcap & HWCAP_SM3 && hwcap & HWCAP_SM4)
240	setCPUFeature(FEAT_SM4);
241	if (hwcap & HWCAP_ASIMDDP)
242	setCPUFeature(FEAT_DOTPROD);
243	if (hwcap & HWCAP_ASIMDFHM)
244	setCPUFeature(FEAT_FP16FML);
245	if (hwcap & HWCAP_FPHP) {
246	setCPUFeature(FEAT_FP16);
247	setCPUFeature(FEAT_FP);
248	}
249	if (hwcap & HWCAP_DIT)
250	setCPUFeature(FEAT_DIT);
251	if (hwcap & HWCAP_ASIMDRDM)
252	setCPUFeature(FEAT_RDM);
253	if (hwcap & HWCAP_ILRCPC)
254	setCPUFeature(FEAT_RCPC2);
255	if (hwcap & HWCAP_AES)
256	setCPUFeature(FEAT_AES);
257	if (hwcap & HWCAP_SHA1)
258	setCPUFeature(FEAT_SHA1);
259	if (hwcap & HWCAP_SHA2)
260	setCPUFeature(FEAT_SHA2);
261	if (hwcap & HWCAP_JSCVT)
262	setCPUFeature(FEAT_JSCVT);
263	if (hwcap & HWCAP_FCMA)
264	setCPUFeature(FEAT_FCMA);
265	if (hwcap & HWCAP_SB)
266	setCPUFeature(FEAT_SB);
267	if (hwcap & HWCAP_SSBS)
268	setCPUFeature(FEAT_SSBS2);
269	if (hwcap2 & HWCAP2_MTE) {
270	setCPUFeature(FEAT_MEMTAG);
271	setCPUFeature(FEAT_MEMTAG2);
272	}
273	if (hwcap2 & HWCAP2_MTE3) {
274	setCPUFeature(FEAT_MEMTAG);
275	setCPUFeature(FEAT_MEMTAG2);
276	setCPUFeature(FEAT_MEMTAG3);
277	}
278	if (hwcap2 & HWCAP2_SVEAES)
279	setCPUFeature(FEAT_SVE_AES);
280	if (hwcap2 & HWCAP2_SVEPMULL) {
281	setCPUFeature(FEAT_SVE_AES);
282	setCPUFeature(FEAT_SVE_PMULL128);
283	}
284	if (hwcap2 & HWCAP2_SVEBITPERM)
285	setCPUFeature(FEAT_SVE_BITPERM);
286	if (hwcap2 & HWCAP2_SVESHA3)
287	setCPUFeature(FEAT_SVE_SHA3);
288	if (hwcap2 & HWCAP2_SVESM4)
289	setCPUFeature(FEAT_SVE_SM4);
290	if (hwcap2 & HWCAP2_DCPODP)
291	setCPUFeature(FEAT_DPB2);
292	if (hwcap & HWCAP_ATOMICS)
293	setCPUFeature(FEAT_LSE);
294	if (hwcap2 & HWCAP2_RNG)
295	setCPUFeature(FEAT_RNG);
296	if (hwcap2 & HWCAP2_I8MM)
297	setCPUFeature(FEAT_I8MM);
298	if (hwcap2 & HWCAP2_EBF16)
299	setCPUFeature(FEAT_EBF16);
300	if (hwcap2 & HWCAP2_SVE_EBF16)
301	setCPUFeature(FEAT_SVE_EBF16);
302	if (hwcap2 & HWCAP2_DGH)
303	setCPUFeature(FEAT_DGH);
304	if (hwcap2 & HWCAP2_FRINT)
305	setCPUFeature(FEAT_FRINTTS);
306	if (hwcap2 & HWCAP2_SVEI8MM)
307	setCPUFeature(FEAT_SVE_I8MM);
308	if (hwcap2 & HWCAP2_SVEF32MM)
309	setCPUFeature(FEAT_SVE_F32MM);
310	if (hwcap2 & HWCAP2_SVEF64MM)
311	setCPUFeature(FEAT_SVE_F64MM);
312	if (hwcap2 & HWCAP2_BTI)
313	setCPUFeature(FEAT_BTI);
314	if (hwcap2 & HWCAP2_RPRES)
315	setCPUFeature(FEAT_RPRES);
316	if (hwcap2 & HWCAP2_WFXT)
317	setCPUFeature(FEAT_WFXT);
318	if (hwcap2 & HWCAP2_SME)
319	setCPUFeature(FEAT_SME);
320	if (hwcap2 & HWCAP2_SME_I16I64)
321	setCPUFeature(FEAT_SME_I64);
322	if (hwcap2 & HWCAP2_SME_F64F64)
323	setCPUFeature(FEAT_SME_F64);
324	if (hwcap & HWCAP_CPUID) {
325	unsigned long ftr;
326	getCPUFeature(ID_AA64PFR1_EL1, ftr);
327	/* ID_AA64PFR1_EL1.MTE >= 0b0001 */
328	if (extractBits(ftr, 8, 4) >= 0x1)
329	setCPUFeature(FEAT_MEMTAG);
330	/* ID_AA64PFR1_EL1.SSBS == 0b0001 */
331	if (extractBits(ftr, 4, 4) == 0x1)
332	setCPUFeature(FEAT_SSBS);
333	/* ID_AA64PFR1_EL1.SME == 0b0010 */
334	if (extractBits(ftr, 24, 4) == 0x2)
335	setCPUFeature(FEAT_SME2);
336	getCPUFeature(ID_AA64PFR0_EL1, ftr);
337	/* ID_AA64PFR0_EL1.FP != 0b1111 */
338	if (extractBits(ftr, 16, 4) != 0xF) {
339	setCPUFeature(FEAT_FP);
340	/* ID_AA64PFR0_EL1.AdvSIMD has the same value as ID_AA64PFR0_EL1.FP */
341	setCPUFeature(FEAT_SIMD);
342	}
343	/* ID_AA64PFR0_EL1.SVE != 0b0000 */
344	if (extractBits(ftr, 32, 4) != 0x0) {
345	/* get ID_AA64ZFR0_EL1, that name supported if sve enabled only */
346	getCPUFeature(S3_0_C0_C4_4, ftr);
347	/* ID_AA64ZFR0_EL1.SVEver == 0b0000 */
348	if (extractBits(ftr, 0, 4) == 0x0)
349	setCPUFeature(FEAT_SVE);
350	/* ID_AA64ZFR0_EL1.SVEver == 0b0001 */
351	if (extractBits(ftr, 0, 4) == 0x1)
352	setCPUFeature(FEAT_SVE2);
353	/* ID_AA64ZFR0_EL1.BF16 != 0b0000 */
354	if (extractBits(ftr, 20, 4) != 0x0)
355	setCPUFeature(FEAT_SVE_BF16);
356	}
357	getCPUFeature(ID_AA64ISAR0_EL1, ftr);
358	/* ID_AA64ISAR0_EL1.SHA3 != 0b0000 */
359	if (extractBits(ftr, 32, 4) != 0x0)
360	setCPUFeature(FEAT_SHA3);
361	getCPUFeature(ID_AA64ISAR1_EL1, ftr);
362	/* ID_AA64ISAR1_EL1.DPB >= 0b0001 */
363	if (extractBits(ftr, 0, 4) >= 0x1)
364	setCPUFeature(FEAT_DPB);
365	/* ID_AA64ISAR1_EL1.LRCPC != 0b0000 */
366	if (extractBits(ftr, 20, 4) != 0x0)
367	setCPUFeature(FEAT_RCPC);
368	/* ID_AA64ISAR1_EL1.LRCPC == 0b0011 */
369	if (extractBits(ftr, 20, 4) == 0x3)
370	setCPUFeature(FEAT_RCPC3);
371	/* ID_AA64ISAR1_EL1.SPECRES == 0b0001 */
372	if (extractBits(ftr, 40, 4) == 0x2)
373	setCPUFeature(FEAT_PREDRES);
374	/* ID_AA64ISAR1_EL1.BF16 != 0b0000 */
375	if (extractBits(ftr, 44, 4) != 0x0)
376	setCPUFeature(FEAT_BF16);
377	/* ID_AA64ISAR1_EL1.LS64 >= 0b0001 */
378	if (extractBits(ftr, 60, 4) >= 0x1)
379	setCPUFeature(FEAT_LS64);
380	/* ID_AA64ISAR1_EL1.LS64 >= 0b0010 */
381	if (extractBits(ftr, 60, 4) >= 0x2)
382	setCPUFeature(FEAT_LS64_V);
383	/* ID_AA64ISAR1_EL1.LS64 >= 0b0011 */
384	if (extractBits(ftr, 60, 4) >= 0x3)
385	setCPUFeature(FEAT_LS64_ACCDATA);
386	} else {
387	/* Set some features in case of no CPUID support. */
388	if (hwcap & (HWCAP_FP \| HWCAP_FPHP)) {
389	setCPUFeature(FEAT_FP);
390	/* FP and AdvSIMD fields have the same value. */
391	setCPUFeature(FEAT_SIMD);
392	}
393	if (hwcap & HWCAP_DCPOP \|\| hwcap2 & HWCAP2_DCPODP)
394	setCPUFeature(FEAT_DPB);
395	if (hwcap & HWCAP_LRCPC \|\| hwcap & HWCAP_ILRCPC)
396	setCPUFeature(FEAT_RCPC);
397	if (hwcap2 & HWCAP2_BF16 \|\| hwcap2 & HWCAP2_EBF16)
398	setCPUFeature(FEAT_BF16);
399	if (hwcap2 & HWCAP2_SVEBF16)
400	setCPUFeature(FEAT_SVE_BF16);
401	if (hwcap2 & HWCAP2_SVE2 && hwcap & HWCAP_SVE)
402	setCPUFeature(FEAT_SVE2);
403	if (hwcap & HWCAP_SHA3)
404	setCPUFeature(FEAT_SHA3);
405	}
406	setCPUFeature(FEAT_INIT);
407	}
408
409	void
410	__init_cpu_features_resolver(unsigned long hwcap, const __ifunc_arg_t *arg) {
411	if (__aarch64_cpu_features.features)
412	return;
413	__init_cpu_features_constructor(hwcap, arg);
414	}
415
416	void __attribute__ ((constructor))
417	__init_cpu_features(void) {
418	unsigned long hwcap;
419	unsigned long hwcap2;
420	/* CPU features already initialized. */
421	if (__aarch64_cpu_features.features)
422	return;
423	hwcap = getauxval(AT_HWCAP);
424	hwcap2 = getauxval(AT_HWCAP2);
425	__ifunc_arg_t arg;
426	arg._size = sizeof(__ifunc_arg_t);
427	arg._hwcap = hwcap;
428	arg._hwcap2 = hwcap2;
429	__init_cpu_features_constructor(hwcap \| _IFUNC_ARG_HWCAP, &arg);
430	#undef extractBits
431	#undef getCPUFeature
432	#undef setCPUFeature
433	}
434	#endif /* __has_include(<asm/hwcap.h>) */
435	#endif /* __has_include(<sys/auxv.h>) */