[thirdparty/gcc.git] / libstdc++-v3 / include / experimental / bits / simd_neon.h

// Simd NEON specific implementations -*- C++ -*-

// Copyright (C) 2020-2024 Free Software Foundation, Inc.
//
// This file is part of the GNU ISO C++ Library.  This library is free
// software; you can redistribute it and/or modify it under the
// terms of the GNU General Public License as published by the
// Free Software Foundation; either version 3, or (at your option)
// any later version.

// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// Under Section 7 of GPL version 3, you are granted additional
// permissions described in the GCC Runtime Library Exception, version
// 3.1, as published by the Free Software Foundation.

// You should have received a copy of the GNU General Public License and
// a copy of the GCC Runtime Library Exception along with this program;
// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
// <http://www.gnu.org/licenses/>.

#ifndef _GLIBCXX_EXPERIMENTAL_SIMD_NEON_H_
#define _GLIBCXX_EXPERIMENTAL_SIMD_NEON_H_

#if __cplusplus >= 201703L

#if !_GLIBCXX_SIMD_HAVE_NEON
#error "simd_neon.h may only be included when NEON on ARM is available"
#endif

_GLIBCXX_SIMD_BEGIN_NAMESPACE

// _CommonImplNeon {{{
struct _CommonImplNeon : _CommonImplBuiltin
{
  // _S_store {{{
  using _CommonImplBuiltin::_S_store;

  // }}}
};

// }}}
// _SimdImplNeon {{{
template <typename _Abi, typename>
  struct _SimdImplNeon : _SimdImplBuiltin<_Abi>
  {
    using _Base = _SimdImplBuiltin<_Abi>;

    template <typename _Tp>
      using _MaskMember = typename _Base::template _MaskMember<_Tp>;

    template <typename _Tp>
      static constexpr size_t _S_max_store_size = 16;

    // _S_masked_load {{{
    template <typename _Tp, size_t _Np, typename _Up>
      static inline _SimdWrapper<_Tp, _Np>
      _S_masked_load(_SimdWrapper<_Tp, _Np> __merge, _MaskMember<_Tp> __k,
		     const _Up* __mem) noexcept
      {
	__execute_n_times<_Np>([&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
	  if (__k[__i] != 0)
	    __merge._M_set(__i, static_cast<_Tp>(__mem[__i]));
	});
	return __merge;
      }

    // }}}
    // _S_masked_store_nocvt {{{
    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static void
      _S_masked_store_nocvt(_SimdWrapper<_Tp, _Np> __v, _Tp* __mem,
			    _MaskMember<_Tp> __k)
      {
	__execute_n_times<_Np>([&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
	  if (__k[__i] != 0)
	    __mem[__i] = __v[__i];
	});
      }

    // }}}
    // _S_reduce {{{
    template <typename _Tp, typename _BinaryOperation>
      _GLIBCXX_SIMD_INTRINSIC static constexpr _Tp
      _S_reduce(simd<_Tp, _Abi> __x, _BinaryOperation&& __binary_op)
      {
	if (not __builtin_is_constant_evaluated())
	  {
	    constexpr size_t _Np = __x.size();
	    if constexpr (sizeof(__x) == 16 && _Np >= 4
			    && !_Abi::template _S_is_partial<_Tp>)
	      {
		const auto __halves = split<simd<_Tp, simd_abi::_Neon<8>>>(__x);
		const auto __y = __binary_op(__halves[0], __halves[1]);
		return _SimdImplNeon<simd_abi::_Neon<8>>::_S_reduce(
			 __y, static_cast<_BinaryOperation&&>(__binary_op));
	      }
	    else if constexpr (_Np == 8)
	      {
		__x = __binary_op(__x, _Base::template _M_make_simd<_Tp, _Np>(
					 __vector_permute<1, 0, 3, 2, 5, 4, 7, 6>(__x._M_data)));
		__x = __binary_op(__x, _Base::template _M_make_simd<_Tp, _Np>(
					 __vector_permute<3, 2, 1, 0, 7, 6, 5, 4>(__x._M_data)));
		__x = __binary_op(__x, _Base::template _M_make_simd<_Tp, _Np>(
					 __vector_permute<7, 6, 5, 4, 3, 2, 1, 0>(__x._M_data)));
		return __x[0];
	      }
	    else if constexpr (_Np == 4)
	      {
		__x = __binary_op(__x, _Base::template _M_make_simd<_Tp, _Np>(
					 __vector_permute<1, 0, 3, 2>(__x._M_data)));
		__x = __binary_op(__x, _Base::template _M_make_simd<_Tp, _Np>(
					 __vector_permute<3, 2, 1, 0>(__x._M_data)));
		return __x[0];
	      }
	    else if constexpr (_Np == 2)
	      {
		__x = __binary_op(__x, _Base::template _M_make_simd<_Tp, _Np>(
					 __vector_permute<1, 0>(__x._M_data)));
		return __x[0];
	      }
	  }
	return _Base::_S_reduce(__x, static_cast<_BinaryOperation&&>(__binary_op));
      }

    // }}}
    // math {{{
    // _S_sqrt {{{
    template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
      _GLIBCXX_SIMD_INTRINSIC static _Tp
      _S_sqrt(_Tp __x)
      {
	if constexpr (__have_neon_a64)
	  {
	    const auto __intrin = __to_intrin(__x);
	    if constexpr (_TVT::template _S_is<float, 2>)
	      return vsqrt_f32(__intrin);
	    else if constexpr (_TVT::template _S_is<float, 4>)
	      return vsqrtq_f32(__intrin);
	    else if constexpr (_TVT::template _S_is<double, 1>)
	      return vsqrt_f64(__intrin);
	    else if constexpr (_TVT::template _S_is<double, 2>)
	      return vsqrtq_f64(__intrin);
	    else
	      __assert_unreachable<_Tp>();
	  }
	else
	  return _Base::_S_sqrt(__x);
      }

    // }}}
    // _S_trunc {{{
    template <typename _TW, typename _TVT = _VectorTraits<_TW>>
      _GLIBCXX_SIMD_INTRINSIC static _TW
      _S_trunc(_TW __x)
      {
	using _Tp = typename _TVT::value_type;
	if constexpr (__have_neon_a32)
	  {
	    const auto __intrin = __to_intrin(__x);
	    if constexpr (_TVT::template _S_is<float, 2>)
	      return vrnd_f32(__intrin);
	    else if constexpr (_TVT::template _S_is<float, 4>)
	      return vrndq_f32(__intrin);
	    else if constexpr (_TVT::template _S_is<double, 1>)
	      return vrnd_f64(__intrin);
	    else if constexpr (_TVT::template _S_is<double, 2>)
	      return vrndq_f64(__intrin);
	    else
	      __assert_unreachable<_Tp>();
	  }
	else if constexpr (is_same_v<_Tp, float>)
	  {
	    auto __intrin = __to_intrin(__x);
	    if constexpr (sizeof(__x) == 16)
	      __intrin = vcvtq_f32_s32(vcvtq_s32_f32(__intrin));
	    else
	      __intrin = vcvt_f32_s32(vcvt_s32_f32(__intrin));
	    return _Base::_S_abs(__x)._M_data < 0x1p23f
		     ? __vector_bitcast<float>(__intrin)
		     : __x._M_data;
	  }
	else
	  return _Base::_S_trunc(__x);
      }

    // }}}
    // _S_round {{{
    template <typename _Tp, size_t _Np>
      _GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np>
      _S_round(_SimdWrapper<_Tp, _Np> __x)
      {
	if constexpr (__have_neon_a32)
	  {
	    const auto __intrin = __to_intrin(__x);
	    if constexpr (sizeof(_Tp) == 4 && sizeof(__x) == 8)
	      return vrnda_f32(__intrin);
	    else if constexpr (sizeof(_Tp) == 4 && sizeof(__x) == 16)
	      return vrndaq_f32(__intrin);
	    else if constexpr (sizeof(_Tp) == 8 && sizeof(__x) == 8)
	      return vrnda_f64(__intrin);
	    else if constexpr (sizeof(_Tp) == 8 && sizeof(__x) == 16)
	      return vrndaq_f64(__intrin);
	    else
	      __assert_unreachable<_Tp>();
	  }
	else
	  return _Base::_S_round(__x);
      }

    // }}}
    // _S_floor {{{
    template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
      _GLIBCXX_SIMD_INTRINSIC static _Tp
      _S_floor(_Tp __x)
      {
	if constexpr (__have_neon_a32)
	  {
	    const auto __intrin = __to_intrin(__x);
	    if constexpr (_TVT::template _S_is<float, 2>)
	      return vrndm_f32(__intrin);
	    else if constexpr (_TVT::template _S_is<float, 4>)
	      return vrndmq_f32(__intrin);
	    else if constexpr (_TVT::template _S_is<double, 1>)
	      return vrndm_f64(__intrin);
	    else if constexpr (_TVT::template _S_is<double, 2>)
	      return vrndmq_f64(__intrin);
	    else
	      __assert_unreachable<_Tp>();
	  }
	else
	  return _Base::_S_floor(__x);
      }

    // }}}
    // _S_ceil {{{
    template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
      _GLIBCXX_SIMD_INTRINSIC static _Tp
      _S_ceil(_Tp __x)
      {
	if constexpr (__have_neon_a32)
	  {
	    const auto __intrin = __to_intrin(__x);
	    if constexpr (_TVT::template _S_is<float, 2>)
	      return vrndp_f32(__intrin);
	    else if constexpr (_TVT::template _S_is<float, 4>)
	      return vrndpq_f32(__intrin);
	    else if constexpr (_TVT::template _S_is<double, 1>)
	      return vrndp_f64(__intrin);
	    else if constexpr (_TVT::template _S_is<double, 2>)
	      return vrndpq_f64(__intrin);
	    else
	      __assert_unreachable<_Tp>();
	  }
	else
	  return _Base::_S_ceil(__x);
      }

    //}}} }}}
  }; // }}}
// _MaskImplNeonMixin {{{
struct _MaskImplNeonMixin
{
  using _Base = _MaskImplBuiltinMixin;

  template <typename _Tp, size_t _Np>
    _GLIBCXX_SIMD_INTRINSIC static constexpr _SanitizedBitMask<_Np>
    _S_to_bits(_SimdWrapper<_Tp, _Np> __x)
    {
      if (__builtin_is_constant_evaluated())
	return _Base::_S_to_bits(__x);

      using _I = __int_for_sizeof_t<_Tp>;
      if constexpr (sizeof(__x) == 16)
	{
	  auto __asint = __vector_bitcast<_I>(__x);
#ifdef __aarch64__
	  [[maybe_unused]] constexpr auto __zero = decltype(__asint)();
#else
	  [[maybe_unused]] constexpr auto __zero = decltype(__lo64(__asint))();
#endif
	  if constexpr (sizeof(_Tp) == 1)
	    {
	      constexpr auto __bitsel
		= __generate_from_n_evaluations<16, __vector_type_t<_I, 16>>(
		  [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
		    return static_cast<_I>(
		      __i < _Np ? (__i < 8 ? 1 << __i : 1 << (__i - 8)) : 0);
		  });
	      __asint &= __bitsel;
#ifdef __aarch64__
	      return __vector_bitcast<_UShort>(
		vpaddq_s8(vpaddq_s8(vpaddq_s8(__asint, __zero), __zero),
			  __zero))[0];
#else
	      return __vector_bitcast<_UShort>(
		vpadd_s8(vpadd_s8(vpadd_s8(__lo64(__asint), __hi64(__asint)),
				  __zero),
			 __zero))[0];
#endif
	    }
	  else if constexpr (sizeof(_Tp) == 2)
	    {
	      constexpr auto __bitsel
		= __generate_from_n_evaluations<8, __vector_type_t<_I, 8>>(
		  [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
		    return static_cast<_I>(__i < _Np ? 1 << __i : 0);
		  });
	      __asint &= __bitsel;
#ifdef __aarch64__
	      return vaddvq_s16(__asint);
#else
	      return vpadd_s16(
		vpadd_s16(vpadd_s16(__lo64(__asint), __hi64(__asint)), __zero),
		__zero)[0];
#endif
	    }
	  else if constexpr (sizeof(_Tp) == 4)
	    {
	      constexpr auto __bitsel
		= __generate_from_n_evaluations<4, __vector_type_t<_I, 4>>(
		  [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
		    return static_cast<_I>(__i < _Np ? 1 << __i : 0);
		  });
	      __asint &= __bitsel;
#ifdef __aarch64__
	      return vaddvq_s32(__asint);
#else
	      return vpadd_s32(vpadd_s32(__lo64(__asint), __hi64(__asint)),
			       __zero)[0];
#endif
	    }
	  else if constexpr (sizeof(_Tp) == 8)
	    return (__asint[0] & 1) | (__asint[1] & 2);
	  else
	    __assert_unreachable<_Tp>();
	}
      else if constexpr (sizeof(__x) == 8)
	{
	  auto __asint = __vector_bitcast<_I>(__x);
	  [[maybe_unused]] constexpr auto __zero = decltype(__asint)();
	  if constexpr (sizeof(_Tp) == 1)
	    {
	      constexpr auto __bitsel
		= __generate_from_n_evaluations<8, __vector_type_t<_I, 8>>(
		  [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
		    return static_cast<_I>(__i < _Np ? 1 << __i : 0);
		  });
	      __asint &= __bitsel;
#ifdef __aarch64__
	      return vaddv_s8(__asint);
#else
	      return vpadd_s8(vpadd_s8(vpadd_s8(__asint, __zero), __zero),
			      __zero)[0];
#endif
	    }
	  else if constexpr (sizeof(_Tp) == 2)
	    {
	      constexpr auto __bitsel
		= __generate_from_n_evaluations<4, __vector_type_t<_I, 4>>(
		  [&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
		    return static_cast<_I>(__i < _Np ? 1 << __i : 0);
		  });
	      __asint &= __bitsel;
#ifdef __aarch64__
	      return vaddv_s16(__asint);
#else
	      return vpadd_s16(vpadd_s16(__asint, __zero), __zero)[0];
#endif
	    }
	  else if constexpr (sizeof(_Tp) == 4)
	    {
	      __asint &= __make_vector<_I>(0x1, 0x2);
#ifdef __aarch64__
	      return vaddv_s32(__asint);
#else
	      return vpadd_s32(__asint, __zero)[0];
#endif
	    }
	  else
	    __assert_unreachable<_Tp>();
	}
      else
	return _Base::_S_to_bits(__x);
    }
};

// }}}
// _MaskImplNeon {{{
template <typename _Abi, typename>
  struct _MaskImplNeon : _MaskImplNeonMixin, _MaskImplBuiltin<_Abi>
  {
    using _MaskImplBuiltinMixin::_S_to_maskvector;
    using _MaskImplNeonMixin::_S_to_bits;
    using _Base = _MaskImplBuiltin<_Abi>;
    using _Base::_S_convert;

    // _S_all_of {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static bool
      _S_all_of(simd_mask<_Tp, _Abi> __k)
      {
	const auto __kk
	  = __vector_bitcast<char>(__k._M_data)
	    | ~__vector_bitcast<char>(_Abi::template _S_implicit_mask<_Tp>());
	if constexpr (sizeof(__k) == 16)
	  {
	    const auto __x = __vector_bitcast<long long>(__kk);
	    return __x[0] + __x[1] == -2;
	  }
	else if constexpr (sizeof(__k) <= 8)
	  return __bit_cast<__int_for_sizeof_t<decltype(__kk)>>(__kk) == -1;
	else
	  __assert_unreachable<_Tp>();
      }

    // }}}
    // _S_any_of {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static bool
      _S_any_of(simd_mask<_Tp, _Abi> __k)
      {
	const auto __kk
	  = __vector_bitcast<char>(__k._M_data)
	    | ~__vector_bitcast<char>(_Abi::template _S_implicit_mask<_Tp>());
	if constexpr (sizeof(__k) == 16)
	  {
	    const auto __x = __vector_bitcast<long long>(__kk);
	    return (__x[0] | __x[1]) != 0;
	  }
	else if constexpr (sizeof(__k) <= 8)
	  return __bit_cast<__int_for_sizeof_t<decltype(__kk)>>(__kk) != 0;
	else
	  __assert_unreachable<_Tp>();
      }

    // }}}
    // _S_none_of {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static bool
      _S_none_of(simd_mask<_Tp, _Abi> __k)
      {
	const auto __kk = _Abi::_S_masked(__k._M_data);
	if constexpr (sizeof(__k) == 16)
	  {
	    const auto __x = __vector_bitcast<long long>(__kk);
	    return (__x[0] | __x[1]) == 0;
	  }
	else if constexpr (sizeof(__k) <= 8)
	  return __bit_cast<__int_for_sizeof_t<decltype(__kk)>>(__kk) == 0;
	else
	  __assert_unreachable<_Tp>();
      }

    // }}}
    // _S_some_of {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static bool _S_some_of(simd_mask<_Tp, _Abi> __k)
      {
	if constexpr (sizeof(__k) <= 8)
	  {
	    const auto __kk = __vector_bitcast<char>(__k._M_data)
			      | ~__vector_bitcast<char>(
				_Abi::template _S_implicit_mask<_Tp>());
	    using _Up = make_unsigned_t<__int_for_sizeof_t<decltype(__kk)>>;
	    return __bit_cast<_Up>(__kk) + 1 > 1;
	  }
	else
	  return _Base::_S_some_of(__k);
      }

    // }}}
    // _S_popcount {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static int
      _S_popcount(simd_mask<_Tp, _Abi> __k)
      {
	if constexpr (sizeof(_Tp) == 1)
	  {
	    const auto __s8 = __vector_bitcast<_SChar>(__k._M_data);
	    int8x8_t __tmp = __lo64(__s8) + __hi64z(__s8);
	    return -vpadd_s8(vpadd_s8(vpadd_s8(__tmp, int8x8_t()), int8x8_t()),
			     int8x8_t())[0];
	  }
	else if constexpr (sizeof(_Tp) == 2)
	  {
	    const auto __s16 = __vector_bitcast<short>(__k._M_data);
	    int16x4_t __tmp = __lo64(__s16) + __hi64z(__s16);
	    return -vpadd_s16(vpadd_s16(__tmp, int16x4_t()), int16x4_t())[0];
	  }
	else if constexpr (sizeof(_Tp) == 4)
	  {
	    const auto __s32 = __vector_bitcast<int>(__k._M_data);
	    int32x2_t __tmp = __lo64(__s32) + __hi64z(__s32);
	    return -vpadd_s32(__tmp, int32x2_t())[0];
	  }
	else if constexpr (sizeof(_Tp) == 8)
	  {
	    static_assert(sizeof(__k) == 16);
	    const auto __s64 = __vector_bitcast<long>(__k._M_data);
	    return -(__s64[0] + __s64[1]);
	  }
      }

    // }}}
    // _S_find_first_set {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static int
      _S_find_first_set(simd_mask<_Tp, _Abi> __k)
      {
	// TODO: the _Base implementation is not optimal for NEON
	return _Base::_S_find_first_set(__k);
      }

    // }}}
    // _S_find_last_set {{{
    template <typename _Tp>
      _GLIBCXX_SIMD_INTRINSIC static int
      _S_find_last_set(simd_mask<_Tp, _Abi> __k)
      {
	// TODO: the _Base implementation is not optimal for NEON
	return _Base::_S_find_last_set(__k);
      }

    // }}}
  }; // }}}

_GLIBCXX_SIMD_END_NAMESPACE
#endif // __cplusplus >= 201703L
#endif // _GLIBCXX_EXPERIMENTAL_SIMD_NEON_H_
// vim: foldmethod=marker sw=2 noet ts=8 sts=2 tw=80
Commit	Line	Data
2bcceb6f MK	1	// Simd NEON specific implementations -- C++ --
2bcceb6f MK	2
a945c346	3	// Copyright (C) 2020-2024 Free Software Foundation, Inc.
2bcceb6f MK	4	//
	5	// This file is part of the GNU ISO C++ Library. This library is free
	6	// software; you can redistribute it and/or modify it under the
	7	// terms of the GNU General Public License as published by the
	8	// Free Software Foundation; either version 3, or (at your option)
	9	// any later version.
	10
	11	// This library is distributed in the hope that it will be useful,
	12	// but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	14	// GNU General Public License for more details.
	15
	16	// Under Section 7 of GPL version 3, you are granted additional
	17	// permissions described in the GCC Runtime Library Exception, version
	18	// 3.1, as published by the Free Software Foundation.
	19
	20	// You should have received a copy of the GNU General Public License and
	21	// a copy of the GCC Runtime Library Exception along with this program;
	22	// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
	23	// <http://www.gnu.org/licenses/>.
	24
	25	#ifndef _GLIBCXX_EXPERIMENTAL_SIMD_NEON_H_
	26	#define _GLIBCXX_EXPERIMENTAL_SIMD_NEON_H_
	27
	28	#if __cplusplus >= 201703L
	29
	30	#if !_GLIBCXX_SIMD_HAVE_NEON
	31	#error "simd_neon.h may only be included when NEON on ARM is available"
	32	#endif
	33
	34	_GLIBCXX_SIMD_BEGIN_NAMESPACE
	35
	36	// _CommonImplNeon {{{
	37	struct _CommonImplNeon : _CommonImplBuiltin
	38	{
	39	// _S_store {{{
	40	using _CommonImplBuiltin::_S_store;
	41
	42	// }}}
	43	};
	44
	45	// }}}
	46	// _SimdImplNeon {{{
52d28210	47	template <typename _Abi, typename>
2bcceb6f MK	48	struct _SimdImplNeon : _SimdImplBuiltin<_Abi>
	49	{
	50	using _Base = _SimdImplBuiltin<_Abi>;
	51
	52	template <typename _Tp>
	53	using _MaskMember = typename _Base::template _MaskMember<_Tp>;
	54
	55	template <typename _Tp>
	56	static constexpr size_t _S_max_store_size = 16;
	57
	58	// _S_masked_load {{{
	59	template <typename _Tp, size_t _Np, typename _Up>
	60	static inline _SimdWrapper<_Tp, _Np>
	61	_S_masked_load(_SimdWrapper<_Tp, _Np> __merge, _MaskMember<_Tp> __k,
	62	const _Up* __mem) noexcept
	63	{
53b55701	64	__execute_n_times<_Np>([&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2bcceb6f MK	65	if (__k[__i] != 0)
	66	__merge._M_set(__i, static_cast<_Tp>(__mem[__i]));
	67	});
	68	return __merge;
	69	}
	70
	71	// }}}
	72	// _S_masked_store_nocvt {{{
	73	template <typename _Tp, size_t _Np>
	74	_GLIBCXX_SIMD_INTRINSIC static void
	75	_S_masked_store_nocvt(_SimdWrapper<_Tp, _Np> __v, _Tp* __mem,
	76	_MaskMember<_Tp> __k)
	77	{
53b55701	78	__execute_n_times<_Np>([&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2bcceb6f MK	79	if (__k[__i] != 0)
	80	__mem[__i] = __v[__i];
	81	});
	82	}
	83
	84	// }}}
	85	// _S_reduce {{{
	86	template <typename _Tp, typename _BinaryOperation>
b0a483b0	87	_GLIBCXX_SIMD_INTRINSIC static constexpr _Tp
2bcceb6f MK	88	_S_reduce(simd<_Tp, _Abi> __x, _BinaryOperation&& __binary_op)
2bcceb6f MK	89	{
b0a483b0	90	if (not __builtin_is_constant_evaluated())
2bcceb6f	91	{
b0a483b0 MK	92	constexpr size_t _Np = __x.size();
	93	if constexpr (sizeof(__x) == 16 && _Np >= 4
	94	&& !_Abi::template _S_is_partial<_Tp>)
	95	{
	96	const auto __halves = split<simd<_Tp, simd_abi::_Neon<8>>>(__x);
	97	const auto __y = __binary_op(__halves[0], __halves[1]);
	98	return _SimdImplNeon<simd_abi::_Neon<8>>::_S_reduce(
	99	__y, static_cast<_BinaryOperation&&>(__binary_op));
	100	}
	101	else if constexpr (_Np == 8)
	102	{
	103	__x = __binary_op(__x, _Base::template _M_make_simd<_Tp, _Np>(
	104	__vector_permute<1, 0, 3, 2, 5, 4, 7, 6>(__x._M_data)));
	105	__x = __binary_op(__x, _Base::template _M_make_simd<_Tp, _Np>(
	106	__vector_permute<3, 2, 1, 0, 7, 6, 5, 4>(__x._M_data)));
	107	__x = __binary_op(__x, _Base::template _M_make_simd<_Tp, _Np>(
	108	__vector_permute<7, 6, 5, 4, 3, 2, 1, 0>(__x._M_data)));
	109	return __x[0];
	110	}
	111	else if constexpr (_Np == 4)
	112	{
	113	__x = __binary_op(__x, _Base::template _M_make_simd<_Tp, _Np>(
	114	__vector_permute<1, 0, 3, 2>(__x._M_data)));
	115	__x = __binary_op(__x, _Base::template _M_make_simd<_Tp, _Np>(
	116	__vector_permute<3, 2, 1, 0>(__x._M_data)));
	117	return __x[0];
	118	}
	119	else if constexpr (_Np == 2)
	120	{
	121	__x = __binary_op(__x, _Base::template _M_make_simd<_Tp, _Np>(
	122	__vector_permute<1, 0>(__x._M_data)));
	123	return __x[0];
	124	}
2bcceb6f	125	}
b0a483b0	126	return _Base::_S_reduce(__x, static_cast<_BinaryOperation&&>(__binary_op));
2bcceb6f MK	127	}
	128
	129	// }}}
	130	// math {{{
	131	// _S_sqrt {{{
	132	template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
b31186e5 MK	133	_GLIBCXX_SIMD_INTRINSIC static _Tp
b31186e5 MK	134	_S_sqrt(_Tp __x)
2bcceb6f MK	135	{
	136	if constexpr (__have_neon_a64)
	137	{
	138	const auto __intrin = __to_intrin(__x);
	139	if constexpr (_TVT::template _S_is<float, 2>)
	140	return vsqrt_f32(__intrin);
	141	else if constexpr (_TVT::template _S_is<float, 4>)
	142	return vsqrtq_f32(__intrin);
	143	else if constexpr (_TVT::template _S_is<double, 1>)
	144	return vsqrt_f64(__intrin);
	145	else if constexpr (_TVT::template _S_is<double, 2>)
	146	return vsqrtq_f64(__intrin);
	147	else
	148	__assert_unreachable<_Tp>();
	149	}
	150	else
	151	return _Base::_S_sqrt(__x);
	152	}
	153
	154	// }}}
	155	// _S_trunc {{{
	156	template <typename _TW, typename _TVT = _VectorTraits<_TW>>
b31186e5 MK	157	_GLIBCXX_SIMD_INTRINSIC static _TW
b31186e5 MK	158	_S_trunc(_TW __x)
2bcceb6f MK	159	{
	160	using _Tp = typename _TVT::value_type;
	161	if constexpr (__have_neon_a32)
	162	{
	163	const auto __intrin = __to_intrin(__x);
	164	if constexpr (_TVT::template _S_is<float, 2>)
	165	return vrnd_f32(__intrin);
	166	else if constexpr (_TVT::template _S_is<float, 4>)
	167	return vrndq_f32(__intrin);
	168	else if constexpr (_TVT::template _S_is<double, 1>)
	169	return vrnd_f64(__intrin);
	170	else if constexpr (_TVT::template _S_is<double, 2>)
	171	return vrndq_f64(__intrin);
	172	else
	173	__assert_unreachable<_Tp>();
	174	}
	175	else if constexpr (is_same_v<_Tp, float>)
	176	{
	177	auto __intrin = __to_intrin(__x);
	178	if constexpr (sizeof(__x) == 16)
	179	__intrin = vcvtq_f32_s32(vcvtq_s32_f32(__intrin));
	180	else
	181	__intrin = vcvt_f32_s32(vcvt_s32_f32(__intrin));
	182	return _Base::_S_abs(__x)._M_data < 0x1p23f
	183	? __vector_bitcast<float>(__intrin)
	184	: __x._M_data;
	185	}
	186	else
	187	return _Base::_S_trunc(__x);
	188	}
	189
	190	// }}}
	191	// _S_round {{{
	192	template <typename _Tp, size_t _Np>
	193	_GLIBCXX_SIMD_INTRINSIC static _SimdWrapper<_Tp, _Np>
	194	_S_round(_SimdWrapper<_Tp, _Np> __x)
	195	{
	196	if constexpr (__have_neon_a32)
	197	{
	198	const auto __intrin = __to_intrin(__x);
	199	if constexpr (sizeof(_Tp) == 4 && sizeof(__x) == 8)
	200	return vrnda_f32(__intrin);
	201	else if constexpr (sizeof(_Tp) == 4 && sizeof(__x) == 16)
	202	return vrndaq_f32(__intrin);
	203	else if constexpr (sizeof(_Tp) == 8 && sizeof(__x) == 8)
	204	return vrnda_f64(__intrin);
	205	else if constexpr (sizeof(_Tp) == 8 && sizeof(__x) == 16)
	206	return vrndaq_f64(__intrin);
	207	else
	208	__assert_unreachable<_Tp>();
	209	}
	210	else
	211	return _Base::_S_round(__x);
	212	}
	213
	214	// }}}
	215	// _S_floor {{{
	216	template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
b31186e5 MK	217	_GLIBCXX_SIMD_INTRINSIC static _Tp
b31186e5 MK	218	_S_floor(_Tp __x)
2bcceb6f MK	219	{
	220	if constexpr (__have_neon_a32)
	221	{
	222	const auto __intrin = __to_intrin(__x);
	223	if constexpr (_TVT::template _S_is<float, 2>)
	224	return vrndm_f32(__intrin);
	225	else if constexpr (_TVT::template _S_is<float, 4>)
	226	return vrndmq_f32(__intrin);
	227	else if constexpr (_TVT::template _S_is<double, 1>)
	228	return vrndm_f64(__intrin);
	229	else if constexpr (_TVT::template _S_is<double, 2>)
	230	return vrndmq_f64(__intrin);
	231	else
	232	__assert_unreachable<_Tp>();
	233	}
	234	else
	235	return _Base::_S_floor(__x);
	236	}
	237
	238	// }}}
	239	// _S_ceil {{{
	240	template <typename _Tp, typename _TVT = _VectorTraits<_Tp>>
b31186e5 MK	241	_GLIBCXX_SIMD_INTRINSIC static _Tp
b31186e5 MK	242	_S_ceil(_Tp __x)
2bcceb6f MK	243	{
	244	if constexpr (__have_neon_a32)
	245	{
	246	const auto __intrin = __to_intrin(__x);
	247	if constexpr (_TVT::template _S_is<float, 2>)
	248	return vrndp_f32(__intrin);
	249	else if constexpr (_TVT::template _S_is<float, 4>)
	250	return vrndpq_f32(__intrin);
	251	else if constexpr (_TVT::template _S_is<double, 1>)
	252	return vrndp_f64(__intrin);
	253	else if constexpr (_TVT::template _S_is<double, 2>)
	254	return vrndpq_f64(__intrin);
	255	else
	256	__assert_unreachable<_Tp>();
	257	}
	258	else
	259	return _Base::_S_ceil(__x);
	260	}
	261
	262	//}}} }}}
	263	}; // }}}
	264	// _MaskImplNeonMixin {{{
	265	struct _MaskImplNeonMixin
	266	{
	267	using _Base = _MaskImplBuiltinMixin;
	268
	269	template <typename _Tp, size_t _Np>
	270	_GLIBCXX_SIMD_INTRINSIC static constexpr _SanitizedBitMask<_Np>
	271	_S_to_bits(_SimdWrapper<_Tp, _Np> __x)
	272	{
	273	if (__builtin_is_constant_evaluated())
	274	return _Base::_S_to_bits(__x);
	275
	276	using _I = __int_for_sizeof_t<_Tp>;
	277	if constexpr (sizeof(__x) == 16)
	278	{
	279	auto __asint = __vector_bitcast<_I>(__x);
	280	#ifdef __aarch64__
	281	[[maybe_unused]] constexpr auto __zero = decltype(__asint)();
	282	#else
	283	[[maybe_unused]] constexpr auto __zero = decltype(__lo64(__asint))();
	284	#endif
	285	if constexpr (sizeof(_Tp) == 1)
	286	{
	287	constexpr auto __bitsel
	288	= __generate_from_n_evaluations<16, __vector_type_t<_I, 16>>(
53b55701	289	[&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2bcceb6f MK	290	return static_cast<_I>(
	291	__i < _Np ? (__i < 8 ? 1 << __i : 1 << (__i - 8)) : 0);
	292	});
	293	__asint &= __bitsel;
	294	#ifdef __aarch64__
	295	return __vector_bitcast<_UShort>(
	296	vpaddq_s8(vpaddq_s8(vpaddq_s8(__asint, __zero), __zero),
	297	__zero))[0];
	298	#else
	299	return __vector_bitcast<_UShort>(
	300	vpadd_s8(vpadd_s8(vpadd_s8(__lo64(__asint), __hi64(__asint)),
	301	__zero),
	302	__zero))[0];
	303	#endif
	304	}
	305	else if constexpr (sizeof(_Tp) == 2)
	306	{
	307	constexpr auto __bitsel
	308	= __generate_from_n_evaluations<8, __vector_type_t<_I, 8>>(
53b55701	309	[&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2bcceb6f MK	310	return static_cast<_I>(__i < _Np ? 1 << __i : 0);
	311	});
	312	__asint &= __bitsel;
	313	#ifdef __aarch64__
59887657	314	return vaddvq_s16(__asint);
2bcceb6f MK	315	#else
	316	return vpadd_s16(
	317	vpadd_s16(vpadd_s16(__lo64(__asint), __hi64(__asint)), __zero),
	318	__zero)[0];
	319	#endif
	320	}
	321	else if constexpr (sizeof(_Tp) == 4)
	322	{
	323	constexpr auto __bitsel
	324	= __generate_from_n_evaluations<4, __vector_type_t<_I, 4>>(
53b55701	325	[&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2bcceb6f MK	326	return static_cast<_I>(__i < _Np ? 1 << __i : 0);
	327	});
	328	__asint &= __bitsel;
	329	#ifdef __aarch64__
59887657	330	return vaddvq_s32(__asint);
2bcceb6f MK	331	#else
	332	return vpadd_s32(vpadd_s32(__lo64(__asint), __hi64(__asint)),
	333	__zero)[0];
	334	#endif
	335	}
	336	else if constexpr (sizeof(_Tp) == 8)
	337	return (__asint[0] & 1) \| (__asint[1] & 2);
	338	else
	339	__assert_unreachable<_Tp>();
	340	}
	341	else if constexpr (sizeof(__x) == 8)
	342	{
	343	auto __asint = __vector_bitcast<_I>(__x);
	344	[[maybe_unused]] constexpr auto __zero = decltype(__asint)();
	345	if constexpr (sizeof(_Tp) == 1)
	346	{
	347	constexpr auto __bitsel
	348	= __generate_from_n_evaluations<8, __vector_type_t<_I, 8>>(
53b55701	349	[&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2bcceb6f MK	350	return static_cast<_I>(__i < _Np ? 1 << __i : 0);
	351	});
	352	__asint &= __bitsel;
59887657	353	#ifdef __aarch64__
	354	return vaddv_s8(__asint);
	355	#else
2bcceb6f MK	356	return vpadd_s8(vpadd_s8(vpadd_s8(__asint, __zero), __zero),
2bcceb6f MK	357	__zero)[0];
59887657	358	#endif
2bcceb6f MK	359	}
	360	else if constexpr (sizeof(_Tp) == 2)
	361	{
	362	constexpr auto __bitsel
	363	= __generate_from_n_evaluations<4, __vector_type_t<_I, 4>>(
53b55701	364	[&](auto __i) _GLIBCXX_SIMD_ALWAYS_INLINE_LAMBDA {
2bcceb6f MK	365	return static_cast<_I>(__i < _Np ? 1 << __i : 0);
	366	});
	367	__asint &= __bitsel;
59887657	368	#ifdef __aarch64__
	369	return vaddv_s16(__asint);
	370	#else
2bcceb6f	371	return vpadd_s16(vpadd_s16(__asint, __zero), __zero)[0];
59887657	372	#endif
2bcceb6f MK	373	}
	374	else if constexpr (sizeof(_Tp) == 4)
	375	{
	376	__asint &= __make_vector<_I>(0x1, 0x2);
59887657	377	#ifdef __aarch64__
	378	return vaddv_s32(__asint);
	379	#else
2bcceb6f	380	return vpadd_s32(__asint, __zero)[0];
59887657	381	#endif
2bcceb6f MK	382	}
	383	else
	384	__assert_unreachable<_Tp>();
	385	}
	386	else
	387	return _Base::_S_to_bits(__x);
	388	}
	389	};
	390
	391	// }}}
	392	// _MaskImplNeon {{{
52d28210	393	template <typename _Abi, typename>
2bcceb6f MK	394	struct _MaskImplNeon : _MaskImplNeonMixin, _MaskImplBuiltin<_Abi>
	395	{
	396	using _MaskImplBuiltinMixin::_S_to_maskvector;
	397	using _MaskImplNeonMixin::_S_to_bits;
	398	using _Base = _MaskImplBuiltin<_Abi>;
	399	using _Base::_S_convert;
	400
	401	// _S_all_of {{{
	402	template <typename _Tp>
b31186e5 MK	403	_GLIBCXX_SIMD_INTRINSIC static bool
b31186e5 MK	404	_S_all_of(simd_mask<_Tp, _Abi> __k)
2bcceb6f MK	405	{
	406	const auto __kk
	407	= __vector_bitcast<char>(__k._M_data)
	408	\| ~__vector_bitcast<char>(_Abi::template _S_implicit_mask<_Tp>());
	409	if constexpr (sizeof(__k) == 16)
	410	{
	411	const auto __x = __vector_bitcast<long long>(__kk);
	412	return __x[0] + __x[1] == -2;
	413	}
	414	else if constexpr (sizeof(__k) <= 8)
	415	return __bit_cast<__int_for_sizeof_t<decltype(__kk)>>(__kk) == -1;
	416	else
	417	__assert_unreachable<_Tp>();
	418	}
	419
	420	// }}}
	421	// _S_any_of {{{
	422	template <typename _Tp>
b31186e5 MK	423	_GLIBCXX_SIMD_INTRINSIC static bool
b31186e5 MK	424	_S_any_of(simd_mask<_Tp, _Abi> __k)
2bcceb6f MK	425	{
	426	const auto __kk
	427	= __vector_bitcast<char>(__k._M_data)
	428	\| ~__vector_bitcast<char>(_Abi::template _S_implicit_mask<_Tp>());
	429	if constexpr (sizeof(__k) == 16)
	430	{
	431	const auto __x = __vector_bitcast<long long>(__kk);
	432	return (__x[0] \| __x[1]) != 0;
	433	}
	434	else if constexpr (sizeof(__k) <= 8)
	435	return __bit_cast<__int_for_sizeof_t<decltype(__kk)>>(__kk) != 0;
	436	else
	437	__assert_unreachable<_Tp>();
	438	}
	439
	440	// }}}
	441	// _S_none_of {{{
	442	template <typename _Tp>
b31186e5 MK	443	_GLIBCXX_SIMD_INTRINSIC static bool
b31186e5 MK	444	_S_none_of(simd_mask<_Tp, _Abi> __k)
2bcceb6f MK	445	{
	446	const auto __kk = _Abi::_S_masked(__k._M_data);
	447	if constexpr (sizeof(__k) == 16)
	448	{
	449	const auto __x = __vector_bitcast<long long>(__kk);
	450	return (__x[0] \| __x[1]) == 0;
	451	}
	452	else if constexpr (sizeof(__k) <= 8)
	453	return __bit_cast<__int_for_sizeof_t<decltype(__kk)>>(__kk) == 0;
	454	else
	455	__assert_unreachable<_Tp>();
	456	}
	457
	458	// }}}
	459	// _S_some_of {{{
	460	template <typename _Tp>
	461	_GLIBCXX_SIMD_INTRINSIC static bool _S_some_of(simd_mask<_Tp, _Abi> __k)
	462	{
	463	if constexpr (sizeof(__k) <= 8)
	464	{
	465	const auto __kk = __vector_bitcast<char>(__k._M_data)
	466	\| ~__vector_bitcast<char>(
	467	_Abi::template _S_implicit_mask<_Tp>());
	468	using _Up = make_unsigned_t<__int_for_sizeof_t<decltype(__kk)>>;
	469	return __bit_cast<_Up>(__kk) + 1 > 1;
	470	}
	471	else
	472	return _Base::_S_some_of(__k);
	473	}
	474
	475	// }}}
	476	// _S_popcount {{{
	477	template <typename _Tp>
b31186e5 MK	478	_GLIBCXX_SIMD_INTRINSIC static int
b31186e5 MK	479	_S_popcount(simd_mask<_Tp, _Abi> __k)
2bcceb6f MK	480	{
	481	if constexpr (sizeof(_Tp) == 1)
	482	{
	483	const auto __s8 = __vector_bitcast<_SChar>(__k._M_data);
	484	int8x8_t __tmp = __lo64(__s8) + __hi64z(__s8);
	485	return -vpadd_s8(vpadd_s8(vpadd_s8(__tmp, int8x8_t()), int8x8_t()),
	486	int8x8_t())[0];
	487	}
	488	else if constexpr (sizeof(_Tp) == 2)
	489	{
	490	const auto __s16 = __vector_bitcast<short>(__k._M_data);
	491	int16x4_t __tmp = __lo64(__s16) + __hi64z(__s16);
	492	return -vpadd_s16(vpadd_s16(__tmp, int16x4_t()), int16x4_t())[0];
	493	}
	494	else if constexpr (sizeof(_Tp) == 4)
	495	{
	496	const auto __s32 = __vector_bitcast<int>(__k._M_data);
	497	int32x2_t __tmp = __lo64(__s32) + __hi64z(__s32);
	498	return -vpadd_s32(__tmp, int32x2_t())[0];
	499	}
	500	else if constexpr (sizeof(_Tp) == 8)
	501	{
	502	static_assert(sizeof(__k) == 16);
	503	const auto __s64 = __vector_bitcast<long>(__k._M_data);
	504	return -(__s64[0] + __s64[1]);
	505	}
	506	}
	507
	508	// }}}
	509	// _S_find_first_set {{{
	510	template <typename _Tp>
	511	_GLIBCXX_SIMD_INTRINSIC static int
	512	_S_find_first_set(simd_mask<_Tp, _Abi> __k)
	513	{
	514	// TODO: the _Base implementation is not optimal for NEON
	515	return _Base::_S_find_first_set(__k);
	516	}
	517
	518	// }}}
	519	// _S_find_last_set {{{
	520	template <typename _Tp>
	521	_GLIBCXX_SIMD_INTRINSIC static int
	522	_S_find_last_set(simd_mask<_Tp, _Abi> __k)
	523	{
	524	// TODO: the _Base implementation is not optimal for NEON
	525	return _Base::_S_find_last_set(__k);
	526	}
	527
	528	// }}}
	529	}; // }}}
	530
	531	_GLIBCXX_SIMD_END_NAMESPACE
	532	#endif // __cplusplus >= 201703L
	533	#endif // _GLIBCXX_EXPERIMENTAL_SIMD_NEON_H_
	534	// vim: foldmethod=marker sw=2 noet ts=8 sts=2 tw=80