]>
Commit | Line | Data |
---|---|---|
2bcceb6f MK |
1 | // Generic simd conversions -*- C++ -*- |
2 | ||
7adcbafe | 3 | // Copyright (C) 2020-2022 Free Software Foundation, Inc. |
2bcceb6f MK |
4 | // |
5 | // This file is part of the GNU ISO C++ Library. This library is free | |
6 | // software; you can redistribute it and/or modify it under the | |
7 | // terms of the GNU General Public License as published by the | |
8 | // Free Software Foundation; either version 3, or (at your option) | |
9 | // any later version. | |
10 | ||
11 | // This library is distributed in the hope that it will be useful, | |
12 | // but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
14 | // GNU General Public License for more details. | |
15 | ||
16 | // Under Section 7 of GPL version 3, you are granted additional | |
17 | // permissions described in the GCC Runtime Library Exception, version | |
18 | // 3.1, as published by the Free Software Foundation. | |
19 | ||
20 | // You should have received a copy of the GNU General Public License and | |
21 | // a copy of the GCC Runtime Library Exception along with this program; | |
22 | // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see | |
23 | // <http://www.gnu.org/licenses/>. | |
24 | ||
25 | #ifndef _GLIBCXX_EXPERIMENTAL_SIMD_CONVERTER_H_ | |
26 | #define _GLIBCXX_EXPERIMENTAL_SIMD_CONVERTER_H_ | |
27 | ||
28 | #if __cplusplus >= 201703L | |
29 | ||
30 | _GLIBCXX_SIMD_BEGIN_NAMESPACE | |
31 | // _SimdConverter scalar -> scalar {{{ | |
32 | template <typename _From, typename _To> | |
33 | struct _SimdConverter<_From, simd_abi::scalar, _To, simd_abi::scalar, | |
34 | enable_if_t<!is_same_v<_From, _To>>> | |
35 | { | |
36 | _GLIBCXX_SIMD_INTRINSIC constexpr _To operator()(_From __a) const noexcept | |
37 | { return static_cast<_To>(__a); } | |
38 | }; | |
39 | ||
40 | // }}} | |
41 | // _SimdConverter scalar -> "native" {{{ | |
42 | template <typename _From, typename _To, typename _Abi> | |
43 | struct _SimdConverter<_From, simd_abi::scalar, _To, _Abi, | |
44 | enable_if_t<!is_same_v<_Abi, simd_abi::scalar>>> | |
45 | { | |
46 | using _Ret = typename _Abi::template __traits<_To>::_SimdMember; | |
47 | ||
48 | template <typename... _More> | |
49 | _GLIBCXX_SIMD_INTRINSIC constexpr _Ret | |
50 | operator()(_From __a, _More... __more) const noexcept | |
51 | { | |
52 | static_assert(sizeof...(_More) + 1 == _Abi::template _S_size<_To>); | |
53 | static_assert(conjunction_v<is_same<_From, _More>...>); | |
54 | return __make_vector<_To>(__a, __more...); | |
55 | } | |
56 | }; | |
57 | ||
58 | // }}} | |
59 | // _SimdConverter "native 1" -> "native 2" {{{ | |
60 | template <typename _From, typename _To, typename _AFrom, typename _ATo> | |
61 | struct _SimdConverter< | |
62 | _From, _AFrom, _To, _ATo, | |
63 | enable_if_t<!disjunction_v< | |
64 | __is_fixed_size_abi<_AFrom>, __is_fixed_size_abi<_ATo>, | |
65 | is_same<_AFrom, simd_abi::scalar>, is_same<_ATo, simd_abi::scalar>, | |
66 | conjunction<is_same<_From, _To>, is_same<_AFrom, _ATo>>>>> | |
67 | { | |
68 | using _Arg = typename _AFrom::template __traits<_From>::_SimdMember; | |
69 | using _Ret = typename _ATo::template __traits<_To>::_SimdMember; | |
70 | using _V = __vector_type_t<_To, simd_size_v<_To, _ATo>>; | |
71 | ||
72 | template <typename... _More> | |
73 | _GLIBCXX_SIMD_INTRINSIC constexpr _Ret | |
74 | operator()(_Arg __a, _More... __more) const noexcept | |
75 | { return __vector_convert<_V>(__a, __more...); } | |
76 | }; | |
77 | ||
78 | // }}} | |
79 | // _SimdConverter scalar -> fixed_size<1> {{{1 | |
80 | template <typename _From, typename _To> | |
81 | struct _SimdConverter<_From, simd_abi::scalar, _To, simd_abi::fixed_size<1>, | |
82 | void> | |
83 | { | |
84 | _GLIBCXX_SIMD_INTRINSIC constexpr _SimdTuple<_To, simd_abi::scalar> | |
85 | operator()(_From __x) const noexcept | |
86 | { return {static_cast<_To>(__x)}; } | |
87 | }; | |
88 | ||
89 | // _SimdConverter fixed_size<1> -> scalar {{{1 | |
90 | template <typename _From, typename _To> | |
91 | struct _SimdConverter<_From, simd_abi::fixed_size<1>, _To, simd_abi::scalar, | |
92 | void> | |
93 | { | |
94 | _GLIBCXX_SIMD_INTRINSIC constexpr _To | |
95 | operator()(_SimdTuple<_From, simd_abi::scalar> __x) const noexcept | |
96 | { return {static_cast<_To>(__x.first)}; } | |
97 | }; | |
98 | ||
99 | // _SimdConverter fixed_size<_Np> -> fixed_size<_Np> {{{1 | |
100 | template <typename _From, typename _To, int _Np> | |
101 | struct _SimdConverter<_From, simd_abi::fixed_size<_Np>, _To, | |
102 | simd_abi::fixed_size<_Np>, | |
103 | enable_if_t<!is_same_v<_From, _To>>> | |
104 | { | |
105 | using _Ret = __fixed_size_storage_t<_To, _Np>; | |
106 | using _Arg = __fixed_size_storage_t<_From, _Np>; | |
107 | ||
108 | _GLIBCXX_SIMD_INTRINSIC constexpr _Ret | |
109 | operator()(const _Arg& __x) const noexcept | |
110 | { | |
111 | if constexpr (is_same_v<_From, _To>) | |
112 | return __x; | |
113 | ||
114 | // special case (optimize) int signedness casts | |
115 | else if constexpr (sizeof(_From) == sizeof(_To) | |
116 | && is_integral_v<_From> && is_integral_v<_To>) | |
117 | return __bit_cast<_Ret>(__x); | |
118 | ||
119 | // special case if all ABI tags in _Ret are scalar | |
120 | else if constexpr (__is_scalar_abi<typename _Ret::_FirstAbi>()) | |
121 | { | |
122 | return __call_with_subscripts( | |
123 | __x, make_index_sequence<_Np>(), | |
124 | [](auto... __values) constexpr->_Ret { | |
125 | return __make_simd_tuple<_To, decltype((void) __values, | |
126 | simd_abi::scalar())...>( | |
127 | static_cast<_To>(__values)...); | |
128 | }); | |
129 | } | |
130 | ||
131 | // from one vector to one vector | |
132 | else if constexpr (_Arg::_S_first_size == _Ret::_S_first_size) | |
133 | { | |
134 | _SimdConverter<_From, typename _Arg::_FirstAbi, _To, | |
135 | typename _Ret::_FirstAbi> | |
136 | __native_cvt; | |
137 | if constexpr (_Arg::_S_tuple_size == 1) | |
138 | return {__native_cvt(__x.first)}; | |
139 | else | |
140 | { | |
141 | constexpr size_t _NRemain = _Np - _Arg::_S_first_size; | |
142 | _SimdConverter<_From, simd_abi::fixed_size<_NRemain>, _To, | |
143 | simd_abi::fixed_size<_NRemain>> | |
144 | __remainder_cvt; | |
145 | return {__native_cvt(__x.first), __remainder_cvt(__x.second)}; | |
146 | } | |
147 | } | |
148 | ||
149 | // from one vector to multiple vectors | |
150 | else if constexpr (_Arg::_S_first_size > _Ret::_S_first_size) | |
151 | { | |
152 | const auto __multiple_return_chunks | |
153 | = __convert_all<__vector_type_t<_To, _Ret::_S_first_size>>( | |
154 | __x.first); | |
155 | constexpr auto __converted = __multiple_return_chunks.size() | |
156 | * _Ret::_FirstAbi::template _S_size<_To>; | |
157 | constexpr auto __remaining = _Np - __converted; | |
158 | if constexpr (_Arg::_S_tuple_size == 1 && __remaining == 0) | |
159 | return __to_simd_tuple<_To, _Np>(__multiple_return_chunks); | |
160 | else if constexpr (_Arg::_S_tuple_size == 1) | |
161 | { // e.g. <int, 3> -> <double, 2, 1> or <short, 7> -> <double, 4, 2, | |
162 | // 1> | |
163 | using _RetRem | |
164 | = __remove_cvref_t<decltype(__simd_tuple_pop_front<__converted>( | |
165 | _Ret()))>; | |
166 | const auto __return_chunks2 | |
167 | = __convert_all<__vector_type_t<_To, _RetRem::_S_first_size>, 0, | |
168 | __converted>(__x.first); | |
169 | constexpr auto __converted2 | |
170 | = __converted | |
171 | + __return_chunks2.size() * _RetRem::_S_first_size; | |
172 | if constexpr (__converted2 == _Np) | |
173 | return __to_simd_tuple<_To, _Np>(__multiple_return_chunks, | |
174 | __return_chunks2); | |
175 | else | |
176 | { | |
177 | using _RetRem2 = __remove_cvref_t< | |
178 | decltype(__simd_tuple_pop_front<__return_chunks2.size() | |
179 | * _RetRem::_S_first_size>( | |
180 | _RetRem()))>; | |
181 | const auto __return_chunks3 = __convert_all< | |
182 | __vector_type_t<_To, _RetRem2::_S_first_size>, 0, | |
183 | __converted2>(__x.first); | |
184 | constexpr auto __converted3 | |
185 | = __converted2 | |
186 | + __return_chunks3.size() * _RetRem2::_S_first_size; | |
187 | if constexpr (__converted3 == _Np) | |
188 | return __to_simd_tuple<_To, _Np>(__multiple_return_chunks, | |
189 | __return_chunks2, | |
190 | __return_chunks3); | |
191 | else | |
192 | { | |
193 | using _RetRem3 | |
194 | = __remove_cvref_t<decltype(__simd_tuple_pop_front< | |
195 | __return_chunks3.size() | |
196 | * _RetRem2::_S_first_size>( | |
197 | _RetRem2()))>; | |
198 | const auto __return_chunks4 = __convert_all< | |
199 | __vector_type_t<_To, _RetRem3::_S_first_size>, 0, | |
200 | __converted3>(__x.first); | |
201 | constexpr auto __converted4 | |
202 | = __converted3 | |
203 | + __return_chunks4.size() * _RetRem3::_S_first_size; | |
204 | if constexpr (__converted4 == _Np) | |
205 | return __to_simd_tuple<_To, _Np>( | |
206 | __multiple_return_chunks, __return_chunks2, | |
207 | __return_chunks3, __return_chunks4); | |
208 | else | |
209 | __assert_unreachable<_To>(); | |
210 | } | |
211 | } | |
212 | } | |
213 | else | |
214 | { | |
215 | constexpr size_t _NRemain = _Np - _Arg::_S_first_size; | |
216 | _SimdConverter<_From, simd_abi::fixed_size<_NRemain>, _To, | |
217 | simd_abi::fixed_size<_NRemain>> | |
218 | __remainder_cvt; | |
219 | return __simd_tuple_concat( | |
220 | __to_simd_tuple<_To, _Arg::_S_first_size>( | |
221 | __multiple_return_chunks), | |
222 | __remainder_cvt(__x.second)); | |
223 | } | |
224 | } | |
225 | ||
226 | // from multiple vectors to one vector | |
227 | // _Arg::_S_first_size < _Ret::_S_first_size | |
228 | // a) heterogeneous input at the end of the tuple (possible with partial | |
229 | // native registers in _Ret) | |
230 | else if constexpr (_Ret::_S_tuple_size == 1 | |
231 | && _Np % _Arg::_S_first_size != 0) | |
232 | { | |
233 | static_assert(_Ret::_FirstAbi::template _S_is_partial<_To>); | |
234 | return _Ret{__generate_from_n_evaluations< | |
235 | _Np, typename _VectorTraits<typename _Ret::_FirstType>::type>( | |
236 | [&](auto __i) { return static_cast<_To>(__x[__i]); })}; | |
237 | } | |
238 | else | |
239 | { | |
240 | static_assert(_Arg::_S_tuple_size > 1); | |
241 | constexpr auto __n | |
242 | = __div_roundup(_Ret::_S_first_size, _Arg::_S_first_size); | |
243 | return __call_with_n_evaluations<__n>( | |
244 | [&__x](auto... __uncvted) { | |
245 | // assuming _Arg Abi tags for all __i are _Arg::_FirstAbi | |
246 | _SimdConverter<_From, typename _Arg::_FirstAbi, _To, | |
247 | typename _Ret::_FirstAbi> | |
248 | __native_cvt; | |
249 | if constexpr (_Ret::_S_tuple_size == 1) | |
250 | return _Ret{__native_cvt(__uncvted...)}; | |
251 | else | |
252 | return _Ret{ | |
253 | __native_cvt(__uncvted...), | |
254 | _SimdConverter< | |
255 | _From, simd_abi::fixed_size<_Np - _Ret::_S_first_size>, _To, | |
256 | simd_abi::fixed_size<_Np - _Ret::_S_first_size>>()( | |
257 | __simd_tuple_pop_front<_Ret::_S_first_size>(__x))}; | |
258 | }, | |
259 | [&__x](auto __i) { return __get_tuple_at<__i>(__x); }); | |
260 | } | |
261 | } | |
262 | }; | |
263 | ||
264 | // _SimdConverter "native" -> fixed_size<_Np> {{{1 | |
265 | // i.e. 1 register to ? registers | |
266 | template <typename _From, typename _Ap, typename _To, int _Np> | |
267 | struct _SimdConverter<_From, _Ap, _To, simd_abi::fixed_size<_Np>, | |
268 | enable_if_t<!__is_fixed_size_abi_v<_Ap>>> | |
269 | { | |
270 | static_assert( | |
271 | _Np == simd_size_v<_From, _Ap>, | |
272 | "_SimdConverter to fixed_size only works for equal element counts"); | |
273 | ||
274 | using _Ret = __fixed_size_storage_t<_To, _Np>; | |
275 | ||
276 | _GLIBCXX_SIMD_INTRINSIC constexpr _Ret | |
277 | operator()(typename _SimdTraits<_From, _Ap>::_SimdMember __x) const noexcept | |
278 | { | |
279 | if constexpr (_Ret::_S_tuple_size == 1) | |
280 | return {__vector_convert<typename _Ret::_FirstType::_BuiltinType>(__x)}; | |
281 | else | |
282 | { | |
283 | using _FixedNp = simd_abi::fixed_size<_Np>; | |
284 | _SimdConverter<_From, _FixedNp, _To, _FixedNp> __fixed_cvt; | |
285 | using _FromFixedStorage = __fixed_size_storage_t<_From, _Np>; | |
286 | if constexpr (_FromFixedStorage::_S_tuple_size == 1) | |
287 | return __fixed_cvt(_FromFixedStorage{__x}); | |
288 | else if constexpr (_FromFixedStorage::_S_tuple_size == 2) | |
289 | { | |
290 | _FromFixedStorage __tmp; | |
291 | static_assert(sizeof(__tmp) <= sizeof(__x)); | |
292 | __builtin_memcpy(&__tmp.first, &__x, sizeof(__tmp.first)); | |
293 | __builtin_memcpy(&__tmp.second.first, | |
294 | reinterpret_cast<const char*>(&__x) | |
295 | + sizeof(__tmp.first), | |
296 | sizeof(__tmp.second.first)); | |
297 | return __fixed_cvt(__tmp); | |
298 | } | |
299 | else | |
300 | __assert_unreachable<_From>(); | |
301 | } | |
302 | } | |
303 | }; | |
304 | ||
305 | // _SimdConverter fixed_size<_Np> -> "native" {{{1 | |
306 | // i.e. ? register to 1 registers | |
307 | template <typename _From, int _Np, typename _To, typename _Ap> | |
308 | struct _SimdConverter<_From, simd_abi::fixed_size<_Np>, _To, _Ap, | |
309 | enable_if_t<!__is_fixed_size_abi_v<_Ap>>> | |
310 | { | |
311 | static_assert( | |
312 | _Np == simd_size_v<_To, _Ap>, | |
313 | "_SimdConverter to fixed_size only works for equal element counts"); | |
314 | ||
315 | using _Arg = __fixed_size_storage_t<_From, _Np>; | |
316 | ||
317 | _GLIBCXX_SIMD_INTRINSIC constexpr | |
318 | typename _SimdTraits<_To, _Ap>::_SimdMember | |
343f01f4 | 319 | operator()(const _Arg& __x) const noexcept |
2bcceb6f MK |
320 | { |
321 | if constexpr (_Arg::_S_tuple_size == 1) | |
322 | return __vector_convert<__vector_type_t<_To, _Np>>(__x.first); | |
323 | else if constexpr (_Arg::_S_is_homogeneous) | |
324 | return __call_with_n_evaluations<_Arg::_S_tuple_size>( | |
325 | [](auto... __members) { | |
326 | if constexpr ((is_convertible_v<decltype(__members), _To> && ...)) | |
327 | return __vector_type_t<_To, _Np>{static_cast<_To>(__members)...}; | |
328 | else | |
329 | return __vector_convert<__vector_type_t<_To, _Np>>(__members...); | |
330 | }, | |
331 | [&](auto __i) { return __get_tuple_at<__i>(__x); }); | |
332 | else if constexpr (__fixed_size_storage_t<_To, _Np>::_S_tuple_size == 1) | |
333 | { | |
334 | _SimdConverter<_From, simd_abi::fixed_size<_Np>, _To, | |
335 | simd_abi::fixed_size<_Np>> | |
336 | __fixed_cvt; | |
337 | return __fixed_cvt(__x).first; | |
338 | } | |
339 | else | |
340 | { | |
341 | const _SimdWrapper<_From, _Np> __xv | |
342 | = __generate_from_n_evaluations<_Np, __vector_type_t<_From, _Np>>( | |
343 | [&](auto __i) { return __x[__i]; }); | |
344 | return __vector_convert<__vector_type_t<_To, _Np>>(__xv); | |
345 | } | |
346 | } | |
347 | }; | |
348 | ||
349 | // }}}1 | |
350 | _GLIBCXX_SIMD_END_NAMESPACE | |
351 | #endif // __cplusplus >= 201703L | |
352 | #endif // _GLIBCXX_EXPERIMENTAL_SIMD_CONVERTER_H_ | |
353 | ||
354 | // vim: foldmethod=marker sw=2 noet ts=8 sts=2 tw=80 |