1 /* Function sincos vectorized with AVX-512. KNL and SKX versions.
2 Copyright (C) 2014-2019 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
20 #include "svml_d_trig_data.h"
21 #include "svml_d_wrapper_impl.h"
24 ALGORITHM DESCRIPTION:
26 ( low accuracy ( < 4ulp ) or enhanced performance
27 ( half of correct mantissa ) implementation )
29 Argument representation:
33 sin(arg) = sin(N*Pi + R) = (-1)^N * sin(R)
34 arg + Pi/2 = (N'*Pi + R')
35 cos(arg) = sin(arg+Pi/2) = sin(N'*Pi + R') = (-1)^N' * sin(R')
36 sin(R), sin(R') are approximated by corresponding polynomial. */
39 ENTRY (_ZGVeN8vl8l8_sincos_knl)
40 #ifndef HAVE_AVX512DQ_ASM_SUPPORT
41 WRAPPER_IMPL_AVX512_fFF _ZGVdN4vl8l8_sincos
44 cfi_adjust_cfa_offset (8)
45 cfi_rel_offset (%rbp, 0)
47 cfi_def_cfa_register (%rbp)
50 movq __svml_d_trig_data@GOTPCREL(%rip), %rax
53 vmovups __dSignMask(%rax), %zmm12
54 vmovups __dInvPI(%rax), %zmm5
56 /* ARGUMENT RANGE REDUCTION:
57 Absolute argument: X' = |X| */
58 vpandnq %zmm4, %zmm12, %zmm3
59 vmovups __dPI1_FMA(%rax), %zmm7
60 vmovups __dPI3_FMA(%rax), %zmm9
62 /* SinR = X' - SinN*Pi1 */
65 /* CosR = SinX - CosN*Pi1 */
68 /* SinY = X'*InvPi + RS : right shifter add */
69 vfmadd213pd __dRShifter(%rax), %zmm3, %zmm5
70 vmovups __dC6(%rax), %zmm13
72 /* SinN = Y - RS : right shifter sub */
73 vsubpd __dRShifter(%rax), %zmm5, %zmm1
74 vmovaps %zmm13, %zmm14
76 /* SinSignRes = Y<<63 : shift LSB to MSB place for result sign */
77 vpsllq $63, %zmm5, %zmm2
78 vcmppd $22, __dRangeVal(%rax), %zmm3, %k1
80 /* Update CosRSign and CosSignRes signs */
82 vfnmadd231pd %zmm1, %zmm7, %zmm8
84 /* SinR = SinR - SinN*Pi1 */
85 vfnmadd231pd __dPI2_FMA(%rax), %zmm1, %zmm8
87 /* Sine result sign: SinRSign = SignMask & SinR */
88 vpandq %zmm8, %zmm12, %zmm11
90 /* Set SinRSign to 0.5 */
91 vporq __dOneHalf(%rax), %zmm11, %zmm6
92 vpternlogq $150, %zmm2, %zmm11, %zmm5
94 /* Update sign SinSignRes */
95 vpternlogq $120, %zmm4, %zmm12, %zmm2
97 /* Polynomial approximation */
98 vmovups __dC7(%rax), %zmm11
100 /* CosN = SinN +(-)0.5 */
101 vaddpd %zmm6, %zmm1, %zmm0
103 /* SinR = SinR - SinN*Pi3 */
104 vfnmadd213pd %zmm8, %zmm9, %zmm1
105 vfnmadd231pd %zmm0, %zmm7, %zmm10
108 vmulpd %zmm1, %zmm1, %zmm15
111 CosR = CosR - CosN*Pi2 */
112 vfnmadd231pd __dPI2_FMA(%rax), %zmm0, %zmm10
113 vfmadd231pd __dC7(%rax), %zmm15, %zmm14
115 /* CosR = CosR - CosN*Pi3 */
116 vfnmadd213pd %zmm10, %zmm9, %zmm0
117 vfmadd213pd __dC5(%rax), %zmm15, %zmm14
120 vmulpd %zmm0, %zmm0, %zmm12
121 vfmadd213pd __dC4(%rax), %zmm15, %zmm14
122 vfmadd213pd %zmm13, %zmm12, %zmm11
124 /* SinPoly = C3 + SinR2*(C4 + SinR2*(C5 + SinR2*(C6 + SinR2*C7))) */
125 vfmadd213pd __dC3(%rax), %zmm15, %zmm14
126 vfmadd213pd __dC5(%rax), %zmm12, %zmm11
128 /* SinPoly = C2 + SinR2*SinPoly */
129 vfmadd213pd __dC2(%rax), %zmm15, %zmm14
130 vfmadd213pd __dC4(%rax), %zmm12, %zmm11
132 /* SinPoly = C1 + SinR2*SinPoly */
133 vfmadd213pd __dC1(%rax), %zmm15, %zmm14
135 /* CosPoly = C3 + CosR2*(C4 + CosR2*(C5 + CosR2*(C6 + CosR2*C7))) */
136 vfmadd213pd __dC3(%rax), %zmm12, %zmm11
138 /* SinPoly = SinR2*SinPoly */
139 vmulpd %zmm15, %zmm14, %zmm13
141 /* CosPoly = C2 + CosR2*CosPoly */
142 vfmadd213pd __dC2(%rax), %zmm12, %zmm11
144 /* SinPoly = SinR*SinPoly */
145 vfmadd213pd %zmm1, %zmm1, %zmm13
146 vpbroadcastq %rdx, %zmm1{%k1}{z}
148 /* CosPoly = C1 + CosR2*CosPoly */
149 vfmadd213pd __dC1(%rax), %zmm12, %zmm11
150 vptestmq %zmm1, %zmm1, %k0
153 /* CosPoly = CosR2*CosPoly */
154 vmulpd %zmm12, %zmm11, %zmm14
157 /* CosPoly = CosR*CosPoly */
158 vfmadd213pd %zmm0, %zmm0, %zmm14
160 /* Final reconstruction.
161 Update Sin result's sign */
162 vpxorq %zmm2, %zmm13, %zmm0
164 /* Update Cos result's sign */
165 vpxorq %zmm5, %zmm14, %zmm2
171 vmovups %zmm0, (%rdi)
172 vmovups %zmm2, (%rsi)
174 cfi_def_cfa_register (%rsp)
176 cfi_adjust_cfa_offset (-8)
182 vmovups %zmm4, 1152(%rsp)
183 vmovups %zmm0, 1216(%rsp)
184 vmovups %zmm2, 1280(%rsp)
188 kmovw %k4, 1048(%rsp)
190 kmovw %k5, 1040(%rsp)
191 kmovw %k6, 1032(%rsp)
192 kmovw %k7, 1024(%rsp)
193 vmovups %zmm16, 960(%rsp)
194 vmovups %zmm17, 896(%rsp)
195 vmovups %zmm18, 832(%rsp)
196 vmovups %zmm19, 768(%rsp)
197 vmovups %zmm20, 704(%rsp)
198 vmovups %zmm21, 640(%rsp)
199 vmovups %zmm22, 576(%rsp)
200 vmovups %zmm23, 512(%rsp)
201 vmovups %zmm24, 448(%rsp)
202 vmovups %zmm25, 384(%rsp)
203 vmovups %zmm26, 320(%rsp)
204 vmovups %zmm27, 256(%rsp)
205 vmovups %zmm28, 192(%rsp)
206 vmovups %zmm29, 128(%rsp)
207 vmovups %zmm30, 64(%rsp)
208 vmovups %zmm31, (%rsp)
209 movq %rsi, 1056(%rsp)
210 movq %r12, 1096(%rsp)
211 cfi_offset_rel_rsp (12, 1096)
213 movq %r13, 1088(%rsp)
214 cfi_offset_rel_rsp (13, 1088)
216 movq %r14, 1080(%rsp)
217 cfi_offset_rel_rsp (14, 1080)
219 movq %r15, 1072(%rsp)
220 cfi_offset_rel_rsp (15, 1072)
221 movq %rbx, 1064(%rsp)
241 kmovw 1048(%rsp), %k4
242 movq 1056(%rsp), %rsi
243 kmovw 1040(%rsp), %k5
244 movq 1096(%rsp), %r12
246 kmovw 1032(%rsp), %k6
247 movq 1088(%rsp), %r13
249 kmovw 1024(%rsp), %k7
250 vmovups 960(%rsp), %zmm16
251 vmovups 896(%rsp), %zmm17
252 vmovups 832(%rsp), %zmm18
253 vmovups 768(%rsp), %zmm19
254 vmovups 704(%rsp), %zmm20
255 vmovups 640(%rsp), %zmm21
256 vmovups 576(%rsp), %zmm22
257 vmovups 512(%rsp), %zmm23
258 vmovups 448(%rsp), %zmm24
259 vmovups 384(%rsp), %zmm25
260 vmovups 320(%rsp), %zmm26
261 vmovups 256(%rsp), %zmm27
262 vmovups 192(%rsp), %zmm28
263 vmovups 128(%rsp), %zmm29
264 vmovups 64(%rsp), %zmm30
265 vmovups (%rsp), %zmm31
266 movq 1080(%rsp), %r14
268 movq 1072(%rsp), %r15
270 movq 1064(%rsp), %rbx
271 vmovups 1216(%rsp), %zmm0
272 vmovups 1280(%rsp), %zmm2
279 vmovsd 1160(%rsp,%r15), %xmm0
283 vmovsd %xmm0, 1224(%rsp,%r15)
284 vmovsd 1160(%rsp,%r15), %xmm0
288 vmovsd %xmm0, 1288(%rsp,%r15)
294 vmovsd 1152(%rsp,%r15), %xmm0
298 vmovsd %xmm0, 1216(%rsp,%r15)
299 vmovsd 1152(%rsp,%r15), %xmm0
303 vmovsd %xmm0, 1280(%rsp,%r15)
307 END (_ZGVeN8vl8l8_sincos_knl)
308 libmvec_hidden_def(_ZGVeN8vl8l8_sincos_knl)
310 ENTRY (_ZGVeN8vl8l8_sincos_skx)
311 #ifndef HAVE_AVX512DQ_ASM_SUPPORT
312 WRAPPER_IMPL_AVX512_fFF _ZGVdN4vl8l8_sincos
315 cfi_adjust_cfa_offset (8)
316 cfi_rel_offset (%rbp, 0)
318 cfi_def_cfa_register (%rbp)
321 movq __svml_d_trig_data@GOTPCREL(%rip), %rax
323 vmovups __dSignMask(%rax), %zmm4
324 vmovups __dInvPI(%rax), %zmm9
325 vmovups __dRShifter(%rax), %zmm10
326 vmovups __dPI1_FMA(%rax), %zmm13
327 vmovups __dPI2_FMA(%rax), %zmm14
328 vmovups __dOneHalf(%rax), %zmm11
329 vmovups __dPI3_FMA(%rax), %zmm2
331 /* ARGUMENT RANGE REDUCTION:
332 Absolute argument: X' = |X| */
333 vandnpd %zmm8, %zmm4, %zmm7
335 /* SinY = X'*InvPi + RS : right shifter add */
336 vfmadd213pd %zmm10, %zmm7, %zmm9
337 vcmppd $18, __dRangeVal(%rax), %zmm7, %k1
339 /* SinSignRes = Y<<63 : shift LSB to MSB place for result sign */
340 vpsllq $63, %zmm9, %zmm6
342 /* SinN = Y - RS : right shifter sub */
343 vsubpd %zmm10, %zmm9, %zmm5
344 vmovups __dC5(%rax), %zmm9
345 vmovups __dC4(%rax), %zmm10
347 /* SinR = X' - SinN*Pi1 */
348 vmovaps %zmm7, %zmm15
349 vfnmadd231pd %zmm5, %zmm13, %zmm15
351 /* SinR = SinR - SinN*Pi1 */
352 vfnmadd231pd %zmm5, %zmm14, %zmm15
354 /* Sine result sign: SinRSign = SignMask & SinR */
355 vandpd %zmm15, %zmm4, %zmm1
357 /* Set SinRSign to 0.5 */
358 vorpd %zmm1, %zmm11, %zmm12
359 vmovups __dC3(%rax), %zmm11
361 /* CosN = SinN +(-)0.5 */
362 vaddpd %zmm12, %zmm5, %zmm3
364 /* SinR = SinR - SinN*Pi3 */
365 vfnmadd213pd %zmm15, %zmm2, %zmm5
366 vmovups __dC2(%rax), %zmm12
369 vmulpd %zmm5, %zmm5, %zmm15
371 /* CosR = SinX - CosN*Pi1 */
373 vfnmadd231pd %zmm3, %zmm13, %zmm0
374 vmovups __dC1(%rax), %zmm13
377 CosR = CosR - CosN*Pi2 */
378 vfnmadd231pd %zmm3, %zmm14, %zmm0
380 /* CosR = CosR - CosN*Pi3 */
381 vfnmadd213pd %zmm0, %zmm2, %zmm3
383 /* Polynomial approximation */
384 vmovups __dC7(%rax), %zmm0
386 /* Update CosRSign and CosSignRes signs */
388 vpternlogq $150, %zmm6, %zmm1, %zmm2
390 /* Update sign SinSignRes */
391 vpternlogq $120, %zmm8, %zmm4, %zmm6
394 vmulpd %zmm3, %zmm3, %zmm1
395 vmovups __dC6(%rax), %zmm4
396 vmovaps %zmm0, %zmm14
397 vfmadd213pd %zmm4, %zmm1, %zmm0
398 vfmadd213pd %zmm4, %zmm15, %zmm14
399 vfmadd213pd %zmm9, %zmm1, %zmm0
400 vfmadd213pd %zmm9, %zmm15, %zmm14
401 vfmadd213pd %zmm10, %zmm1, %zmm0
402 vfmadd213pd %zmm10, %zmm15, %zmm14
404 /* CosPoly = C3 + CosR2*(C4 + CosR2*(C5 + CosR2*(C6 + CosR2*C7))) */
405 vfmadd213pd %zmm11, %zmm1, %zmm0
407 /* SinPoly = C3 + SinR2*(C4 + SinR2*(C5 + SinR2*(C6 + SinR2*C7))) */
408 vfmadd213pd %zmm11, %zmm15, %zmm14
410 /* CosPoly = C2 + CosR2*CosPoly */
411 vfmadd213pd %zmm12, %zmm1, %zmm0
413 /* SinPoly = C2 + SinR2*SinPoly */
414 vfmadd213pd %zmm12, %zmm15, %zmm14
416 /* CosPoly = C1 + CosR2*CosPoly */
417 vfmadd213pd %zmm13, %zmm1, %zmm0
419 /* SinPoly = C1 + SinR2*SinPoly */
420 vfmadd213pd %zmm13, %zmm15, %zmm14
422 /* CosPoly = CosR2*CosPoly */
423 vmulpd %zmm1, %zmm0, %zmm1
425 /* SinPoly = SinR2*SinPoly */
426 vmulpd %zmm15, %zmm14, %zmm4
428 /* CosPoly = CosR*CosPoly */
429 vfmadd213pd %zmm3, %zmm3, %zmm1
431 /* SinPoly = SinR*SinPoly */
432 vfmadd213pd %zmm5, %zmm5, %zmm4
433 vpbroadcastq .L_2il0floatpacket.15(%rip), %zmm3
435 /* Update Cos result's sign */
436 vxorpd %zmm2, %zmm1, %zmm1
438 /* Final reconstruction.
439 Update Sin result's sign */
440 vxorpd %zmm6, %zmm4, %zmm0
441 vpandnq %zmm7, %zmm7, %zmm3{%k1}
442 vcmppd $3, %zmm3, %zmm3, %k0
449 vmovups %zmm0, (%rdi)
450 vmovups %zmm1, (%rsi)
452 cfi_def_cfa_register (%rsp)
454 cfi_adjust_cfa_offset (-8)
460 vmovups %zmm8, 1152(%rsp)
461 vmovups %zmm0, 1216(%rsp)
462 vmovups %zmm1, 1280(%rsp)
467 kmovw %k4, 1048(%rsp)
468 kmovw %k5, 1040(%rsp)
469 kmovw %k6, 1032(%rsp)
470 kmovw %k7, 1024(%rsp)
471 vmovups %zmm16, 960(%rsp)
472 vmovups %zmm17, 896(%rsp)
473 vmovups %zmm18, 832(%rsp)
474 vmovups %zmm19, 768(%rsp)
475 vmovups %zmm20, 704(%rsp)
476 vmovups %zmm21, 640(%rsp)
477 vmovups %zmm22, 576(%rsp)
478 vmovups %zmm23, 512(%rsp)
479 vmovups %zmm24, 448(%rsp)
480 vmovups %zmm25, 384(%rsp)
481 vmovups %zmm26, 320(%rsp)
482 vmovups %zmm27, 256(%rsp)
483 vmovups %zmm28, 192(%rsp)
484 vmovups %zmm29, 128(%rsp)
485 vmovups %zmm30, 64(%rsp)
486 vmovups %zmm31, (%rsp)
487 movq %rsi, 1056(%rsp)
488 movq %r12, 1096(%rsp)
489 cfi_offset_rel_rsp (12, 1096)
491 movq %r13, 1088(%rsp)
492 cfi_offset_rel_rsp (13, 1088)
494 movq %r14, 1080(%rsp)
495 cfi_offset_rel_rsp (14, 1080)
497 movq %r15, 1072(%rsp)
498 cfi_offset_rel_rsp (15, 1072)
499 movq %rbx, 1064(%rsp)
518 kmovw 1048(%rsp), %k4
520 kmovw 1040(%rsp), %k5
521 kmovw 1032(%rsp), %k6
522 kmovw 1024(%rsp), %k7
523 vmovups 960(%rsp), %zmm16
524 vmovups 896(%rsp), %zmm17
525 vmovups 832(%rsp), %zmm18
526 vmovups 768(%rsp), %zmm19
527 vmovups 704(%rsp), %zmm20
528 vmovups 640(%rsp), %zmm21
529 vmovups 576(%rsp), %zmm22
530 vmovups 512(%rsp), %zmm23
531 vmovups 448(%rsp), %zmm24
532 vmovups 384(%rsp), %zmm25
533 vmovups 320(%rsp), %zmm26
534 vmovups 256(%rsp), %zmm27
535 vmovups 192(%rsp), %zmm28
536 vmovups 128(%rsp), %zmm29
537 vmovups 64(%rsp), %zmm30
538 vmovups (%rsp), %zmm31
539 vmovups 1216(%rsp), %zmm0
540 vmovups 1280(%rsp), %zmm1
541 movq 1056(%rsp), %rsi
542 movq 1096(%rsp), %r12
544 movq 1088(%rsp), %r13
546 movq 1080(%rsp), %r14
548 movq 1072(%rsp), %r15
550 movq 1064(%rsp), %rbx
557 vmovsd 1160(%rsp,%r15), %xmm0
559 vmovsd 1160(%rsp,%r15), %xmm0
563 vmovsd %xmm0, 1224(%rsp,%r15)
564 vmovsd 1160(%rsp,%r15), %xmm0
568 vmovsd %xmm0, 1288(%rsp,%r15)
574 vmovsd 1152(%rsp,%r15), %xmm0
576 vmovsd 1152(%rsp,%r15), %xmm0
580 vmovsd %xmm0, 1216(%rsp,%r15)
581 vmovsd 1152(%rsp,%r15), %xmm0
585 vmovsd %xmm0, 1280(%rsp,%r15)
589 END (_ZGVeN8vl8l8_sincos_skx)
590 libmvec_hidden_def(_ZGVeN8vl8l8_sincos_skx)
592 /* Wrapper between vvv and vl8l8 vector variants. */
593 .macro WRAPPER_AVX512_vvv_vl8l8 callee
596 cfi_adjust_cfa_offset (8)
597 cfi_rel_offset (%rbp, 0)
599 cfi_def_cfa_register (%rbp)
602 vmovups %zmm1, 128(%rsp)
604 vmovups %zmm2, 192(%rdi)
606 call HIDDEN_JUMPTARGET(\callee)
656 cfi_def_cfa_register (%rsp)
658 cfi_adjust_cfa_offset (-8)
667 .cfi_escape 0x10,0x6,0x2,0x76,0
670 .cfi_escape 0xf,0x3,0x76,0x78,0x6
671 leal -112(%rbp), %esi
672 leal -176(%rbp), %edi
674 vmovdqa %ymm1, -208(%ebp)
675 vmovdqa %ymm2, -240(%ebp)
676 call HIDDEN_JUMPTARGET(\callee)
677 vmovdqa -208(%ebp), %xmm0
679 vmovsd -176(%ebp), %xmm0
682 vmovsd -168(%ebp), %xmm0
684 movq -200(%ebp), %rax
685 vmovsd -160(%ebp), %xmm0
688 vmovsd -152(%ebp), %xmm0
690 movq -192(%ebp), %rax
691 vmovsd -144(%ebp), %xmm0
694 vmovsd -136(%ebp), %xmm0
696 movq -184(%ebp), %rax
697 vmovsd -128(%ebp), %xmm0
700 vmovsd -120(%ebp), %xmm0
702 vmovdqa -240(%ebp), %xmm0
704 vmovsd -112(%ebp), %xmm0
707 vmovsd -104(%ebp), %xmm0
709 movq -232(%ebp), %rax
710 vmovsd -96(%ebp), %xmm0
713 vmovsd -88(%ebp), %xmm0
715 movq -224(%ebp), %rax
716 vmovsd -80(%ebp), %xmm0
719 vmovsd -72(%ebp), %xmm0
721 movq -216(%ebp), %rax
722 vmovsd -64(%ebp), %xmm0
725 vmovsd -56(%ebp), %xmm0
737 ENTRY (_ZGVeN8vvv_sincos_knl)
738 WRAPPER_AVX512_vvv_vl8l8 _ZGVeN8vl8l8_sincos_knl
739 END (_ZGVeN8vvv_sincos_knl)
741 ENTRY (_ZGVeN8vvv_sincos_skx)
742 WRAPPER_AVX512_vvv_vl8l8 _ZGVeN8vl8l8_sincos_skx
743 END (_ZGVeN8vvv_sincos_skx)
745 .section .rodata, "a"
746 .L_2il0floatpacket.15:
747 .long 0xffffffff,0xffffffff
748 .type .L_2il0floatpacket.15,@object