1 /* OMP constructs' SIMD clone supporting code.
3 Copyright (C) 2005-2017 Free Software Foundation, Inc.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
23 #include "coretypes.h"
29 #include "alloc-pool.h"
30 #include "tree-pass.h"
33 #include "pretty-print.h"
34 #include "diagnostic-core.h"
35 #include "fold-const.h"
36 #include "stor-layout.h"
39 #include "gimple-iterator.h"
40 #include "gimplify-me.h"
41 #include "gimple-walk.h"
42 #include "langhooks.h"
44 #include "tree-into-ssa.h"
47 #include "symbol-summary.h"
53 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
54 of arguments to reserve space for. */
56 static struct cgraph_simd_clone
*
57 simd_clone_struct_alloc (int nargs
)
59 struct cgraph_simd_clone
*clone_info
;
60 size_t len
= (sizeof (struct cgraph_simd_clone
)
61 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
62 clone_info
= (struct cgraph_simd_clone
*)
63 ggc_internal_cleared_alloc (len
);
67 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
70 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
71 struct cgraph_simd_clone
*from
)
73 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
74 + ((from
->nargs
- from
->inbranch
)
75 * sizeof (struct cgraph_simd_clone_arg
))));
78 /* Return vector of parameter types of function FNDECL. This uses
79 TYPE_ARG_TYPES if available, otherwise falls back to types of
80 DECL_ARGUMENTS types. */
83 simd_clone_vector_of_formal_parm_types (tree fndecl
)
85 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
86 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
87 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
90 FOR_EACH_VEC_ELT (args
, i
, arg
)
91 args
[i
] = TREE_TYPE (args
[i
]);
95 /* Given a simd function in NODE, extract the simd specific
96 information from the OMP clauses passed in CLAUSES, and return
97 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
98 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
99 otherwise set to FALSE. */
101 static struct cgraph_simd_clone
*
102 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
103 bool *inbranch_specified
)
105 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
108 *inbranch_specified
= false;
111 if (n
> 0 && args
.last () == void_type_node
)
114 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
115 be cloned have a distinctive artificial label in addition to "omp
119 && lookup_attribute ("cilk simd function",
120 DECL_ATTRIBUTES (node
->decl
)));
122 /* Allocate one more than needed just in case this is an in-branch
123 clone which will require a mask argument. */
124 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
125 clone_info
->nargs
= n
;
126 clone_info
->cilk_elemental
= cilk_clone
;
131 clauses
= TREE_VALUE (clauses
);
132 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
135 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
137 switch (OMP_CLAUSE_CODE (t
))
139 case OMP_CLAUSE_INBRANCH
:
140 clone_info
->inbranch
= 1;
141 *inbranch_specified
= true;
143 case OMP_CLAUSE_NOTINBRANCH
:
144 clone_info
->inbranch
= 0;
145 *inbranch_specified
= true;
147 case OMP_CLAUSE_SIMDLEN
:
149 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
151 case OMP_CLAUSE_LINEAR
:
153 tree decl
= OMP_CLAUSE_DECL (t
);
154 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
155 int argno
= TREE_INT_CST_LOW (decl
);
156 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
158 enum cgraph_simd_clone_arg_type arg_type
;
159 if (TREE_CODE (args
[argno
]) == REFERENCE_TYPE
)
160 switch (OMP_CLAUSE_LINEAR_KIND (t
))
162 case OMP_CLAUSE_LINEAR_REF
:
164 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
;
166 case OMP_CLAUSE_LINEAR_UVAL
:
168 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
;
170 case OMP_CLAUSE_LINEAR_VAL
:
171 case OMP_CLAUSE_LINEAR_DEFAULT
:
173 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
;
179 arg_type
= SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
180 clone_info
->args
[argno
].arg_type
= arg_type
;
181 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
182 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
183 && clone_info
->args
[argno
].linear_step
< n
);
187 if (POINTER_TYPE_P (args
[argno
]))
188 step
= fold_convert (ssizetype
, step
);
189 if (!tree_fits_shwi_p (step
))
191 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
192 "ignoring large linear step");
196 else if (integer_zerop (step
))
198 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
199 "ignoring zero linear step");
205 enum cgraph_simd_clone_arg_type arg_type
;
206 if (TREE_CODE (args
[argno
]) == REFERENCE_TYPE
)
207 switch (OMP_CLAUSE_LINEAR_KIND (t
))
209 case OMP_CLAUSE_LINEAR_REF
:
211 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
;
213 case OMP_CLAUSE_LINEAR_UVAL
:
215 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
;
217 case OMP_CLAUSE_LINEAR_VAL
:
218 case OMP_CLAUSE_LINEAR_DEFAULT
:
220 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
;
226 arg_type
= SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
227 clone_info
->args
[argno
].arg_type
= arg_type
;
228 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
233 case OMP_CLAUSE_UNIFORM
:
235 tree decl
= OMP_CLAUSE_DECL (t
);
236 int argno
= tree_to_uhwi (decl
);
237 clone_info
->args
[argno
].arg_type
238 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
241 case OMP_CLAUSE_ALIGNED
:
243 tree decl
= OMP_CLAUSE_DECL (t
);
244 int argno
= tree_to_uhwi (decl
);
245 clone_info
->args
[argno
].alignment
246 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
255 if (TYPE_ATOMIC (TREE_TYPE (TREE_TYPE (node
->decl
))))
257 warning_at (DECL_SOURCE_LOCATION (node
->decl
), 0,
258 "ignoring %<#pragma omp declare simd%> on function "
259 "with %<_Atomic%> qualified return type");
264 for (unsigned int argno
= 0; argno
< clone_info
->nargs
; argno
++)
265 if (TYPE_ATOMIC (args
[argno
])
266 && clone_info
->args
[argno
].arg_type
!= SIMD_CLONE_ARG_TYPE_UNIFORM
)
268 warning_at (DECL_SOURCE_LOCATION (node
->decl
), 0,
269 "ignoring %<#pragma omp declare simd%> on function "
270 "with %<_Atomic%> qualified non-%<uniform%> argument");
279 /* Given a SIMD clone in NODE, calculate the characteristic data
280 type and return the coresponding type. The characteristic data
281 type is computed as described in the Intel Vector ABI. */
284 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
285 struct cgraph_simd_clone
*clone_info
)
287 tree type
= integer_type_node
;
288 tree fndecl
= node
->decl
;
290 /* a) For non-void function, the characteristic data type is the
292 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
293 type
= TREE_TYPE (TREE_TYPE (fndecl
));
295 /* b) If the function has any non-uniform, non-linear parameters,
296 then the characteristic data type is the type of the first
300 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
301 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
302 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
310 /* c) If the characteristic data type determined by a) or b) above
311 is struct, union, or class type which is pass-by-value (except
312 for the type that maps to the built-in complex data type), the
313 characteristic data type is int. */
314 if (RECORD_OR_UNION_TYPE_P (type
)
315 && !aggregate_value_p (type
, NULL
)
316 && TREE_CODE (type
) != COMPLEX_TYPE
)
317 return integer_type_node
;
319 /* d) If none of the above three classes is applicable, the
320 characteristic data type is int. */
324 /* e) For Intel Xeon Phi native and offload compilation, if the
325 resulting characteristic data type is 8-bit or 16-bit integer
326 data type, the characteristic data type is int. */
327 /* Well, we don't handle Xeon Phi yet. */
331 simd_clone_mangle (struct cgraph_node
*node
,
332 struct cgraph_simd_clone
*clone_info
)
334 char vecsize_mangle
= clone_info
->vecsize_mangle
;
335 char mask
= clone_info
->inbranch
? 'M' : 'N';
336 unsigned int simdlen
= clone_info
->simdlen
;
340 gcc_assert (vecsize_mangle
&& simdlen
);
342 pp_string (&pp
, "_ZGV");
343 pp_character (&pp
, vecsize_mangle
);
344 pp_character (&pp
, mask
);
345 pp_decimal_int (&pp
, simdlen
);
347 for (n
= 0; n
< clone_info
->nargs
; ++n
)
349 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
351 switch (arg
.arg_type
)
353 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
354 pp_character (&pp
, 'u');
356 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
357 pp_character (&pp
, 'l');
359 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
360 pp_character (&pp
, 'R');
362 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
363 pp_character (&pp
, 'L');
365 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
366 pp_character (&pp
, 'U');
369 gcc_assert (arg
.linear_step
!= 0);
370 if (arg
.linear_step
> 1)
371 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
372 else if (arg
.linear_step
< 0)
374 pp_character (&pp
, 'n');
375 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
379 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
380 pp_string (&pp
, "ls");
381 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
383 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
384 pp_string (&pp
, "Rs");
385 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
387 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
388 pp_string (&pp
, "Ls");
389 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
391 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
392 pp_string (&pp
, "Us");
393 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
396 pp_character (&pp
, 'v');
400 pp_character (&pp
, 'a');
401 pp_decimal_int (&pp
, arg
.alignment
);
406 const char *str
= IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
));
409 pp_string (&pp
, str
);
410 str
= pp_formatted_text (&pp
);
412 /* If there already is a SIMD clone with the same mangled name, don't
413 add another one. This can happen e.g. for
414 #pragma omp declare simd
415 #pragma omp declare simd simdlen(8)
417 if the simdlen is assumed to be 8 for the first one, etc. */
418 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
419 clone
= clone
->simdclone
->next_clone
)
420 if (id_equal (DECL_ASSEMBLER_NAME (clone
->decl
), str
))
423 return get_identifier (str
);
426 /* Create a simd clone of OLD_NODE and return it. */
428 static struct cgraph_node
*
429 simd_clone_create (struct cgraph_node
*old_node
)
431 struct cgraph_node
*new_node
;
432 if (old_node
->definition
)
434 if (!old_node
->has_gimple_body_p ())
436 old_node
->get_body ();
437 new_node
= old_node
->create_version_clone_with_body (vNULL
, NULL
, NULL
,
443 tree old_decl
= old_node
->decl
;
444 tree new_decl
= copy_node (old_node
->decl
);
445 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
446 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
447 SET_DECL_RTL (new_decl
, NULL
);
448 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
449 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
450 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
451 if (old_node
->in_other_partition
)
452 new_node
->in_other_partition
= 1;
454 if (new_node
== NULL
)
457 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
458 DECL_COMDAT (new_node
->decl
) = DECL_COMDAT (old_node
->decl
);
459 DECL_WEAK (new_node
->decl
) = DECL_WEAK (old_node
->decl
);
460 DECL_EXTERNAL (new_node
->decl
) = DECL_EXTERNAL (old_node
->decl
);
461 DECL_VISIBILITY_SPECIFIED (new_node
->decl
)
462 = DECL_VISIBILITY_SPECIFIED (old_node
->decl
);
463 DECL_VISIBILITY (new_node
->decl
) = DECL_VISIBILITY (old_node
->decl
);
464 DECL_DLLIMPORT_P (new_node
->decl
) = DECL_DLLIMPORT_P (old_node
->decl
);
465 if (DECL_ONE_ONLY (old_node
->decl
))
466 make_decl_one_only (new_node
->decl
, DECL_ASSEMBLER_NAME (new_node
->decl
));
468 /* The method cgraph_version_clone_with_body () will force the new
469 symbol local. Undo this, and inherit external visibility from
471 new_node
->local
.local
= old_node
->local
.local
;
472 new_node
->externally_visible
= old_node
->externally_visible
;
477 /* Adjust the return type of the given function to its appropriate
478 vector counterpart. Returns a simd array to be used throughout the
479 function as a return value. */
482 simd_clone_adjust_return_type (struct cgraph_node
*node
)
484 tree fndecl
= node
->decl
;
485 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
489 /* Adjust the function return type. */
490 if (orig_rettype
== void_type_node
)
492 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
493 t
= TREE_TYPE (TREE_TYPE (fndecl
));
494 if (INTEGRAL_TYPE_P (t
) || POINTER_TYPE_P (t
))
495 veclen
= node
->simdclone
->vecsize_int
;
497 veclen
= node
->simdclone
->vecsize_float
;
498 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (t
));
499 if (veclen
> node
->simdclone
->simdlen
)
500 veclen
= node
->simdclone
->simdlen
;
501 if (POINTER_TYPE_P (t
))
502 t
= pointer_sized_int_node
;
503 if (veclen
== node
->simdclone
->simdlen
)
504 t
= build_vector_type (t
, node
->simdclone
->simdlen
);
507 t
= build_vector_type (t
, veclen
);
508 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
510 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
511 if (!node
->definition
)
514 t
= DECL_RESULT (fndecl
);
515 /* Adjust the DECL_RESULT. */
516 gcc_assert (TREE_TYPE (t
) != void_type_node
);
517 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
520 tree atype
= build_array_type_nelts (orig_rettype
,
521 node
->simdclone
->simdlen
);
522 if (veclen
!= node
->simdclone
->simdlen
)
523 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
525 /* Set up a SIMD array to use as the return value. */
526 tree retval
= create_tmp_var_raw (atype
, "retval");
527 gimple_add_tmp_var (retval
);
531 /* Each vector argument has a corresponding array to be used locally
532 as part of the eventual loop. Create such temporary array and
535 PREFIX is the prefix to be used for the temporary.
537 TYPE is the inner element type.
539 SIMDLEN is the number of elements. */
542 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
544 tree atype
= build_array_type_nelts (type
, simdlen
);
545 tree avar
= create_tmp_var_raw (atype
, prefix
);
546 gimple_add_tmp_var (avar
);
550 /* Modify the function argument types to their corresponding vector
551 counterparts if appropriate. Also, create one array for each simd
552 argument to be used locally when using the function arguments as
555 NODE is the function whose arguments are to be adjusted.
557 Returns an adjustment vector that will be filled describing how the
558 argument types will be adjusted. */
560 static ipa_parm_adjustment_vec
561 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
564 ipa_parm_adjustment_vec adjustments
;
566 if (node
->definition
)
567 args
= ipa_get_vector_of_formal_parms (node
->decl
);
569 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
570 adjustments
.create (args
.length ());
571 unsigned i
, j
, veclen
;
572 struct ipa_parm_adjustment adj
;
573 struct cgraph_simd_clone
*sc
= node
->simdclone
;
575 for (i
= 0; i
< sc
->nargs
; ++i
)
577 memset (&adj
, 0, sizeof (adj
));
579 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
583 sc
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
584 sc
->args
[i
].orig_type
= parm_type
;
586 switch (sc
->args
[i
].arg_type
)
589 /* No adjustment necessary for scalar arguments. */
590 adj
.op
= IPA_PARM_OP_COPY
;
592 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
593 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
594 if (node
->definition
)
595 sc
->args
[i
].simd_array
596 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
597 TREE_TYPE (parm_type
),
599 adj
.op
= IPA_PARM_OP_COPY
;
601 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
602 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
603 case SIMD_CLONE_ARG_TYPE_VECTOR
:
604 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
605 veclen
= sc
->vecsize_int
;
607 veclen
= sc
->vecsize_float
;
608 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
609 if (veclen
> sc
->simdlen
)
610 veclen
= sc
->simdlen
;
611 adj
.arg_prefix
= "simd";
612 if (POINTER_TYPE_P (parm_type
))
613 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
615 adj
.type
= build_vector_type (parm_type
, veclen
);
616 sc
->args
[i
].vector_type
= adj
.type
;
617 for (j
= veclen
; j
< sc
->simdlen
; j
+= veclen
)
619 adjustments
.safe_push (adj
);
622 memset (&adj
, 0, sizeof (adj
));
623 adj
.op
= IPA_PARM_OP_NEW
;
624 adj
.arg_prefix
= "simd";
626 adj
.type
= sc
->args
[i
].vector_type
;
630 if (node
->definition
)
631 sc
->args
[i
].simd_array
632 = create_tmp_simd_array (DECL_NAME (parm
)
633 ? IDENTIFIER_POINTER (DECL_NAME (parm
))
634 : NULL
, parm_type
, sc
->simdlen
);
636 adjustments
.safe_push (adj
);
641 tree base_type
= simd_clone_compute_base_data_type (sc
->origin
, sc
);
643 memset (&adj
, 0, sizeof (adj
));
644 adj
.op
= IPA_PARM_OP_NEW
;
645 adj
.arg_prefix
= "mask";
648 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
649 veclen
= sc
->vecsize_int
;
651 veclen
= sc
->vecsize_float
;
652 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
653 if (veclen
> sc
->simdlen
)
654 veclen
= sc
->simdlen
;
655 if (sc
->mask_mode
!= VOIDmode
)
657 = lang_hooks
.types
.type_for_mode (sc
->mask_mode
, 1);
658 else if (POINTER_TYPE_P (base_type
))
659 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
661 adj
.type
= build_vector_type (base_type
, veclen
);
662 adjustments
.safe_push (adj
);
664 for (j
= veclen
; j
< sc
->simdlen
; j
+= veclen
)
665 adjustments
.safe_push (adj
);
667 /* We have previously allocated one extra entry for the mask. Use
670 if (sc
->mask_mode
!= VOIDmode
)
671 base_type
= boolean_type_node
;
672 if (node
->definition
)
675 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
676 if (sc
->mask_mode
== VOIDmode
)
677 sc
->args
[i
].simd_array
678 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
679 else if (veclen
< sc
->simdlen
)
680 sc
->args
[i
].simd_array
681 = create_tmp_simd_array ("mask", adj
.type
, sc
->simdlen
/ veclen
);
683 sc
->args
[i
].simd_array
= NULL_TREE
;
685 sc
->args
[i
].orig_type
= base_type
;
686 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
689 if (node
->definition
)
690 ipa_modify_formal_parameters (node
->decl
, adjustments
);
693 tree new_arg_types
= NULL_TREE
, new_reversed
;
694 bool last_parm_void
= false;
695 if (args
.length () > 0 && args
.last () == void_type_node
)
696 last_parm_void
= true;
698 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
699 j
= adjustments
.length ();
700 for (i
= 0; i
< j
; i
++)
702 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
704 if (adj
->op
== IPA_PARM_OP_COPY
)
705 ptype
= args
[adj
->base_index
];
708 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
710 new_reversed
= nreverse (new_arg_types
);
714 TREE_CHAIN (new_arg_types
) = void_list_node
;
716 new_reversed
= void_list_node
;
719 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
720 TYPE_ARG_TYPES (new_type
) = new_reversed
;
721 TREE_TYPE (node
->decl
) = new_type
;
723 adjustments
.release ();
729 /* Initialize and copy the function arguments in NODE to their
730 corresponding local simd arrays. Returns a fresh gimple_seq with
731 the instruction sequence generated. */
734 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
735 ipa_parm_adjustment_vec adjustments
)
737 gimple_seq seq
= NULL
;
738 unsigned i
= 0, j
= 0, k
;
740 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
742 arg
= DECL_CHAIN (arg
), i
++, j
++)
744 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
745 || POINTER_TYPE_P (TREE_TYPE (arg
)))
748 node
->simdclone
->args
[i
].vector_arg
= arg
;
750 tree array
= node
->simdclone
->args
[i
].simd_array
;
751 if (node
->simdclone
->mask_mode
!= VOIDmode
752 && node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_MASK
)
754 if (array
== NULL_TREE
)
757 = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (array
))));
758 for (k
= 0; k
<= l
; k
++)
762 arg
= DECL_CHAIN (arg
);
765 tree t
= build4 (ARRAY_REF
, TREE_TYPE (TREE_TYPE (array
)),
766 array
, size_int (k
), NULL
, NULL
);
767 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
768 gimplify_and_add (t
, &seq
);
772 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
774 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
775 tree ptr
= build_fold_addr_expr (array
);
776 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
777 build_int_cst (ptype
, 0));
778 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
779 gimplify_and_add (t
, &seq
);
783 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
784 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
785 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
787 tree ptr
= build_fold_addr_expr (array
);
791 arg
= DECL_CHAIN (arg
);
795 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
796 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
797 build_int_cst (ptype
, k
* elemsize
));
798 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
799 gimplify_and_add (t
, &seq
);
806 /* Callback info for ipa_simd_modify_stmt_ops below. */
808 struct modify_stmt_info
{
809 ipa_parm_adjustment_vec adjustments
;
811 /* True if the parent statement was modified by
812 ipa_simd_modify_stmt_ops. */
816 /* Callback for walk_gimple_op.
818 Adjust operands from a given statement as specified in the
819 adjustments vector in the callback data. */
822 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
824 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
825 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
827 if (TREE_CODE (*tp
) == ADDR_EXPR
)
828 tp
= &TREE_OPERAND (*tp
, 0);
829 struct ipa_parm_adjustment
*cand
= NULL
;
830 if (TREE_CODE (*tp
) == PARM_DECL
)
831 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
838 tree repl
= NULL_TREE
;
840 repl
= unshare_expr (cand
->new_decl
);
846 bool modified
= info
->modified
;
847 info
->modified
= false;
848 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
851 info
->modified
= modified
;
854 info
->modified
= modified
;
863 repl
= build_fold_addr_expr (repl
);
865 if (is_gimple_debug (info
->stmt
))
867 tree vexpr
= make_node (DEBUG_EXPR_DECL
);
868 stmt
= gimple_build_debug_source_bind (vexpr
, repl
, NULL
);
869 DECL_ARTIFICIAL (vexpr
) = 1;
870 TREE_TYPE (vexpr
) = TREE_TYPE (repl
);
871 SET_DECL_MODE (vexpr
, TYPE_MODE (TREE_TYPE (repl
)));
876 stmt
= gimple_build_assign (make_ssa_name (TREE_TYPE (repl
)), repl
);
877 repl
= gimple_assign_lhs (stmt
);
879 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
880 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
883 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
885 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
891 info
->modified
= true;
895 /* Traverse the function body and perform all modifications as
896 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
897 modified such that the replacement/reduction value will now be an
898 offset into the corresponding simd_array.
900 This function will replace all function argument uses with their
901 corresponding simd array elements, and ajust the return values
905 ipa_simd_modify_function_body (struct cgraph_node
*node
,
906 ipa_parm_adjustment_vec adjustments
,
907 tree retval_array
, tree iter
)
910 unsigned int i
, j
, l
;
912 /* Re-use the adjustments array, but this time use it to replace
913 every function argument use to an offset into the corresponding
915 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
917 if (!node
->simdclone
->args
[i
].vector_arg
)
920 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
921 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
922 adjustments
[j
].new_decl
925 node
->simdclone
->args
[i
].simd_array
,
927 NULL_TREE
, NULL_TREE
);
928 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
929 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
930 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
933 l
= adjustments
.length ();
936 FOR_EACH_SSA_NAME (i
, name
, cfun
)
938 if (SSA_NAME_VAR (name
)
939 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
941 for (j
= 0; j
< l
; j
++)
942 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
943 && adjustments
[j
].new_decl
)
946 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
949 = copy_var_decl (adjustments
[j
].base
,
950 DECL_NAME (adjustments
[j
].base
),
951 TREE_TYPE (adjustments
[j
].base
));
952 adjustments
[j
].new_ssa_base
= base_var
;
955 base_var
= adjustments
[j
].new_ssa_base
;
956 if (SSA_NAME_IS_DEFAULT_DEF (name
))
958 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
959 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
960 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
961 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
962 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
963 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
964 gimple
*stmt
= gimple_build_assign (name
, new_decl
);
965 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
968 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
973 struct modify_stmt_info info
;
974 info
.adjustments
= adjustments
;
976 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
978 gimple_stmt_iterator gsi
;
980 gsi
= gsi_start_bb (bb
);
981 while (!gsi_end_p (gsi
))
983 gimple
*stmt
= gsi_stmt (gsi
);
985 struct walk_stmt_info wi
;
987 memset (&wi
, 0, sizeof (wi
));
988 info
.modified
= false;
990 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
992 if (greturn
*return_stmt
= dyn_cast
<greturn
*> (stmt
))
994 tree retval
= gimple_return_retval (return_stmt
);
997 gsi_remove (&gsi
, true);
1001 /* Replace `return foo' with `retval_array[iter] = foo'. */
1002 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
1003 retval_array
, iter
, NULL
, NULL
);
1004 stmt
= gimple_build_assign (ref
, retval
);
1005 gsi_replace (&gsi
, stmt
, true);
1006 info
.modified
= true;
1012 if (maybe_clean_eh_stmt (stmt
))
1013 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
1020 /* Helper function of simd_clone_adjust, return linear step addend
1024 simd_clone_linear_addend (struct cgraph_node
*node
, unsigned int i
,
1025 tree addtype
, basic_block entry_bb
)
1027 tree ptype
= NULL_TREE
;
1028 switch (node
->simdclone
->args
[i
].arg_type
)
1030 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
1031 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
1032 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
1033 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
1034 return build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
1035 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
1036 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
1037 ptype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
1039 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
1040 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
1041 ptype
= TREE_TYPE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
));
1047 unsigned int idx
= node
->simdclone
->args
[i
].linear_step
;
1048 tree arg
= node
->simdclone
->args
[idx
].orig_arg
;
1049 gcc_assert (is_gimple_reg_type (TREE_TYPE (arg
)));
1050 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
1053 if (is_gimple_reg (arg
))
1054 ret
= get_or_create_ssa_default_def (cfun
, arg
);
1057 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (arg
)), arg
);
1058 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1059 ret
= gimple_assign_lhs (g
);
1061 if (TREE_CODE (TREE_TYPE (arg
)) == REFERENCE_TYPE
)
1063 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (TREE_TYPE (arg
))),
1064 build_simple_mem_ref (ret
));
1065 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1066 ret
= gimple_assign_lhs (g
);
1068 if (!useless_type_conversion_p (addtype
, TREE_TYPE (ret
)))
1070 g
= gimple_build_assign (make_ssa_name (addtype
), NOP_EXPR
, ret
);
1071 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1072 ret
= gimple_assign_lhs (g
);
1074 if (POINTER_TYPE_P (ptype
))
1076 tree size
= TYPE_SIZE_UNIT (TREE_TYPE (ptype
));
1077 if (size
&& TREE_CODE (size
) == INTEGER_CST
)
1079 g
= gimple_build_assign (make_ssa_name (addtype
), MULT_EXPR
,
1080 ret
, fold_convert (addtype
, size
));
1081 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1082 ret
= gimple_assign_lhs (g
);
1088 /* Adjust the argument types in NODE to their appropriate vector
1092 simd_clone_adjust (struct cgraph_node
*node
)
1094 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
1096 targetm
.simd_clone
.adjust (node
);
1098 tree retval
= simd_clone_adjust_return_type (node
);
1099 ipa_parm_adjustment_vec adjustments
1100 = simd_clone_adjust_argument_types (node
);
1102 push_gimplify_context ();
1104 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
1106 /* Adjust all uses of vector arguments accordingly. Adjust all
1107 return values accordingly. */
1108 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
1109 tree iter1
= make_ssa_name (iter
);
1110 tree iter2
= NULL_TREE
;
1111 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
1112 adjustments
.release ();
1114 /* Initialize the iteration variable. */
1115 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
1116 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
1117 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
1118 /* Insert the SIMD array and iv initialization at function
1120 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
1122 pop_gimplify_context (NULL
);
1125 basic_block incr_bb
= NULL
;
1126 struct loop
*loop
= NULL
;
1128 /* Create a new BB right before the original exit BB, to hold the
1129 iteration increment and the condition/branch. */
1130 if (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
))
1132 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
1133 incr_bb
= create_empty_bb (orig_exit
);
1134 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
1135 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
1136 flag. Set it now to be a FALLTHRU_EDGE. */
1137 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
1138 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
1139 for (unsigned i
= 0;
1140 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
1142 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
1143 redirect_edge_succ (e
, incr_bb
);
1146 else if (node
->simdclone
->inbranch
)
1148 incr_bb
= create_empty_bb (entry_bb
);
1149 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
1154 make_single_succ_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
1155 gsi
= gsi_last_bb (incr_bb
);
1156 iter2
= make_ssa_name (iter
);
1157 g
= gimple_build_assign (iter2
, PLUS_EXPR
, iter1
,
1158 build_int_cst (unsigned_type_node
, 1));
1159 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1161 /* Mostly annotate the loop for the vectorizer (the rest is done
1163 loop
= alloc_loop ();
1164 cfun
->has_force_vectorize_loops
= true;
1165 loop
->safelen
= node
->simdclone
->simdlen
;
1166 loop
->force_vectorize
= true;
1167 loop
->header
= body_bb
;
1170 /* Branch around the body if the mask applies. */
1171 if (node
->simdclone
->inbranch
)
1173 gsi
= gsi_last_bb (loop
->header
);
1175 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
1177 if (node
->simdclone
->mask_mode
!= VOIDmode
)
1180 if (mask_array
== NULL_TREE
)
1182 tree arg
= node
->simdclone
->args
[node
->simdclone
->nargs
1184 mask
= get_or_create_ssa_default_def (cfun
, arg
);
1189 tree maskt
= TREE_TYPE (mask_array
);
1190 int c
= tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (maskt
)));
1191 c
= node
->simdclone
->simdlen
/ (c
+ 1);
1192 int s
= exact_log2 (c
);
1195 tree idx
= make_ssa_name (TREE_TYPE (iter1
));
1196 g
= gimple_build_assign (idx
, RSHIFT_EXPR
, iter1
,
1197 build_int_cst (NULL_TREE
, s
));
1198 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1199 mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)));
1200 tree aref
= build4 (ARRAY_REF
,
1201 TREE_TYPE (TREE_TYPE (mask_array
)),
1202 mask_array
, idx
, NULL
, NULL
);
1203 g
= gimple_build_assign (mask
, aref
);
1204 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1205 shift_cnt
= make_ssa_name (TREE_TYPE (iter1
));
1206 g
= gimple_build_assign (shift_cnt
, BIT_AND_EXPR
, iter1
,
1207 build_int_cst (TREE_TYPE (iter1
), c
));
1208 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1210 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (mask
)),
1211 RSHIFT_EXPR
, mask
, shift_cnt
);
1212 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1213 mask
= gimple_assign_lhs (g
);
1214 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (mask
)),
1216 build_int_cst (TREE_TYPE (mask
), 1));
1217 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1218 mask
= gimple_assign_lhs (g
);
1222 mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)));
1223 tree aref
= build4 (ARRAY_REF
,
1224 TREE_TYPE (TREE_TYPE (mask_array
)),
1225 mask_array
, iter1
, NULL
, NULL
);
1226 g
= gimple_build_assign (mask
, aref
);
1227 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1228 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
1229 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
1231 aref
= build1 (VIEW_CONVERT_EXPR
,
1232 build_nonstandard_integer_type (bitsize
, 0),
1234 mask
= make_ssa_name (TREE_TYPE (aref
));
1235 g
= gimple_build_assign (mask
, aref
);
1236 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1240 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
1242 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1243 edge e
= make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
1244 e
->probability
= profile_probability::unlikely ().guessed ();
1245 edge fallthru
= FALLTHRU_EDGE (loop
->header
);
1246 fallthru
->flags
= EDGE_FALSE_VALUE
;
1247 fallthru
->probability
= profile_probability::likely ().guessed ();
1250 basic_block latch_bb
= NULL
;
1251 basic_block new_exit_bb
= NULL
;
1253 /* Generate the condition. */
1256 gsi
= gsi_last_bb (incr_bb
);
1257 g
= gimple_build_cond (LT_EXPR
, iter2
,
1258 build_int_cst (unsigned_type_node
,
1259 node
->simdclone
->simdlen
),
1261 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1262 edge e
= split_block (incr_bb
, gsi_stmt (gsi
));
1264 new_exit_bb
= split_block_after_labels (latch_bb
)->dest
;
1265 loop
->latch
= latch_bb
;
1267 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
1269 edge new_e
= make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
1271 /* FIXME: Do we need to distribute probabilities for the conditional? */
1272 new_e
->probability
= profile_probability::guessed_never ();
1273 /* The successor of incr_bb is already pointing to latch_bb; just
1275 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
1276 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
1279 gphi
*phi
= create_phi_node (iter1
, body_bb
);
1280 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
1281 edge latch_edge
= NULL
;
1282 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
1286 latch_edge
= single_succ_edge (latch_bb
);
1287 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
1289 /* Generate the new return. */
1290 gsi
= gsi_last_bb (new_exit_bb
);
1292 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
1293 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
1294 retval
= TREE_OPERAND (retval
, 0);
1297 retval
= build1 (VIEW_CONVERT_EXPR
,
1298 TREE_TYPE (TREE_TYPE (node
->decl
)),
1300 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
1301 false, GSI_CONTINUE_LINKING
);
1303 g
= gimple_build_return (retval
);
1304 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1307 /* Handle aligned clauses by replacing default defs of the aligned
1308 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
1309 lhs. Handle linear by adding PHIs. */
1310 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
1311 if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
1312 && (TREE_ADDRESSABLE (node
->simdclone
->args
[i
].orig_arg
)
1313 || !is_gimple_reg_type
1314 (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
1316 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1317 if (is_gimple_reg_type (TREE_TYPE (orig_arg
)))
1318 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
1321 iter1
= create_tmp_var_raw (TREE_TYPE (orig_arg
));
1322 gimple_add_tmp_var (iter1
);
1324 gsi
= gsi_after_labels (entry_bb
);
1325 g
= gimple_build_assign (iter1
, orig_arg
);
1326 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1327 gsi
= gsi_after_labels (body_bb
);
1328 g
= gimple_build_assign (orig_arg
, iter1
);
1329 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1331 else if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
1332 && DECL_BY_REFERENCE (node
->simdclone
->args
[i
].orig_arg
)
1333 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
1336 (TREE_TYPE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
1338 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1339 tree def
= ssa_default_def (cfun
, orig_arg
);
1340 if (def
&& !has_zero_uses (def
))
1342 iter1
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg
)));
1343 gimple_add_tmp_var (iter1
);
1344 gsi
= gsi_after_labels (entry_bb
);
1345 g
= gimple_build_assign (iter1
, build_simple_mem_ref (def
));
1346 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1347 gsi
= gsi_after_labels (body_bb
);
1348 g
= gimple_build_assign (build_simple_mem_ref (def
), iter1
);
1349 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1352 else if (node
->simdclone
->args
[i
].alignment
1353 && node
->simdclone
->args
[i
].arg_type
1354 == SIMD_CLONE_ARG_TYPE_UNIFORM
1355 && (node
->simdclone
->args
[i
].alignment
1356 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
1357 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
1360 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
1361 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1362 tree def
= ssa_default_def (cfun
, orig_arg
);
1363 if (def
&& !has_zero_uses (def
))
1365 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
1366 gimple_seq seq
= NULL
;
1367 bool need_cvt
= false;
1369 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
1371 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
1374 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
);
1375 gimple_call_set_lhs (g
, t
);
1376 gimple_seq_add_stmt_without_update (&seq
, g
);
1379 t
= make_ssa_name (orig_arg
);
1380 g
= gimple_build_assign (t
, NOP_EXPR
, gimple_call_lhs (g
));
1381 gimple_seq_add_stmt_without_update (&seq
, g
);
1383 gsi_insert_seq_on_edge_immediate
1384 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
1386 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
1387 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
1389 node
->create_edge (cgraph_node::get_create (fn
),
1390 call
, entry_bb
->count
, freq
);
1392 imm_use_iterator iter
;
1393 use_operand_p use_p
;
1395 tree repl
= gimple_get_lhs (g
);
1396 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
1397 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
1400 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
1401 SET_USE (use_p
, repl
);
1404 else if ((node
->simdclone
->args
[i
].arg_type
1405 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
1406 || (node
->simdclone
->args
[i
].arg_type
1407 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
)
1408 || (node
->simdclone
->args
[i
].arg_type
1409 == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
1410 || (node
->simdclone
->args
[i
].arg_type
1411 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
))
1413 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1414 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
1415 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
1416 tree def
= NULL_TREE
;
1417 if (TREE_ADDRESSABLE (orig_arg
))
1419 def
= make_ssa_name (TREE_TYPE (orig_arg
));
1420 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
1422 iter2
= make_ssa_name (TREE_TYPE (orig_arg
));
1423 gsi
= gsi_after_labels (entry_bb
);
1424 g
= gimple_build_assign (def
, orig_arg
);
1425 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1429 def
= ssa_default_def (cfun
, orig_arg
);
1430 if (!def
|| has_zero_uses (def
))
1434 iter1
= make_ssa_name (orig_arg
);
1436 iter2
= make_ssa_name (orig_arg
);
1441 phi
= create_phi_node (iter1
, body_bb
);
1442 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
1445 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
1446 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
1447 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
1448 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
1449 ? TREE_TYPE (orig_arg
) : sizetype
;
1450 tree addcst
= simd_clone_linear_addend (node
, i
, addtype
,
1452 gsi
= gsi_last_bb (incr_bb
);
1453 g
= gimple_build_assign (iter2
, code
, iter1
, addcst
);
1454 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1457 imm_use_iterator iter
;
1458 use_operand_p use_p
;
1460 if (TREE_ADDRESSABLE (orig_arg
))
1462 gsi
= gsi_after_labels (body_bb
);
1463 g
= gimple_build_assign (orig_arg
, iter1
);
1464 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1467 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
1468 if (use_stmt
== phi
)
1471 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
1472 SET_USE (use_p
, iter1
);
1475 else if (node
->simdclone
->args
[i
].arg_type
1476 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
1477 || (node
->simdclone
->args
[i
].arg_type
1478 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
))
1480 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1481 tree def
= ssa_default_def (cfun
, orig_arg
);
1482 gcc_assert (!TREE_ADDRESSABLE (orig_arg
)
1483 && TREE_CODE (TREE_TYPE (orig_arg
)) == REFERENCE_TYPE
);
1484 if (def
&& !has_zero_uses (def
))
1486 tree rtype
= TREE_TYPE (TREE_TYPE (orig_arg
));
1487 iter1
= make_ssa_name (orig_arg
);
1489 iter2
= make_ssa_name (orig_arg
);
1490 tree iter3
= make_ssa_name (rtype
);
1491 tree iter4
= make_ssa_name (rtype
);
1492 tree iter5
= incr_bb
? make_ssa_name (rtype
) : NULL_TREE
;
1493 gsi
= gsi_after_labels (entry_bb
);
1495 = gimple_build_assign (iter3
, build_simple_mem_ref (def
));
1496 gsi_insert_before (&gsi
, load
, GSI_NEW_STMT
);
1498 tree array
= node
->simdclone
->args
[i
].simd_array
;
1499 TREE_ADDRESSABLE (array
) = 1;
1500 tree ptr
= build_fold_addr_expr (array
);
1501 phi
= create_phi_node (iter1
, body_bb
);
1502 add_phi_arg (phi
, ptr
, preheader_edge
, UNKNOWN_LOCATION
);
1505 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
1506 g
= gimple_build_assign (iter2
, POINTER_PLUS_EXPR
, iter1
,
1507 TYPE_SIZE_UNIT (TREE_TYPE (iter3
)));
1508 gsi
= gsi_last_bb (incr_bb
);
1509 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1512 phi
= create_phi_node (iter4
, body_bb
);
1513 add_phi_arg (phi
, iter3
, preheader_edge
, UNKNOWN_LOCATION
);
1516 add_phi_arg (phi
, iter5
, latch_edge
, UNKNOWN_LOCATION
);
1517 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (iter3
))
1518 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
1519 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (iter3
))
1520 ? TREE_TYPE (iter3
) : sizetype
;
1521 tree addcst
= simd_clone_linear_addend (node
, i
, addtype
,
1523 g
= gimple_build_assign (iter5
, code
, iter4
, addcst
);
1524 gsi
= gsi_last_bb (incr_bb
);
1525 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1528 g
= gimple_build_assign (build_simple_mem_ref (iter1
), iter4
);
1529 gsi
= gsi_after_labels (body_bb
);
1530 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1532 imm_use_iterator iter
;
1533 use_operand_p use_p
;
1535 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
1536 if (use_stmt
== load
)
1539 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
1540 SET_USE (use_p
, iter1
);
1542 if (!TYPE_READONLY (rtype
) && incr_bb
)
1544 tree v
= make_ssa_name (rtype
);
1545 tree aref
= build4 (ARRAY_REF
, rtype
, array
,
1546 size_zero_node
, NULL_TREE
,
1548 gsi
= gsi_after_labels (new_exit_bb
);
1549 g
= gimple_build_assign (v
, aref
);
1550 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1551 g
= gimple_build_assign (build_simple_mem_ref (def
), v
);
1552 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1557 calculate_dominance_info (CDI_DOMINATORS
);
1559 add_loop (loop
, loop
->header
->loop_father
);
1560 update_ssa (TODO_update_ssa
);
1565 /* If the function in NODE is tagged as an elemental SIMD function,
1566 create the appropriate SIMD clones. */
1569 expand_simd_clones (struct cgraph_node
*node
)
1571 tree attr
= lookup_attribute ("omp declare simd",
1572 DECL_ATTRIBUTES (node
->decl
));
1573 if (attr
== NULL_TREE
1574 || node
->global
.inlined_to
1575 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
1579 #pragma omp declare simd
1581 in C, there we don't know the argument types at all. */
1582 if (!node
->definition
1583 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
1586 /* Call this before creating clone_info, as it might ggc_collect. */
1587 if (node
->definition
&& node
->has_gimple_body_p ())
1592 /* Start with parsing the "omp declare simd" attribute(s). */
1593 bool inbranch_clause_specified
;
1594 struct cgraph_simd_clone
*clone_info
1595 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
1596 &inbranch_clause_specified
);
1597 if (clone_info
== NULL
)
1600 int orig_simdlen
= clone_info
->simdlen
;
1601 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
1602 /* The target can return 0 (no simd clones should be created),
1603 1 (just one ISA of simd clones should be created) or higher
1604 count of ISA variants. In that case, clone_info is initialized
1605 for the first ISA variant. */
1607 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
1612 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
1613 also create one inbranch and one !inbranch clone of it. */
1614 for (int i
= 0; i
< count
* 2; i
++)
1616 struct cgraph_simd_clone
*clone
= clone_info
;
1617 if (inbranch_clause_specified
&& (i
& 1) != 0)
1622 clone
= simd_clone_struct_alloc (clone_info
->nargs
1624 simd_clone_struct_copy (clone
, clone_info
);
1625 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
1626 and simd_clone_adjust_argument_types did to the first
1628 clone
->nargs
-= clone_info
->inbranch
;
1629 clone
->simdlen
= orig_simdlen
;
1630 /* And call the target hook again to get the right ISA. */
1631 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
1635 clone
->inbranch
= 1;
1638 /* simd_clone_mangle might fail if such a clone has been created
1640 tree id
= simd_clone_mangle (node
, clone
);
1641 if (id
== NULL_TREE
)
1644 /* Only when we are sure we want to create the clone actually
1645 clone the function (or definitions) or create another
1646 extern FUNCTION_DECL (for prototypes without definitions). */
1647 struct cgraph_node
*n
= simd_clone_create (node
);
1651 n
->simdclone
= clone
;
1652 clone
->origin
= node
;
1653 clone
->next_clone
= NULL
;
1654 if (node
->simd_clones
== NULL
)
1656 clone
->prev_clone
= n
;
1657 node
->simd_clones
= n
;
1661 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
1662 clone
->prev_clone
->simdclone
->next_clone
= n
;
1663 node
->simd_clones
->simdclone
->prev_clone
= n
;
1665 symtab
->change_decl_assembler_name (n
->decl
, id
);
1666 /* And finally adjust the return type, parameters and for
1667 definitions also function body. */
1668 if (node
->definition
)
1669 simd_clone_adjust (n
);
1672 simd_clone_adjust_return_type (n
);
1673 simd_clone_adjust_argument_types (n
);
1677 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
1680 /* Entry point for IPA simd clone creation pass. */
1683 ipa_omp_simd_clone (void)
1685 struct cgraph_node
*node
;
1686 FOR_EACH_FUNCTION (node
)
1687 expand_simd_clones (node
);
1693 const pass_data pass_data_omp_simd_clone
=
1695 SIMPLE_IPA_PASS
, /* type */
1696 "simdclone", /* name */
1697 OPTGROUP_OMP
, /* optinfo_flags */
1698 TV_NONE
, /* tv_id */
1699 ( PROP_ssa
| PROP_cfg
), /* properties_required */
1700 0, /* properties_provided */
1701 0, /* properties_destroyed */
1702 0, /* todo_flags_start */
1703 0, /* todo_flags_finish */
1706 class pass_omp_simd_clone
: public simple_ipa_opt_pass
1709 pass_omp_simd_clone(gcc::context
*ctxt
)
1710 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
1713 /* opt_pass methods: */
1714 virtual bool gate (function
*);
1715 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
1719 pass_omp_simd_clone::gate (function
*)
1721 return targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
;
1726 simple_ipa_opt_pass
*
1727 make_pass_omp_simd_clone (gcc::context
*ctxt
)
1729 return new pass_omp_simd_clone (ctxt
);