1 /* OMP constructs' SIMD clone supporting code.
3 Copyright (C) 2005-2019 Free Software Foundation, Inc.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
23 #include "coretypes.h"
29 #include "alloc-pool.h"
30 #include "tree-pass.h"
33 #include "pretty-print.h"
34 #include "diagnostic-core.h"
35 #include "fold-const.h"
36 #include "stor-layout.h"
39 #include "gimple-iterator.h"
40 #include "gimplify-me.h"
41 #include "gimple-walk.h"
42 #include "langhooks.h"
44 #include "tree-into-ssa.h"
47 #include "symbol-summary.h"
48 #include "ipa-param-manipulation.h"
51 #include "stringpool.h"
53 #include "omp-simd-clone.h"
55 /* Return the number of elements in vector type VECTYPE, which is associated
56 with a SIMD clone. At present these always have a constant length. */
58 static unsigned HOST_WIDE_INT
59 simd_clone_subparts (tree vectype
)
61 return TYPE_VECTOR_SUBPARTS (vectype
).to_constant ();
64 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
65 of arguments to reserve space for. */
67 static struct cgraph_simd_clone
*
68 simd_clone_struct_alloc (int nargs
)
70 struct cgraph_simd_clone
*clone_info
;
71 size_t len
= (sizeof (struct cgraph_simd_clone
)
72 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
73 clone_info
= (struct cgraph_simd_clone
*)
74 ggc_internal_cleared_alloc (len
);
78 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
81 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
82 struct cgraph_simd_clone
*from
)
84 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
85 + ((from
->nargs
- from
->inbranch
)
86 * sizeof (struct cgraph_simd_clone_arg
))));
89 /* Return vector of parameter types of function FNDECL. This uses
90 TYPE_ARG_TYPES if available, otherwise falls back to types of
91 DECL_ARGUMENTS types. */
94 simd_clone_vector_of_formal_parm_types (tree fndecl
)
96 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
97 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
98 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
101 FOR_EACH_VEC_ELT (args
, i
, arg
)
102 args
[i
] = TREE_TYPE (args
[i
]);
106 /* Given a simd function in NODE, extract the simd specific
107 information from the OMP clauses passed in CLAUSES, and return
108 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
109 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
110 otherwise set to FALSE. */
112 static struct cgraph_simd_clone
*
113 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
114 bool *inbranch_specified
)
116 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
119 *inbranch_specified
= false;
122 if (n
> 0 && args
.last () == void_type_node
)
125 /* Allocate one more than needed just in case this is an in-branch
126 clone which will require a mask argument. */
127 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
128 clone_info
->nargs
= n
;
133 clauses
= TREE_VALUE (clauses
);
134 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
137 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
139 switch (OMP_CLAUSE_CODE (t
))
141 case OMP_CLAUSE_INBRANCH
:
142 clone_info
->inbranch
= 1;
143 *inbranch_specified
= true;
145 case OMP_CLAUSE_NOTINBRANCH
:
146 clone_info
->inbranch
= 0;
147 *inbranch_specified
= true;
149 case OMP_CLAUSE_SIMDLEN
:
151 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
153 case OMP_CLAUSE_LINEAR
:
155 tree decl
= OMP_CLAUSE_DECL (t
);
156 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
157 int argno
= TREE_INT_CST_LOW (decl
);
158 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
160 enum cgraph_simd_clone_arg_type arg_type
;
161 if (TREE_CODE (args
[argno
]) == REFERENCE_TYPE
)
162 switch (OMP_CLAUSE_LINEAR_KIND (t
))
164 case OMP_CLAUSE_LINEAR_REF
:
166 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
;
168 case OMP_CLAUSE_LINEAR_UVAL
:
170 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
;
172 case OMP_CLAUSE_LINEAR_VAL
:
173 case OMP_CLAUSE_LINEAR_DEFAULT
:
175 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
;
181 arg_type
= SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
182 clone_info
->args
[argno
].arg_type
= arg_type
;
183 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
184 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
185 && clone_info
->args
[argno
].linear_step
< n
);
189 if (POINTER_TYPE_P (args
[argno
]))
190 step
= fold_convert (ssizetype
, step
);
191 if (!tree_fits_shwi_p (step
))
193 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
194 "ignoring large linear step");
198 else if (integer_zerop (step
))
200 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
201 "ignoring zero linear step");
207 enum cgraph_simd_clone_arg_type arg_type
;
208 if (TREE_CODE (args
[argno
]) == REFERENCE_TYPE
)
209 switch (OMP_CLAUSE_LINEAR_KIND (t
))
211 case OMP_CLAUSE_LINEAR_REF
:
213 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
;
215 case OMP_CLAUSE_LINEAR_UVAL
:
217 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
;
219 case OMP_CLAUSE_LINEAR_VAL
:
220 case OMP_CLAUSE_LINEAR_DEFAULT
:
222 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
;
228 arg_type
= SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
229 clone_info
->args
[argno
].arg_type
= arg_type
;
230 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
235 case OMP_CLAUSE_UNIFORM
:
237 tree decl
= OMP_CLAUSE_DECL (t
);
238 int argno
= tree_to_uhwi (decl
);
239 clone_info
->args
[argno
].arg_type
240 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
243 case OMP_CLAUSE_ALIGNED
:
245 /* Ignore aligned (x) for declare simd, for the ABI we really
246 need an alignment specified. */
247 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
) == NULL_TREE
)
249 tree decl
= OMP_CLAUSE_DECL (t
);
250 int argno
= tree_to_uhwi (decl
);
251 clone_info
->args
[argno
].alignment
252 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
261 if (TYPE_ATOMIC (TREE_TYPE (TREE_TYPE (node
->decl
))))
263 warning_at (DECL_SOURCE_LOCATION (node
->decl
), 0,
264 "ignoring %<#pragma omp declare simd%> on function "
265 "with %<_Atomic%> qualified return type");
270 for (unsigned int argno
= 0; argno
< clone_info
->nargs
; argno
++)
271 if (TYPE_ATOMIC (args
[argno
])
272 && clone_info
->args
[argno
].arg_type
!= SIMD_CLONE_ARG_TYPE_UNIFORM
)
274 warning_at (DECL_SOURCE_LOCATION (node
->decl
), 0,
275 "ignoring %<#pragma omp declare simd%> on function "
276 "with %<_Atomic%> qualified non-%<uniform%> argument");
285 /* Given a SIMD clone in NODE, calculate the characteristic data
286 type and return the coresponding type. The characteristic data
287 type is computed as described in the Intel Vector ABI. */
290 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
291 struct cgraph_simd_clone
*clone_info
)
293 tree type
= integer_type_node
;
294 tree fndecl
= node
->decl
;
296 /* a) For non-void function, the characteristic data type is the
298 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
299 type
= TREE_TYPE (TREE_TYPE (fndecl
));
301 /* b) If the function has any non-uniform, non-linear parameters,
302 then the characteristic data type is the type of the first
306 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
307 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
308 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
316 /* c) If the characteristic data type determined by a) or b) above
317 is struct, union, or class type which is pass-by-value (except
318 for the type that maps to the built-in complex data type), the
319 characteristic data type is int. */
320 if (RECORD_OR_UNION_TYPE_P (type
)
321 && !aggregate_value_p (type
, NULL
)
322 && TREE_CODE (type
) != COMPLEX_TYPE
)
323 return integer_type_node
;
325 /* d) If none of the above three classes is applicable, the
326 characteristic data type is int. */
330 /* e) For Intel Xeon Phi native and offload compilation, if the
331 resulting characteristic data type is 8-bit or 16-bit integer
332 data type, the characteristic data type is int. */
333 /* Well, we don't handle Xeon Phi yet. */
337 simd_clone_mangle (struct cgraph_node
*node
,
338 struct cgraph_simd_clone
*clone_info
)
340 char vecsize_mangle
= clone_info
->vecsize_mangle
;
341 char mask
= clone_info
->inbranch
? 'M' : 'N';
342 unsigned int simdlen
= clone_info
->simdlen
;
346 gcc_assert (vecsize_mangle
&& simdlen
);
348 pp_string (&pp
, "_ZGV");
349 pp_character (&pp
, vecsize_mangle
);
350 pp_character (&pp
, mask
);
351 pp_decimal_int (&pp
, simdlen
);
353 for (n
= 0; n
< clone_info
->nargs
; ++n
)
355 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
357 switch (arg
.arg_type
)
359 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
360 pp_character (&pp
, 'u');
362 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
363 pp_character (&pp
, 'l');
365 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
366 pp_character (&pp
, 'R');
368 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
369 pp_character (&pp
, 'L');
371 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
372 pp_character (&pp
, 'U');
375 gcc_assert (arg
.linear_step
!= 0);
376 if (arg
.linear_step
> 1)
377 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
378 else if (arg
.linear_step
< 0)
380 pp_character (&pp
, 'n');
381 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
385 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
386 pp_string (&pp
, "ls");
387 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
389 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
390 pp_string (&pp
, "Rs");
391 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
393 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
394 pp_string (&pp
, "Ls");
395 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
397 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
398 pp_string (&pp
, "Us");
399 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
402 pp_character (&pp
, 'v');
406 pp_character (&pp
, 'a');
407 pp_decimal_int (&pp
, arg
.alignment
);
412 const char *str
= IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
));
415 pp_string (&pp
, str
);
416 str
= pp_formatted_text (&pp
);
418 /* If there already is a SIMD clone with the same mangled name, don't
419 add another one. This can happen e.g. for
420 #pragma omp declare simd
421 #pragma omp declare simd simdlen(8)
423 if the simdlen is assumed to be 8 for the first one, etc. */
424 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
425 clone
= clone
->simdclone
->next_clone
)
426 if (id_equal (DECL_ASSEMBLER_NAME (clone
->decl
), str
))
429 return get_identifier (str
);
432 /* Create a simd clone of OLD_NODE and return it. */
434 static struct cgraph_node
*
435 simd_clone_create (struct cgraph_node
*old_node
)
437 struct cgraph_node
*new_node
;
438 if (old_node
->definition
)
440 if (!old_node
->has_gimple_body_p ())
442 old_node
->get_body ();
443 new_node
= old_node
->create_version_clone_with_body (vNULL
, NULL
, NULL
,
449 tree old_decl
= old_node
->decl
;
450 tree new_decl
= copy_node (old_node
->decl
);
451 DECL_NAME (new_decl
) = clone_function_name_numbered (old_decl
,
453 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
454 SET_DECL_RTL (new_decl
, NULL
);
455 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
456 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
457 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
458 if (old_node
->in_other_partition
)
459 new_node
->in_other_partition
= 1;
461 if (new_node
== NULL
)
464 set_decl_built_in_function (new_node
->decl
, NOT_BUILT_IN
, 0);
465 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
466 DECL_COMDAT (new_node
->decl
) = DECL_COMDAT (old_node
->decl
);
467 DECL_WEAK (new_node
->decl
) = DECL_WEAK (old_node
->decl
);
468 DECL_EXTERNAL (new_node
->decl
) = DECL_EXTERNAL (old_node
->decl
);
469 DECL_VISIBILITY_SPECIFIED (new_node
->decl
)
470 = DECL_VISIBILITY_SPECIFIED (old_node
->decl
);
471 DECL_VISIBILITY (new_node
->decl
) = DECL_VISIBILITY (old_node
->decl
);
472 DECL_DLLIMPORT_P (new_node
->decl
) = DECL_DLLIMPORT_P (old_node
->decl
);
473 if (DECL_ONE_ONLY (old_node
->decl
))
474 make_decl_one_only (new_node
->decl
, DECL_ASSEMBLER_NAME (new_node
->decl
));
476 /* The method cgraph_version_clone_with_body () will force the new
477 symbol local. Undo this, and inherit external visibility from
479 new_node
->local
.local
= old_node
->local
.local
;
480 new_node
->externally_visible
= old_node
->externally_visible
;
485 /* Adjust the return type of the given function to its appropriate
486 vector counterpart. Returns a simd array to be used throughout the
487 function as a return value. */
490 simd_clone_adjust_return_type (struct cgraph_node
*node
)
492 tree fndecl
= node
->decl
;
493 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
497 /* Adjust the function return type. */
498 if (orig_rettype
== void_type_node
)
500 t
= TREE_TYPE (TREE_TYPE (fndecl
));
501 if (INTEGRAL_TYPE_P (t
) || POINTER_TYPE_P (t
))
502 veclen
= node
->simdclone
->vecsize_int
;
504 veclen
= node
->simdclone
->vecsize_float
;
505 veclen
/= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (t
));
506 if (veclen
> node
->simdclone
->simdlen
)
507 veclen
= node
->simdclone
->simdlen
;
508 if (POINTER_TYPE_P (t
))
509 t
= pointer_sized_int_node
;
510 if (veclen
== node
->simdclone
->simdlen
)
511 t
= build_vector_type (t
, node
->simdclone
->simdlen
);
514 t
= build_vector_type (t
, veclen
);
515 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
517 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
518 if (!node
->definition
)
521 t
= DECL_RESULT (fndecl
);
522 /* Adjust the DECL_RESULT. */
523 gcc_assert (TREE_TYPE (t
) != void_type_node
);
524 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
527 tree atype
= build_array_type_nelts (orig_rettype
,
528 node
->simdclone
->simdlen
);
529 if (veclen
!= node
->simdclone
->simdlen
)
530 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
532 /* Set up a SIMD array to use as the return value. */
533 tree retval
= create_tmp_var_raw (atype
, "retval");
534 gimple_add_tmp_var (retval
);
538 /* Each vector argument has a corresponding array to be used locally
539 as part of the eventual loop. Create such temporary array and
542 PREFIX is the prefix to be used for the temporary.
544 TYPE is the inner element type.
546 SIMDLEN is the number of elements. */
549 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
551 tree atype
= build_array_type_nelts (type
, simdlen
);
552 tree avar
= create_tmp_var_raw (atype
, prefix
);
553 gimple_add_tmp_var (avar
);
557 /* Modify the function argument types to their corresponding vector
558 counterparts if appropriate. Also, create one array for each simd
559 argument to be used locally when using the function arguments as
562 NODE is the function whose arguments are to be adjusted.
564 Returns an adjustment vector that will be filled describing how the
565 argument types will be adjusted. */
567 static ipa_parm_adjustment_vec
568 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
571 ipa_parm_adjustment_vec adjustments
;
573 if (node
->definition
)
574 args
= ipa_get_vector_of_formal_parms (node
->decl
);
576 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
577 adjustments
.create (args
.length ());
578 unsigned i
, j
, veclen
;
579 struct ipa_parm_adjustment adj
;
580 struct cgraph_simd_clone
*sc
= node
->simdclone
;
582 for (i
= 0; i
< sc
->nargs
; ++i
)
584 memset (&adj
, 0, sizeof (adj
));
586 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
590 sc
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
591 sc
->args
[i
].orig_type
= parm_type
;
593 switch (sc
->args
[i
].arg_type
)
596 /* No adjustment necessary for scalar arguments. */
597 adj
.op
= IPA_PARM_OP_COPY
;
599 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
600 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
601 if (node
->definition
)
602 sc
->args
[i
].simd_array
603 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
604 TREE_TYPE (parm_type
),
606 adj
.op
= IPA_PARM_OP_COPY
;
608 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
609 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
610 case SIMD_CLONE_ARG_TYPE_VECTOR
:
611 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
612 veclen
= sc
->vecsize_int
;
614 veclen
= sc
->vecsize_float
;
615 veclen
/= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (parm_type
));
616 if (veclen
> sc
->simdlen
)
617 veclen
= sc
->simdlen
;
618 adj
.arg_prefix
= "simd";
619 if (POINTER_TYPE_P (parm_type
))
620 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
622 adj
.type
= build_vector_type (parm_type
, veclen
);
623 sc
->args
[i
].vector_type
= adj
.type
;
624 for (j
= veclen
; j
< sc
->simdlen
; j
+= veclen
)
626 adjustments
.safe_push (adj
);
629 memset (&adj
, 0, sizeof (adj
));
630 adj
.op
= IPA_PARM_OP_NEW
;
631 adj
.arg_prefix
= "simd";
633 adj
.type
= sc
->args
[i
].vector_type
;
637 if (node
->definition
)
638 sc
->args
[i
].simd_array
639 = create_tmp_simd_array (DECL_NAME (parm
)
640 ? IDENTIFIER_POINTER (DECL_NAME (parm
))
641 : NULL
, parm_type
, sc
->simdlen
);
643 adjustments
.safe_push (adj
);
648 tree base_type
= simd_clone_compute_base_data_type (sc
->origin
, sc
);
650 memset (&adj
, 0, sizeof (adj
));
651 adj
.op
= IPA_PARM_OP_NEW
;
652 adj
.arg_prefix
= "mask";
655 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
656 veclen
= sc
->vecsize_int
;
658 veclen
= sc
->vecsize_float
;
659 veclen
/= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (base_type
));
660 if (veclen
> sc
->simdlen
)
661 veclen
= sc
->simdlen
;
662 if (sc
->mask_mode
!= VOIDmode
)
664 = lang_hooks
.types
.type_for_mode (sc
->mask_mode
, 1);
665 else if (POINTER_TYPE_P (base_type
))
666 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
668 adj
.type
= build_vector_type (base_type
, veclen
);
669 adjustments
.safe_push (adj
);
671 for (j
= veclen
; j
< sc
->simdlen
; j
+= veclen
)
672 adjustments
.safe_push (adj
);
674 /* We have previously allocated one extra entry for the mask. Use
677 if (sc
->mask_mode
!= VOIDmode
)
678 base_type
= boolean_type_node
;
679 if (node
->definition
)
682 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
683 if (sc
->mask_mode
== VOIDmode
)
684 sc
->args
[i
].simd_array
685 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
686 else if (veclen
< sc
->simdlen
)
687 sc
->args
[i
].simd_array
688 = create_tmp_simd_array ("mask", adj
.type
, sc
->simdlen
/ veclen
);
690 sc
->args
[i
].simd_array
= NULL_TREE
;
692 sc
->args
[i
].orig_type
= base_type
;
693 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
696 if (node
->definition
)
697 ipa_modify_formal_parameters (node
->decl
, adjustments
);
700 tree new_arg_types
= NULL_TREE
, new_reversed
;
701 bool last_parm_void
= false;
702 if (args
.length () > 0 && args
.last () == void_type_node
)
703 last_parm_void
= true;
705 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
706 j
= adjustments
.length ();
707 for (i
= 0; i
< j
; i
++)
709 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
711 if (adj
->op
== IPA_PARM_OP_COPY
)
712 ptype
= args
[adj
->base_index
];
715 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
717 new_reversed
= nreverse (new_arg_types
);
721 TREE_CHAIN (new_arg_types
) = void_list_node
;
723 new_reversed
= void_list_node
;
725 TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) = new_reversed
;
726 adjustments
.release ();
732 /* Initialize and copy the function arguments in NODE to their
733 corresponding local simd arrays. Returns a fresh gimple_seq with
734 the instruction sequence generated. */
737 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
738 ipa_parm_adjustment_vec adjustments
)
740 gimple_seq seq
= NULL
;
741 unsigned i
= 0, j
= 0, k
;
743 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
745 arg
= DECL_CHAIN (arg
), i
++, j
++)
747 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
748 || POINTER_TYPE_P (TREE_TYPE (arg
)))
751 node
->simdclone
->args
[i
].vector_arg
= arg
;
753 tree array
= node
->simdclone
->args
[i
].simd_array
;
754 if (node
->simdclone
->mask_mode
!= VOIDmode
755 && node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_MASK
)
757 if (array
== NULL_TREE
)
760 = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (array
))));
761 for (k
= 0; k
<= l
; k
++)
765 arg
= DECL_CHAIN (arg
);
768 tree t
= build4 (ARRAY_REF
, TREE_TYPE (TREE_TYPE (array
)),
769 array
, size_int (k
), NULL
, NULL
);
770 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
771 gimplify_and_add (t
, &seq
);
775 if (simd_clone_subparts (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
777 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
778 tree ptr
= build_fold_addr_expr (array
);
779 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
780 build_int_cst (ptype
, 0));
781 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
782 gimplify_and_add (t
, &seq
);
786 unsigned int simdlen
= simd_clone_subparts (TREE_TYPE (arg
));
787 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
788 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
790 tree ptr
= build_fold_addr_expr (array
);
794 arg
= DECL_CHAIN (arg
);
797 tree elemtype
= TREE_TYPE (TREE_TYPE (arg
));
798 elemsize
= GET_MODE_SIZE (SCALAR_TYPE_MODE (elemtype
));
799 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
800 build_int_cst (ptype
, k
* elemsize
));
801 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
802 gimplify_and_add (t
, &seq
);
809 /* Callback info for ipa_simd_modify_stmt_ops below. */
811 struct modify_stmt_info
{
812 ipa_parm_adjustment_vec adjustments
;
814 /* True if the parent statement was modified by
815 ipa_simd_modify_stmt_ops. */
819 /* Callback for walk_gimple_op.
821 Adjust operands from a given statement as specified in the
822 adjustments vector in the callback data. */
825 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
827 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
828 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
830 if (TREE_CODE (*tp
) == ADDR_EXPR
)
831 tp
= &TREE_OPERAND (*tp
, 0);
832 struct ipa_parm_adjustment
*cand
= NULL
;
833 if (TREE_CODE (*tp
) == PARM_DECL
)
834 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
835 else if (TYPE_P (*tp
))
838 tree repl
= NULL_TREE
;
840 repl
= unshare_expr (cand
->new_decl
);
846 bool modified
= info
->modified
;
847 info
->modified
= false;
848 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
851 info
->modified
= modified
;
854 info
->modified
= modified
;
863 if (gimple_code (info
->stmt
) == GIMPLE_PHI
865 && TREE_CODE (*orig_tp
) == ADDR_EXPR
866 && TREE_CODE (TREE_OPERAND (*orig_tp
, 0)) == PARM_DECL
867 && cand
->alias_ptr_type
)
869 gcc_assert (TREE_CODE (cand
->alias_ptr_type
) == SSA_NAME
);
870 *orig_tp
= cand
->alias_ptr_type
;
871 info
->modified
= true;
875 repl
= build_fold_addr_expr (repl
);
877 if (is_gimple_debug (info
->stmt
))
879 tree vexpr
= make_node (DEBUG_EXPR_DECL
);
880 stmt
= gimple_build_debug_source_bind (vexpr
, repl
, NULL
);
881 DECL_ARTIFICIAL (vexpr
) = 1;
882 TREE_TYPE (vexpr
) = TREE_TYPE (repl
);
883 SET_DECL_MODE (vexpr
, TYPE_MODE (TREE_TYPE (repl
)));
888 stmt
= gimple_build_assign (make_ssa_name (TREE_TYPE (repl
)), repl
);
889 repl
= gimple_assign_lhs (stmt
);
891 gimple_stmt_iterator gsi
;
892 if (gimple_code (info
->stmt
) == GIMPLE_PHI
)
894 gsi
= gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)));
895 /* Cache SSA_NAME for next time. */
897 && TREE_CODE (*orig_tp
) == ADDR_EXPR
898 && TREE_CODE (TREE_OPERAND (*orig_tp
, 0)) == PARM_DECL
)
899 cand
->alias_ptr_type
= repl
;
902 gsi
= gsi_for_stmt (info
->stmt
);
903 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
906 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
908 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
914 info
->modified
= true;
918 /* Traverse the function body and perform all modifications as
919 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
920 modified such that the replacement/reduction value will now be an
921 offset into the corresponding simd_array.
923 This function will replace all function argument uses with their
924 corresponding simd array elements, and ajust the return values
928 ipa_simd_modify_function_body (struct cgraph_node
*node
,
929 ipa_parm_adjustment_vec adjustments
,
930 tree retval_array
, tree iter
)
933 unsigned int i
, j
, l
;
935 /* Re-use the adjustments array, but this time use it to replace
936 every function argument use to an offset into the corresponding
938 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
940 if (!node
->simdclone
->args
[i
].vector_arg
)
943 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
944 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
945 adjustments
[j
].new_decl
948 node
->simdclone
->args
[i
].simd_array
,
950 NULL_TREE
, NULL_TREE
);
951 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
952 && simd_clone_subparts (vectype
) < node
->simdclone
->simdlen
)
953 j
+= node
->simdclone
->simdlen
/ simd_clone_subparts (vectype
) - 1;
956 l
= adjustments
.length ();
959 FOR_EACH_SSA_NAME (i
, name
, cfun
)
961 if (SSA_NAME_VAR (name
)
962 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
964 for (j
= 0; j
< l
; j
++)
965 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
966 && adjustments
[j
].new_decl
)
969 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
972 = copy_var_decl (adjustments
[j
].base
,
973 DECL_NAME (adjustments
[j
].base
),
974 TREE_TYPE (adjustments
[j
].base
));
975 adjustments
[j
].new_ssa_base
= base_var
;
978 base_var
= adjustments
[j
].new_ssa_base
;
979 if (SSA_NAME_IS_DEFAULT_DEF (name
))
981 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
982 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
983 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
984 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
985 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
986 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
987 gimple
*stmt
= gimple_build_assign (name
, new_decl
);
988 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
991 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
996 struct modify_stmt_info info
;
997 info
.adjustments
= adjustments
;
999 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
1001 gimple_stmt_iterator gsi
;
1003 for (gsi
= gsi_start_phis (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1005 gphi
*phi
= as_a
<gphi
*> (gsi_stmt (gsi
));
1006 int i
, n
= gimple_phi_num_args (phi
);
1008 struct walk_stmt_info wi
;
1009 memset (&wi
, 0, sizeof (wi
));
1010 info
.modified
= false;
1012 for (i
= 0; i
< n
; ++i
)
1014 int walk_subtrees
= 1;
1015 tree arg
= gimple_phi_arg_def (phi
, i
);
1017 ipa_simd_modify_stmt_ops (&op
, &walk_subtrees
, &wi
);
1020 SET_PHI_ARG_DEF (phi
, i
, op
);
1021 gcc_assert (TREE_CODE (op
) == SSA_NAME
);
1022 if (gimple_phi_arg_edge (phi
, i
)->flags
& EDGE_ABNORMAL
)
1023 SSA_NAME_OCCURS_IN_ABNORMAL_PHI (op
) = 1;
1028 gsi
= gsi_start_bb (bb
);
1029 while (!gsi_end_p (gsi
))
1031 gimple
*stmt
= gsi_stmt (gsi
);
1033 struct walk_stmt_info wi
;
1035 memset (&wi
, 0, sizeof (wi
));
1036 info
.modified
= false;
1038 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
1040 if (greturn
*return_stmt
= dyn_cast
<greturn
*> (stmt
))
1042 tree retval
= gimple_return_retval (return_stmt
);
1043 edge e
= find_edge (bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
));
1044 e
->flags
|= EDGE_FALLTHRU
;
1047 gsi_remove (&gsi
, true);
1051 /* Replace `return foo' with `retval_array[iter] = foo'. */
1052 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
1053 retval_array
, iter
, NULL
, NULL
);
1054 stmt
= gimple_build_assign (ref
, retval
);
1055 gsi_replace (&gsi
, stmt
, true);
1056 info
.modified
= true;
1062 /* If the above changed the var of a debug bind into something
1063 different, remove the debug stmt. We could also for all the
1064 replaced parameters add VAR_DECLs for debug info purposes,
1065 add debug stmts for those to be the simd array accesses and
1066 replace debug stmt var operand with that var. Debugging of
1067 vectorized loops doesn't work too well, so don't bother for
1069 if ((gimple_debug_bind_p (stmt
)
1070 && !DECL_P (gimple_debug_bind_get_var (stmt
)))
1071 || (gimple_debug_source_bind_p (stmt
)
1072 && !DECL_P (gimple_debug_source_bind_get_var (stmt
))))
1074 gsi_remove (&gsi
, true);
1077 if (maybe_clean_eh_stmt (stmt
))
1078 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
1085 /* Helper function of simd_clone_adjust, return linear step addend
1089 simd_clone_linear_addend (struct cgraph_node
*node
, unsigned int i
,
1090 tree addtype
, basic_block entry_bb
)
1092 tree ptype
= NULL_TREE
;
1093 switch (node
->simdclone
->args
[i
].arg_type
)
1095 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
1096 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
1097 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
1098 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
1099 return build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
1100 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
1101 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
1102 ptype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
1104 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
1105 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
1106 ptype
= TREE_TYPE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
));
1112 unsigned int idx
= node
->simdclone
->args
[i
].linear_step
;
1113 tree arg
= node
->simdclone
->args
[idx
].orig_arg
;
1114 gcc_assert (is_gimple_reg_type (TREE_TYPE (arg
)));
1115 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
1118 if (is_gimple_reg (arg
))
1119 ret
= get_or_create_ssa_default_def (cfun
, arg
);
1122 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (arg
)), arg
);
1123 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1124 ret
= gimple_assign_lhs (g
);
1126 if (TREE_CODE (TREE_TYPE (arg
)) == REFERENCE_TYPE
)
1128 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (TREE_TYPE (arg
))),
1129 build_simple_mem_ref (ret
));
1130 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1131 ret
= gimple_assign_lhs (g
);
1133 if (!useless_type_conversion_p (addtype
, TREE_TYPE (ret
)))
1135 g
= gimple_build_assign (make_ssa_name (addtype
), NOP_EXPR
, ret
);
1136 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1137 ret
= gimple_assign_lhs (g
);
1139 if (POINTER_TYPE_P (ptype
))
1141 tree size
= TYPE_SIZE_UNIT (TREE_TYPE (ptype
));
1142 if (size
&& TREE_CODE (size
) == INTEGER_CST
)
1144 g
= gimple_build_assign (make_ssa_name (addtype
), MULT_EXPR
,
1145 ret
, fold_convert (addtype
, size
));
1146 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1147 ret
= gimple_assign_lhs (g
);
1153 /* Adjust the argument types in NODE to their appropriate vector
1157 simd_clone_adjust (struct cgraph_node
*node
)
1159 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
1161 TREE_TYPE (node
->decl
) = build_distinct_type_copy (TREE_TYPE (node
->decl
));
1162 targetm
.simd_clone
.adjust (node
);
1164 tree retval
= simd_clone_adjust_return_type (node
);
1165 ipa_parm_adjustment_vec adjustments
1166 = simd_clone_adjust_argument_types (node
);
1168 push_gimplify_context ();
1170 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
1172 /* Adjust all uses of vector arguments accordingly. Adjust all
1173 return values accordingly. */
1174 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
1175 tree iter1
= make_ssa_name (iter
);
1176 tree iter2
= NULL_TREE
;
1177 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
1178 adjustments
.release ();
1180 /* Initialize the iteration variable. */
1181 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
1182 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
1183 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
1184 /* Insert the SIMD array and iv initialization at function
1186 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
1188 pop_gimplify_context (NULL
);
1191 basic_block incr_bb
= NULL
;
1192 class loop
*loop
= NULL
;
1194 /* Create a new BB right before the original exit BB, to hold the
1195 iteration increment and the condition/branch. */
1196 if (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
))
1198 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
1199 incr_bb
= create_empty_bb (orig_exit
);
1200 incr_bb
->count
= profile_count::zero ();
1201 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
1202 while (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
))
1204 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
1205 redirect_edge_succ (e
, incr_bb
);
1206 incr_bb
->count
+= e
->count ();
1209 else if (node
->simdclone
->inbranch
)
1211 incr_bb
= create_empty_bb (entry_bb
);
1212 incr_bb
->count
= profile_count::zero ();
1213 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
1218 make_single_succ_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
1219 gsi
= gsi_last_bb (incr_bb
);
1220 iter2
= make_ssa_name (iter
);
1221 g
= gimple_build_assign (iter2
, PLUS_EXPR
, iter1
,
1222 build_int_cst (unsigned_type_node
, 1));
1223 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1225 /* Mostly annotate the loop for the vectorizer (the rest is done
1227 loop
= alloc_loop ();
1228 cfun
->has_force_vectorize_loops
= true;
1229 loop
->safelen
= node
->simdclone
->simdlen
;
1230 loop
->force_vectorize
= true;
1231 loop
->header
= body_bb
;
1234 /* Branch around the body if the mask applies. */
1235 if (node
->simdclone
->inbranch
)
1237 gsi
= gsi_last_bb (loop
->header
);
1239 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
1241 if (node
->simdclone
->mask_mode
!= VOIDmode
)
1244 if (mask_array
== NULL_TREE
)
1246 tree arg
= node
->simdclone
->args
[node
->simdclone
->nargs
1248 mask
= get_or_create_ssa_default_def (cfun
, arg
);
1253 tree maskt
= TREE_TYPE (mask_array
);
1254 int c
= tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (maskt
)));
1255 c
= node
->simdclone
->simdlen
/ (c
+ 1);
1256 int s
= exact_log2 (c
);
1259 tree idx
= make_ssa_name (TREE_TYPE (iter1
));
1260 g
= gimple_build_assign (idx
, RSHIFT_EXPR
, iter1
,
1261 build_int_cst (NULL_TREE
, s
));
1262 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1263 mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)));
1264 tree aref
= build4 (ARRAY_REF
,
1265 TREE_TYPE (TREE_TYPE (mask_array
)),
1266 mask_array
, idx
, NULL
, NULL
);
1267 g
= gimple_build_assign (mask
, aref
);
1268 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1269 shift_cnt
= make_ssa_name (TREE_TYPE (iter1
));
1270 g
= gimple_build_assign (shift_cnt
, BIT_AND_EXPR
, iter1
,
1271 build_int_cst (TREE_TYPE (iter1
), c
));
1272 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1274 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (mask
)),
1275 RSHIFT_EXPR
, mask
, shift_cnt
);
1276 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1277 mask
= gimple_assign_lhs (g
);
1278 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (mask
)),
1280 build_int_cst (TREE_TYPE (mask
), 1));
1281 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1282 mask
= gimple_assign_lhs (g
);
1286 mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)));
1287 tree aref
= build4 (ARRAY_REF
,
1288 TREE_TYPE (TREE_TYPE (mask_array
)),
1289 mask_array
, iter1
, NULL
, NULL
);
1290 g
= gimple_build_assign (mask
, aref
);
1291 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1292 int bitsize
= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (TREE_TYPE (aref
)));
1293 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
1295 aref
= build1 (VIEW_CONVERT_EXPR
,
1296 build_nonstandard_integer_type (bitsize
, 0),
1298 mask
= make_ssa_name (TREE_TYPE (aref
));
1299 g
= gimple_build_assign (mask
, aref
);
1300 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1304 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
1306 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1307 edge e
= make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
1308 e
->probability
= profile_probability::unlikely ().guessed ();
1309 incr_bb
->count
+= e
->count ();
1310 edge fallthru
= FALLTHRU_EDGE (loop
->header
);
1311 fallthru
->flags
= EDGE_FALSE_VALUE
;
1312 fallthru
->probability
= profile_probability::likely ().guessed ();
1315 basic_block latch_bb
= NULL
;
1316 basic_block new_exit_bb
= NULL
;
1318 /* Generate the condition. */
1321 gsi
= gsi_last_bb (incr_bb
);
1322 g
= gimple_build_cond (LT_EXPR
, iter2
,
1323 build_int_cst (unsigned_type_node
,
1324 node
->simdclone
->simdlen
),
1326 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1327 edge e
= split_block (incr_bb
, gsi_stmt (gsi
));
1329 new_exit_bb
= split_block_after_labels (latch_bb
)->dest
;
1330 loop
->latch
= latch_bb
;
1332 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
1334 edge new_e
= make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
1336 /* FIXME: Do we need to distribute probabilities for the conditional? */
1337 new_e
->probability
= profile_probability::guessed_never ();
1338 /* The successor of incr_bb is already pointing to latch_bb; just
1340 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
1341 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
1344 gphi
*phi
= create_phi_node (iter1
, body_bb
);
1345 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
1346 edge latch_edge
= NULL
;
1347 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
1351 latch_edge
= single_succ_edge (latch_bb
);
1352 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
1354 /* Generate the new return. */
1355 gsi
= gsi_last_bb (new_exit_bb
);
1357 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
1358 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
1359 retval
= TREE_OPERAND (retval
, 0);
1362 retval
= build1 (VIEW_CONVERT_EXPR
,
1363 TREE_TYPE (TREE_TYPE (node
->decl
)),
1365 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
1366 false, GSI_CONTINUE_LINKING
);
1368 g
= gimple_build_return (retval
);
1369 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1372 /* Handle aligned clauses by replacing default defs of the aligned
1373 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
1374 lhs. Handle linear by adding PHIs. */
1375 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
1376 if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
1377 && (TREE_ADDRESSABLE (node
->simdclone
->args
[i
].orig_arg
)
1378 || !is_gimple_reg_type
1379 (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
1381 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1382 if (is_gimple_reg_type (TREE_TYPE (orig_arg
)))
1383 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
1386 iter1
= create_tmp_var_raw (TREE_TYPE (orig_arg
));
1387 gimple_add_tmp_var (iter1
);
1389 gsi
= gsi_after_labels (entry_bb
);
1390 g
= gimple_build_assign (iter1
, orig_arg
);
1391 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1392 gsi
= gsi_after_labels (body_bb
);
1393 g
= gimple_build_assign (orig_arg
, iter1
);
1394 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1396 else if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
1397 && DECL_BY_REFERENCE (node
->simdclone
->args
[i
].orig_arg
)
1398 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
1401 (TREE_TYPE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
1403 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1404 tree def
= ssa_default_def (cfun
, orig_arg
);
1405 if (def
&& !has_zero_uses (def
))
1407 iter1
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg
)));
1408 gimple_add_tmp_var (iter1
);
1409 gsi
= gsi_after_labels (entry_bb
);
1410 g
= gimple_build_assign (iter1
, build_simple_mem_ref (def
));
1411 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1412 gsi
= gsi_after_labels (body_bb
);
1413 g
= gimple_build_assign (build_simple_mem_ref (def
), iter1
);
1414 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1417 else if (node
->simdclone
->args
[i
].alignment
1418 && node
->simdclone
->args
[i
].arg_type
1419 == SIMD_CLONE_ARG_TYPE_UNIFORM
1420 && (node
->simdclone
->args
[i
].alignment
1421 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
1422 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
1425 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
1426 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1427 tree def
= ssa_default_def (cfun
, orig_arg
);
1428 if (def
&& !has_zero_uses (def
))
1430 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
1431 gimple_seq seq
= NULL
;
1432 bool need_cvt
= false;
1434 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
1436 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
1439 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
);
1440 gimple_call_set_lhs (g
, t
);
1441 gimple_seq_add_stmt_without_update (&seq
, g
);
1444 t
= make_ssa_name (orig_arg
);
1445 g
= gimple_build_assign (t
, NOP_EXPR
, gimple_call_lhs (g
));
1446 gimple_seq_add_stmt_without_update (&seq
, g
);
1448 gsi_insert_seq_on_edge_immediate
1449 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
1451 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
1452 node
->create_edge (cgraph_node::get_create (fn
),
1453 call
, entry_bb
->count
);
1455 imm_use_iterator iter
;
1456 use_operand_p use_p
;
1458 tree repl
= gimple_get_lhs (g
);
1459 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
1460 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
1463 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
1464 SET_USE (use_p
, repl
);
1467 else if ((node
->simdclone
->args
[i
].arg_type
1468 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
1469 || (node
->simdclone
->args
[i
].arg_type
1470 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
)
1471 || (node
->simdclone
->args
[i
].arg_type
1472 == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
1473 || (node
->simdclone
->args
[i
].arg_type
1474 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
))
1476 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1477 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
1478 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
1479 tree def
= NULL_TREE
;
1480 if (TREE_ADDRESSABLE (orig_arg
))
1482 def
= make_ssa_name (TREE_TYPE (orig_arg
));
1483 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
1485 iter2
= make_ssa_name (TREE_TYPE (orig_arg
));
1486 gsi
= gsi_after_labels (entry_bb
);
1487 g
= gimple_build_assign (def
, orig_arg
);
1488 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1492 def
= ssa_default_def (cfun
, orig_arg
);
1493 if (!def
|| has_zero_uses (def
))
1497 iter1
= make_ssa_name (orig_arg
);
1499 iter2
= make_ssa_name (orig_arg
);
1504 phi
= create_phi_node (iter1
, body_bb
);
1505 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
1508 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
1509 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
1510 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
1511 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
1512 ? TREE_TYPE (orig_arg
) : sizetype
;
1513 tree addcst
= simd_clone_linear_addend (node
, i
, addtype
,
1515 gsi
= gsi_last_bb (incr_bb
);
1516 g
= gimple_build_assign (iter2
, code
, iter1
, addcst
);
1517 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1520 imm_use_iterator iter
;
1521 use_operand_p use_p
;
1523 if (TREE_ADDRESSABLE (orig_arg
))
1525 gsi
= gsi_after_labels (body_bb
);
1526 g
= gimple_build_assign (orig_arg
, iter1
);
1527 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1530 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
1531 if (use_stmt
== phi
)
1534 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
1535 SET_USE (use_p
, iter1
);
1538 else if (node
->simdclone
->args
[i
].arg_type
1539 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
1540 || (node
->simdclone
->args
[i
].arg_type
1541 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
))
1543 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1544 tree def
= ssa_default_def (cfun
, orig_arg
);
1545 gcc_assert (!TREE_ADDRESSABLE (orig_arg
)
1546 && TREE_CODE (TREE_TYPE (orig_arg
)) == REFERENCE_TYPE
);
1547 if (def
&& !has_zero_uses (def
))
1549 tree rtype
= TREE_TYPE (TREE_TYPE (orig_arg
));
1550 iter1
= make_ssa_name (orig_arg
);
1552 iter2
= make_ssa_name (orig_arg
);
1553 tree iter3
= make_ssa_name (rtype
);
1554 tree iter4
= make_ssa_name (rtype
);
1555 tree iter5
= incr_bb
? make_ssa_name (rtype
) : NULL_TREE
;
1556 gsi
= gsi_after_labels (entry_bb
);
1558 = gimple_build_assign (iter3
, build_simple_mem_ref (def
));
1559 gsi_insert_before (&gsi
, load
, GSI_NEW_STMT
);
1561 tree array
= node
->simdclone
->args
[i
].simd_array
;
1562 TREE_ADDRESSABLE (array
) = 1;
1563 tree ptr
= build_fold_addr_expr (array
);
1564 phi
= create_phi_node (iter1
, body_bb
);
1565 add_phi_arg (phi
, ptr
, preheader_edge
, UNKNOWN_LOCATION
);
1568 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
1569 g
= gimple_build_assign (iter2
, POINTER_PLUS_EXPR
, iter1
,
1570 TYPE_SIZE_UNIT (TREE_TYPE (iter3
)));
1571 gsi
= gsi_last_bb (incr_bb
);
1572 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1575 phi
= create_phi_node (iter4
, body_bb
);
1576 add_phi_arg (phi
, iter3
, preheader_edge
, UNKNOWN_LOCATION
);
1579 add_phi_arg (phi
, iter5
, latch_edge
, UNKNOWN_LOCATION
);
1580 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (iter3
))
1581 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
1582 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (iter3
))
1583 ? TREE_TYPE (iter3
) : sizetype
;
1584 tree addcst
= simd_clone_linear_addend (node
, i
, addtype
,
1586 g
= gimple_build_assign (iter5
, code
, iter4
, addcst
);
1587 gsi
= gsi_last_bb (incr_bb
);
1588 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1591 g
= gimple_build_assign (build_simple_mem_ref (iter1
), iter4
);
1592 gsi
= gsi_after_labels (body_bb
);
1593 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1595 imm_use_iterator iter
;
1596 use_operand_p use_p
;
1598 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
1599 if (use_stmt
== load
)
1602 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
1603 SET_USE (use_p
, iter1
);
1605 if (!TYPE_READONLY (rtype
) && incr_bb
)
1607 tree v
= make_ssa_name (rtype
);
1608 tree aref
= build4 (ARRAY_REF
, rtype
, array
,
1609 size_zero_node
, NULL_TREE
,
1611 gsi
= gsi_after_labels (new_exit_bb
);
1612 g
= gimple_build_assign (v
, aref
);
1613 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1614 g
= gimple_build_assign (build_simple_mem_ref (def
), v
);
1615 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1620 calculate_dominance_info (CDI_DOMINATORS
);
1622 add_loop (loop
, loop
->header
->loop_father
);
1623 update_ssa (TODO_update_ssa
);
1628 /* If the function in NODE is tagged as an elemental SIMD function,
1629 create the appropriate SIMD clones. */
1632 expand_simd_clones (struct cgraph_node
*node
)
1634 tree attr
= lookup_attribute ("omp declare simd",
1635 DECL_ATTRIBUTES (node
->decl
));
1636 if (attr
== NULL_TREE
1637 || node
->global
.inlined_to
1638 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
1642 #pragma omp declare simd
1644 in C, there we don't know the argument types at all. */
1645 if (!node
->definition
1646 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
1649 /* Call this before creating clone_info, as it might ggc_collect. */
1650 if (node
->definition
&& node
->has_gimple_body_p ())
1655 /* Start with parsing the "omp declare simd" attribute(s). */
1656 bool inbranch_clause_specified
;
1657 struct cgraph_simd_clone
*clone_info
1658 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
1659 &inbranch_clause_specified
);
1660 if (clone_info
== NULL
)
1663 int orig_simdlen
= clone_info
->simdlen
;
1664 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
1665 /* The target can return 0 (no simd clones should be created),
1666 1 (just one ISA of simd clones should be created) or higher
1667 count of ISA variants. In that case, clone_info is initialized
1668 for the first ISA variant. */
1670 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
1675 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
1676 also create one inbranch and one !inbranch clone of it. */
1677 for (int i
= 0; i
< count
* 2; i
++)
1679 struct cgraph_simd_clone
*clone
= clone_info
;
1680 if (inbranch_clause_specified
&& (i
& 1) != 0)
1685 clone
= simd_clone_struct_alloc (clone_info
->nargs
1687 simd_clone_struct_copy (clone
, clone_info
);
1688 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
1689 and simd_clone_adjust_argument_types did to the first
1691 clone
->nargs
-= clone_info
->inbranch
;
1692 clone
->simdlen
= orig_simdlen
;
1693 /* And call the target hook again to get the right ISA. */
1694 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
1698 clone
->inbranch
= 1;
1701 /* simd_clone_mangle might fail if such a clone has been created
1703 tree id
= simd_clone_mangle (node
, clone
);
1704 if (id
== NULL_TREE
)
1707 /* Only when we are sure we want to create the clone actually
1708 clone the function (or definitions) or create another
1709 extern FUNCTION_DECL (for prototypes without definitions). */
1710 struct cgraph_node
*n
= simd_clone_create (node
);
1714 n
->simdclone
= clone
;
1715 clone
->origin
= node
;
1716 clone
->next_clone
= NULL
;
1717 if (node
->simd_clones
== NULL
)
1719 clone
->prev_clone
= n
;
1720 node
->simd_clones
= n
;
1724 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
1725 clone
->prev_clone
->simdclone
->next_clone
= n
;
1726 node
->simd_clones
->simdclone
->prev_clone
= n
;
1728 symtab
->change_decl_assembler_name (n
->decl
, id
);
1729 /* And finally adjust the return type, parameters and for
1730 definitions also function body. */
1731 if (node
->definition
)
1732 simd_clone_adjust (n
);
1736 = build_distinct_type_copy (TREE_TYPE (n
->decl
));
1737 targetm
.simd_clone
.adjust (n
);
1738 simd_clone_adjust_return_type (n
);
1739 simd_clone_adjust_argument_types (n
);
1743 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
1746 /* Entry point for IPA simd clone creation pass. */
1749 ipa_omp_simd_clone (void)
1751 struct cgraph_node
*node
;
1752 FOR_EACH_FUNCTION (node
)
1753 expand_simd_clones (node
);
1759 const pass_data pass_data_omp_simd_clone
=
1761 SIMPLE_IPA_PASS
, /* type */
1762 "simdclone", /* name */
1763 OPTGROUP_OMP
, /* optinfo_flags */
1764 TV_NONE
, /* tv_id */
1765 ( PROP_ssa
| PROP_cfg
), /* properties_required */
1766 0, /* properties_provided */
1767 0, /* properties_destroyed */
1768 0, /* todo_flags_start */
1769 0, /* todo_flags_finish */
1772 class pass_omp_simd_clone
: public simple_ipa_opt_pass
1775 pass_omp_simd_clone(gcc::context
*ctxt
)
1776 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
1779 /* opt_pass methods: */
1780 virtual bool gate (function
*);
1781 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
1785 pass_omp_simd_clone::gate (function
*)
1787 return targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
;
1792 simple_ipa_opt_pass
*
1793 make_pass_omp_simd_clone (gcc::context
*ctxt
)
1795 return new pass_omp_simd_clone (ctxt
);