1 /* Lowering pass for OMP directives. Converts OMP directives into explicit
2 calls to the runtime library (libgomp), data marshalling to implement data
3 sharing and copying clauses, offloading to accelerators, and more.
5 Contributed by Diego Novillo <dnovillo@redhat.com>
7 Copyright (C) 2005-2015 Free Software Foundation, Inc.
9 This file is part of GCC.
11 GCC is free software; you can redistribute it and/or modify it under
12 the terms of the GNU General Public License as published by the Free
13 Software Foundation; either version 3, or (at your option) any later
16 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
17 WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
21 You should have received a copy of the GNU General Public License
22 along with GCC; see the file COPYING3. If not see
23 <http://www.gnu.org/licenses/>. */
27 #include "coretypes.h"
34 #include "alloc-pool.h"
35 #include "tree-pass.h"
41 #include "pretty-print.h"
42 #include "diagnostic-core.h"
44 #include "fold-const.h"
45 #include "stor-layout.h"
47 #include "internal-fn.h"
48 #include "gimple-fold.h"
50 #include "gimple-iterator.h"
51 #include "gimplify-me.h"
52 #include "gimple-walk.h"
53 #include "tree-iterator.h"
54 #include "tree-inline.h"
55 #include "langhooks.h"
57 #include "tree-into-ssa.h"
68 #include "splay-tree.h"
70 #include "common/common-target.h"
72 #include "gimple-low.h"
73 #include "tree-cfgcleanup.h"
74 #include "symbol-summary.h"
76 #include "tree-nested.h"
80 #include "lto-section-names.h"
81 #include "gomp-constants.h"
82 #include "gimple-pretty-print.h"
84 /* Lowering of OMP parallel and workshare constructs proceeds in two
85 phases. The first phase scans the function looking for OMP statements
86 and then for variables that must be replaced to satisfy data sharing
87 clauses. The second phase expands code for the constructs, as well as
88 re-gimplifying things when variables have been replaced with complex
91 Final code generation is done by pass_expand_omp. The flowgraph is
92 scanned for regions which are then moved to a new
93 function, to be invoked by the thread library, or offloaded. */
95 /* OMP region information. Every parallel and workshare
96 directive is enclosed between two markers, the OMP_* directive
97 and a corresponding GIMPLE_OMP_RETURN statement. */
101 /* The enclosing region. */
102 struct omp_region
*outer
;
104 /* First child region. */
105 struct omp_region
*inner
;
107 /* Next peer region. */
108 struct omp_region
*next
;
110 /* Block containing the omp directive as its last stmt. */
113 /* Block containing the GIMPLE_OMP_RETURN as its last stmt. */
116 /* Block containing the GIMPLE_OMP_CONTINUE as its last stmt. */
119 /* If this is a combined parallel+workshare region, this is a list
120 of additional arguments needed by the combined parallel+workshare
122 vec
<tree
, va_gc
> *ws_args
;
124 /* The code for the omp directive of this region. */
125 enum gimple_code type
;
127 /* Schedule kind, only used for GIMPLE_OMP_FOR type regions. */
128 enum omp_clause_schedule_kind sched_kind
;
130 /* Schedule modifiers. */
131 unsigned char sched_modifiers
;
133 /* True if this is a combined parallel+workshare region. */
134 bool is_combined_parallel
;
136 /* The ordered stmt if type is GIMPLE_OMP_ORDERED and it has
138 gomp_ordered
*ord_stmt
;
141 /* Context structure. Used to store information about each parallel
142 directive in the code. */
146 /* This field must be at the beginning, as we do "inheritance": Some
147 callback functions for tree-inline.c (e.g., omp_copy_decl)
148 receive a copy_body_data pointer that is up-casted to an
149 omp_context pointer. */
152 /* The tree of contexts corresponding to the encountered constructs. */
153 struct omp_context
*outer
;
156 /* Map variables to fields in a structure that allows communication
157 between sending and receiving threads. */
158 splay_tree field_map
;
163 /* These are used just by task contexts, if task firstprivate fn is
164 needed. srecord_type is used to communicate from the thread
165 that encountered the task construct to task firstprivate fn,
166 record_type is allocated by GOMP_task, initialized by task firstprivate
167 fn and passed to the task body fn. */
168 splay_tree sfield_map
;
171 /* A chain of variables to add to the top-level block surrounding the
172 construct. In the case of a parallel, this is in the child function. */
175 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
176 barriers should jump to during omplower pass. */
179 /* What to do with variables with implicitly determined sharing
181 enum omp_clause_default_kind default_kind
;
183 /* Nesting depth of this context. Used to beautify error messages re
184 invalid gotos. The outermost ctx is depth 1, with depth 0 being
185 reserved for the main body of the function. */
188 /* True if this parallel directive is nested within another. */
191 /* True if this construct can be cancelled. */
195 /* A structure holding the elements of:
196 for (V = N1; V cond N2; V += STEP) [...] */
198 struct omp_for_data_loop
200 tree v
, n1
, n2
, step
;
201 enum tree_code cond_code
;
204 /* A structure describing the main elements of a parallel loop. */
208 struct omp_for_data_loop loop
;
214 bool have_nowait
, have_ordered
, simd_schedule
;
215 unsigned char sched_modifiers
;
216 enum omp_clause_schedule_kind sched_kind
;
217 struct omp_for_data_loop
*loops
;
220 /* Describe the OpenACC looping structure of a function. The entire
221 function is held in a 'NULL' loop. */
225 oacc_loop
*parent
; /* Containing loop. */
227 oacc_loop
*child
; /* First inner loop. */
229 oacc_loop
*sibling
; /* Next loop within same parent. */
231 location_t loc
; /* Location of the loop start. */
233 gcall
*marker
; /* Initial head marker. */
235 gcall
*heads
[GOMP_DIM_MAX
]; /* Head marker functions. */
236 gcall
*tails
[GOMP_DIM_MAX
]; /* Tail marker functions. */
238 tree routine
; /* Pseudo-loop enclosing a routine. */
240 unsigned mask
; /* Partitioning mask. */
241 unsigned flags
; /* Partitioning flags. */
242 tree chunk_size
; /* Chunk size. */
243 gcall
*head_end
; /* Final marker of head sequence. */
246 /* Flags for an OpenACC loop. */
248 enum oacc_loop_flags
{
249 OLF_SEQ
= 1u << 0, /* Explicitly sequential */
250 OLF_AUTO
= 1u << 1, /* Compiler chooses axes. */
251 OLF_INDEPENDENT
= 1u << 2, /* Iterations are known independent. */
252 OLF_GANG_STATIC
= 1u << 3, /* Gang partitioning is static (has op). */
254 /* Explicitly specified loop axes. */
256 OLF_DIM_GANG
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_GANG
),
257 OLF_DIM_WORKER
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_WORKER
),
258 OLF_DIM_VECTOR
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_VECTOR
),
260 OLF_MAX
= OLF_DIM_BASE
+ GOMP_DIM_MAX
264 static splay_tree all_contexts
;
265 static int taskreg_nesting_level
;
266 static int target_nesting_level
;
267 static struct omp_region
*root_omp_region
;
268 static bitmap task_shared_vars
;
269 static vec
<omp_context
*> taskreg_contexts
;
270 static bool omp_any_child_fn_dumped
;
272 static void scan_omp (gimple_seq
*, omp_context
*);
273 static tree
scan_omp_1_op (tree
*, int *, void *);
274 static gphi
*find_phi_with_arg_on_edge (tree
, edge
);
276 #define WALK_SUBSTMTS \
280 case GIMPLE_EH_FILTER: \
281 case GIMPLE_TRANSACTION: \
282 /* The sub-statements for these should be walked. */ \
283 *handled_ops_p = false; \
286 /* Return true if CTX corresponds to an oacc parallel region. */
289 is_oacc_parallel (omp_context
*ctx
)
291 enum gimple_code outer_type
= gimple_code (ctx
->stmt
);
292 return ((outer_type
== GIMPLE_OMP_TARGET
)
293 && (gimple_omp_target_kind (ctx
->stmt
)
294 == GF_OMP_TARGET_KIND_OACC_PARALLEL
));
297 /* Return true if CTX corresponds to an oacc kernels region. */
300 is_oacc_kernels (omp_context
*ctx
)
302 enum gimple_code outer_type
= gimple_code (ctx
->stmt
);
303 return ((outer_type
== GIMPLE_OMP_TARGET
)
304 && (gimple_omp_target_kind (ctx
->stmt
)
305 == GF_OMP_TARGET_KIND_OACC_KERNELS
));
308 /* If DECL is the artificial dummy VAR_DECL created for non-static
309 data member privatization, return the underlying "this" parameter,
310 otherwise return NULL. */
313 omp_member_access_dummy_var (tree decl
)
316 || !DECL_ARTIFICIAL (decl
)
317 || !DECL_IGNORED_P (decl
)
318 || !DECL_HAS_VALUE_EXPR_P (decl
)
319 || !lang_hooks
.decls
.omp_disregard_value_expr (decl
, false))
322 tree v
= DECL_VALUE_EXPR (decl
);
323 if (TREE_CODE (v
) != COMPONENT_REF
)
327 switch (TREE_CODE (v
))
333 case POINTER_PLUS_EXPR
:
334 v
= TREE_OPERAND (v
, 0);
337 if (DECL_CONTEXT (v
) == current_function_decl
338 && DECL_ARTIFICIAL (v
)
339 && TREE_CODE (TREE_TYPE (v
)) == POINTER_TYPE
)
347 /* Helper for unshare_and_remap, called through walk_tree. */
350 unshare_and_remap_1 (tree
*tp
, int *walk_subtrees
, void *data
)
352 tree
*pair
= (tree
*) data
;
355 *tp
= unshare_expr (pair
[1]);
358 else if (IS_TYPE_OR_DECL_P (*tp
))
363 /* Return unshare_expr (X) with all occurrences of FROM
367 unshare_and_remap (tree x
, tree from
, tree to
)
369 tree pair
[2] = { from
, to
};
370 x
= unshare_expr (x
);
371 walk_tree (&x
, unshare_and_remap_1
, pair
, NULL
);
375 /* Holds offload tables with decls. */
376 vec
<tree
, va_gc
> *offload_funcs
, *offload_vars
;
378 /* Convenience function for calling scan_omp_1_op on tree operands. */
381 scan_omp_op (tree
*tp
, omp_context
*ctx
)
383 struct walk_stmt_info wi
;
385 memset (&wi
, 0, sizeof (wi
));
387 wi
.want_locations
= true;
389 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
392 static void lower_omp (gimple_seq
*, omp_context
*);
393 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
394 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
396 /* Find an OMP clause of type KIND within CLAUSES. */
399 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
401 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
402 if (OMP_CLAUSE_CODE (clauses
) == kind
)
408 /* Return true if CTX is for an omp parallel. */
411 is_parallel_ctx (omp_context
*ctx
)
413 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
417 /* Return true if CTX is for an omp task. */
420 is_task_ctx (omp_context
*ctx
)
422 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
426 /* Return true if CTX is for an omp taskloop. */
429 is_taskloop_ctx (omp_context
*ctx
)
431 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
432 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_TASKLOOP
;
436 /* Return true if CTX is for an omp parallel or omp task. */
439 is_taskreg_ctx (omp_context
*ctx
)
441 return is_parallel_ctx (ctx
) || is_task_ctx (ctx
);
445 /* Return true if REGION is a combined parallel+workshare region. */
448 is_combined_parallel (struct omp_region
*region
)
450 return region
->is_combined_parallel
;
454 /* Extract the header elements of parallel loop FOR_STMT and store
458 extract_omp_for_data (gomp_for
*for_stmt
, struct omp_for_data
*fd
,
459 struct omp_for_data_loop
*loops
)
461 tree t
, var
, *collapse_iter
, *collapse_count
;
462 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
463 struct omp_for_data_loop
*loop
;
465 struct omp_for_data_loop dummy_loop
;
466 location_t loc
= gimple_location (for_stmt
);
467 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_SIMD
;
468 bool distribute
= gimple_omp_for_kind (for_stmt
)
469 == GF_OMP_FOR_KIND_DISTRIBUTE
;
470 bool taskloop
= gimple_omp_for_kind (for_stmt
)
471 == GF_OMP_FOR_KIND_TASKLOOP
;
474 fd
->for_stmt
= for_stmt
;
476 if (gimple_omp_for_collapse (for_stmt
) > 1)
479 fd
->loops
= &fd
->loop
;
481 fd
->have_nowait
= distribute
|| simd
;
482 fd
->have_ordered
= false;
485 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
486 fd
->sched_modifiers
= 0;
487 fd
->chunk_size
= NULL_TREE
;
488 fd
->simd_schedule
= false;
489 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
490 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_CILKFOR
;
491 collapse_iter
= NULL
;
492 collapse_count
= NULL
;
494 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
495 switch (OMP_CLAUSE_CODE (t
))
497 case OMP_CLAUSE_NOWAIT
:
498 fd
->have_nowait
= true;
500 case OMP_CLAUSE_ORDERED
:
501 fd
->have_ordered
= true;
502 if (OMP_CLAUSE_ORDERED_EXPR (t
))
503 fd
->ordered
= tree_to_shwi (OMP_CLAUSE_ORDERED_EXPR (t
));
505 case OMP_CLAUSE_SCHEDULE
:
506 gcc_assert (!distribute
&& !taskloop
);
508 = (enum omp_clause_schedule_kind
)
509 (OMP_CLAUSE_SCHEDULE_KIND (t
) & OMP_CLAUSE_SCHEDULE_MASK
);
510 fd
->sched_modifiers
= (OMP_CLAUSE_SCHEDULE_KIND (t
)
511 & ~OMP_CLAUSE_SCHEDULE_MASK
);
512 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
513 fd
->simd_schedule
= OMP_CLAUSE_SCHEDULE_SIMD (t
);
515 case OMP_CLAUSE_DIST_SCHEDULE
:
516 gcc_assert (distribute
);
517 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
519 case OMP_CLAUSE_COLLAPSE
:
520 fd
->collapse
= tree_to_shwi (OMP_CLAUSE_COLLAPSE_EXPR (t
));
521 if (fd
->collapse
> 1)
523 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
524 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
530 if (fd
->ordered
&& fd
->collapse
== 1 && loops
!= NULL
)
535 collapse_iter
= &iterv
;
536 collapse_count
= &countv
;
539 /* FIXME: for now map schedule(auto) to schedule(static).
540 There should be analysis to determine whether all iterations
541 are approximately the same amount of work (then schedule(static)
542 is best) or if it varies (then schedule(dynamic,N) is better). */
543 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
545 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
546 gcc_assert (fd
->chunk_size
== NULL
);
548 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
550 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_RUNTIME
;
551 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
552 gcc_assert (fd
->chunk_size
== NULL
);
553 else if (fd
->chunk_size
== NULL
)
555 /* We only need to compute a default chunk size for ordered
556 static loops and dynamic loops. */
557 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
559 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
560 ? integer_zero_node
: integer_one_node
;
563 int cnt
= fd
->ordered
? fd
->ordered
: fd
->collapse
;
564 for (i
= 0; i
< cnt
; i
++)
566 if (i
== 0 && fd
->collapse
== 1 && (fd
->ordered
== 0 || loops
== NULL
))
568 else if (loops
!= NULL
)
573 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
574 gcc_assert (SSA_VAR_P (loop
->v
));
575 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
576 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
577 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
578 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
580 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
581 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
582 switch (loop
->cond_code
)
588 gcc_assert (gimple_omp_for_kind (for_stmt
)
589 == GF_OMP_FOR_KIND_CILKSIMD
590 || (gimple_omp_for_kind (for_stmt
)
591 == GF_OMP_FOR_KIND_CILKFOR
));
594 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
595 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, 1);
597 loop
->n2
= fold_build2_loc (loc
,
598 PLUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
599 build_int_cst (TREE_TYPE (loop
->n2
), 1));
600 loop
->cond_code
= LT_EXPR
;
603 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
604 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, -1);
606 loop
->n2
= fold_build2_loc (loc
,
607 MINUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
608 build_int_cst (TREE_TYPE (loop
->n2
), 1));
609 loop
->cond_code
= GT_EXPR
;
615 t
= gimple_omp_for_incr (for_stmt
, i
);
616 gcc_assert (TREE_OPERAND (t
, 0) == var
);
617 switch (TREE_CODE (t
))
620 loop
->step
= TREE_OPERAND (t
, 1);
622 case POINTER_PLUS_EXPR
:
623 loop
->step
= fold_convert (ssizetype
, TREE_OPERAND (t
, 1));
626 loop
->step
= TREE_OPERAND (t
, 1);
627 loop
->step
= fold_build1_loc (loc
,
628 NEGATE_EXPR
, TREE_TYPE (loop
->step
),
636 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
637 && !fd
->have_ordered
))
639 if (fd
->collapse
== 1)
640 iter_type
= TREE_TYPE (loop
->v
);
642 || TYPE_PRECISION (iter_type
)
643 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
645 = build_nonstandard_integer_type
646 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
648 else if (iter_type
!= long_long_unsigned_type_node
)
650 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
651 iter_type
= long_long_unsigned_type_node
;
652 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
653 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
654 >= TYPE_PRECISION (iter_type
))
658 if (loop
->cond_code
== LT_EXPR
)
659 n
= fold_build2_loc (loc
,
660 PLUS_EXPR
, TREE_TYPE (loop
->v
),
661 loop
->n2
, loop
->step
);
664 if (TREE_CODE (n
) != INTEGER_CST
665 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
666 iter_type
= long_long_unsigned_type_node
;
668 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
669 > TYPE_PRECISION (iter_type
))
673 if (loop
->cond_code
== LT_EXPR
)
676 n2
= fold_build2_loc (loc
,
677 PLUS_EXPR
, TREE_TYPE (loop
->v
),
678 loop
->n2
, loop
->step
);
682 n1
= fold_build2_loc (loc
,
683 MINUS_EXPR
, TREE_TYPE (loop
->v
),
684 loop
->n2
, loop
->step
);
687 if (TREE_CODE (n1
) != INTEGER_CST
688 || TREE_CODE (n2
) != INTEGER_CST
689 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
690 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
691 iter_type
= long_long_unsigned_type_node
;
695 if (i
>= fd
->collapse
)
698 if (collapse_count
&& *collapse_count
== NULL
)
700 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
701 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
702 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
703 if (t
&& integer_zerop (t
))
704 count
= build_zero_cst (long_long_unsigned_type_node
);
705 else if ((i
== 0 || count
!= NULL_TREE
)
706 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
707 && TREE_CONSTANT (loop
->n1
)
708 && TREE_CONSTANT (loop
->n2
)
709 && TREE_CODE (loop
->step
) == INTEGER_CST
)
711 tree itype
= TREE_TYPE (loop
->v
);
713 if (POINTER_TYPE_P (itype
))
714 itype
= signed_type_for (itype
);
715 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
716 t
= fold_build2_loc (loc
,
718 fold_convert_loc (loc
, itype
, loop
->step
), t
);
719 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
720 fold_convert_loc (loc
, itype
, loop
->n2
));
721 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
722 fold_convert_loc (loc
, itype
, loop
->n1
));
723 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
724 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
725 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
726 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
727 fold_convert_loc (loc
, itype
,
730 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
731 fold_convert_loc (loc
, itype
, loop
->step
));
732 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
733 if (count
!= NULL_TREE
)
734 count
= fold_build2_loc (loc
,
735 MULT_EXPR
, long_long_unsigned_type_node
,
739 if (TREE_CODE (count
) != INTEGER_CST
)
742 else if (count
&& !integer_zerop (count
))
749 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
750 || fd
->have_ordered
))
752 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
753 iter_type
= long_long_unsigned_type_node
;
755 iter_type
= long_integer_type_node
;
757 else if (collapse_iter
&& *collapse_iter
!= NULL
)
758 iter_type
= TREE_TYPE (*collapse_iter
);
759 fd
->iter_type
= iter_type
;
760 if (collapse_iter
&& *collapse_iter
== NULL
)
761 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
762 if (collapse_count
&& *collapse_count
== NULL
)
765 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
767 *collapse_count
= create_tmp_var (iter_type
, ".count");
770 if (fd
->collapse
> 1 || (fd
->ordered
&& loops
))
772 fd
->loop
.v
= *collapse_iter
;
773 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
774 fd
->loop
.n2
= *collapse_count
;
775 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
776 fd
->loop
.cond_code
= LT_EXPR
;
783 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
784 is the immediate dominator of PAR_ENTRY_BB, return true if there
785 are no data dependencies that would prevent expanding the parallel
786 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
788 When expanding a combined parallel+workshare region, the call to
789 the child function may need additional arguments in the case of
790 GIMPLE_OMP_FOR regions. In some cases, these arguments are
791 computed out of variables passed in from the parent to the child
792 via 'struct .omp_data_s'. For instance:
794 #pragma omp parallel for schedule (guided, i * 4)
799 # BLOCK 2 (PAR_ENTRY_BB)
801 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
803 # BLOCK 3 (WS_ENTRY_BB)
804 .omp_data_i = &.omp_data_o;
805 D.1667 = .omp_data_i->i;
807 #pragma omp for schedule (guided, D.1598)
809 When we outline the parallel region, the call to the child function
810 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
811 that value is computed *after* the call site. So, in principle we
812 cannot do the transformation.
814 To see whether the code in WS_ENTRY_BB blocks the combined
815 parallel+workshare call, we collect all the variables used in the
816 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
817 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
820 FIXME. If we had the SSA form built at this point, we could merely
821 hoist the code in block 3 into block 2 and be done with it. But at
822 this point we don't have dataflow information and though we could
823 hack something up here, it is really not worth the aggravation. */
826 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
828 struct omp_for_data fd
;
829 gimple
*ws_stmt
= last_stmt (ws_entry_bb
);
831 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
834 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
836 extract_omp_for_data (as_a
<gomp_for
*> (ws_stmt
), &fd
, NULL
);
838 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
840 if (fd
.iter_type
!= long_integer_type_node
)
843 /* FIXME. We give up too easily here. If any of these arguments
844 are not constants, they will likely involve variables that have
845 been mapped into fields of .omp_data_s for sharing with the child
846 function. With appropriate data flow, it would be possible to
848 if (!is_gimple_min_invariant (fd
.loop
.n1
)
849 || !is_gimple_min_invariant (fd
.loop
.n2
)
850 || !is_gimple_min_invariant (fd
.loop
.step
)
851 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
858 static int omp_max_vf (void);
860 /* Adjust CHUNK_SIZE from SCHEDULE clause, depending on simd modifier
861 presence (SIMD_SCHEDULE). */
864 omp_adjust_chunk_size (tree chunk_size
, bool simd_schedule
)
869 int vf
= omp_max_vf ();
873 tree type
= TREE_TYPE (chunk_size
);
874 chunk_size
= fold_build2 (PLUS_EXPR
, type
, chunk_size
,
875 build_int_cst (type
, vf
- 1));
876 return fold_build2 (BIT_AND_EXPR
, type
, chunk_size
,
877 build_int_cst (type
, -vf
));
881 /* Collect additional arguments needed to emit a combined
882 parallel+workshare call. WS_STMT is the workshare directive being
885 static vec
<tree
, va_gc
> *
886 get_ws_args_for (gimple
*par_stmt
, gimple
*ws_stmt
)
889 location_t loc
= gimple_location (ws_stmt
);
890 vec
<tree
, va_gc
> *ws_args
;
892 if (gomp_for
*for_stmt
= dyn_cast
<gomp_for
*> (ws_stmt
))
894 struct omp_for_data fd
;
897 extract_omp_for_data (for_stmt
, &fd
, NULL
);
901 if (gimple_omp_for_combined_into_p (for_stmt
))
904 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
905 OMP_CLAUSE__LOOPTEMP_
);
907 n1
= OMP_CLAUSE_DECL (innerc
);
908 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
909 OMP_CLAUSE__LOOPTEMP_
);
911 n2
= OMP_CLAUSE_DECL (innerc
);
914 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
916 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
917 ws_args
->quick_push (t
);
919 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
920 ws_args
->quick_push (t
);
922 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
923 ws_args
->quick_push (t
);
927 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
928 t
= omp_adjust_chunk_size (t
, fd
.simd_schedule
);
929 ws_args
->quick_push (t
);
934 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
936 /* Number of sections is equal to the number of edges from the
937 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
938 the exit of the sections region. */
939 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
940 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
941 vec_alloc (ws_args
, 1);
942 ws_args
->quick_push (t
);
950 /* Discover whether REGION is a combined parallel+workshare region. */
953 determine_parallel_type (struct omp_region
*region
)
955 basic_block par_entry_bb
, par_exit_bb
;
956 basic_block ws_entry_bb
, ws_exit_bb
;
958 if (region
== NULL
|| region
->inner
== NULL
959 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
960 || region
->inner
->cont
== NULL
)
963 /* We only support parallel+for and parallel+sections. */
964 if (region
->type
!= GIMPLE_OMP_PARALLEL
965 || (region
->inner
->type
!= GIMPLE_OMP_FOR
966 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
969 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
970 WS_EXIT_BB -> PAR_EXIT_BB. */
971 par_entry_bb
= region
->entry
;
972 par_exit_bb
= region
->exit
;
973 ws_entry_bb
= region
->inner
->entry
;
974 ws_exit_bb
= region
->inner
->exit
;
976 if (single_succ (par_entry_bb
) == ws_entry_bb
977 && single_succ (ws_exit_bb
) == par_exit_bb
978 && workshare_safe_to_combine_p (ws_entry_bb
)
979 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
980 || (last_and_only_stmt (ws_entry_bb
)
981 && last_and_only_stmt (par_exit_bb
))))
983 gimple
*par_stmt
= last_stmt (par_entry_bb
);
984 gimple
*ws_stmt
= last_stmt (ws_entry_bb
);
986 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
988 /* If this is a combined parallel loop, we need to determine
989 whether or not to use the combined library calls. There
990 are two cases where we do not apply the transformation:
991 static loops and any kind of ordered loop. In the first
992 case, we already open code the loop so there is no need
993 to do anything else. In the latter case, the combined
994 parallel loop call would still need extra synchronization
995 to implement ordered semantics, so there would not be any
996 gain in using the combined call. */
997 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
998 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
1000 || ((OMP_CLAUSE_SCHEDULE_KIND (c
) & OMP_CLAUSE_SCHEDULE_MASK
)
1001 == OMP_CLAUSE_SCHEDULE_STATIC
)
1002 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
1004 region
->is_combined_parallel
= false;
1005 region
->inner
->is_combined_parallel
= false;
1010 region
->is_combined_parallel
= true;
1011 region
->inner
->is_combined_parallel
= true;
1012 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
1017 /* Return true if EXPR is variable sized. */
1020 is_variable_sized (const_tree expr
)
1022 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
1025 /* Return true if DECL is a reference type. */
1028 is_reference (tree decl
)
1030 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
1033 /* Return the type of a decl. If the decl is reference type,
1034 return its base type. */
1036 get_base_type (tree decl
)
1038 tree type
= TREE_TYPE (decl
);
1039 if (is_reference (decl
))
1040 type
= TREE_TYPE (type
);
1044 /* Lookup variables. The "maybe" form
1045 allows for the variable form to not have been entered, otherwise we
1046 assert that the variable must have been entered. */
1049 lookup_decl (tree var
, omp_context
*ctx
)
1051 tree
*n
= ctx
->cb
.decl_map
->get (var
);
1056 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
1058 tree
*n
= ctx
->cb
.decl_map
->get (const_cast<tree
> (var
));
1059 return n
? *n
: NULL_TREE
;
1063 lookup_field (tree var
, omp_context
*ctx
)
1066 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
1067 return (tree
) n
->value
;
1071 lookup_sfield (splay_tree_key key
, omp_context
*ctx
)
1074 n
= splay_tree_lookup (ctx
->sfield_map
1075 ? ctx
->sfield_map
: ctx
->field_map
, key
);
1076 return (tree
) n
->value
;
1080 lookup_sfield (tree var
, omp_context
*ctx
)
1082 return lookup_sfield ((splay_tree_key
) var
, ctx
);
1086 maybe_lookup_field (splay_tree_key key
, omp_context
*ctx
)
1089 n
= splay_tree_lookup (ctx
->field_map
, key
);
1090 return n
? (tree
) n
->value
: NULL_TREE
;
1094 maybe_lookup_field (tree var
, omp_context
*ctx
)
1096 return maybe_lookup_field ((splay_tree_key
) var
, ctx
);
1099 /* Return true if DECL should be copied by pointer. SHARED_CTX is
1100 the parallel context if DECL is to be shared. */
1103 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
1105 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
1108 /* We can only use copy-in/copy-out semantics for shared variables
1109 when we know the value is not accessible from an outer scope. */
1112 gcc_assert (!is_gimple_omp_oacc (shared_ctx
->stmt
));
1114 /* ??? Trivially accessible from anywhere. But why would we even
1115 be passing an address in this case? Should we simply assert
1116 this to be false, or should we have a cleanup pass that removes
1117 these from the list of mappings? */
1118 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
1121 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
1122 without analyzing the expression whether or not its location
1123 is accessible to anyone else. In the case of nested parallel
1124 regions it certainly may be. */
1125 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
1128 /* Do not use copy-in/copy-out for variables that have their
1130 if (TREE_ADDRESSABLE (decl
))
1133 /* lower_send_shared_vars only uses copy-in, but not copy-out
1135 if (TREE_READONLY (decl
)
1136 || ((TREE_CODE (decl
) == RESULT_DECL
1137 || TREE_CODE (decl
) == PARM_DECL
)
1138 && DECL_BY_REFERENCE (decl
)))
1141 /* Disallow copy-in/out in nested parallel if
1142 decl is shared in outer parallel, otherwise
1143 each thread could store the shared variable
1144 in its own copy-in location, making the
1145 variable no longer really shared. */
1146 if (shared_ctx
->is_nested
)
1150 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
1151 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
1158 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
1159 c
; c
= OMP_CLAUSE_CHAIN (c
))
1160 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
1161 && OMP_CLAUSE_DECL (c
) == decl
)
1165 goto maybe_mark_addressable_and_ret
;
1169 /* For tasks avoid using copy-in/out. As tasks can be
1170 deferred or executed in different thread, when GOMP_task
1171 returns, the task hasn't necessarily terminated. */
1172 if (is_task_ctx (shared_ctx
))
1175 maybe_mark_addressable_and_ret
:
1176 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
1177 if (is_gimple_reg (outer
) && !omp_member_access_dummy_var (outer
))
1179 /* Taking address of OUTER in lower_send_shared_vars
1180 might need regimplification of everything that uses the
1182 if (!task_shared_vars
)
1183 task_shared_vars
= BITMAP_ALLOC (NULL
);
1184 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
1185 TREE_ADDRESSABLE (outer
) = 1;
1194 /* Construct a new automatic decl similar to VAR. */
1197 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
1199 tree copy
= copy_var_decl (var
, name
, type
);
1201 DECL_CONTEXT (copy
) = current_function_decl
;
1202 DECL_CHAIN (copy
) = ctx
->block_vars
;
1203 /* If VAR is listed in task_shared_vars, it means it wasn't
1204 originally addressable and is just because task needs to take
1205 it's address. But we don't need to take address of privatizations
1207 if (TREE_ADDRESSABLE (var
)
1209 && bitmap_bit_p (task_shared_vars
, DECL_UID (var
)))
1210 TREE_ADDRESSABLE (copy
) = 0;
1211 ctx
->block_vars
= copy
;
1217 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
1219 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
1222 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
1225 omp_build_component_ref (tree obj
, tree field
)
1227 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
1228 if (TREE_THIS_VOLATILE (field
))
1229 TREE_THIS_VOLATILE (ret
) |= 1;
1230 if (TREE_READONLY (field
))
1231 TREE_READONLY (ret
) |= 1;
1235 /* Build tree nodes to access the field for VAR on the receiver side. */
1238 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
1240 tree x
, field
= lookup_field (var
, ctx
);
1242 /* If the receiver record type was remapped in the child function,
1243 remap the field into the new record type. */
1244 x
= maybe_lookup_field (field
, ctx
);
1248 x
= build_simple_mem_ref (ctx
->receiver_decl
);
1249 TREE_THIS_NOTRAP (x
) = 1;
1250 x
= omp_build_component_ref (x
, field
);
1252 x
= build_simple_mem_ref (x
);
1257 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1258 of a parallel, this is a component reference; for workshare constructs
1259 this is some variable. */
1262 build_outer_var_ref (tree var
, omp_context
*ctx
, bool lastprivate
= false)
1266 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1268 else if (is_variable_sized (var
))
1270 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1271 x
= build_outer_var_ref (x
, ctx
, lastprivate
);
1272 x
= build_simple_mem_ref (x
);
1274 else if (is_taskreg_ctx (ctx
))
1276 bool by_ref
= use_pointer_for_field (var
, NULL
);
1277 x
= build_receiver_ref (var
, by_ref
, ctx
);
1279 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1280 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
1282 /* #pragma omp simd isn't a worksharing construct, and can reference even
1283 private vars in its linear etc. clauses. */
1285 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1286 x
= lookup_decl (var
, ctx
->outer
);
1287 else if (ctx
->outer
)
1288 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1292 else if (lastprivate
&& is_taskloop_ctx (ctx
))
1294 gcc_assert (ctx
->outer
);
1296 = splay_tree_lookup (ctx
->outer
->field_map
,
1297 (splay_tree_key
) &DECL_UID (var
));
1300 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
->outer
)))
1303 x
= lookup_decl (var
, ctx
->outer
);
1307 tree field
= (tree
) n
->value
;
1308 /* If the receiver record type was remapped in the child function,
1309 remap the field into the new record type. */
1310 x
= maybe_lookup_field (field
, ctx
->outer
);
1314 x
= build_simple_mem_ref (ctx
->outer
->receiver_decl
);
1315 x
= omp_build_component_ref (x
, field
);
1316 if (use_pointer_for_field (var
, ctx
->outer
))
1317 x
= build_simple_mem_ref (x
);
1320 else if (ctx
->outer
)
1321 x
= lookup_decl (var
, ctx
->outer
);
1322 else if (is_reference (var
))
1323 /* This can happen with orphaned constructs. If var is reference, it is
1324 possible it is shared and as such valid. */
1326 else if (omp_member_access_dummy_var (var
))
1333 tree t
= omp_member_access_dummy_var (var
);
1336 x
= DECL_VALUE_EXPR (var
);
1337 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx
);
1339 x
= unshare_and_remap (x
, t
, o
);
1341 x
= unshare_expr (x
);
1345 if (is_reference (var
))
1346 x
= build_simple_mem_ref (x
);
1351 /* Build tree nodes to access the field for VAR on the sender side. */
1354 build_sender_ref (splay_tree_key key
, omp_context
*ctx
)
1356 tree field
= lookup_sfield (key
, ctx
);
1357 return omp_build_component_ref (ctx
->sender_decl
, field
);
1361 build_sender_ref (tree var
, omp_context
*ctx
)
1363 return build_sender_ref ((splay_tree_key
) var
, ctx
);
1366 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1369 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
)
1371 tree field
, type
, sfield
= NULL_TREE
;
1372 splay_tree_key key
= (splay_tree_key
) var
;
1374 if ((mask
& 8) != 0)
1376 key
= (splay_tree_key
) &DECL_UID (var
);
1377 gcc_checking_assert (key
!= (splay_tree_key
) var
);
1379 gcc_assert ((mask
& 1) == 0
1380 || !splay_tree_lookup (ctx
->field_map
, key
));
1381 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1382 || !splay_tree_lookup (ctx
->sfield_map
, key
));
1383 gcc_assert ((mask
& 3) == 3
1384 || !is_gimple_omp_oacc (ctx
->stmt
));
1386 type
= TREE_TYPE (var
);
1389 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1390 type
= build_pointer_type (build_pointer_type (type
));
1393 type
= build_pointer_type (type
);
1394 else if ((mask
& 3) == 1 && is_reference (var
))
1395 type
= TREE_TYPE (type
);
1397 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1398 FIELD_DECL
, DECL_NAME (var
), type
);
1400 /* Remember what variable this field was created for. This does have a
1401 side effect of making dwarf2out ignore this member, so for helpful
1402 debugging we clear it later in delete_omp_context. */
1403 DECL_ABSTRACT_ORIGIN (field
) = var
;
1404 if (type
== TREE_TYPE (var
))
1406 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1407 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1408 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1411 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1413 if ((mask
& 3) == 3)
1415 insert_field_into_struct (ctx
->record_type
, field
);
1416 if (ctx
->srecord_type
)
1418 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1419 FIELD_DECL
, DECL_NAME (var
), type
);
1420 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1421 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1422 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1423 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1424 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1429 if (ctx
->srecord_type
== NULL_TREE
)
1433 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1434 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1435 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1437 sfield
= build_decl (DECL_SOURCE_LOCATION (t
),
1438 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1439 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1440 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1441 splay_tree_insert (ctx
->sfield_map
,
1442 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1443 (splay_tree_value
) sfield
);
1447 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1448 : ctx
->srecord_type
, field
);
1452 splay_tree_insert (ctx
->field_map
, key
, (splay_tree_value
) field
);
1453 if ((mask
& 2) && ctx
->sfield_map
)
1454 splay_tree_insert (ctx
->sfield_map
, key
, (splay_tree_value
) sfield
);
1458 install_var_local (tree var
, omp_context
*ctx
)
1460 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1461 insert_decl_map (&ctx
->cb
, var
, new_var
);
1465 /* Adjust the replacement for DECL in CTX for the new context. This means
1466 copying the DECL_VALUE_EXPR, and fixing up the type. */
1469 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1471 tree new_decl
, size
;
1473 new_decl
= lookup_decl (decl
, ctx
);
1475 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1477 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1478 && DECL_HAS_VALUE_EXPR_P (decl
))
1480 tree ve
= DECL_VALUE_EXPR (decl
);
1481 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1482 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1483 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1486 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1488 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1489 if (size
== error_mark_node
)
1490 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1491 DECL_SIZE (new_decl
) = size
;
1493 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1494 if (size
== error_mark_node
)
1495 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1496 DECL_SIZE_UNIT (new_decl
) = size
;
1500 /* The callback for remap_decl. Search all containing contexts for a
1501 mapping of the variable; this avoids having to duplicate the splay
1502 tree ahead of time. We know a mapping doesn't already exist in the
1503 given context. Create new mappings to implement default semantics. */
1506 omp_copy_decl (tree var
, copy_body_data
*cb
)
1508 omp_context
*ctx
= (omp_context
*) cb
;
1511 if (TREE_CODE (var
) == LABEL_DECL
)
1513 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1514 DECL_CONTEXT (new_var
) = current_function_decl
;
1515 insert_decl_map (&ctx
->cb
, var
, new_var
);
1519 while (!is_taskreg_ctx (ctx
))
1524 new_var
= maybe_lookup_decl (var
, ctx
);
1529 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1532 return error_mark_node
;
1536 /* Debugging dumps for parallel regions. */
1537 void dump_omp_region (FILE *, struct omp_region
*, int);
1538 void debug_omp_region (struct omp_region
*);
1539 void debug_all_omp_regions (void);
1541 /* Dump the parallel region tree rooted at REGION. */
1544 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1546 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1547 gimple_code_name
[region
->type
]);
1550 dump_omp_region (file
, region
->inner
, indent
+ 4);
1554 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1555 region
->cont
->index
);
1559 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1560 region
->exit
->index
);
1562 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1565 dump_omp_region (file
, region
->next
, indent
);
1569 debug_omp_region (struct omp_region
*region
)
1571 dump_omp_region (stderr
, region
, 0);
1575 debug_all_omp_regions (void)
1577 dump_omp_region (stderr
, root_omp_region
, 0);
1581 /* Create a new parallel region starting at STMT inside region PARENT. */
1583 static struct omp_region
*
1584 new_omp_region (basic_block bb
, enum gimple_code type
,
1585 struct omp_region
*parent
)
1587 struct omp_region
*region
= XCNEW (struct omp_region
);
1589 region
->outer
= parent
;
1591 region
->type
= type
;
1595 /* This is a nested region. Add it to the list of inner
1596 regions in PARENT. */
1597 region
->next
= parent
->inner
;
1598 parent
->inner
= region
;
1602 /* This is a toplevel region. Add it to the list of toplevel
1603 regions in ROOT_OMP_REGION. */
1604 region
->next
= root_omp_region
;
1605 root_omp_region
= region
;
1611 /* Release the memory associated with the region tree rooted at REGION. */
1614 free_omp_region_1 (struct omp_region
*region
)
1616 struct omp_region
*i
, *n
;
1618 for (i
= region
->inner
; i
; i
= n
)
1621 free_omp_region_1 (i
);
1627 /* Release the memory for the entire omp region tree. */
1630 free_omp_regions (void)
1632 struct omp_region
*r
, *n
;
1633 for (r
= root_omp_region
; r
; r
= n
)
1636 free_omp_region_1 (r
);
1638 root_omp_region
= NULL
;
1642 /* Create a new context, with OUTER_CTX being the surrounding context. */
1644 static omp_context
*
1645 new_omp_context (gimple
*stmt
, omp_context
*outer_ctx
)
1647 omp_context
*ctx
= XCNEW (omp_context
);
1649 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1650 (splay_tree_value
) ctx
);
1655 ctx
->outer
= outer_ctx
;
1656 ctx
->cb
= outer_ctx
->cb
;
1657 ctx
->cb
.block
= NULL
;
1658 ctx
->depth
= outer_ctx
->depth
+ 1;
1662 ctx
->cb
.src_fn
= current_function_decl
;
1663 ctx
->cb
.dst_fn
= current_function_decl
;
1664 ctx
->cb
.src_node
= cgraph_node::get (current_function_decl
);
1665 gcc_checking_assert (ctx
->cb
.src_node
);
1666 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1667 ctx
->cb
.src_cfun
= cfun
;
1668 ctx
->cb
.copy_decl
= omp_copy_decl
;
1669 ctx
->cb
.eh_lp_nr
= 0;
1670 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1674 ctx
->cb
.decl_map
= new hash_map
<tree
, tree
>;
1679 static gimple_seq
maybe_catch_exception (gimple_seq
);
1681 /* Finalize task copyfn. */
1684 finalize_task_copyfn (gomp_task
*task_stmt
)
1686 struct function
*child_cfun
;
1688 gimple_seq seq
= NULL
, new_seq
;
1691 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1692 if (child_fn
== NULL_TREE
)
1695 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1696 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1698 push_cfun (child_cfun
);
1699 bind
= gimplify_body (child_fn
, false);
1700 gimple_seq_add_stmt (&seq
, bind
);
1701 new_seq
= maybe_catch_exception (seq
);
1704 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1706 gimple_seq_add_stmt (&seq
, bind
);
1708 gimple_set_body (child_fn
, seq
);
1711 /* Inform the callgraph about the new function. */
1712 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
1713 node
->parallelized_function
= 1;
1714 cgraph_node::add_new_function (child_fn
, false);
1717 /* Destroy a omp_context data structures. Called through the splay tree
1718 value delete callback. */
1721 delete_omp_context (splay_tree_value value
)
1723 omp_context
*ctx
= (omp_context
*) value
;
1725 delete ctx
->cb
.decl_map
;
1728 splay_tree_delete (ctx
->field_map
);
1729 if (ctx
->sfield_map
)
1730 splay_tree_delete (ctx
->sfield_map
);
1732 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1733 it produces corrupt debug information. */
1734 if (ctx
->record_type
)
1737 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1738 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1740 if (ctx
->srecord_type
)
1743 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1744 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1747 if (is_task_ctx (ctx
))
1748 finalize_task_copyfn (as_a
<gomp_task
*> (ctx
->stmt
));
1753 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1757 fixup_child_record_type (omp_context
*ctx
)
1759 tree f
, type
= ctx
->record_type
;
1761 /* ??? It isn't sufficient to just call remap_type here, because
1762 variably_modified_type_p doesn't work the way we expect for
1763 record types. Testing each field for whether it needs remapping
1764 and creating a new record by hand works, however. */
1765 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1766 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1770 tree name
, new_fields
= NULL
;
1772 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1773 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1774 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1775 TYPE_DECL
, name
, type
);
1776 TYPE_NAME (type
) = name
;
1778 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1780 tree new_f
= copy_node (f
);
1781 DECL_CONTEXT (new_f
) = type
;
1782 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1783 DECL_CHAIN (new_f
) = new_fields
;
1784 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1785 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1787 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1791 /* Arrange to be able to look up the receiver field
1792 given the sender field. */
1793 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1794 (splay_tree_value
) new_f
);
1796 TYPE_FIELDS (type
) = nreverse (new_fields
);
1800 /* In a target region we never modify any of the pointers in *.omp_data_i,
1801 so attempt to help the optimizers. */
1802 if (is_gimple_omp_offloaded (ctx
->stmt
))
1803 type
= build_qualified_type (type
, TYPE_QUAL_CONST
);
1805 TREE_TYPE (ctx
->receiver_decl
)
1806 = build_qualified_type (build_reference_type (type
), TYPE_QUAL_RESTRICT
);
1809 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1810 specified by CLAUSES. */
1813 scan_sharing_clauses (tree clauses
, omp_context
*ctx
)
1816 bool scan_array_reductions
= false;
1818 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1822 switch (OMP_CLAUSE_CODE (c
))
1824 case OMP_CLAUSE_PRIVATE
:
1825 decl
= OMP_CLAUSE_DECL (c
);
1826 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1828 else if (!is_variable_sized (decl
))
1829 install_var_local (decl
, ctx
);
1832 case OMP_CLAUSE_SHARED
:
1833 decl
= OMP_CLAUSE_DECL (c
);
1834 /* Ignore shared directives in teams construct. */
1835 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1837 /* Global variables don't need to be copied,
1838 the receiver side will use them directly. */
1839 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1840 if (is_global_var (odecl
))
1842 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1845 gcc_assert (is_taskreg_ctx (ctx
));
1846 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1847 || !is_variable_sized (decl
));
1848 /* Global variables don't need to be copied,
1849 the receiver side will use them directly. */
1850 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1852 by_ref
= use_pointer_for_field (decl
, ctx
);
1853 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
1855 if (! TREE_READONLY (decl
)
1856 || TREE_ADDRESSABLE (decl
)
1858 || is_reference (decl
))
1860 install_var_field (decl
, by_ref
, 3, ctx
);
1861 install_var_local (decl
, ctx
);
1864 /* We don't need to copy const scalar vars back. */
1865 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1868 case OMP_CLAUSE_REDUCTION
:
1869 decl
= OMP_CLAUSE_DECL (c
);
1870 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1871 && TREE_CODE (decl
) == MEM_REF
)
1873 tree t
= TREE_OPERAND (decl
, 0);
1874 if (TREE_CODE (t
) == POINTER_PLUS_EXPR
)
1875 t
= TREE_OPERAND (t
, 0);
1876 if (TREE_CODE (t
) == INDIRECT_REF
1877 || TREE_CODE (t
) == ADDR_EXPR
)
1878 t
= TREE_OPERAND (t
, 0);
1879 install_var_local (t
, ctx
);
1880 if (is_taskreg_ctx (ctx
)
1881 && !is_global_var (maybe_lookup_decl_in_outer_ctx (t
, ctx
))
1882 && !is_variable_sized (t
))
1884 by_ref
= use_pointer_for_field (t
, ctx
);
1885 install_var_field (t
, by_ref
, 3, ctx
);
1891 case OMP_CLAUSE_LASTPRIVATE
:
1892 /* Let the corresponding firstprivate clause create
1894 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1898 case OMP_CLAUSE_FIRSTPRIVATE
:
1899 if (is_gimple_omp_oacc (ctx
->stmt
))
1901 sorry ("clause not supported yet");
1905 case OMP_CLAUSE_LINEAR
:
1906 decl
= OMP_CLAUSE_DECL (c
);
1908 if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
1909 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_IS_DEVICE_PTR
)
1910 && is_gimple_omp_offloaded (ctx
->stmt
))
1912 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
1913 install_var_field (decl
, !is_reference (decl
), 3, ctx
);
1914 else if (TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1915 install_var_field (decl
, true, 3, ctx
);
1917 install_var_field (decl
, false, 3, ctx
);
1919 if (is_variable_sized (decl
))
1921 if (is_task_ctx (ctx
))
1922 install_var_field (decl
, false, 1, ctx
);
1925 else if (is_taskreg_ctx (ctx
))
1928 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1929 by_ref
= use_pointer_for_field (decl
, NULL
);
1931 if (is_task_ctx (ctx
)
1932 && (global
|| by_ref
|| is_reference (decl
)))
1934 install_var_field (decl
, false, 1, ctx
);
1936 install_var_field (decl
, by_ref
, 2, ctx
);
1939 install_var_field (decl
, by_ref
, 3, ctx
);
1941 install_var_local (decl
, ctx
);
1944 case OMP_CLAUSE_USE_DEVICE_PTR
:
1945 decl
= OMP_CLAUSE_DECL (c
);
1946 if (TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1947 install_var_field (decl
, true, 3, ctx
);
1949 install_var_field (decl
, false, 3, ctx
);
1950 if (DECL_SIZE (decl
)
1951 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1953 tree decl2
= DECL_VALUE_EXPR (decl
);
1954 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1955 decl2
= TREE_OPERAND (decl2
, 0);
1956 gcc_assert (DECL_P (decl2
));
1957 install_var_local (decl2
, ctx
);
1959 install_var_local (decl
, ctx
);
1962 case OMP_CLAUSE_IS_DEVICE_PTR
:
1963 decl
= OMP_CLAUSE_DECL (c
);
1966 case OMP_CLAUSE__LOOPTEMP_
:
1967 gcc_assert (is_taskreg_ctx (ctx
));
1968 decl
= OMP_CLAUSE_DECL (c
);
1969 install_var_field (decl
, false, 3, ctx
);
1970 install_var_local (decl
, ctx
);
1973 case OMP_CLAUSE_COPYPRIVATE
:
1974 case OMP_CLAUSE_COPYIN
:
1975 decl
= OMP_CLAUSE_DECL (c
);
1976 by_ref
= use_pointer_for_field (decl
, NULL
);
1977 install_var_field (decl
, by_ref
, 3, ctx
);
1980 case OMP_CLAUSE_DEFAULT
:
1981 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
1984 case OMP_CLAUSE_FINAL
:
1986 case OMP_CLAUSE_NUM_THREADS
:
1987 case OMP_CLAUSE_NUM_TEAMS
:
1988 case OMP_CLAUSE_THREAD_LIMIT
:
1989 case OMP_CLAUSE_DEVICE
:
1990 case OMP_CLAUSE_SCHEDULE
:
1991 case OMP_CLAUSE_DIST_SCHEDULE
:
1992 case OMP_CLAUSE_DEPEND
:
1993 case OMP_CLAUSE_PRIORITY
:
1994 case OMP_CLAUSE_GRAINSIZE
:
1995 case OMP_CLAUSE_NUM_TASKS
:
1996 case OMP_CLAUSE__CILK_FOR_COUNT_
:
1997 case OMP_CLAUSE_NUM_GANGS
:
1998 case OMP_CLAUSE_NUM_WORKERS
:
1999 case OMP_CLAUSE_VECTOR_LENGTH
:
2001 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
2005 case OMP_CLAUSE_FROM
:
2006 case OMP_CLAUSE_MAP
:
2008 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
2009 decl
= OMP_CLAUSE_DECL (c
);
2010 /* Global variables with "omp declare target" attribute
2011 don't need to be copied, the receiver side will use them
2013 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2015 && ((OMP_CLAUSE_MAP_KIND (c
) != GOMP_MAP_FIRSTPRIVATE_POINTER
2016 && (OMP_CLAUSE_MAP_KIND (c
)
2017 != GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
2018 || TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2019 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
2020 && varpool_node::get_create (decl
)->offloadable
)
2022 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2023 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
)
2025 /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are
2026 not offloaded; there is nothing to map for those. */
2027 if (!is_gimple_omp_offloaded (ctx
->stmt
)
2028 && !POINTER_TYPE_P (TREE_TYPE (decl
))
2029 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
2032 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2033 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
2034 || (OMP_CLAUSE_MAP_KIND (c
)
2035 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
2037 if (TREE_CODE (decl
) == COMPONENT_REF
2038 || (TREE_CODE (decl
) == INDIRECT_REF
2039 && TREE_CODE (TREE_OPERAND (decl
, 0)) == COMPONENT_REF
2040 && (TREE_CODE (TREE_TYPE (TREE_OPERAND (decl
, 0)))
2041 == REFERENCE_TYPE
)))
2043 if (DECL_SIZE (decl
)
2044 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2046 tree decl2
= DECL_VALUE_EXPR (decl
);
2047 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2048 decl2
= TREE_OPERAND (decl2
, 0);
2049 gcc_assert (DECL_P (decl2
));
2050 install_var_local (decl2
, ctx
);
2052 install_var_local (decl
, ctx
);
2057 if (DECL_SIZE (decl
)
2058 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2060 tree decl2
= DECL_VALUE_EXPR (decl
);
2061 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2062 decl2
= TREE_OPERAND (decl2
, 0);
2063 gcc_assert (DECL_P (decl2
));
2064 install_var_field (decl2
, true, 3, ctx
);
2065 install_var_local (decl2
, ctx
);
2066 install_var_local (decl
, ctx
);
2070 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2071 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
2072 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
2073 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2074 install_var_field (decl
, true, 7, ctx
);
2076 install_var_field (decl
, true, 3, ctx
);
2077 if (is_gimple_omp_offloaded (ctx
->stmt
))
2078 install_var_local (decl
, ctx
);
2083 tree base
= get_base_address (decl
);
2084 tree nc
= OMP_CLAUSE_CHAIN (c
);
2087 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
2088 && OMP_CLAUSE_DECL (nc
) == base
2089 && OMP_CLAUSE_MAP_KIND (nc
) == GOMP_MAP_POINTER
2090 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
2092 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
2093 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
2099 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
2100 decl
= OMP_CLAUSE_DECL (c
);
2102 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
2103 (splay_tree_key
) decl
));
2105 = build_decl (OMP_CLAUSE_LOCATION (c
),
2106 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
2107 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
2108 insert_field_into_struct (ctx
->record_type
, field
);
2109 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
2110 (splay_tree_value
) field
);
2115 case OMP_CLAUSE_NOWAIT
:
2116 case OMP_CLAUSE_ORDERED
:
2117 case OMP_CLAUSE_COLLAPSE
:
2118 case OMP_CLAUSE_UNTIED
:
2119 case OMP_CLAUSE_MERGEABLE
:
2120 case OMP_CLAUSE_PROC_BIND
:
2121 case OMP_CLAUSE_SAFELEN
:
2122 case OMP_CLAUSE_SIMDLEN
:
2123 case OMP_CLAUSE_THREADS
:
2124 case OMP_CLAUSE_SIMD
:
2125 case OMP_CLAUSE_NOGROUP
:
2126 case OMP_CLAUSE_DEFAULTMAP
:
2127 case OMP_CLAUSE_ASYNC
:
2128 case OMP_CLAUSE_WAIT
:
2129 case OMP_CLAUSE_GANG
:
2130 case OMP_CLAUSE_WORKER
:
2131 case OMP_CLAUSE_VECTOR
:
2134 case OMP_CLAUSE_ALIGNED
:
2135 decl
= OMP_CLAUSE_DECL (c
);
2136 if (is_global_var (decl
)
2137 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2138 install_var_local (decl
, ctx
);
2141 case OMP_CLAUSE_DEVICE_RESIDENT
:
2142 case OMP_CLAUSE_USE_DEVICE
:
2143 case OMP_CLAUSE__CACHE_
:
2144 case OMP_CLAUSE_INDEPENDENT
:
2145 case OMP_CLAUSE_AUTO
:
2146 case OMP_CLAUSE_SEQ
:
2147 sorry ("Clause not supported yet");
2155 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2157 switch (OMP_CLAUSE_CODE (c
))
2159 case OMP_CLAUSE_LASTPRIVATE
:
2160 /* Let the corresponding firstprivate clause create
2162 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2163 scan_array_reductions
= true;
2164 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
2168 case OMP_CLAUSE_FIRSTPRIVATE
:
2169 if (is_gimple_omp_oacc (ctx
->stmt
))
2171 sorry ("clause not supported yet");
2175 case OMP_CLAUSE_PRIVATE
:
2176 case OMP_CLAUSE_LINEAR
:
2177 case OMP_CLAUSE_IS_DEVICE_PTR
:
2178 decl
= OMP_CLAUSE_DECL (c
);
2179 if (is_variable_sized (decl
))
2181 if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
2182 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_IS_DEVICE_PTR
)
2183 && is_gimple_omp_offloaded (ctx
->stmt
))
2185 tree decl2
= DECL_VALUE_EXPR (decl
);
2186 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2187 decl2
= TREE_OPERAND (decl2
, 0);
2188 gcc_assert (DECL_P (decl2
));
2189 install_var_local (decl2
, ctx
);
2190 fixup_remapped_decl (decl2
, ctx
, false);
2192 install_var_local (decl
, ctx
);
2194 fixup_remapped_decl (decl
, ctx
,
2195 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
2196 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
2197 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2198 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2199 scan_array_reductions
= true;
2202 case OMP_CLAUSE_REDUCTION
:
2203 decl
= OMP_CLAUSE_DECL (c
);
2204 if (TREE_CODE (decl
) != MEM_REF
)
2206 if (is_variable_sized (decl
))
2207 install_var_local (decl
, ctx
);
2208 fixup_remapped_decl (decl
, ctx
, false);
2210 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2211 scan_array_reductions
= true;
2214 case OMP_CLAUSE_SHARED
:
2215 /* Ignore shared directives in teams construct. */
2216 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2218 decl
= OMP_CLAUSE_DECL (c
);
2219 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2221 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
2223 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
,
2226 bool by_ref
= use_pointer_for_field (decl
, ctx
);
2227 install_var_field (decl
, by_ref
, 11, ctx
);
2230 fixup_remapped_decl (decl
, ctx
, false);
2233 case OMP_CLAUSE_MAP
:
2234 if (!is_gimple_omp_offloaded (ctx
->stmt
))
2236 decl
= OMP_CLAUSE_DECL (c
);
2238 && ((OMP_CLAUSE_MAP_KIND (c
) != GOMP_MAP_FIRSTPRIVATE_POINTER
2239 && (OMP_CLAUSE_MAP_KIND (c
)
2240 != GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
2241 || TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2242 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
2243 && varpool_node::get_create (decl
)->offloadable
)
2247 if ((OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
2248 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
)
2249 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
2250 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
2252 tree new_decl
= lookup_decl (decl
, ctx
);
2253 TREE_TYPE (new_decl
)
2254 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
2256 else if (DECL_SIZE (decl
)
2257 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2259 tree decl2
= DECL_VALUE_EXPR (decl
);
2260 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2261 decl2
= TREE_OPERAND (decl2
, 0);
2262 gcc_assert (DECL_P (decl2
));
2263 fixup_remapped_decl (decl2
, ctx
, false);
2264 fixup_remapped_decl (decl
, ctx
, true);
2267 fixup_remapped_decl (decl
, ctx
, false);
2271 case OMP_CLAUSE_COPYPRIVATE
:
2272 case OMP_CLAUSE_COPYIN
:
2273 case OMP_CLAUSE_DEFAULT
:
2275 case OMP_CLAUSE_NUM_THREADS
:
2276 case OMP_CLAUSE_NUM_TEAMS
:
2277 case OMP_CLAUSE_THREAD_LIMIT
:
2278 case OMP_CLAUSE_DEVICE
:
2279 case OMP_CLAUSE_SCHEDULE
:
2280 case OMP_CLAUSE_DIST_SCHEDULE
:
2281 case OMP_CLAUSE_NOWAIT
:
2282 case OMP_CLAUSE_ORDERED
:
2283 case OMP_CLAUSE_COLLAPSE
:
2284 case OMP_CLAUSE_UNTIED
:
2285 case OMP_CLAUSE_FINAL
:
2286 case OMP_CLAUSE_MERGEABLE
:
2287 case OMP_CLAUSE_PROC_BIND
:
2288 case OMP_CLAUSE_SAFELEN
:
2289 case OMP_CLAUSE_SIMDLEN
:
2290 case OMP_CLAUSE_ALIGNED
:
2291 case OMP_CLAUSE_DEPEND
:
2292 case OMP_CLAUSE__LOOPTEMP_
:
2294 case OMP_CLAUSE_FROM
:
2295 case OMP_CLAUSE_PRIORITY
:
2296 case OMP_CLAUSE_GRAINSIZE
:
2297 case OMP_CLAUSE_NUM_TASKS
:
2298 case OMP_CLAUSE_THREADS
:
2299 case OMP_CLAUSE_SIMD
:
2300 case OMP_CLAUSE_NOGROUP
:
2301 case OMP_CLAUSE_DEFAULTMAP
:
2302 case OMP_CLAUSE_USE_DEVICE_PTR
:
2303 case OMP_CLAUSE__CILK_FOR_COUNT_
:
2304 case OMP_CLAUSE_ASYNC
:
2305 case OMP_CLAUSE_WAIT
:
2306 case OMP_CLAUSE_NUM_GANGS
:
2307 case OMP_CLAUSE_NUM_WORKERS
:
2308 case OMP_CLAUSE_VECTOR_LENGTH
:
2309 case OMP_CLAUSE_GANG
:
2310 case OMP_CLAUSE_WORKER
:
2311 case OMP_CLAUSE_VECTOR
:
2314 case OMP_CLAUSE_DEVICE_RESIDENT
:
2315 case OMP_CLAUSE_USE_DEVICE
:
2316 case OMP_CLAUSE__CACHE_
:
2317 case OMP_CLAUSE_INDEPENDENT
:
2318 case OMP_CLAUSE_AUTO
:
2319 case OMP_CLAUSE_SEQ
:
2320 sorry ("Clause not supported yet");
2328 gcc_checking_assert (!scan_array_reductions
2329 || !is_gimple_omp_oacc (ctx
->stmt
));
2330 if (scan_array_reductions
)
2331 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2332 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
2333 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2335 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
2336 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
2338 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
2339 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2340 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
2341 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2342 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2343 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
2346 /* Create a new name for omp child function. Returns an identifier. If
2347 IS_CILK_FOR is true then the suffix for the child function is
2351 create_omp_child_function_name (bool task_copy
, bool is_cilk_for
)
2354 return clone_function_name (current_function_decl
, "_cilk_for_fn");
2355 return clone_function_name (current_function_decl
,
2356 task_copy
? "_omp_cpyfn" : "_omp_fn");
2359 /* Returns the type of the induction variable for the child function for
2360 _Cilk_for and the types for _high and _low variables based on TYPE. */
2363 cilk_for_check_loop_diff_type (tree type
)
2365 if (TYPE_PRECISION (type
) <= TYPE_PRECISION (uint32_type_node
))
2367 if (TYPE_UNSIGNED (type
))
2368 return uint32_type_node
;
2370 return integer_type_node
;
2374 if (TYPE_UNSIGNED (type
))
2375 return uint64_type_node
;
2377 return long_long_integer_type_node
;
2381 /* Build a decl for the omp child function. It'll not contain a body
2382 yet, just the bare decl. */
2385 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
2387 tree decl
, type
, name
, t
;
2390 = (flag_cilkplus
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2391 ? find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
2392 OMP_CLAUSE__CILK_FOR_COUNT_
) : NULL_TREE
;
2393 tree cilk_var_type
= NULL_TREE
;
2395 name
= create_omp_child_function_name (task_copy
,
2396 cilk_for_count
!= NULL_TREE
);
2398 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2399 ptr_type_node
, NULL_TREE
);
2400 else if (cilk_for_count
)
2402 type
= TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count
, 0));
2403 cilk_var_type
= cilk_for_check_loop_diff_type (type
);
2404 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2405 cilk_var_type
, cilk_var_type
, NULL_TREE
);
2408 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
2410 decl
= build_decl (gimple_location (ctx
->stmt
), FUNCTION_DECL
, name
, type
);
2412 gcc_checking_assert (!is_gimple_omp_oacc (ctx
->stmt
)
2415 ctx
->cb
.dst_fn
= decl
;
2417 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
2419 TREE_STATIC (decl
) = 1;
2420 TREE_USED (decl
) = 1;
2421 DECL_ARTIFICIAL (decl
) = 1;
2422 DECL_IGNORED_P (decl
) = 0;
2423 TREE_PUBLIC (decl
) = 0;
2424 DECL_UNINLINABLE (decl
) = 1;
2425 DECL_EXTERNAL (decl
) = 0;
2426 DECL_CONTEXT (decl
) = NULL_TREE
;
2427 DECL_INITIAL (decl
) = make_node (BLOCK
);
2428 if (cgraph_node::get (current_function_decl
)->offloadable
)
2429 cgraph_node::get_create (decl
)->offloadable
= 1;
2433 for (octx
= ctx
; octx
; octx
= octx
->outer
)
2434 if (is_gimple_omp_offloaded (octx
->stmt
))
2436 cgraph_node::get_create (decl
)->offloadable
= 1;
2437 #ifdef ENABLE_OFFLOADING
2438 g
->have_offload
= true;
2444 if (cgraph_node::get_create (decl
)->offloadable
2445 && !lookup_attribute ("omp declare target",
2446 DECL_ATTRIBUTES (current_function_decl
)))
2447 DECL_ATTRIBUTES (decl
)
2448 = tree_cons (get_identifier ("omp target entrypoint"),
2449 NULL_TREE
, DECL_ATTRIBUTES (decl
));
2451 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2452 RESULT_DECL
, NULL_TREE
, void_type_node
);
2453 DECL_ARTIFICIAL (t
) = 1;
2454 DECL_IGNORED_P (t
) = 1;
2455 DECL_CONTEXT (t
) = decl
;
2456 DECL_RESULT (decl
) = t
;
2458 /* _Cilk_for's child function requires two extra parameters called
2459 __low and __high that are set the by Cilk runtime when it calls this
2463 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2464 PARM_DECL
, get_identifier ("__high"), cilk_var_type
);
2465 DECL_ARTIFICIAL (t
) = 1;
2466 DECL_NAMELESS (t
) = 1;
2467 DECL_ARG_TYPE (t
) = ptr_type_node
;
2468 DECL_CONTEXT (t
) = current_function_decl
;
2470 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2471 DECL_ARGUMENTS (decl
) = t
;
2473 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2474 PARM_DECL
, get_identifier ("__low"), cilk_var_type
);
2475 DECL_ARTIFICIAL (t
) = 1;
2476 DECL_NAMELESS (t
) = 1;
2477 DECL_ARG_TYPE (t
) = ptr_type_node
;
2478 DECL_CONTEXT (t
) = current_function_decl
;
2480 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2481 DECL_ARGUMENTS (decl
) = t
;
2484 tree data_name
= get_identifier (".omp_data_i");
2485 t
= build_decl (DECL_SOURCE_LOCATION (decl
), PARM_DECL
, data_name
,
2487 DECL_ARTIFICIAL (t
) = 1;
2488 DECL_NAMELESS (t
) = 1;
2489 DECL_ARG_TYPE (t
) = ptr_type_node
;
2490 DECL_CONTEXT (t
) = current_function_decl
;
2492 TREE_READONLY (t
) = 1;
2494 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2495 DECL_ARGUMENTS (decl
) = t
;
2497 ctx
->receiver_decl
= t
;
2500 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2501 PARM_DECL
, get_identifier (".omp_data_o"),
2503 DECL_ARTIFICIAL (t
) = 1;
2504 DECL_NAMELESS (t
) = 1;
2505 DECL_ARG_TYPE (t
) = ptr_type_node
;
2506 DECL_CONTEXT (t
) = current_function_decl
;
2508 TREE_ADDRESSABLE (t
) = 1;
2509 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2510 DECL_ARGUMENTS (decl
) = t
;
2513 /* Allocate memory for the function structure. The call to
2514 allocate_struct_function clobbers CFUN, so we need to restore
2516 push_struct_function (decl
);
2517 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
2521 /* Callback for walk_gimple_seq. Check if combined parallel
2522 contains gimple_omp_for_combined_into_p OMP_FOR. */
2525 find_combined_for (gimple_stmt_iterator
*gsi_p
,
2526 bool *handled_ops_p
,
2527 struct walk_stmt_info
*wi
)
2529 gimple
*stmt
= gsi_stmt (*gsi_p
);
2531 *handled_ops_p
= true;
2532 switch (gimple_code (stmt
))
2536 case GIMPLE_OMP_FOR
:
2537 if (gimple_omp_for_combined_into_p (stmt
)
2538 && gimple_omp_for_kind (stmt
)
2539 == *(const enum gf_mask
*) (wi
->info
))
2542 return integer_zero_node
;
2551 /* Add _LOOPTEMP_ clauses on OpenMP parallel or task. */
2554 add_taskreg_looptemp_clauses (enum gf_mask msk
, gimple
*stmt
,
2555 omp_context
*outer_ctx
)
2557 struct walk_stmt_info wi
;
2559 memset (&wi
, 0, sizeof (wi
));
2561 wi
.info
= (void *) &msk
;
2562 walk_gimple_seq (gimple_omp_body (stmt
), find_combined_for
, NULL
, &wi
);
2563 if (wi
.info
!= (void *) &msk
)
2565 gomp_for
*for_stmt
= as_a
<gomp_for
*> ((gimple
*) wi
.info
);
2566 struct omp_for_data fd
;
2567 extract_omp_for_data (for_stmt
, &fd
, NULL
);
2568 /* We need two temporaries with fd.loop.v type (istart/iend)
2569 and then (fd.collapse - 1) temporaries with the same
2570 type for count2 ... countN-1 vars if not constant. */
2571 size_t count
= 2, i
;
2572 tree type
= fd
.iter_type
;
2574 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
2576 count
+= fd
.collapse
- 1;
2577 /* If there are lastprivate clauses on the inner
2578 GIMPLE_OMP_FOR, add one more temporaries for the total number
2579 of iterations (product of count1 ... countN-1). */
2580 if (find_omp_clause (gimple_omp_for_clauses (for_stmt
),
2581 OMP_CLAUSE_LASTPRIVATE
))
2583 else if (msk
== GF_OMP_FOR_KIND_FOR
2584 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2585 OMP_CLAUSE_LASTPRIVATE
))
2588 for (i
= 0; i
< count
; i
++)
2590 tree temp
= create_tmp_var (type
);
2591 tree c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
2592 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
2593 OMP_CLAUSE_DECL (c
) = temp
;
2594 OMP_CLAUSE_CHAIN (c
) = gimple_omp_taskreg_clauses (stmt
);
2595 gimple_omp_taskreg_set_clauses (stmt
, c
);
2600 /* Scan an OpenMP parallel directive. */
2603 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2607 gomp_parallel
*stmt
= as_a
<gomp_parallel
*> (gsi_stmt (*gsi
));
2609 /* Ignore parallel directives with empty bodies, unless there
2610 are copyin clauses. */
2612 && empty_body_p (gimple_omp_body (stmt
))
2613 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2614 OMP_CLAUSE_COPYIN
) == NULL
)
2616 gsi_replace (gsi
, gimple_build_nop (), false);
2620 if (gimple_omp_parallel_combined_p (stmt
))
2621 add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_FOR
, stmt
, outer_ctx
);
2623 ctx
= new_omp_context (stmt
, outer_ctx
);
2624 taskreg_contexts
.safe_push (ctx
);
2625 if (taskreg_nesting_level
> 1)
2626 ctx
->is_nested
= true;
2627 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2628 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2629 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2630 name
= create_tmp_var_name (".omp_data_s");
2631 name
= build_decl (gimple_location (stmt
),
2632 TYPE_DECL
, name
, ctx
->record_type
);
2633 DECL_ARTIFICIAL (name
) = 1;
2634 DECL_NAMELESS (name
) = 1;
2635 TYPE_NAME (ctx
->record_type
) = name
;
2636 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2637 create_omp_child_function (ctx
, false);
2638 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2640 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2641 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2643 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2644 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2647 /* Scan an OpenMP task directive. */
2650 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2654 gomp_task
*stmt
= as_a
<gomp_task
*> (gsi_stmt (*gsi
));
2656 /* Ignore task directives with empty bodies. */
2658 && empty_body_p (gimple_omp_body (stmt
)))
2660 gsi_replace (gsi
, gimple_build_nop (), false);
2664 if (gimple_omp_task_taskloop_p (stmt
))
2665 add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_TASKLOOP
, stmt
, outer_ctx
);
2667 ctx
= new_omp_context (stmt
, outer_ctx
);
2668 taskreg_contexts
.safe_push (ctx
);
2669 if (taskreg_nesting_level
> 1)
2670 ctx
->is_nested
= true;
2671 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2672 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2673 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2674 name
= create_tmp_var_name (".omp_data_s");
2675 name
= build_decl (gimple_location (stmt
),
2676 TYPE_DECL
, name
, ctx
->record_type
);
2677 DECL_ARTIFICIAL (name
) = 1;
2678 DECL_NAMELESS (name
) = 1;
2679 TYPE_NAME (ctx
->record_type
) = name
;
2680 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2681 create_omp_child_function (ctx
, false);
2682 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2684 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2686 if (ctx
->srecord_type
)
2688 name
= create_tmp_var_name (".omp_data_a");
2689 name
= build_decl (gimple_location (stmt
),
2690 TYPE_DECL
, name
, ctx
->srecord_type
);
2691 DECL_ARTIFICIAL (name
) = 1;
2692 DECL_NAMELESS (name
) = 1;
2693 TYPE_NAME (ctx
->srecord_type
) = name
;
2694 TYPE_ARTIFICIAL (ctx
->srecord_type
) = 1;
2695 create_omp_child_function (ctx
, true);
2698 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2700 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2702 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2703 t
= build_int_cst (long_integer_type_node
, 0);
2704 gimple_omp_task_set_arg_size (stmt
, t
);
2705 t
= build_int_cst (long_integer_type_node
, 1);
2706 gimple_omp_task_set_arg_align (stmt
, t
);
2711 /* If any decls have been made addressable during scan_omp,
2712 adjust their fields if needed, and layout record types
2713 of parallel/task constructs. */
2716 finish_taskreg_scan (omp_context
*ctx
)
2718 if (ctx
->record_type
== NULL_TREE
)
2721 /* If any task_shared_vars were needed, verify all
2722 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2723 statements if use_pointer_for_field hasn't changed
2724 because of that. If it did, update field types now. */
2725 if (task_shared_vars
)
2729 for (c
= gimple_omp_taskreg_clauses (ctx
->stmt
);
2730 c
; c
= OMP_CLAUSE_CHAIN (c
))
2731 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
2732 && !OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
2734 tree decl
= OMP_CLAUSE_DECL (c
);
2736 /* Global variables don't need to be copied,
2737 the receiver side will use them directly. */
2738 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2740 if (!bitmap_bit_p (task_shared_vars
, DECL_UID (decl
))
2741 || !use_pointer_for_field (decl
, ctx
))
2743 tree field
= lookup_field (decl
, ctx
);
2744 if (TREE_CODE (TREE_TYPE (field
)) == POINTER_TYPE
2745 && TREE_TYPE (TREE_TYPE (field
)) == TREE_TYPE (decl
))
2747 TREE_TYPE (field
) = build_pointer_type (TREE_TYPE (decl
));
2748 TREE_THIS_VOLATILE (field
) = 0;
2749 DECL_USER_ALIGN (field
) = 0;
2750 DECL_ALIGN (field
) = TYPE_ALIGN (TREE_TYPE (field
));
2751 if (TYPE_ALIGN (ctx
->record_type
) < DECL_ALIGN (field
))
2752 TYPE_ALIGN (ctx
->record_type
) = DECL_ALIGN (field
);
2753 if (ctx
->srecord_type
)
2755 tree sfield
= lookup_sfield (decl
, ctx
);
2756 TREE_TYPE (sfield
) = TREE_TYPE (field
);
2757 TREE_THIS_VOLATILE (sfield
) = 0;
2758 DECL_USER_ALIGN (sfield
) = 0;
2759 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
2760 if (TYPE_ALIGN (ctx
->srecord_type
) < DECL_ALIGN (sfield
))
2761 TYPE_ALIGN (ctx
->srecord_type
) = DECL_ALIGN (sfield
);
2766 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2768 layout_type (ctx
->record_type
);
2769 fixup_child_record_type (ctx
);
2773 location_t loc
= gimple_location (ctx
->stmt
);
2774 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2775 /* Move VLA fields to the end. */
2776 p
= &TYPE_FIELDS (ctx
->record_type
);
2778 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2779 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2782 *p
= TREE_CHAIN (*p
);
2783 TREE_CHAIN (*q
) = NULL_TREE
;
2784 q
= &TREE_CHAIN (*q
);
2787 p
= &DECL_CHAIN (*p
);
2789 if (gimple_omp_task_taskloop_p (ctx
->stmt
))
2791 /* Move fields corresponding to first and second _looptemp_
2792 clause first. There are filled by GOMP_taskloop
2793 and thus need to be in specific positions. */
2794 tree c1
= gimple_omp_task_clauses (ctx
->stmt
);
2795 c1
= find_omp_clause (c1
, OMP_CLAUSE__LOOPTEMP_
);
2796 tree c2
= find_omp_clause (OMP_CLAUSE_CHAIN (c1
),
2797 OMP_CLAUSE__LOOPTEMP_
);
2798 tree f1
= lookup_field (OMP_CLAUSE_DECL (c1
), ctx
);
2799 tree f2
= lookup_field (OMP_CLAUSE_DECL (c2
), ctx
);
2800 p
= &TYPE_FIELDS (ctx
->record_type
);
2802 if (*p
== f1
|| *p
== f2
)
2803 *p
= DECL_CHAIN (*p
);
2805 p
= &DECL_CHAIN (*p
);
2806 DECL_CHAIN (f1
) = f2
;
2807 DECL_CHAIN (f2
) = TYPE_FIELDS (ctx
->record_type
);
2808 TYPE_FIELDS (ctx
->record_type
) = f1
;
2809 if (ctx
->srecord_type
)
2811 f1
= lookup_sfield (OMP_CLAUSE_DECL (c1
), ctx
);
2812 f2
= lookup_sfield (OMP_CLAUSE_DECL (c2
), ctx
);
2813 p
= &TYPE_FIELDS (ctx
->srecord_type
);
2815 if (*p
== f1
|| *p
== f2
)
2816 *p
= DECL_CHAIN (*p
);
2818 p
= &DECL_CHAIN (*p
);
2819 DECL_CHAIN (f1
) = f2
;
2820 DECL_CHAIN (f2
) = TYPE_FIELDS (ctx
->srecord_type
);
2821 TYPE_FIELDS (ctx
->srecord_type
) = f1
;
2824 layout_type (ctx
->record_type
);
2825 fixup_child_record_type (ctx
);
2826 if (ctx
->srecord_type
)
2827 layout_type (ctx
->srecord_type
);
2828 tree t
= fold_convert_loc (loc
, long_integer_type_node
,
2829 TYPE_SIZE_UNIT (ctx
->record_type
));
2830 gimple_omp_task_set_arg_size (ctx
->stmt
, t
);
2831 t
= build_int_cst (long_integer_type_node
,
2832 TYPE_ALIGN_UNIT (ctx
->record_type
));
2833 gimple_omp_task_set_arg_align (ctx
->stmt
, t
);
2837 /* Find the enclosing offload context. */
2839 static omp_context
*
2840 enclosing_target_ctx (omp_context
*ctx
)
2842 for (; ctx
; ctx
= ctx
->outer
)
2843 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
)
2849 /* Return true if ctx is part of an oacc kernels region. */
2852 ctx_in_oacc_kernels_region (omp_context
*ctx
)
2854 for (;ctx
!= NULL
; ctx
= ctx
->outer
)
2856 gimple
*stmt
= ctx
->stmt
;
2857 if (gimple_code (stmt
) == GIMPLE_OMP_TARGET
2858 && gimple_omp_target_kind (stmt
) == GF_OMP_TARGET_KIND_OACC_KERNELS
)
2865 /* Check the parallelism clauses inside a kernels regions.
2866 Until kernels handling moves to use the same loop indirection
2867 scheme as parallel, we need to do this checking early. */
2870 check_oacc_kernel_gwv (gomp_for
*stmt
, omp_context
*ctx
)
2872 bool checking
= true;
2873 unsigned outer_mask
= 0;
2874 unsigned this_mask
= 0;
2875 bool has_seq
= false, has_auto
= false;
2878 outer_mask
= check_oacc_kernel_gwv (NULL
, ctx
->outer
);
2882 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
)
2884 stmt
= as_a
<gomp_for
*> (ctx
->stmt
);
2887 for (tree c
= gimple_omp_for_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
2889 switch (OMP_CLAUSE_CODE (c
))
2891 case OMP_CLAUSE_GANG
:
2892 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_GANG
);
2894 case OMP_CLAUSE_WORKER
:
2895 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_WORKER
);
2897 case OMP_CLAUSE_VECTOR
:
2898 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_VECTOR
);
2900 case OMP_CLAUSE_SEQ
:
2903 case OMP_CLAUSE_AUTO
:
2913 if (has_seq
&& (this_mask
|| has_auto
))
2914 error_at (gimple_location (stmt
), "%<seq%> overrides other"
2915 " OpenACC loop specifiers");
2916 else if (has_auto
&& this_mask
)
2917 error_at (gimple_location (stmt
), "%<auto%> conflicts with other"
2918 " OpenACC loop specifiers");
2920 if (this_mask
& outer_mask
)
2921 error_at (gimple_location (stmt
), "inner loop uses same"
2922 " OpenACC parallelism as containing loop");
2925 return outer_mask
| this_mask
;
2928 /* Scan a GIMPLE_OMP_FOR. */
2931 scan_omp_for (gomp_for
*stmt
, omp_context
*outer_ctx
)
2935 tree clauses
= gimple_omp_for_clauses (stmt
);
2937 ctx
= new_omp_context (stmt
, outer_ctx
);
2939 if (is_gimple_omp_oacc (stmt
))
2941 omp_context
*tgt
= enclosing_target_ctx (outer_ctx
);
2943 if (!tgt
|| is_oacc_parallel (tgt
))
2944 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2946 char const *check
= NULL
;
2948 switch (OMP_CLAUSE_CODE (c
))
2950 case OMP_CLAUSE_GANG
:
2954 case OMP_CLAUSE_WORKER
:
2958 case OMP_CLAUSE_VECTOR
:
2966 if (check
&& OMP_CLAUSE_OPERAND (c
, 0))
2967 error_at (gimple_location (stmt
),
2968 "argument not permitted on %qs clause in"
2969 " OpenACC %<parallel%>", check
);
2972 if (tgt
&& is_oacc_kernels (tgt
))
2974 /* Strip out reductions, as they are not handled yet. */
2975 tree
*prev_ptr
= &clauses
;
2977 while (tree probe
= *prev_ptr
)
2979 tree
*next_ptr
= &OMP_CLAUSE_CHAIN (probe
);
2981 if (OMP_CLAUSE_CODE (probe
) == OMP_CLAUSE_REDUCTION
)
2982 *prev_ptr
= *next_ptr
;
2984 prev_ptr
= next_ptr
;
2987 gimple_omp_for_set_clauses (stmt
, clauses
);
2988 check_oacc_kernel_gwv (stmt
, ctx
);
2992 scan_sharing_clauses (clauses
, ctx
);
2994 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
2995 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
2997 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
2998 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
2999 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
3000 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
3002 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3005 /* Scan an OpenMP sections directive. */
3008 scan_omp_sections (gomp_sections
*stmt
, omp_context
*outer_ctx
)
3012 ctx
= new_omp_context (stmt
, outer_ctx
);
3013 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
3014 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3017 /* Scan an OpenMP single directive. */
3020 scan_omp_single (gomp_single
*stmt
, omp_context
*outer_ctx
)
3025 ctx
= new_omp_context (stmt
, outer_ctx
);
3026 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
3027 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
3028 name
= create_tmp_var_name (".omp_copy_s");
3029 name
= build_decl (gimple_location (stmt
),
3030 TYPE_DECL
, name
, ctx
->record_type
);
3031 TYPE_NAME (ctx
->record_type
) = name
;
3033 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
3034 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3036 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
3037 ctx
->record_type
= NULL
;
3039 layout_type (ctx
->record_type
);
3042 /* Scan a GIMPLE_OMP_TARGET. */
3045 scan_omp_target (gomp_target
*stmt
, omp_context
*outer_ctx
)
3049 bool offloaded
= is_gimple_omp_offloaded (stmt
);
3050 tree clauses
= gimple_omp_target_clauses (stmt
);
3052 ctx
= new_omp_context (stmt
, outer_ctx
);
3053 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
3054 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
3055 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
3056 name
= create_tmp_var_name (".omp_data_t");
3057 name
= build_decl (gimple_location (stmt
),
3058 TYPE_DECL
, name
, ctx
->record_type
);
3059 DECL_ARTIFICIAL (name
) = 1;
3060 DECL_NAMELESS (name
) = 1;
3061 TYPE_NAME (ctx
->record_type
) = name
;
3062 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
3065 create_omp_child_function (ctx
, false);
3066 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
3069 scan_sharing_clauses (clauses
, ctx
);
3070 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3072 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
3073 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
3076 TYPE_FIELDS (ctx
->record_type
)
3077 = nreverse (TYPE_FIELDS (ctx
->record_type
));
3080 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
3081 for (tree field
= TYPE_FIELDS (ctx
->record_type
);
3083 field
= DECL_CHAIN (field
))
3084 gcc_assert (DECL_ALIGN (field
) == align
);
3086 layout_type (ctx
->record_type
);
3088 fixup_child_record_type (ctx
);
3092 /* Scan an OpenMP teams directive. */
3095 scan_omp_teams (gomp_teams
*stmt
, omp_context
*outer_ctx
)
3097 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
3098 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
3099 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3102 /* Check nesting restrictions. */
3104 check_omp_nesting_restrictions (gimple
*stmt
, omp_context
*ctx
)
3108 /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
3109 inside an OpenACC CTX. */
3110 if (!(is_gimple_omp (stmt
)
3111 && is_gimple_omp_oacc (stmt
)))
3113 for (omp_context
*ctx_
= ctx
; ctx_
!= NULL
; ctx_
= ctx_
->outer
)
3114 if (is_gimple_omp (ctx_
->stmt
)
3115 && is_gimple_omp_oacc (ctx_
->stmt
)
3116 /* Except for atomic codes that we share with OpenMP. */
3117 && ! (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
3118 || gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
))
3120 error_at (gimple_location (stmt
),
3121 "non-OpenACC construct inside of OpenACC region");
3128 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3129 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3132 if (gimple_code (stmt
) == GIMPLE_OMP_ORDERED
)
3134 c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3135 if (c
&& OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SIMD
)
3138 error_at (gimple_location (stmt
),
3139 "OpenMP constructs other than %<#pragma omp ordered simd%>"
3140 " may not be nested inside simd region");
3143 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3145 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
3146 || (gimple_omp_for_kind (stmt
)
3147 != GF_OMP_FOR_KIND_DISTRIBUTE
))
3148 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
3150 error_at (gimple_location (stmt
),
3151 "only distribute or parallel constructs are allowed to "
3152 "be closely nested inside teams construct");
3157 switch (gimple_code (stmt
))
3159 case GIMPLE_OMP_FOR
:
3160 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
3162 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
3164 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
3166 error_at (gimple_location (stmt
),
3167 "distribute construct must be closely nested inside "
3173 /* We split taskloop into task and nested taskloop in it. */
3174 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
3176 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
3181 switch (gimple_code (ctx
->stmt
))
3183 case GIMPLE_OMP_FOR
:
3184 ok
= (gimple_omp_for_kind (ctx
->stmt
)
3185 == GF_OMP_FOR_KIND_OACC_LOOP
);
3188 case GIMPLE_OMP_TARGET
:
3189 switch (gimple_omp_target_kind (ctx
->stmt
))
3191 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
3192 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
3203 else if (get_oacc_fn_attrib (current_function_decl
))
3207 error_at (gimple_location (stmt
),
3208 "OpenACC loop directive must be associated with"
3209 " an OpenACC compute region");
3215 if (is_gimple_call (stmt
)
3216 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3217 == BUILT_IN_GOMP_CANCEL
3218 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3219 == BUILT_IN_GOMP_CANCELLATION_POINT
))
3221 const char *bad
= NULL
;
3222 const char *kind
= NULL
;
3225 error_at (gimple_location (stmt
), "orphaned %qs construct",
3226 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3227 == BUILT_IN_GOMP_CANCEL
3228 ? "#pragma omp cancel"
3229 : "#pragma omp cancellation point");
3232 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
3233 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
3237 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
3238 bad
= "#pragma omp parallel";
3239 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3240 == BUILT_IN_GOMP_CANCEL
3241 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3242 ctx
->cancellable
= true;
3246 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3247 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
3248 bad
= "#pragma omp for";
3249 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3250 == BUILT_IN_GOMP_CANCEL
3251 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3253 ctx
->cancellable
= true;
3254 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3256 warning_at (gimple_location (stmt
), 0,
3257 "%<#pragma omp cancel for%> inside "
3258 "%<nowait%> for construct");
3259 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3260 OMP_CLAUSE_ORDERED
))
3261 warning_at (gimple_location (stmt
), 0,
3262 "%<#pragma omp cancel for%> inside "
3263 "%<ordered%> for construct");
3268 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
3269 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
3270 bad
= "#pragma omp sections";
3271 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3272 == BUILT_IN_GOMP_CANCEL
3273 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3275 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
3277 ctx
->cancellable
= true;
3278 if (find_omp_clause (gimple_omp_sections_clauses
3281 warning_at (gimple_location (stmt
), 0,
3282 "%<#pragma omp cancel sections%> inside "
3283 "%<nowait%> sections construct");
3287 gcc_assert (ctx
->outer
3288 && gimple_code (ctx
->outer
->stmt
)
3289 == GIMPLE_OMP_SECTIONS
);
3290 ctx
->outer
->cancellable
= true;
3291 if (find_omp_clause (gimple_omp_sections_clauses
3294 warning_at (gimple_location (stmt
), 0,
3295 "%<#pragma omp cancel sections%> inside "
3296 "%<nowait%> sections construct");
3302 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
3303 bad
= "#pragma omp task";
3305 ctx
->cancellable
= true;
3309 error_at (gimple_location (stmt
), "invalid arguments");
3314 error_at (gimple_location (stmt
),
3315 "%<%s %s%> construct not closely nested inside of %qs",
3316 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3317 == BUILT_IN_GOMP_CANCEL
3318 ? "#pragma omp cancel"
3319 : "#pragma omp cancellation point", kind
, bad
);
3324 case GIMPLE_OMP_SECTIONS
:
3325 case GIMPLE_OMP_SINGLE
:
3326 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3327 switch (gimple_code (ctx
->stmt
))
3329 case GIMPLE_OMP_FOR
:
3330 case GIMPLE_OMP_SECTIONS
:
3331 case GIMPLE_OMP_SINGLE
:
3332 case GIMPLE_OMP_ORDERED
:
3333 case GIMPLE_OMP_MASTER
:
3334 case GIMPLE_OMP_TASK
:
3335 case GIMPLE_OMP_CRITICAL
:
3336 if (is_gimple_call (stmt
))
3338 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3339 != BUILT_IN_GOMP_BARRIER
)
3341 error_at (gimple_location (stmt
),
3342 "barrier region may not be closely nested inside "
3343 "of work-sharing, critical, ordered, master or "
3344 "explicit task region");
3347 error_at (gimple_location (stmt
),
3348 "work-sharing region may not be closely nested inside "
3349 "of work-sharing, critical, ordered, master or explicit "
3352 case GIMPLE_OMP_PARALLEL
:
3358 case GIMPLE_OMP_MASTER
:
3359 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3360 switch (gimple_code (ctx
->stmt
))
3362 case GIMPLE_OMP_FOR
:
3363 case GIMPLE_OMP_SECTIONS
:
3364 case GIMPLE_OMP_SINGLE
:
3365 case GIMPLE_OMP_TASK
:
3366 error_at (gimple_location (stmt
),
3367 "master region may not be closely nested inside "
3368 "of work-sharing or explicit task region");
3370 case GIMPLE_OMP_PARALLEL
:
3376 case GIMPLE_OMP_TASK
:
3377 for (c
= gimple_omp_task_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
3378 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
3379 && (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
3380 || OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
))
3382 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3383 error_at (OMP_CLAUSE_LOCATION (c
),
3384 "%<depend(%s)%> is only allowed in %<omp ordered%>",
3385 kind
== OMP_CLAUSE_DEPEND_SOURCE
? "source" : "sink");
3389 case GIMPLE_OMP_ORDERED
:
3390 for (c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3391 c
; c
= OMP_CLAUSE_CHAIN (c
))
3393 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_DEPEND
)
3395 gcc_assert (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_THREADS
3397 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SIMD
));
3400 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3401 if (kind
== OMP_CLAUSE_DEPEND_SOURCE
3402 || kind
== OMP_CLAUSE_DEPEND_SINK
)
3405 /* Look for containing ordered(N) loop. */
3407 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3409 = find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3410 OMP_CLAUSE_ORDERED
)) == NULL_TREE
)
3412 error_at (OMP_CLAUSE_LOCATION (c
),
3413 "%<depend%> clause must be closely nested "
3414 "inside an ordered loop");
3417 else if (OMP_CLAUSE_ORDERED_EXPR (oclause
) == NULL_TREE
)
3419 error_at (OMP_CLAUSE_LOCATION (c
),
3420 "%<depend%> clause must be closely nested "
3421 "inside a loop with %<ordered%> clause with "
3428 error_at (OMP_CLAUSE_LOCATION (c
),
3429 "invalid depend kind in omp ordered depend");
3433 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3434 switch (gimple_code (ctx
->stmt
))
3436 case GIMPLE_OMP_CRITICAL
:
3437 case GIMPLE_OMP_TASK
:
3438 error_at (gimple_location (stmt
),
3439 "ordered region may not be closely nested inside "
3440 "of critical or explicit task region");
3442 case GIMPLE_OMP_FOR
:
3443 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3444 OMP_CLAUSE_ORDERED
) == NULL
)
3446 error_at (gimple_location (stmt
),
3447 "ordered region must be closely nested inside "
3448 "a loop region with an ordered clause");
3452 case GIMPLE_OMP_PARALLEL
:
3453 error_at (gimple_location (stmt
),
3454 "ordered region must be closely nested inside "
3455 "a loop region with an ordered clause");
3461 case GIMPLE_OMP_CRITICAL
:
3464 = gimple_omp_critical_name (as_a
<gomp_critical
*> (stmt
));
3465 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3466 if (gomp_critical
*other_crit
3467 = dyn_cast
<gomp_critical
*> (ctx
->stmt
))
3468 if (this_stmt_name
== gimple_omp_critical_name (other_crit
))
3470 error_at (gimple_location (stmt
),
3471 "critical region may not be nested inside a critical "
3472 "region with the same name");
3477 case GIMPLE_OMP_TEAMS
:
3479 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
3480 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
3482 error_at (gimple_location (stmt
),
3483 "teams construct not closely nested inside of target "
3488 case GIMPLE_OMP_TARGET
:
3489 for (c
= gimple_omp_target_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
3490 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
3491 && (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
3492 || OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
))
3494 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3495 error_at (OMP_CLAUSE_LOCATION (c
),
3496 "%<depend(%s)%> is only allowed in %<omp ordered%>",
3497 kind
== OMP_CLAUSE_DEPEND_SOURCE
? "source" : "sink");
3500 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3502 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
)
3504 if (is_gimple_omp (stmt
)
3505 && is_gimple_omp_oacc (stmt
)
3506 && is_gimple_omp (ctx
->stmt
))
3508 error_at (gimple_location (stmt
),
3509 "OpenACC construct inside of non-OpenACC region");
3515 const char *stmt_name
, *ctx_stmt_name
;
3516 switch (gimple_omp_target_kind (stmt
))
3518 case GF_OMP_TARGET_KIND_REGION
: stmt_name
= "target"; break;
3519 case GF_OMP_TARGET_KIND_DATA
: stmt_name
= "target data"; break;
3520 case GF_OMP_TARGET_KIND_UPDATE
: stmt_name
= "target update"; break;
3521 case GF_OMP_TARGET_KIND_ENTER_DATA
:
3522 stmt_name
= "target enter data"; break;
3523 case GF_OMP_TARGET_KIND_EXIT_DATA
:
3524 stmt_name
= "target exit data"; break;
3525 case GF_OMP_TARGET_KIND_OACC_PARALLEL
: stmt_name
= "parallel"; break;
3526 case GF_OMP_TARGET_KIND_OACC_KERNELS
: stmt_name
= "kernels"; break;
3527 case GF_OMP_TARGET_KIND_OACC_DATA
: stmt_name
= "data"; break;
3528 case GF_OMP_TARGET_KIND_OACC_UPDATE
: stmt_name
= "update"; break;
3529 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
3530 stmt_name
= "enter/exit data"; break;
3531 default: gcc_unreachable ();
3533 switch (gimple_omp_target_kind (ctx
->stmt
))
3535 case GF_OMP_TARGET_KIND_REGION
: ctx_stmt_name
= "target"; break;
3536 case GF_OMP_TARGET_KIND_DATA
: ctx_stmt_name
= "target data"; break;
3537 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
3538 ctx_stmt_name
= "parallel"; break;
3539 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
3540 ctx_stmt_name
= "kernels"; break;
3541 case GF_OMP_TARGET_KIND_OACC_DATA
: ctx_stmt_name
= "data"; break;
3542 default: gcc_unreachable ();
3545 /* OpenACC/OpenMP mismatch? */
3546 if (is_gimple_omp_oacc (stmt
)
3547 != is_gimple_omp_oacc (ctx
->stmt
))
3549 error_at (gimple_location (stmt
),
3550 "%s %s construct inside of %s %s region",
3551 (is_gimple_omp_oacc (stmt
)
3552 ? "OpenACC" : "OpenMP"), stmt_name
,
3553 (is_gimple_omp_oacc (ctx
->stmt
)
3554 ? "OpenACC" : "OpenMP"), ctx_stmt_name
);
3557 if (is_gimple_omp_offloaded (ctx
->stmt
))
3559 /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */
3560 if (is_gimple_omp_oacc (ctx
->stmt
))
3562 error_at (gimple_location (stmt
),
3563 "%s construct inside of %s region",
3564 stmt_name
, ctx_stmt_name
);
3569 gcc_checking_assert (!is_gimple_omp_oacc (stmt
));
3570 warning_at (gimple_location (stmt
), 0,
3571 "%s construct inside of %s region",
3572 stmt_name
, ctx_stmt_name
);
3584 /* Helper function scan_omp.
3586 Callback for walk_tree or operators in walk_gimple_stmt used to
3587 scan for OMP directives in TP. */
3590 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
3592 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
3593 omp_context
*ctx
= (omp_context
*) wi
->info
;
3596 switch (TREE_CODE (t
))
3603 *tp
= remap_decl (t
, &ctx
->cb
);
3607 if (ctx
&& TYPE_P (t
))
3608 *tp
= remap_type (t
, &ctx
->cb
);
3609 else if (!DECL_P (t
))
3614 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
3615 if (tem
!= TREE_TYPE (t
))
3617 if (TREE_CODE (t
) == INTEGER_CST
)
3618 *tp
= wide_int_to_tree (tem
, t
);
3620 TREE_TYPE (t
) = tem
;
3630 /* Return true if FNDECL is a setjmp or a longjmp. */
3633 setjmp_or_longjmp_p (const_tree fndecl
)
3635 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
3636 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
3637 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
3640 tree declname
= DECL_NAME (fndecl
);
3643 const char *name
= IDENTIFIER_POINTER (declname
);
3644 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
3648 /* Helper function for scan_omp.
3650 Callback for walk_gimple_stmt used to scan for OMP directives in
3651 the current statement in GSI. */
3654 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
3655 struct walk_stmt_info
*wi
)
3657 gimple
*stmt
= gsi_stmt (*gsi
);
3658 omp_context
*ctx
= (omp_context
*) wi
->info
;
3660 if (gimple_has_location (stmt
))
3661 input_location
= gimple_location (stmt
);
3663 /* Check the nesting restrictions. */
3664 bool remove
= false;
3665 if (is_gimple_omp (stmt
))
3666 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3667 else if (is_gimple_call (stmt
))
3669 tree fndecl
= gimple_call_fndecl (stmt
);
3672 if (setjmp_or_longjmp_p (fndecl
)
3674 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3675 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3678 error_at (gimple_location (stmt
),
3679 "setjmp/longjmp inside simd construct");
3681 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
3682 switch (DECL_FUNCTION_CODE (fndecl
))
3684 case BUILT_IN_GOMP_BARRIER
:
3685 case BUILT_IN_GOMP_CANCEL
:
3686 case BUILT_IN_GOMP_CANCELLATION_POINT
:
3687 case BUILT_IN_GOMP_TASKYIELD
:
3688 case BUILT_IN_GOMP_TASKWAIT
:
3689 case BUILT_IN_GOMP_TASKGROUP_START
:
3690 case BUILT_IN_GOMP_TASKGROUP_END
:
3691 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3700 stmt
= gimple_build_nop ();
3701 gsi_replace (gsi
, stmt
, false);
3704 *handled_ops_p
= true;
3706 switch (gimple_code (stmt
))
3708 case GIMPLE_OMP_PARALLEL
:
3709 taskreg_nesting_level
++;
3710 scan_omp_parallel (gsi
, ctx
);
3711 taskreg_nesting_level
--;
3714 case GIMPLE_OMP_TASK
:
3715 taskreg_nesting_level
++;
3716 scan_omp_task (gsi
, ctx
);
3717 taskreg_nesting_level
--;
3720 case GIMPLE_OMP_FOR
:
3721 scan_omp_for (as_a
<gomp_for
*> (stmt
), ctx
);
3724 case GIMPLE_OMP_SECTIONS
:
3725 scan_omp_sections (as_a
<gomp_sections
*> (stmt
), ctx
);
3728 case GIMPLE_OMP_SINGLE
:
3729 scan_omp_single (as_a
<gomp_single
*> (stmt
), ctx
);
3732 case GIMPLE_OMP_SECTION
:
3733 case GIMPLE_OMP_MASTER
:
3734 case GIMPLE_OMP_TASKGROUP
:
3735 case GIMPLE_OMP_ORDERED
:
3736 case GIMPLE_OMP_CRITICAL
:
3737 ctx
= new_omp_context (stmt
, ctx
);
3738 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3741 case GIMPLE_OMP_TARGET
:
3742 scan_omp_target (as_a
<gomp_target
*> (stmt
), ctx
);
3745 case GIMPLE_OMP_TEAMS
:
3746 scan_omp_teams (as_a
<gomp_teams
*> (stmt
), ctx
);
3753 *handled_ops_p
= false;
3755 for (var
= gimple_bind_vars (as_a
<gbind
*> (stmt
));
3757 var
= DECL_CHAIN (var
))
3758 insert_decl_map (&ctx
->cb
, var
, var
);
3762 *handled_ops_p
= false;
3770 /* Scan all the statements starting at the current statement. CTX
3771 contains context information about the OMP directives and
3772 clauses found during the scan. */
3775 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
3777 location_t saved_location
;
3778 struct walk_stmt_info wi
;
3780 memset (&wi
, 0, sizeof (wi
));
3782 wi
.want_locations
= true;
3784 saved_location
= input_location
;
3785 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
3786 input_location
= saved_location
;
3789 /* Re-gimplification and code generation routines. */
3791 /* Build a call to GOMP_barrier. */
3794 build_omp_barrier (tree lhs
)
3796 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
3797 : BUILT_IN_GOMP_BARRIER
);
3798 gcall
*g
= gimple_build_call (fndecl
, 0);
3800 gimple_call_set_lhs (g
, lhs
);
3804 /* If a context was created for STMT when it was scanned, return it. */
3806 static omp_context
*
3807 maybe_lookup_ctx (gimple
*stmt
)
3810 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
3811 return n
? (omp_context
*) n
->value
: NULL
;
3815 /* Find the mapping for DECL in CTX or the immediately enclosing
3816 context that has a mapping for DECL.
3818 If CTX is a nested parallel directive, we may have to use the decl
3819 mappings created in CTX's parent context. Suppose that we have the
3820 following parallel nesting (variable UIDs showed for clarity):
3823 #omp parallel shared(iD.1562) -> outer parallel
3824 iD.1562 = iD.1562 + 1;
3826 #omp parallel shared (iD.1562) -> inner parallel
3827 iD.1562 = iD.1562 - 1;
3829 Each parallel structure will create a distinct .omp_data_s structure
3830 for copying iD.1562 in/out of the directive:
3832 outer parallel .omp_data_s.1.i -> iD.1562
3833 inner parallel .omp_data_s.2.i -> iD.1562
3835 A shared variable mapping will produce a copy-out operation before
3836 the parallel directive and a copy-in operation after it. So, in
3837 this case we would have:
3840 .omp_data_o.1.i = iD.1562;
3841 #omp parallel shared(iD.1562) -> outer parallel
3842 .omp_data_i.1 = &.omp_data_o.1
3843 .omp_data_i.1->i = .omp_data_i.1->i + 1;
3845 .omp_data_o.2.i = iD.1562; -> **
3846 #omp parallel shared(iD.1562) -> inner parallel
3847 .omp_data_i.2 = &.omp_data_o.2
3848 .omp_data_i.2->i = .omp_data_i.2->i - 1;
3851 ** This is a problem. The symbol iD.1562 cannot be referenced
3852 inside the body of the outer parallel region. But since we are
3853 emitting this copy operation while expanding the inner parallel
3854 directive, we need to access the CTX structure of the outer
3855 parallel directive to get the correct mapping:
3857 .omp_data_o.2.i = .omp_data_i.1->i
3859 Since there may be other workshare or parallel directives enclosing
3860 the parallel directive, it may be necessary to walk up the context
3861 parent chain. This is not a problem in general because nested
3862 parallelism happens only rarely. */
3865 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3870 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3871 t
= maybe_lookup_decl (decl
, up
);
3873 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
3875 return t
? t
: decl
;
3879 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
3880 in outer contexts. */
3883 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3888 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3889 t
= maybe_lookup_decl (decl
, up
);
3891 return t
? t
: decl
;
3895 /* Construct the initialization value for reduction operation OP. */
3898 omp_reduction_init_op (location_t loc
, enum tree_code op
, tree type
)
3907 case TRUTH_ORIF_EXPR
:
3908 case TRUTH_XOR_EXPR
:
3910 return build_zero_cst (type
);
3913 case TRUTH_AND_EXPR
:
3914 case TRUTH_ANDIF_EXPR
:
3916 return fold_convert_loc (loc
, type
, integer_one_node
);
3919 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
3922 if (SCALAR_FLOAT_TYPE_P (type
))
3924 REAL_VALUE_TYPE max
, min
;
3925 if (HONOR_INFINITIES (type
))
3928 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
3931 real_maxval (&min
, 1, TYPE_MODE (type
));
3932 return build_real (type
, min
);
3934 else if (POINTER_TYPE_P (type
))
3937 = wi::min_value (TYPE_PRECISION (type
), TYPE_SIGN (type
));
3938 return wide_int_to_tree (type
, min
);
3942 gcc_assert (INTEGRAL_TYPE_P (type
));
3943 return TYPE_MIN_VALUE (type
);
3947 if (SCALAR_FLOAT_TYPE_P (type
))
3949 REAL_VALUE_TYPE max
;
3950 if (HONOR_INFINITIES (type
))
3953 real_maxval (&max
, 0, TYPE_MODE (type
));
3954 return build_real (type
, max
);
3956 else if (POINTER_TYPE_P (type
))
3959 = wi::max_value (TYPE_PRECISION (type
), TYPE_SIGN (type
));
3960 return wide_int_to_tree (type
, max
);
3964 gcc_assert (INTEGRAL_TYPE_P (type
));
3965 return TYPE_MAX_VALUE (type
);
3973 /* Construct the initialization value for reduction CLAUSE. */
3976 omp_reduction_init (tree clause
, tree type
)
3978 return omp_reduction_init_op (OMP_CLAUSE_LOCATION (clause
),
3979 OMP_CLAUSE_REDUCTION_CODE (clause
), type
);
3982 /* Return alignment to be assumed for var in CLAUSE, which should be
3983 OMP_CLAUSE_ALIGNED. */
3986 omp_clause_aligned_alignment (tree clause
)
3988 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
3989 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
3991 /* Otherwise return implementation defined alignment. */
3992 unsigned int al
= 1;
3993 machine_mode mode
, vmode
;
3994 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
3996 vs
= 1 << floor_log2 (vs
);
3997 static enum mode_class classes
[]
3998 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
3999 for (int i
= 0; i
< 4; i
+= 2)
4000 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
4002 mode
= GET_MODE_WIDER_MODE (mode
))
4004 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
4005 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
4008 && GET_MODE_SIZE (vmode
) < vs
4009 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
4010 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
4012 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
4013 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
4015 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
4016 / GET_MODE_SIZE (mode
));
4017 if (TYPE_MODE (type
) != vmode
)
4019 if (TYPE_ALIGN_UNIT (type
) > al
)
4020 al
= TYPE_ALIGN_UNIT (type
);
4022 return build_int_cst (integer_type_node
, al
);
4025 /* Return maximum possible vectorization factor for the target. */
4032 || !flag_tree_loop_optimize
4033 || (!flag_tree_loop_vectorize
4034 && (global_options_set
.x_flag_tree_loop_vectorize
4035 || global_options_set
.x_flag_tree_vectorize
)))
4038 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
4041 vs
= 1 << floor_log2 (vs
);
4044 machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
4045 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
4046 return GET_MODE_NUNITS (vqimode
);
4050 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
4054 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
4055 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
4059 max_vf
= omp_max_vf ();
4062 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
4063 OMP_CLAUSE_SAFELEN
);
4064 if (c
&& TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) != INTEGER_CST
)
4066 else if (c
&& compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
4068 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
4072 idx
= create_tmp_var (unsigned_type_node
);
4073 lane
= create_tmp_var (unsigned_type_node
);
4079 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
4080 tree avar
= create_tmp_var_raw (atype
);
4081 if (TREE_ADDRESSABLE (new_var
))
4082 TREE_ADDRESSABLE (avar
) = 1;
4083 DECL_ATTRIBUTES (avar
)
4084 = tree_cons (get_identifier ("omp simd array"), NULL
,
4085 DECL_ATTRIBUTES (avar
));
4086 gimple_add_tmp_var (avar
);
4087 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
4088 NULL_TREE
, NULL_TREE
);
4089 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
4090 NULL_TREE
, NULL_TREE
);
4091 if (DECL_P (new_var
))
4093 SET_DECL_VALUE_EXPR (new_var
, lvar
);
4094 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4099 /* Helper function of lower_rec_input_clauses. For a reference
4100 in simd reduction, add an underlying variable it will reference. */
4103 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
4105 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
4106 if (TREE_CONSTANT (z
))
4108 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)),
4109 get_name (new_vard
));
4110 gimple_add_tmp_var (z
);
4111 TREE_ADDRESSABLE (z
) = 1;
4112 z
= build_fold_addr_expr_loc (loc
, z
);
4113 gimplify_assign (new_vard
, z
, ilist
);
4117 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
4118 from the receiver (aka child) side and initializers for REFERENCE_TYPE
4119 private variables. Initialization statements go in ILIST, while calls
4120 to destructors go in DLIST. */
4123 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
4124 omp_context
*ctx
, struct omp_for_data
*fd
)
4126 tree c
, dtor
, copyin_seq
, x
, ptr
;
4127 bool copyin_by_ref
= false;
4128 bool lastprivate_firstprivate
= false;
4129 bool reduction_omp_orig_ref
= false;
4131 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4132 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
);
4134 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
4135 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
4136 gimple_seq llist
[2] = { NULL
, NULL
};
4140 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
4141 with data sharing clauses referencing variable sized vars. That
4142 is unnecessarily hard to support and very unlikely to result in
4143 vectorized code anyway. */
4145 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4146 switch (OMP_CLAUSE_CODE (c
))
4148 case OMP_CLAUSE_LINEAR
:
4149 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
4152 case OMP_CLAUSE_PRIVATE
:
4153 case OMP_CLAUSE_FIRSTPRIVATE
:
4154 case OMP_CLAUSE_LASTPRIVATE
:
4155 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
4158 case OMP_CLAUSE_REDUCTION
:
4159 if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
4160 || is_variable_sized (OMP_CLAUSE_DECL (c
)))
4167 /* Do all the fixed sized types in the first pass, and the variable sized
4168 types in the second pass. This makes sure that the scalar arguments to
4169 the variable sized types are processed before we use them in the
4170 variable sized operations. */
4171 for (pass
= 0; pass
< 2; ++pass
)
4173 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4175 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
4178 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4182 case OMP_CLAUSE_PRIVATE
:
4183 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
4186 case OMP_CLAUSE_SHARED
:
4187 /* Ignore shared directives in teams construct. */
4188 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
4190 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
4192 gcc_assert (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
)
4193 || is_global_var (OMP_CLAUSE_DECL (c
)));
4196 case OMP_CLAUSE_FIRSTPRIVATE
:
4197 case OMP_CLAUSE_COPYIN
:
4199 case OMP_CLAUSE_LINEAR
:
4200 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
)
4201 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
4202 lastprivate_firstprivate
= true;
4204 case OMP_CLAUSE_REDUCTION
:
4205 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
4206 reduction_omp_orig_ref
= true;
4208 case OMP_CLAUSE__LOOPTEMP_
:
4209 /* Handle _looptemp_ clauses only on parallel/task. */
4213 case OMP_CLAUSE_LASTPRIVATE
:
4214 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4216 lastprivate_firstprivate
= true;
4217 if (pass
!= 0 || is_taskloop_ctx (ctx
))
4220 /* Even without corresponding firstprivate, if
4221 decl is Fortran allocatable, it needs outer var
4224 && lang_hooks
.decls
.omp_private_outer_ref
4225 (OMP_CLAUSE_DECL (c
)))
4226 lastprivate_firstprivate
= true;
4228 case OMP_CLAUSE_ALIGNED
:
4231 var
= OMP_CLAUSE_DECL (c
);
4232 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
4233 && !is_global_var (var
))
4235 new_var
= maybe_lookup_decl (var
, ctx
);
4236 if (new_var
== NULL_TREE
)
4237 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
4238 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
4239 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
4240 omp_clause_aligned_alignment (c
));
4241 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4242 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
4243 gimplify_and_add (x
, ilist
);
4245 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
4246 && is_global_var (var
))
4248 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
4249 new_var
= lookup_decl (var
, ctx
);
4250 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
4251 t
= build_fold_addr_expr_loc (clause_loc
, t
);
4252 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
4253 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
4254 omp_clause_aligned_alignment (c
));
4255 t
= fold_convert_loc (clause_loc
, ptype
, t
);
4256 x
= create_tmp_var (ptype
);
4257 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
4258 gimplify_and_add (t
, ilist
);
4259 t
= build_simple_mem_ref_loc (clause_loc
, x
);
4260 SET_DECL_VALUE_EXPR (new_var
, t
);
4261 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4268 new_var
= var
= OMP_CLAUSE_DECL (c
);
4269 if (c_kind
== OMP_CLAUSE_REDUCTION
&& TREE_CODE (var
) == MEM_REF
)
4271 var
= TREE_OPERAND (var
, 0);
4272 if (TREE_CODE (var
) == POINTER_PLUS_EXPR
)
4273 var
= TREE_OPERAND (var
, 0);
4274 if (TREE_CODE (var
) == INDIRECT_REF
4275 || TREE_CODE (var
) == ADDR_EXPR
)
4276 var
= TREE_OPERAND (var
, 0);
4277 if (is_variable_sized (var
))
4279 gcc_assert (DECL_HAS_VALUE_EXPR_P (var
));
4280 var
= DECL_VALUE_EXPR (var
);
4281 gcc_assert (TREE_CODE (var
) == INDIRECT_REF
);
4282 var
= TREE_OPERAND (var
, 0);
4283 gcc_assert (DECL_P (var
));
4287 if (c_kind
!= OMP_CLAUSE_COPYIN
)
4288 new_var
= lookup_decl (var
, ctx
);
4290 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
4295 /* C/C++ array section reductions. */
4296 else if (c_kind
== OMP_CLAUSE_REDUCTION
4297 && var
!= OMP_CLAUSE_DECL (c
))
4302 tree bias
= TREE_OPERAND (OMP_CLAUSE_DECL (c
), 1);
4303 tree orig_var
= TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0);
4304 if (TREE_CODE (orig_var
) == POINTER_PLUS_EXPR
)
4306 tree b
= TREE_OPERAND (orig_var
, 1);
4307 b
= maybe_lookup_decl (b
, ctx
);
4310 b
= TREE_OPERAND (orig_var
, 1);
4311 b
= maybe_lookup_decl_in_outer_ctx (b
, ctx
);
4313 if (integer_zerop (bias
))
4317 bias
= fold_convert_loc (clause_loc
,
4318 TREE_TYPE (b
), bias
);
4319 bias
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
4320 TREE_TYPE (b
), b
, bias
);
4322 orig_var
= TREE_OPERAND (orig_var
, 0);
4324 if (TREE_CODE (orig_var
) == INDIRECT_REF
4325 || TREE_CODE (orig_var
) == ADDR_EXPR
)
4326 orig_var
= TREE_OPERAND (orig_var
, 0);
4327 tree d
= OMP_CLAUSE_DECL (c
);
4328 tree type
= TREE_TYPE (d
);
4329 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
4330 tree v
= TYPE_MAX_VALUE (TYPE_DOMAIN (type
));
4331 const char *name
= get_name (orig_var
);
4332 if (TREE_CONSTANT (v
))
4334 x
= create_tmp_var_raw (type
, name
);
4335 gimple_add_tmp_var (x
);
4336 TREE_ADDRESSABLE (x
) = 1;
4337 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4342 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4343 tree t
= maybe_lookup_decl (v
, ctx
);
4347 v
= maybe_lookup_decl_in_outer_ctx (v
, ctx
);
4348 gimplify_expr (&v
, ilist
, NULL
, is_gimple_val
, fb_rvalue
);
4349 t
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
4351 build_int_cst (TREE_TYPE (v
), 1));
4352 t
= fold_build2_loc (clause_loc
, MULT_EXPR
,
4354 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4355 tree al
= size_int (TYPE_ALIGN (TREE_TYPE (type
)));
4356 x
= build_call_expr_loc (clause_loc
, atmp
, 2, t
, al
);
4359 tree ptype
= build_pointer_type (TREE_TYPE (type
));
4360 x
= fold_convert_loc (clause_loc
, ptype
, x
);
4361 tree y
= create_tmp_var (ptype
, name
);
4362 gimplify_assign (y
, x
, ilist
);
4366 if (!integer_zerop (bias
))
4368 bias
= fold_convert_loc (clause_loc
, sizetype
, bias
);
4369 bias
= fold_build1_loc (clause_loc
, NEGATE_EXPR
,
4371 x
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
4372 TREE_TYPE (x
), x
, bias
);
4373 yb
= create_tmp_var (ptype
, name
);
4374 gimplify_assign (yb
, x
, ilist
);
4378 d
= TREE_OPERAND (d
, 0);
4379 if (TREE_CODE (d
) == POINTER_PLUS_EXPR
)
4380 d
= TREE_OPERAND (d
, 0);
4381 if (TREE_CODE (d
) == ADDR_EXPR
)
4383 if (orig_var
!= var
)
4385 gcc_assert (is_variable_sized (orig_var
));
4386 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
),
4388 gimplify_assign (new_var
, x
, ilist
);
4389 tree new_orig_var
= lookup_decl (orig_var
, ctx
);
4390 tree t
= build_fold_indirect_ref (new_var
);
4391 DECL_IGNORED_P (new_var
) = 0;
4392 TREE_THIS_NOTRAP (t
);
4393 SET_DECL_VALUE_EXPR (new_orig_var
, t
);
4394 DECL_HAS_VALUE_EXPR_P (new_orig_var
) = 1;
4398 x
= build2 (MEM_REF
, TREE_TYPE (new_var
), x
,
4399 build_int_cst (ptype
, 0));
4400 SET_DECL_VALUE_EXPR (new_var
, x
);
4401 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4406 gcc_assert (orig_var
== var
);
4407 if (TREE_CODE (d
) == INDIRECT_REF
)
4409 x
= create_tmp_var (ptype
, name
);
4410 TREE_ADDRESSABLE (x
) = 1;
4411 gimplify_assign (x
, yb
, ilist
);
4412 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4414 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4415 gimplify_assign (new_var
, x
, ilist
);
4417 tree y1
= create_tmp_var (ptype
, NULL
);
4418 gimplify_assign (y1
, y
, ilist
);
4419 tree i2
= NULL_TREE
, y2
= NULL_TREE
;
4420 tree body2
= NULL_TREE
, end2
= NULL_TREE
;
4421 tree y3
= NULL_TREE
, y4
= NULL_TREE
;
4422 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) || is_simd
)
4424 y2
= create_tmp_var (ptype
, NULL
);
4425 gimplify_assign (y2
, y
, ilist
);
4426 tree ref
= build_outer_var_ref (var
, ctx
);
4427 /* For ref build_outer_var_ref already performs this. */
4428 if (TREE_CODE (d
) == INDIRECT_REF
)
4429 gcc_assert (is_reference (var
));
4430 else if (TREE_CODE (d
) == ADDR_EXPR
)
4431 ref
= build_fold_addr_expr (ref
);
4432 else if (is_reference (var
))
4433 ref
= build_fold_addr_expr (ref
);
4434 ref
= fold_convert_loc (clause_loc
, ptype
, ref
);
4435 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
)
4436 && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
4438 y3
= create_tmp_var (ptype
, NULL
);
4439 gimplify_assign (y3
, unshare_expr (ref
), ilist
);
4443 y4
= create_tmp_var (ptype
, NULL
);
4444 gimplify_assign (y4
, ref
, dlist
);
4447 tree i
= create_tmp_var (TREE_TYPE (v
), NULL
);
4448 gimplify_assign (i
, build_int_cst (TREE_TYPE (v
), 0), ilist
);
4449 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
4450 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
4451 gimple_seq_add_stmt (ilist
, gimple_build_label (body
));
4454 i2
= create_tmp_var (TREE_TYPE (v
), NULL
);
4455 gimplify_assign (i2
, build_int_cst (TREE_TYPE (v
), 0), dlist
);
4456 body2
= create_artificial_label (UNKNOWN_LOCATION
);
4457 end2
= create_artificial_label (UNKNOWN_LOCATION
);
4458 gimple_seq_add_stmt (dlist
, gimple_build_label (body2
));
4460 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4462 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4463 tree decl_placeholder
4464 = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
);
4465 SET_DECL_VALUE_EXPR (decl_placeholder
,
4466 build_simple_mem_ref (y1
));
4467 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 1;
4468 SET_DECL_VALUE_EXPR (placeholder
,
4469 y3
? build_simple_mem_ref (y3
)
4471 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4472 x
= lang_hooks
.decls
.omp_clause_default_ctor
4473 (c
, build_simple_mem_ref (y1
),
4474 y3
? build_simple_mem_ref (y3
) : NULL_TREE
);
4476 gimplify_and_add (x
, ilist
);
4477 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4479 gimple_seq tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4480 lower_omp (&tseq
, ctx
);
4481 gimple_seq_add_seq (ilist
, tseq
);
4483 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4486 SET_DECL_VALUE_EXPR (decl_placeholder
,
4487 build_simple_mem_ref (y2
));
4488 SET_DECL_VALUE_EXPR (placeholder
,
4489 build_simple_mem_ref (y4
));
4490 gimple_seq tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4491 lower_omp (&tseq
, ctx
);
4492 gimple_seq_add_seq (dlist
, tseq
);
4493 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4495 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4496 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 0;
4497 x
= lang_hooks
.decls
.omp_clause_dtor
4498 (c
, build_simple_mem_ref (y2
));
4501 gimple_seq tseq
= NULL
;
4503 gimplify_stmt (&dtor
, &tseq
);
4504 gimple_seq_add_seq (dlist
, tseq
);
4509 x
= omp_reduction_init (c
, TREE_TYPE (type
));
4510 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4512 /* reduction(-:var) sums up the partial results, so it
4513 acts identically to reduction(+:var). */
4514 if (code
== MINUS_EXPR
)
4517 gimplify_assign (build_simple_mem_ref (y1
), x
, ilist
);
4520 x
= build2 (code
, TREE_TYPE (type
),
4521 build_simple_mem_ref (y4
),
4522 build_simple_mem_ref (y2
));
4523 gimplify_assign (build_simple_mem_ref (y4
), x
, dlist
);
4527 = gimple_build_assign (y1
, POINTER_PLUS_EXPR
, y1
,
4528 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4529 gimple_seq_add_stmt (ilist
, g
);
4532 g
= gimple_build_assign (y3
, POINTER_PLUS_EXPR
, y3
,
4533 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4534 gimple_seq_add_stmt (ilist
, g
);
4536 g
= gimple_build_assign (i
, PLUS_EXPR
, i
,
4537 build_int_cst (TREE_TYPE (i
), 1));
4538 gimple_seq_add_stmt (ilist
, g
);
4539 g
= gimple_build_cond (LE_EXPR
, i
, v
, body
, end
);
4540 gimple_seq_add_stmt (ilist
, g
);
4541 gimple_seq_add_stmt (ilist
, gimple_build_label (end
));
4544 g
= gimple_build_assign (y2
, POINTER_PLUS_EXPR
, y2
,
4545 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4546 gimple_seq_add_stmt (dlist
, g
);
4549 g
= gimple_build_assign
4550 (y4
, POINTER_PLUS_EXPR
, y4
,
4551 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4552 gimple_seq_add_stmt (dlist
, g
);
4554 g
= gimple_build_assign (i2
, PLUS_EXPR
, i2
,
4555 build_int_cst (TREE_TYPE (i2
), 1));
4556 gimple_seq_add_stmt (dlist
, g
);
4557 g
= gimple_build_cond (LE_EXPR
, i2
, v
, body2
, end2
);
4558 gimple_seq_add_stmt (dlist
, g
);
4559 gimple_seq_add_stmt (dlist
, gimple_build_label (end2
));
4563 else if (is_variable_sized (var
))
4565 /* For variable sized types, we need to allocate the
4566 actual storage here. Call alloca and store the
4567 result in the pointer decl that we created elsewhere. */
4571 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
4576 ptr
= DECL_VALUE_EXPR (new_var
);
4577 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
4578 ptr
= TREE_OPERAND (ptr
, 0);
4579 gcc_assert (DECL_P (ptr
));
4580 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
4582 /* void *tmp = __builtin_alloca */
4583 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4584 stmt
= gimple_build_call (atmp
, 2, x
,
4585 size_int (DECL_ALIGN (var
)));
4586 tmp
= create_tmp_var_raw (ptr_type_node
);
4587 gimple_add_tmp_var (tmp
);
4588 gimple_call_set_lhs (stmt
, tmp
);
4590 gimple_seq_add_stmt (ilist
, stmt
);
4592 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
4593 gimplify_assign (ptr
, x
, ilist
);
4596 else if (is_reference (var
))
4598 /* For references that are being privatized for Fortran,
4599 allocate new backing storage for the new pointer
4600 variable. This allows us to avoid changing all the
4601 code that expects a pointer to something that expects
4602 a direct variable. */
4606 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
4607 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
4609 x
= build_receiver_ref (var
, false, ctx
);
4610 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4612 else if (TREE_CONSTANT (x
))
4614 /* For reduction in SIMD loop, defer adding the
4615 initialization of the reference, because if we decide
4616 to use SIMD array for it, the initilization could cause
4618 if (c_kind
== OMP_CLAUSE_REDUCTION
&& is_simd
)
4622 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
4624 gimple_add_tmp_var (x
);
4625 TREE_ADDRESSABLE (x
) = 1;
4626 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4632 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4633 tree rtype
= TREE_TYPE (TREE_TYPE (new_var
));
4634 tree al
= size_int (TYPE_ALIGN (rtype
));
4635 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
4640 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4641 gimplify_assign (new_var
, x
, ilist
);
4644 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4646 else if (c_kind
== OMP_CLAUSE_REDUCTION
4647 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4655 switch (OMP_CLAUSE_CODE (c
))
4657 case OMP_CLAUSE_SHARED
:
4658 /* Ignore shared directives in teams construct. */
4659 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
4661 /* Shared global vars are just accessed directly. */
4662 if (is_global_var (new_var
))
4664 /* For taskloop firstprivate/lastprivate, represented
4665 as firstprivate and shared clause on the task, new_var
4666 is the firstprivate var. */
4667 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
4669 /* Set up the DECL_VALUE_EXPR for shared variables now. This
4670 needs to be delayed until after fixup_child_record_type so
4671 that we get the correct type during the dereference. */
4672 by_ref
= use_pointer_for_field (var
, ctx
);
4673 x
= build_receiver_ref (var
, by_ref
, ctx
);
4674 SET_DECL_VALUE_EXPR (new_var
, x
);
4675 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4677 /* ??? If VAR is not passed by reference, and the variable
4678 hasn't been initialized yet, then we'll get a warning for
4679 the store into the omp_data_s structure. Ideally, we'd be
4680 able to notice this and not store anything at all, but
4681 we're generating code too early. Suppress the warning. */
4683 TREE_NO_WARNING (var
) = 1;
4686 case OMP_CLAUSE_LASTPRIVATE
:
4687 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4691 case OMP_CLAUSE_PRIVATE
:
4692 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
4693 x
= build_outer_var_ref (var
, ctx
);
4694 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4696 if (is_task_ctx (ctx
))
4697 x
= build_receiver_ref (var
, false, ctx
);
4699 x
= build_outer_var_ref (var
, ctx
);
4705 nx
= lang_hooks
.decls
.omp_clause_default_ctor
4706 (c
, unshare_expr (new_var
), x
);
4709 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
4710 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
4711 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
4712 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4713 idx
, lane
, ivar
, lvar
))
4716 x
= lang_hooks
.decls
.omp_clause_default_ctor
4717 (c
, unshare_expr (ivar
), x
);
4719 gimplify_and_add (x
, &llist
[0]);
4722 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4725 gimple_seq tseq
= NULL
;
4728 gimplify_stmt (&dtor
, &tseq
);
4729 gimple_seq_add_seq (&llist
[1], tseq
);
4736 gimplify_and_add (nx
, ilist
);
4740 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
4743 gimple_seq tseq
= NULL
;
4746 gimplify_stmt (&dtor
, &tseq
);
4747 gimple_seq_add_seq (dlist
, tseq
);
4751 case OMP_CLAUSE_LINEAR
:
4752 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
4753 goto do_firstprivate
;
4754 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
4757 x
= build_outer_var_ref (var
, ctx
);
4760 case OMP_CLAUSE_FIRSTPRIVATE
:
4761 if (is_task_ctx (ctx
))
4763 if (is_reference (var
) || is_variable_sized (var
))
4765 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
4767 || use_pointer_for_field (var
, NULL
))
4769 x
= build_receiver_ref (var
, false, ctx
);
4770 SET_DECL_VALUE_EXPR (new_var
, x
);
4771 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4776 x
= build_outer_var_ref (var
, ctx
);
4779 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
4780 && gimple_omp_for_combined_into_p (ctx
->stmt
))
4782 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
4783 tree stept
= TREE_TYPE (t
);
4784 tree ct
= find_omp_clause (clauses
,
4785 OMP_CLAUSE__LOOPTEMP_
);
4787 tree l
= OMP_CLAUSE_DECL (ct
);
4788 tree n1
= fd
->loop
.n1
;
4789 tree step
= fd
->loop
.step
;
4790 tree itype
= TREE_TYPE (l
);
4791 if (POINTER_TYPE_P (itype
))
4792 itype
= signed_type_for (itype
);
4793 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
4794 if (TYPE_UNSIGNED (itype
)
4795 && fd
->loop
.cond_code
== GT_EXPR
)
4796 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
4797 fold_build1 (NEGATE_EXPR
, itype
, l
),
4798 fold_build1 (NEGATE_EXPR
,
4801 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
4802 t
= fold_build2 (MULT_EXPR
, stept
,
4803 fold_convert (stept
, l
), t
);
4805 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
4807 x
= lang_hooks
.decls
.omp_clause_linear_ctor
4809 gimplify_and_add (x
, ilist
);
4813 if (POINTER_TYPE_P (TREE_TYPE (x
)))
4814 x
= fold_build2 (POINTER_PLUS_EXPR
,
4815 TREE_TYPE (x
), x
, t
);
4817 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
4820 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
4821 || TREE_ADDRESSABLE (new_var
))
4822 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4823 idx
, lane
, ivar
, lvar
))
4825 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
4827 tree iv
= create_tmp_var (TREE_TYPE (new_var
));
4828 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
4829 gimplify_and_add (x
, ilist
);
4830 gimple_stmt_iterator gsi
4831 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
4833 = gimple_build_assign (unshare_expr (lvar
), iv
);
4834 gsi_insert_before_without_update (&gsi
, g
,
4836 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
4837 enum tree_code code
= PLUS_EXPR
;
4838 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
4839 code
= POINTER_PLUS_EXPR
;
4840 g
= gimple_build_assign (iv
, code
, iv
, t
);
4841 gsi_insert_before_without_update (&gsi
, g
,
4845 x
= lang_hooks
.decls
.omp_clause_copy_ctor
4846 (c
, unshare_expr (ivar
), x
);
4847 gimplify_and_add (x
, &llist
[0]);
4848 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4851 gimple_seq tseq
= NULL
;
4854 gimplify_stmt (&dtor
, &tseq
);
4855 gimple_seq_add_seq (&llist
[1], tseq
);
4860 x
= lang_hooks
.decls
.omp_clause_copy_ctor
4861 (c
, unshare_expr (new_var
), x
);
4862 gimplify_and_add (x
, ilist
);
4865 case OMP_CLAUSE__LOOPTEMP_
:
4866 gcc_assert (is_taskreg_ctx (ctx
));
4867 x
= build_outer_var_ref (var
, ctx
);
4868 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
4869 gimplify_and_add (x
, ilist
);
4872 case OMP_CLAUSE_COPYIN
:
4873 by_ref
= use_pointer_for_field (var
, NULL
);
4874 x
= build_receiver_ref (var
, by_ref
, ctx
);
4875 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
4876 append_to_statement_list (x
, ©in_seq
);
4877 copyin_by_ref
|= by_ref
;
4880 case OMP_CLAUSE_REDUCTION
:
4881 /* OpenACC reductions are initialized using the
4882 GOACC_REDUCTION internal function. */
4883 if (is_gimple_omp_oacc (ctx
->stmt
))
4885 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4887 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4889 x
= build_outer_var_ref (var
, ctx
);
4891 if (is_reference (var
)
4892 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
4894 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4895 SET_DECL_VALUE_EXPR (placeholder
, x
);
4896 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4897 tree new_vard
= new_var
;
4898 if (is_reference (var
))
4900 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
4901 new_vard
= TREE_OPERAND (new_var
, 0);
4902 gcc_assert (DECL_P (new_vard
));
4905 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4906 idx
, lane
, ivar
, lvar
))
4908 if (new_vard
== new_var
)
4910 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
4911 SET_DECL_VALUE_EXPR (new_var
, ivar
);
4915 SET_DECL_VALUE_EXPR (new_vard
,
4916 build_fold_addr_expr (ivar
));
4917 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
4919 x
= lang_hooks
.decls
.omp_clause_default_ctor
4920 (c
, unshare_expr (ivar
),
4921 build_outer_var_ref (var
, ctx
));
4923 gimplify_and_add (x
, &llist
[0]);
4924 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4926 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4927 lower_omp (&tseq
, ctx
);
4928 gimple_seq_add_seq (&llist
[0], tseq
);
4930 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4931 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4932 lower_omp (&tseq
, ctx
);
4933 gimple_seq_add_seq (&llist
[1], tseq
);
4934 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4935 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4936 if (new_vard
== new_var
)
4937 SET_DECL_VALUE_EXPR (new_var
, lvar
);
4939 SET_DECL_VALUE_EXPR (new_vard
,
4940 build_fold_addr_expr (lvar
));
4941 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4946 gimplify_stmt (&dtor
, &tseq
);
4947 gimple_seq_add_seq (&llist
[1], tseq
);
4951 /* If this is a reference to constant size reduction var
4952 with placeholder, we haven't emitted the initializer
4953 for it because it is undesirable if SIMD arrays are used.
4954 But if they aren't used, we need to emit the deferred
4955 initialization now. */
4956 else if (is_reference (var
) && is_simd
)
4957 handle_simd_reference (clause_loc
, new_vard
, ilist
);
4958 x
= lang_hooks
.decls
.omp_clause_default_ctor
4959 (c
, unshare_expr (new_var
),
4960 build_outer_var_ref (var
, ctx
));
4962 gimplify_and_add (x
, ilist
);
4963 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4965 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4966 lower_omp (&tseq
, ctx
);
4967 gimple_seq_add_seq (ilist
, tseq
);
4969 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4972 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4973 lower_omp (&tseq
, ctx
);
4974 gimple_seq_add_seq (dlist
, tseq
);
4975 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4977 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4982 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
4983 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
4984 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4986 /* reduction(-:var) sums up the partial results, so it
4987 acts identically to reduction(+:var). */
4988 if (code
== MINUS_EXPR
)
4991 tree new_vard
= new_var
;
4992 if (is_simd
&& is_reference (var
))
4994 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
4995 new_vard
= TREE_OPERAND (new_var
, 0);
4996 gcc_assert (DECL_P (new_vard
));
4999 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
5000 idx
, lane
, ivar
, lvar
))
5002 tree ref
= build_outer_var_ref (var
, ctx
);
5004 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
5006 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
5007 ref
= build_outer_var_ref (var
, ctx
);
5008 gimplify_assign (ref
, x
, &llist
[1]);
5010 if (new_vard
!= new_var
)
5012 SET_DECL_VALUE_EXPR (new_vard
,
5013 build_fold_addr_expr (lvar
));
5014 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
5019 if (is_reference (var
) && is_simd
)
5020 handle_simd_reference (clause_loc
, new_vard
, ilist
);
5021 gimplify_assign (new_var
, x
, ilist
);
5024 tree ref
= build_outer_var_ref (var
, ctx
);
5026 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
5027 ref
= build_outer_var_ref (var
, ctx
);
5028 gimplify_assign (ref
, x
, dlist
);
5042 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
5043 /* Don't want uninit warnings on simduid, it is always uninitialized,
5044 but we use it not for the value, but for the DECL_UID only. */
5045 TREE_NO_WARNING (uid
) = 1;
5047 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
5048 gimple_call_set_lhs (g
, lane
);
5049 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
5050 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
5051 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
5052 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
5053 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
5054 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
5055 g
= gimple_build_assign (lane
, INTEGER_CST
,
5056 build_int_cst (unsigned_type_node
, 0));
5057 gimple_seq_add_stmt (ilist
, g
);
5058 for (int i
= 0; i
< 2; i
++)
5061 tree vf
= create_tmp_var (unsigned_type_node
);
5062 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
5063 gimple_call_set_lhs (g
, vf
);
5064 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
5065 gimple_seq_add_stmt (seq
, g
);
5066 tree t
= build_int_cst (unsigned_type_node
, 0);
5067 g
= gimple_build_assign (idx
, INTEGER_CST
, t
);
5068 gimple_seq_add_stmt (seq
, g
);
5069 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
5070 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
5071 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
5072 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
5073 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
5074 gimple_seq_add_seq (seq
, llist
[i
]);
5075 t
= build_int_cst (unsigned_type_node
, 1);
5076 g
= gimple_build_assign (idx
, PLUS_EXPR
, idx
, t
);
5077 gimple_seq_add_stmt (seq
, g
);
5078 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
5079 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
5080 gimple_seq_add_stmt (seq
, g
);
5081 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
5085 /* The copyin sequence is not to be executed by the main thread, since
5086 that would result in self-copies. Perhaps not visible to scalars,
5087 but it certainly is to C++ operator=. */
5090 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
5092 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
5093 build_int_cst (TREE_TYPE (x
), 0));
5094 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
5095 gimplify_and_add (x
, ilist
);
5098 /* If any copyin variable is passed by reference, we must ensure the
5099 master thread doesn't modify it before it is copied over in all
5100 threads. Similarly for variables in both firstprivate and
5101 lastprivate clauses we need to ensure the lastprivate copying
5102 happens after firstprivate copying in all threads. And similarly
5103 for UDRs if initializer expression refers to omp_orig. */
5104 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
5106 /* Don't add any barrier for #pragma omp simd or
5107 #pragma omp distribute. */
5108 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
5109 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
5110 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
5113 /* If max_vf is non-zero, then we can use only a vectorization factor
5114 up to the max_vf we chose. So stick it into the safelen clause. */
5117 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
5118 OMP_CLAUSE_SAFELEN
);
5120 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) == INTEGER_CST
5121 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
5124 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
5125 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
5127 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
5128 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
5134 /* Generate code to implement the LASTPRIVATE clauses. This is used for
5135 both parallel and workshare constructs. PREDICATE may be NULL if it's
5139 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
5142 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
5143 bool par_clauses
= false;
5144 tree simduid
= NULL
, lastlane
= NULL
;
5146 /* Early exit if there are no lastprivate or linear clauses. */
5147 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
5148 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
5149 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
5150 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
5152 if (clauses
== NULL
)
5154 /* If this was a workshare clause, see if it had been combined
5155 with its parallel. In that case, look for the clauses on the
5156 parallel statement itself. */
5157 if (is_parallel_ctx (ctx
))
5161 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
5164 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
5165 OMP_CLAUSE_LASTPRIVATE
);
5166 if (clauses
== NULL
)
5174 tree label_true
, arm1
, arm2
;
5176 label
= create_artificial_label (UNKNOWN_LOCATION
);
5177 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
5178 arm1
= TREE_OPERAND (predicate
, 0);
5179 arm2
= TREE_OPERAND (predicate
, 1);
5180 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
5181 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
5182 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
5184 gimple_seq_add_stmt (stmt_list
, stmt
);
5185 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
5188 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
5189 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
5191 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
5193 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
5196 for (c
= clauses
; c
;)
5199 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5201 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5202 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
5203 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
5205 var
= OMP_CLAUSE_DECL (c
);
5206 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5207 && OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
)
5208 && is_taskloop_ctx (ctx
))
5210 gcc_checking_assert (ctx
->outer
&& is_task_ctx (ctx
->outer
));
5211 new_var
= lookup_decl (var
, ctx
->outer
);
5214 new_var
= lookup_decl (var
, ctx
);
5216 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
5218 tree val
= DECL_VALUE_EXPR (new_var
);
5219 if (TREE_CODE (val
) == ARRAY_REF
5220 && VAR_P (TREE_OPERAND (val
, 0))
5221 && lookup_attribute ("omp simd array",
5222 DECL_ATTRIBUTES (TREE_OPERAND (val
,
5225 if (lastlane
== NULL
)
5227 lastlane
= create_tmp_var (unsigned_type_node
);
5229 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
5231 TREE_OPERAND (val
, 1));
5232 gimple_call_set_lhs (g
, lastlane
);
5233 gimple_seq_add_stmt (stmt_list
, g
);
5235 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
5236 TREE_OPERAND (val
, 0), lastlane
,
5237 NULL_TREE
, NULL_TREE
);
5241 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5242 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
5244 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
5245 gimple_seq_add_seq (stmt_list
,
5246 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
5247 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
5249 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
5250 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
5252 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
5253 gimple_seq_add_seq (stmt_list
,
5254 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
5255 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
5259 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5260 && OMP_CLAUSE_LASTPRIVATE_TASKLOOP_IV (c
))
5262 gcc_checking_assert (is_taskloop_ctx (ctx
));
5263 tree ovar
= maybe_lookup_decl_in_outer_ctx (var
,
5265 if (is_global_var (ovar
))
5269 x
= build_outer_var_ref (var
, ctx
, true);
5270 if (is_reference (var
))
5271 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5272 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
5273 gimplify_and_add (x
, stmt_list
);
5275 c
= OMP_CLAUSE_CHAIN (c
);
5276 if (c
== NULL
&& !par_clauses
)
5278 /* If this was a workshare clause, see if it had been combined
5279 with its parallel. In that case, continue looking for the
5280 clauses also on the parallel statement itself. */
5281 if (is_parallel_ctx (ctx
))
5285 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
5288 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
5289 OMP_CLAUSE_LASTPRIVATE
);
5295 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
5298 /* Lower the OpenACC reductions of CLAUSES for compute axis LEVEL
5299 (which might be a placeholder). INNER is true if this is an inner
5300 axis of a multi-axis loop. FORK and JOIN are (optional) fork and
5301 join markers. Generate the before-loop forking sequence in
5302 FORK_SEQ and the after-loop joining sequence to JOIN_SEQ. The
5303 general form of these sequences is
5305 GOACC_REDUCTION_SETUP
5307 GOACC_REDUCTION_INIT
5309 GOACC_REDUCTION_FINI
5311 GOACC_REDUCTION_TEARDOWN. */
5314 lower_oacc_reductions (location_t loc
, tree clauses
, tree level
, bool inner
,
5315 gcall
*fork
, gcall
*join
, gimple_seq
*fork_seq
,
5316 gimple_seq
*join_seq
, omp_context
*ctx
)
5318 gimple_seq before_fork
= NULL
;
5319 gimple_seq after_fork
= NULL
;
5320 gimple_seq before_join
= NULL
;
5321 gimple_seq after_join
= NULL
;
5322 tree init_code
= NULL_TREE
, fini_code
= NULL_TREE
,
5323 setup_code
= NULL_TREE
, teardown_code
= NULL_TREE
;
5324 unsigned offset
= 0;
5326 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5327 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
5329 tree orig
= OMP_CLAUSE_DECL (c
);
5330 tree var
= maybe_lookup_decl (orig
, ctx
);
5331 tree ref_to_res
= NULL_TREE
;
5332 tree incoming
, outgoing
;
5334 enum tree_code rcode
= OMP_CLAUSE_REDUCTION_CODE (c
);
5335 if (rcode
== MINUS_EXPR
)
5337 else if (rcode
== TRUTH_ANDIF_EXPR
)
5338 rcode
= BIT_AND_EXPR
;
5339 else if (rcode
== TRUTH_ORIF_EXPR
)
5340 rcode
= BIT_IOR_EXPR
;
5341 tree op
= build_int_cst (unsigned_type_node
, rcode
);
5345 gcc_assert (!is_reference (var
));
5347 incoming
= outgoing
= var
;
5351 /* See if an outer construct also reduces this variable. */
5352 omp_context
*outer
= ctx
;
5354 while (omp_context
*probe
= outer
->outer
)
5356 enum gimple_code type
= gimple_code (probe
->stmt
);
5361 case GIMPLE_OMP_FOR
:
5362 cls
= gimple_omp_for_clauses (probe
->stmt
);
5365 case GIMPLE_OMP_TARGET
:
5366 if (gimple_omp_target_kind (probe
->stmt
)
5367 != GF_OMP_TARGET_KIND_OACC_PARALLEL
)
5370 cls
= gimple_omp_target_clauses (probe
->stmt
);
5378 for (; cls
; cls
= OMP_CLAUSE_CHAIN (cls
))
5379 if (OMP_CLAUSE_CODE (cls
) == OMP_CLAUSE_REDUCTION
5380 && orig
== OMP_CLAUSE_DECL (cls
))
5381 goto has_outer_reduction
;
5385 /* This is the outermost construct with this reduction,
5386 see if there's a mapping for it. */
5387 if (gimple_code (outer
->stmt
) == GIMPLE_OMP_TARGET
5388 && maybe_lookup_field (orig
, outer
))
5390 ref_to_res
= build_receiver_ref (orig
, false, outer
);
5391 if (is_reference (orig
))
5392 ref_to_res
= build_simple_mem_ref (ref_to_res
);
5395 incoming
= omp_reduction_init_op (loc
, rcode
, TREE_TYPE (var
));
5398 incoming
= outgoing
= orig
;
5400 has_outer_reduction
:;
5404 ref_to_res
= integer_zero_node
;
5406 /* Determine position in reduction buffer, which may be used
5408 enum machine_mode mode
= TYPE_MODE (TREE_TYPE (var
));
5409 unsigned align
= GET_MODE_ALIGNMENT (mode
) / BITS_PER_UNIT
;
5410 offset
= (offset
+ align
- 1) & ~(align
- 1);
5411 tree off
= build_int_cst (sizetype
, offset
);
5412 offset
+= GET_MODE_SIZE (mode
);
5416 init_code
= build_int_cst (integer_type_node
,
5417 IFN_GOACC_REDUCTION_INIT
);
5418 fini_code
= build_int_cst (integer_type_node
,
5419 IFN_GOACC_REDUCTION_FINI
);
5420 setup_code
= build_int_cst (integer_type_node
,
5421 IFN_GOACC_REDUCTION_SETUP
);
5422 teardown_code
= build_int_cst (integer_type_node
,
5423 IFN_GOACC_REDUCTION_TEARDOWN
);
5427 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5428 TREE_TYPE (var
), 6, setup_code
,
5429 unshare_expr (ref_to_res
),
5430 incoming
, level
, op
, off
);
5432 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5433 TREE_TYPE (var
), 6, init_code
,
5434 unshare_expr (ref_to_res
),
5435 var
, level
, op
, off
);
5437 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5438 TREE_TYPE (var
), 6, fini_code
,
5439 unshare_expr (ref_to_res
),
5440 var
, level
, op
, off
);
5442 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5443 TREE_TYPE (var
), 6, teardown_code
,
5444 ref_to_res
, var
, level
, op
, off
);
5446 gimplify_assign (var
, setup_call
, &before_fork
);
5447 gimplify_assign (var
, init_call
, &after_fork
);
5448 gimplify_assign (var
, fini_call
, &before_join
);
5449 gimplify_assign (outgoing
, teardown_call
, &after_join
);
5452 /* Now stitch things together. */
5453 gimple_seq_add_seq (fork_seq
, before_fork
);
5455 gimple_seq_add_stmt (fork_seq
, fork
);
5456 gimple_seq_add_seq (fork_seq
, after_fork
);
5458 gimple_seq_add_seq (join_seq
, before_join
);
5460 gimple_seq_add_stmt (join_seq
, join
);
5461 gimple_seq_add_seq (join_seq
, after_join
);
5464 /* Generate code to implement the REDUCTION clauses. */
5467 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
5469 gimple_seq sub_seq
= NULL
;
5471 tree x
, c
, tid
= NULL_TREE
;
5474 /* OpenACC loop reductions are handled elsewhere. */
5475 if (is_gimple_omp_oacc (ctx
->stmt
))
5478 /* SIMD reductions are handled in lower_rec_input_clauses. */
5479 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
5480 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
5483 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
5484 update in that case, otherwise use a lock. */
5485 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
5486 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
5488 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
)
5489 || TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
)
5491 /* Never use OMP_ATOMIC for array reductions or UDRs. */
5501 /* Initialize thread info for OpenACC. */
5502 if (is_gimple_omp_oacc (ctx
->stmt
))
5504 /* Get the current thread id. */
5505 tree call
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
5506 tid
= create_tmp_var (TREE_TYPE (TREE_TYPE (call
)));
5507 gimple
*stmt
= gimple_build_call (call
, 0);
5508 gimple_call_set_lhs (stmt
, tid
);
5509 gimple_seq_add_stmt (stmt_seqp
, stmt
);
5512 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5514 tree var
, ref
, new_var
, orig_var
;
5515 enum tree_code code
;
5516 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5518 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
5521 orig_var
= var
= OMP_CLAUSE_DECL (c
);
5522 if (TREE_CODE (var
) == MEM_REF
)
5524 var
= TREE_OPERAND (var
, 0);
5525 if (TREE_CODE (var
) == POINTER_PLUS_EXPR
)
5526 var
= TREE_OPERAND (var
, 0);
5527 if (TREE_CODE (var
) == INDIRECT_REF
5528 || TREE_CODE (var
) == ADDR_EXPR
)
5529 var
= TREE_OPERAND (var
, 0);
5531 if (is_variable_sized (var
))
5533 gcc_assert (DECL_HAS_VALUE_EXPR_P (var
));
5534 var
= DECL_VALUE_EXPR (var
);
5535 gcc_assert (TREE_CODE (var
) == INDIRECT_REF
);
5536 var
= TREE_OPERAND (var
, 0);
5537 gcc_assert (DECL_P (var
));
5540 new_var
= lookup_decl (var
, ctx
);
5541 if (var
== OMP_CLAUSE_DECL (c
) && is_reference (var
))
5542 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5543 ref
= build_outer_var_ref (var
, ctx
);
5544 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
5546 /* reduction(-:var) sums up the partial results, so it acts
5547 identically to reduction(+:var). */
5548 if (code
== MINUS_EXPR
)
5553 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
5555 addr
= save_expr (addr
);
5556 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
5557 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
5558 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
5559 gimplify_and_add (x
, stmt_seqp
);
5562 else if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
)
5564 tree d
= OMP_CLAUSE_DECL (c
);
5565 tree type
= TREE_TYPE (d
);
5566 tree v
= TYPE_MAX_VALUE (TYPE_DOMAIN (type
));
5567 tree i
= create_tmp_var (TREE_TYPE (v
), NULL
);
5568 tree ptype
= build_pointer_type (TREE_TYPE (type
));
5569 tree bias
= TREE_OPERAND (d
, 1);
5570 d
= TREE_OPERAND (d
, 0);
5571 if (TREE_CODE (d
) == POINTER_PLUS_EXPR
)
5573 tree b
= TREE_OPERAND (d
, 1);
5574 b
= maybe_lookup_decl (b
, ctx
);
5577 b
= TREE_OPERAND (d
, 1);
5578 b
= maybe_lookup_decl_in_outer_ctx (b
, ctx
);
5580 if (integer_zerop (bias
))
5584 bias
= fold_convert_loc (clause_loc
, TREE_TYPE (b
), bias
);
5585 bias
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
5586 TREE_TYPE (b
), b
, bias
);
5588 d
= TREE_OPERAND (d
, 0);
5590 /* For ref build_outer_var_ref already performs this, so
5591 only new_var needs a dereference. */
5592 if (TREE_CODE (d
) == INDIRECT_REF
)
5594 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5595 gcc_assert (is_reference (var
) && var
== orig_var
);
5597 else if (TREE_CODE (d
) == ADDR_EXPR
)
5599 if (orig_var
== var
)
5601 new_var
= build_fold_addr_expr (new_var
);
5602 ref
= build_fold_addr_expr (ref
);
5607 gcc_assert (orig_var
== var
);
5608 if (is_reference (var
))
5609 ref
= build_fold_addr_expr (ref
);
5613 tree t
= maybe_lookup_decl (v
, ctx
);
5617 v
= maybe_lookup_decl_in_outer_ctx (v
, ctx
);
5618 gimplify_expr (&v
, stmt_seqp
, NULL
, is_gimple_val
, fb_rvalue
);
5620 if (!integer_zerop (bias
))
5622 bias
= fold_convert_loc (clause_loc
, sizetype
, bias
);
5623 new_var
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
5624 TREE_TYPE (new_var
), new_var
,
5625 unshare_expr (bias
));
5626 ref
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
5627 TREE_TYPE (ref
), ref
, bias
);
5629 new_var
= fold_convert_loc (clause_loc
, ptype
, new_var
);
5630 ref
= fold_convert_loc (clause_loc
, ptype
, ref
);
5631 tree m
= create_tmp_var (ptype
, NULL
);
5632 gimplify_assign (m
, new_var
, stmt_seqp
);
5634 m
= create_tmp_var (ptype
, NULL
);
5635 gimplify_assign (m
, ref
, stmt_seqp
);
5637 gimplify_assign (i
, build_int_cst (TREE_TYPE (v
), 0), stmt_seqp
);
5638 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
5639 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
5640 gimple_seq_add_stmt (&sub_seq
, gimple_build_label (body
));
5641 tree priv
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5642 tree out
= build_simple_mem_ref_loc (clause_loc
, ref
);
5643 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5645 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
5646 tree decl_placeholder
5647 = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
);
5648 SET_DECL_VALUE_EXPR (placeholder
, out
);
5649 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
5650 SET_DECL_VALUE_EXPR (decl_placeholder
, priv
);
5651 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 1;
5652 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
5653 gimple_seq_add_seq (&sub_seq
,
5654 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
5655 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5656 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
5657 OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
) = NULL
;
5661 x
= build2 (code
, TREE_TYPE (out
), out
, priv
);
5662 out
= unshare_expr (out
);
5663 gimplify_assign (out
, x
, &sub_seq
);
5665 gimple
*g
= gimple_build_assign (new_var
, POINTER_PLUS_EXPR
, new_var
,
5666 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
5667 gimple_seq_add_stmt (&sub_seq
, g
);
5668 g
= gimple_build_assign (ref
, POINTER_PLUS_EXPR
, ref
,
5669 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
5670 gimple_seq_add_stmt (&sub_seq
, g
);
5671 g
= gimple_build_assign (i
, PLUS_EXPR
, i
,
5672 build_int_cst (TREE_TYPE (i
), 1));
5673 gimple_seq_add_stmt (&sub_seq
, g
);
5674 g
= gimple_build_cond (LE_EXPR
, i
, v
, body
, end
);
5675 gimple_seq_add_stmt (&sub_seq
, g
);
5676 gimple_seq_add_stmt (&sub_seq
, gimple_build_label (end
));
5678 else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5680 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
5682 if (is_reference (var
)
5683 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
5685 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
5686 SET_DECL_VALUE_EXPR (placeholder
, ref
);
5687 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
5688 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
5689 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
5690 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5691 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
5695 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
5696 ref
= build_outer_var_ref (var
, ctx
);
5697 gimplify_assign (ref
, x
, &sub_seq
);
5701 if (is_gimple_omp_oacc (ctx
->stmt
))
5704 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
5706 gimple_seq_add_stmt (stmt_seqp
, stmt
);
5708 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
5710 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
5712 gimple_seq_add_stmt (stmt_seqp
, stmt
);
5716 /* Generate code to implement the COPYPRIVATE clauses. */
5719 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
5724 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5726 tree var
, new_var
, ref
, x
;
5728 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5730 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
5733 var
= OMP_CLAUSE_DECL (c
);
5734 by_ref
= use_pointer_for_field (var
, NULL
);
5736 ref
= build_sender_ref (var
, ctx
);
5737 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
5740 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
5741 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
5743 gimplify_assign (ref
, x
, slist
);
5745 ref
= build_receiver_ref (var
, false, ctx
);
5748 ref
= fold_convert_loc (clause_loc
,
5749 build_pointer_type (TREE_TYPE (new_var
)),
5751 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
5753 if (is_reference (var
))
5755 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
5756 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
5757 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5759 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
5760 gimplify_and_add (x
, rlist
);
5765 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
5766 and REDUCTION from the sender (aka parent) side. */
5769 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
5773 int ignored_looptemp
= 0;
5774 bool is_taskloop
= false;
5776 /* For taskloop, ignore first two _looptemp_ clauses, those are initialized
5777 by GOMP_taskloop. */
5778 if (is_task_ctx (ctx
) && gimple_omp_task_taskloop_p (ctx
->stmt
))
5780 ignored_looptemp
= 2;
5784 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5786 tree val
, ref
, x
, var
;
5787 bool by_ref
, do_in
= false, do_out
= false;
5788 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5790 switch (OMP_CLAUSE_CODE (c
))
5792 case OMP_CLAUSE_PRIVATE
:
5793 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
5796 case OMP_CLAUSE_FIRSTPRIVATE
:
5797 case OMP_CLAUSE_COPYIN
:
5798 case OMP_CLAUSE_LASTPRIVATE
:
5799 case OMP_CLAUSE_REDUCTION
:
5801 case OMP_CLAUSE_SHARED
:
5802 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
5805 case OMP_CLAUSE__LOOPTEMP_
:
5806 if (ignored_looptemp
)
5816 val
= OMP_CLAUSE_DECL (c
);
5817 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
5818 && TREE_CODE (val
) == MEM_REF
)
5820 val
= TREE_OPERAND (val
, 0);
5821 if (TREE_CODE (val
) == POINTER_PLUS_EXPR
)
5822 val
= TREE_OPERAND (val
, 0);
5823 if (TREE_CODE (val
) == INDIRECT_REF
5824 || TREE_CODE (val
) == ADDR_EXPR
)
5825 val
= TREE_OPERAND (val
, 0);
5826 if (is_variable_sized (val
))
5830 /* For OMP_CLAUSE_SHARED_FIRSTPRIVATE, look beyond the
5831 outer taskloop region. */
5832 omp_context
*ctx_for_o
= ctx
;
5834 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
5835 && OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
5836 ctx_for_o
= ctx
->outer
;
5838 var
= lookup_decl_in_outer_ctx (val
, ctx_for_o
);
5840 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
5841 && is_global_var (var
))
5844 t
= omp_member_access_dummy_var (var
);
5847 var
= DECL_VALUE_EXPR (var
);
5848 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx_for_o
);
5850 var
= unshare_and_remap (var
, t
, o
);
5852 var
= unshare_expr (var
);
5855 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
)
5857 /* Handle taskloop firstprivate/lastprivate, where the
5858 lastprivate on GIMPLE_OMP_TASK is represented as
5859 OMP_CLAUSE_SHARED_FIRSTPRIVATE. */
5860 tree f
= lookup_sfield ((splay_tree_key
) &DECL_UID (val
), ctx
);
5861 x
= omp_build_component_ref (ctx
->sender_decl
, f
);
5862 if (use_pointer_for_field (val
, ctx
))
5863 var
= build_fold_addr_expr (var
);
5864 gimplify_assign (x
, var
, ilist
);
5865 DECL_ABSTRACT_ORIGIN (f
) = NULL
;
5869 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
5870 || val
== OMP_CLAUSE_DECL (c
))
5871 && is_variable_sized (val
))
5873 by_ref
= use_pointer_for_field (val
, NULL
);
5875 switch (OMP_CLAUSE_CODE (c
))
5877 case OMP_CLAUSE_PRIVATE
:
5878 case OMP_CLAUSE_FIRSTPRIVATE
:
5879 case OMP_CLAUSE_COPYIN
:
5880 case OMP_CLAUSE__LOOPTEMP_
:
5884 case OMP_CLAUSE_LASTPRIVATE
:
5885 if (by_ref
|| is_reference (val
))
5887 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
5894 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
5899 case OMP_CLAUSE_REDUCTION
:
5901 if (val
== OMP_CLAUSE_DECL (c
))
5902 do_out
= !(by_ref
|| is_reference (val
));
5904 by_ref
= TREE_CODE (TREE_TYPE (val
)) == ARRAY_TYPE
;
5913 ref
= build_sender_ref (val
, ctx
);
5914 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
5915 gimplify_assign (ref
, x
, ilist
);
5916 if (is_task_ctx (ctx
))
5917 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
5922 ref
= build_sender_ref (val
, ctx
);
5923 gimplify_assign (var
, ref
, olist
);
5928 /* Generate code to implement SHARED from the sender (aka parent)
5929 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
5930 list things that got automatically shared. */
5933 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
5935 tree var
, ovar
, nvar
, t
, f
, x
, record_type
;
5937 if (ctx
->record_type
== NULL
)
5940 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
5941 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
5943 ovar
= DECL_ABSTRACT_ORIGIN (f
);
5944 if (!ovar
|| TREE_CODE (ovar
) == FIELD_DECL
)
5947 nvar
= maybe_lookup_decl (ovar
, ctx
);
5948 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
5951 /* If CTX is a nested parallel directive. Find the immediately
5952 enclosing parallel or workshare construct that contains a
5953 mapping for OVAR. */
5954 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
5956 t
= omp_member_access_dummy_var (var
);
5959 var
= DECL_VALUE_EXPR (var
);
5960 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx
);
5962 var
= unshare_and_remap (var
, t
, o
);
5964 var
= unshare_expr (var
);
5967 if (use_pointer_for_field (ovar
, ctx
))
5969 x
= build_sender_ref (ovar
, ctx
);
5970 var
= build_fold_addr_expr (var
);
5971 gimplify_assign (x
, var
, ilist
);
5975 x
= build_sender_ref (ovar
, ctx
);
5976 gimplify_assign (x
, var
, ilist
);
5978 if (!TREE_READONLY (var
)
5979 /* We don't need to receive a new reference to a result
5980 or parm decl. In fact we may not store to it as we will
5981 invalidate any pending RSO and generate wrong gimple
5983 && !((TREE_CODE (var
) == RESULT_DECL
5984 || TREE_CODE (var
) == PARM_DECL
)
5985 && DECL_BY_REFERENCE (var
)))
5987 x
= build_sender_ref (ovar
, ctx
);
5988 gimplify_assign (var
, x
, olist
);
5994 /* Emit an OpenACC head marker call, encapulating the partitioning and
5995 other information that must be processed by the target compiler.
5996 Return the maximum number of dimensions the associated loop might
5997 be partitioned over. */
6000 lower_oacc_head_mark (location_t loc
, tree ddvar
, tree clauses
,
6001 gimple_seq
*seq
, omp_context
*ctx
)
6003 unsigned levels
= 0;
6005 tree gang_static
= NULL_TREE
;
6006 auto_vec
<tree
, 5> args
;
6008 args
.quick_push (build_int_cst
6009 (integer_type_node
, IFN_UNIQUE_OACC_HEAD_MARK
));
6010 args
.quick_push (ddvar
);
6011 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
6013 switch (OMP_CLAUSE_CODE (c
))
6015 case OMP_CLAUSE_GANG
:
6016 tag
|= OLF_DIM_GANG
;
6017 gang_static
= OMP_CLAUSE_GANG_STATIC_EXPR (c
);
6018 /* static:* is represented by -1, and we can ignore it, as
6019 scheduling is always static. */
6020 if (gang_static
&& integer_minus_onep (gang_static
))
6021 gang_static
= NULL_TREE
;
6025 case OMP_CLAUSE_WORKER
:
6026 tag
|= OLF_DIM_WORKER
;
6030 case OMP_CLAUSE_VECTOR
:
6031 tag
|= OLF_DIM_VECTOR
;
6035 case OMP_CLAUSE_SEQ
:
6039 case OMP_CLAUSE_AUTO
:
6043 case OMP_CLAUSE_INDEPENDENT
:
6044 tag
|= OLF_INDEPENDENT
;
6054 if (DECL_P (gang_static
))
6055 gang_static
= build_outer_var_ref (gang_static
, ctx
);
6056 tag
|= OLF_GANG_STATIC
;
6059 /* In a parallel region, loops are implicitly INDEPENDENT. */
6060 omp_context
*tgt
= enclosing_target_ctx (ctx
);
6061 if (!tgt
|| is_oacc_parallel (tgt
))
6062 tag
|= OLF_INDEPENDENT
;
6064 /* A loop lacking SEQ, GANG, WORKER and/or VECTOR is implicitly AUTO. */
6065 if (!(tag
& (((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1) << OLF_DIM_BASE
)
6069 /* Ensure at least one level. */
6073 args
.quick_push (build_int_cst (integer_type_node
, levels
));
6074 args
.quick_push (build_int_cst (integer_type_node
, tag
));
6076 args
.quick_push (gang_static
);
6078 gcall
*call
= gimple_build_call_internal_vec (IFN_UNIQUE
, args
);
6079 gimple_set_location (call
, loc
);
6080 gimple_set_lhs (call
, ddvar
);
6081 gimple_seq_add_stmt (seq
, call
);
6086 /* Emit an OpenACC lopp head or tail marker to SEQ. LEVEL is the
6087 partitioning level of the enclosed region. */
6090 lower_oacc_loop_marker (location_t loc
, tree ddvar
, bool head
,
6091 tree tofollow
, gimple_seq
*seq
)
6093 int marker_kind
= (head
? IFN_UNIQUE_OACC_HEAD_MARK
6094 : IFN_UNIQUE_OACC_TAIL_MARK
);
6095 tree marker
= build_int_cst (integer_type_node
, marker_kind
);
6096 int nargs
= 2 + (tofollow
!= NULL_TREE
);
6097 gcall
*call
= gimple_build_call_internal (IFN_UNIQUE
, nargs
,
6098 marker
, ddvar
, tofollow
);
6099 gimple_set_location (call
, loc
);
6100 gimple_set_lhs (call
, ddvar
);
6101 gimple_seq_add_stmt (seq
, call
);
6104 /* Generate the before and after OpenACC loop sequences. CLAUSES are
6105 the loop clauses, from which we extract reductions. Initialize
6109 lower_oacc_head_tail (location_t loc
, tree clauses
,
6110 gimple_seq
*head
, gimple_seq
*tail
, omp_context
*ctx
)
6113 tree ddvar
= create_tmp_var (integer_type_node
, ".data_dep");
6114 gimple_seq_add_stmt (head
, gimple_build_assign (ddvar
, integer_zero_node
));
6116 unsigned count
= lower_oacc_head_mark (loc
, ddvar
, clauses
, head
, ctx
);
6118 lower_oacc_loop_marker (loc
, ddvar
, false, integer_zero_node
, tail
);
6120 tree fork_kind
= build_int_cst (unsigned_type_node
, IFN_UNIQUE_OACC_FORK
);
6121 tree join_kind
= build_int_cst (unsigned_type_node
, IFN_UNIQUE_OACC_JOIN
);
6123 for (unsigned done
= 1; count
; count
--, done
++)
6125 gimple_seq fork_seq
= NULL
;
6126 gimple_seq join_seq
= NULL
;
6128 tree place
= build_int_cst (integer_type_node
, -1);
6129 gcall
*fork
= gimple_build_call_internal (IFN_UNIQUE
, 3,
6130 fork_kind
, ddvar
, place
);
6131 gimple_set_location (fork
, loc
);
6132 gimple_set_lhs (fork
, ddvar
);
6134 gcall
*join
= gimple_build_call_internal (IFN_UNIQUE
, 3,
6135 join_kind
, ddvar
, place
);
6136 gimple_set_location (join
, loc
);
6137 gimple_set_lhs (join
, ddvar
);
6139 /* Mark the beginning of this level sequence. */
6141 lower_oacc_loop_marker (loc
, ddvar
, true,
6142 build_int_cst (integer_type_node
, count
),
6144 lower_oacc_loop_marker (loc
, ddvar
, false,
6145 build_int_cst (integer_type_node
, done
),
6148 lower_oacc_reductions (loc
, clauses
, place
, inner
,
6149 fork
, join
, &fork_seq
, &join_seq
, ctx
);
6151 /* Append this level to head. */
6152 gimple_seq_add_seq (head
, fork_seq
);
6153 /* Prepend it to tail. */
6154 gimple_seq_add_seq (&join_seq
, *tail
);
6160 /* Mark the end of the sequence. */
6161 lower_oacc_loop_marker (loc
, ddvar
, true, NULL_TREE
, head
);
6162 lower_oacc_loop_marker (loc
, ddvar
, false, NULL_TREE
, tail
);
6165 /* A convenience function to build an empty GIMPLE_COND with just the
6169 gimple_build_cond_empty (tree cond
)
6171 enum tree_code pred_code
;
6174 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
6175 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
6178 static void expand_omp_build_assign (gimple_stmt_iterator
*, tree
, tree
,
6181 /* Build the function calls to GOMP_parallel_start etc to actually
6182 generate the parallel operation. REGION is the parallel region
6183 being expanded. BB is the block where to insert the code. WS_ARGS
6184 will be set if this is a call to a combined parallel+workshare
6185 construct, it contains the list of additional arguments needed by
6186 the workshare construct. */
6189 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
6190 gomp_parallel
*entry_stmt
,
6191 vec
<tree
, va_gc
> *ws_args
)
6193 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
6194 gimple_stmt_iterator gsi
;
6196 enum built_in_function start_ix
;
6198 location_t clause_loc
;
6199 vec
<tree
, va_gc
> *args
;
6201 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
6203 /* Determine what flavor of GOMP_parallel we will be
6205 start_ix
= BUILT_IN_GOMP_PARALLEL
;
6206 if (is_combined_parallel (region
))
6208 switch (region
->inner
->type
)
6210 case GIMPLE_OMP_FOR
:
6211 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
6212 switch (region
->inner
->sched_kind
)
6214 case OMP_CLAUSE_SCHEDULE_RUNTIME
:
6217 case OMP_CLAUSE_SCHEDULE_DYNAMIC
:
6218 case OMP_CLAUSE_SCHEDULE_GUIDED
:
6219 if (region
->inner
->sched_modifiers
6220 & OMP_CLAUSE_SCHEDULE_NONMONOTONIC
)
6222 start_ix2
= 3 + region
->inner
->sched_kind
;
6227 start_ix2
= region
->inner
->sched_kind
;
6230 start_ix2
+= (int) BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
;
6231 start_ix
= (enum built_in_function
) start_ix2
;
6233 case GIMPLE_OMP_SECTIONS
:
6234 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
6241 /* By default, the value of NUM_THREADS is zero (selected at run time)
6242 and there is no conditional. */
6244 val
= build_int_cst (unsigned_type_node
, 0);
6245 flags
= build_int_cst (unsigned_type_node
, 0);
6247 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
6249 cond
= OMP_CLAUSE_IF_EXPR (c
);
6251 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
6254 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
6255 clause_loc
= OMP_CLAUSE_LOCATION (c
);
6258 clause_loc
= gimple_location (entry_stmt
);
6260 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
6262 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
6264 /* Ensure 'val' is of the correct type. */
6265 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
6267 /* If we found the clause 'if (cond)', build either
6268 (cond != 0) or (cond ? val : 1u). */
6271 cond
= gimple_boolify (cond
);
6273 if (integer_zerop (val
))
6274 val
= fold_build2_loc (clause_loc
,
6275 EQ_EXPR
, unsigned_type_node
, cond
,
6276 build_int_cst (TREE_TYPE (cond
), 0));
6279 basic_block cond_bb
, then_bb
, else_bb
;
6280 edge e
, e_then
, e_else
;
6281 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
6283 tmp_var
= create_tmp_var (TREE_TYPE (val
));
6284 if (gimple_in_ssa_p (cfun
))
6286 tmp_then
= make_ssa_name (tmp_var
);
6287 tmp_else
= make_ssa_name (tmp_var
);
6288 tmp_join
= make_ssa_name (tmp_var
);
6297 e
= split_block_after_labels (bb
);
6302 then_bb
= create_empty_bb (cond_bb
);
6303 else_bb
= create_empty_bb (then_bb
);
6304 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
6305 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
6307 stmt
= gimple_build_cond_empty (cond
);
6308 gsi
= gsi_start_bb (cond_bb
);
6309 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6311 gsi
= gsi_start_bb (then_bb
);
6312 expand_omp_build_assign (&gsi
, tmp_then
, val
, true);
6314 gsi
= gsi_start_bb (else_bb
);
6315 expand_omp_build_assign (&gsi
, tmp_else
,
6316 build_int_cst (unsigned_type_node
, 1),
6319 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
6320 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
6321 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
6322 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
6323 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
6324 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
6326 if (gimple_in_ssa_p (cfun
))
6328 gphi
*phi
= create_phi_node (tmp_join
, bb
);
6329 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
6330 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
6336 gsi
= gsi_start_bb (bb
);
6337 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
6338 false, GSI_CONTINUE_LINKING
);
6341 gsi
= gsi_last_bb (bb
);
6342 t
= gimple_omp_parallel_data_arg (entry_stmt
);
6344 t1
= null_pointer_node
;
6346 t1
= build_fold_addr_expr (t
);
6347 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
6349 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
6350 args
->quick_push (t2
);
6351 args
->quick_push (t1
);
6352 args
->quick_push (val
);
6354 args
->splice (*ws_args
);
6355 args
->quick_push (flags
);
6357 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
6358 builtin_decl_explicit (start_ix
), args
);
6360 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6361 false, GSI_CONTINUE_LINKING
);
6364 /* Insert a function call whose name is FUNC_NAME with the information from
6365 ENTRY_STMT into the basic_block BB. */
6368 expand_cilk_for_call (basic_block bb
, gomp_parallel
*entry_stmt
,
6369 vec
<tree
, va_gc
> *ws_args
)
6372 gimple_stmt_iterator gsi
;
6373 vec
<tree
, va_gc
> *args
;
6375 gcc_assert (vec_safe_length (ws_args
) == 2);
6376 tree func_name
= (*ws_args
)[0];
6377 tree grain
= (*ws_args
)[1];
6379 tree clauses
= gimple_omp_parallel_clauses (entry_stmt
);
6380 tree count
= find_omp_clause (clauses
, OMP_CLAUSE__CILK_FOR_COUNT_
);
6381 gcc_assert (count
!= NULL_TREE
);
6382 count
= OMP_CLAUSE_OPERAND (count
, 0);
6384 gsi
= gsi_last_bb (bb
);
6385 t
= gimple_omp_parallel_data_arg (entry_stmt
);
6387 t1
= null_pointer_node
;
6389 t1
= build_fold_addr_expr (t
);
6390 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
6392 vec_alloc (args
, 4);
6393 args
->quick_push (t2
);
6394 args
->quick_push (t1
);
6395 args
->quick_push (count
);
6396 args
->quick_push (grain
);
6397 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
, func_name
, args
);
6399 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, false,
6400 GSI_CONTINUE_LINKING
);
6403 /* Build the function call to GOMP_task to actually
6404 generate the task operation. BB is the block where to insert the code. */
6407 expand_task_call (struct omp_region
*region
, basic_block bb
,
6408 gomp_task
*entry_stmt
)
6411 gimple_stmt_iterator gsi
;
6412 location_t loc
= gimple_location (entry_stmt
);
6414 tree clauses
= gimple_omp_task_clauses (entry_stmt
);
6416 tree ifc
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
6417 tree untied
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
6418 tree mergeable
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
6419 tree depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
6420 tree finalc
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
6421 tree priority
= find_omp_clause (clauses
, OMP_CLAUSE_PRIORITY
);
6424 = (untied
? GOMP_TASK_FLAG_UNTIED
: 0)
6425 | (mergeable
? GOMP_TASK_FLAG_MERGEABLE
: 0)
6426 | (depend
? GOMP_TASK_FLAG_DEPEND
: 0);
6428 bool taskloop_p
= gimple_omp_task_taskloop_p (entry_stmt
);
6429 tree startvar
= NULL_TREE
, endvar
= NULL_TREE
, step
= NULL_TREE
;
6430 tree num_tasks
= NULL_TREE
;
6434 gimple
*g
= last_stmt (region
->outer
->entry
);
6435 gcc_assert (gimple_code (g
) == GIMPLE_OMP_FOR
6436 && gimple_omp_for_kind (g
) == GF_OMP_FOR_KIND_TASKLOOP
);
6437 struct omp_for_data fd
;
6438 extract_omp_for_data (as_a
<gomp_for
*> (g
), &fd
, NULL
);
6439 startvar
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6440 endvar
= find_omp_clause (OMP_CLAUSE_CHAIN (startvar
),
6441 OMP_CLAUSE__LOOPTEMP_
);
6442 startvar
= OMP_CLAUSE_DECL (startvar
);
6443 endvar
= OMP_CLAUSE_DECL (endvar
);
6444 step
= fold_convert_loc (loc
, fd
.iter_type
, fd
.loop
.step
);
6445 if (fd
.loop
.cond_code
== LT_EXPR
)
6446 iflags
|= GOMP_TASK_FLAG_UP
;
6447 tree tclauses
= gimple_omp_for_clauses (g
);
6448 num_tasks
= find_omp_clause (tclauses
, OMP_CLAUSE_NUM_TASKS
);
6450 num_tasks
= OMP_CLAUSE_NUM_TASKS_EXPR (num_tasks
);
6453 num_tasks
= find_omp_clause (tclauses
, OMP_CLAUSE_GRAINSIZE
);
6456 iflags
|= GOMP_TASK_FLAG_GRAINSIZE
;
6457 num_tasks
= OMP_CLAUSE_GRAINSIZE_EXPR (num_tasks
);
6460 num_tasks
= integer_zero_node
;
6462 num_tasks
= fold_convert_loc (loc
, long_integer_type_node
, num_tasks
);
6463 if (ifc
== NULL_TREE
)
6464 iflags
|= GOMP_TASK_FLAG_IF
;
6465 if (find_omp_clause (tclauses
, OMP_CLAUSE_NOGROUP
))
6466 iflags
|= GOMP_TASK_FLAG_NOGROUP
;
6467 ull
= fd
.iter_type
== long_long_unsigned_type_node
;
6470 iflags
|= GOMP_TASK_FLAG_PRIORITY
;
6472 tree flags
= build_int_cst (unsigned_type_node
, iflags
);
6474 tree cond
= boolean_true_node
;
6479 tree t
= gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc
));
6480 t
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, t
,
6481 build_int_cst (unsigned_type_node
,
6483 build_int_cst (unsigned_type_node
, 0));
6484 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
,
6488 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc
));
6493 tree t
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (finalc
));
6494 t
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, t
,
6495 build_int_cst (unsigned_type_node
,
6496 GOMP_TASK_FLAG_FINAL
),
6497 build_int_cst (unsigned_type_node
, 0));
6498 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, t
);
6501 depend
= OMP_CLAUSE_DECL (depend
);
6503 depend
= build_int_cst (ptr_type_node
, 0);
6505 priority
= fold_convert (integer_type_node
,
6506 OMP_CLAUSE_PRIORITY_EXPR (priority
));
6508 priority
= integer_zero_node
;
6510 gsi
= gsi_last_bb (bb
);
6511 tree t
= gimple_omp_task_data_arg (entry_stmt
);
6513 t2
= null_pointer_node
;
6515 t2
= build_fold_addr_expr_loc (loc
, t
);
6516 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
6517 t
= gimple_omp_task_copy_fn (entry_stmt
);
6519 t3
= null_pointer_node
;
6521 t3
= build_fold_addr_expr_loc (loc
, t
);
6524 t
= build_call_expr (ull
6525 ? builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP_ULL
)
6526 : builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP
),
6528 gimple_omp_task_arg_size (entry_stmt
),
6529 gimple_omp_task_arg_align (entry_stmt
), flags
,
6530 num_tasks
, priority
, startvar
, endvar
, step
);
6532 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
6534 gimple_omp_task_arg_size (entry_stmt
),
6535 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
6538 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6539 false, GSI_CONTINUE_LINKING
);
6543 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
6544 catch handler and return it. This prevents programs from violating the
6545 structured block semantics with throws. */
6548 maybe_catch_exception (gimple_seq body
)
6553 if (!flag_exceptions
)
6556 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
6557 decl
= lang_hooks
.eh_protect_cleanup_actions ();
6559 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
6561 g
= gimple_build_eh_must_not_throw (decl
);
6562 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
6565 return gimple_seq_alloc_with_stmt (g
);
6568 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
6571 vec2chain (vec
<tree
, va_gc
> *v
)
6573 tree chain
= NULL_TREE
, t
;
6576 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
6578 DECL_CHAIN (t
) = chain
;
6586 /* Remove barriers in REGION->EXIT's block. Note that this is only
6587 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
6588 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
6589 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
6593 remove_exit_barrier (struct omp_region
*region
)
6595 gimple_stmt_iterator gsi
;
6596 basic_block exit_bb
;
6600 int any_addressable_vars
= -1;
6602 exit_bb
= region
->exit
;
6604 /* If the parallel region doesn't return, we don't have REGION->EXIT
6609 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
6610 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
6611 statements that can appear in between are extremely limited -- no
6612 memory operations at all. Here, we allow nothing at all, so the
6613 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
6614 gsi
= gsi_last_bb (exit_bb
);
6615 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
6617 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
6620 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
6622 gsi
= gsi_last_bb (e
->src
);
6623 if (gsi_end_p (gsi
))
6625 stmt
= gsi_stmt (gsi
);
6626 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
6627 && !gimple_omp_return_nowait_p (stmt
))
6629 /* OpenMP 3.0 tasks unfortunately prevent this optimization
6630 in many cases. If there could be tasks queued, the barrier
6631 might be needed to let the tasks run before some local
6632 variable of the parallel that the task uses as shared
6633 runs out of scope. The task can be spawned either
6634 from within current function (this would be easy to check)
6635 or from some function it calls and gets passed an address
6636 of such a variable. */
6637 if (any_addressable_vars
< 0)
6639 gomp_parallel
*parallel_stmt
6640 = as_a
<gomp_parallel
*> (last_stmt (region
->entry
));
6641 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
6642 tree local_decls
, block
, decl
;
6645 any_addressable_vars
= 0;
6646 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
6647 if (TREE_ADDRESSABLE (decl
))
6649 any_addressable_vars
= 1;
6652 for (block
= gimple_block (stmt
);
6653 !any_addressable_vars
6655 && TREE_CODE (block
) == BLOCK
;
6656 block
= BLOCK_SUPERCONTEXT (block
))
6658 for (local_decls
= BLOCK_VARS (block
);
6660 local_decls
= DECL_CHAIN (local_decls
))
6661 if (TREE_ADDRESSABLE (local_decls
))
6663 any_addressable_vars
= 1;
6666 if (block
== gimple_block (parallel_stmt
))
6670 if (!any_addressable_vars
)
6671 gimple_omp_return_set_nowait (stmt
);
6677 remove_exit_barriers (struct omp_region
*region
)
6679 if (region
->type
== GIMPLE_OMP_PARALLEL
)
6680 remove_exit_barrier (region
);
6684 region
= region
->inner
;
6685 remove_exit_barriers (region
);
6686 while (region
->next
)
6688 region
= region
->next
;
6689 remove_exit_barriers (region
);
6694 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
6695 calls. These can't be declared as const functions, but
6696 within one parallel body they are constant, so they can be
6697 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
6698 which are declared const. Similarly for task body, except
6699 that in untied task omp_get_thread_num () can change at any task
6700 scheduling point. */
6703 optimize_omp_library_calls (gimple
*entry_stmt
)
6706 gimple_stmt_iterator gsi
;
6707 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
6708 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
6709 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
6710 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
6711 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
6712 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
6713 OMP_CLAUSE_UNTIED
) != NULL
);
6715 FOR_EACH_BB_FN (bb
, cfun
)
6716 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
6718 gimple
*call
= gsi_stmt (gsi
);
6721 if (is_gimple_call (call
)
6722 && (decl
= gimple_call_fndecl (call
))
6723 && DECL_EXTERNAL (decl
)
6724 && TREE_PUBLIC (decl
)
6725 && DECL_INITIAL (decl
) == NULL
)
6729 if (DECL_NAME (decl
) == thr_num_id
)
6731 /* In #pragma omp task untied omp_get_thread_num () can change
6732 during the execution of the task region. */
6735 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
6737 else if (DECL_NAME (decl
) == num_thr_id
)
6738 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
6742 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
6743 || gimple_call_num_args (call
) != 0)
6746 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
6749 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
6750 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
6751 TREE_TYPE (TREE_TYPE (built_in
))))
6754 gimple_call_set_fndecl (call
, built_in
);
6759 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
6763 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
6767 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
6768 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
6771 if (TREE_CODE (t
) == ADDR_EXPR
)
6772 recompute_tree_invariant_for_addr_expr (t
);
6774 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
6778 /* Prepend or append TO = FROM assignment before or after *GSI_P. */
6781 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
,
6784 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
6785 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
6786 !after
, after
? GSI_CONTINUE_LINKING
6788 gimple
*stmt
= gimple_build_assign (to
, from
);
6790 gsi_insert_after (gsi_p
, stmt
, GSI_CONTINUE_LINKING
);
6792 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
6793 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
6794 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
6796 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
6797 gimple_regimplify_operands (stmt
, &gsi
);
6801 /* Expand the OpenMP parallel or task directive starting at REGION. */
6804 expand_omp_taskreg (struct omp_region
*region
)
6806 basic_block entry_bb
, exit_bb
, new_bb
;
6807 struct function
*child_cfun
;
6808 tree child_fn
, block
, t
;
6809 gimple_stmt_iterator gsi
;
6810 gimple
*entry_stmt
, *stmt
;
6812 vec
<tree
, va_gc
> *ws_args
;
6814 entry_stmt
= last_stmt (region
->entry
);
6815 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
6816 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
6818 entry_bb
= region
->entry
;
6819 if (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
)
6820 exit_bb
= region
->cont
;
6822 exit_bb
= region
->exit
;
6826 && gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
6827 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt
),
6828 OMP_CLAUSE__CILK_FOR_COUNT_
) != NULL_TREE
);
6831 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for,
6832 and the inner statement contains the name of the built-in function
6834 ws_args
= region
->inner
->ws_args
;
6835 else if (is_combined_parallel (region
))
6836 ws_args
= region
->ws_args
;
6840 if (child_cfun
->cfg
)
6842 /* Due to inlining, it may happen that we have already outlined
6843 the region, in which case all we need to do is make the
6844 sub-graph unreachable and emit the parallel call. */
6845 edge entry_succ_e
, exit_succ_e
;
6847 entry_succ_e
= single_succ_edge (entry_bb
);
6849 gsi
= gsi_last_bb (entry_bb
);
6850 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
6851 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
6852 gsi_remove (&gsi
, true);
6857 exit_succ_e
= single_succ_edge (exit_bb
);
6858 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
6860 remove_edge_and_dominated_blocks (entry_succ_e
);
6864 unsigned srcidx
, dstidx
, num
;
6866 /* If the parallel region needs data sent from the parent
6867 function, then the very first statement (except possible
6868 tree profile counter updates) of the parallel body
6869 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
6870 &.OMP_DATA_O is passed as an argument to the child function,
6871 we need to replace it with the argument as seen by the child
6874 In most cases, this will end up being the identity assignment
6875 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
6876 a function call that has been inlined, the original PARM_DECL
6877 .OMP_DATA_I may have been converted into a different local
6878 variable. In which case, we need to keep the assignment. */
6879 if (gimple_omp_taskreg_data_arg (entry_stmt
))
6881 basic_block entry_succ_bb
6882 = single_succ_p (entry_bb
) ? single_succ (entry_bb
)
6883 : FALLTHRU_EDGE (entry_bb
)->dest
;
6885 gimple
*parcopy_stmt
= NULL
;
6887 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
6891 gcc_assert (!gsi_end_p (gsi
));
6892 stmt
= gsi_stmt (gsi
);
6893 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
6896 if (gimple_num_ops (stmt
) == 2)
6898 tree arg
= gimple_assign_rhs1 (stmt
);
6900 /* We're ignore the subcode because we're
6901 effectively doing a STRIP_NOPS. */
6903 if (TREE_CODE (arg
) == ADDR_EXPR
6904 && TREE_OPERAND (arg
, 0)
6905 == gimple_omp_taskreg_data_arg (entry_stmt
))
6907 parcopy_stmt
= stmt
;
6913 gcc_assert (parcopy_stmt
!= NULL
);
6914 arg
= DECL_ARGUMENTS (child_fn
);
6916 if (!gimple_in_ssa_p (cfun
))
6918 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
6919 gsi_remove (&gsi
, true);
6922 /* ?? Is setting the subcode really necessary ?? */
6923 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
6924 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
6929 tree lhs
= gimple_assign_lhs (parcopy_stmt
);
6930 gcc_assert (SSA_NAME_VAR (lhs
) == arg
);
6931 /* We'd like to set the rhs to the default def in the child_fn,
6932 but it's too early to create ssa names in the child_fn.
6933 Instead, we set the rhs to the parm. In
6934 move_sese_region_to_fn, we introduce a default def for the
6935 parm, map the parm to it's default def, and once we encounter
6936 this stmt, replace the parm with the default def. */
6937 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
6938 update_stmt (parcopy_stmt
);
6942 /* Declare local variables needed in CHILD_CFUN. */
6943 block
= DECL_INITIAL (child_fn
);
6944 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
6945 /* The gimplifier could record temporaries in parallel/task block
6946 rather than in containing function's local_decls chain,
6947 which would mean cgraph missed finalizing them. Do it now. */
6948 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
6949 if (TREE_CODE (t
) == VAR_DECL
6951 && !DECL_EXTERNAL (t
))
6952 varpool_node::finalize_decl (t
);
6953 DECL_SAVED_TREE (child_fn
) = NULL
;
6954 /* We'll create a CFG for child_fn, so no gimple body is needed. */
6955 gimple_set_body (child_fn
, NULL
);
6956 TREE_USED (block
) = 1;
6958 /* Reset DECL_CONTEXT on function arguments. */
6959 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
6960 DECL_CONTEXT (t
) = child_fn
;
6962 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
6963 so that it can be moved to the child function. */
6964 gsi
= gsi_last_bb (entry_bb
);
6965 stmt
= gsi_stmt (gsi
);
6966 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
6967 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
6968 e
= split_block (entry_bb
, stmt
);
6969 gsi_remove (&gsi
, true);
6972 if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
6973 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
6976 e2
= make_edge (e
->src
, BRANCH_EDGE (entry_bb
)->dest
, EDGE_ABNORMAL
);
6977 gcc_assert (e2
->dest
== region
->exit
);
6978 remove_edge (BRANCH_EDGE (entry_bb
));
6979 set_immediate_dominator (CDI_DOMINATORS
, e2
->dest
, e
->src
);
6980 gsi
= gsi_last_bb (region
->exit
);
6981 gcc_assert (!gsi_end_p (gsi
)
6982 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
6983 gsi_remove (&gsi
, true);
6986 /* Convert GIMPLE_OMP_{RETURN,CONTINUE} into a RETURN_EXPR. */
6989 gsi
= gsi_last_bb (exit_bb
);
6990 gcc_assert (!gsi_end_p (gsi
)
6991 && (gimple_code (gsi_stmt (gsi
))
6992 == (e2
? GIMPLE_OMP_CONTINUE
: GIMPLE_OMP_RETURN
)));
6993 stmt
= gimple_build_return (NULL
);
6994 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
6995 gsi_remove (&gsi
, true);
6998 /* Move the parallel region into CHILD_CFUN. */
7000 if (gimple_in_ssa_p (cfun
))
7002 init_tree_ssa (child_cfun
);
7003 init_ssa_operands (child_cfun
);
7004 child_cfun
->gimple_df
->in_ssa_p
= true;
7008 block
= gimple_block (entry_stmt
);
7010 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
7012 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
7015 basic_block dest_bb
= e2
->dest
;
7017 make_edge (new_bb
, dest_bb
, EDGE_FALLTHRU
);
7019 set_immediate_dominator (CDI_DOMINATORS
, dest_bb
, new_bb
);
7021 /* When the OMP expansion process cannot guarantee an up-to-date
7022 loop tree arrange for the child function to fixup loops. */
7023 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
7024 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
7026 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
7027 num
= vec_safe_length (child_cfun
->local_decls
);
7028 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
7030 t
= (*child_cfun
->local_decls
)[srcidx
];
7031 if (DECL_CONTEXT (t
) == cfun
->decl
)
7033 if (srcidx
!= dstidx
)
7034 (*child_cfun
->local_decls
)[dstidx
] = t
;
7038 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
7040 /* Inform the callgraph about the new function. */
7041 child_cfun
->curr_properties
= cfun
->curr_properties
;
7042 child_cfun
->has_simduid_loops
|= cfun
->has_simduid_loops
;
7043 child_cfun
->has_force_vectorize_loops
|= cfun
->has_force_vectorize_loops
;
7044 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
7045 node
->parallelized_function
= 1;
7046 cgraph_node::add_new_function (child_fn
, true);
7048 bool need_asm
= DECL_ASSEMBLER_NAME_SET_P (current_function_decl
)
7049 && !DECL_ASSEMBLER_NAME_SET_P (child_fn
);
7051 /* Fix the callgraph edges for child_cfun. Those for cfun will be
7052 fixed in a following pass. */
7053 push_cfun (child_cfun
);
7055 assign_assembler_name_if_neeeded (child_fn
);
7058 optimize_omp_library_calls (entry_stmt
);
7059 cgraph_edge::rebuild_edges ();
7061 /* Some EH regions might become dead, see PR34608. If
7062 pass_cleanup_cfg isn't the first pass to happen with the
7063 new child, these dead EH edges might cause problems.
7064 Clean them up now. */
7065 if (flag_exceptions
)
7068 bool changed
= false;
7070 FOR_EACH_BB_FN (bb
, cfun
)
7071 changed
|= gimple_purge_dead_eh_edges (bb
);
7073 cleanup_tree_cfg ();
7075 if (gimple_in_ssa_p (cfun
))
7076 update_ssa (TODO_update_ssa
);
7077 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
7078 verify_loop_structure ();
7081 if (dump_file
&& !gimple_in_ssa_p (cfun
))
7083 omp_any_child_fn_dumped
= true;
7084 dump_function_header (dump_file
, child_fn
, dump_flags
);
7085 dump_function_to_file (child_fn
, dump_file
, dump_flags
);
7089 /* Emit a library call to launch the children threads. */
7091 expand_cilk_for_call (new_bb
,
7092 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
7093 else if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
7094 expand_parallel_call (region
, new_bb
,
7095 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
7097 expand_task_call (region
, new_bb
, as_a
<gomp_task
*> (entry_stmt
));
7098 if (gimple_in_ssa_p (cfun
))
7099 update_ssa (TODO_update_ssa_only_virtuals
);
7102 /* Information about members of an OpenACC collapsed loop nest. */
7104 struct oacc_collapse
7106 tree base
; /* Base value. */
7107 tree iters
; /* Number of steps. */
7108 tree step
; /* step size. */
7111 /* Helper for expand_oacc_for. Determine collapsed loop information.
7112 Fill in COUNTS array. Emit any initialization code before GSI.
7113 Return the calculated outer loop bound of BOUND_TYPE. */
7116 expand_oacc_collapse_init (const struct omp_for_data
*fd
,
7117 gimple_stmt_iterator
*gsi
,
7118 oacc_collapse
*counts
, tree bound_type
)
7120 tree total
= build_int_cst (bound_type
, 1);
7123 gcc_assert (integer_onep (fd
->loop
.step
));
7124 gcc_assert (integer_zerop (fd
->loop
.n1
));
7126 for (ix
= 0; ix
!= fd
->collapse
; ix
++)
7128 const omp_for_data_loop
*loop
= &fd
->loops
[ix
];
7130 tree iter_type
= TREE_TYPE (loop
->v
);
7131 tree diff_type
= iter_type
;
7132 tree plus_type
= iter_type
;
7134 gcc_assert (loop
->cond_code
== fd
->loop
.cond_code
);
7136 if (POINTER_TYPE_P (iter_type
))
7137 plus_type
= sizetype
;
7138 if (POINTER_TYPE_P (diff_type
) || TYPE_UNSIGNED (diff_type
))
7139 diff_type
= signed_type_for (diff_type
);
7143 tree s
= loop
->step
;
7144 bool up
= loop
->cond_code
== LT_EXPR
;
7145 tree dir
= build_int_cst (diff_type
, up
? +1 : -1);
7149 b
= force_gimple_operand_gsi (gsi
, b
, true, NULL_TREE
,
7150 true, GSI_SAME_STMT
);
7151 e
= force_gimple_operand_gsi (gsi
, e
, true, NULL_TREE
,
7152 true, GSI_SAME_STMT
);
7154 /* Convert the step, avoiding possible unsigned->signed overflow. */
7155 negating
= !up
&& TYPE_UNSIGNED (TREE_TYPE (s
));
7157 s
= fold_build1 (NEGATE_EXPR
, TREE_TYPE (s
), s
);
7158 s
= fold_convert (diff_type
, s
);
7160 s
= fold_build1 (NEGATE_EXPR
, diff_type
, s
);
7161 s
= force_gimple_operand_gsi (gsi
, s
, true, NULL_TREE
,
7162 true, GSI_SAME_STMT
);
7164 /* Determine the range, avoiding possible unsigned->signed overflow. */
7165 negating
= !up
&& TYPE_UNSIGNED (iter_type
);
7166 expr
= fold_build2 (MINUS_EXPR
, plus_type
,
7167 fold_convert (plus_type
, negating
? b
: e
),
7168 fold_convert (plus_type
, negating
? e
: b
));
7169 expr
= fold_convert (diff_type
, expr
);
7171 expr
= fold_build1 (NEGATE_EXPR
, diff_type
, expr
);
7172 tree range
= force_gimple_operand_gsi
7173 (gsi
, expr
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7175 /* Determine number of iterations. */
7176 expr
= fold_build2 (MINUS_EXPR
, diff_type
, range
, dir
);
7177 expr
= fold_build2 (PLUS_EXPR
, diff_type
, expr
, s
);
7178 expr
= fold_build2 (TRUNC_DIV_EXPR
, diff_type
, expr
, s
);
7180 tree iters
= force_gimple_operand_gsi (gsi
, expr
, true, NULL_TREE
,
7181 true, GSI_SAME_STMT
);
7183 counts
[ix
].base
= b
;
7184 counts
[ix
].iters
= iters
;
7185 counts
[ix
].step
= s
;
7187 total
= fold_build2 (MULT_EXPR
, bound_type
, total
,
7188 fold_convert (bound_type
, iters
));
7194 /* Emit initializers for collapsed loop members. IVAR is the outer
7195 loop iteration variable, from which collapsed loop iteration values
7196 are calculated. COUNTS array has been initialized by
7197 expand_oacc_collapse_inits. */
7200 expand_oacc_collapse_vars (const struct omp_for_data
*fd
,
7201 gimple_stmt_iterator
*gsi
,
7202 const oacc_collapse
*counts
, tree ivar
)
7204 tree ivar_type
= TREE_TYPE (ivar
);
7206 /* The most rapidly changing iteration variable is the innermost
7208 for (int ix
= fd
->collapse
; ix
--;)
7210 const omp_for_data_loop
*loop
= &fd
->loops
[ix
];
7211 const oacc_collapse
*collapse
= &counts
[ix
];
7212 tree iter_type
= TREE_TYPE (loop
->v
);
7213 tree diff_type
= TREE_TYPE (collapse
->step
);
7214 tree plus_type
= iter_type
;
7215 enum tree_code plus_code
= PLUS_EXPR
;
7218 if (POINTER_TYPE_P (iter_type
))
7220 plus_code
= POINTER_PLUS_EXPR
;
7221 plus_type
= sizetype
;
7224 expr
= fold_build2 (TRUNC_MOD_EXPR
, ivar_type
, ivar
,
7225 fold_convert (ivar_type
, collapse
->iters
));
7226 expr
= fold_build2 (MULT_EXPR
, diff_type
, fold_convert (diff_type
, expr
),
7228 expr
= fold_build2 (plus_code
, iter_type
, collapse
->base
,
7229 fold_convert (plus_type
, expr
));
7230 expr
= force_gimple_operand_gsi (gsi
, expr
, false, NULL_TREE
,
7231 true, GSI_SAME_STMT
);
7232 gassign
*ass
= gimple_build_assign (loop
->v
, expr
);
7233 gsi_insert_before (gsi
, ass
, GSI_SAME_STMT
);
7237 expr
= fold_build2 (TRUNC_DIV_EXPR
, ivar_type
, ivar
,
7238 fold_convert (ivar_type
, collapse
->iters
));
7239 ivar
= force_gimple_operand_gsi (gsi
, expr
, true, NULL_TREE
,
7240 true, GSI_SAME_STMT
);
7246 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
7247 of the combined collapse > 1 loop constructs, generate code like:
7248 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
7253 count3 = (adj + N32 - N31) / STEP3;
7254 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
7259 count2 = (adj + N22 - N21) / STEP2;
7260 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
7265 count1 = (adj + N12 - N11) / STEP1;
7266 count = count1 * count2 * count3;
7267 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
7269 and set ZERO_ITER_BB to that bb. If this isn't the outermost
7270 of the combined loop constructs, just initialize COUNTS array
7271 from the _looptemp_ clauses. */
7273 /* NOTE: It *could* be better to moosh all of the BBs together,
7274 creating one larger BB with all the computation and the unexpected
7275 jump at the end. I.e.
7277 bool zero3, zero2, zero1, zero;
7280 count3 = (N32 - N31) /[cl] STEP3;
7282 count2 = (N22 - N21) /[cl] STEP2;
7284 count1 = (N12 - N11) /[cl] STEP1;
7285 zero = zero3 || zero2 || zero1;
7286 count = count1 * count2 * count3;
7287 if (__builtin_expect(zero, false)) goto zero_iter_bb;
7289 After all, we expect the zero=false, and thus we expect to have to
7290 evaluate all of the comparison expressions, so short-circuiting
7291 oughtn't be a win. Since the condition isn't protecting a
7292 denominator, we're not concerned about divide-by-zero, so we can
7293 fully evaluate count even if a numerator turned out to be wrong.
7295 It seems like putting this all together would create much better
7296 scheduling opportunities, and less pressure on the chip's branch
7300 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
7301 basic_block
&entry_bb
, tree
*counts
,
7302 basic_block
&zero_iter1_bb
, int &first_zero_iter1
,
7303 basic_block
&zero_iter2_bb
, int &first_zero_iter2
,
7304 basic_block
&l2_dom_bb
)
7306 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
7310 /* Collapsed loops need work for expansion into SSA form. */
7311 gcc_assert (!gimple_in_ssa_p (cfun
));
7313 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
7314 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
7316 gcc_assert (fd
->ordered
== 0);
7317 /* First two _looptemp_ clauses are for istart/iend, counts[0]
7318 isn't supposed to be handled, as the inner loop doesn't
7320 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7321 OMP_CLAUSE__LOOPTEMP_
);
7322 gcc_assert (innerc
);
7323 for (i
= 0; i
< fd
->collapse
; i
++)
7325 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7326 OMP_CLAUSE__LOOPTEMP_
);
7327 gcc_assert (innerc
);
7329 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
7331 counts
[0] = NULL_TREE
;
7336 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
7338 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7339 counts
[i
] = NULL_TREE
;
7340 t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
7341 fold_convert (itype
, fd
->loops
[i
].n1
),
7342 fold_convert (itype
, fd
->loops
[i
].n2
));
7343 if (t
&& integer_zerop (t
))
7345 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
7346 counts
[i
] = build_int_cst (type
, 0);
7350 for (i
= 0; i
< (fd
->ordered
? fd
->ordered
: fd
->collapse
); i
++)
7352 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7354 if (i
>= fd
->collapse
&& counts
[i
])
7356 if ((SSA_VAR_P (fd
->loop
.n2
) || i
>= fd
->collapse
)
7357 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
7358 fold_convert (itype
, fd
->loops
[i
].n1
),
7359 fold_convert (itype
, fd
->loops
[i
].n2
)))
7360 == NULL_TREE
|| !integer_onep (t
)))
7364 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
7365 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
7366 true, GSI_SAME_STMT
);
7367 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
7368 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
7369 true, GSI_SAME_STMT
);
7370 cond_stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
7371 NULL_TREE
, NULL_TREE
);
7372 gsi_insert_before (gsi
, cond_stmt
, GSI_SAME_STMT
);
7373 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
7374 expand_omp_regimplify_p
, NULL
, NULL
)
7375 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
7376 expand_omp_regimplify_p
, NULL
, NULL
))
7378 *gsi
= gsi_for_stmt (cond_stmt
);
7379 gimple_regimplify_operands (cond_stmt
, gsi
);
7381 e
= split_block (entry_bb
, cond_stmt
);
7382 basic_block
&zero_iter_bb
7383 = i
< fd
->collapse
? zero_iter1_bb
: zero_iter2_bb
;
7384 int &first_zero_iter
7385 = i
< fd
->collapse
? first_zero_iter1
: first_zero_iter2
;
7386 if (zero_iter_bb
== NULL
)
7388 gassign
*assign_stmt
;
7389 first_zero_iter
= i
;
7390 zero_iter_bb
= create_empty_bb (entry_bb
);
7391 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
7392 *gsi
= gsi_after_labels (zero_iter_bb
);
7393 if (i
< fd
->collapse
)
7394 assign_stmt
= gimple_build_assign (fd
->loop
.n2
,
7395 build_zero_cst (type
));
7398 counts
[i
] = create_tmp_reg (type
, ".count");
7400 = gimple_build_assign (counts
[i
], build_zero_cst (type
));
7402 gsi_insert_before (gsi
, assign_stmt
, GSI_SAME_STMT
);
7403 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
7406 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
7407 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
7408 e
->flags
= EDGE_TRUE_VALUE
;
7409 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
7410 if (l2_dom_bb
== NULL
)
7411 l2_dom_bb
= entry_bb
;
7413 *gsi
= gsi_last_bb (entry_bb
);
7416 if (POINTER_TYPE_P (itype
))
7417 itype
= signed_type_for (itype
);
7418 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
7420 t
= fold_build2 (PLUS_EXPR
, itype
,
7421 fold_convert (itype
, fd
->loops
[i
].step
), t
);
7422 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
7423 fold_convert (itype
, fd
->loops
[i
].n2
));
7424 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
7425 fold_convert (itype
, fd
->loops
[i
].n1
));
7426 /* ?? We could probably use CEIL_DIV_EXPR instead of
7427 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
7428 generate the same code in the end because generically we
7429 don't know that the values involved must be negative for
7431 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
7432 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
7433 fold_build1 (NEGATE_EXPR
, itype
, t
),
7434 fold_build1 (NEGATE_EXPR
, itype
,
7435 fold_convert (itype
,
7436 fd
->loops
[i
].step
)));
7438 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
7439 fold_convert (itype
, fd
->loops
[i
].step
));
7440 t
= fold_convert (type
, t
);
7441 if (TREE_CODE (t
) == INTEGER_CST
)
7445 if (i
< fd
->collapse
|| i
!= first_zero_iter2
)
7446 counts
[i
] = create_tmp_reg (type
, ".count");
7447 expand_omp_build_assign (gsi
, counts
[i
], t
);
7449 if (SSA_VAR_P (fd
->loop
.n2
) && i
< fd
->collapse
)
7454 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
7455 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
7461 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
7463 V3 = N31 + (T % count3) * STEP3;
7465 V2 = N21 + (T % count2) * STEP2;
7467 V1 = N11 + T * STEP1;
7468 if this loop doesn't have an inner loop construct combined with it.
7469 If it does have an inner loop construct combined with it and the
7470 iteration count isn't known constant, store values from counts array
7471 into its _looptemp_ temporaries instead. */
7474 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
7475 tree
*counts
, gimple
*inner_stmt
, tree startvar
)
7478 if (gimple_omp_for_combined_p (fd
->for_stmt
))
7480 /* If fd->loop.n2 is constant, then no propagation of the counts
7481 is needed, they are constant. */
7482 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
7485 tree clauses
= gimple_code (inner_stmt
) != GIMPLE_OMP_FOR
7486 ? gimple_omp_taskreg_clauses (inner_stmt
)
7487 : gimple_omp_for_clauses (inner_stmt
);
7488 /* First two _looptemp_ clauses are for istart/iend, counts[0]
7489 isn't supposed to be handled, as the inner loop doesn't
7491 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
7492 gcc_assert (innerc
);
7493 for (i
= 0; i
< fd
->collapse
; i
++)
7495 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7496 OMP_CLAUSE__LOOPTEMP_
);
7497 gcc_assert (innerc
);
7500 tree tem
= OMP_CLAUSE_DECL (innerc
);
7501 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
7502 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
7503 false, GSI_CONTINUE_LINKING
);
7504 gassign
*stmt
= gimple_build_assign (tem
, t
);
7505 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7511 tree type
= TREE_TYPE (fd
->loop
.v
);
7512 tree tem
= create_tmp_reg (type
, ".tem");
7513 gassign
*stmt
= gimple_build_assign (tem
, startvar
);
7514 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7516 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
7518 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
7520 if (POINTER_TYPE_P (vtype
))
7521 itype
= signed_type_for (vtype
);
7523 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
7526 t
= fold_convert (itype
, t
);
7527 t
= fold_build2 (MULT_EXPR
, itype
, t
,
7528 fold_convert (itype
, fd
->loops
[i
].step
));
7529 if (POINTER_TYPE_P (vtype
))
7530 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
7532 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
7533 t
= force_gimple_operand_gsi (gsi
, t
,
7534 DECL_P (fd
->loops
[i
].v
)
7535 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
7537 GSI_CONTINUE_LINKING
);
7538 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
7539 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7542 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
7543 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
7544 false, GSI_CONTINUE_LINKING
);
7545 stmt
= gimple_build_assign (tem
, t
);
7546 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7552 /* Helper function for expand_omp_for_*. Generate code like:
7555 if (V3 cond3 N32) goto BODY_BB; else goto L11;
7559 if (V2 cond2 N22) goto BODY_BB; else goto L12;
7566 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
7567 basic_block body_bb
)
7569 basic_block last_bb
, bb
, collapse_bb
= NULL
;
7571 gimple_stmt_iterator gsi
;
7577 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
7579 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
7581 bb
= create_empty_bb (last_bb
);
7582 add_bb_to_loop (bb
, last_bb
->loop_father
);
7583 gsi
= gsi_start_bb (bb
);
7585 if (i
< fd
->collapse
- 1)
7587 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
7588 e
->probability
= REG_BR_PROB_BASE
/ 8;
7590 t
= fd
->loops
[i
+ 1].n1
;
7591 t
= force_gimple_operand_gsi (&gsi
, t
,
7592 DECL_P (fd
->loops
[i
+ 1].v
)
7593 && TREE_ADDRESSABLE (fd
->loops
[i
7596 GSI_CONTINUE_LINKING
);
7597 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
7598 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7603 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
7605 if (POINTER_TYPE_P (vtype
))
7606 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
7608 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
7609 t
= force_gimple_operand_gsi (&gsi
, t
,
7610 DECL_P (fd
->loops
[i
].v
)
7611 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
7612 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
7613 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
7614 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7618 t
= fd
->loops
[i
].n2
;
7619 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7620 false, GSI_CONTINUE_LINKING
);
7621 tree v
= fd
->loops
[i
].v
;
7622 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
7623 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
7624 false, GSI_CONTINUE_LINKING
);
7625 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
7626 stmt
= gimple_build_cond_empty (t
);
7627 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7628 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
7629 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7632 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
7640 /* Expand #pragma omp ordered depend(source). */
7643 expand_omp_ordered_source (gimple_stmt_iterator
*gsi
, struct omp_for_data
*fd
,
7644 tree
*counts
, location_t loc
)
7646 enum built_in_function source_ix
7647 = fd
->iter_type
== long_integer_type_node
7648 ? BUILT_IN_GOMP_DOACROSS_POST
: BUILT_IN_GOMP_DOACROSS_ULL_POST
;
7650 = gimple_build_call (builtin_decl_explicit (source_ix
), 1,
7651 build_fold_addr_expr (counts
[fd
->ordered
]));
7652 gimple_set_location (g
, loc
);
7653 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
7656 /* Expand a single depend from #pragma omp ordered depend(sink:...). */
7659 expand_omp_ordered_sink (gimple_stmt_iterator
*gsi
, struct omp_for_data
*fd
,
7660 tree
*counts
, tree c
, location_t loc
)
7662 auto_vec
<tree
, 10> args
;
7663 enum built_in_function sink_ix
7664 = fd
->iter_type
== long_integer_type_node
7665 ? BUILT_IN_GOMP_DOACROSS_WAIT
: BUILT_IN_GOMP_DOACROSS_ULL_WAIT
;
7666 tree t
, off
, coff
= NULL_TREE
, deps
= OMP_CLAUSE_DECL (c
), cond
= NULL_TREE
;
7668 gimple_stmt_iterator gsi2
= *gsi
;
7669 bool warned_step
= false;
7671 for (i
= 0; i
< fd
->ordered
; i
++)
7673 off
= TREE_PURPOSE (deps
);
7674 if (!integer_zerop (off
))
7676 gcc_assert (fd
->loops
[i
].cond_code
== LT_EXPR
7677 || fd
->loops
[i
].cond_code
== GT_EXPR
);
7678 bool forward
= fd
->loops
[i
].cond_code
== LT_EXPR
;
7679 if (forward
^ OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7680 warning_at (loc
, 0, "%<depend(sink)%> clause waiting for "
7681 "lexically later iteration");
7684 deps
= TREE_CHAIN (deps
);
7686 /* If all offsets corresponding to the collapsed loops are zero,
7687 this depend clause can be ignored. FIXME: but there is still a
7688 flush needed. We need to emit one __sync_synchronize () for it
7689 though (perhaps conditionally)? Solve this together with the
7690 conservative dependence folding optimization.
7691 if (i >= fd->collapse)
7694 deps
= OMP_CLAUSE_DECL (c
);
7696 edge e1
= split_block (gsi_bb (gsi2
), gsi_stmt (gsi2
));
7697 edge e2
= split_block_after_labels (e1
->dest
);
7699 *gsi
= gsi_after_labels (e1
->dest
);
7700 for (i
= 0; i
< fd
->ordered
; i
++)
7702 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7703 if (POINTER_TYPE_P (itype
))
7706 deps
= TREE_CHAIN (deps
);
7707 off
= TREE_PURPOSE (deps
);
7708 tree s
= fold_convert_loc (loc
, itype
, fd
->loops
[i
].step
);
7710 if (integer_zerop (off
))
7711 t
= boolean_true_node
;
7715 tree co
= fold_convert_loc (loc
, itype
, off
);
7716 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
7718 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7719 co
= fold_build1_loc (loc
, NEGATE_EXPR
, itype
, co
);
7720 a
= fold_build2_loc (loc
, POINTER_PLUS_EXPR
,
7721 TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].v
,
7724 else if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7725 a
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7726 fd
->loops
[i
].v
, co
);
7728 a
= fold_build2_loc (loc
, PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7729 fd
->loops
[i
].v
, co
);
7730 if (fd
->loops
[i
].cond_code
== LT_EXPR
)
7732 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7733 t
= fold_build2_loc (loc
, GE_EXPR
, boolean_type_node
, a
,
7736 t
= fold_build2_loc (loc
, LT_EXPR
, boolean_type_node
, a
,
7739 else if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7740 t
= fold_build2_loc (loc
, GT_EXPR
, boolean_type_node
, a
,
7743 t
= fold_build2_loc (loc
, LE_EXPR
, boolean_type_node
, a
,
7747 cond
= fold_build2_loc (loc
, BIT_AND_EXPR
, boolean_type_node
, cond
, t
);
7751 off
= fold_convert_loc (loc
, itype
, off
);
7753 if (fd
->loops
[i
].cond_code
== LT_EXPR
7754 ? !integer_onep (fd
->loops
[i
].step
)
7755 : !integer_minus_onep (fd
->loops
[i
].step
))
7757 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
7758 t
= fold_build2_loc (loc
, TRUNC_MOD_EXPR
, itype
, off
,
7759 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
7762 t
= fold_build2_loc (loc
, TRUNC_MOD_EXPR
, itype
, off
, s
);
7763 t
= fold_build2_loc (loc
, EQ_EXPR
, boolean_type_node
, t
,
7764 build_int_cst (itype
, 0));
7765 if (integer_zerop (t
) && !warned_step
)
7767 warning_at (loc
, 0, "%<depend(sink)%> refers to iteration never "
7768 "in the iteration space");
7771 cond
= fold_build2_loc (loc
, BIT_AND_EXPR
, boolean_type_node
,
7775 if (i
<= fd
->collapse
- 1 && fd
->collapse
> 1)
7781 t
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7782 fd
->loops
[i
].v
, fd
->loops
[i
].n1
);
7783 t
= fold_convert_loc (loc
, fd
->iter_type
, t
);
7785 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
7786 off
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, off
,
7787 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
7790 off
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, off
, s
);
7791 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7792 off
= fold_build1_loc (loc
, NEGATE_EXPR
, itype
, off
);
7793 off
= fold_convert_loc (loc
, fd
->iter_type
, off
);
7794 if (i
<= fd
->collapse
- 1 && fd
->collapse
> 1)
7797 off
= fold_build2_loc (loc
, PLUS_EXPR
, fd
->iter_type
, coff
,
7799 if (i
< fd
->collapse
- 1)
7801 coff
= fold_build2_loc (loc
, MULT_EXPR
, fd
->iter_type
, off
,
7806 off
= unshare_expr (off
);
7807 t
= fold_build2_loc (loc
, PLUS_EXPR
, fd
->iter_type
, t
, off
);
7808 t
= force_gimple_operand_gsi (gsi
, t
, true, NULL_TREE
,
7809 true, GSI_SAME_STMT
);
7812 gimple
*g
= gimple_build_call_vec (builtin_decl_explicit (sink_ix
), args
);
7813 gimple_set_location (g
, loc
);
7814 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
7816 *gsi
= gsi_last_bb (e1
->src
);
7817 cond
= unshare_expr (cond
);
7818 cond
= force_gimple_operand_gsi (gsi
, cond
, true, NULL_TREE
, false,
7819 GSI_CONTINUE_LINKING
);
7820 gsi_insert_after (gsi
, gimple_build_cond_empty (cond
), GSI_NEW_STMT
);
7821 edge e3
= make_edge (e1
->src
, e2
->dest
, EDGE_FALSE_VALUE
);
7822 e3
->probability
= REG_BR_PROB_BASE
/ 8;
7823 e1
->probability
= REG_BR_PROB_BASE
- e3
->probability
;
7824 e1
->flags
= EDGE_TRUE_VALUE
;
7825 set_immediate_dominator (CDI_DOMINATORS
, e2
->dest
, e1
->src
);
7827 *gsi
= gsi_after_labels (e2
->dest
);
7830 /* Expand all #pragma omp ordered depend(source) and
7831 #pragma omp ordered depend(sink:...) constructs in the current
7832 #pragma omp for ordered(n) region. */
7835 expand_omp_ordered_source_sink (struct omp_region
*region
,
7836 struct omp_for_data
*fd
, tree
*counts
,
7837 basic_block cont_bb
)
7839 struct omp_region
*inner
;
7841 for (i
= fd
->collapse
- 1; i
< fd
->ordered
; i
++)
7842 if (i
== fd
->collapse
- 1 && fd
->collapse
> 1)
7843 counts
[i
] = NULL_TREE
;
7844 else if (i
>= fd
->collapse
&& !cont_bb
)
7845 counts
[i
] = build_zero_cst (fd
->iter_type
);
7846 else if (!POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
))
7847 && integer_onep (fd
->loops
[i
].step
))
7848 counts
[i
] = NULL_TREE
;
7850 counts
[i
] = create_tmp_var (fd
->iter_type
, ".orditer");
7852 = build_array_type_nelts (fd
->iter_type
, fd
->ordered
- fd
->collapse
+ 1);
7853 counts
[fd
->ordered
] = create_tmp_var (atype
, ".orditera");
7854 TREE_ADDRESSABLE (counts
[fd
->ordered
]) = 1;
7856 for (inner
= region
->inner
; inner
; inner
= inner
->next
)
7857 if (inner
->type
== GIMPLE_OMP_ORDERED
)
7859 gomp_ordered
*ord_stmt
= inner
->ord_stmt
;
7860 gimple_stmt_iterator gsi
= gsi_for_stmt (ord_stmt
);
7861 location_t loc
= gimple_location (ord_stmt
);
7863 for (c
= gimple_omp_ordered_clauses (ord_stmt
);
7864 c
; c
= OMP_CLAUSE_CHAIN (c
))
7865 if (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
)
7868 expand_omp_ordered_source (&gsi
, fd
, counts
, loc
);
7869 for (c
= gimple_omp_ordered_clauses (ord_stmt
);
7870 c
; c
= OMP_CLAUSE_CHAIN (c
))
7871 if (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
)
7872 expand_omp_ordered_sink (&gsi
, fd
, counts
, c
, loc
);
7873 gsi_remove (&gsi
, true);
7877 /* Wrap the body into fd->ordered - fd->collapse loops that aren't
7881 expand_omp_for_ordered_loops (struct omp_for_data
*fd
, tree
*counts
,
7882 basic_block cont_bb
, basic_block body_bb
,
7883 bool ordered_lastprivate
)
7885 if (fd
->ordered
== fd
->collapse
)
7890 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
7891 for (int i
= fd
->collapse
; i
< fd
->ordered
; i
++)
7893 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
7894 tree n1
= fold_convert (type
, fd
->loops
[i
].n1
);
7895 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, n1
);
7896 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
7897 size_int (i
- fd
->collapse
+ 1),
7898 NULL_TREE
, NULL_TREE
);
7899 expand_omp_build_assign (&gsi
, aref
, build_zero_cst (fd
->iter_type
));
7904 for (int i
= fd
->ordered
- 1; i
>= fd
->collapse
; i
--)
7906 tree t
, type
= TREE_TYPE (fd
->loops
[i
].v
);
7907 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
7908 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
,
7909 fold_convert (type
, fd
->loops
[i
].n1
));
7911 expand_omp_build_assign (&gsi
, counts
[i
],
7912 build_zero_cst (fd
->iter_type
));
7913 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
7914 size_int (i
- fd
->collapse
+ 1),
7915 NULL_TREE
, NULL_TREE
);
7916 expand_omp_build_assign (&gsi
, aref
, build_zero_cst (fd
->iter_type
));
7917 if (!gsi_end_p (gsi
))
7920 gsi
= gsi_last_bb (body_bb
);
7921 edge e1
= split_block (body_bb
, gsi_stmt (gsi
));
7922 basic_block new_body
= e1
->dest
;
7923 if (body_bb
== cont_bb
)
7926 basic_block new_header
;
7927 if (EDGE_COUNT (cont_bb
->preds
) > 0)
7929 gsi
= gsi_last_bb (cont_bb
);
7930 if (POINTER_TYPE_P (type
))
7931 t
= fold_build_pointer_plus (fd
->loops
[i
].v
,
7932 fold_convert (sizetype
,
7933 fd
->loops
[i
].step
));
7935 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loops
[i
].v
,
7936 fold_convert (type
, fd
->loops
[i
].step
));
7937 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7940 t
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, counts
[i
],
7941 build_int_cst (fd
->iter_type
, 1));
7942 expand_omp_build_assign (&gsi
, counts
[i
], t
);
7947 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7948 fd
->loops
[i
].v
, fd
->loops
[i
].n1
);
7949 t
= fold_convert (fd
->iter_type
, t
);
7950 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7951 true, GSI_SAME_STMT
);
7953 aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
7954 size_int (i
- fd
->collapse
+ 1),
7955 NULL_TREE
, NULL_TREE
);
7956 expand_omp_build_assign (&gsi
, aref
, t
);
7958 e2
= split_block (cont_bb
, gsi_stmt (gsi
));
7959 new_header
= e2
->dest
;
7962 new_header
= cont_bb
;
7963 gsi
= gsi_after_labels (new_header
);
7964 tree v
= force_gimple_operand_gsi (&gsi
, fd
->loops
[i
].v
, true, NULL_TREE
,
7965 true, GSI_SAME_STMT
);
7967 = force_gimple_operand_gsi (&gsi
, fold_convert (type
, fd
->loops
[i
].n2
),
7968 true, NULL_TREE
, true, GSI_SAME_STMT
);
7969 t
= build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, n2
);
7970 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_NEW_STMT
);
7971 edge e3
= split_block (new_header
, gsi_stmt (gsi
));
7974 make_edge (body_bb
, new_header
, EDGE_FALLTHRU
);
7975 e3
->flags
= EDGE_FALSE_VALUE
;
7976 e3
->probability
= REG_BR_PROB_BASE
/ 8;
7977 e1
= make_edge (new_header
, new_body
, EDGE_TRUE_VALUE
);
7978 e1
->probability
= REG_BR_PROB_BASE
- e3
->probability
;
7980 set_immediate_dominator (CDI_DOMINATORS
, new_header
, body_bb
);
7981 set_immediate_dominator (CDI_DOMINATORS
, new_body
, new_header
);
7985 struct loop
*loop
= alloc_loop ();
7986 loop
->header
= new_header
;
7987 loop
->latch
= e2
->src
;
7988 add_loop (loop
, body_bb
->loop_father
);
7992 /* If there are any lastprivate clauses and it is possible some loops
7993 might have zero iterations, ensure all the decls are initialized,
7994 otherwise we could crash evaluating C++ class iterators with lastprivate
7996 bool need_inits
= false;
7997 for (int i
= fd
->collapse
; ordered_lastprivate
&& i
< fd
->ordered
; i
++)
8000 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8001 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
8002 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
,
8003 fold_convert (type
, fd
->loops
[i
].n1
));
8007 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8008 tree this_cond
= fold_build2 (fd
->loops
[i
].cond_code
,
8010 fold_convert (type
, fd
->loops
[i
].n1
),
8011 fold_convert (type
, fd
->loops
[i
].n2
));
8012 if (!integer_onep (this_cond
))
8020 /* A subroutine of expand_omp_for. Generate code for a parallel
8021 loop with any schedule. Given parameters:
8023 for (V = N1; V cond N2; V += STEP) BODY;
8025 where COND is "<" or ">", we generate pseudocode
8027 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
8028 if (more) goto L0; else goto L3;
8035 if (V cond iend) goto L1; else goto L2;
8037 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
8040 If this is a combined omp parallel loop, instead of the call to
8041 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
8042 If this is gimple_omp_for_combined_p loop, then instead of assigning
8043 V and iend in L0 we assign the first two _looptemp_ clause decls of the
8044 inner GIMPLE_OMP_FOR and V += STEP; and
8045 if (V cond iend) goto L1; else goto L2; are removed.
8047 For collapsed loops, given parameters:
8049 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
8050 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
8051 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
8054 we generate pseudocode
8056 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
8061 count3 = (adj + N32 - N31) / STEP3;
8062 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
8067 count2 = (adj + N22 - N21) / STEP2;
8068 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
8073 count1 = (adj + N12 - N11) / STEP1;
8074 count = count1 * count2 * count3;
8079 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
8080 if (more) goto L0; else goto L3;
8084 V3 = N31 + (T % count3) * STEP3;
8086 V2 = N21 + (T % count2) * STEP2;
8088 V1 = N11 + T * STEP1;
8093 if (V < iend) goto L10; else goto L2;
8096 if (V3 cond3 N32) goto L1; else goto L11;
8100 if (V2 cond2 N22) goto L1; else goto L12;
8106 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
8112 expand_omp_for_generic (struct omp_region
*region
,
8113 struct omp_for_data
*fd
,
8114 enum built_in_function start_fn
,
8115 enum built_in_function next_fn
,
8118 tree type
, istart0
, iend0
, iend
;
8119 tree t
, vmain
, vback
, bias
= NULL_TREE
;
8120 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
8121 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
8122 gimple_stmt_iterator gsi
;
8123 gassign
*assign_stmt
;
8124 bool in_combined_parallel
= is_combined_parallel (region
);
8125 bool broken_loop
= region
->cont
== NULL
;
8127 tree
*counts
= NULL
;
8129 bool ordered_lastprivate
= false;
8131 gcc_assert (!broken_loop
|| !in_combined_parallel
);
8132 gcc_assert (fd
->iter_type
== long_integer_type_node
8133 || !in_combined_parallel
);
8135 entry_bb
= region
->entry
;
8136 cont_bb
= region
->cont
;
8138 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
8139 gcc_assert (broken_loop
8140 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
8141 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
8142 l1_bb
= single_succ (l0_bb
);
8145 l2_bb
= create_empty_bb (cont_bb
);
8146 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
8147 || (single_succ_edge (BRANCH_EDGE (cont_bb
)->dest
)->dest
8149 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
8153 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
8154 exit_bb
= region
->exit
;
8156 gsi
= gsi_last_bb (entry_bb
);
8158 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
8160 && find_omp_clause (gimple_omp_for_clauses (gsi_stmt (gsi
)),
8161 OMP_CLAUSE_LASTPRIVATE
))
8162 ordered_lastprivate
= false;
8163 if (fd
->collapse
> 1 || fd
->ordered
)
8165 int first_zero_iter1
= -1, first_zero_iter2
= -1;
8166 basic_block zero_iter1_bb
= NULL
, zero_iter2_bb
= NULL
, l2_dom_bb
= NULL
;
8168 counts
= XALLOCAVEC (tree
, fd
->ordered
? fd
->ordered
+ 1 : fd
->collapse
);
8169 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
8170 zero_iter1_bb
, first_zero_iter1
,
8171 zero_iter2_bb
, first_zero_iter2
, l2_dom_bb
);
8175 /* Some counts[i] vars might be uninitialized if
8176 some loop has zero iterations. But the body shouldn't
8177 be executed in that case, so just avoid uninit warnings. */
8178 for (i
= first_zero_iter1
;
8179 i
< (fd
->ordered
? fd
->ordered
: fd
->collapse
); i
++)
8180 if (SSA_VAR_P (counts
[i
]))
8181 TREE_NO_WARNING (counts
[i
]) = 1;
8183 e
= split_block (entry_bb
, gsi_stmt (gsi
));
8185 make_edge (zero_iter1_bb
, entry_bb
, EDGE_FALLTHRU
);
8186 gsi
= gsi_last_bb (entry_bb
);
8187 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
8188 get_immediate_dominator (CDI_DOMINATORS
,
8193 /* Some counts[i] vars might be uninitialized if
8194 some loop has zero iterations. But the body shouldn't
8195 be executed in that case, so just avoid uninit warnings. */
8196 for (i
= first_zero_iter2
; i
< fd
->ordered
; i
++)
8197 if (SSA_VAR_P (counts
[i
]))
8198 TREE_NO_WARNING (counts
[i
]) = 1;
8200 make_edge (zero_iter2_bb
, entry_bb
, EDGE_FALLTHRU
);
8204 e
= split_block (entry_bb
, gsi_stmt (gsi
));
8206 make_edge (zero_iter2_bb
, entry_bb
, EDGE_FALLTHRU
);
8207 gsi
= gsi_last_bb (entry_bb
);
8208 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
8209 get_immediate_dominator
8210 (CDI_DOMINATORS
, zero_iter2_bb
));
8213 if (fd
->collapse
== 1)
8215 counts
[0] = fd
->loop
.n2
;
8216 fd
->loop
= fd
->loops
[0];
8220 type
= TREE_TYPE (fd
->loop
.v
);
8221 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
8222 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
8223 TREE_ADDRESSABLE (istart0
) = 1;
8224 TREE_ADDRESSABLE (iend0
) = 1;
8226 /* See if we need to bias by LLONG_MIN. */
8227 if (fd
->iter_type
== long_long_unsigned_type_node
8228 && TREE_CODE (type
) == INTEGER_TYPE
8229 && !TYPE_UNSIGNED (type
)
8230 && fd
->ordered
== 0)
8234 if (fd
->loop
.cond_code
== LT_EXPR
)
8237 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
8241 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
8244 if (TREE_CODE (n1
) != INTEGER_CST
8245 || TREE_CODE (n2
) != INTEGER_CST
8246 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
8247 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
8250 gimple_stmt_iterator gsif
= gsi
;
8253 tree arr
= NULL_TREE
;
8254 if (in_combined_parallel
)
8256 gcc_assert (fd
->ordered
== 0);
8257 /* In a combined parallel loop, emit a call to
8258 GOMP_loop_foo_next. */
8259 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
8260 build_fold_addr_expr (istart0
),
8261 build_fold_addr_expr (iend0
));
8265 tree t0
, t1
, t2
, t3
, t4
;
8266 /* If this is not a combined parallel loop, emit a call to
8267 GOMP_loop_foo_start in ENTRY_BB. */
8268 t4
= build_fold_addr_expr (iend0
);
8269 t3
= build_fold_addr_expr (istart0
);
8272 t0
= build_int_cst (unsigned_type_node
,
8273 fd
->ordered
- fd
->collapse
+ 1);
8274 arr
= create_tmp_var (build_array_type_nelts (fd
->iter_type
,
8276 - fd
->collapse
+ 1),
8278 DECL_NAMELESS (arr
) = 1;
8279 TREE_ADDRESSABLE (arr
) = 1;
8280 TREE_STATIC (arr
) = 1;
8281 vec
<constructor_elt
, va_gc
> *v
;
8282 vec_alloc (v
, fd
->ordered
- fd
->collapse
+ 1);
8285 for (idx
= 0; idx
< fd
->ordered
- fd
->collapse
+ 1; idx
++)
8288 if (idx
== 0 && fd
->collapse
> 1)
8291 c
= counts
[idx
+ fd
->collapse
- 1];
8292 tree purpose
= size_int (idx
);
8293 CONSTRUCTOR_APPEND_ELT (v
, purpose
, c
);
8294 if (TREE_CODE (c
) != INTEGER_CST
)
8295 TREE_STATIC (arr
) = 0;
8298 DECL_INITIAL (arr
) = build_constructor (TREE_TYPE (arr
), v
);
8299 if (!TREE_STATIC (arr
))
8300 force_gimple_operand_gsi (&gsi
, build1 (DECL_EXPR
,
8301 void_type_node
, arr
),
8302 true, NULL_TREE
, true, GSI_SAME_STMT
);
8303 t1
= build_fold_addr_expr (arr
);
8308 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
8311 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
8314 = find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
8315 OMP_CLAUSE__LOOPTEMP_
);
8316 gcc_assert (innerc
);
8317 t0
= OMP_CLAUSE_DECL (innerc
);
8318 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
8319 OMP_CLAUSE__LOOPTEMP_
);
8320 gcc_assert (innerc
);
8321 t1
= OMP_CLAUSE_DECL (innerc
);
8323 if (POINTER_TYPE_P (TREE_TYPE (t0
))
8324 && TYPE_PRECISION (TREE_TYPE (t0
))
8325 != TYPE_PRECISION (fd
->iter_type
))
8327 /* Avoid casting pointers to integer of a different size. */
8328 tree itype
= signed_type_for (type
);
8329 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
8330 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
8334 t1
= fold_convert (fd
->iter_type
, t1
);
8335 t0
= fold_convert (fd
->iter_type
, t0
);
8339 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
8340 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
8343 if (fd
->iter_type
== long_integer_type_node
|| fd
->ordered
)
8347 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
8348 t
= omp_adjust_chunk_size (t
, fd
->simd_schedule
);
8350 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8351 5, t0
, t1
, t
, t3
, t4
);
8353 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8354 6, t0
, t1
, t2
, t
, t3
, t4
);
8356 else if (fd
->ordered
)
8357 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8360 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8361 5, t0
, t1
, t2
, t3
, t4
);
8369 /* The GOMP_loop_ull_*start functions have additional boolean
8370 argument, true for < loops and false for > loops.
8371 In Fortran, the C bool type can be different from
8372 boolean_type_node. */
8373 bfn_decl
= builtin_decl_explicit (start_fn
);
8374 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
8375 t5
= build_int_cst (c_bool_type
,
8376 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
8379 tree bfn_decl
= builtin_decl_explicit (start_fn
);
8380 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
8381 t
= omp_adjust_chunk_size (t
, fd
->simd_schedule
);
8382 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
8385 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8386 6, t5
, t0
, t1
, t2
, t3
, t4
);
8389 if (TREE_TYPE (t
) != boolean_type_node
)
8390 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
8391 t
, build_int_cst (TREE_TYPE (t
), 0));
8392 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8393 true, GSI_SAME_STMT
);
8394 if (arr
&& !TREE_STATIC (arr
))
8396 tree clobber
= build_constructor (TREE_TYPE (arr
), NULL
);
8397 TREE_THIS_VOLATILE (clobber
) = 1;
8398 gsi_insert_before (&gsi
, gimple_build_assign (arr
, clobber
),
8401 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
8403 /* Remove the GIMPLE_OMP_FOR statement. */
8404 gsi_remove (&gsi
, true);
8406 if (gsi_end_p (gsif
))
8407 gsif
= gsi_after_labels (gsi_bb (gsif
));
8410 /* Iteration setup for sequential loop goes in L0_BB. */
8411 tree startvar
= fd
->loop
.v
;
8412 tree endvar
= NULL_TREE
;
8414 if (gimple_omp_for_combined_p (fd
->for_stmt
))
8416 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
8417 && gimple_omp_for_kind (inner_stmt
)
8418 == GF_OMP_FOR_KIND_SIMD
);
8419 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
8420 OMP_CLAUSE__LOOPTEMP_
);
8421 gcc_assert (innerc
);
8422 startvar
= OMP_CLAUSE_DECL (innerc
);
8423 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
8424 OMP_CLAUSE__LOOPTEMP_
);
8425 gcc_assert (innerc
);
8426 endvar
= OMP_CLAUSE_DECL (innerc
);
8429 gsi
= gsi_start_bb (l0_bb
);
8431 if (fd
->ordered
&& fd
->collapse
== 1)
8432 t
= fold_build2 (MULT_EXPR
, fd
->iter_type
, t
,
8433 fold_convert (fd
->iter_type
, fd
->loop
.step
));
8435 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
8436 if (fd
->ordered
&& fd
->collapse
== 1)
8438 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8439 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (startvar
),
8440 fd
->loop
.n1
, fold_convert (sizetype
, t
));
8443 t
= fold_convert (TREE_TYPE (startvar
), t
);
8444 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (startvar
),
8450 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8451 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
8452 t
= fold_convert (TREE_TYPE (startvar
), t
);
8454 t
= force_gimple_operand_gsi (&gsi
, t
,
8456 && TREE_ADDRESSABLE (startvar
),
8457 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
8458 assign_stmt
= gimple_build_assign (startvar
, t
);
8459 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8462 if (fd
->ordered
&& fd
->collapse
== 1)
8463 t
= fold_build2 (MULT_EXPR
, fd
->iter_type
, t
,
8464 fold_convert (fd
->iter_type
, fd
->loop
.step
));
8466 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
8467 if (fd
->ordered
&& fd
->collapse
== 1)
8469 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8470 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (startvar
),
8471 fd
->loop
.n1
, fold_convert (sizetype
, t
));
8474 t
= fold_convert (TREE_TYPE (startvar
), t
);
8475 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (startvar
),
8481 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8482 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
8483 t
= fold_convert (TREE_TYPE (startvar
), t
);
8485 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8486 false, GSI_CONTINUE_LINKING
);
8489 assign_stmt
= gimple_build_assign (endvar
, iend
);
8490 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8491 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
8492 assign_stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
8494 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, iend
);
8495 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8497 /* Handle linear clause adjustments. */
8498 tree itercnt
= NULL_TREE
;
8499 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
8500 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
8501 c
; c
= OMP_CLAUSE_CHAIN (c
))
8502 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
8503 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
8505 tree d
= OMP_CLAUSE_DECL (c
);
8506 bool is_ref
= is_reference (d
);
8507 tree t
= d
, a
, dest
;
8509 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
8510 tree type
= TREE_TYPE (t
);
8511 if (POINTER_TYPE_P (type
))
8513 dest
= unshare_expr (t
);
8514 tree v
= create_tmp_var (TREE_TYPE (t
), NULL
);
8515 expand_omp_build_assign (&gsif
, v
, t
);
8516 if (itercnt
== NULL_TREE
)
8519 tree n1
= fd
->loop
.n1
;
8520 if (POINTER_TYPE_P (TREE_TYPE (itercnt
)))
8523 = fold_convert (signed_type_for (TREE_TYPE (itercnt
)),
8525 n1
= fold_convert (TREE_TYPE (itercnt
), n1
);
8527 itercnt
= fold_build2 (MINUS_EXPR
, TREE_TYPE (itercnt
),
8529 itercnt
= fold_build2 (EXACT_DIV_EXPR
, TREE_TYPE (itercnt
),
8530 itercnt
, fd
->loop
.step
);
8531 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
8533 GSI_CONTINUE_LINKING
);
8535 a
= fold_build2 (MULT_EXPR
, type
,
8536 fold_convert (type
, itercnt
),
8537 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
8538 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
8539 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), v
, a
);
8540 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8541 false, GSI_CONTINUE_LINKING
);
8542 assign_stmt
= gimple_build_assign (dest
, t
);
8543 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8545 if (fd
->collapse
> 1)
8546 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
8550 /* Until now, counts array contained number of iterations or
8551 variable containing it for ith loop. From now on, we need
8552 those counts only for collapsed loops, and only for the 2nd
8553 till the last collapsed one. Move those one element earlier,
8554 we'll use counts[fd->collapse - 1] for the first source/sink
8555 iteration counter and so on and counts[fd->ordered]
8556 as the array holding the current counter values for
8558 if (fd
->collapse
> 1)
8559 memmove (counts
, counts
+ 1, (fd
->collapse
- 1) * sizeof (counts
[0]));
8563 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
8565 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8567 = fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
8568 fold_convert (type
, fd
->loops
[i
].n1
),
8569 fold_convert (type
, fd
->loops
[i
].n2
));
8570 if (!integer_onep (this_cond
))
8573 if (i
< fd
->ordered
)
8576 = create_empty_bb (EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
);
8577 add_bb_to_loop (cont_bb
, l1_bb
->loop_father
);
8578 gimple_stmt_iterator gsi
= gsi_after_labels (cont_bb
);
8579 gimple
*g
= gimple_build_omp_continue (fd
->loop
.v
, fd
->loop
.v
);
8580 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
8581 make_edge (cont_bb
, l3_bb
, EDGE_FALLTHRU
);
8582 make_edge (cont_bb
, l1_bb
, 0);
8583 l2_bb
= create_empty_bb (cont_bb
);
8584 broken_loop
= false;
8587 expand_omp_ordered_source_sink (region
, fd
, counts
, cont_bb
);
8588 cont_bb
= expand_omp_for_ordered_loops (fd
, counts
, cont_bb
, l1_bb
,
8589 ordered_lastprivate
);
8590 if (counts
[fd
->collapse
- 1])
8592 gcc_assert (fd
->collapse
== 1);
8593 gsi
= gsi_last_bb (l0_bb
);
8594 expand_omp_build_assign (&gsi
, counts
[fd
->collapse
- 1],
8596 gsi
= gsi_last_bb (cont_bb
);
8597 t
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, counts
[fd
->collapse
- 1],
8598 build_int_cst (fd
->iter_type
, 1));
8599 expand_omp_build_assign (&gsi
, counts
[fd
->collapse
- 1], t
);
8600 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8601 size_zero_node
, NULL_TREE
, NULL_TREE
);
8602 expand_omp_build_assign (&gsi
, aref
, counts
[fd
->collapse
- 1]);
8603 t
= counts
[fd
->collapse
- 1];
8605 else if (fd
->collapse
> 1)
8609 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[0].v
),
8610 fd
->loops
[0].v
, fd
->loops
[0].n1
);
8611 t
= fold_convert (fd
->iter_type
, t
);
8613 gsi
= gsi_last_bb (l0_bb
);
8614 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8615 size_zero_node
, NULL_TREE
, NULL_TREE
);
8616 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8617 false, GSI_CONTINUE_LINKING
);
8618 expand_omp_build_assign (&gsi
, aref
, t
, true);
8623 /* Code to control the increment and predicate for the sequential
8624 loop goes in the CONT_BB. */
8625 gsi
= gsi_last_bb (cont_bb
);
8626 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
8627 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
8628 vmain
= gimple_omp_continue_control_use (cont_stmt
);
8629 vback
= gimple_omp_continue_control_def (cont_stmt
);
8631 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
8633 if (POINTER_TYPE_P (type
))
8634 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
8636 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
8637 t
= force_gimple_operand_gsi (&gsi
, t
,
8639 && TREE_ADDRESSABLE (vback
),
8640 NULL_TREE
, true, GSI_SAME_STMT
);
8641 assign_stmt
= gimple_build_assign (vback
, t
);
8642 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
8644 if (fd
->ordered
&& counts
[fd
->collapse
- 1] == NULL_TREE
)
8646 if (fd
->collapse
> 1)
8650 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[0].v
),
8651 fd
->loops
[0].v
, fd
->loops
[0].n1
);
8652 t
= fold_convert (fd
->iter_type
, t
);
8654 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
,
8655 counts
[fd
->ordered
], size_zero_node
,
8656 NULL_TREE
, NULL_TREE
);
8657 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8658 true, GSI_SAME_STMT
);
8659 expand_omp_build_assign (&gsi
, aref
, t
);
8662 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
8663 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
8665 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
8666 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
8669 /* Remove GIMPLE_OMP_CONTINUE. */
8670 gsi_remove (&gsi
, true);
8672 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
8673 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
8675 /* Emit code to get the next parallel iteration in L2_BB. */
8676 gsi
= gsi_start_bb (l2_bb
);
8678 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
8679 build_fold_addr_expr (istart0
),
8680 build_fold_addr_expr (iend0
));
8681 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8682 false, GSI_CONTINUE_LINKING
);
8683 if (TREE_TYPE (t
) != boolean_type_node
)
8684 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
8685 t
, build_int_cst (TREE_TYPE (t
), 0));
8686 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
8687 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
8690 /* Add the loop cleanup function. */
8691 gsi
= gsi_last_bb (exit_bb
);
8692 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
8693 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
8694 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
8695 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
8697 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
8698 gcall
*call_stmt
= gimple_build_call (t
, 0);
8699 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
8700 gimple_call_set_lhs (call_stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
8701 gsi_insert_after (&gsi
, call_stmt
, GSI_SAME_STMT
);
8704 tree arr
= counts
[fd
->ordered
];
8705 tree clobber
= build_constructor (TREE_TYPE (arr
), NULL
);
8706 TREE_THIS_VOLATILE (clobber
) = 1;
8707 gsi_insert_after (&gsi
, gimple_build_assign (arr
, clobber
),
8710 gsi_remove (&gsi
, true);
8712 /* Connect the new blocks. */
8713 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
8714 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
8720 e
= find_edge (cont_bb
, l3_bb
);
8721 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
8723 phis
= phi_nodes (l3_bb
);
8724 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
8726 gimple
*phi
= gsi_stmt (gsi
);
8727 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
8728 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
8732 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
8733 e
= find_edge (cont_bb
, l1_bb
);
8736 e
= BRANCH_EDGE (cont_bb
);
8737 gcc_assert (single_succ (e
->dest
) == l1_bb
);
8739 if (gimple_omp_for_combined_p (fd
->for_stmt
))
8744 else if (fd
->collapse
> 1)
8747 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
8750 e
->flags
= EDGE_TRUE_VALUE
;
8753 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
8754 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
8758 e
= find_edge (cont_bb
, l2_bb
);
8759 e
->flags
= EDGE_FALLTHRU
;
8761 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
8763 if (gimple_in_ssa_p (cfun
))
8765 /* Add phis to the outer loop that connect to the phis in the inner,
8766 original loop, and move the loop entry value of the inner phi to
8767 the loop entry value of the outer phi. */
8769 for (psi
= gsi_start_phis (l3_bb
); !gsi_end_p (psi
); gsi_next (&psi
))
8771 source_location locus
;
8773 gphi
*exit_phi
= psi
.phi ();
8775 edge l2_to_l3
= find_edge (l2_bb
, l3_bb
);
8776 tree exit_res
= PHI_ARG_DEF_FROM_EDGE (exit_phi
, l2_to_l3
);
8778 basic_block latch
= BRANCH_EDGE (cont_bb
)->dest
;
8779 edge latch_to_l1
= find_edge (latch
, l1_bb
);
8781 = find_phi_with_arg_on_edge (exit_res
, latch_to_l1
);
8783 tree t
= gimple_phi_result (exit_phi
);
8784 tree new_res
= copy_ssa_name (t
, NULL
);
8785 nphi
= create_phi_node (new_res
, l0_bb
);
8787 edge l0_to_l1
= find_edge (l0_bb
, l1_bb
);
8788 t
= PHI_ARG_DEF_FROM_EDGE (inner_phi
, l0_to_l1
);
8789 locus
= gimple_phi_arg_location_from_edge (inner_phi
, l0_to_l1
);
8790 edge entry_to_l0
= find_edge (entry_bb
, l0_bb
);
8791 add_phi_arg (nphi
, t
, entry_to_l0
, locus
);
8793 edge l2_to_l0
= find_edge (l2_bb
, l0_bb
);
8794 add_phi_arg (nphi
, exit_res
, l2_to_l0
, UNKNOWN_LOCATION
);
8796 add_phi_arg (inner_phi
, new_res
, l0_to_l1
, UNKNOWN_LOCATION
);
8800 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
8801 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
8802 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
8803 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
8804 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
8805 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
8806 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
8807 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
8809 /* We enter expand_omp_for_generic with a loop. This original loop may
8810 have its own loop struct, or it may be part of an outer loop struct
8811 (which may be the fake loop). */
8812 struct loop
*outer_loop
= entry_bb
->loop_father
;
8813 bool orig_loop_has_loop_struct
= l1_bb
->loop_father
!= outer_loop
;
8815 add_bb_to_loop (l2_bb
, outer_loop
);
8817 /* We've added a new loop around the original loop. Allocate the
8818 corresponding loop struct. */
8819 struct loop
*new_loop
= alloc_loop ();
8820 new_loop
->header
= l0_bb
;
8821 new_loop
->latch
= l2_bb
;
8822 add_loop (new_loop
, outer_loop
);
8824 /* Allocate a loop structure for the original loop unless we already
8826 if (!orig_loop_has_loop_struct
8827 && !gimple_omp_for_combined_p (fd
->for_stmt
))
8829 struct loop
*orig_loop
= alloc_loop ();
8830 orig_loop
->header
= l1_bb
;
8831 /* The loop may have multiple latches. */
8832 add_loop (orig_loop
, new_loop
);
8838 /* A subroutine of expand_omp_for. Generate code for a parallel
8839 loop with static schedule and no specified chunk size. Given
8842 for (V = N1; V cond N2; V += STEP) BODY;
8844 where COND is "<" or ">", we generate pseudocode
8846 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
8851 if ((__typeof (V)) -1 > 0 && cond is >)
8852 n = -(adj + N2 - N1) / -STEP;
8854 n = (adj + N2 - N1) / STEP;
8857 if (threadid < tt) goto L3; else goto L4;
8862 s0 = q * threadid + tt;
8865 if (s0 >= e0) goto L2; else goto L0;
8871 if (V cond e) goto L1;
8876 expand_omp_for_static_nochunk (struct omp_region
*region
,
8877 struct omp_for_data
*fd
,
8880 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
8881 tree type
, itype
, vmain
, vback
;
8882 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
8883 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
8885 gimple_stmt_iterator gsi
;
8887 bool broken_loop
= region
->cont
== NULL
;
8888 tree
*counts
= NULL
;
8891 itype
= type
= TREE_TYPE (fd
->loop
.v
);
8892 if (POINTER_TYPE_P (type
))
8893 itype
= signed_type_for (type
);
8895 entry_bb
= region
->entry
;
8896 cont_bb
= region
->cont
;
8897 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
8898 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
8899 gcc_assert (broken_loop
8900 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
8901 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
8902 body_bb
= single_succ (seq_start_bb
);
8905 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
8906 || single_succ (BRANCH_EDGE (cont_bb
)->dest
) == body_bb
);
8907 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
8909 exit_bb
= region
->exit
;
8911 /* Iteration space partitioning goes in ENTRY_BB. */
8912 gsi
= gsi_last_bb (entry_bb
);
8913 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
8915 if (fd
->collapse
> 1)
8917 int first_zero_iter
= -1, dummy
= -1;
8918 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
8920 counts
= XALLOCAVEC (tree
, fd
->collapse
);
8921 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
8922 fin_bb
, first_zero_iter
,
8923 dummy_bb
, dummy
, l2_dom_bb
);
8926 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
8927 t
= integer_one_node
;
8929 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
8930 fold_convert (type
, fd
->loop
.n1
),
8931 fold_convert (type
, fd
->loop
.n2
));
8932 if (fd
->collapse
== 1
8933 && TYPE_UNSIGNED (type
)
8934 && (t
== NULL_TREE
|| !integer_onep (t
)))
8936 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
8937 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
8938 true, GSI_SAME_STMT
);
8939 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
8940 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
8941 true, GSI_SAME_STMT
);
8942 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
8943 NULL_TREE
, NULL_TREE
);
8944 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
8945 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
8946 expand_omp_regimplify_p
, NULL
, NULL
)
8947 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
8948 expand_omp_regimplify_p
, NULL
, NULL
))
8950 gsi
= gsi_for_stmt (cond_stmt
);
8951 gimple_regimplify_operands (cond_stmt
, &gsi
);
8953 ep
= split_block (entry_bb
, cond_stmt
);
8954 ep
->flags
= EDGE_TRUE_VALUE
;
8955 entry_bb
= ep
->dest
;
8956 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
8957 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
8958 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
8959 if (gimple_in_ssa_p (cfun
))
8961 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
8962 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
8963 !gsi_end_p (gpi
); gsi_next (&gpi
))
8965 gphi
*phi
= gpi
.phi ();
8966 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
8967 ep
, UNKNOWN_LOCATION
);
8970 gsi
= gsi_last_bb (entry_bb
);
8973 switch (gimple_omp_for_kind (fd
->for_stmt
))
8975 case GF_OMP_FOR_KIND_FOR
:
8976 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
8977 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
8979 case GF_OMP_FOR_KIND_DISTRIBUTE
:
8980 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
8981 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
8986 nthreads
= build_call_expr (nthreads
, 0);
8987 nthreads
= fold_convert (itype
, nthreads
);
8988 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
8989 true, GSI_SAME_STMT
);
8990 threadid
= build_call_expr (threadid
, 0);
8991 threadid
= fold_convert (itype
, threadid
);
8992 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
8993 true, GSI_SAME_STMT
);
8997 step
= fd
->loop
.step
;
8998 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9000 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
9001 OMP_CLAUSE__LOOPTEMP_
);
9002 gcc_assert (innerc
);
9003 n1
= OMP_CLAUSE_DECL (innerc
);
9004 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9005 OMP_CLAUSE__LOOPTEMP_
);
9006 gcc_assert (innerc
);
9007 n2
= OMP_CLAUSE_DECL (innerc
);
9009 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
9010 true, NULL_TREE
, true, GSI_SAME_STMT
);
9011 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
9012 true, NULL_TREE
, true, GSI_SAME_STMT
);
9013 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
9014 true, NULL_TREE
, true, GSI_SAME_STMT
);
9016 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
9017 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
9018 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
9019 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
9020 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
9021 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
9022 fold_build1 (NEGATE_EXPR
, itype
, t
),
9023 fold_build1 (NEGATE_EXPR
, itype
, step
));
9025 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
9026 t
= fold_convert (itype
, t
);
9027 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
9029 q
= create_tmp_reg (itype
, "q");
9030 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
9031 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
9032 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
9034 tt
= create_tmp_reg (itype
, "tt");
9035 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
9036 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
9037 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
9039 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
9040 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
9041 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
9043 second_bb
= split_block (entry_bb
, cond_stmt
)->dest
;
9044 gsi
= gsi_last_bb (second_bb
);
9045 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9047 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
9049 gassign
*assign_stmt
9050 = gimple_build_assign (q
, PLUS_EXPR
, q
, build_int_cst (itype
, 1));
9051 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9053 third_bb
= split_block (second_bb
, assign_stmt
)->dest
;
9054 gsi
= gsi_last_bb (third_bb
);
9055 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9057 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
9058 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
9059 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
9061 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
9062 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
9064 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
9065 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
9067 /* Remove the GIMPLE_OMP_FOR statement. */
9068 gsi_remove (&gsi
, true);
9070 /* Setup code for sequential iteration goes in SEQ_START_BB. */
9071 gsi
= gsi_start_bb (seq_start_bb
);
9073 tree startvar
= fd
->loop
.v
;
9074 tree endvar
= NULL_TREE
;
9076 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9078 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
9079 ? gimple_omp_parallel_clauses (inner_stmt
)
9080 : gimple_omp_for_clauses (inner_stmt
);
9081 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
9082 gcc_assert (innerc
);
9083 startvar
= OMP_CLAUSE_DECL (innerc
);
9084 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9085 OMP_CLAUSE__LOOPTEMP_
);
9086 gcc_assert (innerc
);
9087 endvar
= OMP_CLAUSE_DECL (innerc
);
9088 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
9089 && gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
9092 for (i
= 1; i
< fd
->collapse
; i
++)
9094 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9095 OMP_CLAUSE__LOOPTEMP_
);
9096 gcc_assert (innerc
);
9098 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9099 OMP_CLAUSE__LOOPTEMP_
);
9102 /* If needed (distribute parallel for with lastprivate),
9103 propagate down the total number of iterations. */
9104 tree t
= fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (innerc
)),
9106 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, false,
9107 GSI_CONTINUE_LINKING
);
9108 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
9109 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9113 t
= fold_convert (itype
, s0
);
9114 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9115 if (POINTER_TYPE_P (type
))
9116 t
= fold_build_pointer_plus (n1
, t
);
9118 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9119 t
= fold_convert (TREE_TYPE (startvar
), t
);
9120 t
= force_gimple_operand_gsi (&gsi
, t
,
9122 && TREE_ADDRESSABLE (startvar
),
9123 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
9124 assign_stmt
= gimple_build_assign (startvar
, t
);
9125 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9127 t
= fold_convert (itype
, e0
);
9128 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9129 if (POINTER_TYPE_P (type
))
9130 t
= fold_build_pointer_plus (n1
, t
);
9132 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9133 t
= fold_convert (TREE_TYPE (startvar
), t
);
9134 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9135 false, GSI_CONTINUE_LINKING
);
9138 assign_stmt
= gimple_build_assign (endvar
, e
);
9139 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9140 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
9141 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
9143 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
9144 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9146 /* Handle linear clause adjustments. */
9147 tree itercnt
= NULL_TREE
;
9148 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
9149 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
9150 c
; c
= OMP_CLAUSE_CHAIN (c
))
9151 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
9152 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
9154 tree d
= OMP_CLAUSE_DECL (c
);
9155 bool is_ref
= is_reference (d
);
9156 tree t
= d
, a
, dest
;
9158 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
9159 if (itercnt
== NULL_TREE
)
9161 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9163 itercnt
= fold_build2 (MINUS_EXPR
, itype
,
9164 fold_convert (itype
, n1
),
9165 fold_convert (itype
, fd
->loop
.n1
));
9166 itercnt
= fold_build2 (EXACT_DIV_EXPR
, itype
, itercnt
, step
);
9167 itercnt
= fold_build2 (PLUS_EXPR
, itype
, itercnt
, s0
);
9168 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
9170 GSI_CONTINUE_LINKING
);
9175 tree type
= TREE_TYPE (t
);
9176 if (POINTER_TYPE_P (type
))
9178 a
= fold_build2 (MULT_EXPR
, type
,
9179 fold_convert (type
, itercnt
),
9180 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
9181 dest
= unshare_expr (t
);
9182 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
9183 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), t
, a
);
9184 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9185 false, GSI_CONTINUE_LINKING
);
9186 assign_stmt
= gimple_build_assign (dest
, t
);
9187 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9189 if (fd
->collapse
> 1)
9190 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
9194 /* The code controlling the sequential loop replaces the
9195 GIMPLE_OMP_CONTINUE. */
9196 gsi
= gsi_last_bb (cont_bb
);
9197 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
9198 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
9199 vmain
= gimple_omp_continue_control_use (cont_stmt
);
9200 vback
= gimple_omp_continue_control_def (cont_stmt
);
9202 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
9204 if (POINTER_TYPE_P (type
))
9205 t
= fold_build_pointer_plus (vmain
, step
);
9207 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
9208 t
= force_gimple_operand_gsi (&gsi
, t
,
9210 && TREE_ADDRESSABLE (vback
),
9211 NULL_TREE
, true, GSI_SAME_STMT
);
9212 assign_stmt
= gimple_build_assign (vback
, t
);
9213 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9215 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
9216 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
9218 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
9221 /* Remove the GIMPLE_OMP_CONTINUE statement. */
9222 gsi_remove (&gsi
, true);
9224 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
9225 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
9228 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
9229 gsi
= gsi_last_bb (exit_bb
);
9230 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
9232 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
9233 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
9235 gsi_remove (&gsi
, true);
9237 /* Connect all the blocks. */
9238 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
9239 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
9240 ep
= find_edge (entry_bb
, second_bb
);
9241 ep
->flags
= EDGE_TRUE_VALUE
;
9242 ep
->probability
= REG_BR_PROB_BASE
/ 4;
9243 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
9244 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
9248 ep
= find_edge (cont_bb
, body_bb
);
9251 ep
= BRANCH_EDGE (cont_bb
);
9252 gcc_assert (single_succ (ep
->dest
) == body_bb
);
9254 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9259 else if (fd
->collapse
> 1)
9262 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
9265 ep
->flags
= EDGE_TRUE_VALUE
;
9266 find_edge (cont_bb
, fin_bb
)->flags
9267 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
9270 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
9271 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
9272 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
9274 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
9275 recompute_dominator (CDI_DOMINATORS
, body_bb
));
9276 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
9277 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
9279 struct loop
*loop
= body_bb
->loop_father
;
9280 if (loop
!= entry_bb
->loop_father
)
9282 gcc_assert (loop
->header
== body_bb
);
9283 gcc_assert (broken_loop
9284 || loop
->latch
== region
->cont
9285 || single_pred (loop
->latch
) == region
->cont
);
9289 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
9291 loop
= alloc_loop ();
9292 loop
->header
= body_bb
;
9293 if (collapse_bb
== NULL
)
9294 loop
->latch
= cont_bb
;
9295 add_loop (loop
, body_bb
->loop_father
);
9299 /* Return phi in E->DEST with ARG on edge E. */
9302 find_phi_with_arg_on_edge (tree arg
, edge e
)
9304 basic_block bb
= e
->dest
;
9306 for (gphi_iterator gpi
= gsi_start_phis (bb
);
9310 gphi
*phi
= gpi
.phi ();
9311 if (PHI_ARG_DEF_FROM_EDGE (phi
, e
) == arg
)
9318 /* A subroutine of expand_omp_for. Generate code for a parallel
9319 loop with static schedule and a specified chunk size. Given
9322 for (V = N1; V cond N2; V += STEP) BODY;
9324 where COND is "<" or ">", we generate pseudocode
9326 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
9331 if ((__typeof (V)) -1 > 0 && cond is >)
9332 n = -(adj + N2 - N1) / -STEP;
9334 n = (adj + N2 - N1) / STEP;
9336 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
9337 here so that V is defined
9338 if the loop is not entered
9340 s0 = (trip * nthreads + threadid) * CHUNK;
9341 e0 = min(s0 + CHUNK, n);
9342 if (s0 < n) goto L1; else goto L4;
9349 if (V cond e) goto L2; else goto L3;
9357 expand_omp_for_static_chunk (struct omp_region
*region
,
9358 struct omp_for_data
*fd
, gimple
*inner_stmt
)
9360 tree n
, s0
, e0
, e
, t
;
9361 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
9362 tree type
, itype
, vmain
, vback
, vextra
;
9363 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
9364 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
9365 gimple_stmt_iterator gsi
;
9367 bool broken_loop
= region
->cont
== NULL
;
9368 tree
*counts
= NULL
;
9371 itype
= type
= TREE_TYPE (fd
->loop
.v
);
9372 if (POINTER_TYPE_P (type
))
9373 itype
= signed_type_for (type
);
9375 entry_bb
= region
->entry
;
9376 se
= split_block (entry_bb
, last_stmt (entry_bb
));
9378 iter_part_bb
= se
->dest
;
9379 cont_bb
= region
->cont
;
9380 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
9381 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
9382 gcc_assert (broken_loop
9383 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
9384 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
9385 body_bb
= single_succ (seq_start_bb
);
9388 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
9389 || single_succ (BRANCH_EDGE (cont_bb
)->dest
) == body_bb
);
9390 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
9391 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
9393 exit_bb
= region
->exit
;
9395 /* Trip and adjustment setup goes in ENTRY_BB. */
9396 gsi
= gsi_last_bb (entry_bb
);
9397 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9399 if (fd
->collapse
> 1)
9401 int first_zero_iter
= -1, dummy
= -1;
9402 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
9404 counts
= XALLOCAVEC (tree
, fd
->collapse
);
9405 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
9406 fin_bb
, first_zero_iter
,
9407 dummy_bb
, dummy
, l2_dom_bb
);
9410 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9411 t
= integer_one_node
;
9413 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
9414 fold_convert (type
, fd
->loop
.n1
),
9415 fold_convert (type
, fd
->loop
.n2
));
9416 if (fd
->collapse
== 1
9417 && TYPE_UNSIGNED (type
)
9418 && (t
== NULL_TREE
|| !integer_onep (t
)))
9420 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
9421 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
9422 true, GSI_SAME_STMT
);
9423 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
9424 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
9425 true, GSI_SAME_STMT
);
9426 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
9427 NULL_TREE
, NULL_TREE
);
9428 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
9429 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
9430 expand_omp_regimplify_p
, NULL
, NULL
)
9431 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
9432 expand_omp_regimplify_p
, NULL
, NULL
))
9434 gsi
= gsi_for_stmt (cond_stmt
);
9435 gimple_regimplify_operands (cond_stmt
, &gsi
);
9437 se
= split_block (entry_bb
, cond_stmt
);
9438 se
->flags
= EDGE_TRUE_VALUE
;
9439 entry_bb
= se
->dest
;
9440 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
9441 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
9442 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
9443 if (gimple_in_ssa_p (cfun
))
9445 int dest_idx
= find_edge (iter_part_bb
, fin_bb
)->dest_idx
;
9446 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
9447 !gsi_end_p (gpi
); gsi_next (&gpi
))
9449 gphi
*phi
= gpi
.phi ();
9450 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
9451 se
, UNKNOWN_LOCATION
);
9454 gsi
= gsi_last_bb (entry_bb
);
9457 switch (gimple_omp_for_kind (fd
->for_stmt
))
9459 case GF_OMP_FOR_KIND_FOR
:
9460 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
9461 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
9463 case GF_OMP_FOR_KIND_DISTRIBUTE
:
9464 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
9465 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
9470 nthreads
= build_call_expr (nthreads
, 0);
9471 nthreads
= fold_convert (itype
, nthreads
);
9472 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
9473 true, GSI_SAME_STMT
);
9474 threadid
= build_call_expr (threadid
, 0);
9475 threadid
= fold_convert (itype
, threadid
);
9476 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
9477 true, GSI_SAME_STMT
);
9481 step
= fd
->loop
.step
;
9482 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9484 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
9485 OMP_CLAUSE__LOOPTEMP_
);
9486 gcc_assert (innerc
);
9487 n1
= OMP_CLAUSE_DECL (innerc
);
9488 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9489 OMP_CLAUSE__LOOPTEMP_
);
9490 gcc_assert (innerc
);
9491 n2
= OMP_CLAUSE_DECL (innerc
);
9493 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
9494 true, NULL_TREE
, true, GSI_SAME_STMT
);
9495 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
9496 true, NULL_TREE
, true, GSI_SAME_STMT
);
9497 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
9498 true, NULL_TREE
, true, GSI_SAME_STMT
);
9499 tree chunk_size
= fold_convert (itype
, fd
->chunk_size
);
9500 chunk_size
= omp_adjust_chunk_size (chunk_size
, fd
->simd_schedule
);
9502 = force_gimple_operand_gsi (&gsi
, chunk_size
, true, NULL_TREE
, true,
9505 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
9506 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
9507 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
9508 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
9509 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
9510 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
9511 fold_build1 (NEGATE_EXPR
, itype
, t
),
9512 fold_build1 (NEGATE_EXPR
, itype
, step
));
9514 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
9515 t
= fold_convert (itype
, t
);
9516 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9517 true, GSI_SAME_STMT
);
9519 trip_var
= create_tmp_reg (itype
, ".trip");
9520 if (gimple_in_ssa_p (cfun
))
9522 trip_init
= make_ssa_name (trip_var
);
9523 trip_main
= make_ssa_name (trip_var
);
9524 trip_back
= make_ssa_name (trip_var
);
9528 trip_init
= trip_var
;
9529 trip_main
= trip_var
;
9530 trip_back
= trip_var
;
9533 gassign
*assign_stmt
9534 = gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
9535 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9537 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, chunk_size
);
9538 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9539 if (POINTER_TYPE_P (type
))
9540 t
= fold_build_pointer_plus (n1
, t
);
9542 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9543 vextra
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9544 true, GSI_SAME_STMT
);
9546 /* Remove the GIMPLE_OMP_FOR. */
9547 gsi_remove (&gsi
, true);
9549 gimple_stmt_iterator gsif
= gsi
;
9551 /* Iteration space partitioning goes in ITER_PART_BB. */
9552 gsi
= gsi_last_bb (iter_part_bb
);
9554 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
9555 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
9556 t
= fold_build2 (MULT_EXPR
, itype
, t
, chunk_size
);
9557 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9558 false, GSI_CONTINUE_LINKING
);
9560 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, chunk_size
);
9561 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
9562 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9563 false, GSI_CONTINUE_LINKING
);
9565 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
9566 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
9568 /* Setup code for sequential iteration goes in SEQ_START_BB. */
9569 gsi
= gsi_start_bb (seq_start_bb
);
9571 tree startvar
= fd
->loop
.v
;
9572 tree endvar
= NULL_TREE
;
9574 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9576 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
9577 ? gimple_omp_parallel_clauses (inner_stmt
)
9578 : gimple_omp_for_clauses (inner_stmt
);
9579 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
9580 gcc_assert (innerc
);
9581 startvar
= OMP_CLAUSE_DECL (innerc
);
9582 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9583 OMP_CLAUSE__LOOPTEMP_
);
9584 gcc_assert (innerc
);
9585 endvar
= OMP_CLAUSE_DECL (innerc
);
9586 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
9587 && gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
9590 for (i
= 1; i
< fd
->collapse
; i
++)
9592 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9593 OMP_CLAUSE__LOOPTEMP_
);
9594 gcc_assert (innerc
);
9596 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9597 OMP_CLAUSE__LOOPTEMP_
);
9600 /* If needed (distribute parallel for with lastprivate),
9601 propagate down the total number of iterations. */
9602 tree t
= fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (innerc
)),
9604 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, false,
9605 GSI_CONTINUE_LINKING
);
9606 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
9607 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9612 t
= fold_convert (itype
, s0
);
9613 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9614 if (POINTER_TYPE_P (type
))
9615 t
= fold_build_pointer_plus (n1
, t
);
9617 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9618 t
= fold_convert (TREE_TYPE (startvar
), t
);
9619 t
= force_gimple_operand_gsi (&gsi
, t
,
9621 && TREE_ADDRESSABLE (startvar
),
9622 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
9623 assign_stmt
= gimple_build_assign (startvar
, t
);
9624 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9626 t
= fold_convert (itype
, e0
);
9627 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9628 if (POINTER_TYPE_P (type
))
9629 t
= fold_build_pointer_plus (n1
, t
);
9631 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9632 t
= fold_convert (TREE_TYPE (startvar
), t
);
9633 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9634 false, GSI_CONTINUE_LINKING
);
9637 assign_stmt
= gimple_build_assign (endvar
, e
);
9638 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9639 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
9640 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
9642 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
9643 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9645 /* Handle linear clause adjustments. */
9646 tree itercnt
= NULL_TREE
, itercntbias
= NULL_TREE
;
9647 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
9648 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
9649 c
; c
= OMP_CLAUSE_CHAIN (c
))
9650 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
9651 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
9653 tree d
= OMP_CLAUSE_DECL (c
);
9654 bool is_ref
= is_reference (d
);
9655 tree t
= d
, a
, dest
;
9657 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
9658 tree type
= TREE_TYPE (t
);
9659 if (POINTER_TYPE_P (type
))
9661 dest
= unshare_expr (t
);
9662 tree v
= create_tmp_var (TREE_TYPE (t
), NULL
);
9663 expand_omp_build_assign (&gsif
, v
, t
);
9664 if (itercnt
== NULL_TREE
)
9666 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9669 = fold_build2 (MINUS_EXPR
, itype
, fold_convert (itype
, n1
),
9670 fold_convert (itype
, fd
->loop
.n1
));
9671 itercntbias
= fold_build2 (EXACT_DIV_EXPR
, itype
,
9674 = force_gimple_operand_gsi (&gsif
, itercntbias
, true,
9677 itercnt
= fold_build2 (PLUS_EXPR
, itype
, itercntbias
, s0
);
9678 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
9680 GSI_CONTINUE_LINKING
);
9685 a
= fold_build2 (MULT_EXPR
, type
,
9686 fold_convert (type
, itercnt
),
9687 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
9688 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
9689 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), v
, a
);
9690 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9691 false, GSI_CONTINUE_LINKING
);
9692 assign_stmt
= gimple_build_assign (dest
, t
);
9693 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9695 if (fd
->collapse
> 1)
9696 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
9700 /* The code controlling the sequential loop goes in CONT_BB,
9701 replacing the GIMPLE_OMP_CONTINUE. */
9702 gsi
= gsi_last_bb (cont_bb
);
9703 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
9704 vmain
= gimple_omp_continue_control_use (cont_stmt
);
9705 vback
= gimple_omp_continue_control_def (cont_stmt
);
9707 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
9709 if (POINTER_TYPE_P (type
))
9710 t
= fold_build_pointer_plus (vmain
, step
);
9712 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
9713 if (DECL_P (vback
) && TREE_ADDRESSABLE (vback
))
9714 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9715 true, GSI_SAME_STMT
);
9716 assign_stmt
= gimple_build_assign (vback
, t
);
9717 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9719 if (tree_int_cst_equal (fd
->chunk_size
, integer_one_node
))
9720 t
= build2 (EQ_EXPR
, boolean_type_node
,
9721 build_int_cst (itype
, 0),
9722 build_int_cst (itype
, 1));
9724 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
9725 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
9727 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
9730 /* Remove GIMPLE_OMP_CONTINUE. */
9731 gsi_remove (&gsi
, true);
9733 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
9734 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
9736 /* Trip update code goes into TRIP_UPDATE_BB. */
9737 gsi
= gsi_start_bb (trip_update_bb
);
9739 t
= build_int_cst (itype
, 1);
9740 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
9741 assign_stmt
= gimple_build_assign (trip_back
, t
);
9742 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9745 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
9746 gsi
= gsi_last_bb (exit_bb
);
9747 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
9749 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
9750 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
9752 gsi_remove (&gsi
, true);
9754 /* Connect the new blocks. */
9755 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
9756 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
9760 se
= find_edge (cont_bb
, body_bb
);
9763 se
= BRANCH_EDGE (cont_bb
);
9764 gcc_assert (single_succ (se
->dest
) == body_bb
);
9766 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9771 else if (fd
->collapse
> 1)
9774 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
9777 se
->flags
= EDGE_TRUE_VALUE
;
9778 find_edge (cont_bb
, trip_update_bb
)->flags
9779 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
9781 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
9784 if (gimple_in_ssa_p (cfun
))
9792 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
9794 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
9795 remove arguments of the phi nodes in fin_bb. We need to create
9796 appropriate phi nodes in iter_part_bb instead. */
9797 se
= find_edge (iter_part_bb
, fin_bb
);
9798 re
= single_succ_edge (trip_update_bb
);
9799 vec
<edge_var_map
> *head
= redirect_edge_var_map_vector (re
);
9800 ene
= single_succ_edge (entry_bb
);
9802 psi
= gsi_start_phis (fin_bb
);
9803 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
9804 gsi_next (&psi
), ++i
)
9807 source_location locus
;
9810 t
= gimple_phi_result (phi
);
9811 gcc_assert (t
== redirect_edge_var_map_result (vm
));
9813 if (!single_pred_p (fin_bb
))
9814 t
= copy_ssa_name (t
, phi
);
9816 nphi
= create_phi_node (t
, iter_part_bb
);
9818 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
9819 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
9821 /* A special case -- fd->loop.v is not yet computed in
9822 iter_part_bb, we need to use vextra instead. */
9823 if (t
== fd
->loop
.v
)
9825 add_phi_arg (nphi
, t
, ene
, locus
);
9826 locus
= redirect_edge_var_map_location (vm
);
9827 tree back_arg
= redirect_edge_var_map_def (vm
);
9828 add_phi_arg (nphi
, back_arg
, re
, locus
);
9829 edge ce
= find_edge (cont_bb
, body_bb
);
9832 ce
= BRANCH_EDGE (cont_bb
);
9833 gcc_assert (single_succ (ce
->dest
) == body_bb
);
9834 ce
= single_succ_edge (ce
->dest
);
9836 gphi
*inner_loop_phi
= find_phi_with_arg_on_edge (back_arg
, ce
);
9837 gcc_assert (inner_loop_phi
!= NULL
);
9838 add_phi_arg (inner_loop_phi
, gimple_phi_result (nphi
),
9839 find_edge (seq_start_bb
, body_bb
), locus
);
9841 if (!single_pred_p (fin_bb
))
9842 add_phi_arg (phi
, gimple_phi_result (nphi
), se
, locus
);
9844 gcc_assert (gsi_end_p (psi
) && (head
== NULL
|| i
== head
->length ()));
9845 redirect_edge_var_map_clear (re
);
9846 if (single_pred_p (fin_bb
))
9849 psi
= gsi_start_phis (fin_bb
);
9850 if (gsi_end_p (psi
))
9852 remove_phi_node (&psi
, false);
9855 /* Make phi node for trip. */
9856 phi
= create_phi_node (trip_main
, iter_part_bb
);
9857 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
9859 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
9864 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
9865 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
9866 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
9867 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
9868 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
9869 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
9870 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
9871 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
9872 recompute_dominator (CDI_DOMINATORS
, body_bb
));
9876 struct loop
*loop
= body_bb
->loop_father
;
9877 struct loop
*trip_loop
= alloc_loop ();
9878 trip_loop
->header
= iter_part_bb
;
9879 trip_loop
->latch
= trip_update_bb
;
9880 add_loop (trip_loop
, iter_part_bb
->loop_father
);
9882 if (loop
!= entry_bb
->loop_father
)
9884 gcc_assert (loop
->header
== body_bb
);
9885 gcc_assert (loop
->latch
== region
->cont
9886 || single_pred (loop
->latch
) == region
->cont
);
9887 trip_loop
->inner
= loop
;
9891 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
9893 loop
= alloc_loop ();
9894 loop
->header
= body_bb
;
9895 if (collapse_bb
== NULL
)
9896 loop
->latch
= cont_bb
;
9897 add_loop (loop
, trip_loop
);
9902 /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop.
9904 for (V = N1; V cond N2; V += STEP) BODY;
9906 where COND is "<" or ">" or "!=", we generate pseudocode
9908 for (ind_var = low; ind_var < high; ind_var++)
9910 V = n1 + (ind_var * STEP)
9915 In the above pseudocode, low and high are function parameters of the
9916 child function. In the function below, we are inserting a temp.
9917 variable that will be making a call to two OMP functions that will not be
9918 found in the body of _Cilk_for (since OMP_FOR cannot be mixed
9919 with _Cilk_for). These functions are replaced with low and high
9920 by the function that handles taskreg. */
9924 expand_cilk_for (struct omp_region
*region
, struct omp_for_data
*fd
)
9926 bool broken_loop
= region
->cont
== NULL
;
9927 basic_block entry_bb
= region
->entry
;
9928 basic_block cont_bb
= region
->cont
;
9930 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
9931 gcc_assert (broken_loop
9932 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
9933 basic_block l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
9934 basic_block l1_bb
, l2_bb
;
9938 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
9939 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
9940 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
9941 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
9945 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
9946 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
9947 l2_bb
= single_succ (l1_bb
);
9949 basic_block exit_bb
= region
->exit
;
9950 basic_block l2_dom_bb
= NULL
;
9952 gimple_stmt_iterator gsi
= gsi_last_bb (entry_bb
);
9954 /* Below statements until the "tree high_val = ..." are pseudo statements
9955 used to pass information to be used by expand_omp_taskreg.
9956 low_val and high_val will be replaced by the __low and __high
9957 parameter from the child function.
9959 The call_exprs part is a place-holder, it is mainly used
9960 to distinctly identify to the top-level part that this is
9961 where we should put low and high (reasoning given in header
9965 = gimple_omp_parallel_child_fn (
9966 as_a
<gomp_parallel
*> (last_stmt (region
->outer
->entry
)));
9967 tree t
, low_val
= NULL_TREE
, high_val
= NULL_TREE
;
9968 for (t
= DECL_ARGUMENTS (child_fndecl
); t
; t
= TREE_CHAIN (t
))
9970 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__high"))
9972 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__low"))
9975 gcc_assert (low_val
&& high_val
);
9977 tree type
= TREE_TYPE (low_val
);
9978 tree ind_var
= create_tmp_reg (type
, "__cilk_ind_var");
9979 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9981 /* Not needed in SSA form right now. */
9982 gcc_assert (!gimple_in_ssa_p (cfun
));
9983 if (l2_dom_bb
== NULL
)
9989 gimple
*stmt
= gimple_build_assign (ind_var
, n1
);
9991 /* Replace the GIMPLE_OMP_FOR statement. */
9992 gsi_replace (&gsi
, stmt
, true);
9996 /* Code to control the increment goes in the CONT_BB. */
9997 gsi
= gsi_last_bb (cont_bb
);
9998 stmt
= gsi_stmt (gsi
);
9999 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
10000 stmt
= gimple_build_assign (ind_var
, PLUS_EXPR
, ind_var
,
10001 build_one_cst (type
));
10003 /* Replace GIMPLE_OMP_CONTINUE. */
10004 gsi_replace (&gsi
, stmt
, true);
10007 /* Emit the condition in L1_BB. */
10008 gsi
= gsi_after_labels (l1_bb
);
10009 t
= fold_build2 (MULT_EXPR
, TREE_TYPE (fd
->loop
.step
),
10010 fold_convert (TREE_TYPE (fd
->loop
.step
), ind_var
),
10012 if (POINTER_TYPE_P (TREE_TYPE (fd
->loop
.n1
)))
10013 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
10014 fd
->loop
.n1
, fold_convert (sizetype
, t
));
10016 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
10017 fd
->loop
.n1
, fold_convert (TREE_TYPE (fd
->loop
.n1
), t
));
10018 t
= fold_convert (TREE_TYPE (fd
->loop
.v
), t
);
10019 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
10021 /* The condition is always '<' since the runtime will fill in the low
10022 and high values. */
10023 stmt
= gimple_build_cond (LT_EXPR
, ind_var
, n2
, NULL_TREE
, NULL_TREE
);
10024 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
10026 /* Remove GIMPLE_OMP_RETURN. */
10027 gsi
= gsi_last_bb (exit_bb
);
10028 gsi_remove (&gsi
, true);
10030 /* Connect the new blocks. */
10031 remove_edge (FALLTHRU_EDGE (entry_bb
));
10036 remove_edge (BRANCH_EDGE (entry_bb
));
10037 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
10039 e
= BRANCH_EDGE (l1_bb
);
10040 ne
= FALLTHRU_EDGE (l1_bb
);
10041 e
->flags
= EDGE_TRUE_VALUE
;
10045 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
10047 ne
= single_succ_edge (l1_bb
);
10048 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
10051 ne
->flags
= EDGE_FALSE_VALUE
;
10052 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
10053 ne
->probability
= REG_BR_PROB_BASE
/ 8;
10055 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
10056 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
10057 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
10061 struct loop
*loop
= alloc_loop ();
10062 loop
->header
= l1_bb
;
10063 loop
->latch
= cont_bb
;
10064 add_loop (loop
, l1_bb
->loop_father
);
10065 loop
->safelen
= INT_MAX
;
10068 /* Pick the correct library function based on the precision of the
10069 induction variable type. */
10070 tree lib_fun
= NULL_TREE
;
10071 if (TYPE_PRECISION (type
) == 32)
10072 lib_fun
= cilk_for_32_fndecl
;
10073 else if (TYPE_PRECISION (type
) == 64)
10074 lib_fun
= cilk_for_64_fndecl
;
10076 gcc_unreachable ();
10078 gcc_assert (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_CILKFOR
);
10080 /* WS_ARGS contains the library function flavor to call:
10081 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the
10082 user-defined grain value. If the user does not define one, then zero
10083 is passed in by the parser. */
10084 vec_alloc (region
->ws_args
, 2);
10085 region
->ws_args
->quick_push (lib_fun
);
10086 region
->ws_args
->quick_push (fd
->chunk_size
);
10089 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
10090 loop. Given parameters:
10092 for (V = N1; V cond N2; V += STEP) BODY;
10094 where COND is "<" or ">", we generate pseudocode
10102 if (V cond N2) goto L0; else goto L2;
10105 For collapsed loops, given parameters:
10107 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
10108 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
10109 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
10112 we generate pseudocode
10118 count3 = (adj + N32 - N31) / STEP3;
10123 count2 = (adj + N22 - N21) / STEP2;
10128 count1 = (adj + N12 - N11) / STEP1;
10129 count = count1 * count2 * count3;
10139 V2 += (V3 cond3 N32) ? 0 : STEP2;
10140 V3 = (V3 cond3 N32) ? V3 : N31;
10141 V1 += (V2 cond2 N22) ? 0 : STEP1;
10142 V2 = (V2 cond2 N22) ? V2 : N21;
10144 if (V < count) goto L0; else goto L2;
10150 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
10153 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
10154 gimple_stmt_iterator gsi
;
10157 bool broken_loop
= region
->cont
== NULL
;
10159 tree
*counts
= NULL
;
10161 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10162 OMP_CLAUSE_SAFELEN
);
10163 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10164 OMP_CLAUSE__SIMDUID_
);
10167 type
= TREE_TYPE (fd
->loop
.v
);
10168 entry_bb
= region
->entry
;
10169 cont_bb
= region
->cont
;
10170 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10171 gcc_assert (broken_loop
10172 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
10173 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10176 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
10177 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
10178 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
10179 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
10183 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
10184 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
10185 l2_bb
= single_succ (l1_bb
);
10187 exit_bb
= region
->exit
;
10190 gsi
= gsi_last_bb (entry_bb
);
10192 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
10193 /* Not needed in SSA form right now. */
10194 gcc_assert (!gimple_in_ssa_p (cfun
));
10195 if (fd
->collapse
> 1)
10197 int first_zero_iter
= -1, dummy
= -1;
10198 basic_block zero_iter_bb
= l2_bb
, dummy_bb
= NULL
;
10200 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10201 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10202 zero_iter_bb
, first_zero_iter
,
10203 dummy_bb
, dummy
, l2_dom_bb
);
10205 if (l2_dom_bb
== NULL
)
10210 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
10212 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10213 OMP_CLAUSE__LOOPTEMP_
);
10214 gcc_assert (innerc
);
10215 n1
= OMP_CLAUSE_DECL (innerc
);
10216 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10217 OMP_CLAUSE__LOOPTEMP_
);
10218 gcc_assert (innerc
);
10219 n2
= OMP_CLAUSE_DECL (innerc
);
10220 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
10221 fold_convert (type
, n1
));
10222 if (fd
->collapse
> 1)
10225 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
10231 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
10232 fold_convert (type
, fd
->loop
.n1
));
10233 if (fd
->collapse
> 1)
10234 for (i
= 0; i
< fd
->collapse
; i
++)
10236 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
10237 if (POINTER_TYPE_P (itype
))
10238 itype
= signed_type_for (itype
);
10239 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
10240 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10244 /* Remove the GIMPLE_OMP_FOR statement. */
10245 gsi_remove (&gsi
, true);
10249 /* Code to control the increment goes in the CONT_BB. */
10250 gsi
= gsi_last_bb (cont_bb
);
10251 stmt
= gsi_stmt (gsi
);
10252 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
10254 if (POINTER_TYPE_P (type
))
10255 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
10257 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
10258 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
10260 if (fd
->collapse
> 1)
10262 i
= fd
->collapse
- 1;
10263 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
10265 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
10266 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
10270 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
10271 fd
->loops
[i
].step
);
10272 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
10273 fd
->loops
[i
].v
, t
);
10275 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10277 for (i
= fd
->collapse
- 1; i
> 0; i
--)
10279 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
10280 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
10281 if (POINTER_TYPE_P (itype2
))
10282 itype2
= signed_type_for (itype2
);
10283 t
= build3 (COND_EXPR
, itype2
,
10284 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
10286 fold_convert (itype
, fd
->loops
[i
].n2
)),
10287 build_int_cst (itype2
, 0),
10288 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
10289 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
10290 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
10292 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
10293 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
10295 t
= build3 (COND_EXPR
, itype
,
10296 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
10298 fold_convert (itype
, fd
->loops
[i
].n2
)),
10300 fold_convert (itype
, fd
->loops
[i
].n1
));
10301 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10305 /* Remove GIMPLE_OMP_CONTINUE. */
10306 gsi_remove (&gsi
, true);
10309 /* Emit the condition in L1_BB. */
10310 gsi
= gsi_start_bb (l1_bb
);
10312 t
= fold_convert (type
, n2
);
10313 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
10314 false, GSI_CONTINUE_LINKING
);
10315 tree v
= fd
->loop
.v
;
10316 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
10317 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
10318 false, GSI_CONTINUE_LINKING
);
10319 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, v
, t
);
10320 cond_stmt
= gimple_build_cond_empty (t
);
10321 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
10322 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
10324 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
10327 gsi
= gsi_for_stmt (cond_stmt
);
10328 gimple_regimplify_operands (cond_stmt
, &gsi
);
10331 /* Remove GIMPLE_OMP_RETURN. */
10332 gsi
= gsi_last_bb (exit_bb
);
10333 gsi_remove (&gsi
, true);
10335 /* Connect the new blocks. */
10336 remove_edge (FALLTHRU_EDGE (entry_bb
));
10340 remove_edge (BRANCH_EDGE (entry_bb
));
10341 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
10343 e
= BRANCH_EDGE (l1_bb
);
10344 ne
= FALLTHRU_EDGE (l1_bb
);
10345 e
->flags
= EDGE_TRUE_VALUE
;
10349 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
10351 ne
= single_succ_edge (l1_bb
);
10352 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
10355 ne
->flags
= EDGE_FALSE_VALUE
;
10356 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
10357 ne
->probability
= REG_BR_PROB_BASE
/ 8;
10359 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
10360 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
10361 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
10365 struct loop
*loop
= alloc_loop ();
10366 loop
->header
= l1_bb
;
10367 loop
->latch
= cont_bb
;
10368 add_loop (loop
, l1_bb
->loop_father
);
10369 if (safelen
== NULL_TREE
)
10370 loop
->safelen
= INT_MAX
;
10373 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
10374 if (TREE_CODE (safelen
) != INTEGER_CST
)
10376 else if (!tree_fits_uhwi_p (safelen
)
10377 || tree_to_uhwi (safelen
) > INT_MAX
)
10378 loop
->safelen
= INT_MAX
;
10380 loop
->safelen
= tree_to_uhwi (safelen
);
10381 if (loop
->safelen
== 1)
10386 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
10387 cfun
->has_simduid_loops
= true;
10389 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
10391 if ((flag_tree_loop_vectorize
10392 || (!global_options_set
.x_flag_tree_loop_vectorize
10393 && !global_options_set
.x_flag_tree_vectorize
))
10394 && flag_tree_loop_optimize
10395 && loop
->safelen
> 1)
10397 loop
->force_vectorize
= true;
10398 cfun
->has_force_vectorize_loops
= true;
10402 cfun
->has_simduid_loops
= true;
10405 /* Taskloop construct is represented after gimplification with
10406 two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
10407 in between them. This routine expands the outer GIMPLE_OMP_FOR,
10408 which should just compute all the needed loop temporaries
10409 for GIMPLE_OMP_TASK. */
10412 expand_omp_taskloop_for_outer (struct omp_region
*region
,
10413 struct omp_for_data
*fd
,
10414 gimple
*inner_stmt
)
10416 tree type
, bias
= NULL_TREE
;
10417 basic_block entry_bb
, cont_bb
, exit_bb
;
10418 gimple_stmt_iterator gsi
;
10419 gassign
*assign_stmt
;
10420 tree
*counts
= NULL
;
10423 gcc_assert (inner_stmt
);
10424 gcc_assert (region
->cont
);
10425 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_TASK
10426 && gimple_omp_task_taskloop_p (inner_stmt
));
10427 type
= TREE_TYPE (fd
->loop
.v
);
10429 /* See if we need to bias by LLONG_MIN. */
10430 if (fd
->iter_type
== long_long_unsigned_type_node
10431 && TREE_CODE (type
) == INTEGER_TYPE
10432 && !TYPE_UNSIGNED (type
))
10436 if (fd
->loop
.cond_code
== LT_EXPR
)
10439 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10443 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10446 if (TREE_CODE (n1
) != INTEGER_CST
10447 || TREE_CODE (n2
) != INTEGER_CST
10448 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
10449 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
10452 entry_bb
= region
->entry
;
10453 cont_bb
= region
->cont
;
10454 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10455 gcc_assert (BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
10456 exit_bb
= region
->exit
;
10458 gsi
= gsi_last_bb (entry_bb
);
10459 gimple
*for_stmt
= gsi_stmt (gsi
);
10460 gcc_assert (gimple_code (for_stmt
) == GIMPLE_OMP_FOR
);
10461 if (fd
->collapse
> 1)
10463 int first_zero_iter
= -1, dummy
= -1;
10464 basic_block zero_iter_bb
= NULL
, dummy_bb
= NULL
, l2_dom_bb
= NULL
;
10466 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10467 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10468 zero_iter_bb
, first_zero_iter
,
10469 dummy_bb
, dummy
, l2_dom_bb
);
10473 /* Some counts[i] vars might be uninitialized if
10474 some loop has zero iterations. But the body shouldn't
10475 be executed in that case, so just avoid uninit warnings. */
10476 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
10477 if (SSA_VAR_P (counts
[i
]))
10478 TREE_NO_WARNING (counts
[i
]) = 1;
10480 edge e
= split_block (entry_bb
, gsi_stmt (gsi
));
10481 entry_bb
= e
->dest
;
10482 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
10483 gsi
= gsi_last_bb (entry_bb
);
10484 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
10485 get_immediate_dominator (CDI_DOMINATORS
,
10493 if (POINTER_TYPE_P (TREE_TYPE (t0
))
10494 && TYPE_PRECISION (TREE_TYPE (t0
))
10495 != TYPE_PRECISION (fd
->iter_type
))
10497 /* Avoid casting pointers to integer of a different size. */
10498 tree itype
= signed_type_for (type
);
10499 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
10500 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
10504 t1
= fold_convert (fd
->iter_type
, t1
);
10505 t0
= fold_convert (fd
->iter_type
, t0
);
10509 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
10510 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
10513 tree innerc
= find_omp_clause (gimple_omp_task_clauses (inner_stmt
),
10514 OMP_CLAUSE__LOOPTEMP_
);
10515 gcc_assert (innerc
);
10516 tree startvar
= OMP_CLAUSE_DECL (innerc
);
10517 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
), OMP_CLAUSE__LOOPTEMP_
);
10518 gcc_assert (innerc
);
10519 tree endvar
= OMP_CLAUSE_DECL (innerc
);
10520 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
10522 gcc_assert (innerc
);
10523 for (i
= 1; i
< fd
->collapse
; i
++)
10525 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10526 OMP_CLAUSE__LOOPTEMP_
);
10527 gcc_assert (innerc
);
10529 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10530 OMP_CLAUSE__LOOPTEMP_
);
10533 /* If needed (inner taskloop has lastprivate clause), propagate
10534 down the total number of iterations. */
10535 tree t
= force_gimple_operand_gsi (&gsi
, fd
->loop
.n2
, false,
10537 GSI_CONTINUE_LINKING
);
10538 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
10539 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10543 t0
= force_gimple_operand_gsi (&gsi
, t0
, false, NULL_TREE
, false,
10544 GSI_CONTINUE_LINKING
);
10545 assign_stmt
= gimple_build_assign (startvar
, t0
);
10546 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10548 t1
= force_gimple_operand_gsi (&gsi
, t1
, false, NULL_TREE
, false,
10549 GSI_CONTINUE_LINKING
);
10550 assign_stmt
= gimple_build_assign (endvar
, t1
);
10551 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10552 if (fd
->collapse
> 1)
10553 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
10555 /* Remove the GIMPLE_OMP_FOR statement. */
10556 gsi
= gsi_for_stmt (for_stmt
);
10557 gsi_remove (&gsi
, true);
10559 gsi
= gsi_last_bb (cont_bb
);
10560 gsi_remove (&gsi
, true);
10562 gsi
= gsi_last_bb (exit_bb
);
10563 gsi_remove (&gsi
, true);
10565 FALLTHRU_EDGE (entry_bb
)->probability
= REG_BR_PROB_BASE
;
10566 remove_edge (BRANCH_EDGE (entry_bb
));
10567 FALLTHRU_EDGE (cont_bb
)->probability
= REG_BR_PROB_BASE
;
10568 remove_edge (BRANCH_EDGE (cont_bb
));
10569 set_immediate_dominator (CDI_DOMINATORS
, exit_bb
, cont_bb
);
10570 set_immediate_dominator (CDI_DOMINATORS
, region
->entry
,
10571 recompute_dominator (CDI_DOMINATORS
, region
->entry
));
10574 /* Taskloop construct is represented after gimplification with
10575 two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
10576 in between them. This routine expands the inner GIMPLE_OMP_FOR.
10577 GOMP_taskloop{,_ull} function arranges for each task to be given just
10578 a single range of iterations. */
10581 expand_omp_taskloop_for_inner (struct omp_region
*region
,
10582 struct omp_for_data
*fd
,
10583 gimple
*inner_stmt
)
10585 tree e
, t
, type
, itype
, vmain
, vback
, bias
= NULL_TREE
;
10586 basic_block entry_bb
, exit_bb
, body_bb
, cont_bb
, collapse_bb
= NULL
;
10587 basic_block fin_bb
;
10588 gimple_stmt_iterator gsi
;
10590 bool broken_loop
= region
->cont
== NULL
;
10591 tree
*counts
= NULL
;
10594 itype
= type
= TREE_TYPE (fd
->loop
.v
);
10595 if (POINTER_TYPE_P (type
))
10596 itype
= signed_type_for (type
);
10598 /* See if we need to bias by LLONG_MIN. */
10599 if (fd
->iter_type
== long_long_unsigned_type_node
10600 && TREE_CODE (type
) == INTEGER_TYPE
10601 && !TYPE_UNSIGNED (type
))
10605 if (fd
->loop
.cond_code
== LT_EXPR
)
10608 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10612 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10615 if (TREE_CODE (n1
) != INTEGER_CST
10616 || TREE_CODE (n2
) != INTEGER_CST
10617 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
10618 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
10621 entry_bb
= region
->entry
;
10622 cont_bb
= region
->cont
;
10623 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10624 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
10625 gcc_assert (broken_loop
10626 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
10627 body_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10630 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
10631 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
10633 exit_bb
= region
->exit
;
10635 /* Iteration space partitioning goes in ENTRY_BB. */
10636 gsi
= gsi_last_bb (entry_bb
);
10637 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
10639 if (fd
->collapse
> 1)
10641 int first_zero_iter
= -1, dummy
= -1;
10642 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
10644 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10645 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10646 fin_bb
, first_zero_iter
,
10647 dummy_bb
, dummy
, l2_dom_bb
);
10651 t
= integer_one_node
;
10653 step
= fd
->loop
.step
;
10654 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10655 OMP_CLAUSE__LOOPTEMP_
);
10656 gcc_assert (innerc
);
10657 n1
= OMP_CLAUSE_DECL (innerc
);
10658 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
), OMP_CLAUSE__LOOPTEMP_
);
10659 gcc_assert (innerc
);
10660 n2
= OMP_CLAUSE_DECL (innerc
);
10663 n1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, n1
, bias
);
10664 n2
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, n2
, bias
);
10666 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
10667 true, NULL_TREE
, true, GSI_SAME_STMT
);
10668 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
10669 true, NULL_TREE
, true, GSI_SAME_STMT
);
10670 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
10671 true, NULL_TREE
, true, GSI_SAME_STMT
);
10673 tree startvar
= fd
->loop
.v
;
10674 tree endvar
= NULL_TREE
;
10676 if (gimple_omp_for_combined_p (fd
->for_stmt
))
10678 tree clauses
= gimple_omp_for_clauses (inner_stmt
);
10679 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
10680 gcc_assert (innerc
);
10681 startvar
= OMP_CLAUSE_DECL (innerc
);
10682 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10683 OMP_CLAUSE__LOOPTEMP_
);
10684 gcc_assert (innerc
);
10685 endvar
= OMP_CLAUSE_DECL (innerc
);
10687 t
= fold_convert (TREE_TYPE (startvar
), n1
);
10688 t
= force_gimple_operand_gsi (&gsi
, t
,
10690 && TREE_ADDRESSABLE (startvar
),
10691 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
10692 gimple
*assign_stmt
= gimple_build_assign (startvar
, t
);
10693 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10695 t
= fold_convert (TREE_TYPE (startvar
), n2
);
10696 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
10697 false, GSI_CONTINUE_LINKING
);
10700 assign_stmt
= gimple_build_assign (endvar
, e
);
10701 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10702 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
10703 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
10705 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
10706 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10708 if (fd
->collapse
> 1)
10709 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
10713 /* The code controlling the sequential loop replaces the
10714 GIMPLE_OMP_CONTINUE. */
10715 gsi
= gsi_last_bb (cont_bb
);
10716 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
10717 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
10718 vmain
= gimple_omp_continue_control_use (cont_stmt
);
10719 vback
= gimple_omp_continue_control_def (cont_stmt
);
10721 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
10723 if (POINTER_TYPE_P (type
))
10724 t
= fold_build_pointer_plus (vmain
, step
);
10726 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
10727 t
= force_gimple_operand_gsi (&gsi
, t
,
10729 && TREE_ADDRESSABLE (vback
),
10730 NULL_TREE
, true, GSI_SAME_STMT
);
10731 assign_stmt
= gimple_build_assign (vback
, t
);
10732 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
10734 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
10735 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
10737 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
10740 /* Remove the GIMPLE_OMP_CONTINUE statement. */
10741 gsi_remove (&gsi
, true);
10743 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
10744 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
10747 /* Remove the GIMPLE_OMP_FOR statement. */
10748 gsi
= gsi_for_stmt (fd
->for_stmt
);
10749 gsi_remove (&gsi
, true);
10751 /* Remove the GIMPLE_OMP_RETURN statement. */
10752 gsi
= gsi_last_bb (exit_bb
);
10753 gsi_remove (&gsi
, true);
10755 FALLTHRU_EDGE (entry_bb
)->probability
= REG_BR_PROB_BASE
;
10757 remove_edge (BRANCH_EDGE (entry_bb
));
10760 remove_edge_and_dominated_blocks (BRANCH_EDGE (entry_bb
));
10761 region
->outer
->cont
= NULL
;
10764 /* Connect all the blocks. */
10767 ep
= find_edge (cont_bb
, body_bb
);
10768 if (gimple_omp_for_combined_p (fd
->for_stmt
))
10773 else if (fd
->collapse
> 1)
10776 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
10779 ep
->flags
= EDGE_TRUE_VALUE
;
10780 find_edge (cont_bb
, fin_bb
)->flags
10781 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
10784 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
10785 recompute_dominator (CDI_DOMINATORS
, body_bb
));
10787 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
10788 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
10790 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
10792 struct loop
*loop
= alloc_loop ();
10793 loop
->header
= body_bb
;
10794 if (collapse_bb
== NULL
)
10795 loop
->latch
= cont_bb
;
10796 add_loop (loop
, body_bb
->loop_father
);
10800 /* A subroutine of expand_omp_for. Generate code for an OpenACC
10801 partitioned loop. The lowering here is abstracted, in that the
10802 loop parameters are passed through internal functions, which are
10803 further lowered by oacc_device_lower, once we get to the target
10804 compiler. The loop is of the form:
10806 for (V = B; V LTGT E; V += S) {BODY}
10808 where LTGT is < or >. We may have a specified chunking size, CHUNKING
10809 (constant 0 for no chunking) and we will have a GWV partitioning
10810 mask, specifying dimensions over which the loop is to be
10811 partitioned (see note below). We generate code that looks like:
10813 <entry_bb> [incoming FALL->body, BRANCH->exit]
10814 typedef signedintify (typeof (V)) T; // underlying signed integral type
10817 T DIR = LTGT == '<' ? +1 : -1;
10818 T chunk_max = GOACC_LOOP_CHUNK (dir, range, S, CHUNK_SIZE, GWV);
10819 T step = GOACC_LOOP_STEP (dir, range, S, CHUNK_SIZE, GWV);
10821 <head_bb> [created by splitting end of entry_bb]
10822 T offset = GOACC_LOOP_OFFSET (dir, range, S, CHUNK_SIZE, GWV, chunk_no);
10823 T bound = GOACC_LOOP_BOUND (dir, range, S, CHUNK_SIZE, GWV, offset);
10824 if (!(offset LTGT bound)) goto bottom_bb;
10826 <body_bb> [incoming]
10830 <cont_bb> [incoming, may == body_bb FALL->exit_bb, BRANCH->body_bb]
10832 if (offset LTGT bound) goto body_bb; [*]
10834 <bottom_bb> [created by splitting start of exit_bb] insert BRANCH->head_bb
10836 if (chunk < chunk_max) goto head_bb;
10838 <exit_bb> [incoming]
10839 V = B + ((range -/+ 1) / S +/- 1) * S [*]
10841 [*] Needed if V live at end of loop
10843 Note: CHUNKING & GWV mask are specified explicitly here. This is a
10844 transition, and will be specified by a more general mechanism shortly.
10848 expand_oacc_for (struct omp_region
*region
, struct omp_for_data
*fd
)
10850 tree v
= fd
->loop
.v
;
10851 enum tree_code cond_code
= fd
->loop
.cond_code
;
10852 enum tree_code plus_code
= PLUS_EXPR
;
10854 tree chunk_size
= integer_minus_one_node
;
10855 tree gwv
= integer_zero_node
;
10856 tree iter_type
= TREE_TYPE (v
);
10857 tree diff_type
= iter_type
;
10858 tree plus_type
= iter_type
;
10859 struct oacc_collapse
*counts
= NULL
;
10861 gcc_checking_assert (gimple_omp_for_kind (fd
->for_stmt
)
10862 == GF_OMP_FOR_KIND_OACC_LOOP
);
10863 gcc_assert (!gimple_omp_for_combined_into_p (fd
->for_stmt
));
10864 gcc_assert (cond_code
== LT_EXPR
|| cond_code
== GT_EXPR
);
10866 if (POINTER_TYPE_P (iter_type
))
10868 plus_code
= POINTER_PLUS_EXPR
;
10869 plus_type
= sizetype
;
10871 if (POINTER_TYPE_P (diff_type
) || TYPE_UNSIGNED (diff_type
))
10872 diff_type
= signed_type_for (diff_type
);
10874 basic_block entry_bb
= region
->entry
; /* BB ending in OMP_FOR */
10875 basic_block exit_bb
= region
->exit
; /* BB ending in OMP_RETURN */
10876 basic_block cont_bb
= region
->cont
; /* BB ending in OMP_CONTINUE */
10877 basic_block bottom_bb
= NULL
;
10879 /* entry_bb has two sucessors; the branch edge is to the exit
10880 block, fallthrough edge to body. */
10881 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2
10882 && BRANCH_EDGE (entry_bb
)->dest
== exit_bb
);
10884 /* If cont_bb non-NULL, it has 2 successors. The branch successor is
10885 body_bb, or to a block whose only successor is the body_bb. Its
10886 fallthrough successor is the final block (same as the branch
10887 successor of the entry_bb). */
10890 basic_block body_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10891 basic_block bed
= BRANCH_EDGE (cont_bb
)->dest
;
10893 gcc_assert (FALLTHRU_EDGE (cont_bb
)->dest
== exit_bb
);
10894 gcc_assert (bed
== body_bb
|| single_succ_edge (bed
)->dest
== body_bb
);
10897 gcc_assert (!gimple_in_ssa_p (cfun
));
10899 /* The exit block only has entry_bb and cont_bb as predecessors. */
10900 gcc_assert (EDGE_COUNT (exit_bb
->preds
) == 1 + (cont_bb
!= NULL
));
10903 tree chunk_max
= NULL_TREE
;
10904 tree bound
, offset
;
10905 tree step
= create_tmp_var (diff_type
, ".step");
10906 bool up
= cond_code
== LT_EXPR
;
10907 tree dir
= build_int_cst (diff_type
, up
? +1 : -1);
10908 bool chunking
= !gimple_in_ssa_p (cfun
);;
10911 /* SSA instances. */
10912 tree offset_incr
= NULL_TREE
;
10913 tree offset_init
= NULL_TREE
;
10915 gimple_stmt_iterator gsi
;
10921 edge split
, be
, fte
;
10923 /* Split the end of entry_bb to create head_bb. */
10924 split
= split_block (entry_bb
, last_stmt (entry_bb
));
10925 basic_block head_bb
= split
->dest
;
10926 entry_bb
= split
->src
;
10928 /* Chunk setup goes at end of entry_bb, replacing the omp_for. */
10929 gsi
= gsi_last_bb (entry_bb
);
10930 gomp_for
*for_stmt
= as_a
<gomp_for
*> (gsi_stmt (gsi
));
10931 loc
= gimple_location (for_stmt
);
10933 if (gimple_in_ssa_p (cfun
))
10935 offset_init
= gimple_omp_for_index (for_stmt
, 0);
10936 gcc_assert (integer_zerop (fd
->loop
.n1
));
10937 /* The SSA parallelizer does gang parallelism. */
10938 gwv
= build_int_cst (integer_type_node
, GOMP_DIM_MASK (GOMP_DIM_GANG
));
10941 if (fd
->collapse
> 1)
10943 counts
= XALLOCAVEC (struct oacc_collapse
, fd
->collapse
);
10944 tree total
= expand_oacc_collapse_init (fd
, &gsi
, counts
,
10945 TREE_TYPE (fd
->loop
.n2
));
10947 if (SSA_VAR_P (fd
->loop
.n2
))
10949 total
= force_gimple_operand_gsi (&gsi
, total
, false, NULL_TREE
,
10950 true, GSI_SAME_STMT
);
10951 ass
= gimple_build_assign (fd
->loop
.n2
, total
);
10952 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
10957 tree b
= fd
->loop
.n1
;
10958 tree e
= fd
->loop
.n2
;
10959 tree s
= fd
->loop
.step
;
10961 b
= force_gimple_operand_gsi (&gsi
, b
, true, NULL_TREE
, true, GSI_SAME_STMT
);
10962 e
= force_gimple_operand_gsi (&gsi
, e
, true, NULL_TREE
, true, GSI_SAME_STMT
);
10964 /* Convert the step, avoiding possible unsigned->signed overflow. */
10965 negating
= !up
&& TYPE_UNSIGNED (TREE_TYPE (s
));
10967 s
= fold_build1 (NEGATE_EXPR
, TREE_TYPE (s
), s
);
10968 s
= fold_convert (diff_type
, s
);
10970 s
= fold_build1 (NEGATE_EXPR
, diff_type
, s
);
10971 s
= force_gimple_operand_gsi (&gsi
, s
, true, NULL_TREE
, true, GSI_SAME_STMT
);
10974 chunk_size
= integer_zero_node
;
10975 expr
= fold_convert (diff_type
, chunk_size
);
10976 chunk_size
= force_gimple_operand_gsi (&gsi
, expr
, true,
10977 NULL_TREE
, true, GSI_SAME_STMT
);
10978 /* Determine the range, avoiding possible unsigned->signed overflow. */
10979 negating
= !up
&& TYPE_UNSIGNED (iter_type
);
10980 expr
= fold_build2 (MINUS_EXPR
, plus_type
,
10981 fold_convert (plus_type
, negating
? b
: e
),
10982 fold_convert (plus_type
, negating
? e
: b
));
10983 expr
= fold_convert (diff_type
, expr
);
10985 expr
= fold_build1 (NEGATE_EXPR
, diff_type
, expr
);
10986 tree range
= force_gimple_operand_gsi (&gsi
, expr
, true,
10987 NULL_TREE
, true, GSI_SAME_STMT
);
10989 chunk_no
= build_int_cst (diff_type
, 0);
10992 gcc_assert (!gimple_in_ssa_p (cfun
));
10995 chunk_max
= create_tmp_var (diff_type
, ".chunk_max");
10996 chunk_no
= create_tmp_var (diff_type
, ".chunk_no");
10998 ass
= gimple_build_assign (chunk_no
, expr
);
10999 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11001 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 6,
11002 build_int_cst (integer_type_node
,
11003 IFN_GOACC_LOOP_CHUNKS
),
11004 dir
, range
, s
, chunk_size
, gwv
);
11005 gimple_call_set_lhs (call
, chunk_max
);
11006 gimple_set_location (call
, loc
);
11007 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
11010 chunk_size
= chunk_no
;
11012 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 6,
11013 build_int_cst (integer_type_node
,
11014 IFN_GOACC_LOOP_STEP
),
11015 dir
, range
, s
, chunk_size
, gwv
);
11016 gimple_call_set_lhs (call
, step
);
11017 gimple_set_location (call
, loc
);
11018 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
11020 /* Remove the GIMPLE_OMP_FOR. */
11021 gsi_remove (&gsi
, true);
11023 /* Fixup edges from head_bb */
11024 be
= BRANCH_EDGE (head_bb
);
11025 fte
= FALLTHRU_EDGE (head_bb
);
11026 be
->flags
|= EDGE_FALSE_VALUE
;
11027 fte
->flags
^= EDGE_FALLTHRU
| EDGE_TRUE_VALUE
;
11029 basic_block body_bb
= fte
->dest
;
11031 if (gimple_in_ssa_p (cfun
))
11033 gsi
= gsi_last_bb (cont_bb
);
11034 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
11036 offset
= gimple_omp_continue_control_use (cont_stmt
);
11037 offset_incr
= gimple_omp_continue_control_def (cont_stmt
);
11041 offset
= create_tmp_var (diff_type
, ".offset");
11042 offset_init
= offset_incr
= offset
;
11044 bound
= create_tmp_var (TREE_TYPE (offset
), ".bound");
11046 /* Loop offset & bound go into head_bb. */
11047 gsi
= gsi_start_bb (head_bb
);
11049 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 7,
11050 build_int_cst (integer_type_node
,
11051 IFN_GOACC_LOOP_OFFSET
),
11053 chunk_size
, gwv
, chunk_no
);
11054 gimple_call_set_lhs (call
, offset_init
);
11055 gimple_set_location (call
, loc
);
11056 gsi_insert_after (&gsi
, call
, GSI_CONTINUE_LINKING
);
11058 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 7,
11059 build_int_cst (integer_type_node
,
11060 IFN_GOACC_LOOP_BOUND
),
11062 chunk_size
, gwv
, offset_init
);
11063 gimple_call_set_lhs (call
, bound
);
11064 gimple_set_location (call
, loc
);
11065 gsi_insert_after (&gsi
, call
, GSI_CONTINUE_LINKING
);
11067 expr
= build2 (cond_code
, boolean_type_node
, offset_init
, bound
);
11068 gsi_insert_after (&gsi
, gimple_build_cond_empty (expr
),
11069 GSI_CONTINUE_LINKING
);
11071 /* V assignment goes into body_bb. */
11072 if (!gimple_in_ssa_p (cfun
))
11074 gsi
= gsi_start_bb (body_bb
);
11076 expr
= build2 (plus_code
, iter_type
, b
,
11077 fold_convert (plus_type
, offset
));
11078 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
11079 true, GSI_SAME_STMT
);
11080 ass
= gimple_build_assign (v
, expr
);
11081 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11082 if (fd
->collapse
> 1)
11083 expand_oacc_collapse_vars (fd
, &gsi
, counts
, v
);
11086 /* Loop increment goes into cont_bb. If this is not a loop, we
11087 will have spawned threads as if it was, and each one will
11088 execute one iteration. The specification is not explicit about
11089 whether such constructs are ill-formed or not, and they can
11090 occur, especially when noreturn routines are involved. */
11093 gsi
= gsi_last_bb (cont_bb
);
11094 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
11095 loc
= gimple_location (cont_stmt
);
11097 /* Increment offset. */
11098 if (gimple_in_ssa_p (cfun
))
11099 expr
= build2 (plus_code
, iter_type
, offset
,
11100 fold_convert (plus_type
, step
));
11102 expr
= build2 (PLUS_EXPR
, diff_type
, offset
, step
);
11103 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
11104 true, GSI_SAME_STMT
);
11105 ass
= gimple_build_assign (offset_incr
, expr
);
11106 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11107 expr
= build2 (cond_code
, boolean_type_node
, offset_incr
, bound
);
11108 gsi_insert_before (&gsi
, gimple_build_cond_empty (expr
), GSI_SAME_STMT
);
11110 /* Remove the GIMPLE_OMP_CONTINUE. */
11111 gsi_remove (&gsi
, true);
11113 /* Fixup edges from cont_bb */
11114 be
= BRANCH_EDGE (cont_bb
);
11115 fte
= FALLTHRU_EDGE (cont_bb
);
11116 be
->flags
|= EDGE_TRUE_VALUE
;
11117 fte
->flags
^= EDGE_FALLTHRU
| EDGE_FALSE_VALUE
;
11121 /* Split the beginning of exit_bb to make bottom_bb. We
11122 need to insert a nop at the start, because splitting is
11123 after a stmt, not before. */
11124 gsi
= gsi_start_bb (exit_bb
);
11125 stmt
= gimple_build_nop ();
11126 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
11127 split
= split_block (exit_bb
, stmt
);
11128 bottom_bb
= split
->src
;
11129 exit_bb
= split
->dest
;
11130 gsi
= gsi_last_bb (bottom_bb
);
11132 /* Chunk increment and test goes into bottom_bb. */
11133 expr
= build2 (PLUS_EXPR
, diff_type
, chunk_no
,
11134 build_int_cst (diff_type
, 1));
11135 ass
= gimple_build_assign (chunk_no
, expr
);
11136 gsi_insert_after (&gsi
, ass
, GSI_CONTINUE_LINKING
);
11138 /* Chunk test at end of bottom_bb. */
11139 expr
= build2 (LT_EXPR
, boolean_type_node
, chunk_no
, chunk_max
);
11140 gsi_insert_after (&gsi
, gimple_build_cond_empty (expr
),
11141 GSI_CONTINUE_LINKING
);
11143 /* Fixup edges from bottom_bb. */
11144 split
->flags
^= EDGE_FALLTHRU
| EDGE_FALSE_VALUE
;
11145 make_edge (bottom_bb
, head_bb
, EDGE_TRUE_VALUE
);
11149 gsi
= gsi_last_bb (exit_bb
);
11150 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
11151 loc
= gimple_location (gsi_stmt (gsi
));
11153 if (!gimple_in_ssa_p (cfun
))
11155 /* Insert the final value of V, in case it is live. This is the
11156 value for the only thread that survives past the join. */
11157 expr
= fold_build2 (MINUS_EXPR
, diff_type
, range
, dir
);
11158 expr
= fold_build2 (PLUS_EXPR
, diff_type
, expr
, s
);
11159 expr
= fold_build2 (TRUNC_DIV_EXPR
, diff_type
, expr
, s
);
11160 expr
= fold_build2 (MULT_EXPR
, diff_type
, expr
, s
);
11161 expr
= build2 (plus_code
, iter_type
, b
, fold_convert (plus_type
, expr
));
11162 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
11163 true, GSI_SAME_STMT
);
11164 ass
= gimple_build_assign (v
, expr
);
11165 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11168 /* Remove the OMP_RETURN. */
11169 gsi_remove (&gsi
, true);
11173 /* We now have one or two nested loops. Update the loop
11175 struct loop
*parent
= entry_bb
->loop_father
;
11176 struct loop
*body
= body_bb
->loop_father
;
11180 struct loop
*chunk_loop
= alloc_loop ();
11181 chunk_loop
->header
= head_bb
;
11182 chunk_loop
->latch
= bottom_bb
;
11183 add_loop (chunk_loop
, parent
);
11184 parent
= chunk_loop
;
11186 else if (parent
!= body
)
11188 gcc_assert (body
->header
== body_bb
);
11189 gcc_assert (body
->latch
== cont_bb
11190 || single_pred (body
->latch
) == cont_bb
);
11196 struct loop
*body_loop
= alloc_loop ();
11197 body_loop
->header
= body_bb
;
11198 body_loop
->latch
= cont_bb
;
11199 add_loop (body_loop
, parent
);
11204 /* Expand the OMP loop defined by REGION. */
11207 expand_omp_for (struct omp_region
*region
, gimple
*inner_stmt
)
11209 struct omp_for_data fd
;
11210 struct omp_for_data_loop
*loops
;
11213 = (struct omp_for_data_loop
*)
11214 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
11215 * sizeof (struct omp_for_data_loop
));
11216 extract_omp_for_data (as_a
<gomp_for
*> (last_stmt (region
->entry
)),
11218 region
->sched_kind
= fd
.sched_kind
;
11219 region
->sched_modifiers
= fd
.sched_modifiers
;
11221 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
11222 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
11223 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
11226 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
11227 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
11228 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
11231 /* If there isn't a continue then this is a degerate case where
11232 the introduction of abnormal edges during lowering will prevent
11233 original loops from being detected. Fix that up. */
11234 loops_state_set (LOOPS_NEED_FIXUP
);
11236 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_SIMD
)
11237 expand_omp_simd (region
, &fd
);
11238 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
11239 expand_cilk_for (region
, &fd
);
11240 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
11242 gcc_assert (!inner_stmt
);
11243 expand_oacc_for (region
, &fd
);
11245 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
11247 if (gimple_omp_for_combined_into_p (fd
.for_stmt
))
11248 expand_omp_taskloop_for_inner (region
, &fd
, inner_stmt
);
11250 expand_omp_taskloop_for_outer (region
, &fd
, inner_stmt
);
11252 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
11253 && !fd
.have_ordered
)
11255 if (fd
.chunk_size
== NULL
)
11256 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
11258 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
11262 int fn_index
, start_ix
, next_ix
;
11264 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
11265 == GF_OMP_FOR_KIND_FOR
);
11266 if (fd
.chunk_size
== NULL
11267 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
11268 fd
.chunk_size
= integer_zero_node
;
11269 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
11270 switch (fd
.sched_kind
)
11272 case OMP_CLAUSE_SCHEDULE_RUNTIME
:
11275 case OMP_CLAUSE_SCHEDULE_DYNAMIC
:
11276 case OMP_CLAUSE_SCHEDULE_GUIDED
:
11277 if ((fd
.sched_modifiers
& OMP_CLAUSE_SCHEDULE_NONMONOTONIC
)
11279 && !fd
.have_ordered
)
11281 fn_index
= 3 + fd
.sched_kind
;
11286 fn_index
= fd
.sched_kind
;
11290 fn_index
+= fd
.have_ordered
* 6;
11292 start_ix
= ((int)BUILT_IN_GOMP_LOOP_DOACROSS_STATIC_START
) + fn_index
;
11294 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
11295 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
11296 if (fd
.iter_type
== long_long_unsigned_type_node
)
11298 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
11299 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
11300 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
11301 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
11303 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
11304 (enum built_in_function
) next_ix
, inner_stmt
);
11307 if (gimple_in_ssa_p (cfun
))
11308 update_ssa (TODO_update_ssa_only_virtuals
);
11312 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
11314 v = GOMP_sections_start (n);
11331 v = GOMP_sections_next ();
11336 If this is a combined parallel sections, replace the call to
11337 GOMP_sections_start with call to GOMP_sections_next. */
11340 expand_omp_sections (struct omp_region
*region
)
11342 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
11344 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
11345 gimple_stmt_iterator si
, switch_si
;
11346 gomp_sections
*sections_stmt
;
11348 gomp_continue
*cont
;
11351 struct omp_region
*inner
;
11353 bool exit_reachable
= region
->cont
!= NULL
;
11355 gcc_assert (region
->exit
!= NULL
);
11356 entry_bb
= region
->entry
;
11357 l0_bb
= single_succ (entry_bb
);
11358 l1_bb
= region
->cont
;
11359 l2_bb
= region
->exit
;
11360 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
11361 l2
= gimple_block_label (l2_bb
);
11364 /* This can happen if there are reductions. */
11365 len
= EDGE_COUNT (l0_bb
->succs
);
11366 gcc_assert (len
> 0);
11367 e
= EDGE_SUCC (l0_bb
, len
- 1);
11368 si
= gsi_last_bb (e
->dest
);
11371 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
11372 l2
= gimple_block_label (e
->dest
);
11374 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
11376 si
= gsi_last_bb (e
->dest
);
11378 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
11380 l2
= gimple_block_label (e
->dest
);
11385 if (exit_reachable
)
11386 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
11388 default_bb
= create_empty_bb (l0_bb
);
11390 /* We will build a switch() with enough cases for all the
11391 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
11392 and a default case to abort if something goes wrong. */
11393 len
= EDGE_COUNT (l0_bb
->succs
);
11395 /* Use vec::quick_push on label_vec throughout, since we know the size
11397 auto_vec
<tree
> label_vec (len
);
11399 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
11400 GIMPLE_OMP_SECTIONS statement. */
11401 si
= gsi_last_bb (entry_bb
);
11402 sections_stmt
= as_a
<gomp_sections
*> (gsi_stmt (si
));
11403 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
11404 vin
= gimple_omp_sections_control (sections_stmt
);
11405 if (!is_combined_parallel (region
))
11407 /* If we are not inside a combined parallel+sections region,
11408 call GOMP_sections_start. */
11409 t
= build_int_cst (unsigned_type_node
, len
- 1);
11410 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
11411 stmt
= gimple_build_call (u
, 1, t
);
11415 /* Otherwise, call GOMP_sections_next. */
11416 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
11417 stmt
= gimple_build_call (u
, 0);
11419 gimple_call_set_lhs (stmt
, vin
);
11420 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11421 gsi_remove (&si
, true);
11423 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
11425 switch_si
= gsi_last_bb (l0_bb
);
11426 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
11427 if (exit_reachable
)
11429 cont
= as_a
<gomp_continue
*> (last_stmt (l1_bb
));
11430 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
11431 vmain
= gimple_omp_continue_control_use (cont
);
11432 vnext
= gimple_omp_continue_control_def (cont
);
11440 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
11441 label_vec
.quick_push (t
);
11444 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
11445 for (inner
= region
->inner
, casei
= 1;
11447 inner
= inner
->next
, i
++, casei
++)
11449 basic_block s_entry_bb
, s_exit_bb
;
11451 /* Skip optional reduction region. */
11452 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
11459 s_entry_bb
= inner
->entry
;
11460 s_exit_bb
= inner
->exit
;
11462 t
= gimple_block_label (s_entry_bb
);
11463 u
= build_int_cst (unsigned_type_node
, casei
);
11464 u
= build_case_label (u
, NULL
, t
);
11465 label_vec
.quick_push (u
);
11467 si
= gsi_last_bb (s_entry_bb
);
11468 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
11469 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
11470 gsi_remove (&si
, true);
11471 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
11473 if (s_exit_bb
== NULL
)
11476 si
= gsi_last_bb (s_exit_bb
);
11477 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
11478 gsi_remove (&si
, true);
11480 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
11483 /* Error handling code goes in DEFAULT_BB. */
11484 t
= gimple_block_label (default_bb
);
11485 u
= build_case_label (NULL
, NULL
, t
);
11486 make_edge (l0_bb
, default_bb
, 0);
11487 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
11489 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
11490 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
11491 gsi_remove (&switch_si
, true);
11493 si
= gsi_start_bb (default_bb
);
11494 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
11495 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
11497 if (exit_reachable
)
11501 /* Code to get the next section goes in L1_BB. */
11502 si
= gsi_last_bb (l1_bb
);
11503 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
11505 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
11506 stmt
= gimple_build_call (bfn_decl
, 0);
11507 gimple_call_set_lhs (stmt
, vnext
);
11508 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11509 gsi_remove (&si
, true);
11511 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
11514 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
11515 si
= gsi_last_bb (l2_bb
);
11516 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
11517 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
11518 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
11519 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
11521 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
11522 stmt
= gimple_build_call (t
, 0);
11523 if (gimple_omp_return_lhs (gsi_stmt (si
)))
11524 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
11525 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11526 gsi_remove (&si
, true);
11528 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
11532 /* Expand code for an OpenMP single directive. We've already expanded
11533 much of the code, here we simply place the GOMP_barrier call. */
11536 expand_omp_single (struct omp_region
*region
)
11538 basic_block entry_bb
, exit_bb
;
11539 gimple_stmt_iterator si
;
11541 entry_bb
= region
->entry
;
11542 exit_bb
= region
->exit
;
11544 si
= gsi_last_bb (entry_bb
);
11545 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
11546 gsi_remove (&si
, true);
11547 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
11549 si
= gsi_last_bb (exit_bb
);
11550 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
11552 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
11553 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
11555 gsi_remove (&si
, true);
11556 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
11560 /* Generic expansion for OpenMP synchronization directives: master,
11561 ordered and critical. All we need to do here is remove the entry
11562 and exit markers for REGION. */
11565 expand_omp_synch (struct omp_region
*region
)
11567 basic_block entry_bb
, exit_bb
;
11568 gimple_stmt_iterator si
;
11570 entry_bb
= region
->entry
;
11571 exit_bb
= region
->exit
;
11573 si
= gsi_last_bb (entry_bb
);
11574 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
11575 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
11576 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
11577 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
11578 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
11579 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
11580 gsi_remove (&si
, true);
11581 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
11585 si
= gsi_last_bb (exit_bb
);
11586 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
11587 gsi_remove (&si
, true);
11588 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
11592 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
11593 operation as a normal volatile load. */
11596 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
11597 tree loaded_val
, int index
)
11599 enum built_in_function tmpbase
;
11600 gimple_stmt_iterator gsi
;
11601 basic_block store_bb
;
11604 tree decl
, call
, type
, itype
;
11606 gsi
= gsi_last_bb (load_bb
);
11607 stmt
= gsi_stmt (gsi
);
11608 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
11609 loc
= gimple_location (stmt
);
11611 /* ??? If the target does not implement atomic_load_optab[mode], and mode
11612 is smaller than word size, then expand_atomic_load assumes that the load
11613 is atomic. We could avoid the builtin entirely in this case. */
11615 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
11616 decl
= builtin_decl_explicit (tmpbase
);
11617 if (decl
== NULL_TREE
)
11620 type
= TREE_TYPE (loaded_val
);
11621 itype
= TREE_TYPE (TREE_TYPE (decl
));
11623 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
11624 build_int_cst (NULL
,
11625 gimple_omp_atomic_seq_cst_p (stmt
)
11627 : MEMMODEL_RELAXED
));
11628 if (!useless_type_conversion_p (type
, itype
))
11629 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
11630 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
11632 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11633 gsi_remove (&gsi
, true);
11635 store_bb
= single_succ (load_bb
);
11636 gsi
= gsi_last_bb (store_bb
);
11637 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
11638 gsi_remove (&gsi
, true);
11640 if (gimple_in_ssa_p (cfun
))
11641 update_ssa (TODO_update_ssa_no_phi
);
11646 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
11647 operation as a normal volatile store. */
11650 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
11651 tree loaded_val
, tree stored_val
, int index
)
11653 enum built_in_function tmpbase
;
11654 gimple_stmt_iterator gsi
;
11655 basic_block store_bb
= single_succ (load_bb
);
11658 tree decl
, call
, type
, itype
;
11659 machine_mode imode
;
11662 gsi
= gsi_last_bb (load_bb
);
11663 stmt
= gsi_stmt (gsi
);
11664 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
11666 /* If the load value is needed, then this isn't a store but an exchange. */
11667 exchange
= gimple_omp_atomic_need_value_p (stmt
);
11669 gsi
= gsi_last_bb (store_bb
);
11670 stmt
= gsi_stmt (gsi
);
11671 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
11672 loc
= gimple_location (stmt
);
11674 /* ??? If the target does not implement atomic_store_optab[mode], and mode
11675 is smaller than word size, then expand_atomic_store assumes that the store
11676 is atomic. We could avoid the builtin entirely in this case. */
11678 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
11679 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
11680 decl
= builtin_decl_explicit (tmpbase
);
11681 if (decl
== NULL_TREE
)
11684 type
= TREE_TYPE (stored_val
);
11686 /* Dig out the type of the function's second argument. */
11687 itype
= TREE_TYPE (decl
);
11688 itype
= TYPE_ARG_TYPES (itype
);
11689 itype
= TREE_CHAIN (itype
);
11690 itype
= TREE_VALUE (itype
);
11691 imode
= TYPE_MODE (itype
);
11693 if (exchange
&& !can_atomic_exchange_p (imode
, true))
11696 if (!useless_type_conversion_p (itype
, type
))
11697 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
11698 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
11699 build_int_cst (NULL
,
11700 gimple_omp_atomic_seq_cst_p (stmt
)
11702 : MEMMODEL_RELAXED
));
11705 if (!useless_type_conversion_p (type
, itype
))
11706 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
11707 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
11710 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11711 gsi_remove (&gsi
, true);
11713 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
11714 gsi
= gsi_last_bb (load_bb
);
11715 gsi_remove (&gsi
, true);
11717 if (gimple_in_ssa_p (cfun
))
11718 update_ssa (TODO_update_ssa_no_phi
);
11723 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
11724 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
11725 size of the data type, and thus usable to find the index of the builtin
11726 decl. Returns false if the expression is not of the proper form. */
11729 expand_omp_atomic_fetch_op (basic_block load_bb
,
11730 tree addr
, tree loaded_val
,
11731 tree stored_val
, int index
)
11733 enum built_in_function oldbase
, newbase
, tmpbase
;
11734 tree decl
, itype
, call
;
11736 basic_block store_bb
= single_succ (load_bb
);
11737 gimple_stmt_iterator gsi
;
11740 enum tree_code code
;
11741 bool need_old
, need_new
;
11742 machine_mode imode
;
11745 /* We expect to find the following sequences:
11748 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
11751 val = tmp OP something; (or: something OP tmp)
11752 GIMPLE_OMP_STORE (val)
11754 ???FIXME: Allow a more flexible sequence.
11755 Perhaps use data flow to pick the statements.
11759 gsi
= gsi_after_labels (store_bb
);
11760 stmt
= gsi_stmt (gsi
);
11761 loc
= gimple_location (stmt
);
11762 if (!is_gimple_assign (stmt
))
11765 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
11767 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
11768 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
11769 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
11770 gcc_checking_assert (!need_old
|| !need_new
);
11772 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
11775 /* Check for one of the supported fetch-op operations. */
11776 code
= gimple_assign_rhs_code (stmt
);
11780 case POINTER_PLUS_EXPR
:
11781 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
11782 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
11785 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
11786 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
11789 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
11790 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
11793 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
11794 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
11797 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
11798 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
11804 /* Make sure the expression is of the proper form. */
11805 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
11806 rhs
= gimple_assign_rhs2 (stmt
);
11807 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
11808 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
11809 rhs
= gimple_assign_rhs1 (stmt
);
11813 tmpbase
= ((enum built_in_function
)
11814 ((need_new
? newbase
: oldbase
) + index
+ 1));
11815 decl
= builtin_decl_explicit (tmpbase
);
11816 if (decl
== NULL_TREE
)
11818 itype
= TREE_TYPE (TREE_TYPE (decl
));
11819 imode
= TYPE_MODE (itype
);
11821 /* We could test all of the various optabs involved, but the fact of the
11822 matter is that (with the exception of i486 vs i586 and xadd) all targets
11823 that support any atomic operaton optab also implements compare-and-swap.
11824 Let optabs.c take care of expanding any compare-and-swap loop. */
11825 if (!can_compare_and_swap_p (imode
, true))
11828 gsi
= gsi_last_bb (load_bb
);
11829 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
11831 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
11832 It only requires that the operation happen atomically. Thus we can
11833 use the RELAXED memory model. */
11834 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
11835 fold_convert_loc (loc
, itype
, rhs
),
11836 build_int_cst (NULL
,
11837 seq_cst
? MEMMODEL_SEQ_CST
11838 : MEMMODEL_RELAXED
));
11840 if (need_old
|| need_new
)
11842 lhs
= need_old
? loaded_val
: stored_val
;
11843 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
11844 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
11847 call
= fold_convert_loc (loc
, void_type_node
, call
);
11848 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11849 gsi_remove (&gsi
, true);
11851 gsi
= gsi_last_bb (store_bb
);
11852 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
11853 gsi_remove (&gsi
, true);
11854 gsi
= gsi_last_bb (store_bb
);
11855 gsi_remove (&gsi
, true);
11857 if (gimple_in_ssa_p (cfun
))
11858 update_ssa (TODO_update_ssa_no_phi
);
11863 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
11867 newval = rhs; // with oldval replacing *addr in rhs
11868 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
11869 if (oldval != newval)
11872 INDEX is log2 of the size of the data type, and thus usable to find the
11873 index of the builtin decl. */
11876 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
11877 tree addr
, tree loaded_val
, tree stored_val
,
11880 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
11881 tree type
, itype
, cmpxchg
, iaddr
;
11882 gimple_stmt_iterator si
;
11883 basic_block loop_header
= single_succ (load_bb
);
11884 gimple
*phi
, *stmt
;
11886 enum built_in_function fncode
;
11888 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
11889 order to use the RELAXED memory model effectively. */
11890 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
11892 cmpxchg
= builtin_decl_explicit (fncode
);
11893 if (cmpxchg
== NULL_TREE
)
11895 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
11896 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
11898 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
11901 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
11902 si
= gsi_last_bb (load_bb
);
11903 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
11905 /* For floating-point values, we'll need to view-convert them to integers
11906 so that we can perform the atomic compare and swap. Simplify the
11907 following code by always setting up the "i"ntegral variables. */
11908 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
11912 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
11915 = force_gimple_operand_gsi (&si
,
11916 fold_convert (TREE_TYPE (iaddr
), addr
),
11917 false, NULL_TREE
, true, GSI_SAME_STMT
);
11918 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
11919 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
11920 loadedi
= create_tmp_var (itype
);
11921 if (gimple_in_ssa_p (cfun
))
11922 loadedi
= make_ssa_name (loadedi
);
11927 loadedi
= loaded_val
;
11930 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
11931 tree loaddecl
= builtin_decl_explicit (fncode
);
11934 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr
)),
11935 build_call_expr (loaddecl
, 2, iaddr
,
11936 build_int_cst (NULL_TREE
,
11937 MEMMODEL_RELAXED
)));
11939 initial
= build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)), iaddr
,
11940 build_int_cst (TREE_TYPE (iaddr
), 0));
11943 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
11946 /* Move the value to the LOADEDI temporary. */
11947 if (gimple_in_ssa_p (cfun
))
11949 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
11950 phi
= create_phi_node (loadedi
, loop_header
);
11951 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
11955 gsi_insert_before (&si
,
11956 gimple_build_assign (loadedi
, initial
),
11958 if (loadedi
!= loaded_val
)
11960 gimple_stmt_iterator gsi2
;
11963 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
11964 gsi2
= gsi_start_bb (loop_header
);
11965 if (gimple_in_ssa_p (cfun
))
11968 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
11969 true, GSI_SAME_STMT
);
11970 stmt
= gimple_build_assign (loaded_val
, x
);
11971 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
11975 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
11976 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
11977 true, GSI_SAME_STMT
);
11980 gsi_remove (&si
, true);
11982 si
= gsi_last_bb (store_bb
);
11983 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
11986 storedi
= stored_val
;
11989 force_gimple_operand_gsi (&si
,
11990 build1 (VIEW_CONVERT_EXPR
, itype
,
11991 stored_val
), true, NULL_TREE
, true,
11994 /* Build the compare&swap statement. */
11995 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
11996 new_storedi
= force_gimple_operand_gsi (&si
,
11997 fold_convert (TREE_TYPE (loadedi
),
12000 true, GSI_SAME_STMT
);
12002 if (gimple_in_ssa_p (cfun
))
12003 old_vali
= loadedi
;
12006 old_vali
= create_tmp_var (TREE_TYPE (loadedi
));
12007 stmt
= gimple_build_assign (old_vali
, loadedi
);
12008 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12010 stmt
= gimple_build_assign (loadedi
, new_storedi
);
12011 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12014 /* Note that we always perform the comparison as an integer, even for
12015 floating point. This allows the atomic operation to properly
12016 succeed even with NaNs and -0.0. */
12017 stmt
= gimple_build_cond_empty
12018 (build2 (NE_EXPR
, boolean_type_node
,
12019 new_storedi
, old_vali
));
12020 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12023 e
= single_succ_edge (store_bb
);
12024 e
->flags
&= ~EDGE_FALLTHRU
;
12025 e
->flags
|= EDGE_FALSE_VALUE
;
12027 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
12029 /* Copy the new value to loadedi (we already did that before the condition
12030 if we are not in SSA). */
12031 if (gimple_in_ssa_p (cfun
))
12033 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
12034 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
12037 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
12038 gsi_remove (&si
, true);
12040 struct loop
*loop
= alloc_loop ();
12041 loop
->header
= loop_header
;
12042 loop
->latch
= store_bb
;
12043 add_loop (loop
, loop_header
->loop_father
);
12045 if (gimple_in_ssa_p (cfun
))
12046 update_ssa (TODO_update_ssa_no_phi
);
12051 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
12053 GOMP_atomic_start ();
12055 GOMP_atomic_end ();
12057 The result is not globally atomic, but works so long as all parallel
12058 references are within #pragma omp atomic directives. According to
12059 responses received from omp@openmp.org, appears to be within spec.
12060 Which makes sense, since that's how several other compilers handle
12061 this situation as well.
12062 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
12063 expanding. STORED_VAL is the operand of the matching
12064 GIMPLE_OMP_ATOMIC_STORE.
12067 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
12068 loaded_val = *addr;
12071 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
12072 *addr = stored_val;
12076 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
12077 tree addr
, tree loaded_val
, tree stored_val
)
12079 gimple_stmt_iterator si
;
12083 si
= gsi_last_bb (load_bb
);
12084 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
12086 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
12087 t
= build_call_expr (t
, 0);
12088 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
12090 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
12091 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12092 gsi_remove (&si
, true);
12094 si
= gsi_last_bb (store_bb
);
12095 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
12097 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
12099 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12101 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
12102 t
= build_call_expr (t
, 0);
12103 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
12104 gsi_remove (&si
, true);
12106 if (gimple_in_ssa_p (cfun
))
12107 update_ssa (TODO_update_ssa_no_phi
);
12111 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
12112 using expand_omp_atomic_fetch_op. If it failed, we try to
12113 call expand_omp_atomic_pipeline, and if it fails too, the
12114 ultimate fallback is wrapping the operation in a mutex
12115 (expand_omp_atomic_mutex). REGION is the atomic region built
12116 by build_omp_regions_1(). */
12119 expand_omp_atomic (struct omp_region
*region
)
12121 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
12122 gomp_atomic_load
*load
= as_a
<gomp_atomic_load
*> (last_stmt (load_bb
));
12123 gomp_atomic_store
*store
= as_a
<gomp_atomic_store
*> (last_stmt (store_bb
));
12124 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
12125 tree addr
= gimple_omp_atomic_load_rhs (load
);
12126 tree stored_val
= gimple_omp_atomic_store_val (store
);
12127 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
12128 HOST_WIDE_INT index
;
12130 /* Make sure the type is one of the supported sizes. */
12131 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
12132 index
= exact_log2 (index
);
12133 if (index
>= 0 && index
<= 4)
12135 unsigned int align
= TYPE_ALIGN_UNIT (type
);
12137 /* __sync builtins require strict data alignment. */
12138 if (exact_log2 (align
) >= index
)
12141 if (loaded_val
== stored_val
12142 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
12143 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
12144 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
12145 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
12148 /* Atomic store. */
12149 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
12150 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
12151 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
12152 && store_bb
== single_succ (load_bb
)
12153 && first_stmt (store_bb
) == store
12154 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
12155 stored_val
, index
))
12158 /* When possible, use specialized atomic update functions. */
12159 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
12160 && store_bb
== single_succ (load_bb
)
12161 && expand_omp_atomic_fetch_op (load_bb
, addr
,
12162 loaded_val
, stored_val
, index
))
12165 /* If we don't have specialized __sync builtins, try and implement
12166 as a compare and swap loop. */
12167 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
12168 loaded_val
, stored_val
, index
))
12173 /* The ultimate fallback is wrapping the operation in a mutex. */
12174 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
12178 /* Encode an oacc launc argument. This matches the GOMP_LAUNCH_PACK
12179 macro on gomp-constants.h. We do not check for overflow. */
12182 oacc_launch_pack (unsigned code
, tree device
, unsigned op
)
12186 res
= build_int_cst (unsigned_type_node
, GOMP_LAUNCH_PACK (code
, 0, op
));
12189 device
= fold_build2 (LSHIFT_EXPR
, unsigned_type_node
,
12190 device
, build_int_cst (unsigned_type_node
,
12191 GOMP_LAUNCH_DEVICE_SHIFT
));
12192 res
= fold_build2 (BIT_IOR_EXPR
, unsigned_type_node
, res
, device
);
12197 /* Look for compute grid dimension clauses and convert to an attribute
12198 attached to FN. This permits the target-side code to (a) massage
12199 the dimensions, (b) emit that data and (c) optimize. Non-constant
12200 dimensions are pushed onto ARGS.
12202 The attribute value is a TREE_LIST. A set of dimensions is
12203 represented as a list of INTEGER_CST. Those that are runtime
12204 expres are represented as an INTEGER_CST of zero.
12206 TOOO. Normally the attribute will just contain a single such list. If
12207 however it contains a list of lists, this will represent the use of
12208 device_type. Each member of the outer list is an assoc list of
12209 dimensions, keyed by the device type. The first entry will be the
12210 default. Well, that's the plan. */
12212 #define OACC_FN_ATTRIB "oacc function"
12214 /* Replace any existing oacc fn attribute with updated dimensions. */
12217 replace_oacc_fn_attrib (tree fn
, tree dims
)
12219 tree ident
= get_identifier (OACC_FN_ATTRIB
);
12220 tree attribs
= DECL_ATTRIBUTES (fn
);
12222 /* If we happen to be present as the first attrib, drop it. */
12223 if (attribs
&& TREE_PURPOSE (attribs
) == ident
)
12224 attribs
= TREE_CHAIN (attribs
);
12225 DECL_ATTRIBUTES (fn
) = tree_cons (ident
, dims
, attribs
);
12228 /* Scan CLAUSES for launch dimensions and attach them to the oacc
12229 function attribute. Push any that are non-constant onto the ARGS
12230 list, along with an appropriate GOMP_LAUNCH_DIM tag. */
12233 set_oacc_fn_attrib (tree fn
, tree clauses
, vec
<tree
> *args
)
12235 /* Must match GOMP_DIM ordering. */
12236 static const omp_clause_code ids
[]
12237 = { OMP_CLAUSE_NUM_GANGS
, OMP_CLAUSE_NUM_WORKERS
,
12238 OMP_CLAUSE_VECTOR_LENGTH
};
12240 tree dims
[GOMP_DIM_MAX
];
12241 tree attr
= NULL_TREE
;
12242 unsigned non_const
= 0;
12244 for (ix
= GOMP_DIM_MAX
; ix
--;)
12246 tree clause
= find_omp_clause (clauses
, ids
[ix
]);
12247 tree dim
= NULL_TREE
;
12250 dim
= OMP_CLAUSE_EXPR (clause
, ids
[ix
]);
12252 if (dim
&& TREE_CODE (dim
) != INTEGER_CST
)
12254 dim
= integer_zero_node
;
12255 non_const
|= GOMP_DIM_MASK (ix
);
12257 attr
= tree_cons (NULL_TREE
, dim
, attr
);
12260 replace_oacc_fn_attrib (fn
, attr
);
12264 /* Push a dynamic argument set. */
12265 args
->safe_push (oacc_launch_pack (GOMP_LAUNCH_DIM
,
12266 NULL_TREE
, non_const
));
12267 for (unsigned ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
12268 if (non_const
& GOMP_DIM_MASK (ix
))
12269 args
->safe_push (dims
[ix
]);
12273 /* Retrieve the oacc function attrib and return it. Non-oacc
12274 functions will return NULL. */
12277 get_oacc_fn_attrib (tree fn
)
12279 return lookup_attribute (OACC_FN_ATTRIB
, DECL_ATTRIBUTES (fn
));
12282 /* Extract an oacc execution dimension from FN. FN must be an
12283 offloaded function or routine that has already had its execution
12284 dimensions lowered to the target-specific values. */
12287 get_oacc_fn_dim_size (tree fn
, int axis
)
12289 tree attrs
= get_oacc_fn_attrib (fn
);
12291 gcc_assert (axis
< GOMP_DIM_MAX
);
12293 tree dims
= TREE_VALUE (attrs
);
12295 dims
= TREE_CHAIN (dims
);
12297 int size
= TREE_INT_CST_LOW (TREE_VALUE (dims
));
12302 /* Extract the dimension axis from an IFN_GOACC_DIM_POS or
12303 IFN_GOACC_DIM_SIZE call. */
12306 get_oacc_ifn_dim_arg (const gimple
*stmt
)
12308 gcc_checking_assert (gimple_call_internal_fn (stmt
) == IFN_GOACC_DIM_SIZE
12309 || gimple_call_internal_fn (stmt
) == IFN_GOACC_DIM_POS
);
12310 tree arg
= gimple_call_arg (stmt
, 0);
12311 HOST_WIDE_INT axis
= TREE_INT_CST_LOW (arg
);
12313 gcc_checking_assert (axis
>= 0 && axis
< GOMP_DIM_MAX
);
12317 /* Expand the GIMPLE_OMP_TARGET starting at REGION. */
12320 expand_omp_target (struct omp_region
*region
)
12322 basic_block entry_bb
, exit_bb
, new_bb
;
12323 struct function
*child_cfun
;
12324 tree child_fn
, block
, t
;
12325 gimple_stmt_iterator gsi
;
12326 gomp_target
*entry_stmt
;
12329 bool offloaded
, data_region
;
12331 entry_stmt
= as_a
<gomp_target
*> (last_stmt (region
->entry
));
12332 new_bb
= region
->entry
;
12334 offloaded
= is_gimple_omp_offloaded (entry_stmt
);
12335 switch (gimple_omp_target_kind (entry_stmt
))
12337 case GF_OMP_TARGET_KIND_REGION
:
12338 case GF_OMP_TARGET_KIND_UPDATE
:
12339 case GF_OMP_TARGET_KIND_ENTER_DATA
:
12340 case GF_OMP_TARGET_KIND_EXIT_DATA
:
12341 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
12342 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
12343 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
12344 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
12345 data_region
= false;
12347 case GF_OMP_TARGET_KIND_DATA
:
12348 case GF_OMP_TARGET_KIND_OACC_DATA
:
12349 data_region
= true;
12352 gcc_unreachable ();
12355 child_fn
= NULL_TREE
;
12359 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
12360 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
12363 /* Supported by expand_omp_taskreg, but not here. */
12364 if (child_cfun
!= NULL
)
12365 gcc_checking_assert (!child_cfun
->cfg
);
12366 gcc_checking_assert (!gimple_in_ssa_p (cfun
));
12368 entry_bb
= region
->entry
;
12369 exit_bb
= region
->exit
;
12373 unsigned srcidx
, dstidx
, num
;
12375 /* If the offloading region needs data sent from the parent
12376 function, then the very first statement (except possible
12377 tree profile counter updates) of the offloading body
12378 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
12379 &.OMP_DATA_O is passed as an argument to the child function,
12380 we need to replace it with the argument as seen by the child
12383 In most cases, this will end up being the identity assignment
12384 .OMP_DATA_I = .OMP_DATA_I. However, if the offloading body had
12385 a function call that has been inlined, the original PARM_DECL
12386 .OMP_DATA_I may have been converted into a different local
12387 variable. In which case, we need to keep the assignment. */
12388 tree data_arg
= gimple_omp_target_data_arg (entry_stmt
);
12391 basic_block entry_succ_bb
= single_succ (entry_bb
);
12392 gimple_stmt_iterator gsi
;
12394 gimple
*tgtcopy_stmt
= NULL
;
12395 tree sender
= TREE_VEC_ELT (data_arg
, 0);
12397 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
12399 gcc_assert (!gsi_end_p (gsi
));
12400 stmt
= gsi_stmt (gsi
);
12401 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
12404 if (gimple_num_ops (stmt
) == 2)
12406 tree arg
= gimple_assign_rhs1 (stmt
);
12408 /* We're ignoring the subcode because we're
12409 effectively doing a STRIP_NOPS. */
12411 if (TREE_CODE (arg
) == ADDR_EXPR
12412 && TREE_OPERAND (arg
, 0) == sender
)
12414 tgtcopy_stmt
= stmt
;
12420 gcc_assert (tgtcopy_stmt
!= NULL
);
12421 arg
= DECL_ARGUMENTS (child_fn
);
12423 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
12424 gsi_remove (&gsi
, true);
12427 /* Declare local variables needed in CHILD_CFUN. */
12428 block
= DECL_INITIAL (child_fn
);
12429 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
12430 /* The gimplifier could record temporaries in the offloading block
12431 rather than in containing function's local_decls chain,
12432 which would mean cgraph missed finalizing them. Do it now. */
12433 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
12434 if (TREE_CODE (t
) == VAR_DECL
12436 && !DECL_EXTERNAL (t
))
12437 varpool_node::finalize_decl (t
);
12438 DECL_SAVED_TREE (child_fn
) = NULL
;
12439 /* We'll create a CFG for child_fn, so no gimple body is needed. */
12440 gimple_set_body (child_fn
, NULL
);
12441 TREE_USED (block
) = 1;
12443 /* Reset DECL_CONTEXT on function arguments. */
12444 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
12445 DECL_CONTEXT (t
) = child_fn
;
12447 /* Split ENTRY_BB at GIMPLE_*,
12448 so that it can be moved to the child function. */
12449 gsi
= gsi_last_bb (entry_bb
);
12450 stmt
= gsi_stmt (gsi
);
12452 && gimple_code (stmt
) == gimple_code (entry_stmt
));
12453 e
= split_block (entry_bb
, stmt
);
12454 gsi_remove (&gsi
, true);
12455 entry_bb
= e
->dest
;
12456 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
12458 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
12461 gsi
= gsi_last_bb (exit_bb
);
12462 gcc_assert (!gsi_end_p (gsi
)
12463 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
12464 stmt
= gimple_build_return (NULL
);
12465 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
12466 gsi_remove (&gsi
, true);
12469 /* Move the offloading region into CHILD_CFUN. */
12471 block
= gimple_block (entry_stmt
);
12473 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
12475 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
12476 /* When the OMP expansion process cannot guarantee an up-to-date
12477 loop tree arrange for the child function to fixup loops. */
12478 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
12479 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
12481 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
12482 num
= vec_safe_length (child_cfun
->local_decls
);
12483 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
12485 t
= (*child_cfun
->local_decls
)[srcidx
];
12486 if (DECL_CONTEXT (t
) == cfun
->decl
)
12488 if (srcidx
!= dstidx
)
12489 (*child_cfun
->local_decls
)[dstidx
] = t
;
12493 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
12495 /* Inform the callgraph about the new function. */
12496 child_cfun
->curr_properties
= cfun
->curr_properties
;
12497 child_cfun
->has_simduid_loops
|= cfun
->has_simduid_loops
;
12498 child_cfun
->has_force_vectorize_loops
|= cfun
->has_force_vectorize_loops
;
12499 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
12500 node
->parallelized_function
= 1;
12501 cgraph_node::add_new_function (child_fn
, true);
12503 #ifdef ENABLE_OFFLOADING
12504 /* Add the new function to the offload table. */
12505 vec_safe_push (offload_funcs
, child_fn
);
12508 bool need_asm
= DECL_ASSEMBLER_NAME_SET_P (current_function_decl
)
12509 && !DECL_ASSEMBLER_NAME_SET_P (child_fn
);
12511 /* Fix the callgraph edges for child_cfun. Those for cfun will be
12512 fixed in a following pass. */
12513 push_cfun (child_cfun
);
12515 assign_assembler_name_if_neeeded (child_fn
);
12516 cgraph_edge::rebuild_edges ();
12518 #ifdef ENABLE_OFFLOADING
12519 /* Prevent IPA from removing child_fn as unreachable, since there are no
12520 refs from the parent function to child_fn in offload LTO mode. */
12521 cgraph_node::get (child_fn
)->mark_force_output ();
12524 /* Some EH regions might become dead, see PR34608. If
12525 pass_cleanup_cfg isn't the first pass to happen with the
12526 new child, these dead EH edges might cause problems.
12527 Clean them up now. */
12528 if (flag_exceptions
)
12531 bool changed
= false;
12533 FOR_EACH_BB_FN (bb
, cfun
)
12534 changed
|= gimple_purge_dead_eh_edges (bb
);
12536 cleanup_tree_cfg ();
12538 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
12539 verify_loop_structure ();
12542 if (dump_file
&& !gimple_in_ssa_p (cfun
))
12544 omp_any_child_fn_dumped
= true;
12545 dump_function_header (dump_file
, child_fn
, dump_flags
);
12546 dump_function_to_file (child_fn
, dump_file
, dump_flags
);
12550 /* Emit a library call to launch the offloading region, or do data
12552 tree t1
, t2
, t3
, t4
, device
, cond
, depend
, c
, clauses
;
12553 enum built_in_function start_ix
;
12554 location_t clause_loc
;
12555 unsigned int flags_i
= 0;
12557 switch (gimple_omp_target_kind (entry_stmt
))
12559 case GF_OMP_TARGET_KIND_REGION
:
12560 start_ix
= BUILT_IN_GOMP_TARGET
;
12562 case GF_OMP_TARGET_KIND_DATA
:
12563 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
12565 case GF_OMP_TARGET_KIND_UPDATE
:
12566 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
12568 case GF_OMP_TARGET_KIND_ENTER_DATA
:
12569 start_ix
= BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
;
12571 case GF_OMP_TARGET_KIND_EXIT_DATA
:
12572 start_ix
= BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
;
12573 flags_i
|= GOMP_TARGET_FLAG_EXIT_DATA
;
12575 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
12576 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
12577 start_ix
= BUILT_IN_GOACC_PARALLEL
;
12579 case GF_OMP_TARGET_KIND_OACC_DATA
:
12580 start_ix
= BUILT_IN_GOACC_DATA_START
;
12582 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
12583 start_ix
= BUILT_IN_GOACC_UPDATE
;
12585 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
12586 start_ix
= BUILT_IN_GOACC_ENTER_EXIT_DATA
;
12589 gcc_unreachable ();
12592 clauses
= gimple_omp_target_clauses (entry_stmt
);
12594 /* By default, the value of DEVICE is GOMP_DEVICE_ICV (let runtime
12595 library choose) and there is no conditional. */
12597 device
= build_int_cst (integer_type_node
, GOMP_DEVICE_ICV
);
12599 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
12601 cond
= OMP_CLAUSE_IF_EXPR (c
);
12603 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
12606 /* Even if we pass it to all library function calls, it is currently only
12607 defined/used for the OpenMP target ones. */
12608 gcc_checking_assert (start_ix
== BUILT_IN_GOMP_TARGET
12609 || start_ix
== BUILT_IN_GOMP_TARGET_DATA
12610 || start_ix
== BUILT_IN_GOMP_TARGET_UPDATE
12611 || start_ix
== BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
);
12613 device
= OMP_CLAUSE_DEVICE_ID (c
);
12614 clause_loc
= OMP_CLAUSE_LOCATION (c
);
12617 clause_loc
= gimple_location (entry_stmt
);
12619 c
= find_omp_clause (clauses
, OMP_CLAUSE_NOWAIT
);
12621 flags_i
|= GOMP_TARGET_FLAG_NOWAIT
;
12623 /* Ensure 'device' is of the correct type. */
12624 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
12626 /* If we found the clause 'if (cond)', build
12627 (cond ? device : GOMP_DEVICE_HOST_FALLBACK). */
12630 cond
= gimple_boolify (cond
);
12632 basic_block cond_bb
, then_bb
, else_bb
;
12636 tmp_var
= create_tmp_var (TREE_TYPE (device
));
12638 e
= split_block_after_labels (new_bb
);
12641 gsi
= gsi_last_bb (new_bb
);
12643 e
= split_block (new_bb
, gsi_stmt (gsi
));
12649 then_bb
= create_empty_bb (cond_bb
);
12650 else_bb
= create_empty_bb (then_bb
);
12651 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
12652 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
12654 stmt
= gimple_build_cond_empty (cond
);
12655 gsi
= gsi_last_bb (cond_bb
);
12656 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
12658 gsi
= gsi_start_bb (then_bb
);
12659 stmt
= gimple_build_assign (tmp_var
, device
);
12660 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
12662 gsi
= gsi_start_bb (else_bb
);
12663 stmt
= gimple_build_assign (tmp_var
,
12664 build_int_cst (integer_type_node
,
12665 GOMP_DEVICE_HOST_FALLBACK
));
12666 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
12668 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
12669 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
12670 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
12671 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
12672 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
12673 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
12678 gsi
= gsi_last_bb (new_bb
);
12679 t
= gimple_omp_target_data_arg (entry_stmt
);
12682 t1
= size_zero_node
;
12683 t2
= build_zero_cst (ptr_type_node
);
12689 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
12690 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
12691 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
12692 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
12693 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
12697 bool tagging
= false;
12698 /* The maximum number used by any start_ix, without varargs. */
12699 auto_vec
<tree
, 11> args
;
12700 args
.quick_push (device
);
12702 args
.quick_push (build_fold_addr_expr (child_fn
));
12703 args
.quick_push (t1
);
12704 args
.quick_push (t2
);
12705 args
.quick_push (t3
);
12706 args
.quick_push (t4
);
12709 case BUILT_IN_GOACC_DATA_START
:
12710 case BUILT_IN_GOMP_TARGET_DATA
:
12712 case BUILT_IN_GOMP_TARGET
:
12713 case BUILT_IN_GOMP_TARGET_UPDATE
:
12714 case BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
:
12715 args
.quick_push (build_int_cst (unsigned_type_node
, flags_i
));
12716 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
12718 depend
= OMP_CLAUSE_DECL (c
);
12720 depend
= build_int_cst (ptr_type_node
, 0);
12721 args
.quick_push (depend
);
12722 if (start_ix
== BUILT_IN_GOMP_TARGET
)
12724 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_TEAMS
);
12727 t
= fold_convert (integer_type_node
,
12728 OMP_CLAUSE_NUM_TEAMS_EXPR (c
));
12729 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL
,
12730 true, GSI_SAME_STMT
);
12733 t
= integer_minus_one_node
;
12734 args
.quick_push (t
);
12735 c
= find_omp_clause (clauses
, OMP_CLAUSE_THREAD_LIMIT
);
12738 t
= fold_convert (integer_type_node
,
12739 OMP_CLAUSE_THREAD_LIMIT_EXPR (c
));
12740 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL
,
12741 true, GSI_SAME_STMT
);
12744 t
= integer_minus_one_node
;
12745 args
.quick_push (t
);
12748 case BUILT_IN_GOACC_PARALLEL
:
12750 set_oacc_fn_attrib (child_fn
, clauses
, &args
);
12754 case BUILT_IN_GOACC_ENTER_EXIT_DATA
:
12755 case BUILT_IN_GOACC_UPDATE
:
12757 tree t_async
= NULL_TREE
;
12759 /* If present, use the value specified by the respective
12760 clause, making sure that is of the correct type. */
12761 c
= find_omp_clause (clauses
, OMP_CLAUSE_ASYNC
);
12763 t_async
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
12765 OMP_CLAUSE_ASYNC_EXPR (c
));
12767 /* Default values for t_async. */
12768 t_async
= fold_convert_loc (gimple_location (entry_stmt
),
12770 build_int_cst (integer_type_node
,
12772 if (tagging
&& t_async
)
12774 unsigned HOST_WIDE_INT i_async
;
12776 if (TREE_CODE (t_async
) == INTEGER_CST
)
12778 /* See if we can pack the async arg in to the tag's
12780 i_async
= TREE_INT_CST_LOW (t_async
);
12782 if (i_async
< GOMP_LAUNCH_OP_MAX
)
12783 t_async
= NULL_TREE
;
12786 i_async
= GOMP_LAUNCH_OP_MAX
;
12787 args
.safe_push (oacc_launch_pack
12788 (GOMP_LAUNCH_ASYNC
, NULL_TREE
, i_async
));
12791 args
.safe_push (t_async
);
12793 /* Save the argument index, and ... */
12794 unsigned t_wait_idx
= args
.length ();
12795 unsigned num_waits
= 0;
12796 c
= find_omp_clause (clauses
, OMP_CLAUSE_WAIT
);
12798 /* ... push a placeholder. */
12799 args
.safe_push (integer_zero_node
);
12801 for (; c
; c
= OMP_CLAUSE_CHAIN (c
))
12802 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_WAIT
)
12804 args
.safe_push (fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
12806 OMP_CLAUSE_WAIT_EXPR (c
)));
12810 if (!tagging
|| num_waits
)
12814 /* Now that we know the number, update the placeholder. */
12816 len
= oacc_launch_pack (GOMP_LAUNCH_WAIT
, NULL_TREE
, num_waits
);
12818 len
= build_int_cst (integer_type_node
, num_waits
);
12819 len
= fold_convert_loc (gimple_location (entry_stmt
),
12820 unsigned_type_node
, len
);
12821 args
[t_wait_idx
] = len
;
12826 gcc_unreachable ();
12829 /* Push terminal marker - zero. */
12830 args
.safe_push (oacc_launch_pack (0, NULL_TREE
, 0));
12832 g
= gimple_build_call_vec (builtin_decl_explicit (start_ix
), args
);
12833 gimple_set_location (g
, gimple_location (entry_stmt
));
12834 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
12837 g
= gsi_stmt (gsi
);
12838 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
12839 gsi_remove (&gsi
, true);
12841 if (data_region
&& region
->exit
)
12843 gsi
= gsi_last_bb (region
->exit
);
12844 g
= gsi_stmt (gsi
);
12845 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
12846 gsi_remove (&gsi
, true);
12851 /* Expand the parallel region tree rooted at REGION. Expansion
12852 proceeds in depth-first order. Innermost regions are expanded
12853 first. This way, parallel regions that require a new function to
12854 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
12855 internal dependencies in their body. */
12858 expand_omp (struct omp_region
*region
)
12860 omp_any_child_fn_dumped
= false;
12863 location_t saved_location
;
12864 gimple
*inner_stmt
= NULL
;
12866 /* First, determine whether this is a combined parallel+workshare
12868 if (region
->type
== GIMPLE_OMP_PARALLEL
)
12869 determine_parallel_type (region
);
12871 if (region
->type
== GIMPLE_OMP_FOR
12872 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
12873 inner_stmt
= last_stmt (region
->inner
->entry
);
12876 expand_omp (region
->inner
);
12878 saved_location
= input_location
;
12879 if (gimple_has_location (last_stmt (region
->entry
)))
12880 input_location
= gimple_location (last_stmt (region
->entry
));
12882 switch (region
->type
)
12884 case GIMPLE_OMP_PARALLEL
:
12885 case GIMPLE_OMP_TASK
:
12886 expand_omp_taskreg (region
);
12889 case GIMPLE_OMP_FOR
:
12890 expand_omp_for (region
, inner_stmt
);
12893 case GIMPLE_OMP_SECTIONS
:
12894 expand_omp_sections (region
);
12897 case GIMPLE_OMP_SECTION
:
12898 /* Individual omp sections are handled together with their
12899 parent GIMPLE_OMP_SECTIONS region. */
12902 case GIMPLE_OMP_SINGLE
:
12903 expand_omp_single (region
);
12906 case GIMPLE_OMP_ORDERED
:
12908 gomp_ordered
*ord_stmt
12909 = as_a
<gomp_ordered
*> (last_stmt (region
->entry
));
12910 if (find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
12911 OMP_CLAUSE_DEPEND
))
12913 /* We'll expand these when expanding corresponding
12914 worksharing region with ordered(n) clause. */
12915 gcc_assert (region
->outer
12916 && region
->outer
->type
== GIMPLE_OMP_FOR
);
12917 region
->ord_stmt
= ord_stmt
;
12922 case GIMPLE_OMP_MASTER
:
12923 case GIMPLE_OMP_TASKGROUP
:
12924 case GIMPLE_OMP_CRITICAL
:
12925 case GIMPLE_OMP_TEAMS
:
12926 expand_omp_synch (region
);
12929 case GIMPLE_OMP_ATOMIC_LOAD
:
12930 expand_omp_atomic (region
);
12933 case GIMPLE_OMP_TARGET
:
12934 expand_omp_target (region
);
12938 gcc_unreachable ();
12941 input_location
= saved_location
;
12942 region
= region
->next
;
12944 if (omp_any_child_fn_dumped
)
12947 dump_function_header (dump_file
, current_function_decl
, dump_flags
);
12948 omp_any_child_fn_dumped
= false;
12953 /* Helper for build_omp_regions. Scan the dominator tree starting at
12954 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
12955 true, the function ends once a single tree is built (otherwise, whole
12956 forest of OMP constructs may be built). */
12959 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
12962 gimple_stmt_iterator gsi
;
12966 gsi
= gsi_last_bb (bb
);
12967 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
12969 struct omp_region
*region
;
12970 enum gimple_code code
;
12972 stmt
= gsi_stmt (gsi
);
12973 code
= gimple_code (stmt
);
12974 if (code
== GIMPLE_OMP_RETURN
)
12976 /* STMT is the return point out of region PARENT. Mark it
12977 as the exit point and make PARENT the immediately
12978 enclosing region. */
12979 gcc_assert (parent
);
12982 parent
= parent
->outer
;
12984 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
12986 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
12987 GIMPLE_OMP_RETURN, but matches with
12988 GIMPLE_OMP_ATOMIC_LOAD. */
12989 gcc_assert (parent
);
12990 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
12993 parent
= parent
->outer
;
12995 else if (code
== GIMPLE_OMP_CONTINUE
)
12997 gcc_assert (parent
);
13000 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
13002 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
13003 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
13007 region
= new_omp_region (bb
, code
, parent
);
13009 if (code
== GIMPLE_OMP_TARGET
)
13011 switch (gimple_omp_target_kind (stmt
))
13013 case GF_OMP_TARGET_KIND_REGION
:
13014 case GF_OMP_TARGET_KIND_DATA
:
13015 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
13016 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
13017 case GF_OMP_TARGET_KIND_OACC_DATA
:
13019 case GF_OMP_TARGET_KIND_UPDATE
:
13020 case GF_OMP_TARGET_KIND_ENTER_DATA
:
13021 case GF_OMP_TARGET_KIND_EXIT_DATA
:
13022 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
13023 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
13024 /* ..., other than for those stand-alone directives... */
13028 gcc_unreachable ();
13031 else if (code
== GIMPLE_OMP_ORDERED
13032 && find_omp_clause (gimple_omp_ordered_clauses
13033 (as_a
<gomp_ordered
*> (stmt
)),
13034 OMP_CLAUSE_DEPEND
))
13035 /* #pragma omp ordered depend is also just a stand-alone
13038 /* ..., this directive becomes the parent for a new region. */
13044 if (single_tree
&& !parent
)
13047 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
13049 son
= next_dom_son (CDI_DOMINATORS
, son
))
13050 build_omp_regions_1 (son
, parent
, single_tree
);
13053 /* Builds the tree of OMP regions rooted at ROOT, storing it to
13054 root_omp_region. */
13057 build_omp_regions_root (basic_block root
)
13059 gcc_assert (root_omp_region
== NULL
);
13060 build_omp_regions_1 (root
, NULL
, true);
13061 gcc_assert (root_omp_region
!= NULL
);
13064 /* Expands omp construct (and its subconstructs) starting in HEAD. */
13067 omp_expand_local (basic_block head
)
13069 build_omp_regions_root (head
);
13070 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
13072 fprintf (dump_file
, "\nOMP region tree\n\n");
13073 dump_omp_region (dump_file
, root_omp_region
, 0);
13074 fprintf (dump_file
, "\n");
13077 remove_exit_barriers (root_omp_region
);
13078 expand_omp (root_omp_region
);
13080 free_omp_regions ();
13083 /* Scan the CFG and build a tree of OMP regions. Return the root of
13084 the OMP region tree. */
13087 build_omp_regions (void)
13089 gcc_assert (root_omp_region
== NULL
);
13090 calculate_dominance_info (CDI_DOMINATORS
);
13091 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
13094 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
13096 static unsigned int
13097 execute_expand_omp (void)
13099 build_omp_regions ();
13101 if (!root_omp_region
)
13106 fprintf (dump_file
, "\nOMP region tree\n\n");
13107 dump_omp_region (dump_file
, root_omp_region
, 0);
13108 fprintf (dump_file
, "\n");
13111 remove_exit_barriers (root_omp_region
);
13113 expand_omp (root_omp_region
);
13115 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
13116 verify_loop_structure ();
13117 cleanup_tree_cfg ();
13119 free_omp_regions ();
13124 /* OMP expansion -- the default pass, run before creation of SSA form. */
13128 const pass_data pass_data_expand_omp
=
13130 GIMPLE_PASS
, /* type */
13131 "ompexp", /* name */
13132 OPTGROUP_NONE
, /* optinfo_flags */
13133 TV_NONE
, /* tv_id */
13134 PROP_gimple_any
, /* properties_required */
13135 PROP_gimple_eomp
, /* properties_provided */
13136 0, /* properties_destroyed */
13137 0, /* todo_flags_start */
13138 0, /* todo_flags_finish */
13141 class pass_expand_omp
: public gimple_opt_pass
13144 pass_expand_omp (gcc::context
*ctxt
)
13145 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
13148 /* opt_pass methods: */
13149 virtual unsigned int execute (function
*)
13151 bool gate
= ((flag_cilkplus
!= 0 || flag_openacc
!= 0 || flag_openmp
!= 0
13152 || flag_openmp_simd
!= 0)
13153 && !seen_error ());
13155 /* This pass always runs, to provide PROP_gimple_eomp.
13156 But often, there is nothing to do. */
13160 return execute_expand_omp ();
13163 }; // class pass_expand_omp
13165 } // anon namespace
13168 make_pass_expand_omp (gcc::context
*ctxt
)
13170 return new pass_expand_omp (ctxt
);
13175 const pass_data pass_data_expand_omp_ssa
=
13177 GIMPLE_PASS
, /* type */
13178 "ompexpssa", /* name */
13179 OPTGROUP_NONE
, /* optinfo_flags */
13180 TV_NONE
, /* tv_id */
13181 PROP_cfg
| PROP_ssa
, /* properties_required */
13182 PROP_gimple_eomp
, /* properties_provided */
13183 0, /* properties_destroyed */
13184 0, /* todo_flags_start */
13185 TODO_cleanup_cfg
| TODO_rebuild_alias
, /* todo_flags_finish */
13188 class pass_expand_omp_ssa
: public gimple_opt_pass
13191 pass_expand_omp_ssa (gcc::context
*ctxt
)
13192 : gimple_opt_pass (pass_data_expand_omp_ssa
, ctxt
)
13195 /* opt_pass methods: */
13196 virtual bool gate (function
*fun
)
13198 return !(fun
->curr_properties
& PROP_gimple_eomp
);
13200 virtual unsigned int execute (function
*) { return execute_expand_omp (); }
13202 }; // class pass_expand_omp_ssa
13204 } // anon namespace
13207 make_pass_expand_omp_ssa (gcc::context
*ctxt
)
13209 return new pass_expand_omp_ssa (ctxt
);
13212 /* Routines to lower OMP directives into OMP-GIMPLE. */
13214 /* If ctx is a worksharing context inside of a cancellable parallel
13215 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
13216 and conditional branch to parallel's cancel_label to handle
13217 cancellation in the implicit barrier. */
13220 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
13222 gimple
*omp_return
= gimple_seq_last_stmt (*body
);
13223 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
13224 if (gimple_omp_return_nowait_p (omp_return
))
13227 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
13228 && ctx
->outer
->cancellable
)
13230 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
13231 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
13232 tree lhs
= create_tmp_var (c_bool_type
);
13233 gimple_omp_return_set_lhs (omp_return
, lhs
);
13234 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
13235 gimple
*g
= gimple_build_cond (NE_EXPR
, lhs
,
13236 fold_convert (c_bool_type
,
13237 boolean_false_node
),
13238 ctx
->outer
->cancel_label
, fallthru_label
);
13239 gimple_seq_add_stmt (body
, g
);
13240 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
13244 /* Lower the OpenMP sections directive in the current statement in GSI_P.
13245 CTX is the enclosing OMP context for the current statement. */
13248 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
13250 tree block
, control
;
13251 gimple_stmt_iterator tgsi
;
13252 gomp_sections
*stmt
;
13254 gbind
*new_stmt
, *bind
;
13255 gimple_seq ilist
, dlist
, olist
, new_body
;
13257 stmt
= as_a
<gomp_sections
*> (gsi_stmt (*gsi_p
));
13259 push_gimplify_context ();
13263 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
13264 &ilist
, &dlist
, ctx
, NULL
);
13266 new_body
= gimple_omp_body (stmt
);
13267 gimple_omp_set_body (stmt
, NULL
);
13268 tgsi
= gsi_start (new_body
);
13269 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
13274 sec_start
= gsi_stmt (tgsi
);
13275 sctx
= maybe_lookup_ctx (sec_start
);
13278 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
13279 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
13280 GSI_CONTINUE_LINKING
);
13281 gimple_omp_set_body (sec_start
, NULL
);
13283 if (gsi_one_before_end_p (tgsi
))
13285 gimple_seq l
= NULL
;
13286 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
13288 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
13289 gimple_omp_section_set_last (sec_start
);
13292 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
13293 GSI_CONTINUE_LINKING
);
13296 block
= make_node (BLOCK
);
13297 bind
= gimple_build_bind (NULL
, new_body
, block
);
13300 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
13302 block
= make_node (BLOCK
);
13303 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
13304 gsi_replace (gsi_p
, new_stmt
, true);
13306 pop_gimplify_context (new_stmt
);
13307 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
13308 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
13309 if (BLOCK_VARS (block
))
13310 TREE_USED (block
) = 1;
13313 gimple_seq_add_seq (&new_body
, ilist
);
13314 gimple_seq_add_stmt (&new_body
, stmt
);
13315 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
13316 gimple_seq_add_stmt (&new_body
, bind
);
13318 control
= create_tmp_var (unsigned_type_node
, ".section");
13319 t
= gimple_build_omp_continue (control
, control
);
13320 gimple_omp_sections_set_control (stmt
, control
);
13321 gimple_seq_add_stmt (&new_body
, t
);
13323 gimple_seq_add_seq (&new_body
, olist
);
13324 if (ctx
->cancellable
)
13325 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
13326 gimple_seq_add_seq (&new_body
, dlist
);
13328 new_body
= maybe_catch_exception (new_body
);
13330 t
= gimple_build_omp_return
13331 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
13332 OMP_CLAUSE_NOWAIT
));
13333 gimple_seq_add_stmt (&new_body
, t
);
13334 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
13336 gimple_bind_set_body (new_stmt
, new_body
);
13340 /* A subroutine of lower_omp_single. Expand the simple form of
13341 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
13343 if (GOMP_single_start ())
13345 [ GOMP_barrier (); ] -> unless 'nowait' is present.
13347 FIXME. It may be better to delay expanding the logic of this until
13348 pass_expand_omp. The expanded logic may make the job more difficult
13349 to a synchronization analysis pass. */
13352 lower_omp_single_simple (gomp_single
*single_stmt
, gimple_seq
*pre_p
)
13354 location_t loc
= gimple_location (single_stmt
);
13355 tree tlabel
= create_artificial_label (loc
);
13356 tree flabel
= create_artificial_label (loc
);
13357 gimple
*call
, *cond
;
13360 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
13361 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)));
13362 call
= gimple_build_call (decl
, 0);
13363 gimple_call_set_lhs (call
, lhs
);
13364 gimple_seq_add_stmt (pre_p
, call
);
13366 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
13367 fold_convert_loc (loc
, TREE_TYPE (lhs
),
13368 boolean_true_node
),
13370 gimple_seq_add_stmt (pre_p
, cond
);
13371 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
13372 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
13373 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
13377 /* A subroutine of lower_omp_single. Expand the simple form of
13378 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
13380 #pragma omp single copyprivate (a, b, c)
13382 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
13385 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
13391 GOMP_single_copy_end (©out);
13402 FIXME. It may be better to delay expanding the logic of this until
13403 pass_expand_omp. The expanded logic may make the job more difficult
13404 to a synchronization analysis pass. */
13407 lower_omp_single_copy (gomp_single
*single_stmt
, gimple_seq
*pre_p
,
13410 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
13411 gimple_seq copyin_seq
;
13412 location_t loc
= gimple_location (single_stmt
);
13414 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
13416 ptr_type
= build_pointer_type (ctx
->record_type
);
13417 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
13419 l0
= create_artificial_label (loc
);
13420 l1
= create_artificial_label (loc
);
13421 l2
= create_artificial_label (loc
);
13423 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
13424 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
13425 t
= fold_convert_loc (loc
, ptr_type
, t
);
13426 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
13428 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
13429 build_int_cst (ptr_type
, 0));
13430 t
= build3 (COND_EXPR
, void_type_node
, t
,
13431 build_and_jump (&l0
), build_and_jump (&l1
));
13432 gimplify_and_add (t
, pre_p
);
13434 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
13436 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
13439 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
13442 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
13443 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
13444 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
13445 gimplify_and_add (t
, pre_p
);
13447 t
= build_and_jump (&l2
);
13448 gimplify_and_add (t
, pre_p
);
13450 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
13452 gimple_seq_add_seq (pre_p
, copyin_seq
);
13454 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
13458 /* Expand code for an OpenMP single directive. */
13461 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
13465 gomp_single
*single_stmt
= as_a
<gomp_single
*> (gsi_stmt (*gsi_p
));
13467 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
13469 push_gimplify_context ();
13471 block
= make_node (BLOCK
);
13472 bind
= gimple_build_bind (NULL
, NULL
, block
);
13473 gsi_replace (gsi_p
, bind
, true);
13476 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
13477 &bind_body
, &dlist
, ctx
, NULL
);
13478 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
13480 gimple_seq_add_stmt (&bind_body
, single_stmt
);
13482 if (ctx
->record_type
)
13483 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
13485 lower_omp_single_simple (single_stmt
, &bind_body
);
13487 gimple_omp_set_body (single_stmt
, NULL
);
13489 gimple_seq_add_seq (&bind_body
, dlist
);
13491 bind_body
= maybe_catch_exception (bind_body
);
13493 t
= gimple_build_omp_return
13494 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
13495 OMP_CLAUSE_NOWAIT
));
13496 gimple_seq_add_stmt (&bind_body_tail
, t
);
13497 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
13498 if (ctx
->record_type
)
13500 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
13501 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
13502 TREE_THIS_VOLATILE (clobber
) = 1;
13503 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
13504 clobber
), GSI_SAME_STMT
);
13506 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
13507 gimple_bind_set_body (bind
, bind_body
);
13509 pop_gimplify_context (bind
);
13511 gimple_bind_append_vars (bind
, ctx
->block_vars
);
13512 BLOCK_VARS (block
) = ctx
->block_vars
;
13513 if (BLOCK_VARS (block
))
13514 TREE_USED (block
) = 1;
13518 /* Expand code for an OpenMP master directive. */
13521 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
13523 tree block
, lab
= NULL
, x
, bfn_decl
;
13524 gimple
*stmt
= gsi_stmt (*gsi_p
);
13526 location_t loc
= gimple_location (stmt
);
13529 push_gimplify_context ();
13531 block
= make_node (BLOCK
);
13532 bind
= gimple_build_bind (NULL
, NULL
, block
);
13533 gsi_replace (gsi_p
, bind
, true);
13534 gimple_bind_add_stmt (bind
, stmt
);
13536 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
13537 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
13538 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
13539 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
13541 gimplify_and_add (x
, &tseq
);
13542 gimple_bind_add_seq (bind
, tseq
);
13544 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
13545 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
13546 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
13547 gimple_omp_set_body (stmt
, NULL
);
13549 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
13551 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
13553 pop_gimplify_context (bind
);
13555 gimple_bind_append_vars (bind
, ctx
->block_vars
);
13556 BLOCK_VARS (block
) = ctx
->block_vars
;
13560 /* Expand code for an OpenMP taskgroup directive. */
13563 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
13565 gimple
*stmt
= gsi_stmt (*gsi_p
);
13568 tree block
= make_node (BLOCK
);
13570 bind
= gimple_build_bind (NULL
, NULL
, block
);
13571 gsi_replace (gsi_p
, bind
, true);
13572 gimple_bind_add_stmt (bind
, stmt
);
13574 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
13576 gimple_bind_add_stmt (bind
, x
);
13578 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
13579 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
13580 gimple_omp_set_body (stmt
, NULL
);
13582 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
13584 gimple_bind_append_vars (bind
, ctx
->block_vars
);
13585 BLOCK_VARS (block
) = ctx
->block_vars
;
13589 /* Fold the OMP_ORDERED_CLAUSES for the OMP_ORDERED in STMT if possible. */
13592 lower_omp_ordered_clauses (gimple_stmt_iterator
*gsi_p
, gomp_ordered
*ord_stmt
,
13595 struct omp_for_data fd
;
13596 if (!ctx
->outer
|| gimple_code (ctx
->outer
->stmt
) != GIMPLE_OMP_FOR
)
13599 unsigned int len
= gimple_omp_for_collapse (ctx
->outer
->stmt
);
13600 struct omp_for_data_loop
*loops
= XALLOCAVEC (struct omp_for_data_loop
, len
);
13601 extract_omp_for_data (as_a
<gomp_for
*> (ctx
->outer
->stmt
), &fd
, loops
);
13605 tree
*list_p
= gimple_omp_ordered_clauses_ptr (ord_stmt
);
13606 tree c
= gimple_omp_ordered_clauses (ord_stmt
);
13607 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
13608 && OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
)
13610 /* Merge depend clauses from multiple adjacent
13611 #pragma omp ordered depend(sink:...) constructs
13612 into one #pragma omp ordered depend(sink:...), so that
13613 we can optimize them together. */
13614 gimple_stmt_iterator gsi
= *gsi_p
;
13616 while (!gsi_end_p (gsi
))
13618 gimple
*stmt
= gsi_stmt (gsi
);
13619 if (is_gimple_debug (stmt
)
13620 || gimple_code (stmt
) == GIMPLE_NOP
)
13625 if (gimple_code (stmt
) != GIMPLE_OMP_ORDERED
)
13627 gomp_ordered
*ord_stmt2
= as_a
<gomp_ordered
*> (stmt
);
13628 c
= gimple_omp_ordered_clauses (ord_stmt2
);
13630 || OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_DEPEND
13631 || OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_SINK
)
13634 list_p
= &OMP_CLAUSE_CHAIN (*list_p
);
13636 gsi_remove (&gsi
, true);
13640 /* Canonicalize sink dependence clauses into one folded clause if
13643 The basic algorithm is to create a sink vector whose first
13644 element is the GCD of all the first elements, and whose remaining
13645 elements are the minimum of the subsequent columns.
13647 We ignore dependence vectors whose first element is zero because
13648 such dependencies are known to be executed by the same thread.
13650 We take into account the direction of the loop, so a minimum
13651 becomes a maximum if the loop is iterating forwards. We also
13652 ignore sink clauses where the loop direction is unknown, or where
13653 the offsets are clearly invalid because they are not a multiple
13654 of the loop increment.
13658 #pragma omp for ordered(2)
13659 for (i=0; i < N; ++i)
13660 for (j=0; j < M; ++j)
13662 #pragma omp ordered \
13663 depend(sink:i-8,j-2) \
13664 depend(sink:i,j-1) \ // Completely ignored because i+0.
13665 depend(sink:i-4,j-3) \
13666 depend(sink:i-6,j-4)
13667 #pragma omp ordered depend(source)
13672 depend(sink:-gcd(8,4,6),-min(2,3,4))
13677 /* FIXME: Computing GCD's where the first element is zero is
13678 non-trivial in the presence of collapsed loops. Do this later. */
13679 if (fd
.collapse
> 1)
13682 wide_int
*folded_deps
= XALLOCAVEC (wide_int
, 2 * len
- 1);
13683 memset (folded_deps
, 0, sizeof (*folded_deps
) * (2 * len
- 1));
13684 tree folded_dep
= NULL_TREE
;
13685 /* TRUE if the first dimension's offset is negative. */
13686 bool neg_offset_p
= false;
13688 list_p
= gimple_omp_ordered_clauses_ptr (ord_stmt
);
13690 while ((c
= *list_p
) != NULL
)
13692 bool remove
= false;
13694 gcc_assert (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
);
13695 if (OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_SINK
)
13696 goto next_ordered_clause
;
13699 for (vec
= OMP_CLAUSE_DECL (c
), i
= 0;
13700 vec
&& TREE_CODE (vec
) == TREE_LIST
;
13701 vec
= TREE_CHAIN (vec
), ++i
)
13703 gcc_assert (i
< len
);
13705 /* extract_omp_for_data has canonicalized the condition. */
13706 gcc_assert (fd
.loops
[i
].cond_code
== LT_EXPR
13707 || fd
.loops
[i
].cond_code
== GT_EXPR
);
13708 bool forward
= fd
.loops
[i
].cond_code
== LT_EXPR
;
13709 bool maybe_lexically_later
= true;
13711 /* While the committee makes up its mind, bail if we have any
13712 non-constant steps. */
13713 if (TREE_CODE (fd
.loops
[i
].step
) != INTEGER_CST
)
13714 goto lower_omp_ordered_ret
;
13716 tree itype
= TREE_TYPE (TREE_VALUE (vec
));
13717 if (POINTER_TYPE_P (itype
))
13719 wide_int offset
= wide_int::from (TREE_PURPOSE (vec
),
13720 TYPE_PRECISION (itype
),
13721 TYPE_SIGN (itype
));
13723 /* Ignore invalid offsets that are not multiples of the step. */
13724 if (!wi::multiple_of_p
13725 (wi::abs (offset
), wi::abs ((wide_int
) fd
.loops
[i
].step
),
13728 warning_at (OMP_CLAUSE_LOCATION (c
), 0,
13729 "ignoring sink clause with offset that is not "
13730 "a multiple of the loop step");
13732 goto next_ordered_clause
;
13735 /* Calculate the first dimension. The first dimension of
13736 the folded dependency vector is the GCD of the first
13737 elements, while ignoring any first elements whose offset
13741 /* Ignore dependence vectors whose first dimension is 0. */
13745 goto next_ordered_clause
;
13749 if (!TYPE_UNSIGNED (itype
) && (forward
^ wi::neg_p (offset
)))
13751 error_at (OMP_CLAUSE_LOCATION (c
),
13752 "first offset must be in opposite direction "
13753 "of loop iterations");
13754 goto lower_omp_ordered_ret
;
13758 neg_offset_p
= forward
;
13759 /* Initialize the first time around. */
13760 if (folded_dep
== NULL_TREE
)
13763 folded_deps
[0] = offset
;
13766 folded_deps
[0] = wi::gcd (folded_deps
[0],
13770 /* Calculate minimum for the remaining dimensions. */
13773 folded_deps
[len
+ i
- 1] = offset
;
13774 if (folded_dep
== c
)
13775 folded_deps
[i
] = offset
;
13776 else if (maybe_lexically_later
13777 && !wi::eq_p (folded_deps
[i
], offset
))
13779 if (forward
^ wi::gts_p (folded_deps
[i
], offset
))
13783 for (j
= 1; j
<= i
; j
++)
13784 folded_deps
[j
] = folded_deps
[len
+ j
- 1];
13787 maybe_lexically_later
= false;
13791 gcc_assert (i
== len
);
13795 next_ordered_clause
:
13797 *list_p
= OMP_CLAUSE_CHAIN (c
);
13799 list_p
= &OMP_CLAUSE_CHAIN (c
);
13805 folded_deps
[0] = -folded_deps
[0];
13807 tree itype
= TREE_TYPE (TREE_VALUE (OMP_CLAUSE_DECL (folded_dep
)));
13808 if (POINTER_TYPE_P (itype
))
13811 TREE_PURPOSE (OMP_CLAUSE_DECL (folded_dep
))
13812 = wide_int_to_tree (itype
, folded_deps
[0]);
13813 OMP_CLAUSE_CHAIN (folded_dep
) = gimple_omp_ordered_clauses (ord_stmt
);
13814 *gimple_omp_ordered_clauses_ptr (ord_stmt
) = folded_dep
;
13817 lower_omp_ordered_ret
:
13819 /* Ordered without clauses is #pragma omp threads, while we want
13820 a nop instead if we remove all clauses. */
13821 if (gimple_omp_ordered_clauses (ord_stmt
) == NULL_TREE
)
13822 gsi_replace (gsi_p
, gimple_build_nop (), true);
13826 /* Expand code for an OpenMP ordered directive. */
13829 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
13832 gimple
*stmt
= gsi_stmt (*gsi_p
);
13833 gomp_ordered
*ord_stmt
= as_a
<gomp_ordered
*> (stmt
);
13837 = find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
), OMP_CLAUSE_SIMD
);
13839 if (find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
13840 OMP_CLAUSE_DEPEND
))
13842 /* FIXME: This is needs to be moved to the expansion to verify various
13843 conditions only testable on cfg with dominators computed, and also
13844 all the depend clauses to be merged still might need to be available
13845 for the runtime checks. */
13847 lower_omp_ordered_clauses (gsi_p
, ord_stmt
, ctx
);
13851 push_gimplify_context ();
13853 block
= make_node (BLOCK
);
13854 bind
= gimple_build_bind (NULL
, NULL
, block
);
13855 gsi_replace (gsi_p
, bind
, true);
13856 gimple_bind_add_stmt (bind
, stmt
);
13860 x
= gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_START
, 0);
13861 cfun
->has_simduid_loops
= true;
13864 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
13866 gimple_bind_add_stmt (bind
, x
);
13868 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
13869 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
13870 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
13871 gimple_omp_set_body (stmt
, NULL
);
13874 x
= gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_END
, 0);
13876 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
),
13878 gimple_bind_add_stmt (bind
, x
);
13880 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
13882 pop_gimplify_context (bind
);
13884 gimple_bind_append_vars (bind
, ctx
->block_vars
);
13885 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
13889 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
13890 substitution of a couple of function calls. But in the NAMED case,
13891 requires that languages coordinate a symbol name. It is therefore
13892 best put here in common code. */
13894 static GTY(()) hash_map
<tree
, tree
> *critical_name_mutexes
;
13897 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
13900 tree name
, lock
, unlock
;
13901 gomp_critical
*stmt
= as_a
<gomp_critical
*> (gsi_stmt (*gsi_p
));
13903 location_t loc
= gimple_location (stmt
);
13906 name
= gimple_omp_critical_name (stmt
);
13911 if (!critical_name_mutexes
)
13912 critical_name_mutexes
= hash_map
<tree
, tree
>::create_ggc (10);
13914 tree
*n
= critical_name_mutexes
->get (name
);
13919 decl
= create_tmp_var_raw (ptr_type_node
);
13921 new_str
= ACONCAT ((".gomp_critical_user_",
13922 IDENTIFIER_POINTER (name
), NULL
));
13923 DECL_NAME (decl
) = get_identifier (new_str
);
13924 TREE_PUBLIC (decl
) = 1;
13925 TREE_STATIC (decl
) = 1;
13926 DECL_COMMON (decl
) = 1;
13927 DECL_ARTIFICIAL (decl
) = 1;
13928 DECL_IGNORED_P (decl
) = 1;
13930 varpool_node::finalize_decl (decl
);
13932 critical_name_mutexes
->put (name
, decl
);
13937 /* If '#pragma omp critical' is inside offloaded region or
13938 inside function marked as offloadable, the symbol must be
13939 marked as offloadable too. */
13941 if (cgraph_node::get (current_function_decl
)->offloadable
)
13942 varpool_node::get_create (decl
)->offloadable
= 1;
13944 for (octx
= ctx
->outer
; octx
; octx
= octx
->outer
)
13945 if (is_gimple_omp_offloaded (octx
->stmt
))
13947 varpool_node::get_create (decl
)->offloadable
= 1;
13951 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
13952 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
13954 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
13955 unlock
= build_call_expr_loc (loc
, unlock
, 1,
13956 build_fold_addr_expr_loc (loc
, decl
));
13960 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
13961 lock
= build_call_expr_loc (loc
, lock
, 0);
13963 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
13964 unlock
= build_call_expr_loc (loc
, unlock
, 0);
13967 push_gimplify_context ();
13969 block
= make_node (BLOCK
);
13970 bind
= gimple_build_bind (NULL
, NULL
, block
);
13971 gsi_replace (gsi_p
, bind
, true);
13972 gimple_bind_add_stmt (bind
, stmt
);
13974 tbody
= gimple_bind_body (bind
);
13975 gimplify_and_add (lock
, &tbody
);
13976 gimple_bind_set_body (bind
, tbody
);
13978 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
13979 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
13980 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
13981 gimple_omp_set_body (stmt
, NULL
);
13983 tbody
= gimple_bind_body (bind
);
13984 gimplify_and_add (unlock
, &tbody
);
13985 gimple_bind_set_body (bind
, tbody
);
13987 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
13989 pop_gimplify_context (bind
);
13990 gimple_bind_append_vars (bind
, ctx
->block_vars
);
13991 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
13995 /* A subroutine of lower_omp_for. Generate code to emit the predicate
13996 for a lastprivate clause. Given a loop control predicate of (V
13997 cond N2), we gate the clause on (!(V cond N2)). The lowered form
13998 is appended to *DLIST, iterator initialization is appended to
14002 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
14003 gimple_seq
*dlist
, struct omp_context
*ctx
)
14005 tree clauses
, cond
, vinit
;
14006 enum tree_code cond_code
;
14009 cond_code
= fd
->loop
.cond_code
;
14010 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
14012 /* When possible, use a strict equality expression. This can let VRP
14013 type optimizations deduce the value and remove a copy. */
14014 if (tree_fits_shwi_p (fd
->loop
.step
))
14016 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
14017 if (step
== 1 || step
== -1)
14018 cond_code
= EQ_EXPR
;
14021 tree n2
= fd
->loop
.n2
;
14022 if (fd
->collapse
> 1
14023 && TREE_CODE (n2
) != INTEGER_CST
14024 && gimple_omp_for_combined_into_p (fd
->for_stmt
))
14026 struct omp_context
*taskreg_ctx
= NULL
;
14027 if (gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_FOR
)
14029 gomp_for
*gfor
= as_a
<gomp_for
*> (ctx
->outer
->stmt
);
14030 if (gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_FOR
14031 || gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
14033 if (gimple_omp_for_combined_into_p (gfor
))
14035 gcc_assert (ctx
->outer
->outer
14036 && is_parallel_ctx (ctx
->outer
->outer
));
14037 taskreg_ctx
= ctx
->outer
->outer
;
14041 struct omp_for_data outer_fd
;
14042 extract_omp_for_data (gfor
, &outer_fd
, NULL
);
14043 n2
= fold_convert (TREE_TYPE (n2
), outer_fd
.loop
.n2
);
14046 else if (gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_TASKLOOP
)
14047 taskreg_ctx
= ctx
->outer
->outer
;
14049 else if (is_taskreg_ctx (ctx
->outer
))
14050 taskreg_ctx
= ctx
->outer
;
14055 = find_omp_clause (gimple_omp_taskreg_clauses (taskreg_ctx
->stmt
),
14056 OMP_CLAUSE__LOOPTEMP_
);
14057 gcc_assert (innerc
);
14058 for (i
= 0; i
< fd
->collapse
; i
++)
14060 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
14061 OMP_CLAUSE__LOOPTEMP_
);
14062 gcc_assert (innerc
);
14064 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
14065 OMP_CLAUSE__LOOPTEMP_
);
14067 n2
= fold_convert (TREE_TYPE (n2
),
14068 lookup_decl (OMP_CLAUSE_DECL (innerc
),
14072 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, n2
);
14074 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
14076 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
14077 if (!gimple_seq_empty_p (stmts
))
14079 gimple_seq_add_seq (&stmts
, *dlist
);
14082 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
14083 vinit
= fd
->loop
.n1
;
14084 if (cond_code
== EQ_EXPR
14085 && tree_fits_shwi_p (fd
->loop
.n2
)
14086 && ! integer_zerop (fd
->loop
.n2
))
14087 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
14089 vinit
= unshare_expr (vinit
);
14091 /* Initialize the iterator variable, so that threads that don't execute
14092 any iterations don't execute the lastprivate clauses by accident. */
14093 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
14098 /* Lower code for an OMP loop directive. */
14101 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14103 tree
*rhs_p
, block
;
14104 struct omp_for_data fd
, *fdp
= NULL
;
14105 gomp_for
*stmt
= as_a
<gomp_for
*> (gsi_stmt (*gsi_p
));
14107 gimple_seq omp_for_body
, body
, dlist
;
14108 gimple_seq oacc_head
= NULL
, oacc_tail
= NULL
;
14111 push_gimplify_context ();
14113 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
14115 block
= make_node (BLOCK
);
14116 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
14117 /* Replace at gsi right away, so that 'stmt' is no member
14118 of a sequence anymore as we're going to add to a different
14120 gsi_replace (gsi_p
, new_stmt
, true);
14122 /* Move declaration of temporaries in the loop body before we make
14124 omp_for_body
= gimple_omp_body (stmt
);
14125 if (!gimple_seq_empty_p (omp_for_body
)
14126 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
14129 = as_a
<gbind
*> (gimple_seq_first_stmt (omp_for_body
));
14130 tree vars
= gimple_bind_vars (inner_bind
);
14131 gimple_bind_append_vars (new_stmt
, vars
);
14132 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
14133 keep them on the inner_bind and it's block. */
14134 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
14135 if (gimple_bind_block (inner_bind
))
14136 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
14139 if (gimple_omp_for_combined_into_p (stmt
))
14141 extract_omp_for_data (stmt
, &fd
, NULL
);
14144 /* We need two temporaries with fd.loop.v type (istart/iend)
14145 and then (fd.collapse - 1) temporaries with the same
14146 type for count2 ... countN-1 vars if not constant. */
14148 tree type
= fd
.iter_type
;
14149 if (fd
.collapse
> 1
14150 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
14151 count
+= fd
.collapse
- 1;
14153 = (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
14154 || gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_TASKLOOP
);
14155 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
14156 tree clauses
= *pc
;
14159 = find_omp_clause (gimple_omp_taskreg_clauses (ctx
->outer
->stmt
),
14160 OMP_CLAUSE__LOOPTEMP_
);
14161 for (i
= 0; i
< count
; i
++)
14166 gcc_assert (outerc
);
14167 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
14168 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
14169 OMP_CLAUSE__LOOPTEMP_
);
14173 temp
= create_tmp_var (type
);
14174 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
14176 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
14177 OMP_CLAUSE_DECL (*pc
) = temp
;
14178 pc
= &OMP_CLAUSE_CHAIN (*pc
);
14183 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
14186 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
14188 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
14190 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14192 /* Lower the header expressions. At this point, we can assume that
14193 the header is of the form:
14195 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
14197 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
14198 using the .omp_data_s mapping, if needed. */
14199 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
14201 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
14202 if (!is_gimple_min_invariant (*rhs_p
))
14203 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
14205 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
14206 if (!is_gimple_min_invariant (*rhs_p
))
14207 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
14209 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
14210 if (!is_gimple_min_invariant (*rhs_p
))
14211 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
14214 /* Once lowered, extract the bounds and clauses. */
14215 extract_omp_for_data (stmt
, &fd
, NULL
);
14217 if (is_gimple_omp_oacc (ctx
->stmt
)
14218 && !ctx_in_oacc_kernels_region (ctx
))
14219 lower_oacc_head_tail (gimple_location (stmt
),
14220 gimple_omp_for_clauses (stmt
),
14221 &oacc_head
, &oacc_tail
, ctx
);
14223 /* Add OpenACC partitioning markers just before the loop */
14225 gimple_seq_add_seq (&body
, oacc_head
);
14227 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
14229 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
14230 for (tree c
= gimple_omp_for_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
14231 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
14232 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
14234 OMP_CLAUSE_DECL (c
) = lookup_decl (OMP_CLAUSE_DECL (c
), ctx
);
14235 if (DECL_P (OMP_CLAUSE_LINEAR_STEP (c
)))
14236 OMP_CLAUSE_LINEAR_STEP (c
)
14237 = maybe_lookup_decl_in_outer_ctx (OMP_CLAUSE_LINEAR_STEP (c
),
14241 gimple_seq_add_stmt (&body
, stmt
);
14242 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
14244 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
14247 /* After the loop, add exit clauses. */
14248 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
14250 if (ctx
->cancellable
)
14251 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
14253 gimple_seq_add_seq (&body
, dlist
);
14255 body
= maybe_catch_exception (body
);
14257 /* Region exit marker goes at the end of the loop body. */
14258 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
14259 maybe_add_implicit_barrier_cancel (ctx
, &body
);
14261 /* Add OpenACC joining and reduction markers just after the loop. */
14263 gimple_seq_add_seq (&body
, oacc_tail
);
14265 pop_gimplify_context (new_stmt
);
14267 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
14268 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
14269 if (BLOCK_VARS (block
))
14270 TREE_USED (block
) = 1;
14272 gimple_bind_set_body (new_stmt
, body
);
14273 gimple_omp_set_body (stmt
, NULL
);
14274 gimple_omp_for_set_pre_body (stmt
, NULL
);
14277 /* Callback for walk_stmts. Check if the current statement only contains
14278 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
14281 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
14282 bool *handled_ops_p
,
14283 struct walk_stmt_info
*wi
)
14285 int *info
= (int *) wi
->info
;
14286 gimple
*stmt
= gsi_stmt (*gsi_p
);
14288 *handled_ops_p
= true;
14289 switch (gimple_code (stmt
))
14293 case GIMPLE_OMP_FOR
:
14294 case GIMPLE_OMP_SECTIONS
:
14295 *info
= *info
== 0 ? 1 : -1;
14304 struct omp_taskcopy_context
14306 /* This field must be at the beginning, as we do "inheritance": Some
14307 callback functions for tree-inline.c (e.g., omp_copy_decl)
14308 receive a copy_body_data pointer that is up-casted to an
14309 omp_context pointer. */
14315 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
14317 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
14319 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
14320 return create_tmp_var (TREE_TYPE (var
));
14326 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
14328 tree name
, new_fields
= NULL
, type
, f
;
14330 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
14331 name
= DECL_NAME (TYPE_NAME (orig_type
));
14332 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
14333 TYPE_DECL
, name
, type
);
14334 TYPE_NAME (type
) = name
;
14336 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
14338 tree new_f
= copy_node (f
);
14339 DECL_CONTEXT (new_f
) = type
;
14340 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
14341 TREE_CHAIN (new_f
) = new_fields
;
14342 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
14343 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
14344 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
14346 new_fields
= new_f
;
14347 tcctx
->cb
.decl_map
->put (f
, new_f
);
14349 TYPE_FIELDS (type
) = nreverse (new_fields
);
14350 layout_type (type
);
14354 /* Create task copyfn. */
14357 create_task_copyfn (gomp_task
*task_stmt
, omp_context
*ctx
)
14359 struct function
*child_cfun
;
14360 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
14361 tree record_type
, srecord_type
, bind
, list
;
14362 bool record_needs_remap
= false, srecord_needs_remap
= false;
14364 struct omp_taskcopy_context tcctx
;
14365 location_t loc
= gimple_location (task_stmt
);
14367 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
14368 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
14369 gcc_assert (child_cfun
->cfg
== NULL
);
14370 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
14372 /* Reset DECL_CONTEXT on function arguments. */
14373 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
14374 DECL_CONTEXT (t
) = child_fn
;
14376 /* Populate the function. */
14377 push_gimplify_context ();
14378 push_cfun (child_cfun
);
14380 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
14381 TREE_SIDE_EFFECTS (bind
) = 1;
14383 DECL_SAVED_TREE (child_fn
) = bind
;
14384 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
14386 /* Remap src and dst argument types if needed. */
14387 record_type
= ctx
->record_type
;
14388 srecord_type
= ctx
->srecord_type
;
14389 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
14390 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
14392 record_needs_remap
= true;
14395 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
14396 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
14398 srecord_needs_remap
= true;
14402 if (record_needs_remap
|| srecord_needs_remap
)
14404 memset (&tcctx
, '\0', sizeof (tcctx
));
14405 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
14406 tcctx
.cb
.dst_fn
= child_fn
;
14407 tcctx
.cb
.src_node
= cgraph_node::get (tcctx
.cb
.src_fn
);
14408 gcc_checking_assert (tcctx
.cb
.src_node
);
14409 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
14410 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
14411 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
14412 tcctx
.cb
.eh_lp_nr
= 0;
14413 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
14414 tcctx
.cb
.decl_map
= new hash_map
<tree
, tree
>;
14417 if (record_needs_remap
)
14418 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
14419 if (srecord_needs_remap
)
14420 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
14423 tcctx
.cb
.decl_map
= NULL
;
14425 arg
= DECL_ARGUMENTS (child_fn
);
14426 TREE_TYPE (arg
) = build_pointer_type (record_type
);
14427 sarg
= DECL_CHAIN (arg
);
14428 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
14430 /* First pass: initialize temporaries used in record_type and srecord_type
14431 sizes and field offsets. */
14432 if (tcctx
.cb
.decl_map
)
14433 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
14434 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
14438 decl
= OMP_CLAUSE_DECL (c
);
14439 p
= tcctx
.cb
.decl_map
->get (decl
);
14442 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
14443 sf
= (tree
) n
->value
;
14444 sf
= *tcctx
.cb
.decl_map
->get (sf
);
14445 src
= build_simple_mem_ref_loc (loc
, sarg
);
14446 src
= omp_build_component_ref (src
, sf
);
14447 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
14448 append_to_statement_list (t
, &list
);
14451 /* Second pass: copy shared var pointers and copy construct non-VLA
14452 firstprivate vars. */
14453 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
14454 switch (OMP_CLAUSE_CODE (c
))
14456 splay_tree_key key
;
14457 case OMP_CLAUSE_SHARED
:
14458 decl
= OMP_CLAUSE_DECL (c
);
14459 key
= (splay_tree_key
) decl
;
14460 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
14461 key
= (splay_tree_key
) &DECL_UID (decl
);
14462 n
= splay_tree_lookup (ctx
->field_map
, key
);
14465 f
= (tree
) n
->value
;
14466 if (tcctx
.cb
.decl_map
)
14467 f
= *tcctx
.cb
.decl_map
->get (f
);
14468 n
= splay_tree_lookup (ctx
->sfield_map
, key
);
14469 sf
= (tree
) n
->value
;
14470 if (tcctx
.cb
.decl_map
)
14471 sf
= *tcctx
.cb
.decl_map
->get (sf
);
14472 src
= build_simple_mem_ref_loc (loc
, sarg
);
14473 src
= omp_build_component_ref (src
, sf
);
14474 dst
= build_simple_mem_ref_loc (loc
, arg
);
14475 dst
= omp_build_component_ref (dst
, f
);
14476 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
14477 append_to_statement_list (t
, &list
);
14479 case OMP_CLAUSE_FIRSTPRIVATE
:
14480 decl
= OMP_CLAUSE_DECL (c
);
14481 if (is_variable_sized (decl
))
14483 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
14486 f
= (tree
) n
->value
;
14487 if (tcctx
.cb
.decl_map
)
14488 f
= *tcctx
.cb
.decl_map
->get (f
);
14489 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
14492 sf
= (tree
) n
->value
;
14493 if (tcctx
.cb
.decl_map
)
14494 sf
= *tcctx
.cb
.decl_map
->get (sf
);
14495 src
= build_simple_mem_ref_loc (loc
, sarg
);
14496 src
= omp_build_component_ref (src
, sf
);
14497 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
14498 src
= build_simple_mem_ref_loc (loc
, src
);
14502 dst
= build_simple_mem_ref_loc (loc
, arg
);
14503 dst
= omp_build_component_ref (dst
, f
);
14504 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
14505 append_to_statement_list (t
, &list
);
14507 case OMP_CLAUSE_PRIVATE
:
14508 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
14510 decl
= OMP_CLAUSE_DECL (c
);
14511 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
14512 f
= (tree
) n
->value
;
14513 if (tcctx
.cb
.decl_map
)
14514 f
= *tcctx
.cb
.decl_map
->get (f
);
14515 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
14518 sf
= (tree
) n
->value
;
14519 if (tcctx
.cb
.decl_map
)
14520 sf
= *tcctx
.cb
.decl_map
->get (sf
);
14521 src
= build_simple_mem_ref_loc (loc
, sarg
);
14522 src
= omp_build_component_ref (src
, sf
);
14523 if (use_pointer_for_field (decl
, NULL
))
14524 src
= build_simple_mem_ref_loc (loc
, src
);
14528 dst
= build_simple_mem_ref_loc (loc
, arg
);
14529 dst
= omp_build_component_ref (dst
, f
);
14530 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
14531 append_to_statement_list (t
, &list
);
14537 /* Last pass: handle VLA firstprivates. */
14538 if (tcctx
.cb
.decl_map
)
14539 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
14540 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
14544 decl
= OMP_CLAUSE_DECL (c
);
14545 if (!is_variable_sized (decl
))
14547 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
14550 f
= (tree
) n
->value
;
14551 f
= *tcctx
.cb
.decl_map
->get (f
);
14552 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
14553 ind
= DECL_VALUE_EXPR (decl
);
14554 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
14555 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
14556 n
= splay_tree_lookup (ctx
->sfield_map
,
14557 (splay_tree_key
) TREE_OPERAND (ind
, 0));
14558 sf
= (tree
) n
->value
;
14559 sf
= *tcctx
.cb
.decl_map
->get (sf
);
14560 src
= build_simple_mem_ref_loc (loc
, sarg
);
14561 src
= omp_build_component_ref (src
, sf
);
14562 src
= build_simple_mem_ref_loc (loc
, src
);
14563 dst
= build_simple_mem_ref_loc (loc
, arg
);
14564 dst
= omp_build_component_ref (dst
, f
);
14565 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
14566 append_to_statement_list (t
, &list
);
14567 n
= splay_tree_lookup (ctx
->field_map
,
14568 (splay_tree_key
) TREE_OPERAND (ind
, 0));
14569 df
= (tree
) n
->value
;
14570 df
= *tcctx
.cb
.decl_map
->get (df
);
14571 ptr
= build_simple_mem_ref_loc (loc
, arg
);
14572 ptr
= omp_build_component_ref (ptr
, df
);
14573 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
14574 build_fold_addr_expr_loc (loc
, dst
));
14575 append_to_statement_list (t
, &list
);
14578 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
14579 append_to_statement_list (t
, &list
);
14581 if (tcctx
.cb
.decl_map
)
14582 delete tcctx
.cb
.decl_map
;
14583 pop_gimplify_context (NULL
);
14584 BIND_EXPR_BODY (bind
) = list
;
14589 lower_depend_clauses (tree
*pclauses
, gimple_seq
*iseq
, gimple_seq
*oseq
)
14593 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
14595 clauses
= find_omp_clause (*pclauses
, OMP_CLAUSE_DEPEND
);
14596 gcc_assert (clauses
);
14597 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
14598 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
14599 switch (OMP_CLAUSE_DEPEND_KIND (c
))
14601 case OMP_CLAUSE_DEPEND_IN
:
14604 case OMP_CLAUSE_DEPEND_OUT
:
14605 case OMP_CLAUSE_DEPEND_INOUT
:
14608 case OMP_CLAUSE_DEPEND_SOURCE
:
14609 case OMP_CLAUSE_DEPEND_SINK
:
14612 gcc_unreachable ();
14614 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
14615 tree array
= create_tmp_var (type
);
14616 TREE_ADDRESSABLE (array
) = 1;
14617 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
14619 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
14620 gimple_seq_add_stmt (iseq
, g
);
14621 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
14623 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
14624 gimple_seq_add_stmt (iseq
, g
);
14625 for (i
= 0; i
< 2; i
++)
14627 if ((i
? n_in
: n_out
) == 0)
14629 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
14630 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
14631 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
14633 tree t
= OMP_CLAUSE_DECL (c
);
14634 t
= fold_convert (ptr_type_node
, t
);
14635 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
14636 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
14637 NULL_TREE
, NULL_TREE
);
14638 g
= gimple_build_assign (r
, t
);
14639 gimple_seq_add_stmt (iseq
, g
);
14642 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
14643 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
14644 OMP_CLAUSE_CHAIN (c
) = *pclauses
;
14646 tree clobber
= build_constructor (type
, NULL
);
14647 TREE_THIS_VOLATILE (clobber
) = 1;
14648 g
= gimple_build_assign (array
, clobber
);
14649 gimple_seq_add_stmt (oseq
, g
);
14652 /* Lower the OpenMP parallel or task directive in the current statement
14653 in GSI_P. CTX holds context information for the directive. */
14656 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14660 gimple
*stmt
= gsi_stmt (*gsi_p
);
14661 gbind
*par_bind
, *bind
, *dep_bind
= NULL
;
14662 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
14663 location_t loc
= gimple_location (stmt
);
14665 clauses
= gimple_omp_taskreg_clauses (stmt
);
14667 = as_a
<gbind
*> (gimple_seq_first_stmt (gimple_omp_body (stmt
)));
14668 par_body
= gimple_bind_body (par_bind
);
14669 child_fn
= ctx
->cb
.dst_fn
;
14670 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
14671 && !gimple_omp_parallel_combined_p (stmt
))
14673 struct walk_stmt_info wi
;
14676 memset (&wi
, 0, sizeof (wi
));
14678 wi
.val_only
= true;
14679 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
14681 gimple_omp_parallel_set_combined_p (stmt
, true);
14683 gimple_seq dep_ilist
= NULL
;
14684 gimple_seq dep_olist
= NULL
;
14685 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
14686 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
14688 push_gimplify_context ();
14689 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
14690 lower_depend_clauses (gimple_omp_task_clauses_ptr (stmt
),
14691 &dep_ilist
, &dep_olist
);
14694 if (ctx
->srecord_type
)
14695 create_task_copyfn (as_a
<gomp_task
*> (stmt
), ctx
);
14697 push_gimplify_context ();
14702 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
14703 lower_omp (&par_body
, ctx
);
14704 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
14705 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
14707 /* Declare all the variables created by mapping and the variables
14708 declared in the scope of the parallel body. */
14709 record_vars_into (ctx
->block_vars
, child_fn
);
14710 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
14712 if (ctx
->record_type
)
14715 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
14716 : ctx
->record_type
, ".omp_data_o");
14717 DECL_NAMELESS (ctx
->sender_decl
) = 1;
14718 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
14719 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
14724 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
14725 lower_send_shared_vars (&ilist
, &olist
, ctx
);
14727 if (ctx
->record_type
)
14729 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
14730 TREE_THIS_VOLATILE (clobber
) = 1;
14731 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
14735 /* Once all the expansions are done, sequence all the different
14736 fragments inside gimple_omp_body. */
14740 if (ctx
->record_type
)
14742 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
14743 /* fixup_child_record_type might have changed receiver_decl's type. */
14744 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
14745 gimple_seq_add_stmt (&new_body
,
14746 gimple_build_assign (ctx
->receiver_decl
, t
));
14749 gimple_seq_add_seq (&new_body
, par_ilist
);
14750 gimple_seq_add_seq (&new_body
, par_body
);
14751 gimple_seq_add_seq (&new_body
, par_rlist
);
14752 if (ctx
->cancellable
)
14753 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
14754 gimple_seq_add_seq (&new_body
, par_olist
);
14755 new_body
= maybe_catch_exception (new_body
);
14756 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
)
14757 gimple_seq_add_stmt (&new_body
,
14758 gimple_build_omp_continue (integer_zero_node
,
14759 integer_zero_node
));
14760 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
14761 gimple_omp_set_body (stmt
, new_body
);
14763 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
14764 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
14765 gimple_bind_add_seq (bind
, ilist
);
14766 gimple_bind_add_stmt (bind
, stmt
);
14767 gimple_bind_add_seq (bind
, olist
);
14769 pop_gimplify_context (NULL
);
14773 gimple_bind_add_seq (dep_bind
, dep_ilist
);
14774 gimple_bind_add_stmt (dep_bind
, bind
);
14775 gimple_bind_add_seq (dep_bind
, dep_olist
);
14776 pop_gimplify_context (dep_bind
);
14780 /* Lower the GIMPLE_OMP_TARGET in the current statement
14781 in GSI_P. CTX holds context information for the directive. */
14784 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14787 tree child_fn
, t
, c
;
14788 gomp_target
*stmt
= as_a
<gomp_target
*> (gsi_stmt (*gsi_p
));
14789 gbind
*tgt_bind
, *bind
, *dep_bind
= NULL
;
14790 gimple_seq tgt_body
, olist
, ilist
, new_body
;
14791 location_t loc
= gimple_location (stmt
);
14792 bool offloaded
, data_region
;
14793 unsigned int map_cnt
= 0;
14794 bool has_depend
= false;
14796 offloaded
= is_gimple_omp_offloaded (stmt
);
14797 switch (gimple_omp_target_kind (stmt
))
14799 case GF_OMP_TARGET_KIND_REGION
:
14800 case GF_OMP_TARGET_KIND_UPDATE
:
14801 case GF_OMP_TARGET_KIND_ENTER_DATA
:
14802 case GF_OMP_TARGET_KIND_EXIT_DATA
:
14803 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
14804 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
14805 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
14806 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
14807 data_region
= false;
14809 case GF_OMP_TARGET_KIND_DATA
:
14810 case GF_OMP_TARGET_KIND_OACC_DATA
:
14811 data_region
= true;
14814 gcc_unreachable ();
14817 clauses
= gimple_omp_target_clauses (stmt
);
14819 gimple_seq dep_ilist
= NULL
;
14820 gimple_seq dep_olist
= NULL
;
14821 if (find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
14823 push_gimplify_context ();
14824 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
14825 lower_depend_clauses (gimple_omp_target_clauses_ptr (stmt
),
14826 &dep_ilist
, &dep_olist
);
14834 tgt_bind
= gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt
));
14835 tgt_body
= gimple_bind_body (tgt_bind
);
14837 else if (data_region
)
14838 tgt_body
= gimple_omp_body (stmt
);
14839 child_fn
= ctx
->cb
.dst_fn
;
14841 push_gimplify_context ();
14843 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
14844 switch (OMP_CLAUSE_CODE (c
))
14850 case OMP_CLAUSE_MAP
:
14852 /* First check what we're prepared to handle in the following. */
14853 switch (OMP_CLAUSE_MAP_KIND (c
))
14855 case GOMP_MAP_ALLOC
:
14857 case GOMP_MAP_FROM
:
14858 case GOMP_MAP_TOFROM
:
14859 case GOMP_MAP_POINTER
:
14860 case GOMP_MAP_TO_PSET
:
14861 case GOMP_MAP_FORCE_DEALLOC
:
14862 case GOMP_MAP_RELEASE
:
14863 case GOMP_MAP_ALWAYS_TO
:
14864 case GOMP_MAP_ALWAYS_FROM
:
14865 case GOMP_MAP_ALWAYS_TOFROM
:
14866 case GOMP_MAP_FIRSTPRIVATE_POINTER
:
14867 case GOMP_MAP_FIRSTPRIVATE_REFERENCE
:
14868 case GOMP_MAP_STRUCT
:
14869 case GOMP_MAP_ALWAYS_POINTER
:
14871 case GOMP_MAP_FORCE_ALLOC
:
14872 case GOMP_MAP_FORCE_TO
:
14873 case GOMP_MAP_FORCE_FROM
:
14874 case GOMP_MAP_FORCE_TOFROM
:
14875 case GOMP_MAP_FORCE_PRESENT
:
14876 case GOMP_MAP_FORCE_DEVICEPTR
:
14877 gcc_assert (is_gimple_omp_oacc (stmt
));
14880 gcc_unreachable ();
14884 case OMP_CLAUSE_TO
:
14885 case OMP_CLAUSE_FROM
:
14886 var
= OMP_CLAUSE_DECL (c
);
14889 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
14890 || (!OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
14891 && (OMP_CLAUSE_MAP_KIND (c
)
14892 != GOMP_MAP_FIRSTPRIVATE_POINTER
)))
14897 if (DECL_SIZE (var
)
14898 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
14900 tree var2
= DECL_VALUE_EXPR (var
);
14901 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
14902 var2
= TREE_OPERAND (var2
, 0);
14903 gcc_assert (DECL_P (var2
));
14908 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
14909 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
14911 if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
14913 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
))
14914 && varpool_node::get_create (var
)->offloadable
)
14917 tree type
= build_pointer_type (TREE_TYPE (var
));
14918 tree new_var
= lookup_decl (var
, ctx
);
14919 x
= create_tmp_var_raw (type
, get_name (new_var
));
14920 gimple_add_tmp_var (x
);
14921 x
= build_simple_mem_ref (x
);
14922 SET_DECL_VALUE_EXPR (new_var
, x
);
14923 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
14928 if (!maybe_lookup_field (var
, ctx
))
14933 x
= build_receiver_ref (var
, true, ctx
);
14934 tree new_var
= lookup_decl (var
, ctx
);
14936 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
14937 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
14938 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
14939 x
= build_simple_mem_ref (x
);
14940 SET_DECL_VALUE_EXPR (new_var
, x
);
14941 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
14946 case OMP_CLAUSE_FIRSTPRIVATE
:
14948 var
= OMP_CLAUSE_DECL (c
);
14949 if (!is_reference (var
)
14950 && !is_gimple_reg_type (TREE_TYPE (var
)))
14952 tree new_var
= lookup_decl (var
, ctx
);
14953 if (is_variable_sized (var
))
14955 tree pvar
= DECL_VALUE_EXPR (var
);
14956 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
14957 pvar
= TREE_OPERAND (pvar
, 0);
14958 gcc_assert (DECL_P (pvar
));
14959 tree new_pvar
= lookup_decl (pvar
, ctx
);
14960 x
= build_fold_indirect_ref (new_pvar
);
14961 TREE_THIS_NOTRAP (x
) = 1;
14964 x
= build_receiver_ref (var
, true, ctx
);
14965 SET_DECL_VALUE_EXPR (new_var
, x
);
14966 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
14970 case OMP_CLAUSE_PRIVATE
:
14971 var
= OMP_CLAUSE_DECL (c
);
14972 if (is_variable_sized (var
))
14974 tree new_var
= lookup_decl (var
, ctx
);
14975 tree pvar
= DECL_VALUE_EXPR (var
);
14976 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
14977 pvar
= TREE_OPERAND (pvar
, 0);
14978 gcc_assert (DECL_P (pvar
));
14979 tree new_pvar
= lookup_decl (pvar
, ctx
);
14980 x
= build_fold_indirect_ref (new_pvar
);
14981 TREE_THIS_NOTRAP (x
) = 1;
14982 SET_DECL_VALUE_EXPR (new_var
, x
);
14983 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
14987 case OMP_CLAUSE_USE_DEVICE_PTR
:
14988 case OMP_CLAUSE_IS_DEVICE_PTR
:
14989 var
= OMP_CLAUSE_DECL (c
);
14991 if (is_variable_sized (var
))
14993 tree new_var
= lookup_decl (var
, ctx
);
14994 tree pvar
= DECL_VALUE_EXPR (var
);
14995 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
14996 pvar
= TREE_OPERAND (pvar
, 0);
14997 gcc_assert (DECL_P (pvar
));
14998 tree new_pvar
= lookup_decl (pvar
, ctx
);
14999 x
= build_fold_indirect_ref (new_pvar
);
15000 TREE_THIS_NOTRAP (x
) = 1;
15001 SET_DECL_VALUE_EXPR (new_var
, x
);
15002 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15004 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15006 tree new_var
= lookup_decl (var
, ctx
);
15007 tree type
= build_pointer_type (TREE_TYPE (var
));
15008 x
= create_tmp_var_raw (type
, get_name (new_var
));
15009 gimple_add_tmp_var (x
);
15010 x
= build_simple_mem_ref (x
);
15011 SET_DECL_VALUE_EXPR (new_var
, x
);
15012 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15019 target_nesting_level
++;
15020 lower_omp (&tgt_body
, ctx
);
15021 target_nesting_level
--;
15023 else if (data_region
)
15024 lower_omp (&tgt_body
, ctx
);
15028 /* Declare all the variables created by mapping and the variables
15029 declared in the scope of the target body. */
15030 record_vars_into (ctx
->block_vars
, child_fn
);
15031 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
15036 if (ctx
->record_type
)
15039 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
15040 DECL_NAMELESS (ctx
->sender_decl
) = 1;
15041 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
15042 t
= make_tree_vec (3);
15043 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
15044 TREE_VEC_ELT (t
, 1)
15045 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
15046 ".omp_data_sizes");
15047 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
15048 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
15049 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
15050 tree tkind_type
= short_unsigned_type_node
;
15051 int talign_shift
= 8;
15052 TREE_VEC_ELT (t
, 2)
15053 = create_tmp_var (build_array_type_nelts (tkind_type
, map_cnt
),
15054 ".omp_data_kinds");
15055 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
15056 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
15057 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
15058 gimple_omp_target_set_data_arg (stmt
, t
);
15060 vec
<constructor_elt
, va_gc
> *vsize
;
15061 vec
<constructor_elt
, va_gc
> *vkind
;
15062 vec_alloc (vsize
, map_cnt
);
15063 vec_alloc (vkind
, map_cnt
);
15064 unsigned int map_idx
= 0;
15066 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15067 switch (OMP_CLAUSE_CODE (c
))
15069 tree ovar
, nc
, s
, purpose
, var
, x
, type
;
15070 unsigned int talign
;
15074 case OMP_CLAUSE_MAP
:
15075 case OMP_CLAUSE_TO
:
15076 case OMP_CLAUSE_FROM
:
15078 ovar
= OMP_CLAUSE_DECL (c
);
15079 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15080 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
15081 || (OMP_CLAUSE_MAP_KIND (c
)
15082 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
15084 if (!DECL_P (ovar
))
15086 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15087 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
15089 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
15090 == get_base_address (ovar
));
15091 nc
= OMP_CLAUSE_CHAIN (c
);
15092 ovar
= OMP_CLAUSE_DECL (nc
);
15096 tree x
= build_sender_ref (ovar
, ctx
);
15098 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
15099 gimplify_assign (x
, v
, &ilist
);
15105 if (DECL_SIZE (ovar
)
15106 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
15108 tree ovar2
= DECL_VALUE_EXPR (ovar
);
15109 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
15110 ovar2
= TREE_OPERAND (ovar2
, 0);
15111 gcc_assert (DECL_P (ovar2
));
15114 if (!maybe_lookup_field (ovar
, ctx
))
15118 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
15119 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
15120 talign
= DECL_ALIGN_UNIT (ovar
);
15123 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
15124 x
= build_sender_ref (ovar
, ctx
);
15126 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15127 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
15128 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
15129 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
15131 gcc_assert (offloaded
);
15133 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)));
15134 mark_addressable (avar
);
15135 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
15136 talign
= DECL_ALIGN_UNIT (avar
);
15137 avar
= build_fold_addr_expr (avar
);
15138 gimplify_assign (x
, avar
, &ilist
);
15140 else if (is_gimple_reg (var
))
15142 gcc_assert (offloaded
);
15143 tree avar
= create_tmp_var (TREE_TYPE (var
));
15144 mark_addressable (avar
);
15145 enum gomp_map_kind map_kind
= OMP_CLAUSE_MAP_KIND (c
);
15146 if (GOMP_MAP_COPY_TO_P (map_kind
)
15147 || map_kind
== GOMP_MAP_POINTER
15148 || map_kind
== GOMP_MAP_TO_PSET
15149 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
15150 gimplify_assign (avar
, var
, &ilist
);
15151 avar
= build_fold_addr_expr (avar
);
15152 gimplify_assign (x
, avar
, &ilist
);
15153 if ((GOMP_MAP_COPY_FROM_P (map_kind
)
15154 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
15155 && !TYPE_READONLY (TREE_TYPE (var
)))
15157 x
= unshare_expr (x
);
15158 x
= build_simple_mem_ref (x
);
15159 gimplify_assign (var
, x
, &olist
);
15164 var
= build_fold_addr_expr (var
);
15165 gimplify_assign (x
, var
, &ilist
);
15168 s
= OMP_CLAUSE_SIZE (c
);
15169 if (s
== NULL_TREE
)
15170 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
15171 s
= fold_convert (size_type_node
, s
);
15172 purpose
= size_int (map_idx
++);
15173 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
15174 if (TREE_CODE (s
) != INTEGER_CST
)
15175 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
15177 unsigned HOST_WIDE_INT tkind
, tkind_zero
;
15178 switch (OMP_CLAUSE_CODE (c
))
15180 case OMP_CLAUSE_MAP
:
15181 tkind
= OMP_CLAUSE_MAP_KIND (c
);
15182 tkind_zero
= tkind
;
15183 if (OMP_CLAUSE_MAP_MAYBE_ZERO_LENGTH_ARRAY_SECTION (c
))
15186 case GOMP_MAP_ALLOC
:
15188 case GOMP_MAP_FROM
:
15189 case GOMP_MAP_TOFROM
:
15190 case GOMP_MAP_ALWAYS_TO
:
15191 case GOMP_MAP_ALWAYS_FROM
:
15192 case GOMP_MAP_ALWAYS_TOFROM
:
15193 case GOMP_MAP_RELEASE
:
15194 tkind_zero
= GOMP_MAP_ZERO_LEN_ARRAY_SECTION
;
15196 case GOMP_MAP_DELETE
:
15197 tkind_zero
= GOMP_MAP_DELETE_ZERO_LEN_ARRAY_SECTION
;
15201 if (tkind_zero
!= tkind
)
15203 if (integer_zerop (s
))
15204 tkind
= tkind_zero
;
15205 else if (integer_nonzerop (s
))
15206 tkind_zero
= tkind
;
15209 case OMP_CLAUSE_TO
:
15210 tkind
= GOMP_MAP_TO
;
15211 tkind_zero
= tkind
;
15213 case OMP_CLAUSE_FROM
:
15214 tkind
= GOMP_MAP_FROM
;
15215 tkind_zero
= tkind
;
15218 gcc_unreachable ();
15220 gcc_checking_assert (tkind
15221 < (HOST_WIDE_INT_C (1U) << talign_shift
));
15222 gcc_checking_assert (tkind_zero
15223 < (HOST_WIDE_INT_C (1U) << talign_shift
));
15224 talign
= ceil_log2 (talign
);
15225 tkind
|= talign
<< talign_shift
;
15226 tkind_zero
|= talign
<< talign_shift
;
15227 gcc_checking_assert (tkind
15228 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
15229 gcc_checking_assert (tkind_zero
15230 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
15231 if (tkind
== tkind_zero
)
15232 x
= build_int_cstu (tkind_type
, tkind
);
15235 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 0;
15236 x
= build3 (COND_EXPR
, tkind_type
,
15237 fold_build2 (EQ_EXPR
, boolean_type_node
,
15238 unshare_expr (s
), size_zero_node
),
15239 build_int_cstu (tkind_type
, tkind_zero
),
15240 build_int_cstu (tkind_type
, tkind
));
15242 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
, x
);
15247 case OMP_CLAUSE_FIRSTPRIVATE
:
15248 ovar
= OMP_CLAUSE_DECL (c
);
15249 if (is_reference (ovar
))
15250 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
15252 talign
= DECL_ALIGN_UNIT (ovar
);
15253 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
15254 x
= build_sender_ref (ovar
, ctx
);
15255 tkind
= GOMP_MAP_FIRSTPRIVATE
;
15256 type
= TREE_TYPE (ovar
);
15257 if (is_reference (ovar
))
15258 type
= TREE_TYPE (type
);
15259 bool use_firstprivate_int
, force_addr
;
15260 use_firstprivate_int
= false;
15261 force_addr
= false;
15262 if ((INTEGRAL_TYPE_P (type
)
15263 && TYPE_PRECISION (type
) <= POINTER_SIZE
)
15264 || TREE_CODE (type
) == POINTER_TYPE
)
15265 use_firstprivate_int
= true;
15268 if (is_reference (var
))
15269 use_firstprivate_int
= false;
15270 else if (is_gimple_reg (var
))
15272 if (DECL_HAS_VALUE_EXPR_P (var
))
15274 tree v
= get_base_address (var
);
15275 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
15277 use_firstprivate_int
= false;
15281 switch (TREE_CODE (v
))
15285 use_firstprivate_int
= false;
15294 use_firstprivate_int
= false;
15296 if (use_firstprivate_int
)
15298 tkind
= GOMP_MAP_FIRSTPRIVATE_INT
;
15300 if (is_reference (var
))
15301 t
= build_simple_mem_ref (var
);
15302 if (TREE_CODE (type
) != POINTER_TYPE
)
15303 t
= fold_convert (pointer_sized_int_node
, t
);
15304 t
= fold_convert (TREE_TYPE (x
), t
);
15305 gimplify_assign (x
, t
, &ilist
);
15307 else if (is_reference (var
))
15308 gimplify_assign (x
, var
, &ilist
);
15309 else if (!force_addr
&& is_gimple_reg (var
))
15311 tree avar
= create_tmp_var (TREE_TYPE (var
));
15312 mark_addressable (avar
);
15313 gimplify_assign (avar
, var
, &ilist
);
15314 avar
= build_fold_addr_expr (avar
);
15315 gimplify_assign (x
, avar
, &ilist
);
15319 var
= build_fold_addr_expr (var
);
15320 gimplify_assign (x
, var
, &ilist
);
15322 if (tkind
== GOMP_MAP_FIRSTPRIVATE_INT
)
15324 else if (is_reference (var
))
15325 s
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
15327 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
15328 s
= fold_convert (size_type_node
, s
);
15329 purpose
= size_int (map_idx
++);
15330 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
15331 if (TREE_CODE (s
) != INTEGER_CST
)
15332 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
15334 gcc_checking_assert (tkind
15335 < (HOST_WIDE_INT_C (1U) << talign_shift
));
15336 talign
= ceil_log2 (talign
);
15337 tkind
|= talign
<< talign_shift
;
15338 gcc_checking_assert (tkind
15339 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
15340 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
15341 build_int_cstu (tkind_type
, tkind
));
15344 case OMP_CLAUSE_USE_DEVICE_PTR
:
15345 case OMP_CLAUSE_IS_DEVICE_PTR
:
15346 ovar
= OMP_CLAUSE_DECL (c
);
15347 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
15348 x
= build_sender_ref (ovar
, ctx
);
15349 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_USE_DEVICE_PTR
)
15350 tkind
= GOMP_MAP_USE_DEVICE_PTR
;
15352 tkind
= GOMP_MAP_FIRSTPRIVATE_INT
;
15353 type
= TREE_TYPE (ovar
);
15354 if (TREE_CODE (type
) == ARRAY_TYPE
)
15355 var
= build_fold_addr_expr (var
);
15358 if (is_reference (ovar
))
15360 type
= TREE_TYPE (type
);
15361 if (TREE_CODE (type
) != ARRAY_TYPE
)
15362 var
= build_simple_mem_ref (var
);
15363 var
= fold_convert (TREE_TYPE (x
), var
);
15366 gimplify_assign (x
, var
, &ilist
);
15368 purpose
= size_int (map_idx
++);
15369 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
15370 gcc_checking_assert (tkind
15371 < (HOST_WIDE_INT_C (1U) << talign_shift
));
15372 gcc_checking_assert (tkind
15373 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
15374 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
15375 build_int_cstu (tkind_type
, tkind
));
15379 gcc_assert (map_idx
== map_cnt
);
15381 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
15382 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
15383 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
15384 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
15385 for (int i
= 1; i
<= 2; i
++)
15386 if (!TREE_STATIC (TREE_VEC_ELT (t
, i
)))
15388 gimple_seq initlist
= NULL
;
15389 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
15390 TREE_VEC_ELT (t
, i
)),
15391 &initlist
, true, NULL_TREE
);
15392 gimple_seq_add_seq (&ilist
, initlist
);
15394 tree clobber
= build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, i
)),
15396 TREE_THIS_VOLATILE (clobber
) = 1;
15397 gimple_seq_add_stmt (&olist
,
15398 gimple_build_assign (TREE_VEC_ELT (t
, i
),
15402 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
15403 TREE_THIS_VOLATILE (clobber
) = 1;
15404 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
15408 /* Once all the expansions are done, sequence all the different
15409 fragments inside gimple_omp_body. */
15414 && ctx
->record_type
)
15416 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
15417 /* fixup_child_record_type might have changed receiver_decl's type. */
15418 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
15419 gimple_seq_add_stmt (&new_body
,
15420 gimple_build_assign (ctx
->receiver_decl
, t
));
15423 if (offloaded
|| data_region
)
15425 tree prev
= NULL_TREE
;
15426 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15427 switch (OMP_CLAUSE_CODE (c
))
15432 case OMP_CLAUSE_FIRSTPRIVATE
:
15433 var
= OMP_CLAUSE_DECL (c
);
15434 if (is_reference (var
)
15435 || is_gimple_reg_type (TREE_TYPE (var
)))
15437 tree new_var
= lookup_decl (var
, ctx
);
15439 type
= TREE_TYPE (var
);
15440 if (is_reference (var
))
15441 type
= TREE_TYPE (type
);
15442 bool use_firstprivate_int
;
15443 use_firstprivate_int
= false;
15444 if ((INTEGRAL_TYPE_P (type
)
15445 && TYPE_PRECISION (type
) <= POINTER_SIZE
)
15446 || TREE_CODE (type
) == POINTER_TYPE
)
15447 use_firstprivate_int
= true;
15450 tree v
= lookup_decl_in_outer_ctx (var
, ctx
);
15451 if (is_reference (v
))
15452 use_firstprivate_int
= false;
15453 else if (is_gimple_reg (v
))
15455 if (DECL_HAS_VALUE_EXPR_P (v
))
15457 v
= get_base_address (v
);
15458 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
15459 use_firstprivate_int
= false;
15461 switch (TREE_CODE (v
))
15465 use_firstprivate_int
= false;
15473 use_firstprivate_int
= false;
15475 if (use_firstprivate_int
)
15477 x
= build_receiver_ref (var
, false, ctx
);
15478 if (TREE_CODE (type
) != POINTER_TYPE
)
15479 x
= fold_convert (pointer_sized_int_node
, x
);
15480 x
= fold_convert (type
, x
);
15481 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
15483 if (is_reference (var
))
15485 tree v
= create_tmp_var_raw (type
, get_name (var
));
15486 gimple_add_tmp_var (v
);
15487 TREE_ADDRESSABLE (v
) = 1;
15488 gimple_seq_add_stmt (&new_body
,
15489 gimple_build_assign (v
, x
));
15490 x
= build_fold_addr_expr (v
);
15492 gimple_seq_add_stmt (&new_body
,
15493 gimple_build_assign (new_var
, x
));
15497 x
= build_receiver_ref (var
, !is_reference (var
), ctx
);
15498 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
15500 gimple_seq_add_stmt (&new_body
,
15501 gimple_build_assign (new_var
, x
));
15504 else if (is_variable_sized (var
))
15506 tree pvar
= DECL_VALUE_EXPR (var
);
15507 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15508 pvar
= TREE_OPERAND (pvar
, 0);
15509 gcc_assert (DECL_P (pvar
));
15510 tree new_var
= lookup_decl (pvar
, ctx
);
15511 x
= build_receiver_ref (var
, false, ctx
);
15512 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15513 gimple_seq_add_stmt (&new_body
,
15514 gimple_build_assign (new_var
, x
));
15517 case OMP_CLAUSE_PRIVATE
:
15518 var
= OMP_CLAUSE_DECL (c
);
15519 if (is_reference (var
))
15521 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
15522 tree new_var
= lookup_decl (var
, ctx
);
15523 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
15524 if (TREE_CONSTANT (x
))
15526 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
15528 gimple_add_tmp_var (x
);
15529 TREE_ADDRESSABLE (x
) = 1;
15530 x
= build_fold_addr_expr_loc (clause_loc
, x
);
15535 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
15536 tree rtype
= TREE_TYPE (TREE_TYPE (new_var
));
15537 tree al
= size_int (TYPE_ALIGN (rtype
));
15538 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
15541 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
15542 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15543 gimple_seq_add_stmt (&new_body
,
15544 gimple_build_assign (new_var
, x
));
15547 case OMP_CLAUSE_USE_DEVICE_PTR
:
15548 case OMP_CLAUSE_IS_DEVICE_PTR
:
15549 var
= OMP_CLAUSE_DECL (c
);
15550 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_USE_DEVICE_PTR
)
15551 x
= build_sender_ref (var
, ctx
);
15553 x
= build_receiver_ref (var
, false, ctx
);
15554 if (is_variable_sized (var
))
15556 tree pvar
= DECL_VALUE_EXPR (var
);
15557 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15558 pvar
= TREE_OPERAND (pvar
, 0);
15559 gcc_assert (DECL_P (pvar
));
15560 tree new_var
= lookup_decl (pvar
, ctx
);
15561 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15562 gimple_seq_add_stmt (&new_body
,
15563 gimple_build_assign (new_var
, x
));
15565 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15567 tree new_var
= lookup_decl (var
, ctx
);
15568 new_var
= DECL_VALUE_EXPR (new_var
);
15569 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
15570 new_var
= TREE_OPERAND (new_var
, 0);
15571 gcc_assert (DECL_P (new_var
));
15572 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15573 gimple_seq_add_stmt (&new_body
,
15574 gimple_build_assign (new_var
, x
));
15578 tree type
= TREE_TYPE (var
);
15579 tree new_var
= lookup_decl (var
, ctx
);
15580 if (is_reference (var
))
15582 type
= TREE_TYPE (type
);
15583 if (TREE_CODE (type
) != ARRAY_TYPE
)
15585 tree v
= create_tmp_var_raw (type
, get_name (var
));
15586 gimple_add_tmp_var (v
);
15587 TREE_ADDRESSABLE (v
) = 1;
15588 x
= fold_convert (type
, x
);
15589 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
15591 gimple_seq_add_stmt (&new_body
,
15592 gimple_build_assign (v
, x
));
15593 x
= build_fold_addr_expr (v
);
15596 x
= fold_convert (TREE_TYPE (new_var
), x
);
15597 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15598 gimple_seq_add_stmt (&new_body
,
15599 gimple_build_assign (new_var
, x
));
15603 /* Handle GOMP_MAP_FIRSTPRIVATE_{POINTER,REFERENCE} in second pass,
15604 so that firstprivate vars holding OMP_CLAUSE_SIZE if needed
15605 are already handled. */
15606 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15607 switch (OMP_CLAUSE_CODE (c
))
15612 case OMP_CLAUSE_MAP
:
15613 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
15614 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)
15616 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
15617 HOST_WIDE_INT offset
= 0;
15619 var
= OMP_CLAUSE_DECL (c
);
15621 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
15622 && is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
15624 && varpool_node::get_create (var
)->offloadable
)
15626 if (TREE_CODE (var
) == INDIRECT_REF
15627 && TREE_CODE (TREE_OPERAND (var
, 0)) == COMPONENT_REF
)
15628 var
= TREE_OPERAND (var
, 0);
15629 if (TREE_CODE (var
) == COMPONENT_REF
)
15631 var
= get_addr_base_and_unit_offset (var
, &offset
);
15632 gcc_assert (var
!= NULL_TREE
&& DECL_P (var
));
15634 else if (DECL_SIZE (var
)
15635 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
15637 tree var2
= DECL_VALUE_EXPR (var
);
15638 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
15639 var2
= TREE_OPERAND (var2
, 0);
15640 gcc_assert (DECL_P (var2
));
15643 tree new_var
= lookup_decl (var
, ctx
), x
;
15644 tree type
= TREE_TYPE (new_var
);
15646 if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == INDIRECT_REF
15647 && (TREE_CODE (TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0))
15650 type
= TREE_TYPE (TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0));
15652 new_var
= build2 (MEM_REF
, type
,
15653 build_fold_addr_expr (new_var
),
15654 build_int_cst (build_pointer_type (type
),
15657 else if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == COMPONENT_REF
)
15659 type
= TREE_TYPE (OMP_CLAUSE_DECL (c
));
15660 is_ref
= TREE_CODE (type
) == REFERENCE_TYPE
;
15661 new_var
= build2 (MEM_REF
, type
,
15662 build_fold_addr_expr (new_var
),
15663 build_int_cst (build_pointer_type (type
),
15667 is_ref
= is_reference (var
);
15668 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)
15670 bool ref_to_array
= false;
15673 type
= TREE_TYPE (type
);
15674 if (TREE_CODE (type
) == ARRAY_TYPE
)
15676 type
= build_pointer_type (type
);
15677 ref_to_array
= true;
15680 else if (TREE_CODE (type
) == ARRAY_TYPE
)
15682 tree decl2
= DECL_VALUE_EXPR (new_var
);
15683 gcc_assert (TREE_CODE (decl2
) == MEM_REF
);
15684 decl2
= TREE_OPERAND (decl2
, 0);
15685 gcc_assert (DECL_P (decl2
));
15687 type
= TREE_TYPE (new_var
);
15689 x
= build_receiver_ref (OMP_CLAUSE_DECL (prev
), false, ctx
);
15690 x
= fold_convert_loc (clause_loc
, type
, x
);
15691 if (!integer_zerop (OMP_CLAUSE_SIZE (c
)))
15693 tree bias
= OMP_CLAUSE_SIZE (c
);
15695 bias
= lookup_decl (bias
, ctx
);
15696 bias
= fold_convert_loc (clause_loc
, sizetype
, bias
);
15697 bias
= fold_build1_loc (clause_loc
, NEGATE_EXPR
, sizetype
,
15699 x
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
15700 TREE_TYPE (x
), x
, bias
);
15703 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
15704 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15705 if (is_ref
&& !ref_to_array
)
15707 tree t
= create_tmp_var_raw (type
, get_name (var
));
15708 gimple_add_tmp_var (t
);
15709 TREE_ADDRESSABLE (t
) = 1;
15710 gimple_seq_add_stmt (&new_body
,
15711 gimple_build_assign (t
, x
));
15712 x
= build_fold_addr_expr_loc (clause_loc
, t
);
15714 gimple_seq_add_stmt (&new_body
,
15715 gimple_build_assign (new_var
, x
));
15718 else if (OMP_CLAUSE_CHAIN (c
)
15719 && OMP_CLAUSE_CODE (OMP_CLAUSE_CHAIN (c
))
15721 && (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c
))
15722 == GOMP_MAP_FIRSTPRIVATE_POINTER
15723 || (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c
))
15724 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
15727 case OMP_CLAUSE_PRIVATE
:
15728 var
= OMP_CLAUSE_DECL (c
);
15729 if (is_variable_sized (var
))
15731 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
15732 tree new_var
= lookup_decl (var
, ctx
);
15733 tree pvar
= DECL_VALUE_EXPR (var
);
15734 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15735 pvar
= TREE_OPERAND (pvar
, 0);
15736 gcc_assert (DECL_P (pvar
));
15737 tree new_pvar
= lookup_decl (pvar
, ctx
);
15738 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
15739 tree al
= size_int (DECL_ALIGN (var
));
15740 tree x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
15741 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
15742 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_pvar
), x
);
15743 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15744 gimple_seq_add_stmt (&new_body
,
15745 gimple_build_assign (new_pvar
, x
));
15750 gimple_seq fork_seq
= NULL
;
15751 gimple_seq join_seq
= NULL
;
15753 if (is_oacc_parallel (ctx
))
15755 /* If there are reductions on the offloaded region itself, treat
15756 them as a dummy GANG loop. */
15757 tree level
= build_int_cst (integer_type_node
, GOMP_DIM_GANG
);
15759 lower_oacc_reductions (gimple_location (ctx
->stmt
), clauses
, level
,
15760 false, NULL
, NULL
, &fork_seq
, &join_seq
, ctx
);
15763 gimple_seq_add_seq (&new_body
, fork_seq
);
15764 gimple_seq_add_seq (&new_body
, tgt_body
);
15765 gimple_seq_add_seq (&new_body
, join_seq
);
15768 new_body
= maybe_catch_exception (new_body
);
15770 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
15771 gimple_omp_set_body (stmt
, new_body
);
15774 bind
= gimple_build_bind (NULL
, NULL
,
15775 tgt_bind
? gimple_bind_block (tgt_bind
)
15777 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
15778 gimple_bind_add_seq (bind
, ilist
);
15779 gimple_bind_add_stmt (bind
, stmt
);
15780 gimple_bind_add_seq (bind
, olist
);
15782 pop_gimplify_context (NULL
);
15786 gimple_bind_add_seq (dep_bind
, dep_ilist
);
15787 gimple_bind_add_stmt (dep_bind
, bind
);
15788 gimple_bind_add_seq (dep_bind
, dep_olist
);
15789 pop_gimplify_context (dep_bind
);
15793 /* Expand code for an OpenMP teams directive. */
15796 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
15798 gomp_teams
*teams_stmt
= as_a
<gomp_teams
*> (gsi_stmt (*gsi_p
));
15799 push_gimplify_context ();
15801 tree block
= make_node (BLOCK
);
15802 gbind
*bind
= gimple_build_bind (NULL
, NULL
, block
);
15803 gsi_replace (gsi_p
, bind
, true);
15804 gimple_seq bind_body
= NULL
;
15805 gimple_seq dlist
= NULL
;
15806 gimple_seq olist
= NULL
;
15808 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
15809 OMP_CLAUSE_NUM_TEAMS
);
15810 if (num_teams
== NULL_TREE
)
15811 num_teams
= build_int_cst (unsigned_type_node
, 0);
15814 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
15815 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
15816 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
15818 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
15819 OMP_CLAUSE_THREAD_LIMIT
);
15820 if (thread_limit
== NULL_TREE
)
15821 thread_limit
= build_int_cst (unsigned_type_node
, 0);
15824 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
15825 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
15826 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
15830 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
15831 &bind_body
, &dlist
, ctx
, NULL
);
15832 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
15833 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
15834 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
15836 location_t loc
= gimple_location (teams_stmt
);
15837 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
15838 gimple
*call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
15839 gimple_set_location (call
, loc
);
15840 gimple_seq_add_stmt (&bind_body
, call
);
15842 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
15843 gimple_omp_set_body (teams_stmt
, NULL
);
15844 gimple_seq_add_seq (&bind_body
, olist
);
15845 gimple_seq_add_seq (&bind_body
, dlist
);
15846 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
15847 gimple_bind_set_body (bind
, bind_body
);
15849 pop_gimplify_context (bind
);
15851 gimple_bind_append_vars (bind
, ctx
->block_vars
);
15852 BLOCK_VARS (block
) = ctx
->block_vars
;
15853 if (BLOCK_VARS (block
))
15854 TREE_USED (block
) = 1;
15858 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
15859 regimplified. If DATA is non-NULL, lower_omp_1 is outside
15860 of OMP context, but with task_shared_vars set. */
15863 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
15868 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
15869 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
15872 if (task_shared_vars
15874 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
15877 /* If a global variable has been privatized, TREE_CONSTANT on
15878 ADDR_EXPR might be wrong. */
15879 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
15880 recompute_tree_invariant_for_addr_expr (t
);
15882 *walk_subtrees
= !IS_TYPE_OR_DECL_P (t
);
15886 /* Data to be communicated between lower_omp_regimplify_operands and
15887 lower_omp_regimplify_operands_p. */
15889 struct lower_omp_regimplify_operands_data
15895 /* Helper function for lower_omp_regimplify_operands. Find
15896 omp_member_access_dummy_var vars and adjust temporarily their
15897 DECL_VALUE_EXPRs if needed. */
15900 lower_omp_regimplify_operands_p (tree
*tp
, int *walk_subtrees
,
15903 tree t
= omp_member_access_dummy_var (*tp
);
15906 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
15907 lower_omp_regimplify_operands_data
*ldata
15908 = (lower_omp_regimplify_operands_data
*) wi
->info
;
15909 tree o
= maybe_lookup_decl (t
, ldata
->ctx
);
15912 ldata
->decls
->safe_push (DECL_VALUE_EXPR (*tp
));
15913 ldata
->decls
->safe_push (*tp
);
15914 tree v
= unshare_and_remap (DECL_VALUE_EXPR (*tp
), t
, o
);
15915 SET_DECL_VALUE_EXPR (*tp
, v
);
15918 *walk_subtrees
= !IS_TYPE_OR_DECL_P (*tp
);
15922 /* Wrapper around gimple_regimplify_operands that adjusts DECL_VALUE_EXPRs
15923 of omp_member_access_dummy_var vars during regimplification. */
15926 lower_omp_regimplify_operands (omp_context
*ctx
, gimple
*stmt
,
15927 gimple_stmt_iterator
*gsi_p
)
15929 auto_vec
<tree
, 10> decls
;
15932 struct walk_stmt_info wi
;
15933 memset (&wi
, '\0', sizeof (wi
));
15934 struct lower_omp_regimplify_operands_data data
;
15936 data
.decls
= &decls
;
15938 walk_gimple_op (stmt
, lower_omp_regimplify_operands_p
, &wi
);
15940 gimple_regimplify_operands (stmt
, gsi_p
);
15941 while (!decls
.is_empty ())
15943 tree t
= decls
.pop ();
15944 tree v
= decls
.pop ();
15945 SET_DECL_VALUE_EXPR (t
, v
);
15950 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
15952 gimple
*stmt
= gsi_stmt (*gsi_p
);
15953 struct walk_stmt_info wi
;
15956 if (gimple_has_location (stmt
))
15957 input_location
= gimple_location (stmt
);
15959 if (task_shared_vars
)
15960 memset (&wi
, '\0', sizeof (wi
));
15962 /* If we have issued syntax errors, avoid doing any heavy lifting.
15963 Just replace the OMP directives with a NOP to avoid
15964 confusing RTL expansion. */
15965 if (seen_error () && is_gimple_omp (stmt
))
15967 gsi_replace (gsi_p
, gimple_build_nop (), true);
15971 switch (gimple_code (stmt
))
15975 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
15976 if ((ctx
|| task_shared_vars
)
15977 && (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
15978 lower_omp_regimplify_p
,
15979 ctx
? NULL
: &wi
, NULL
)
15980 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
15981 lower_omp_regimplify_p
,
15982 ctx
? NULL
: &wi
, NULL
)))
15983 lower_omp_regimplify_operands (ctx
, cond_stmt
, gsi_p
);
15987 lower_omp (gimple_catch_handler_ptr (as_a
<gcatch
*> (stmt
)), ctx
);
15989 case GIMPLE_EH_FILTER
:
15990 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
15993 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
15994 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
15996 case GIMPLE_TRANSACTION
:
15997 lower_omp (gimple_transaction_body_ptr (
15998 as_a
<gtransaction
*> (stmt
)),
16002 lower_omp (gimple_bind_body_ptr (as_a
<gbind
*> (stmt
)), ctx
);
16004 case GIMPLE_OMP_PARALLEL
:
16005 case GIMPLE_OMP_TASK
:
16006 ctx
= maybe_lookup_ctx (stmt
);
16008 if (ctx
->cancellable
)
16009 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
16010 lower_omp_taskreg (gsi_p
, ctx
);
16012 case GIMPLE_OMP_FOR
:
16013 ctx
= maybe_lookup_ctx (stmt
);
16015 if (ctx
->cancellable
)
16016 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
16017 lower_omp_for (gsi_p
, ctx
);
16019 case GIMPLE_OMP_SECTIONS
:
16020 ctx
= maybe_lookup_ctx (stmt
);
16022 if (ctx
->cancellable
)
16023 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
16024 lower_omp_sections (gsi_p
, ctx
);
16026 case GIMPLE_OMP_SINGLE
:
16027 ctx
= maybe_lookup_ctx (stmt
);
16029 lower_omp_single (gsi_p
, ctx
);
16031 case GIMPLE_OMP_MASTER
:
16032 ctx
= maybe_lookup_ctx (stmt
);
16034 lower_omp_master (gsi_p
, ctx
);
16036 case GIMPLE_OMP_TASKGROUP
:
16037 ctx
= maybe_lookup_ctx (stmt
);
16039 lower_omp_taskgroup (gsi_p
, ctx
);
16041 case GIMPLE_OMP_ORDERED
:
16042 ctx
= maybe_lookup_ctx (stmt
);
16044 lower_omp_ordered (gsi_p
, ctx
);
16046 case GIMPLE_OMP_CRITICAL
:
16047 ctx
= maybe_lookup_ctx (stmt
);
16049 lower_omp_critical (gsi_p
, ctx
);
16051 case GIMPLE_OMP_ATOMIC_LOAD
:
16052 if ((ctx
|| task_shared_vars
)
16053 && walk_tree (gimple_omp_atomic_load_rhs_ptr (
16054 as_a
<gomp_atomic_load
*> (stmt
)),
16055 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
16056 lower_omp_regimplify_operands (ctx
, stmt
, gsi_p
);
16058 case GIMPLE_OMP_TARGET
:
16059 ctx
= maybe_lookup_ctx (stmt
);
16061 lower_omp_target (gsi_p
, ctx
);
16063 case GIMPLE_OMP_TEAMS
:
16064 ctx
= maybe_lookup_ctx (stmt
);
16066 lower_omp_teams (gsi_p
, ctx
);
16070 call_stmt
= as_a
<gcall
*> (stmt
);
16071 fndecl
= gimple_call_fndecl (call_stmt
);
16073 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
16074 switch (DECL_FUNCTION_CODE (fndecl
))
16076 case BUILT_IN_GOMP_BARRIER
:
16080 case BUILT_IN_GOMP_CANCEL
:
16081 case BUILT_IN_GOMP_CANCELLATION_POINT
:
16084 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
16085 cctx
= cctx
->outer
;
16086 gcc_assert (gimple_call_lhs (call_stmt
) == NULL_TREE
);
16087 if (!cctx
->cancellable
)
16089 if (DECL_FUNCTION_CODE (fndecl
)
16090 == BUILT_IN_GOMP_CANCELLATION_POINT
)
16092 stmt
= gimple_build_nop ();
16093 gsi_replace (gsi_p
, stmt
, false);
16097 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
16099 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
16100 gimple_call_set_fndecl (call_stmt
, fndecl
);
16101 gimple_call_set_fntype (call_stmt
, TREE_TYPE (fndecl
));
16104 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)));
16105 gimple_call_set_lhs (call_stmt
, lhs
);
16106 tree fallthru_label
;
16107 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
16109 g
= gimple_build_label (fallthru_label
);
16110 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
16111 g
= gimple_build_cond (NE_EXPR
, lhs
,
16112 fold_convert (TREE_TYPE (lhs
),
16113 boolean_false_node
),
16114 cctx
->cancel_label
, fallthru_label
);
16115 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
16122 if ((ctx
|| task_shared_vars
)
16123 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
16126 /* Just remove clobbers, this should happen only if we have
16127 "privatized" local addressable variables in SIMD regions,
16128 the clobber isn't needed in that case and gimplifying address
16129 of the ARRAY_REF into a pointer and creating MEM_REF based
16130 clobber would create worse code than we get with the clobber
16132 if (gimple_clobber_p (stmt
))
16134 gsi_replace (gsi_p
, gimple_build_nop (), true);
16137 lower_omp_regimplify_operands (ctx
, stmt
, gsi_p
);
16144 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
16146 location_t saved_location
= input_location
;
16147 gimple_stmt_iterator gsi
;
16148 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
16149 lower_omp_1 (&gsi
, ctx
);
16150 /* During gimplification, we haven't folded statments inside offloading
16151 or taskreg regions (gimplify.c:maybe_fold_stmt); do that now. */
16152 if (target_nesting_level
|| taskreg_nesting_level
)
16153 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
16155 input_location
= saved_location
;
16158 /* Main entry point. */
16160 static unsigned int
16161 execute_lower_omp (void)
16167 /* This pass always runs, to provide PROP_gimple_lomp.
16168 But often, there is nothing to do. */
16169 if (flag_cilkplus
== 0 && flag_openacc
== 0 && flag_openmp
== 0
16170 && flag_openmp_simd
== 0)
16173 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
16174 delete_omp_context
);
16176 body
= gimple_body (current_function_decl
);
16177 scan_omp (&body
, NULL
);
16178 gcc_assert (taskreg_nesting_level
== 0);
16179 FOR_EACH_VEC_ELT (taskreg_contexts
, i
, ctx
)
16180 finish_taskreg_scan (ctx
);
16181 taskreg_contexts
.release ();
16183 if (all_contexts
->root
)
16185 if (task_shared_vars
)
16186 push_gimplify_context ();
16187 lower_omp (&body
, NULL
);
16188 if (task_shared_vars
)
16189 pop_gimplify_context (NULL
);
16194 splay_tree_delete (all_contexts
);
16195 all_contexts
= NULL
;
16197 BITMAP_FREE (task_shared_vars
);
16203 const pass_data pass_data_lower_omp
=
16205 GIMPLE_PASS
, /* type */
16206 "omplower", /* name */
16207 OPTGROUP_NONE
, /* optinfo_flags */
16208 TV_NONE
, /* tv_id */
16209 PROP_gimple_any
, /* properties_required */
16210 PROP_gimple_lomp
, /* properties_provided */
16211 0, /* properties_destroyed */
16212 0, /* todo_flags_start */
16213 0, /* todo_flags_finish */
16216 class pass_lower_omp
: public gimple_opt_pass
16219 pass_lower_omp (gcc::context
*ctxt
)
16220 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
16223 /* opt_pass methods: */
16224 virtual unsigned int execute (function
*) { return execute_lower_omp (); }
16226 }; // class pass_lower_omp
16228 } // anon namespace
16231 make_pass_lower_omp (gcc::context
*ctxt
)
16233 return new pass_lower_omp (ctxt
);
16236 /* The following is a utility to diagnose structured block violations.
16237 It is not part of the "omplower" pass, as that's invoked too late. It
16238 should be invoked by the respective front ends after gimplification. */
16240 static splay_tree all_labels
;
16242 /* Check for mismatched contexts and generate an error if needed. Return
16243 true if an error is detected. */
16246 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
16247 gimple
*branch_ctx
, gimple
*label_ctx
)
16249 gcc_checking_assert (!branch_ctx
|| is_gimple_omp (branch_ctx
));
16250 gcc_checking_assert (!label_ctx
|| is_gimple_omp (label_ctx
));
16252 if (label_ctx
== branch_ctx
)
16255 const char* kind
= NULL
;
16260 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
16261 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
16263 && gimple_code (label_ctx
) == GIMPLE_OMP_FOR
16264 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
16265 kind
= "Cilk Plus";
16269 if ((branch_ctx
&& is_gimple_omp_oacc (branch_ctx
))
16270 || (label_ctx
&& is_gimple_omp_oacc (label_ctx
)))
16272 gcc_checking_assert (kind
== NULL
);
16278 gcc_checking_assert (flag_openmp
);
16283 Previously we kept track of the label's entire context in diagnose_sb_[12]
16284 so we could traverse it and issue a correct "exit" or "enter" error
16285 message upon a structured block violation.
16287 We built the context by building a list with tree_cons'ing, but there is
16288 no easy counterpart in gimple tuples. It seems like far too much work
16289 for issuing exit/enter error messages. If someone really misses the
16290 distinct error message... patches welcome.
16294 /* Try to avoid confusing the user by producing and error message
16295 with correct "exit" or "enter" verbiage. We prefer "exit"
16296 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
16297 if (branch_ctx
== NULL
)
16303 if (TREE_VALUE (label_ctx
) == branch_ctx
)
16308 label_ctx
= TREE_CHAIN (label_ctx
);
16313 error ("invalid exit from %s structured block", kind
);
16315 error ("invalid entry to %s structured block", kind
);
16318 /* If it's obvious we have an invalid entry, be specific about the error. */
16319 if (branch_ctx
== NULL
)
16320 error ("invalid entry to %s structured block", kind
);
16323 /* Otherwise, be vague and lazy, but efficient. */
16324 error ("invalid branch to/from %s structured block", kind
);
16327 gsi_replace (gsi_p
, gimple_build_nop (), false);
16331 /* Pass 1: Create a minimal tree of structured blocks, and record
16332 where each label is found. */
16335 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
16336 struct walk_stmt_info
*wi
)
16338 gimple
*context
= (gimple
*) wi
->info
;
16339 gimple
*inner_context
;
16340 gimple
*stmt
= gsi_stmt (*gsi_p
);
16342 *handled_ops_p
= true;
16344 switch (gimple_code (stmt
))
16348 case GIMPLE_OMP_PARALLEL
:
16349 case GIMPLE_OMP_TASK
:
16350 case GIMPLE_OMP_SECTIONS
:
16351 case GIMPLE_OMP_SINGLE
:
16352 case GIMPLE_OMP_SECTION
:
16353 case GIMPLE_OMP_MASTER
:
16354 case GIMPLE_OMP_ORDERED
:
16355 case GIMPLE_OMP_CRITICAL
:
16356 case GIMPLE_OMP_TARGET
:
16357 case GIMPLE_OMP_TEAMS
:
16358 case GIMPLE_OMP_TASKGROUP
:
16359 /* The minimal context here is just the current OMP construct. */
16360 inner_context
= stmt
;
16361 wi
->info
= inner_context
;
16362 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
16363 wi
->info
= context
;
16366 case GIMPLE_OMP_FOR
:
16367 inner_context
= stmt
;
16368 wi
->info
= inner_context
;
16369 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
16371 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
16372 diagnose_sb_1
, NULL
, wi
);
16373 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
16374 wi
->info
= context
;
16378 splay_tree_insert (all_labels
,
16379 (splay_tree_key
) gimple_label_label (
16380 as_a
<glabel
*> (stmt
)),
16381 (splay_tree_value
) context
);
16391 /* Pass 2: Check each branch and see if its context differs from that of
16392 the destination label's context. */
16395 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
16396 struct walk_stmt_info
*wi
)
16398 gimple
*context
= (gimple
*) wi
->info
;
16400 gimple
*stmt
= gsi_stmt (*gsi_p
);
16402 *handled_ops_p
= true;
16404 switch (gimple_code (stmt
))
16408 case GIMPLE_OMP_PARALLEL
:
16409 case GIMPLE_OMP_TASK
:
16410 case GIMPLE_OMP_SECTIONS
:
16411 case GIMPLE_OMP_SINGLE
:
16412 case GIMPLE_OMP_SECTION
:
16413 case GIMPLE_OMP_MASTER
:
16414 case GIMPLE_OMP_ORDERED
:
16415 case GIMPLE_OMP_CRITICAL
:
16416 case GIMPLE_OMP_TARGET
:
16417 case GIMPLE_OMP_TEAMS
:
16418 case GIMPLE_OMP_TASKGROUP
:
16420 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
16421 wi
->info
= context
;
16424 case GIMPLE_OMP_FOR
:
16426 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
16428 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
16429 diagnose_sb_2
, NULL
, wi
);
16430 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
16431 wi
->info
= context
;
16436 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
16437 tree lab
= gimple_cond_true_label (cond_stmt
);
16440 n
= splay_tree_lookup (all_labels
,
16441 (splay_tree_key
) lab
);
16442 diagnose_sb_0 (gsi_p
, context
,
16443 n
? (gimple
*) n
->value
: NULL
);
16445 lab
= gimple_cond_false_label (cond_stmt
);
16448 n
= splay_tree_lookup (all_labels
,
16449 (splay_tree_key
) lab
);
16450 diagnose_sb_0 (gsi_p
, context
,
16451 n
? (gimple
*) n
->value
: NULL
);
16458 tree lab
= gimple_goto_dest (stmt
);
16459 if (TREE_CODE (lab
) != LABEL_DECL
)
16462 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
16463 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
*) n
->value
: NULL
);
16467 case GIMPLE_SWITCH
:
16469 gswitch
*switch_stmt
= as_a
<gswitch
*> (stmt
);
16471 for (i
= 0; i
< gimple_switch_num_labels (switch_stmt
); ++i
)
16473 tree lab
= CASE_LABEL (gimple_switch_label (switch_stmt
, i
));
16474 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
16475 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
*) n
->value
))
16481 case GIMPLE_RETURN
:
16482 diagnose_sb_0 (gsi_p
, context
, NULL
);
16492 /* Called from tree-cfg.c::make_edges to create cfg edges for all relevant
16495 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
,
16498 gimple
*last
= last_stmt (bb
);
16499 enum gimple_code code
= gimple_code (last
);
16500 struct omp_region
*cur_region
= *region
;
16501 bool fallthru
= false;
16505 case GIMPLE_OMP_PARALLEL
:
16506 case GIMPLE_OMP_TASK
:
16507 case GIMPLE_OMP_FOR
:
16508 case GIMPLE_OMP_SINGLE
:
16509 case GIMPLE_OMP_TEAMS
:
16510 case GIMPLE_OMP_MASTER
:
16511 case GIMPLE_OMP_TASKGROUP
:
16512 case GIMPLE_OMP_CRITICAL
:
16513 case GIMPLE_OMP_SECTION
:
16514 cur_region
= new_omp_region (bb
, code
, cur_region
);
16518 case GIMPLE_OMP_ORDERED
:
16519 cur_region
= new_omp_region (bb
, code
, cur_region
);
16521 if (find_omp_clause (gimple_omp_ordered_clauses
16522 (as_a
<gomp_ordered
*> (last
)),
16523 OMP_CLAUSE_DEPEND
))
16524 cur_region
= cur_region
->outer
;
16527 case GIMPLE_OMP_TARGET
:
16528 cur_region
= new_omp_region (bb
, code
, cur_region
);
16530 switch (gimple_omp_target_kind (last
))
16532 case GF_OMP_TARGET_KIND_REGION
:
16533 case GF_OMP_TARGET_KIND_DATA
:
16534 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
16535 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
16536 case GF_OMP_TARGET_KIND_OACC_DATA
:
16538 case GF_OMP_TARGET_KIND_UPDATE
:
16539 case GF_OMP_TARGET_KIND_ENTER_DATA
:
16540 case GF_OMP_TARGET_KIND_EXIT_DATA
:
16541 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
16542 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
16543 cur_region
= cur_region
->outer
;
16546 gcc_unreachable ();
16550 case GIMPLE_OMP_SECTIONS
:
16551 cur_region
= new_omp_region (bb
, code
, cur_region
);
16555 case GIMPLE_OMP_SECTIONS_SWITCH
:
16559 case GIMPLE_OMP_ATOMIC_LOAD
:
16560 case GIMPLE_OMP_ATOMIC_STORE
:
16564 case GIMPLE_OMP_RETURN
:
16565 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
16566 somewhere other than the next block. This will be
16568 cur_region
->exit
= bb
;
16569 if (cur_region
->type
== GIMPLE_OMP_TASK
)
16570 /* Add an edge corresponding to not scheduling the task
16572 make_edge (cur_region
->entry
, bb
, EDGE_ABNORMAL
);
16573 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
16574 cur_region
= cur_region
->outer
;
16577 case GIMPLE_OMP_CONTINUE
:
16578 cur_region
->cont
= bb
;
16579 switch (cur_region
->type
)
16581 case GIMPLE_OMP_FOR
:
16582 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
16583 succs edges as abnormal to prevent splitting
16585 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
16586 /* Make the loopback edge. */
16587 make_edge (bb
, single_succ (cur_region
->entry
),
16590 /* Create an edge from GIMPLE_OMP_FOR to exit, which
16591 corresponds to the case that the body of the loop
16592 is not executed at all. */
16593 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
16594 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
16598 case GIMPLE_OMP_SECTIONS
:
16599 /* Wire up the edges into and out of the nested sections. */
16601 basic_block switch_bb
= single_succ (cur_region
->entry
);
16603 struct omp_region
*i
;
16604 for (i
= cur_region
->inner
; i
; i
= i
->next
)
16606 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
16607 make_edge (switch_bb
, i
->entry
, 0);
16608 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
16611 /* Make the loopback edge to the block with
16612 GIMPLE_OMP_SECTIONS_SWITCH. */
16613 make_edge (bb
, switch_bb
, 0);
16615 /* Make the edge from the switch to exit. */
16616 make_edge (switch_bb
, bb
->next_bb
, 0);
16621 case GIMPLE_OMP_TASK
:
16626 gcc_unreachable ();
16631 gcc_unreachable ();
16634 if (*region
!= cur_region
)
16636 *region
= cur_region
;
16638 *region_idx
= cur_region
->entry
->index
;
16646 static unsigned int
16647 diagnose_omp_structured_block_errors (void)
16649 struct walk_stmt_info wi
;
16650 gimple_seq body
= gimple_body (current_function_decl
);
16652 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
16654 memset (&wi
, 0, sizeof (wi
));
16655 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
16657 memset (&wi
, 0, sizeof (wi
));
16658 wi
.want_locations
= true;
16659 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
16661 gimple_set_body (current_function_decl
, body
);
16663 splay_tree_delete (all_labels
);
16671 const pass_data pass_data_diagnose_omp_blocks
=
16673 GIMPLE_PASS
, /* type */
16674 "*diagnose_omp_blocks", /* name */
16675 OPTGROUP_NONE
, /* optinfo_flags */
16676 TV_NONE
, /* tv_id */
16677 PROP_gimple_any
, /* properties_required */
16678 0, /* properties_provided */
16679 0, /* properties_destroyed */
16680 0, /* todo_flags_start */
16681 0, /* todo_flags_finish */
16684 class pass_diagnose_omp_blocks
: public gimple_opt_pass
16687 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
16688 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
16691 /* opt_pass methods: */
16692 virtual bool gate (function
*)
16694 return flag_cilkplus
|| flag_openacc
|| flag_openmp
;
16696 virtual unsigned int execute (function
*)
16698 return diagnose_omp_structured_block_errors ();
16701 }; // class pass_diagnose_omp_blocks
16703 } // anon namespace
16706 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
16708 return new pass_diagnose_omp_blocks (ctxt
);
16711 /* SIMD clone supporting code. */
16713 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
16714 of arguments to reserve space for. */
16716 static struct cgraph_simd_clone
*
16717 simd_clone_struct_alloc (int nargs
)
16719 struct cgraph_simd_clone
*clone_info
;
16720 size_t len
= (sizeof (struct cgraph_simd_clone
)
16721 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
16722 clone_info
= (struct cgraph_simd_clone
*)
16723 ggc_internal_cleared_alloc (len
);
16727 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
16730 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
16731 struct cgraph_simd_clone
*from
)
16733 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
16734 + ((from
->nargs
- from
->inbranch
)
16735 * sizeof (struct cgraph_simd_clone_arg
))));
16738 /* Return vector of parameter types of function FNDECL. This uses
16739 TYPE_ARG_TYPES if available, otherwise falls back to types of
16740 DECL_ARGUMENTS types. */
16743 simd_clone_vector_of_formal_parm_types (tree fndecl
)
16745 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
16746 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
16747 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
16750 FOR_EACH_VEC_ELT (args
, i
, arg
)
16751 args
[i
] = TREE_TYPE (args
[i
]);
16755 /* Given a simd function in NODE, extract the simd specific
16756 information from the OMP clauses passed in CLAUSES, and return
16757 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
16758 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
16759 otherwise set to FALSE. */
16761 static struct cgraph_simd_clone
*
16762 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
16763 bool *inbranch_specified
)
16765 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
16768 *inbranch_specified
= false;
16770 n
= args
.length ();
16771 if (n
> 0 && args
.last () == void_type_node
)
16774 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
16775 be cloned have a distinctive artificial label in addition to "omp
16779 && lookup_attribute ("cilk simd function",
16780 DECL_ATTRIBUTES (node
->decl
)));
16782 /* Allocate one more than needed just in case this is an in-branch
16783 clone which will require a mask argument. */
16784 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
16785 clone_info
->nargs
= n
;
16786 clone_info
->cilk_elemental
= cilk_clone
;
16793 clauses
= TREE_VALUE (clauses
);
16794 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
16797 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
16799 switch (OMP_CLAUSE_CODE (t
))
16801 case OMP_CLAUSE_INBRANCH
:
16802 clone_info
->inbranch
= 1;
16803 *inbranch_specified
= true;
16805 case OMP_CLAUSE_NOTINBRANCH
:
16806 clone_info
->inbranch
= 0;
16807 *inbranch_specified
= true;
16809 case OMP_CLAUSE_SIMDLEN
:
16810 clone_info
->simdlen
16811 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
16813 case OMP_CLAUSE_LINEAR
:
16815 tree decl
= OMP_CLAUSE_DECL (t
);
16816 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
16817 int argno
= TREE_INT_CST_LOW (decl
);
16818 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
16820 enum cgraph_simd_clone_arg_type arg_type
;
16821 if (TREE_CODE (args
[argno
]) == REFERENCE_TYPE
)
16822 switch (OMP_CLAUSE_LINEAR_KIND (t
))
16824 case OMP_CLAUSE_LINEAR_REF
:
16826 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
;
16828 case OMP_CLAUSE_LINEAR_UVAL
:
16830 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
;
16832 case OMP_CLAUSE_LINEAR_VAL
:
16833 case OMP_CLAUSE_LINEAR_DEFAULT
:
16835 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
;
16838 gcc_unreachable ();
16841 arg_type
= SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
16842 clone_info
->args
[argno
].arg_type
= arg_type
;
16843 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
16844 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
16845 && clone_info
->args
[argno
].linear_step
< n
);
16849 if (POINTER_TYPE_P (args
[argno
]))
16850 step
= fold_convert (ssizetype
, step
);
16851 if (!tree_fits_shwi_p (step
))
16853 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
16854 "ignoring large linear step");
16858 else if (integer_zerop (step
))
16860 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
16861 "ignoring zero linear step");
16867 enum cgraph_simd_clone_arg_type arg_type
;
16868 if (TREE_CODE (args
[argno
]) == REFERENCE_TYPE
)
16869 switch (OMP_CLAUSE_LINEAR_KIND (t
))
16871 case OMP_CLAUSE_LINEAR_REF
:
16873 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
;
16875 case OMP_CLAUSE_LINEAR_UVAL
:
16877 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
;
16879 case OMP_CLAUSE_LINEAR_VAL
:
16880 case OMP_CLAUSE_LINEAR_DEFAULT
:
16882 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
;
16885 gcc_unreachable ();
16888 arg_type
= SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
16889 clone_info
->args
[argno
].arg_type
= arg_type
;
16890 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
16895 case OMP_CLAUSE_UNIFORM
:
16897 tree decl
= OMP_CLAUSE_DECL (t
);
16898 int argno
= tree_to_uhwi (decl
);
16899 clone_info
->args
[argno
].arg_type
16900 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
16903 case OMP_CLAUSE_ALIGNED
:
16905 tree decl
= OMP_CLAUSE_DECL (t
);
16906 int argno
= tree_to_uhwi (decl
);
16907 clone_info
->args
[argno
].alignment
16908 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
16919 /* Given a SIMD clone in NODE, calculate the characteristic data
16920 type and return the coresponding type. The characteristic data
16921 type is computed as described in the Intel Vector ABI. */
16924 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
16925 struct cgraph_simd_clone
*clone_info
)
16927 tree type
= integer_type_node
;
16928 tree fndecl
= node
->decl
;
16930 /* a) For non-void function, the characteristic data type is the
16932 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
16933 type
= TREE_TYPE (TREE_TYPE (fndecl
));
16935 /* b) If the function has any non-uniform, non-linear parameters,
16936 then the characteristic data type is the type of the first
16940 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
16941 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
16942 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
16950 /* c) If the characteristic data type determined by a) or b) above
16951 is struct, union, or class type which is pass-by-value (except
16952 for the type that maps to the built-in complex data type), the
16953 characteristic data type is int. */
16954 if (RECORD_OR_UNION_TYPE_P (type
)
16955 && !aggregate_value_p (type
, NULL
)
16956 && TREE_CODE (type
) != COMPLEX_TYPE
)
16957 return integer_type_node
;
16959 /* d) If none of the above three classes is applicable, the
16960 characteristic data type is int. */
16964 /* e) For Intel Xeon Phi native and offload compilation, if the
16965 resulting characteristic data type is 8-bit or 16-bit integer
16966 data type, the characteristic data type is int. */
16967 /* Well, we don't handle Xeon Phi yet. */
16971 simd_clone_mangle (struct cgraph_node
*node
,
16972 struct cgraph_simd_clone
*clone_info
)
16974 char vecsize_mangle
= clone_info
->vecsize_mangle
;
16975 char mask
= clone_info
->inbranch
? 'M' : 'N';
16976 unsigned int simdlen
= clone_info
->simdlen
;
16980 gcc_assert (vecsize_mangle
&& simdlen
);
16982 pp_string (&pp
, "_ZGV");
16983 pp_character (&pp
, vecsize_mangle
);
16984 pp_character (&pp
, mask
);
16985 pp_decimal_int (&pp
, simdlen
);
16987 for (n
= 0; n
< clone_info
->nargs
; ++n
)
16989 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
16991 switch (arg
.arg_type
)
16993 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
16994 pp_character (&pp
, 'u');
16996 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
16997 pp_character (&pp
, 'l');
16998 goto mangle_linear
;
16999 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
17000 pp_character (&pp
, 'R');
17001 goto mangle_linear
;
17002 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
17003 pp_character (&pp
, 'L');
17004 goto mangle_linear
;
17005 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
17006 pp_character (&pp
, 'U');
17007 goto mangle_linear
;
17009 gcc_assert (arg
.linear_step
!= 0);
17010 if (arg
.linear_step
> 1)
17011 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
17012 else if (arg
.linear_step
< 0)
17014 pp_character (&pp
, 'n');
17015 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
17019 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
17020 pp_string (&pp
, "ls");
17021 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
17023 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
17024 pp_string (&pp
, "Rs");
17025 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
17027 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
17028 pp_string (&pp
, "Ls");
17029 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
17031 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
17032 pp_string (&pp
, "Us");
17033 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
17036 pp_character (&pp
, 'v');
17040 pp_character (&pp
, 'a');
17041 pp_decimal_int (&pp
, arg
.alignment
);
17045 pp_underscore (&pp
);
17046 const char *str
= IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
));
17049 pp_string (&pp
, str
);
17050 str
= pp_formatted_text (&pp
);
17052 /* If there already is a SIMD clone with the same mangled name, don't
17053 add another one. This can happen e.g. for
17054 #pragma omp declare simd
17055 #pragma omp declare simd simdlen(8)
17056 int foo (int, int);
17057 if the simdlen is assumed to be 8 for the first one, etc. */
17058 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
17059 clone
= clone
->simdclone
->next_clone
)
17060 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone
->decl
)),
17064 return get_identifier (str
);
17067 /* Create a simd clone of OLD_NODE and return it. */
17069 static struct cgraph_node
*
17070 simd_clone_create (struct cgraph_node
*old_node
)
17072 struct cgraph_node
*new_node
;
17073 if (old_node
->definition
)
17075 if (!old_node
->has_gimple_body_p ())
17077 old_node
->get_body ();
17078 new_node
= old_node
->create_version_clone_with_body (vNULL
, NULL
, NULL
,
17084 tree old_decl
= old_node
->decl
;
17085 tree new_decl
= copy_node (old_node
->decl
);
17086 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
17087 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
17088 SET_DECL_RTL (new_decl
, NULL
);
17089 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
17090 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
17091 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
17092 if (old_node
->in_other_partition
)
17093 new_node
->in_other_partition
= 1;
17094 symtab
->call_cgraph_insertion_hooks (new_node
);
17096 if (new_node
== NULL
)
17099 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
17101 /* The function cgraph_function_versioning () will force the new
17102 symbol local. Undo this, and inherit external visability from
17104 new_node
->local
.local
= old_node
->local
.local
;
17105 new_node
->externally_visible
= old_node
->externally_visible
;
17110 /* Adjust the return type of the given function to its appropriate
17111 vector counterpart. Returns a simd array to be used throughout the
17112 function as a return value. */
17115 simd_clone_adjust_return_type (struct cgraph_node
*node
)
17117 tree fndecl
= node
->decl
;
17118 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
17119 unsigned int veclen
;
17122 /* Adjust the function return type. */
17123 if (orig_rettype
== void_type_node
)
17125 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
17126 t
= TREE_TYPE (TREE_TYPE (fndecl
));
17127 if (INTEGRAL_TYPE_P (t
) || POINTER_TYPE_P (t
))
17128 veclen
= node
->simdclone
->vecsize_int
;
17130 veclen
= node
->simdclone
->vecsize_float
;
17131 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (t
));
17132 if (veclen
> node
->simdclone
->simdlen
)
17133 veclen
= node
->simdclone
->simdlen
;
17134 if (POINTER_TYPE_P (t
))
17135 t
= pointer_sized_int_node
;
17136 if (veclen
== node
->simdclone
->simdlen
)
17137 t
= build_vector_type (t
, node
->simdclone
->simdlen
);
17140 t
= build_vector_type (t
, veclen
);
17141 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
17143 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
17144 if (!node
->definition
)
17147 t
= DECL_RESULT (fndecl
);
17148 /* Adjust the DECL_RESULT. */
17149 gcc_assert (TREE_TYPE (t
) != void_type_node
);
17150 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
17153 tree atype
= build_array_type_nelts (orig_rettype
,
17154 node
->simdclone
->simdlen
);
17155 if (veclen
!= node
->simdclone
->simdlen
)
17156 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
17158 /* Set up a SIMD array to use as the return value. */
17159 tree retval
= create_tmp_var_raw (atype
, "retval");
17160 gimple_add_tmp_var (retval
);
17164 /* Each vector argument has a corresponding array to be used locally
17165 as part of the eventual loop. Create such temporary array and
17168 PREFIX is the prefix to be used for the temporary.
17170 TYPE is the inner element type.
17172 SIMDLEN is the number of elements. */
17175 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
17177 tree atype
= build_array_type_nelts (type
, simdlen
);
17178 tree avar
= create_tmp_var_raw (atype
, prefix
);
17179 gimple_add_tmp_var (avar
);
17183 /* Modify the function argument types to their corresponding vector
17184 counterparts if appropriate. Also, create one array for each simd
17185 argument to be used locally when using the function arguments as
17188 NODE is the function whose arguments are to be adjusted.
17190 Returns an adjustment vector that will be filled describing how the
17191 argument types will be adjusted. */
17193 static ipa_parm_adjustment_vec
17194 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
17197 ipa_parm_adjustment_vec adjustments
;
17199 if (node
->definition
)
17200 args
= ipa_get_vector_of_formal_parms (node
->decl
);
17202 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
17203 adjustments
.create (args
.length ());
17204 unsigned i
, j
, veclen
;
17205 struct ipa_parm_adjustment adj
;
17206 for (i
= 0; i
< node
->simdclone
->nargs
; ++i
)
17208 memset (&adj
, 0, sizeof (adj
));
17209 tree parm
= args
[i
];
17210 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
17211 adj
.base_index
= i
;
17214 node
->simdclone
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
17215 node
->simdclone
->args
[i
].orig_type
= parm_type
;
17217 switch (node
->simdclone
->args
[i
].arg_type
)
17220 /* No adjustment necessary for scalar arguments. */
17221 adj
.op
= IPA_PARM_OP_COPY
;
17223 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
17224 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
17225 if (node
->definition
)
17226 node
->simdclone
->args
[i
].simd_array
17227 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
17228 TREE_TYPE (parm_type
),
17229 node
->simdclone
->simdlen
);
17230 adj
.op
= IPA_PARM_OP_COPY
;
17232 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
17233 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
17234 case SIMD_CLONE_ARG_TYPE_VECTOR
:
17235 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
17236 veclen
= node
->simdclone
->vecsize_int
;
17238 veclen
= node
->simdclone
->vecsize_float
;
17239 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
17240 if (veclen
> node
->simdclone
->simdlen
)
17241 veclen
= node
->simdclone
->simdlen
;
17242 adj
.arg_prefix
= "simd";
17243 if (POINTER_TYPE_P (parm_type
))
17244 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
17246 adj
.type
= build_vector_type (parm_type
, veclen
);
17247 node
->simdclone
->args
[i
].vector_type
= adj
.type
;
17248 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
17250 adjustments
.safe_push (adj
);
17253 memset (&adj
, 0, sizeof (adj
));
17254 adj
.op
= IPA_PARM_OP_NEW
;
17255 adj
.arg_prefix
= "simd";
17256 adj
.base_index
= i
;
17257 adj
.type
= node
->simdclone
->args
[i
].vector_type
;
17261 if (node
->definition
)
17262 node
->simdclone
->args
[i
].simd_array
17263 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
17264 parm_type
, node
->simdclone
->simdlen
);
17266 adjustments
.safe_push (adj
);
17269 if (node
->simdclone
->inbranch
)
17272 = simd_clone_compute_base_data_type (node
->simdclone
->origin
,
17275 memset (&adj
, 0, sizeof (adj
));
17276 adj
.op
= IPA_PARM_OP_NEW
;
17277 adj
.arg_prefix
= "mask";
17279 adj
.base_index
= i
;
17280 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
17281 veclen
= node
->simdclone
->vecsize_int
;
17283 veclen
= node
->simdclone
->vecsize_float
;
17284 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
17285 if (veclen
> node
->simdclone
->simdlen
)
17286 veclen
= node
->simdclone
->simdlen
;
17287 if (POINTER_TYPE_P (base_type
))
17288 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
17290 adj
.type
= build_vector_type (base_type
, veclen
);
17291 adjustments
.safe_push (adj
);
17293 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
17294 adjustments
.safe_push (adj
);
17296 /* We have previously allocated one extra entry for the mask. Use
17298 struct cgraph_simd_clone
*sc
= node
->simdclone
;
17300 if (node
->definition
)
17302 sc
->args
[i
].orig_arg
17303 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
17304 sc
->args
[i
].simd_array
17305 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
17307 sc
->args
[i
].orig_type
= base_type
;
17308 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
17311 if (node
->definition
)
17312 ipa_modify_formal_parameters (node
->decl
, adjustments
);
17315 tree new_arg_types
= NULL_TREE
, new_reversed
;
17316 bool last_parm_void
= false;
17317 if (args
.length () > 0 && args
.last () == void_type_node
)
17318 last_parm_void
= true;
17320 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
17321 j
= adjustments
.length ();
17322 for (i
= 0; i
< j
; i
++)
17324 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
17326 if (adj
->op
== IPA_PARM_OP_COPY
)
17327 ptype
= args
[adj
->base_index
];
17330 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
17332 new_reversed
= nreverse (new_arg_types
);
17333 if (last_parm_void
)
17336 TREE_CHAIN (new_arg_types
) = void_list_node
;
17338 new_reversed
= void_list_node
;
17341 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
17342 TYPE_ARG_TYPES (new_type
) = new_reversed
;
17343 TREE_TYPE (node
->decl
) = new_type
;
17345 adjustments
.release ();
17348 return adjustments
;
17351 /* Initialize and copy the function arguments in NODE to their
17352 corresponding local simd arrays. Returns a fresh gimple_seq with
17353 the instruction sequence generated. */
17356 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
17357 ipa_parm_adjustment_vec adjustments
)
17359 gimple_seq seq
= NULL
;
17360 unsigned i
= 0, j
= 0, k
;
17362 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
17364 arg
= DECL_CHAIN (arg
), i
++, j
++)
17366 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
17367 || POINTER_TYPE_P (TREE_TYPE (arg
)))
17370 node
->simdclone
->args
[i
].vector_arg
= arg
;
17372 tree array
= node
->simdclone
->args
[i
].simd_array
;
17373 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
17375 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
17376 tree ptr
= build_fold_addr_expr (array
);
17377 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
17378 build_int_cst (ptype
, 0));
17379 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
17380 gimplify_and_add (t
, &seq
);
17384 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
17385 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
17386 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
17388 tree ptr
= build_fold_addr_expr (array
);
17392 arg
= DECL_CHAIN (arg
);
17396 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
17397 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
17398 build_int_cst (ptype
, k
* elemsize
));
17399 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
17400 gimplify_and_add (t
, &seq
);
17407 /* Callback info for ipa_simd_modify_stmt_ops below. */
17409 struct modify_stmt_info
{
17410 ipa_parm_adjustment_vec adjustments
;
17412 /* True if the parent statement was modified by
17413 ipa_simd_modify_stmt_ops. */
17417 /* Callback for walk_gimple_op.
17419 Adjust operands from a given statement as specified in the
17420 adjustments vector in the callback data. */
17423 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
17425 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
17426 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
17427 tree
*orig_tp
= tp
;
17428 if (TREE_CODE (*tp
) == ADDR_EXPR
)
17429 tp
= &TREE_OPERAND (*tp
, 0);
17430 struct ipa_parm_adjustment
*cand
= NULL
;
17431 if (TREE_CODE (*tp
) == PARM_DECL
)
17432 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
17436 *walk_subtrees
= 0;
17439 tree repl
= NULL_TREE
;
17441 repl
= unshare_expr (cand
->new_decl
);
17446 *walk_subtrees
= 0;
17447 bool modified
= info
->modified
;
17448 info
->modified
= false;
17449 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
17450 if (!info
->modified
)
17452 info
->modified
= modified
;
17455 info
->modified
= modified
;
17464 repl
= build_fold_addr_expr (repl
);
17466 if (is_gimple_debug (info
->stmt
))
17468 tree vexpr
= make_node (DEBUG_EXPR_DECL
);
17469 stmt
= gimple_build_debug_source_bind (vexpr
, repl
, NULL
);
17470 DECL_ARTIFICIAL (vexpr
) = 1;
17471 TREE_TYPE (vexpr
) = TREE_TYPE (repl
);
17472 DECL_MODE (vexpr
) = TYPE_MODE (TREE_TYPE (repl
));
17477 stmt
= gimple_build_assign (make_ssa_name (TREE_TYPE (repl
)), repl
);
17478 repl
= gimple_assign_lhs (stmt
);
17480 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
17481 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
17484 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
17486 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
17492 info
->modified
= true;
17496 /* Traverse the function body and perform all modifications as
17497 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
17498 modified such that the replacement/reduction value will now be an
17499 offset into the corresponding simd_array.
17501 This function will replace all function argument uses with their
17502 corresponding simd array elements, and ajust the return values
17506 ipa_simd_modify_function_body (struct cgraph_node
*node
,
17507 ipa_parm_adjustment_vec adjustments
,
17508 tree retval_array
, tree iter
)
17511 unsigned int i
, j
, l
;
17513 /* Re-use the adjustments array, but this time use it to replace
17514 every function argument use to an offset into the corresponding
17516 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
17518 if (!node
->simdclone
->args
[i
].vector_arg
)
17521 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
17522 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
17523 adjustments
[j
].new_decl
17524 = build4 (ARRAY_REF
,
17526 node
->simdclone
->args
[i
].simd_array
,
17528 NULL_TREE
, NULL_TREE
);
17529 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
17530 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
17531 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
17534 l
= adjustments
.length ();
17535 for (i
= 1; i
< num_ssa_names
; i
++)
17537 tree name
= ssa_name (i
);
17539 && SSA_NAME_VAR (name
)
17540 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
17542 for (j
= 0; j
< l
; j
++)
17543 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
17544 && adjustments
[j
].new_decl
)
17547 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
17550 = copy_var_decl (adjustments
[j
].base
,
17551 DECL_NAME (adjustments
[j
].base
),
17552 TREE_TYPE (adjustments
[j
].base
));
17553 adjustments
[j
].new_ssa_base
= base_var
;
17556 base_var
= adjustments
[j
].new_ssa_base
;
17557 if (SSA_NAME_IS_DEFAULT_DEF (name
))
17559 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
17560 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
17561 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
17562 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
17563 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
17564 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
17565 gimple
*stmt
= gimple_build_assign (name
, new_decl
);
17566 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
17569 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
17574 struct modify_stmt_info info
;
17575 info
.adjustments
= adjustments
;
17577 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
17579 gimple_stmt_iterator gsi
;
17581 gsi
= gsi_start_bb (bb
);
17582 while (!gsi_end_p (gsi
))
17584 gimple
*stmt
= gsi_stmt (gsi
);
17586 struct walk_stmt_info wi
;
17588 memset (&wi
, 0, sizeof (wi
));
17589 info
.modified
= false;
17591 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
17593 if (greturn
*return_stmt
= dyn_cast
<greturn
*> (stmt
))
17595 tree retval
= gimple_return_retval (return_stmt
);
17598 gsi_remove (&gsi
, true);
17602 /* Replace `return foo' with `retval_array[iter] = foo'. */
17603 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
17604 retval_array
, iter
, NULL
, NULL
);
17605 stmt
= gimple_build_assign (ref
, retval
);
17606 gsi_replace (&gsi
, stmt
, true);
17607 info
.modified
= true;
17612 update_stmt (stmt
);
17613 if (maybe_clean_eh_stmt (stmt
))
17614 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
17621 /* Helper function of simd_clone_adjust, return linear step addend
17622 of Ith argument. */
17625 simd_clone_linear_addend (struct cgraph_node
*node
, unsigned int i
,
17626 tree addtype
, basic_block entry_bb
)
17628 tree ptype
= NULL_TREE
;
17629 switch (node
->simdclone
->args
[i
].arg_type
)
17631 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
17632 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
17633 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
17634 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
17635 return build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
17636 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
17637 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
17638 ptype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
17640 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
17641 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
17642 ptype
= TREE_TYPE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
));
17645 gcc_unreachable ();
17648 unsigned int idx
= node
->simdclone
->args
[i
].linear_step
;
17649 tree arg
= node
->simdclone
->args
[idx
].orig_arg
;
17650 gcc_assert (is_gimple_reg_type (TREE_TYPE (arg
)));
17651 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
17654 if (is_gimple_reg (arg
))
17655 ret
= get_or_create_ssa_default_def (cfun
, arg
);
17658 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (arg
)), arg
);
17659 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
17660 ret
= gimple_assign_lhs (g
);
17662 if (TREE_CODE (TREE_TYPE (arg
)) == REFERENCE_TYPE
)
17664 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (TREE_TYPE (arg
))),
17665 build_simple_mem_ref (ret
));
17666 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
17667 ret
= gimple_assign_lhs (g
);
17669 if (!useless_type_conversion_p (addtype
, TREE_TYPE (ret
)))
17671 g
= gimple_build_assign (make_ssa_name (addtype
), NOP_EXPR
, ret
);
17672 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
17673 ret
= gimple_assign_lhs (g
);
17675 if (POINTER_TYPE_P (ptype
))
17677 tree size
= TYPE_SIZE_UNIT (TREE_TYPE (ptype
));
17678 if (size
&& TREE_CODE (size
) == INTEGER_CST
)
17680 g
= gimple_build_assign (make_ssa_name (addtype
), MULT_EXPR
,
17681 ret
, fold_convert (addtype
, size
));
17682 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
17683 ret
= gimple_assign_lhs (g
);
17689 /* Adjust the argument types in NODE to their appropriate vector
17693 simd_clone_adjust (struct cgraph_node
*node
)
17695 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
17697 targetm
.simd_clone
.adjust (node
);
17699 tree retval
= simd_clone_adjust_return_type (node
);
17700 ipa_parm_adjustment_vec adjustments
17701 = simd_clone_adjust_argument_types (node
);
17703 push_gimplify_context ();
17705 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
17707 /* Adjust all uses of vector arguments accordingly. Adjust all
17708 return values accordingly. */
17709 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
17710 tree iter1
= make_ssa_name (iter
);
17711 tree iter2
= make_ssa_name (iter
);
17712 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
17714 /* Initialize the iteration variable. */
17715 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
17716 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
17717 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
17718 /* Insert the SIMD array and iv initialization at function
17720 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
17722 pop_gimplify_context (NULL
);
17724 /* Create a new BB right before the original exit BB, to hold the
17725 iteration increment and the condition/branch. */
17726 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
17727 basic_block incr_bb
= create_empty_bb (orig_exit
);
17728 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
17729 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
17730 flag. Set it now to be a FALLTHRU_EDGE. */
17731 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
17732 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
17733 for (unsigned i
= 0;
17734 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
17736 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
17737 redirect_edge_succ (e
, incr_bb
);
17739 edge e
= make_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
17740 e
->probability
= REG_BR_PROB_BASE
;
17741 gsi
= gsi_last_bb (incr_bb
);
17742 gimple
*g
= gimple_build_assign (iter2
, PLUS_EXPR
, iter1
,
17743 build_int_cst (unsigned_type_node
, 1));
17744 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
17746 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
17747 struct loop
*loop
= alloc_loop ();
17748 cfun
->has_force_vectorize_loops
= true;
17749 loop
->safelen
= node
->simdclone
->simdlen
;
17750 loop
->force_vectorize
= true;
17751 loop
->header
= body_bb
;
17753 /* Branch around the body if the mask applies. */
17754 if (node
->simdclone
->inbranch
)
17756 gimple_stmt_iterator gsi
= gsi_last_bb (loop
->header
);
17758 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
17759 tree mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)));
17760 tree aref
= build4 (ARRAY_REF
,
17761 TREE_TYPE (TREE_TYPE (mask_array
)),
17764 g
= gimple_build_assign (mask
, aref
);
17765 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
17766 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
17767 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
17769 aref
= build1 (VIEW_CONVERT_EXPR
,
17770 build_nonstandard_integer_type (bitsize
, 0), mask
);
17771 mask
= make_ssa_name (TREE_TYPE (aref
));
17772 g
= gimple_build_assign (mask
, aref
);
17773 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
17776 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
17778 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
17779 make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
17780 FALLTHRU_EDGE (loop
->header
)->flags
= EDGE_FALSE_VALUE
;
17783 /* Generate the condition. */
17784 g
= gimple_build_cond (LT_EXPR
,
17786 build_int_cst (unsigned_type_node
,
17787 node
->simdclone
->simdlen
),
17789 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
17790 e
= split_block (incr_bb
, gsi_stmt (gsi
));
17791 basic_block latch_bb
= e
->dest
;
17792 basic_block new_exit_bb
;
17793 new_exit_bb
= split_block_after_labels (latch_bb
)->dest
;
17794 loop
->latch
= latch_bb
;
17796 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
17798 make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
17799 /* The successor of incr_bb is already pointing to latch_bb; just
17801 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
17802 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
17804 gphi
*phi
= create_phi_node (iter1
, body_bb
);
17805 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
17806 edge latch_edge
= single_succ_edge (latch_bb
);
17807 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
17809 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
17811 /* Generate the new return. */
17812 gsi
= gsi_last_bb (new_exit_bb
);
17814 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
17815 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
17816 retval
= TREE_OPERAND (retval
, 0);
17819 retval
= build1 (VIEW_CONVERT_EXPR
,
17820 TREE_TYPE (TREE_TYPE (node
->decl
)),
17822 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
17823 false, GSI_CONTINUE_LINKING
);
17825 g
= gimple_build_return (retval
);
17826 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
17828 /* Handle aligned clauses by replacing default defs of the aligned
17829 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
17830 lhs. Handle linear by adding PHIs. */
17831 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
17832 if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
17833 && (TREE_ADDRESSABLE (node
->simdclone
->args
[i
].orig_arg
)
17834 || !is_gimple_reg_type
17835 (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
17837 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
17838 if (is_gimple_reg_type (TREE_TYPE (orig_arg
)))
17839 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
17842 iter1
= create_tmp_var_raw (TREE_TYPE (orig_arg
));
17843 gimple_add_tmp_var (iter1
);
17845 gsi
= gsi_after_labels (entry_bb
);
17846 g
= gimple_build_assign (iter1
, orig_arg
);
17847 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
17848 gsi
= gsi_after_labels (body_bb
);
17849 g
= gimple_build_assign (orig_arg
, iter1
);
17850 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
17852 else if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
17853 && DECL_BY_REFERENCE (node
->simdclone
->args
[i
].orig_arg
)
17854 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
17856 && TREE_ADDRESSABLE
17857 (TREE_TYPE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
17859 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
17860 tree def
= ssa_default_def (cfun
, orig_arg
);
17861 if (def
&& !has_zero_uses (def
))
17863 iter1
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg
)));
17864 gimple_add_tmp_var (iter1
);
17865 gsi
= gsi_after_labels (entry_bb
);
17866 g
= gimple_build_assign (iter1
, build_simple_mem_ref (def
));
17867 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
17868 gsi
= gsi_after_labels (body_bb
);
17869 g
= gimple_build_assign (build_simple_mem_ref (def
), iter1
);
17870 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
17873 else if (node
->simdclone
->args
[i
].alignment
17874 && node
->simdclone
->args
[i
].arg_type
17875 == SIMD_CLONE_ARG_TYPE_UNIFORM
17876 && (node
->simdclone
->args
[i
].alignment
17877 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
17878 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
17881 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
17882 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
17883 tree def
= ssa_default_def (cfun
, orig_arg
);
17884 if (def
&& !has_zero_uses (def
))
17886 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
17887 gimple_seq seq
= NULL
;
17888 bool need_cvt
= false;
17890 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
17892 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
17895 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
);
17896 gimple_call_set_lhs (g
, t
);
17897 gimple_seq_add_stmt_without_update (&seq
, g
);
17900 t
= make_ssa_name (orig_arg
);
17901 g
= gimple_build_assign (t
, NOP_EXPR
, gimple_call_lhs (g
));
17902 gimple_seq_add_stmt_without_update (&seq
, g
);
17904 gsi_insert_seq_on_edge_immediate
17905 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
17907 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
17908 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
17910 node
->create_edge (cgraph_node::get_create (fn
),
17911 call
, entry_bb
->count
, freq
);
17913 imm_use_iterator iter
;
17914 use_operand_p use_p
;
17916 tree repl
= gimple_get_lhs (g
);
17917 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
17918 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
17921 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
17922 SET_USE (use_p
, repl
);
17925 else if ((node
->simdclone
->args
[i
].arg_type
17926 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
17927 || (node
->simdclone
->args
[i
].arg_type
17928 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
)
17929 || (node
->simdclone
->args
[i
].arg_type
17930 == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
17931 || (node
->simdclone
->args
[i
].arg_type
17932 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
))
17934 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
17935 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
17936 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
17937 tree def
= NULL_TREE
;
17938 if (TREE_ADDRESSABLE (orig_arg
))
17940 def
= make_ssa_name (TREE_TYPE (orig_arg
));
17941 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
17942 iter2
= make_ssa_name (TREE_TYPE (orig_arg
));
17943 gsi
= gsi_after_labels (entry_bb
);
17944 g
= gimple_build_assign (def
, orig_arg
);
17945 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
17949 def
= ssa_default_def (cfun
, orig_arg
);
17950 if (!def
|| has_zero_uses (def
))
17954 iter1
= make_ssa_name (orig_arg
);
17955 iter2
= make_ssa_name (orig_arg
);
17960 phi
= create_phi_node (iter1
, body_bb
);
17961 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
17962 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
17963 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
17964 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
17965 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
17966 ? TREE_TYPE (orig_arg
) : sizetype
;
17967 tree addcst
= simd_clone_linear_addend (node
, i
, addtype
,
17969 gsi
= gsi_last_bb (incr_bb
);
17970 g
= gimple_build_assign (iter2
, code
, iter1
, addcst
);
17971 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
17973 imm_use_iterator iter
;
17974 use_operand_p use_p
;
17976 if (TREE_ADDRESSABLE (orig_arg
))
17978 gsi
= gsi_after_labels (body_bb
);
17979 g
= gimple_build_assign (orig_arg
, iter1
);
17980 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
17983 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
17984 if (use_stmt
== phi
)
17987 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
17988 SET_USE (use_p
, iter1
);
17991 else if (node
->simdclone
->args
[i
].arg_type
17992 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
17993 || (node
->simdclone
->args
[i
].arg_type
17994 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
))
17996 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
17997 tree def
= ssa_default_def (cfun
, orig_arg
);
17998 gcc_assert (!TREE_ADDRESSABLE (orig_arg
)
17999 && TREE_CODE (TREE_TYPE (orig_arg
)) == REFERENCE_TYPE
);
18000 if (def
&& !has_zero_uses (def
))
18002 tree rtype
= TREE_TYPE (TREE_TYPE (orig_arg
));
18003 iter1
= make_ssa_name (orig_arg
);
18004 iter2
= make_ssa_name (orig_arg
);
18005 tree iter3
= make_ssa_name (rtype
);
18006 tree iter4
= make_ssa_name (rtype
);
18007 tree iter5
= make_ssa_name (rtype
);
18008 gsi
= gsi_after_labels (entry_bb
);
18010 = gimple_build_assign (iter3
, build_simple_mem_ref (def
));
18011 gsi_insert_before (&gsi
, load
, GSI_NEW_STMT
);
18013 tree array
= node
->simdclone
->args
[i
].simd_array
;
18014 TREE_ADDRESSABLE (array
) = 1;
18015 tree ptr
= build_fold_addr_expr (array
);
18016 phi
= create_phi_node (iter1
, body_bb
);
18017 add_phi_arg (phi
, ptr
, preheader_edge
, UNKNOWN_LOCATION
);
18018 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
18019 g
= gimple_build_assign (iter2
, POINTER_PLUS_EXPR
, iter1
,
18020 TYPE_SIZE_UNIT (TREE_TYPE (iter3
)));
18021 gsi
= gsi_last_bb (incr_bb
);
18022 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
18024 phi
= create_phi_node (iter4
, body_bb
);
18025 add_phi_arg (phi
, iter3
, preheader_edge
, UNKNOWN_LOCATION
);
18026 add_phi_arg (phi
, iter5
, latch_edge
, UNKNOWN_LOCATION
);
18027 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (iter3
))
18028 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
18029 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (iter3
))
18030 ? TREE_TYPE (iter3
) : sizetype
;
18031 tree addcst
= simd_clone_linear_addend (node
, i
, addtype
,
18033 g
= gimple_build_assign (iter5
, code
, iter4
, addcst
);
18034 gsi
= gsi_last_bb (incr_bb
);
18035 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
18037 g
= gimple_build_assign (build_simple_mem_ref (iter1
), iter4
);
18038 gsi
= gsi_after_labels (body_bb
);
18039 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
18041 imm_use_iterator iter
;
18042 use_operand_p use_p
;
18044 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
18045 if (use_stmt
== load
)
18048 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
18049 SET_USE (use_p
, iter1
);
18051 if (!TYPE_READONLY (rtype
))
18053 tree v
= make_ssa_name (rtype
);
18054 tree aref
= build4 (ARRAY_REF
, rtype
, array
,
18055 size_zero_node
, NULL_TREE
,
18057 gsi
= gsi_after_labels (new_exit_bb
);
18058 g
= gimple_build_assign (v
, aref
);
18059 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
18060 g
= gimple_build_assign (build_simple_mem_ref (def
), v
);
18061 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
18066 calculate_dominance_info (CDI_DOMINATORS
);
18067 add_loop (loop
, loop
->header
->loop_father
);
18068 update_ssa (TODO_update_ssa
);
18073 /* If the function in NODE is tagged as an elemental SIMD function,
18074 create the appropriate SIMD clones. */
18077 expand_simd_clones (struct cgraph_node
*node
)
18079 tree attr
= lookup_attribute ("omp declare simd",
18080 DECL_ATTRIBUTES (node
->decl
));
18081 if (attr
== NULL_TREE
18082 || node
->global
.inlined_to
18083 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
18087 #pragma omp declare simd
18089 in C, there we don't know the argument types at all. */
18090 if (!node
->definition
18091 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
18096 /* Start with parsing the "omp declare simd" attribute(s). */
18097 bool inbranch_clause_specified
;
18098 struct cgraph_simd_clone
*clone_info
18099 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
18100 &inbranch_clause_specified
);
18101 if (clone_info
== NULL
)
18104 int orig_simdlen
= clone_info
->simdlen
;
18105 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
18106 /* The target can return 0 (no simd clones should be created),
18107 1 (just one ISA of simd clones should be created) or higher
18108 count of ISA variants. In that case, clone_info is initialized
18109 for the first ISA variant. */
18111 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
18116 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
18117 also create one inbranch and one !inbranch clone of it. */
18118 for (int i
= 0; i
< count
* 2; i
++)
18120 struct cgraph_simd_clone
*clone
= clone_info
;
18121 if (inbranch_clause_specified
&& (i
& 1) != 0)
18126 clone
= simd_clone_struct_alloc (clone_info
->nargs
18128 simd_clone_struct_copy (clone
, clone_info
);
18129 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
18130 and simd_clone_adjust_argument_types did to the first
18132 clone
->nargs
-= clone_info
->inbranch
;
18133 clone
->simdlen
= orig_simdlen
;
18134 /* And call the target hook again to get the right ISA. */
18135 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
18139 clone
->inbranch
= 1;
18142 /* simd_clone_mangle might fail if such a clone has been created
18144 tree id
= simd_clone_mangle (node
, clone
);
18145 if (id
== NULL_TREE
)
18148 /* Only when we are sure we want to create the clone actually
18149 clone the function (or definitions) or create another
18150 extern FUNCTION_DECL (for prototypes without definitions). */
18151 struct cgraph_node
*n
= simd_clone_create (node
);
18155 n
->simdclone
= clone
;
18156 clone
->origin
= node
;
18157 clone
->next_clone
= NULL
;
18158 if (node
->simd_clones
== NULL
)
18160 clone
->prev_clone
= n
;
18161 node
->simd_clones
= n
;
18165 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
18166 clone
->prev_clone
->simdclone
->next_clone
= n
;
18167 node
->simd_clones
->simdclone
->prev_clone
= n
;
18169 symtab
->change_decl_assembler_name (n
->decl
, id
);
18170 /* And finally adjust the return type, parameters and for
18171 definitions also function body. */
18172 if (node
->definition
)
18173 simd_clone_adjust (n
);
18176 simd_clone_adjust_return_type (n
);
18177 simd_clone_adjust_argument_types (n
);
18181 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
18184 /* Entry point for IPA simd clone creation pass. */
18186 static unsigned int
18187 ipa_omp_simd_clone (void)
18189 struct cgraph_node
*node
;
18190 FOR_EACH_FUNCTION (node
)
18191 expand_simd_clones (node
);
18197 const pass_data pass_data_omp_simd_clone
=
18199 SIMPLE_IPA_PASS
, /* type */
18200 "simdclone", /* name */
18201 OPTGROUP_NONE
, /* optinfo_flags */
18202 TV_NONE
, /* tv_id */
18203 ( PROP_ssa
| PROP_cfg
), /* properties_required */
18204 0, /* properties_provided */
18205 0, /* properties_destroyed */
18206 0, /* todo_flags_start */
18207 0, /* todo_flags_finish */
18210 class pass_omp_simd_clone
: public simple_ipa_opt_pass
18213 pass_omp_simd_clone(gcc::context
*ctxt
)
18214 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
18217 /* opt_pass methods: */
18218 virtual bool gate (function
*);
18219 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
18223 pass_omp_simd_clone::gate (function
*)
18225 return ((flag_openmp
|| flag_openmp_simd
18227 || (in_lto_p
&& !flag_wpa
))
18228 && (targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
));
18231 } // anon namespace
18233 simple_ipa_opt_pass
*
18234 make_pass_omp_simd_clone (gcc::context
*ctxt
)
18236 return new pass_omp_simd_clone (ctxt
);
18239 /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and
18240 adds their addresses and sizes to constructor-vector V_CTOR. */
18242 add_decls_addresses_to_decl_constructor (vec
<tree
, va_gc
> *v_decls
,
18243 vec
<constructor_elt
, va_gc
> *v_ctor
)
18245 unsigned len
= vec_safe_length (v_decls
);
18246 for (unsigned i
= 0; i
< len
; i
++)
18248 tree it
= (*v_decls
)[i
];
18249 bool is_function
= TREE_CODE (it
) != VAR_DECL
;
18251 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, build_fold_addr_expr (it
));
18253 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
,
18254 fold_convert (const_ptr_type_node
,
18255 DECL_SIZE_UNIT (it
)));
18259 /* Create new symbols containing (address, size) pairs for global variables,
18260 marked with "omp declare target" attribute, as well as addresses for the
18261 functions, which are outlined offloading regions. */
18263 omp_finish_file (void)
18265 unsigned num_funcs
= vec_safe_length (offload_funcs
);
18266 unsigned num_vars
= vec_safe_length (offload_vars
);
18268 if (num_funcs
== 0 && num_vars
== 0)
18271 if (targetm_common
.have_named_sections
)
18273 vec
<constructor_elt
, va_gc
> *v_f
, *v_v
;
18274 vec_alloc (v_f
, num_funcs
);
18275 vec_alloc (v_v
, num_vars
* 2);
18277 add_decls_addresses_to_decl_constructor (offload_funcs
, v_f
);
18278 add_decls_addresses_to_decl_constructor (offload_vars
, v_v
);
18280 tree vars_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
18282 tree funcs_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
18284 TYPE_ALIGN (vars_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
18285 TYPE_ALIGN (funcs_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
18286 tree ctor_v
= build_constructor (vars_decl_type
, v_v
);
18287 tree ctor_f
= build_constructor (funcs_decl_type
, v_f
);
18288 TREE_CONSTANT (ctor_v
) = TREE_CONSTANT (ctor_f
) = 1;
18289 TREE_STATIC (ctor_v
) = TREE_STATIC (ctor_f
) = 1;
18290 tree funcs_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
18291 get_identifier (".offload_func_table"),
18293 tree vars_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
18294 get_identifier (".offload_var_table"),
18296 TREE_STATIC (funcs_decl
) = TREE_STATIC (vars_decl
) = 1;
18297 /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node),
18298 otherwise a joint table in a binary will contain padding between
18299 tables from multiple object files. */
18300 DECL_USER_ALIGN (funcs_decl
) = DECL_USER_ALIGN (vars_decl
) = 1;
18301 DECL_ALIGN (funcs_decl
) = TYPE_ALIGN (funcs_decl_type
);
18302 DECL_ALIGN (vars_decl
) = TYPE_ALIGN (vars_decl_type
);
18303 DECL_INITIAL (funcs_decl
) = ctor_f
;
18304 DECL_INITIAL (vars_decl
) = ctor_v
;
18305 set_decl_section_name (funcs_decl
, OFFLOAD_FUNC_TABLE_SECTION_NAME
);
18306 set_decl_section_name (vars_decl
, OFFLOAD_VAR_TABLE_SECTION_NAME
);
18308 varpool_node::finalize_decl (vars_decl
);
18309 varpool_node::finalize_decl (funcs_decl
);
18313 for (unsigned i
= 0; i
< num_funcs
; i
++)
18315 tree it
= (*offload_funcs
)[i
];
18316 targetm
.record_offload_symbol (it
);
18318 for (unsigned i
= 0; i
< num_vars
; i
++)
18320 tree it
= (*offload_vars
)[i
];
18321 targetm
.record_offload_symbol (it
);
18326 /* Find the number of threads (POS = false), or thread number (POS =
18327 true) for an OpenACC region partitioned as MASK. Setup code
18328 required for the calculation is added to SEQ. */
18331 oacc_thread_numbers (bool pos
, int mask
, gimple_seq
*seq
)
18333 tree res
= pos
? NULL_TREE
: build_int_cst (unsigned_type_node
, 1);
18336 /* Start at gang level, and examine relevant dimension indices. */
18337 for (ix
= GOMP_DIM_GANG
; ix
!= GOMP_DIM_MAX
; ix
++)
18338 if (GOMP_DIM_MASK (ix
) & mask
)
18340 tree arg
= build_int_cst (unsigned_type_node
, ix
);
18344 /* We had an outer index, so scale that by the size of
18346 tree n
= create_tmp_var (integer_type_node
);
18348 = gimple_build_call_internal (IFN_GOACC_DIM_SIZE
, 1, arg
);
18350 gimple_call_set_lhs (call
, n
);
18351 gimple_seq_add_stmt (seq
, call
);
18352 res
= fold_build2 (MULT_EXPR
, integer_type_node
, res
, n
);
18356 /* Determine index in this dimension. */
18357 tree id
= create_tmp_var (integer_type_node
);
18358 gimple
*call
= gimple_build_call_internal
18359 (IFN_GOACC_DIM_POS
, 1, arg
);
18361 gimple_call_set_lhs (call
, id
);
18362 gimple_seq_add_stmt (seq
, call
);
18364 res
= fold_build2 (PLUS_EXPR
, integer_type_node
, res
, id
);
18370 if (res
== NULL_TREE
)
18371 res
= integer_zero_node
;
18376 /* Transform IFN_GOACC_LOOP calls to actual code. See
18377 expand_oacc_for for where these are generated. At the vector
18378 level, we stride loops, such that each member of a warp will
18379 operate on adjacent iterations. At the worker and gang level,
18380 each gang/warp executes a set of contiguous iterations. Chunking
18381 can override this such that each iteration engine executes a
18382 contiguous chunk, and then moves on to stride to the next chunk. */
18385 oacc_xform_loop (gcall
*call
)
18387 gimple_stmt_iterator gsi
= gsi_for_stmt (call
);
18388 enum ifn_goacc_loop_kind code
18389 = (enum ifn_goacc_loop_kind
) TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
18390 tree dir
= gimple_call_arg (call
, 1);
18391 tree range
= gimple_call_arg (call
, 2);
18392 tree step
= gimple_call_arg (call
, 3);
18393 tree chunk_size
= NULL_TREE
;
18394 unsigned mask
= (unsigned) TREE_INT_CST_LOW (gimple_call_arg (call
, 5));
18395 tree lhs
= gimple_call_lhs (call
);
18396 tree type
= TREE_TYPE (lhs
);
18397 tree diff_type
= TREE_TYPE (range
);
18398 tree r
= NULL_TREE
;
18399 gimple_seq seq
= NULL
;
18400 bool chunking
= false, striding
= true;
18401 unsigned outer_mask
= mask
& (~mask
+ 1); // Outermost partitioning
18402 unsigned inner_mask
= mask
& ~outer_mask
; // Inner partitioning (if any)
18404 #ifdef ACCEL_COMPILER
18405 chunk_size
= gimple_call_arg (call
, 4);
18406 if (integer_minus_onep (chunk_size
) /* Force static allocation. */
18407 || integer_zerop (chunk_size
)) /* Default (also static). */
18409 /* If we're at the gang level, we want each to execute a
18410 contiguous run of iterations. Otherwise we want each element
18412 striding
= !(outer_mask
& GOMP_DIM_MASK (GOMP_DIM_GANG
));
18417 /* Chunk of size 1 is striding. */
18418 striding
= integer_onep (chunk_size
);
18419 chunking
= !striding
;
18423 /* striding=true, chunking=true
18425 striding=true, chunking=false
18427 striding=false,chunking=true
18428 -> chunks=ceil (range/(chunksize*threads*step))
18429 striding=false,chunking=false
18430 -> chunk_size=ceil(range/(threads*step)),chunks=1 */
18431 push_gimplify_context (true);
18435 default: gcc_unreachable ();
18437 case IFN_GOACC_LOOP_CHUNKS
:
18439 r
= build_int_cst (type
, 1);
18443 = (range - dir) / (chunks * step * num_threads) + dir */
18444 tree per
= oacc_thread_numbers (false, mask
, &seq
);
18445 per
= fold_convert (type
, per
);
18446 chunk_size
= fold_convert (type
, chunk_size
);
18447 per
= fold_build2 (MULT_EXPR
, type
, per
, chunk_size
);
18448 per
= fold_build2 (MULT_EXPR
, type
, per
, step
);
18449 r
= build2 (MINUS_EXPR
, type
, range
, dir
);
18450 r
= build2 (PLUS_EXPR
, type
, r
, per
);
18451 r
= build2 (TRUNC_DIV_EXPR
, type
, r
, per
);
18455 case IFN_GOACC_LOOP_STEP
:
18457 /* If striding, step by the entire compute volume, otherwise
18458 step by the inner volume. */
18459 unsigned volume
= striding
? mask
: inner_mask
;
18461 r
= oacc_thread_numbers (false, volume
, &seq
);
18462 r
= build2 (MULT_EXPR
, type
, fold_convert (type
, r
), step
);
18466 case IFN_GOACC_LOOP_OFFSET
:
18469 r
= oacc_thread_numbers (true, mask
, &seq
);
18470 r
= fold_convert (diff_type
, r
);
18474 tree inner_size
= oacc_thread_numbers (false, inner_mask
, &seq
);
18475 tree outer_size
= oacc_thread_numbers (false, outer_mask
, &seq
);
18476 tree volume
= fold_build2 (MULT_EXPR
, TREE_TYPE (inner_size
),
18477 inner_size
, outer_size
);
18479 volume
= fold_convert (diff_type
, volume
);
18481 chunk_size
= fold_convert (diff_type
, chunk_size
);
18484 tree per
= fold_build2 (MULT_EXPR
, diff_type
, volume
, step
);
18486 chunk_size
= build2 (MINUS_EXPR
, diff_type
, range
, dir
);
18487 chunk_size
= build2 (PLUS_EXPR
, diff_type
, chunk_size
, per
);
18488 chunk_size
= build2 (TRUNC_DIV_EXPR
, diff_type
, chunk_size
, per
);
18491 tree span
= build2 (MULT_EXPR
, diff_type
, chunk_size
,
18492 fold_convert (diff_type
, inner_size
));
18493 r
= oacc_thread_numbers (true, outer_mask
, &seq
);
18494 r
= fold_convert (diff_type
, r
);
18495 r
= build2 (MULT_EXPR
, diff_type
, r
, span
);
18497 tree inner
= oacc_thread_numbers (true, inner_mask
, &seq
);
18498 inner
= fold_convert (diff_type
, inner
);
18499 r
= fold_build2 (PLUS_EXPR
, diff_type
, r
, inner
);
18503 tree chunk
= fold_convert (diff_type
, gimple_call_arg (call
, 6));
18505 = fold_build2 (MULT_EXPR
, diff_type
, volume
, chunk_size
);
18506 per
= build2 (MULT_EXPR
, diff_type
, per
, chunk
);
18508 r
= build2 (PLUS_EXPR
, diff_type
, r
, per
);
18511 r
= fold_build2 (MULT_EXPR
, diff_type
, r
, step
);
18512 if (type
!= diff_type
)
18513 r
= fold_convert (type
, r
);
18516 case IFN_GOACC_LOOP_BOUND
:
18521 tree inner_size
= oacc_thread_numbers (false, inner_mask
, &seq
);
18522 tree outer_size
= oacc_thread_numbers (false, outer_mask
, &seq
);
18523 tree volume
= fold_build2 (MULT_EXPR
, TREE_TYPE (inner_size
),
18524 inner_size
, outer_size
);
18526 volume
= fold_convert (diff_type
, volume
);
18528 chunk_size
= fold_convert (diff_type
, chunk_size
);
18531 tree per
= fold_build2 (MULT_EXPR
, diff_type
, volume
, step
);
18533 chunk_size
= build2 (MINUS_EXPR
, diff_type
, range
, dir
);
18534 chunk_size
= build2 (PLUS_EXPR
, diff_type
, chunk_size
, per
);
18535 chunk_size
= build2 (TRUNC_DIV_EXPR
, diff_type
, chunk_size
, per
);
18538 tree span
= build2 (MULT_EXPR
, diff_type
, chunk_size
,
18539 fold_convert (diff_type
, inner_size
));
18541 r
= fold_build2 (MULT_EXPR
, diff_type
, span
, step
);
18543 tree offset
= gimple_call_arg (call
, 6);
18544 r
= build2 (PLUS_EXPR
, diff_type
, r
,
18545 fold_convert (diff_type
, offset
));
18546 r
= build2 (integer_onep (dir
) ? MIN_EXPR
: MAX_EXPR
,
18547 diff_type
, r
, range
);
18549 if (diff_type
!= type
)
18550 r
= fold_convert (type
, r
);
18554 gimplify_assign (lhs
, r
, &seq
);
18556 pop_gimplify_context (NULL
);
18558 gsi_replace_with_seq (&gsi
, seq
, true);
18561 /* Validate and update the dimensions for offloaded FN. ATTRS is the
18562 raw attribute. DIMS is an array of dimensions, which is returned.
18563 Returns the function level dimensionality -- the level at which an
18564 offload routine wishes to partition a loop. */
18567 oacc_validate_dims (tree fn
, tree attrs
, int *dims
)
18569 tree purpose
[GOMP_DIM_MAX
];
18571 tree pos
= TREE_VALUE (attrs
);
18574 /* Make sure the attribute creator attached the dimension
18578 for (ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
18580 purpose
[ix
] = TREE_PURPOSE (pos
);
18584 if (integer_zerop (purpose
[ix
]))
18586 else if (fn_level
< 0)
18590 tree val
= TREE_VALUE (pos
);
18591 dims
[ix
] = val
? TREE_INT_CST_LOW (val
) : -1;
18592 pos
= TREE_CHAIN (pos
);
18595 bool changed
= targetm
.goacc
.validate_dims (fn
, dims
, fn_level
);
18597 /* Default anything left to 1. */
18598 for (ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
18607 /* Replace the attribute with new values. */
18609 for (ix
= GOMP_DIM_MAX
; ix
--;)
18610 pos
= tree_cons (purpose
[ix
],
18611 build_int_cst (integer_type_node
, dims
[ix
]),
18613 replace_oacc_fn_attrib (fn
, pos
);
18619 /* Create an empty OpenACC loop structure at LOC. */
18622 new_oacc_loop_raw (oacc_loop
*parent
, location_t loc
)
18624 oacc_loop
*loop
= XCNEW (oacc_loop
);
18626 loop
->parent
= parent
;
18627 loop
->child
= loop
->sibling
= NULL
;
18631 loop
->sibling
= parent
->child
;
18632 parent
->child
= loop
;
18636 loop
->marker
= NULL
;
18637 memset (loop
->heads
, 0, sizeof (loop
->heads
));
18638 memset (loop
->tails
, 0, sizeof (loop
->tails
));
18639 loop
->routine
= NULL_TREE
;
18641 loop
->mask
= loop
->flags
= 0;
18642 loop
->chunk_size
= 0;
18643 loop
->head_end
= NULL
;
18648 /* Create an outermost, dummy OpenACC loop for offloaded function
18652 new_oacc_loop_outer (tree decl
)
18654 return new_oacc_loop_raw (NULL
, DECL_SOURCE_LOCATION (decl
));
18657 /* Start a new OpenACC loop structure beginning at head marker HEAD.
18658 Link into PARENT loop. Return the new loop. */
18661 new_oacc_loop (oacc_loop
*parent
, gcall
*marker
)
18663 oacc_loop
*loop
= new_oacc_loop_raw (parent
, gimple_location (marker
));
18665 loop
->marker
= marker
;
18667 /* TODO: This is where device_type flattening would occur for the loop
18670 loop
->flags
= TREE_INT_CST_LOW (gimple_call_arg (marker
, 3));
18672 tree chunk_size
= integer_zero_node
;
18673 if (loop
->flags
& OLF_GANG_STATIC
)
18674 chunk_size
= gimple_call_arg (marker
, 4);
18675 loop
->chunk_size
= chunk_size
;
18680 /* Create a dummy loop encompassing a call to a openACC routine.
18681 Extract the routine's partitioning requirements. */
18684 new_oacc_loop_routine (oacc_loop
*parent
, gcall
*call
, tree decl
, tree attrs
)
18686 oacc_loop
*loop
= new_oacc_loop_raw (parent
, gimple_location (call
));
18687 int dims
[GOMP_DIM_MAX
];
18688 int level
= oacc_validate_dims (decl
, attrs
, dims
);
18690 gcc_assert (level
>= 0);
18692 loop
->marker
= call
;
18693 loop
->routine
= decl
;
18694 loop
->mask
= ((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1)
18695 ^ (GOMP_DIM_MASK (level
) - 1));
18698 /* Finish off the current OpenACC loop ending at tail marker TAIL.
18699 Return the parent loop. */
18702 finish_oacc_loop (oacc_loop
*loop
)
18704 return loop
->parent
;
18707 /* Free all OpenACC loop structures within LOOP (inclusive). */
18710 free_oacc_loop (oacc_loop
*loop
)
18713 free_oacc_loop (loop
->sibling
);
18715 free_oacc_loop (loop
->child
);
18720 /* Dump out the OpenACC loop head or tail beginning at FROM. */
18723 dump_oacc_loop_part (FILE *file
, gcall
*from
, int depth
,
18724 const char *title
, int level
)
18726 enum ifn_unique_kind kind
18727 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (from
, 0));
18729 fprintf (file
, "%*s%s-%d:\n", depth
* 2, "", title
, level
);
18730 for (gimple_stmt_iterator gsi
= gsi_for_stmt (from
);;)
18732 gimple
*stmt
= gsi_stmt (gsi
);
18734 if (is_gimple_call (stmt
)
18735 && gimple_call_internal_p (stmt
)
18736 && gimple_call_internal_fn (stmt
) == IFN_UNIQUE
)
18738 enum ifn_unique_kind k
18739 = ((enum ifn_unique_kind
) TREE_INT_CST_LOW
18740 (gimple_call_arg (stmt
, 0)));
18742 if (k
== kind
&& stmt
!= from
)
18745 print_gimple_stmt (file
, stmt
, depth
* 2 + 2, 0);
18748 while (gsi_end_p (gsi
))
18749 gsi
= gsi_start_bb (single_succ (gsi_bb (gsi
)));
18753 /* Dump OpenACC loops LOOP, its siblings and its children. */
18756 dump_oacc_loop (FILE *file
, oacc_loop
*loop
, int depth
)
18760 fprintf (file
, "%*sLoop %x(%x) %s:%u\n", depth
* 2, "",
18761 loop
->flags
, loop
->mask
,
18762 LOCATION_FILE (loop
->loc
), LOCATION_LINE (loop
->loc
));
18765 print_gimple_stmt (file
, loop
->marker
, depth
* 2, 0);
18768 fprintf (file
, "%*sRoutine %s:%u:%s\n",
18769 depth
* 2, "", DECL_SOURCE_FILE (loop
->routine
),
18770 DECL_SOURCE_LINE (loop
->routine
),
18771 IDENTIFIER_POINTER (DECL_NAME (loop
->routine
)));
18773 for (ix
= GOMP_DIM_GANG
; ix
!= GOMP_DIM_MAX
; ix
++)
18774 if (loop
->heads
[ix
])
18775 dump_oacc_loop_part (file
, loop
->heads
[ix
], depth
, "Head", ix
);
18776 for (ix
= GOMP_DIM_MAX
; ix
--;)
18777 if (loop
->tails
[ix
])
18778 dump_oacc_loop_part (file
, loop
->tails
[ix
], depth
, "Tail", ix
);
18781 dump_oacc_loop (file
, loop
->child
, depth
+ 1);
18783 dump_oacc_loop (file
, loop
->sibling
, depth
);
18786 void debug_oacc_loop (oacc_loop
*);
18788 /* Dump loops to stderr. */
18790 DEBUG_FUNCTION
void
18791 debug_oacc_loop (oacc_loop
*loop
)
18793 dump_oacc_loop (stderr
, loop
, 0);
18796 /* DFS walk of basic blocks BB onwards, creating OpenACC loop
18797 structures as we go. By construction these loops are properly
18801 oacc_loop_discover_walk (oacc_loop
*loop
, basic_block bb
)
18806 if (bb
->flags
& BB_VISITED
)
18810 bb
->flags
|= BB_VISITED
;
18812 /* Scan for loop markers. */
18813 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
);
18816 gimple
*stmt
= gsi_stmt (gsi
);
18818 if (!is_gimple_call (stmt
))
18821 gcall
*call
= as_a
<gcall
*> (stmt
);
18823 /* If this is a routine, make a dummy loop for it. */
18824 if (tree decl
= gimple_call_fndecl (call
))
18825 if (tree attrs
= get_oacc_fn_attrib (decl
))
18827 gcc_assert (!marker
);
18828 new_oacc_loop_routine (loop
, call
, decl
, attrs
);
18831 if (!gimple_call_internal_p (call
))
18834 if (gimple_call_internal_fn (call
) != IFN_UNIQUE
)
18837 enum ifn_unique_kind kind
18838 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
18839 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
18840 || kind
== IFN_UNIQUE_OACC_TAIL_MARK
)
18842 if (gimple_call_num_args (call
) == 2)
18844 gcc_assert (marker
&& !remaining
);
18846 if (kind
== IFN_UNIQUE_OACC_TAIL_MARK
)
18847 loop
= finish_oacc_loop (loop
);
18849 loop
->head_end
= call
;
18853 int count
= TREE_INT_CST_LOW (gimple_call_arg (call
, 2));
18857 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
)
18858 loop
= new_oacc_loop (loop
, call
);
18861 gcc_assert (count
== remaining
);
18865 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
)
18866 loop
->heads
[marker
] = call
;
18868 loop
->tails
[remaining
] = call
;
18874 if (remaining
|| marker
)
18876 bb
= single_succ (bb
);
18877 gcc_assert (single_pred_p (bb
) && !(bb
->flags
& BB_VISITED
));
18881 /* Walk successor blocks. */
18885 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
18886 oacc_loop_discover_walk (loop
, e
->dest
);
18889 /* LOOP is the first sibling. Reverse the order in place and return
18890 the new first sibling. Recurse to child loops. */
18893 oacc_loop_sibling_nreverse (oacc_loop
*loop
)
18895 oacc_loop
*last
= NULL
;
18899 loop
->child
= oacc_loop_sibling_nreverse (loop
->child
);
18901 oacc_loop
*next
= loop
->sibling
;
18902 loop
->sibling
= last
;
18911 /* Discover the OpenACC loops marked up by HEAD and TAIL markers for
18912 the current function. */
18915 oacc_loop_discovery ()
18919 oacc_loop
*top
= new_oacc_loop_outer (current_function_decl
);
18920 oacc_loop_discover_walk (top
, ENTRY_BLOCK_PTR_FOR_FN (cfun
));
18922 /* The siblings were constructed in reverse order, reverse them so
18923 that diagnostics come out in an unsurprising order. */
18924 top
= oacc_loop_sibling_nreverse (top
);
18926 /* Reset the visited flags. */
18927 FOR_ALL_BB_FN (bb
, cfun
)
18928 bb
->flags
&= ~BB_VISITED
;
18933 /* Transform the abstract internal function markers starting at FROM
18934 to be for partitioning level LEVEL. Stop when we meet another HEAD
18938 oacc_loop_xform_head_tail (gcall
*from
, int level
)
18940 enum ifn_unique_kind kind
18941 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (from
, 0));
18942 tree replacement
= build_int_cst (unsigned_type_node
, level
);
18944 for (gimple_stmt_iterator gsi
= gsi_for_stmt (from
);;)
18946 gimple
*stmt
= gsi_stmt (gsi
);
18948 if (is_gimple_call (stmt
)
18949 && gimple_call_internal_p (stmt
)
18950 && gimple_call_internal_fn (stmt
) == IFN_UNIQUE
)
18952 enum ifn_unique_kind k
18953 = ((enum ifn_unique_kind
)
18954 TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0)));
18956 if (k
== IFN_UNIQUE_OACC_FORK
|| k
== IFN_UNIQUE_OACC_JOIN
)
18957 *gimple_call_arg_ptr (stmt
, 2) = replacement
;
18958 else if (k
== kind
&& stmt
!= from
)
18961 else if (is_gimple_call (stmt
)
18962 && gimple_call_internal_p (stmt
)
18963 && gimple_call_internal_fn (stmt
) == IFN_GOACC_REDUCTION
)
18964 *gimple_call_arg_ptr (stmt
, 3) = replacement
;
18967 while (gsi_end_p (gsi
))
18968 gsi
= gsi_start_bb (single_succ (gsi_bb (gsi
)));
18972 /* Transform the IFN_GOACC_LOOP internal functions by providing the
18973 determined partitioning mask and chunking argument. */
18976 oacc_loop_xform_loop (gcall
*end_marker
, tree mask_arg
, tree chunk_arg
)
18978 gimple_stmt_iterator gsi
= gsi_for_stmt (end_marker
);
18982 for (; !gsi_end_p (gsi
); gsi_next (&gsi
))
18984 gimple
*stmt
= gsi_stmt (gsi
);
18986 if (!is_gimple_call (stmt
))
18989 gcall
*call
= as_a
<gcall
*> (stmt
);
18991 if (!gimple_call_internal_p (call
))
18994 if (gimple_call_internal_fn (call
) != IFN_GOACC_LOOP
)
18997 *gimple_call_arg_ptr (call
, 5) = mask_arg
;
18998 *gimple_call_arg_ptr (call
, 4) = chunk_arg
;
18999 if (TREE_INT_CST_LOW (gimple_call_arg (call
, 0))
19000 == IFN_GOACC_LOOP_BOUND
)
19004 /* If we didn't see LOOP_BOUND, it should be in the single
19005 successor block. */
19006 basic_block bb
= single_succ (gsi_bb (gsi
));
19007 gsi
= gsi_start_bb (bb
);
19011 /* Process the discovered OpenACC loops, setting the correct
19012 partitioning level etc. */
19015 oacc_loop_process (oacc_loop
*loop
)
19018 oacc_loop_process (loop
->child
);
19020 if (loop
->mask
&& !loop
->routine
)
19023 unsigned mask
= loop
->mask
;
19024 unsigned dim
= GOMP_DIM_GANG
;
19025 tree mask_arg
= build_int_cst (unsigned_type_node
, mask
);
19026 tree chunk_arg
= loop
->chunk_size
;
19028 oacc_loop_xform_loop (loop
->head_end
, mask_arg
, chunk_arg
);
19030 for (ix
= 0; ix
!= GOMP_DIM_MAX
&& loop
->heads
[ix
]; ix
++)
19034 while (!(GOMP_DIM_MASK (dim
) & mask
))
19037 oacc_loop_xform_head_tail (loop
->heads
[ix
], dim
);
19038 oacc_loop_xform_head_tail (loop
->tails
[ix
], dim
);
19040 mask
^= GOMP_DIM_MASK (dim
);
19045 oacc_loop_process (loop
->sibling
);
19048 /* Walk the OpenACC loop heirarchy checking and assigning the
19049 programmer-specified partitionings. OUTER_MASK is the partitioning
19050 this loop is contained within. Return partitiong mask used within
19054 oacc_loop_fixed_partitions (oacc_loop
*loop
, unsigned outer_mask
)
19056 unsigned this_mask
= loop
->mask
;
19057 bool has_auto
= false;
19060 #ifdef ACCEL_COMPILER
19061 /* When device_type is supported, we want the device compiler to be
19062 noisy, if the loop parameters are device_type-specific. */
19066 if (!loop
->routine
)
19068 bool auto_par
= (loop
->flags
& OLF_AUTO
) != 0;
19069 bool seq_par
= (loop
->flags
& OLF_SEQ
) != 0;
19071 this_mask
= ((loop
->flags
>> OLF_DIM_BASE
)
19072 & (GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1));
19074 if ((this_mask
!= 0) + auto_par
+ seq_par
> 1)
19077 error_at (loop
->loc
,
19079 ? "%<seq%> overrides other OpenACC loop specifiers"
19080 : "%<auto%> conflicts with other OpenACC loop specifiers");
19082 loop
->flags
&= ~OLF_AUTO
;
19086 ~((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1) << OLF_DIM_BASE
);
19090 if (auto_par
&& (loop
->flags
& OLF_INDEPENDENT
))
19094 if (this_mask
& outer_mask
)
19096 const oacc_loop
*outer
;
19097 for (outer
= loop
->parent
; outer
; outer
= outer
->parent
)
19098 if (outer
->mask
& this_mask
)
19105 error_at (loop
->loc
,
19106 "%s uses same OpenACC parallelism as containing loop",
19107 loop
->routine
? "routine call" : "inner loop");
19108 inform (outer
->loc
, "containing loop here");
19111 error_at (loop
->loc
,
19112 "%s uses OpenACC parallelism disallowed by containing routine",
19113 loop
->routine
? "routine call" : "loop");
19116 inform (DECL_SOURCE_LOCATION (loop
->routine
),
19117 "routine %qD declared here", loop
->routine
);
19119 this_mask
&= ~outer_mask
;
19123 unsigned outermost
= this_mask
& -this_mask
;
19125 if (outermost
&& outermost
<= outer_mask
)
19129 error_at (loop
->loc
,
19130 "incorrectly nested OpenACC loop parallelism");
19132 const oacc_loop
*outer
;
19133 for (outer
= loop
->parent
;
19134 outer
->flags
&& outer
->flags
< outermost
;
19135 outer
= outer
->parent
)
19137 inform (outer
->loc
, "containing loop here");
19140 this_mask
&= ~outermost
;
19144 loop
->mask
= this_mask
;
19147 && oacc_loop_fixed_partitions (loop
->child
, outer_mask
| this_mask
))
19151 && oacc_loop_fixed_partitions (loop
->sibling
, outer_mask
))
19157 /* Walk the OpenACC loop heirarchy to check and assign partitioning
19161 oacc_loop_partition (oacc_loop
*loop
, int fn_level
)
19163 unsigned outer_mask
= 0;
19166 outer_mask
= GOMP_DIM_MASK (fn_level
) - 1;
19168 oacc_loop_fixed_partitions (loop
, outer_mask
);
19171 /* Default fork/join early expander. Delete the function calls if
19172 there is no RTL expander. */
19175 default_goacc_fork_join (gcall
*ARG_UNUSED (call
),
19176 const int *ARG_UNUSED (dims
), bool is_fork
)
19179 return targetm
.have_oacc_fork ();
19181 return targetm
.have_oacc_join ();
19184 /* Default goacc.reduction early expander.
19186 LHS-opt = IFN_REDUCTION (KIND, RES_PTR, VAR, LEVEL, OP, OFFSET)
19187 If RES_PTR is not integer-zerop:
19188 SETUP - emit 'LHS = *RES_PTR', LHS = NULL
19189 TEARDOWN - emit '*RES_PTR = VAR'
19191 emit 'LHS = VAR' */
19194 default_goacc_reduction (gcall
*call
)
19196 unsigned code
= (unsigned)TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
19197 gimple_stmt_iterator gsi
= gsi_for_stmt (call
);
19198 tree lhs
= gimple_call_lhs (call
);
19199 tree var
= gimple_call_arg (call
, 2);
19200 gimple_seq seq
= NULL
;
19202 if (code
== IFN_GOACC_REDUCTION_SETUP
19203 || code
== IFN_GOACC_REDUCTION_TEARDOWN
)
19205 /* Setup and Teardown need to copy from/to the receiver object,
19206 if there is one. */
19207 tree ref_to_res
= gimple_call_arg (call
, 1);
19209 if (!integer_zerop (ref_to_res
))
19211 tree dst
= build_simple_mem_ref (ref_to_res
);
19214 if (code
== IFN_GOACC_REDUCTION_SETUP
)
19220 gimple_seq_add_stmt (&seq
, gimple_build_assign (dst
, src
));
19224 /* Copy VAR to LHS, if there is an LHS. */
19226 gimple_seq_add_stmt (&seq
, gimple_build_assign (lhs
, var
));
19228 gsi_replace_with_seq (&gsi
, seq
, true);
19231 /* Main entry point for oacc transformations which run on the device
19232 compiler after LTO, so we know what the target device is at this
19233 point (including the host fallback). */
19235 static unsigned int
19236 execute_oacc_device_lower ()
19238 tree attrs
= get_oacc_fn_attrib (current_function_decl
);
19239 int dims
[GOMP_DIM_MAX
];
19242 /* Not an offloaded function. */
19245 int fn_level
= oacc_validate_dims (current_function_decl
, attrs
, dims
);
19247 /* Discover, partition and process the loops. */
19248 oacc_loop
*loops
= oacc_loop_discovery ();
19249 oacc_loop_partition (loops
, fn_level
);
19250 oacc_loop_process (loops
);
19253 fprintf (dump_file
, "OpenACC loops\n");
19254 dump_oacc_loop (dump_file
, loops
, 0);
19255 fprintf (dump_file
, "\n");
19258 /* Offloaded targets may introduce new basic blocks, which require
19259 dominance information to update SSA. */
19260 calculate_dominance_info (CDI_DOMINATORS
);
19262 /* Now lower internal loop functions to target-specific code
19265 FOR_ALL_BB_FN (bb
, cfun
)
19266 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
);)
19268 gimple
*stmt
= gsi_stmt (gsi
);
19269 if (!is_gimple_call (stmt
))
19275 gcall
*call
= as_a
<gcall
*> (stmt
);
19276 if (!gimple_call_internal_p (call
))
19282 /* Rewind to allow rescan. */
19284 bool rescan
= false, remove
= false;
19285 enum internal_fn ifn_code
= gimple_call_internal_fn (call
);
19291 case IFN_GOACC_LOOP
:
19292 oacc_xform_loop (call
);
19296 case IFN_GOACC_REDUCTION
:
19297 /* Mark the function for SSA renaming. */
19298 mark_virtual_operands_for_renaming (cfun
);
19300 /* If the level is -1, this ended up being an unused
19301 axis. Handle as a default. */
19302 if (integer_minus_onep (gimple_call_arg (call
, 3)))
19303 default_goacc_reduction (call
);
19305 targetm
.goacc
.reduction (call
);
19311 enum ifn_unique_kind kind
19312 = ((enum ifn_unique_kind
)
19313 TREE_INT_CST_LOW (gimple_call_arg (call
, 0)));
19318 gcc_unreachable ();
19320 case IFN_UNIQUE_OACC_FORK
:
19321 case IFN_UNIQUE_OACC_JOIN
:
19322 if (integer_minus_onep (gimple_call_arg (call
, 2)))
19324 else if (!targetm
.goacc
.fork_join
19325 (call
, dims
, kind
== IFN_UNIQUE_OACC_FORK
))
19329 case IFN_UNIQUE_OACC_HEAD_MARK
:
19330 case IFN_UNIQUE_OACC_TAIL_MARK
:
19338 if (gsi_end_p (gsi
))
19339 /* We rewound past the beginning of the BB. */
19340 gsi
= gsi_start_bb (bb
);
19342 /* Undo the rewind. */
19347 if (gimple_vdef (call
))
19348 replace_uses_by (gimple_vdef (call
), gimple_vuse (call
));
19349 if (gimple_call_lhs (call
))
19351 /* Propagate the data dependency var. */
19352 gimple
*ass
= gimple_build_assign (gimple_call_lhs (call
),
19353 gimple_call_arg (call
, 1));
19354 gsi_replace (&gsi
, ass
, false);
19357 gsi_remove (&gsi
, true);
19360 /* If not rescanning, advance over the call. */
19364 free_oacc_loop (loops
);
19369 /* Default launch dimension validator. Force everything to 1. A
19370 backend that wants to provide larger dimensions must override this
19374 default_goacc_validate_dims (tree
ARG_UNUSED (decl
), int *dims
,
19375 int ARG_UNUSED (fn_level
))
19377 bool changed
= false;
19379 for (unsigned ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
19391 /* Default dimension bound is unknown on accelerator and 1 on host. */
19394 default_goacc_dim_limit (int ARG_UNUSED (axis
))
19396 #ifdef ACCEL_COMPILER
19405 const pass_data pass_data_oacc_device_lower
=
19407 GIMPLE_PASS
, /* type */
19408 "oaccdevlow", /* name */
19409 OPTGROUP_NONE
, /* optinfo_flags */
19410 TV_NONE
, /* tv_id */
19411 PROP_cfg
, /* properties_required */
19412 0 /* Possibly PROP_gimple_eomp. */, /* properties_provided */
19413 0, /* properties_destroyed */
19414 0, /* todo_flags_start */
19415 TODO_update_ssa
| TODO_cleanup_cfg
, /* todo_flags_finish */
19418 class pass_oacc_device_lower
: public gimple_opt_pass
19421 pass_oacc_device_lower (gcc::context
*ctxt
)
19422 : gimple_opt_pass (pass_data_oacc_device_lower
, ctxt
)
19425 /* opt_pass methods: */
19426 virtual unsigned int execute (function
*)
19428 bool gate
= flag_openacc
!= 0;
19433 return execute_oacc_device_lower ();
19436 }; // class pass_oacc_transform
19438 } // anon namespace
19441 make_pass_oacc_device_lower (gcc::context
*ctxt
)
19443 return new pass_oacc_device_lower (ctxt
);
19446 #include "gt-omp-low.h"