1 /* Lowering pass for OMP directives. Converts OMP directives into explicit
2 calls to the runtime library (libgomp), data marshalling to implement data
3 sharing and copying clauses, offloading to accelerators, and more.
5 Contributed by Diego Novillo <dnovillo@redhat.com>
7 Copyright (C) 2005-2016 Free Software Foundation, Inc.
9 This file is part of GCC.
11 GCC is free software; you can redistribute it and/or modify it under
12 the terms of the GNU General Public License as published by the Free
13 Software Foundation; either version 3, or (at your option) any later
16 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
17 WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
21 You should have received a copy of the GNU General Public License
22 along with GCC; see the file COPYING3. If not see
23 <http://www.gnu.org/licenses/>. */
27 #include "coretypes.h"
34 #include "alloc-pool.h"
35 #include "tree-pass.h"
41 #include "pretty-print.h"
42 #include "diagnostic-core.h"
44 #include "fold-const.h"
45 #include "stor-layout.h"
47 #include "internal-fn.h"
48 #include "gimple-fold.h"
50 #include "gimple-iterator.h"
51 #include "gimplify-me.h"
52 #include "gimple-walk.h"
53 #include "tree-iterator.h"
54 #include "tree-inline.h"
55 #include "langhooks.h"
57 #include "tree-into-ssa.h"
68 #include "splay-tree.h"
70 #include "common/common-target.h"
72 #include "gimple-low.h"
73 #include "tree-cfgcleanup.h"
74 #include "symbol-summary.h"
76 #include "tree-nested.h"
80 #include "lto-section-names.h"
81 #include "gomp-constants.h"
82 #include "gimple-pretty-print.h"
83 #include "symbol-summary.h"
87 /* Lowering of OMP parallel and workshare constructs proceeds in two
88 phases. The first phase scans the function looking for OMP statements
89 and then for variables that must be replaced to satisfy data sharing
90 clauses. The second phase expands code for the constructs, as well as
91 re-gimplifying things when variables have been replaced with complex
94 Final code generation is done by pass_expand_omp. The flowgraph is
95 scanned for regions which are then moved to a new
96 function, to be invoked by the thread library, or offloaded. */
98 /* OMP region information. Every parallel and workshare
99 directive is enclosed between two markers, the OMP_* directive
100 and a corresponding GIMPLE_OMP_RETURN statement. */
104 /* The enclosing region. */
105 struct omp_region
*outer
;
107 /* First child region. */
108 struct omp_region
*inner
;
110 /* Next peer region. */
111 struct omp_region
*next
;
113 /* Block containing the omp directive as its last stmt. */
116 /* Block containing the GIMPLE_OMP_RETURN as its last stmt. */
119 /* Block containing the GIMPLE_OMP_CONTINUE as its last stmt. */
122 /* If this is a combined parallel+workshare region, this is a list
123 of additional arguments needed by the combined parallel+workshare
125 vec
<tree
, va_gc
> *ws_args
;
127 /* The code for the omp directive of this region. */
128 enum gimple_code type
;
130 /* Schedule kind, only used for GIMPLE_OMP_FOR type regions. */
131 enum omp_clause_schedule_kind sched_kind
;
133 /* Schedule modifiers. */
134 unsigned char sched_modifiers
;
136 /* True if this is a combined parallel+workshare region. */
137 bool is_combined_parallel
;
139 /* The ordered stmt if type is GIMPLE_OMP_ORDERED and it has
141 gomp_ordered
*ord_stmt
;
144 /* Context structure. Used to store information about each parallel
145 directive in the code. */
149 /* This field must be at the beginning, as we do "inheritance": Some
150 callback functions for tree-inline.c (e.g., omp_copy_decl)
151 receive a copy_body_data pointer that is up-casted to an
152 omp_context pointer. */
155 /* The tree of contexts corresponding to the encountered constructs. */
156 struct omp_context
*outer
;
159 /* Map variables to fields in a structure that allows communication
160 between sending and receiving threads. */
161 splay_tree field_map
;
166 /* These are used just by task contexts, if task firstprivate fn is
167 needed. srecord_type is used to communicate from the thread
168 that encountered the task construct to task firstprivate fn,
169 record_type is allocated by GOMP_task, initialized by task firstprivate
170 fn and passed to the task body fn. */
171 splay_tree sfield_map
;
174 /* A chain of variables to add to the top-level block surrounding the
175 construct. In the case of a parallel, this is in the child function. */
178 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
179 barriers should jump to during omplower pass. */
182 /* What to do with variables with implicitly determined sharing
184 enum omp_clause_default_kind default_kind
;
186 /* Nesting depth of this context. Used to beautify error messages re
187 invalid gotos. The outermost ctx is depth 1, with depth 0 being
188 reserved for the main body of the function. */
191 /* True if this parallel directive is nested within another. */
194 /* True if this construct can be cancelled. */
198 /* A structure holding the elements of:
199 for (V = N1; V cond N2; V += STEP) [...] */
201 struct omp_for_data_loop
203 tree v
, n1
, n2
, step
;
204 enum tree_code cond_code
;
207 /* A structure describing the main elements of a parallel loop. */
211 struct omp_for_data_loop loop
;
217 bool have_nowait
, have_ordered
, simd_schedule
;
218 unsigned char sched_modifiers
;
219 enum omp_clause_schedule_kind sched_kind
;
220 struct omp_for_data_loop
*loops
;
223 /* Describe the OpenACC looping structure of a function. The entire
224 function is held in a 'NULL' loop. */
228 oacc_loop
*parent
; /* Containing loop. */
230 oacc_loop
*child
; /* First inner loop. */
232 oacc_loop
*sibling
; /* Next loop within same parent. */
234 location_t loc
; /* Location of the loop start. */
236 gcall
*marker
; /* Initial head marker. */
238 gcall
*heads
[GOMP_DIM_MAX
]; /* Head marker functions. */
239 gcall
*tails
[GOMP_DIM_MAX
]; /* Tail marker functions. */
241 tree routine
; /* Pseudo-loop enclosing a routine. */
243 unsigned mask
; /* Partitioning mask. */
244 unsigned flags
; /* Partitioning flags. */
245 tree chunk_size
; /* Chunk size. */
246 gcall
*head_end
; /* Final marker of head sequence. */
249 /* Flags for an OpenACC loop. */
251 enum oacc_loop_flags
{
252 OLF_SEQ
= 1u << 0, /* Explicitly sequential */
253 OLF_AUTO
= 1u << 1, /* Compiler chooses axes. */
254 OLF_INDEPENDENT
= 1u << 2, /* Iterations are known independent. */
255 OLF_GANG_STATIC
= 1u << 3, /* Gang partitioning is static (has op). */
257 /* Explicitly specified loop axes. */
259 OLF_DIM_GANG
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_GANG
),
260 OLF_DIM_WORKER
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_WORKER
),
261 OLF_DIM_VECTOR
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_VECTOR
),
263 OLF_MAX
= OLF_DIM_BASE
+ GOMP_DIM_MAX
267 static splay_tree all_contexts
;
268 static int taskreg_nesting_level
;
269 static int target_nesting_level
;
270 static struct omp_region
*root_omp_region
;
271 static bitmap task_shared_vars
;
272 static vec
<omp_context
*> taskreg_contexts
;
273 static bool omp_any_child_fn_dumped
;
275 static void scan_omp (gimple_seq
*, omp_context
*);
276 static tree
scan_omp_1_op (tree
*, int *, void *);
277 static gphi
*find_phi_with_arg_on_edge (tree
, edge
);
279 #define WALK_SUBSTMTS \
283 case GIMPLE_EH_FILTER: \
284 case GIMPLE_TRANSACTION: \
285 /* The sub-statements for these should be walked. */ \
286 *handled_ops_p = false; \
289 /* Return true if CTX corresponds to an oacc parallel region. */
292 is_oacc_parallel (omp_context
*ctx
)
294 enum gimple_code outer_type
= gimple_code (ctx
->stmt
);
295 return ((outer_type
== GIMPLE_OMP_TARGET
)
296 && (gimple_omp_target_kind (ctx
->stmt
)
297 == GF_OMP_TARGET_KIND_OACC_PARALLEL
));
300 /* Return true if CTX corresponds to an oacc kernels region. */
303 is_oacc_kernels (omp_context
*ctx
)
305 enum gimple_code outer_type
= gimple_code (ctx
->stmt
);
306 return ((outer_type
== GIMPLE_OMP_TARGET
)
307 && (gimple_omp_target_kind (ctx
->stmt
)
308 == GF_OMP_TARGET_KIND_OACC_KERNELS
));
311 /* If DECL is the artificial dummy VAR_DECL created for non-static
312 data member privatization, return the underlying "this" parameter,
313 otherwise return NULL. */
316 omp_member_access_dummy_var (tree decl
)
319 || !DECL_ARTIFICIAL (decl
)
320 || !DECL_IGNORED_P (decl
)
321 || !DECL_HAS_VALUE_EXPR_P (decl
)
322 || !lang_hooks
.decls
.omp_disregard_value_expr (decl
, false))
325 tree v
= DECL_VALUE_EXPR (decl
);
326 if (TREE_CODE (v
) != COMPONENT_REF
)
330 switch (TREE_CODE (v
))
336 case POINTER_PLUS_EXPR
:
337 v
= TREE_OPERAND (v
, 0);
340 if (DECL_CONTEXT (v
) == current_function_decl
341 && DECL_ARTIFICIAL (v
)
342 && TREE_CODE (TREE_TYPE (v
)) == POINTER_TYPE
)
350 /* Helper for unshare_and_remap, called through walk_tree. */
353 unshare_and_remap_1 (tree
*tp
, int *walk_subtrees
, void *data
)
355 tree
*pair
= (tree
*) data
;
358 *tp
= unshare_expr (pair
[1]);
361 else if (IS_TYPE_OR_DECL_P (*tp
))
366 /* Return unshare_expr (X) with all occurrences of FROM
370 unshare_and_remap (tree x
, tree from
, tree to
)
372 tree pair
[2] = { from
, to
};
373 x
= unshare_expr (x
);
374 walk_tree (&x
, unshare_and_remap_1
, pair
, NULL
);
378 /* Holds offload tables with decls. */
379 vec
<tree
, va_gc
> *offload_funcs
, *offload_vars
;
381 /* Convenience function for calling scan_omp_1_op on tree operands. */
384 scan_omp_op (tree
*tp
, omp_context
*ctx
)
386 struct walk_stmt_info wi
;
388 memset (&wi
, 0, sizeof (wi
));
390 wi
.want_locations
= true;
392 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
395 static void lower_omp (gimple_seq
*, omp_context
*);
396 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
397 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
399 /* Find an OMP clause of type KIND within CLAUSES. */
402 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
404 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
405 if (OMP_CLAUSE_CODE (clauses
) == kind
)
411 /* Return true if CTX is for an omp parallel. */
414 is_parallel_ctx (omp_context
*ctx
)
416 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
420 /* Return true if CTX is for an omp task. */
423 is_task_ctx (omp_context
*ctx
)
425 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
429 /* Return true if CTX is for an omp taskloop. */
432 is_taskloop_ctx (omp_context
*ctx
)
434 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
435 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_TASKLOOP
;
439 /* Return true if CTX is for an omp parallel or omp task. */
442 is_taskreg_ctx (omp_context
*ctx
)
444 return is_parallel_ctx (ctx
) || is_task_ctx (ctx
);
448 /* Return true if REGION is a combined parallel+workshare region. */
451 is_combined_parallel (struct omp_region
*region
)
453 return region
->is_combined_parallel
;
456 /* Adjust *COND_CODE and *N2 so that the former is either LT_EXPR or
460 adjust_for_condition (location_t loc
, enum tree_code
*cond_code
, tree
*n2
)
469 if (POINTER_TYPE_P (TREE_TYPE (*n2
)))
470 *n2
= fold_build_pointer_plus_hwi_loc (loc
, *n2
, 1);
472 *n2
= fold_build2_loc (loc
, PLUS_EXPR
, TREE_TYPE (*n2
), *n2
,
473 build_int_cst (TREE_TYPE (*n2
), 1));
474 *cond_code
= LT_EXPR
;
477 if (POINTER_TYPE_P (TREE_TYPE (*n2
)))
478 *n2
= fold_build_pointer_plus_hwi_loc (loc
, *n2
, -1);
480 *n2
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (*n2
), *n2
,
481 build_int_cst (TREE_TYPE (*n2
), 1));
482 *cond_code
= GT_EXPR
;
489 /* Return the looping step from INCR, extracted from the step of a gimple omp
493 get_omp_for_step_from_incr (location_t loc
, tree incr
)
496 switch (TREE_CODE (incr
))
499 step
= TREE_OPERAND (incr
, 1);
501 case POINTER_PLUS_EXPR
:
502 step
= fold_convert (ssizetype
, TREE_OPERAND (incr
, 1));
505 step
= TREE_OPERAND (incr
, 1);
506 step
= fold_build1_loc (loc
, NEGATE_EXPR
, TREE_TYPE (step
), step
);
514 /* Extract the header elements of parallel loop FOR_STMT and store
518 extract_omp_for_data (gomp_for
*for_stmt
, struct omp_for_data
*fd
,
519 struct omp_for_data_loop
*loops
)
521 tree t
, var
, *collapse_iter
, *collapse_count
;
522 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
523 struct omp_for_data_loop
*loop
;
525 struct omp_for_data_loop dummy_loop
;
526 location_t loc
= gimple_location (for_stmt
);
527 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_SIMD
;
528 bool distribute
= gimple_omp_for_kind (for_stmt
)
529 == GF_OMP_FOR_KIND_DISTRIBUTE
;
530 bool taskloop
= gimple_omp_for_kind (for_stmt
)
531 == GF_OMP_FOR_KIND_TASKLOOP
;
534 fd
->for_stmt
= for_stmt
;
536 if (gimple_omp_for_collapse (for_stmt
) > 1)
539 fd
->loops
= &fd
->loop
;
541 fd
->have_nowait
= distribute
|| simd
;
542 fd
->have_ordered
= false;
545 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
546 fd
->sched_modifiers
= 0;
547 fd
->chunk_size
= NULL_TREE
;
548 fd
->simd_schedule
= false;
549 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
550 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_CILKFOR
;
551 collapse_iter
= NULL
;
552 collapse_count
= NULL
;
554 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
555 switch (OMP_CLAUSE_CODE (t
))
557 case OMP_CLAUSE_NOWAIT
:
558 fd
->have_nowait
= true;
560 case OMP_CLAUSE_ORDERED
:
561 fd
->have_ordered
= true;
562 if (OMP_CLAUSE_ORDERED_EXPR (t
))
563 fd
->ordered
= tree_to_shwi (OMP_CLAUSE_ORDERED_EXPR (t
));
565 case OMP_CLAUSE_SCHEDULE
:
566 gcc_assert (!distribute
&& !taskloop
);
568 = (enum omp_clause_schedule_kind
)
569 (OMP_CLAUSE_SCHEDULE_KIND (t
) & OMP_CLAUSE_SCHEDULE_MASK
);
570 fd
->sched_modifiers
= (OMP_CLAUSE_SCHEDULE_KIND (t
)
571 & ~OMP_CLAUSE_SCHEDULE_MASK
);
572 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
573 fd
->simd_schedule
= OMP_CLAUSE_SCHEDULE_SIMD (t
);
575 case OMP_CLAUSE_DIST_SCHEDULE
:
576 gcc_assert (distribute
);
577 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
579 case OMP_CLAUSE_COLLAPSE
:
580 fd
->collapse
= tree_to_shwi (OMP_CLAUSE_COLLAPSE_EXPR (t
));
581 if (fd
->collapse
> 1)
583 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
584 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
590 if (fd
->ordered
&& fd
->collapse
== 1 && loops
!= NULL
)
595 collapse_iter
= &iterv
;
596 collapse_count
= &countv
;
599 /* FIXME: for now map schedule(auto) to schedule(static).
600 There should be analysis to determine whether all iterations
601 are approximately the same amount of work (then schedule(static)
602 is best) or if it varies (then schedule(dynamic,N) is better). */
603 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
605 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
606 gcc_assert (fd
->chunk_size
== NULL
);
608 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
610 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_RUNTIME
;
611 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
612 gcc_assert (fd
->chunk_size
== NULL
);
613 else if (fd
->chunk_size
== NULL
)
615 /* We only need to compute a default chunk size for ordered
616 static loops and dynamic loops. */
617 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
619 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
620 ? integer_zero_node
: integer_one_node
;
623 int cnt
= fd
->ordered
? fd
->ordered
: fd
->collapse
;
624 for (i
= 0; i
< cnt
; i
++)
626 if (i
== 0 && fd
->collapse
== 1 && (fd
->ordered
== 0 || loops
== NULL
))
628 else if (loops
!= NULL
)
633 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
634 gcc_assert (SSA_VAR_P (loop
->v
));
635 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
636 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
637 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
638 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
640 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
641 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
642 gcc_assert (loop
->cond_code
!= NE_EXPR
643 || gimple_omp_for_kind (for_stmt
) == GF_OMP_FOR_KIND_CILKSIMD
644 || gimple_omp_for_kind (for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
);
645 adjust_for_condition (loc
, &loop
->cond_code
, &loop
->n2
);
647 t
= gimple_omp_for_incr (for_stmt
, i
);
648 gcc_assert (TREE_OPERAND (t
, 0) == var
);
649 loop
->step
= get_omp_for_step_from_incr (loc
, t
);
652 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
653 && !fd
->have_ordered
))
655 if (fd
->collapse
== 1)
656 iter_type
= TREE_TYPE (loop
->v
);
658 || TYPE_PRECISION (iter_type
)
659 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
661 = build_nonstandard_integer_type
662 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
664 else if (iter_type
!= long_long_unsigned_type_node
)
666 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
667 iter_type
= long_long_unsigned_type_node
;
668 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
669 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
670 >= TYPE_PRECISION (iter_type
))
674 if (loop
->cond_code
== LT_EXPR
)
675 n
= fold_build2_loc (loc
,
676 PLUS_EXPR
, TREE_TYPE (loop
->v
),
677 loop
->n2
, loop
->step
);
680 if (TREE_CODE (n
) != INTEGER_CST
681 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
682 iter_type
= long_long_unsigned_type_node
;
684 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
685 > TYPE_PRECISION (iter_type
))
689 if (loop
->cond_code
== LT_EXPR
)
692 n2
= fold_build2_loc (loc
,
693 PLUS_EXPR
, TREE_TYPE (loop
->v
),
694 loop
->n2
, loop
->step
);
698 n1
= fold_build2_loc (loc
,
699 MINUS_EXPR
, TREE_TYPE (loop
->v
),
700 loop
->n2
, loop
->step
);
703 if (TREE_CODE (n1
) != INTEGER_CST
704 || TREE_CODE (n2
) != INTEGER_CST
705 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
706 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
707 iter_type
= long_long_unsigned_type_node
;
711 if (i
>= fd
->collapse
)
714 if (collapse_count
&& *collapse_count
== NULL
)
716 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
717 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
718 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
719 if (t
&& integer_zerop (t
))
720 count
= build_zero_cst (long_long_unsigned_type_node
);
721 else if ((i
== 0 || count
!= NULL_TREE
)
722 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
723 && TREE_CONSTANT (loop
->n1
)
724 && TREE_CONSTANT (loop
->n2
)
725 && TREE_CODE (loop
->step
) == INTEGER_CST
)
727 tree itype
= TREE_TYPE (loop
->v
);
729 if (POINTER_TYPE_P (itype
))
730 itype
= signed_type_for (itype
);
731 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
732 t
= fold_build2_loc (loc
,
734 fold_convert_loc (loc
, itype
, loop
->step
), t
);
735 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
736 fold_convert_loc (loc
, itype
, loop
->n2
));
737 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
738 fold_convert_loc (loc
, itype
, loop
->n1
));
739 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
740 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
741 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
742 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
743 fold_convert_loc (loc
, itype
,
746 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
747 fold_convert_loc (loc
, itype
, loop
->step
));
748 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
749 if (count
!= NULL_TREE
)
750 count
= fold_build2_loc (loc
,
751 MULT_EXPR
, long_long_unsigned_type_node
,
755 if (TREE_CODE (count
) != INTEGER_CST
)
758 else if (count
&& !integer_zerop (count
))
765 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
766 || fd
->have_ordered
))
768 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
769 iter_type
= long_long_unsigned_type_node
;
771 iter_type
= long_integer_type_node
;
773 else if (collapse_iter
&& *collapse_iter
!= NULL
)
774 iter_type
= TREE_TYPE (*collapse_iter
);
775 fd
->iter_type
= iter_type
;
776 if (collapse_iter
&& *collapse_iter
== NULL
)
777 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
778 if (collapse_count
&& *collapse_count
== NULL
)
781 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
783 *collapse_count
= create_tmp_var (iter_type
, ".count");
786 if (fd
->collapse
> 1 || (fd
->ordered
&& loops
))
788 fd
->loop
.v
= *collapse_iter
;
789 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
790 fd
->loop
.n2
= *collapse_count
;
791 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
792 fd
->loop
.cond_code
= LT_EXPR
;
799 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
800 is the immediate dominator of PAR_ENTRY_BB, return true if there
801 are no data dependencies that would prevent expanding the parallel
802 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
804 When expanding a combined parallel+workshare region, the call to
805 the child function may need additional arguments in the case of
806 GIMPLE_OMP_FOR regions. In some cases, these arguments are
807 computed out of variables passed in from the parent to the child
808 via 'struct .omp_data_s'. For instance:
810 #pragma omp parallel for schedule (guided, i * 4)
815 # BLOCK 2 (PAR_ENTRY_BB)
817 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
819 # BLOCK 3 (WS_ENTRY_BB)
820 .omp_data_i = &.omp_data_o;
821 D.1667 = .omp_data_i->i;
823 #pragma omp for schedule (guided, D.1598)
825 When we outline the parallel region, the call to the child function
826 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
827 that value is computed *after* the call site. So, in principle we
828 cannot do the transformation.
830 To see whether the code in WS_ENTRY_BB blocks the combined
831 parallel+workshare call, we collect all the variables used in the
832 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
833 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
836 FIXME. If we had the SSA form built at this point, we could merely
837 hoist the code in block 3 into block 2 and be done with it. But at
838 this point we don't have dataflow information and though we could
839 hack something up here, it is really not worth the aggravation. */
842 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
844 struct omp_for_data fd
;
845 gimple
*ws_stmt
= last_stmt (ws_entry_bb
);
847 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
850 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
852 extract_omp_for_data (as_a
<gomp_for
*> (ws_stmt
), &fd
, NULL
);
854 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
856 if (fd
.iter_type
!= long_integer_type_node
)
859 /* FIXME. We give up too easily here. If any of these arguments
860 are not constants, they will likely involve variables that have
861 been mapped into fields of .omp_data_s for sharing with the child
862 function. With appropriate data flow, it would be possible to
864 if (!is_gimple_min_invariant (fd
.loop
.n1
)
865 || !is_gimple_min_invariant (fd
.loop
.n2
)
866 || !is_gimple_min_invariant (fd
.loop
.step
)
867 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
874 static int omp_max_vf (void);
876 /* Adjust CHUNK_SIZE from SCHEDULE clause, depending on simd modifier
877 presence (SIMD_SCHEDULE). */
880 omp_adjust_chunk_size (tree chunk_size
, bool simd_schedule
)
885 int vf
= omp_max_vf ();
889 tree type
= TREE_TYPE (chunk_size
);
890 chunk_size
= fold_build2 (PLUS_EXPR
, type
, chunk_size
,
891 build_int_cst (type
, vf
- 1));
892 return fold_build2 (BIT_AND_EXPR
, type
, chunk_size
,
893 build_int_cst (type
, -vf
));
897 /* Collect additional arguments needed to emit a combined
898 parallel+workshare call. WS_STMT is the workshare directive being
901 static vec
<tree
, va_gc
> *
902 get_ws_args_for (gimple
*par_stmt
, gimple
*ws_stmt
)
905 location_t loc
= gimple_location (ws_stmt
);
906 vec
<tree
, va_gc
> *ws_args
;
908 if (gomp_for
*for_stmt
= dyn_cast
<gomp_for
*> (ws_stmt
))
910 struct omp_for_data fd
;
913 extract_omp_for_data (for_stmt
, &fd
, NULL
);
917 if (gimple_omp_for_combined_into_p (for_stmt
))
920 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
921 OMP_CLAUSE__LOOPTEMP_
);
923 n1
= OMP_CLAUSE_DECL (innerc
);
924 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
925 OMP_CLAUSE__LOOPTEMP_
);
927 n2
= OMP_CLAUSE_DECL (innerc
);
930 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
932 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
933 ws_args
->quick_push (t
);
935 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
936 ws_args
->quick_push (t
);
938 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
939 ws_args
->quick_push (t
);
943 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
944 t
= omp_adjust_chunk_size (t
, fd
.simd_schedule
);
945 ws_args
->quick_push (t
);
950 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
952 /* Number of sections is equal to the number of edges from the
953 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
954 the exit of the sections region. */
955 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
956 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
957 vec_alloc (ws_args
, 1);
958 ws_args
->quick_push (t
);
966 /* Discover whether REGION is a combined parallel+workshare region. */
969 determine_parallel_type (struct omp_region
*region
)
971 basic_block par_entry_bb
, par_exit_bb
;
972 basic_block ws_entry_bb
, ws_exit_bb
;
974 if (region
== NULL
|| region
->inner
== NULL
975 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
976 || region
->inner
->cont
== NULL
)
979 /* We only support parallel+for and parallel+sections. */
980 if (region
->type
!= GIMPLE_OMP_PARALLEL
981 || (region
->inner
->type
!= GIMPLE_OMP_FOR
982 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
985 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
986 WS_EXIT_BB -> PAR_EXIT_BB. */
987 par_entry_bb
= region
->entry
;
988 par_exit_bb
= region
->exit
;
989 ws_entry_bb
= region
->inner
->entry
;
990 ws_exit_bb
= region
->inner
->exit
;
992 if (single_succ (par_entry_bb
) == ws_entry_bb
993 && single_succ (ws_exit_bb
) == par_exit_bb
994 && workshare_safe_to_combine_p (ws_entry_bb
)
995 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
996 || (last_and_only_stmt (ws_entry_bb
)
997 && last_and_only_stmt (par_exit_bb
))))
999 gimple
*par_stmt
= last_stmt (par_entry_bb
);
1000 gimple
*ws_stmt
= last_stmt (ws_entry_bb
);
1002 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
1004 /* If this is a combined parallel loop, we need to determine
1005 whether or not to use the combined library calls. There
1006 are two cases where we do not apply the transformation:
1007 static loops and any kind of ordered loop. In the first
1008 case, we already open code the loop so there is no need
1009 to do anything else. In the latter case, the combined
1010 parallel loop call would still need extra synchronization
1011 to implement ordered semantics, so there would not be any
1012 gain in using the combined call. */
1013 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
1014 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
1016 || ((OMP_CLAUSE_SCHEDULE_KIND (c
) & OMP_CLAUSE_SCHEDULE_MASK
)
1017 == OMP_CLAUSE_SCHEDULE_STATIC
)
1018 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
1020 region
->is_combined_parallel
= false;
1021 region
->inner
->is_combined_parallel
= false;
1026 region
->is_combined_parallel
= true;
1027 region
->inner
->is_combined_parallel
= true;
1028 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
1033 /* Return true if EXPR is variable sized. */
1036 is_variable_sized (const_tree expr
)
1038 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
1041 /* Return true if DECL is a reference type. */
1044 is_reference (tree decl
)
1046 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
1049 /* Return the type of a decl. If the decl is reference type,
1050 return its base type. */
1052 get_base_type (tree decl
)
1054 tree type
= TREE_TYPE (decl
);
1055 if (is_reference (decl
))
1056 type
= TREE_TYPE (type
);
1060 /* Lookup variables. The "maybe" form
1061 allows for the variable form to not have been entered, otherwise we
1062 assert that the variable must have been entered. */
1065 lookup_decl (tree var
, omp_context
*ctx
)
1067 tree
*n
= ctx
->cb
.decl_map
->get (var
);
1072 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
1074 tree
*n
= ctx
->cb
.decl_map
->get (const_cast<tree
> (var
));
1075 return n
? *n
: NULL_TREE
;
1079 lookup_field (tree var
, omp_context
*ctx
)
1082 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
1083 return (tree
) n
->value
;
1087 lookup_sfield (splay_tree_key key
, omp_context
*ctx
)
1090 n
= splay_tree_lookup (ctx
->sfield_map
1091 ? ctx
->sfield_map
: ctx
->field_map
, key
);
1092 return (tree
) n
->value
;
1096 lookup_sfield (tree var
, omp_context
*ctx
)
1098 return lookup_sfield ((splay_tree_key
) var
, ctx
);
1102 maybe_lookup_field (splay_tree_key key
, omp_context
*ctx
)
1105 n
= splay_tree_lookup (ctx
->field_map
, key
);
1106 return n
? (tree
) n
->value
: NULL_TREE
;
1110 maybe_lookup_field (tree var
, omp_context
*ctx
)
1112 return maybe_lookup_field ((splay_tree_key
) var
, ctx
);
1115 /* Return true if DECL should be copied by pointer. SHARED_CTX is
1116 the parallel context if DECL is to be shared. */
1119 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
1121 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
1124 /* We can only use copy-in/copy-out semantics for shared variables
1125 when we know the value is not accessible from an outer scope. */
1128 gcc_assert (!is_gimple_omp_oacc (shared_ctx
->stmt
));
1130 /* ??? Trivially accessible from anywhere. But why would we even
1131 be passing an address in this case? Should we simply assert
1132 this to be false, or should we have a cleanup pass that removes
1133 these from the list of mappings? */
1134 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
1137 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
1138 without analyzing the expression whether or not its location
1139 is accessible to anyone else. In the case of nested parallel
1140 regions it certainly may be. */
1141 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
1144 /* Do not use copy-in/copy-out for variables that have their
1146 if (TREE_ADDRESSABLE (decl
))
1149 /* lower_send_shared_vars only uses copy-in, but not copy-out
1151 if (TREE_READONLY (decl
)
1152 || ((TREE_CODE (decl
) == RESULT_DECL
1153 || TREE_CODE (decl
) == PARM_DECL
)
1154 && DECL_BY_REFERENCE (decl
)))
1157 /* Disallow copy-in/out in nested parallel if
1158 decl is shared in outer parallel, otherwise
1159 each thread could store the shared variable
1160 in its own copy-in location, making the
1161 variable no longer really shared. */
1162 if (shared_ctx
->is_nested
)
1166 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
1167 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
1174 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
1175 c
; c
= OMP_CLAUSE_CHAIN (c
))
1176 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
1177 && OMP_CLAUSE_DECL (c
) == decl
)
1181 goto maybe_mark_addressable_and_ret
;
1185 /* For tasks avoid using copy-in/out. As tasks can be
1186 deferred or executed in different thread, when GOMP_task
1187 returns, the task hasn't necessarily terminated. */
1188 if (is_task_ctx (shared_ctx
))
1191 maybe_mark_addressable_and_ret
:
1192 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
1193 if (is_gimple_reg (outer
) && !omp_member_access_dummy_var (outer
))
1195 /* Taking address of OUTER in lower_send_shared_vars
1196 might need regimplification of everything that uses the
1198 if (!task_shared_vars
)
1199 task_shared_vars
= BITMAP_ALLOC (NULL
);
1200 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
1201 TREE_ADDRESSABLE (outer
) = 1;
1210 /* Construct a new automatic decl similar to VAR. */
1213 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
1215 tree copy
= copy_var_decl (var
, name
, type
);
1217 DECL_CONTEXT (copy
) = current_function_decl
;
1218 DECL_CHAIN (copy
) = ctx
->block_vars
;
1219 /* If VAR is listed in task_shared_vars, it means it wasn't
1220 originally addressable and is just because task needs to take
1221 it's address. But we don't need to take address of privatizations
1223 if (TREE_ADDRESSABLE (var
)
1225 && bitmap_bit_p (task_shared_vars
, DECL_UID (var
)))
1226 TREE_ADDRESSABLE (copy
) = 0;
1227 ctx
->block_vars
= copy
;
1233 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
1235 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
1238 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
1241 omp_build_component_ref (tree obj
, tree field
)
1243 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
1244 if (TREE_THIS_VOLATILE (field
))
1245 TREE_THIS_VOLATILE (ret
) |= 1;
1246 if (TREE_READONLY (field
))
1247 TREE_READONLY (ret
) |= 1;
1251 /* Build tree nodes to access the field for VAR on the receiver side. */
1254 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
1256 tree x
, field
= lookup_field (var
, ctx
);
1258 /* If the receiver record type was remapped in the child function,
1259 remap the field into the new record type. */
1260 x
= maybe_lookup_field (field
, ctx
);
1264 x
= build_simple_mem_ref (ctx
->receiver_decl
);
1265 TREE_THIS_NOTRAP (x
) = 1;
1266 x
= omp_build_component_ref (x
, field
);
1269 x
= build_simple_mem_ref (x
);
1270 TREE_THIS_NOTRAP (x
) = 1;
1276 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1277 of a parallel, this is a component reference; for workshare constructs
1278 this is some variable. */
1281 build_outer_var_ref (tree var
, omp_context
*ctx
, bool lastprivate
= false)
1285 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1287 else if (is_variable_sized (var
))
1289 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1290 x
= build_outer_var_ref (x
, ctx
, lastprivate
);
1291 x
= build_simple_mem_ref (x
);
1293 else if (is_taskreg_ctx (ctx
))
1295 bool by_ref
= use_pointer_for_field (var
, NULL
);
1296 x
= build_receiver_ref (var
, by_ref
, ctx
);
1298 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1299 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
1301 /* #pragma omp simd isn't a worksharing construct, and can reference even
1302 private vars in its linear etc. clauses. */
1304 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1305 x
= lookup_decl (var
, ctx
->outer
);
1306 else if (ctx
->outer
)
1307 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1311 else if (lastprivate
&& is_taskloop_ctx (ctx
))
1313 gcc_assert (ctx
->outer
);
1315 = splay_tree_lookup (ctx
->outer
->field_map
,
1316 (splay_tree_key
) &DECL_UID (var
));
1319 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
->outer
)))
1322 x
= lookup_decl (var
, ctx
->outer
);
1326 tree field
= (tree
) n
->value
;
1327 /* If the receiver record type was remapped in the child function,
1328 remap the field into the new record type. */
1329 x
= maybe_lookup_field (field
, ctx
->outer
);
1333 x
= build_simple_mem_ref (ctx
->outer
->receiver_decl
);
1334 x
= omp_build_component_ref (x
, field
);
1335 if (use_pointer_for_field (var
, ctx
->outer
))
1336 x
= build_simple_mem_ref (x
);
1339 else if (ctx
->outer
)
1341 omp_context
*outer
= ctx
->outer
;
1342 if (gimple_code (outer
->stmt
) == GIMPLE_OMP_GRID_BODY
)
1344 outer
= outer
->outer
;
1346 && gimple_code (outer
->stmt
) != GIMPLE_OMP_GRID_BODY
);
1348 x
= lookup_decl (var
, outer
);
1350 else if (is_reference (var
))
1351 /* This can happen with orphaned constructs. If var is reference, it is
1352 possible it is shared and as such valid. */
1354 else if (omp_member_access_dummy_var (var
))
1361 tree t
= omp_member_access_dummy_var (var
);
1364 x
= DECL_VALUE_EXPR (var
);
1365 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx
);
1367 x
= unshare_and_remap (x
, t
, o
);
1369 x
= unshare_expr (x
);
1373 if (is_reference (var
))
1374 x
= build_simple_mem_ref (x
);
1379 /* Build tree nodes to access the field for VAR on the sender side. */
1382 build_sender_ref (splay_tree_key key
, omp_context
*ctx
)
1384 tree field
= lookup_sfield (key
, ctx
);
1385 return omp_build_component_ref (ctx
->sender_decl
, field
);
1389 build_sender_ref (tree var
, omp_context
*ctx
)
1391 return build_sender_ref ((splay_tree_key
) var
, ctx
);
1394 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. If
1395 BASE_POINTERS_RESTRICT, declare the field with restrict. */
1398 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
,
1399 bool base_pointers_restrict
= false)
1401 tree field
, type
, sfield
= NULL_TREE
;
1402 splay_tree_key key
= (splay_tree_key
) var
;
1404 if ((mask
& 8) != 0)
1406 key
= (splay_tree_key
) &DECL_UID (var
);
1407 gcc_checking_assert (key
!= (splay_tree_key
) var
);
1409 gcc_assert ((mask
& 1) == 0
1410 || !splay_tree_lookup (ctx
->field_map
, key
));
1411 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1412 || !splay_tree_lookup (ctx
->sfield_map
, key
));
1413 gcc_assert ((mask
& 3) == 3
1414 || !is_gimple_omp_oacc (ctx
->stmt
));
1416 type
= TREE_TYPE (var
);
1417 /* Prevent redeclaring the var in the split-off function with a restrict
1418 pointer type. Note that we only clear type itself, restrict qualifiers in
1419 the pointed-to type will be ignored by points-to analysis. */
1420 if (POINTER_TYPE_P (type
)
1421 && TYPE_RESTRICT (type
))
1422 type
= build_qualified_type (type
, TYPE_QUALS (type
) & ~TYPE_QUAL_RESTRICT
);
1426 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1427 type
= build_pointer_type (build_pointer_type (type
));
1431 type
= build_pointer_type (type
);
1432 if (base_pointers_restrict
)
1433 type
= build_qualified_type (type
, TYPE_QUAL_RESTRICT
);
1435 else if ((mask
& 3) == 1 && is_reference (var
))
1436 type
= TREE_TYPE (type
);
1438 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1439 FIELD_DECL
, DECL_NAME (var
), type
);
1441 /* Remember what variable this field was created for. This does have a
1442 side effect of making dwarf2out ignore this member, so for helpful
1443 debugging we clear it later in delete_omp_context. */
1444 DECL_ABSTRACT_ORIGIN (field
) = var
;
1445 if (type
== TREE_TYPE (var
))
1447 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1448 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1449 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1452 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1454 if ((mask
& 3) == 3)
1456 insert_field_into_struct (ctx
->record_type
, field
);
1457 if (ctx
->srecord_type
)
1459 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1460 FIELD_DECL
, DECL_NAME (var
), type
);
1461 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1462 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1463 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1464 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1465 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1470 if (ctx
->srecord_type
== NULL_TREE
)
1474 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1475 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1476 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1478 sfield
= build_decl (DECL_SOURCE_LOCATION (t
),
1479 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1480 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1481 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1482 splay_tree_insert (ctx
->sfield_map
,
1483 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1484 (splay_tree_value
) sfield
);
1488 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1489 : ctx
->srecord_type
, field
);
1493 splay_tree_insert (ctx
->field_map
, key
, (splay_tree_value
) field
);
1494 if ((mask
& 2) && ctx
->sfield_map
)
1495 splay_tree_insert (ctx
->sfield_map
, key
, (splay_tree_value
) sfield
);
1499 install_var_local (tree var
, omp_context
*ctx
)
1501 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1502 insert_decl_map (&ctx
->cb
, var
, new_var
);
1506 /* Adjust the replacement for DECL in CTX for the new context. This means
1507 copying the DECL_VALUE_EXPR, and fixing up the type. */
1510 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1512 tree new_decl
, size
;
1514 new_decl
= lookup_decl (decl
, ctx
);
1516 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1518 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1519 && DECL_HAS_VALUE_EXPR_P (decl
))
1521 tree ve
= DECL_VALUE_EXPR (decl
);
1522 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1523 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1524 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1527 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1529 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1530 if (size
== error_mark_node
)
1531 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1532 DECL_SIZE (new_decl
) = size
;
1534 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1535 if (size
== error_mark_node
)
1536 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1537 DECL_SIZE_UNIT (new_decl
) = size
;
1541 /* The callback for remap_decl. Search all containing contexts for a
1542 mapping of the variable; this avoids having to duplicate the splay
1543 tree ahead of time. We know a mapping doesn't already exist in the
1544 given context. Create new mappings to implement default semantics. */
1547 omp_copy_decl (tree var
, copy_body_data
*cb
)
1549 omp_context
*ctx
= (omp_context
*) cb
;
1552 if (TREE_CODE (var
) == LABEL_DECL
)
1554 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1555 DECL_CONTEXT (new_var
) = current_function_decl
;
1556 insert_decl_map (&ctx
->cb
, var
, new_var
);
1560 while (!is_taskreg_ctx (ctx
))
1565 new_var
= maybe_lookup_decl (var
, ctx
);
1570 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1573 return error_mark_node
;
1577 /* Debugging dumps for parallel regions. */
1578 void dump_omp_region (FILE *, struct omp_region
*, int);
1579 void debug_omp_region (struct omp_region
*);
1580 void debug_all_omp_regions (void);
1582 /* Dump the parallel region tree rooted at REGION. */
1585 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1587 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1588 gimple_code_name
[region
->type
]);
1591 dump_omp_region (file
, region
->inner
, indent
+ 4);
1595 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1596 region
->cont
->index
);
1600 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1601 region
->exit
->index
);
1603 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1606 dump_omp_region (file
, region
->next
, indent
);
1610 debug_omp_region (struct omp_region
*region
)
1612 dump_omp_region (stderr
, region
, 0);
1616 debug_all_omp_regions (void)
1618 dump_omp_region (stderr
, root_omp_region
, 0);
1622 /* Create a new parallel region starting at STMT inside region PARENT. */
1624 static struct omp_region
*
1625 new_omp_region (basic_block bb
, enum gimple_code type
,
1626 struct omp_region
*parent
)
1628 struct omp_region
*region
= XCNEW (struct omp_region
);
1630 region
->outer
= parent
;
1632 region
->type
= type
;
1636 /* This is a nested region. Add it to the list of inner
1637 regions in PARENT. */
1638 region
->next
= parent
->inner
;
1639 parent
->inner
= region
;
1643 /* This is a toplevel region. Add it to the list of toplevel
1644 regions in ROOT_OMP_REGION. */
1645 region
->next
= root_omp_region
;
1646 root_omp_region
= region
;
1652 /* Release the memory associated with the region tree rooted at REGION. */
1655 free_omp_region_1 (struct omp_region
*region
)
1657 struct omp_region
*i
, *n
;
1659 for (i
= region
->inner
; i
; i
= n
)
1662 free_omp_region_1 (i
);
1668 /* Release the memory for the entire omp region tree. */
1671 free_omp_regions (void)
1673 struct omp_region
*r
, *n
;
1674 for (r
= root_omp_region
; r
; r
= n
)
1677 free_omp_region_1 (r
);
1679 root_omp_region
= NULL
;
1683 /* Create a new context, with OUTER_CTX being the surrounding context. */
1685 static omp_context
*
1686 new_omp_context (gimple
*stmt
, omp_context
*outer_ctx
)
1688 omp_context
*ctx
= XCNEW (omp_context
);
1690 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1691 (splay_tree_value
) ctx
);
1696 ctx
->outer
= outer_ctx
;
1697 ctx
->cb
= outer_ctx
->cb
;
1698 ctx
->cb
.block
= NULL
;
1699 ctx
->depth
= outer_ctx
->depth
+ 1;
1703 ctx
->cb
.src_fn
= current_function_decl
;
1704 ctx
->cb
.dst_fn
= current_function_decl
;
1705 ctx
->cb
.src_node
= cgraph_node::get (current_function_decl
);
1706 gcc_checking_assert (ctx
->cb
.src_node
);
1707 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1708 ctx
->cb
.src_cfun
= cfun
;
1709 ctx
->cb
.copy_decl
= omp_copy_decl
;
1710 ctx
->cb
.eh_lp_nr
= 0;
1711 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1715 ctx
->cb
.decl_map
= new hash_map
<tree
, tree
>;
1720 static gimple_seq
maybe_catch_exception (gimple_seq
);
1722 /* Finalize task copyfn. */
1725 finalize_task_copyfn (gomp_task
*task_stmt
)
1727 struct function
*child_cfun
;
1729 gimple_seq seq
= NULL
, new_seq
;
1732 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1733 if (child_fn
== NULL_TREE
)
1736 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1737 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1739 push_cfun (child_cfun
);
1740 bind
= gimplify_body (child_fn
, false);
1741 gimple_seq_add_stmt (&seq
, bind
);
1742 new_seq
= maybe_catch_exception (seq
);
1745 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1747 gimple_seq_add_stmt (&seq
, bind
);
1749 gimple_set_body (child_fn
, seq
);
1752 /* Inform the callgraph about the new function. */
1753 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
1754 node
->parallelized_function
= 1;
1755 cgraph_node::add_new_function (child_fn
, false);
1758 /* Destroy a omp_context data structures. Called through the splay tree
1759 value delete callback. */
1762 delete_omp_context (splay_tree_value value
)
1764 omp_context
*ctx
= (omp_context
*) value
;
1766 delete ctx
->cb
.decl_map
;
1769 splay_tree_delete (ctx
->field_map
);
1770 if (ctx
->sfield_map
)
1771 splay_tree_delete (ctx
->sfield_map
);
1773 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1774 it produces corrupt debug information. */
1775 if (ctx
->record_type
)
1778 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1779 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1781 if (ctx
->srecord_type
)
1784 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1785 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1788 if (is_task_ctx (ctx
))
1789 finalize_task_copyfn (as_a
<gomp_task
*> (ctx
->stmt
));
1794 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1798 fixup_child_record_type (omp_context
*ctx
)
1800 tree f
, type
= ctx
->record_type
;
1802 if (!ctx
->receiver_decl
)
1804 /* ??? It isn't sufficient to just call remap_type here, because
1805 variably_modified_type_p doesn't work the way we expect for
1806 record types. Testing each field for whether it needs remapping
1807 and creating a new record by hand works, however. */
1808 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1809 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1813 tree name
, new_fields
= NULL
;
1815 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1816 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1817 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1818 TYPE_DECL
, name
, type
);
1819 TYPE_NAME (type
) = name
;
1821 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1823 tree new_f
= copy_node (f
);
1824 DECL_CONTEXT (new_f
) = type
;
1825 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1826 DECL_CHAIN (new_f
) = new_fields
;
1827 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1828 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1830 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1834 /* Arrange to be able to look up the receiver field
1835 given the sender field. */
1836 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1837 (splay_tree_value
) new_f
);
1839 TYPE_FIELDS (type
) = nreverse (new_fields
);
1843 /* In a target region we never modify any of the pointers in *.omp_data_i,
1844 so attempt to help the optimizers. */
1845 if (is_gimple_omp_offloaded (ctx
->stmt
))
1846 type
= build_qualified_type (type
, TYPE_QUAL_CONST
);
1848 TREE_TYPE (ctx
->receiver_decl
)
1849 = build_qualified_type (build_reference_type (type
), TYPE_QUAL_RESTRICT
);
1852 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1853 specified by CLAUSES. If BASE_POINTERS_RESTRICT, install var field with
1857 scan_sharing_clauses (tree clauses
, omp_context
*ctx
,
1858 bool base_pointers_restrict
= false)
1861 bool scan_array_reductions
= false;
1863 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1867 switch (OMP_CLAUSE_CODE (c
))
1869 case OMP_CLAUSE_PRIVATE
:
1870 decl
= OMP_CLAUSE_DECL (c
);
1871 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1873 else if (!is_variable_sized (decl
))
1874 install_var_local (decl
, ctx
);
1877 case OMP_CLAUSE_SHARED
:
1878 decl
= OMP_CLAUSE_DECL (c
);
1879 /* Ignore shared directives in teams construct. */
1880 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1882 /* Global variables don't need to be copied,
1883 the receiver side will use them directly. */
1884 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1885 if (is_global_var (odecl
))
1887 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1890 gcc_assert (is_taskreg_ctx (ctx
));
1891 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1892 || !is_variable_sized (decl
));
1893 /* Global variables don't need to be copied,
1894 the receiver side will use them directly. */
1895 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1897 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
1899 use_pointer_for_field (decl
, ctx
);
1902 by_ref
= use_pointer_for_field (decl
, NULL
);
1903 if ((! TREE_READONLY (decl
) && !OMP_CLAUSE_SHARED_READONLY (c
))
1904 || TREE_ADDRESSABLE (decl
)
1906 || is_reference (decl
))
1908 by_ref
= use_pointer_for_field (decl
, ctx
);
1909 install_var_field (decl
, by_ref
, 3, ctx
);
1910 install_var_local (decl
, ctx
);
1913 /* We don't need to copy const scalar vars back. */
1914 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1917 case OMP_CLAUSE_REDUCTION
:
1918 decl
= OMP_CLAUSE_DECL (c
);
1919 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1920 && TREE_CODE (decl
) == MEM_REF
)
1922 tree t
= TREE_OPERAND (decl
, 0);
1923 if (TREE_CODE (t
) == POINTER_PLUS_EXPR
)
1924 t
= TREE_OPERAND (t
, 0);
1925 if (TREE_CODE (t
) == INDIRECT_REF
1926 || TREE_CODE (t
) == ADDR_EXPR
)
1927 t
= TREE_OPERAND (t
, 0);
1928 install_var_local (t
, ctx
);
1929 if (is_taskreg_ctx (ctx
)
1930 && !is_global_var (maybe_lookup_decl_in_outer_ctx (t
, ctx
))
1931 && !is_variable_sized (t
))
1933 by_ref
= use_pointer_for_field (t
, ctx
);
1934 install_var_field (t
, by_ref
, 3, ctx
);
1940 case OMP_CLAUSE_LASTPRIVATE
:
1941 /* Let the corresponding firstprivate clause create
1943 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1947 case OMP_CLAUSE_FIRSTPRIVATE
:
1948 case OMP_CLAUSE_LINEAR
:
1949 decl
= OMP_CLAUSE_DECL (c
);
1951 if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
1952 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_IS_DEVICE_PTR
)
1953 && is_gimple_omp_offloaded (ctx
->stmt
))
1955 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
1956 install_var_field (decl
, !is_reference (decl
), 3, ctx
);
1957 else if (TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1958 install_var_field (decl
, true, 3, ctx
);
1960 install_var_field (decl
, false, 3, ctx
);
1962 if (is_variable_sized (decl
))
1964 if (is_task_ctx (ctx
))
1965 install_var_field (decl
, false, 1, ctx
);
1968 else if (is_taskreg_ctx (ctx
))
1971 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1972 by_ref
= use_pointer_for_field (decl
, NULL
);
1974 if (is_task_ctx (ctx
)
1975 && (global
|| by_ref
|| is_reference (decl
)))
1977 install_var_field (decl
, false, 1, ctx
);
1979 install_var_field (decl
, by_ref
, 2, ctx
);
1982 install_var_field (decl
, by_ref
, 3, ctx
);
1984 install_var_local (decl
, ctx
);
1987 case OMP_CLAUSE_USE_DEVICE_PTR
:
1988 decl
= OMP_CLAUSE_DECL (c
);
1989 if (TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1990 install_var_field (decl
, true, 3, ctx
);
1992 install_var_field (decl
, false, 3, ctx
);
1993 if (DECL_SIZE (decl
)
1994 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1996 tree decl2
= DECL_VALUE_EXPR (decl
);
1997 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1998 decl2
= TREE_OPERAND (decl2
, 0);
1999 gcc_assert (DECL_P (decl2
));
2000 install_var_local (decl2
, ctx
);
2002 install_var_local (decl
, ctx
);
2005 case OMP_CLAUSE_IS_DEVICE_PTR
:
2006 decl
= OMP_CLAUSE_DECL (c
);
2009 case OMP_CLAUSE__LOOPTEMP_
:
2010 gcc_assert (is_taskreg_ctx (ctx
));
2011 decl
= OMP_CLAUSE_DECL (c
);
2012 install_var_field (decl
, false, 3, ctx
);
2013 install_var_local (decl
, ctx
);
2016 case OMP_CLAUSE_COPYPRIVATE
:
2017 case OMP_CLAUSE_COPYIN
:
2018 decl
= OMP_CLAUSE_DECL (c
);
2019 by_ref
= use_pointer_for_field (decl
, NULL
);
2020 install_var_field (decl
, by_ref
, 3, ctx
);
2023 case OMP_CLAUSE_DEFAULT
:
2024 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
2027 case OMP_CLAUSE_FINAL
:
2029 case OMP_CLAUSE_NUM_THREADS
:
2030 case OMP_CLAUSE_NUM_TEAMS
:
2031 case OMP_CLAUSE_THREAD_LIMIT
:
2032 case OMP_CLAUSE_DEVICE
:
2033 case OMP_CLAUSE_SCHEDULE
:
2034 case OMP_CLAUSE_DIST_SCHEDULE
:
2035 case OMP_CLAUSE_DEPEND
:
2036 case OMP_CLAUSE_PRIORITY
:
2037 case OMP_CLAUSE_GRAINSIZE
:
2038 case OMP_CLAUSE_NUM_TASKS
:
2039 case OMP_CLAUSE__CILK_FOR_COUNT_
:
2040 case OMP_CLAUSE_NUM_GANGS
:
2041 case OMP_CLAUSE_NUM_WORKERS
:
2042 case OMP_CLAUSE_VECTOR_LENGTH
:
2044 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
2048 case OMP_CLAUSE_FROM
:
2049 case OMP_CLAUSE_MAP
:
2051 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
2052 decl
= OMP_CLAUSE_DECL (c
);
2053 /* Global variables with "omp declare target" attribute
2054 don't need to be copied, the receiver side will use them
2055 directly. However, global variables with "omp declare target link"
2056 attribute need to be copied. */
2057 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2059 && ((OMP_CLAUSE_MAP_KIND (c
) != GOMP_MAP_FIRSTPRIVATE_POINTER
2060 && (OMP_CLAUSE_MAP_KIND (c
)
2061 != GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
2062 || TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2063 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
2064 && varpool_node::get_create (decl
)->offloadable
2065 && !lookup_attribute ("omp declare target link",
2066 DECL_ATTRIBUTES (decl
)))
2068 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2069 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
)
2071 /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are
2072 not offloaded; there is nothing to map for those. */
2073 if (!is_gimple_omp_offloaded (ctx
->stmt
)
2074 && !POINTER_TYPE_P (TREE_TYPE (decl
))
2075 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
2078 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2079 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
2080 || (OMP_CLAUSE_MAP_KIND (c
)
2081 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
2083 if (TREE_CODE (decl
) == COMPONENT_REF
2084 || (TREE_CODE (decl
) == INDIRECT_REF
2085 && TREE_CODE (TREE_OPERAND (decl
, 0)) == COMPONENT_REF
2086 && (TREE_CODE (TREE_TYPE (TREE_OPERAND (decl
, 0)))
2087 == REFERENCE_TYPE
)))
2089 if (DECL_SIZE (decl
)
2090 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2092 tree decl2
= DECL_VALUE_EXPR (decl
);
2093 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2094 decl2
= TREE_OPERAND (decl2
, 0);
2095 gcc_assert (DECL_P (decl2
));
2096 install_var_local (decl2
, ctx
);
2098 install_var_local (decl
, ctx
);
2103 if (DECL_SIZE (decl
)
2104 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2106 tree decl2
= DECL_VALUE_EXPR (decl
);
2107 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2108 decl2
= TREE_OPERAND (decl2
, 0);
2109 gcc_assert (DECL_P (decl2
));
2110 install_var_field (decl2
, true, 3, ctx
);
2111 install_var_local (decl2
, ctx
);
2112 install_var_local (decl
, ctx
);
2116 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2117 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
2118 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
2119 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2120 install_var_field (decl
, true, 7, ctx
);
2122 install_var_field (decl
, true, 3, ctx
,
2123 base_pointers_restrict
);
2124 if (is_gimple_omp_offloaded (ctx
->stmt
))
2125 install_var_local (decl
, ctx
);
2130 tree base
= get_base_address (decl
);
2131 tree nc
= OMP_CLAUSE_CHAIN (c
);
2134 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
2135 && OMP_CLAUSE_DECL (nc
) == base
2136 && OMP_CLAUSE_MAP_KIND (nc
) == GOMP_MAP_POINTER
2137 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
2139 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
2140 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
2146 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
2147 decl
= OMP_CLAUSE_DECL (c
);
2149 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
2150 (splay_tree_key
) decl
));
2152 = build_decl (OMP_CLAUSE_LOCATION (c
),
2153 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
2154 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
2155 insert_field_into_struct (ctx
->record_type
, field
);
2156 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
2157 (splay_tree_value
) field
);
2162 case OMP_CLAUSE__GRIDDIM_
:
2165 scan_omp_op (&OMP_CLAUSE__GRIDDIM__SIZE (c
), ctx
->outer
);
2166 scan_omp_op (&OMP_CLAUSE__GRIDDIM__GROUP (c
), ctx
->outer
);
2170 case OMP_CLAUSE_NOWAIT
:
2171 case OMP_CLAUSE_ORDERED
:
2172 case OMP_CLAUSE_COLLAPSE
:
2173 case OMP_CLAUSE_UNTIED
:
2174 case OMP_CLAUSE_MERGEABLE
:
2175 case OMP_CLAUSE_PROC_BIND
:
2176 case OMP_CLAUSE_SAFELEN
:
2177 case OMP_CLAUSE_SIMDLEN
:
2178 case OMP_CLAUSE_THREADS
:
2179 case OMP_CLAUSE_SIMD
:
2180 case OMP_CLAUSE_NOGROUP
:
2181 case OMP_CLAUSE_DEFAULTMAP
:
2182 case OMP_CLAUSE_ASYNC
:
2183 case OMP_CLAUSE_WAIT
:
2184 case OMP_CLAUSE_GANG
:
2185 case OMP_CLAUSE_WORKER
:
2186 case OMP_CLAUSE_VECTOR
:
2187 case OMP_CLAUSE_TILE
:
2188 case OMP_CLAUSE_INDEPENDENT
:
2189 case OMP_CLAUSE_AUTO
:
2190 case OMP_CLAUSE_SEQ
:
2193 case OMP_CLAUSE_ALIGNED
:
2194 decl
= OMP_CLAUSE_DECL (c
);
2195 if (is_global_var (decl
)
2196 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2197 install_var_local (decl
, ctx
);
2200 case OMP_CLAUSE_DEVICE_RESIDENT
:
2201 case OMP_CLAUSE__CACHE_
:
2202 sorry ("Clause not supported yet");
2210 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2212 switch (OMP_CLAUSE_CODE (c
))
2214 case OMP_CLAUSE_LASTPRIVATE
:
2215 /* Let the corresponding firstprivate clause create
2217 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2218 scan_array_reductions
= true;
2219 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
2223 case OMP_CLAUSE_FIRSTPRIVATE
:
2224 case OMP_CLAUSE_PRIVATE
:
2225 case OMP_CLAUSE_LINEAR
:
2226 case OMP_CLAUSE_IS_DEVICE_PTR
:
2227 decl
= OMP_CLAUSE_DECL (c
);
2228 if (is_variable_sized (decl
))
2230 if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
2231 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_IS_DEVICE_PTR
)
2232 && is_gimple_omp_offloaded (ctx
->stmt
))
2234 tree decl2
= DECL_VALUE_EXPR (decl
);
2235 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2236 decl2
= TREE_OPERAND (decl2
, 0);
2237 gcc_assert (DECL_P (decl2
));
2238 install_var_local (decl2
, ctx
);
2239 fixup_remapped_decl (decl2
, ctx
, false);
2241 install_var_local (decl
, ctx
);
2243 fixup_remapped_decl (decl
, ctx
,
2244 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
2245 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
2246 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2247 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2248 scan_array_reductions
= true;
2251 case OMP_CLAUSE_REDUCTION
:
2252 decl
= OMP_CLAUSE_DECL (c
);
2253 if (TREE_CODE (decl
) != MEM_REF
)
2255 if (is_variable_sized (decl
))
2256 install_var_local (decl
, ctx
);
2257 fixup_remapped_decl (decl
, ctx
, false);
2259 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2260 scan_array_reductions
= true;
2263 case OMP_CLAUSE_SHARED
:
2264 /* Ignore shared directives in teams construct. */
2265 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2267 decl
= OMP_CLAUSE_DECL (c
);
2268 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2270 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
2272 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
,
2275 bool by_ref
= use_pointer_for_field (decl
, ctx
);
2276 install_var_field (decl
, by_ref
, 11, ctx
);
2279 fixup_remapped_decl (decl
, ctx
, false);
2282 case OMP_CLAUSE_MAP
:
2283 if (!is_gimple_omp_offloaded (ctx
->stmt
))
2285 decl
= OMP_CLAUSE_DECL (c
);
2287 && ((OMP_CLAUSE_MAP_KIND (c
) != GOMP_MAP_FIRSTPRIVATE_POINTER
2288 && (OMP_CLAUSE_MAP_KIND (c
)
2289 != GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
2290 || TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2291 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
2292 && varpool_node::get_create (decl
)->offloadable
)
2296 if ((OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
2297 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
)
2298 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
2299 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
2301 tree new_decl
= lookup_decl (decl
, ctx
);
2302 TREE_TYPE (new_decl
)
2303 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
2305 else if (DECL_SIZE (decl
)
2306 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2308 tree decl2
= DECL_VALUE_EXPR (decl
);
2309 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2310 decl2
= TREE_OPERAND (decl2
, 0);
2311 gcc_assert (DECL_P (decl2
));
2312 fixup_remapped_decl (decl2
, ctx
, false);
2313 fixup_remapped_decl (decl
, ctx
, true);
2316 fixup_remapped_decl (decl
, ctx
, false);
2320 case OMP_CLAUSE_COPYPRIVATE
:
2321 case OMP_CLAUSE_COPYIN
:
2322 case OMP_CLAUSE_DEFAULT
:
2324 case OMP_CLAUSE_NUM_THREADS
:
2325 case OMP_CLAUSE_NUM_TEAMS
:
2326 case OMP_CLAUSE_THREAD_LIMIT
:
2327 case OMP_CLAUSE_DEVICE
:
2328 case OMP_CLAUSE_SCHEDULE
:
2329 case OMP_CLAUSE_DIST_SCHEDULE
:
2330 case OMP_CLAUSE_NOWAIT
:
2331 case OMP_CLAUSE_ORDERED
:
2332 case OMP_CLAUSE_COLLAPSE
:
2333 case OMP_CLAUSE_UNTIED
:
2334 case OMP_CLAUSE_FINAL
:
2335 case OMP_CLAUSE_MERGEABLE
:
2336 case OMP_CLAUSE_PROC_BIND
:
2337 case OMP_CLAUSE_SAFELEN
:
2338 case OMP_CLAUSE_SIMDLEN
:
2339 case OMP_CLAUSE_ALIGNED
:
2340 case OMP_CLAUSE_DEPEND
:
2341 case OMP_CLAUSE__LOOPTEMP_
:
2343 case OMP_CLAUSE_FROM
:
2344 case OMP_CLAUSE_PRIORITY
:
2345 case OMP_CLAUSE_GRAINSIZE
:
2346 case OMP_CLAUSE_NUM_TASKS
:
2347 case OMP_CLAUSE_THREADS
:
2348 case OMP_CLAUSE_SIMD
:
2349 case OMP_CLAUSE_NOGROUP
:
2350 case OMP_CLAUSE_DEFAULTMAP
:
2351 case OMP_CLAUSE_USE_DEVICE_PTR
:
2352 case OMP_CLAUSE__CILK_FOR_COUNT_
:
2353 case OMP_CLAUSE_ASYNC
:
2354 case OMP_CLAUSE_WAIT
:
2355 case OMP_CLAUSE_NUM_GANGS
:
2356 case OMP_CLAUSE_NUM_WORKERS
:
2357 case OMP_CLAUSE_VECTOR_LENGTH
:
2358 case OMP_CLAUSE_GANG
:
2359 case OMP_CLAUSE_WORKER
:
2360 case OMP_CLAUSE_VECTOR
:
2361 case OMP_CLAUSE_TILE
:
2362 case OMP_CLAUSE_INDEPENDENT
:
2363 case OMP_CLAUSE_AUTO
:
2364 case OMP_CLAUSE_SEQ
:
2365 case OMP_CLAUSE__GRIDDIM_
:
2368 case OMP_CLAUSE_DEVICE_RESIDENT
:
2369 case OMP_CLAUSE__CACHE_
:
2370 sorry ("Clause not supported yet");
2378 gcc_checking_assert (!scan_array_reductions
2379 || !is_gimple_omp_oacc (ctx
->stmt
));
2380 if (scan_array_reductions
)
2381 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2382 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
2383 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2385 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
2386 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
2388 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
2389 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2390 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
2391 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2392 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2393 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
2396 /* Create a new name for omp child function. Returns an identifier. If
2397 IS_CILK_FOR is true then the suffix for the child function is
2401 create_omp_child_function_name (bool task_copy
, bool is_cilk_for
)
2404 return clone_function_name (current_function_decl
, "_cilk_for_fn");
2405 return clone_function_name (current_function_decl
,
2406 task_copy
? "_omp_cpyfn" : "_omp_fn");
2409 /* Returns the type of the induction variable for the child function for
2410 _Cilk_for and the types for _high and _low variables based on TYPE. */
2413 cilk_for_check_loop_diff_type (tree type
)
2415 if (TYPE_PRECISION (type
) <= TYPE_PRECISION (uint32_type_node
))
2417 if (TYPE_UNSIGNED (type
))
2418 return uint32_type_node
;
2420 return integer_type_node
;
2424 if (TYPE_UNSIGNED (type
))
2425 return uint64_type_node
;
2427 return long_long_integer_type_node
;
2431 /* Build a decl for the omp child function. It'll not contain a body
2432 yet, just the bare decl. */
2435 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
2437 tree decl
, type
, name
, t
;
2440 = (flag_cilkplus
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2441 ? find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
2442 OMP_CLAUSE__CILK_FOR_COUNT_
) : NULL_TREE
;
2443 tree cilk_var_type
= NULL_TREE
;
2445 name
= create_omp_child_function_name (task_copy
,
2446 cilk_for_count
!= NULL_TREE
);
2448 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2449 ptr_type_node
, NULL_TREE
);
2450 else if (cilk_for_count
)
2452 type
= TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count
, 0));
2453 cilk_var_type
= cilk_for_check_loop_diff_type (type
);
2454 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2455 cilk_var_type
, cilk_var_type
, NULL_TREE
);
2458 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
2460 decl
= build_decl (gimple_location (ctx
->stmt
), FUNCTION_DECL
, name
, type
);
2462 gcc_checking_assert (!is_gimple_omp_oacc (ctx
->stmt
)
2465 ctx
->cb
.dst_fn
= decl
;
2467 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
2469 TREE_STATIC (decl
) = 1;
2470 TREE_USED (decl
) = 1;
2471 DECL_ARTIFICIAL (decl
) = 1;
2472 DECL_IGNORED_P (decl
) = 0;
2473 TREE_PUBLIC (decl
) = 0;
2474 DECL_UNINLINABLE (decl
) = 1;
2475 DECL_EXTERNAL (decl
) = 0;
2476 DECL_CONTEXT (decl
) = NULL_TREE
;
2477 DECL_INITIAL (decl
) = make_node (BLOCK
);
2478 if (cgraph_node::get (current_function_decl
)->offloadable
)
2479 cgraph_node::get_create (decl
)->offloadable
= 1;
2483 for (octx
= ctx
; octx
; octx
= octx
->outer
)
2484 if (is_gimple_omp_offloaded (octx
->stmt
))
2486 cgraph_node::get_create (decl
)->offloadable
= 1;
2487 if (ENABLE_OFFLOADING
)
2488 g
->have_offload
= true;
2494 if (cgraph_node::get_create (decl
)->offloadable
2495 && !lookup_attribute ("omp declare target",
2496 DECL_ATTRIBUTES (current_function_decl
)))
2497 DECL_ATTRIBUTES (decl
)
2498 = tree_cons (get_identifier ("omp target entrypoint"),
2499 NULL_TREE
, DECL_ATTRIBUTES (decl
));
2501 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2502 RESULT_DECL
, NULL_TREE
, void_type_node
);
2503 DECL_ARTIFICIAL (t
) = 1;
2504 DECL_IGNORED_P (t
) = 1;
2505 DECL_CONTEXT (t
) = decl
;
2506 DECL_RESULT (decl
) = t
;
2508 /* _Cilk_for's child function requires two extra parameters called
2509 __low and __high that are set the by Cilk runtime when it calls this
2513 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2514 PARM_DECL
, get_identifier ("__high"), cilk_var_type
);
2515 DECL_ARTIFICIAL (t
) = 1;
2516 DECL_NAMELESS (t
) = 1;
2517 DECL_ARG_TYPE (t
) = ptr_type_node
;
2518 DECL_CONTEXT (t
) = current_function_decl
;
2520 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2521 DECL_ARGUMENTS (decl
) = t
;
2523 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2524 PARM_DECL
, get_identifier ("__low"), cilk_var_type
);
2525 DECL_ARTIFICIAL (t
) = 1;
2526 DECL_NAMELESS (t
) = 1;
2527 DECL_ARG_TYPE (t
) = ptr_type_node
;
2528 DECL_CONTEXT (t
) = current_function_decl
;
2530 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2531 DECL_ARGUMENTS (decl
) = t
;
2534 tree data_name
= get_identifier (".omp_data_i");
2535 t
= build_decl (DECL_SOURCE_LOCATION (decl
), PARM_DECL
, data_name
,
2537 DECL_ARTIFICIAL (t
) = 1;
2538 DECL_NAMELESS (t
) = 1;
2539 DECL_ARG_TYPE (t
) = ptr_type_node
;
2540 DECL_CONTEXT (t
) = current_function_decl
;
2542 TREE_READONLY (t
) = 1;
2544 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2545 DECL_ARGUMENTS (decl
) = t
;
2547 ctx
->receiver_decl
= t
;
2550 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2551 PARM_DECL
, get_identifier (".omp_data_o"),
2553 DECL_ARTIFICIAL (t
) = 1;
2554 DECL_NAMELESS (t
) = 1;
2555 DECL_ARG_TYPE (t
) = ptr_type_node
;
2556 DECL_CONTEXT (t
) = current_function_decl
;
2558 TREE_ADDRESSABLE (t
) = 1;
2559 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2560 DECL_ARGUMENTS (decl
) = t
;
2563 /* Allocate memory for the function structure. The call to
2564 allocate_struct_function clobbers CFUN, so we need to restore
2566 push_struct_function (decl
);
2567 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
2571 /* Callback for walk_gimple_seq. Check if combined parallel
2572 contains gimple_omp_for_combined_into_p OMP_FOR. */
2575 find_combined_for (gimple_stmt_iterator
*gsi_p
,
2576 bool *handled_ops_p
,
2577 struct walk_stmt_info
*wi
)
2579 gimple
*stmt
= gsi_stmt (*gsi_p
);
2581 *handled_ops_p
= true;
2582 switch (gimple_code (stmt
))
2586 case GIMPLE_OMP_FOR
:
2587 if (gimple_omp_for_combined_into_p (stmt
)
2588 && gimple_omp_for_kind (stmt
)
2589 == *(const enum gf_mask
*) (wi
->info
))
2592 return integer_zero_node
;
2601 /* Add _LOOPTEMP_ clauses on OpenMP parallel or task. */
2604 add_taskreg_looptemp_clauses (enum gf_mask msk
, gimple
*stmt
,
2605 omp_context
*outer_ctx
)
2607 struct walk_stmt_info wi
;
2609 memset (&wi
, 0, sizeof (wi
));
2611 wi
.info
= (void *) &msk
;
2612 walk_gimple_seq (gimple_omp_body (stmt
), find_combined_for
, NULL
, &wi
);
2613 if (wi
.info
!= (void *) &msk
)
2615 gomp_for
*for_stmt
= as_a
<gomp_for
*> ((gimple
*) wi
.info
);
2616 struct omp_for_data fd
;
2617 extract_omp_for_data (for_stmt
, &fd
, NULL
);
2618 /* We need two temporaries with fd.loop.v type (istart/iend)
2619 and then (fd.collapse - 1) temporaries with the same
2620 type for count2 ... countN-1 vars if not constant. */
2621 size_t count
= 2, i
;
2622 tree type
= fd
.iter_type
;
2624 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
2626 count
+= fd
.collapse
- 1;
2627 /* If there are lastprivate clauses on the inner
2628 GIMPLE_OMP_FOR, add one more temporaries for the total number
2629 of iterations (product of count1 ... countN-1). */
2630 if (find_omp_clause (gimple_omp_for_clauses (for_stmt
),
2631 OMP_CLAUSE_LASTPRIVATE
))
2633 else if (msk
== GF_OMP_FOR_KIND_FOR
2634 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2635 OMP_CLAUSE_LASTPRIVATE
))
2638 for (i
= 0; i
< count
; i
++)
2640 tree temp
= create_tmp_var (type
);
2641 tree c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
2642 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
2643 OMP_CLAUSE_DECL (c
) = temp
;
2644 OMP_CLAUSE_CHAIN (c
) = gimple_omp_taskreg_clauses (stmt
);
2645 gimple_omp_taskreg_set_clauses (stmt
, c
);
2650 /* Scan an OpenMP parallel directive. */
2653 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2657 gomp_parallel
*stmt
= as_a
<gomp_parallel
*> (gsi_stmt (*gsi
));
2659 /* Ignore parallel directives with empty bodies, unless there
2660 are copyin clauses. */
2662 && empty_body_p (gimple_omp_body (stmt
))
2663 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2664 OMP_CLAUSE_COPYIN
) == NULL
)
2666 gsi_replace (gsi
, gimple_build_nop (), false);
2670 if (gimple_omp_parallel_combined_p (stmt
))
2671 add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_FOR
, stmt
, outer_ctx
);
2673 ctx
= new_omp_context (stmt
, outer_ctx
);
2674 taskreg_contexts
.safe_push (ctx
);
2675 if (taskreg_nesting_level
> 1)
2676 ctx
->is_nested
= true;
2677 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2678 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2679 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2680 name
= create_tmp_var_name (".omp_data_s");
2681 name
= build_decl (gimple_location (stmt
),
2682 TYPE_DECL
, name
, ctx
->record_type
);
2683 DECL_ARTIFICIAL (name
) = 1;
2684 DECL_NAMELESS (name
) = 1;
2685 TYPE_NAME (ctx
->record_type
) = name
;
2686 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2687 if (!gimple_omp_parallel_grid_phony (stmt
))
2689 create_omp_child_function (ctx
, false);
2690 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2693 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2694 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2696 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2697 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2700 /* Scan an OpenMP task directive. */
2703 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2707 gomp_task
*stmt
= as_a
<gomp_task
*> (gsi_stmt (*gsi
));
2709 /* Ignore task directives with empty bodies. */
2711 && empty_body_p (gimple_omp_body (stmt
)))
2713 gsi_replace (gsi
, gimple_build_nop (), false);
2717 if (gimple_omp_task_taskloop_p (stmt
))
2718 add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_TASKLOOP
, stmt
, outer_ctx
);
2720 ctx
= new_omp_context (stmt
, outer_ctx
);
2721 taskreg_contexts
.safe_push (ctx
);
2722 if (taskreg_nesting_level
> 1)
2723 ctx
->is_nested
= true;
2724 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2725 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2726 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2727 name
= create_tmp_var_name (".omp_data_s");
2728 name
= build_decl (gimple_location (stmt
),
2729 TYPE_DECL
, name
, ctx
->record_type
);
2730 DECL_ARTIFICIAL (name
) = 1;
2731 DECL_NAMELESS (name
) = 1;
2732 TYPE_NAME (ctx
->record_type
) = name
;
2733 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2734 create_omp_child_function (ctx
, false);
2735 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2737 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2739 if (ctx
->srecord_type
)
2741 name
= create_tmp_var_name (".omp_data_a");
2742 name
= build_decl (gimple_location (stmt
),
2743 TYPE_DECL
, name
, ctx
->srecord_type
);
2744 DECL_ARTIFICIAL (name
) = 1;
2745 DECL_NAMELESS (name
) = 1;
2746 TYPE_NAME (ctx
->srecord_type
) = name
;
2747 TYPE_ARTIFICIAL (ctx
->srecord_type
) = 1;
2748 create_omp_child_function (ctx
, true);
2751 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2753 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2755 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2756 t
= build_int_cst (long_integer_type_node
, 0);
2757 gimple_omp_task_set_arg_size (stmt
, t
);
2758 t
= build_int_cst (long_integer_type_node
, 1);
2759 gimple_omp_task_set_arg_align (stmt
, t
);
2764 /* If any decls have been made addressable during scan_omp,
2765 adjust their fields if needed, and layout record types
2766 of parallel/task constructs. */
2769 finish_taskreg_scan (omp_context
*ctx
)
2771 if (ctx
->record_type
== NULL_TREE
)
2774 /* If any task_shared_vars were needed, verify all
2775 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2776 statements if use_pointer_for_field hasn't changed
2777 because of that. If it did, update field types now. */
2778 if (task_shared_vars
)
2782 for (c
= gimple_omp_taskreg_clauses (ctx
->stmt
);
2783 c
; c
= OMP_CLAUSE_CHAIN (c
))
2784 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
2785 && !OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
2787 tree decl
= OMP_CLAUSE_DECL (c
);
2789 /* Global variables don't need to be copied,
2790 the receiver side will use them directly. */
2791 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2793 if (!bitmap_bit_p (task_shared_vars
, DECL_UID (decl
))
2794 || !use_pointer_for_field (decl
, ctx
))
2796 tree field
= lookup_field (decl
, ctx
);
2797 if (TREE_CODE (TREE_TYPE (field
)) == POINTER_TYPE
2798 && TREE_TYPE (TREE_TYPE (field
)) == TREE_TYPE (decl
))
2800 TREE_TYPE (field
) = build_pointer_type (TREE_TYPE (decl
));
2801 TREE_THIS_VOLATILE (field
) = 0;
2802 DECL_USER_ALIGN (field
) = 0;
2803 DECL_ALIGN (field
) = TYPE_ALIGN (TREE_TYPE (field
));
2804 if (TYPE_ALIGN (ctx
->record_type
) < DECL_ALIGN (field
))
2805 TYPE_ALIGN (ctx
->record_type
) = DECL_ALIGN (field
);
2806 if (ctx
->srecord_type
)
2808 tree sfield
= lookup_sfield (decl
, ctx
);
2809 TREE_TYPE (sfield
) = TREE_TYPE (field
);
2810 TREE_THIS_VOLATILE (sfield
) = 0;
2811 DECL_USER_ALIGN (sfield
) = 0;
2812 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
2813 if (TYPE_ALIGN (ctx
->srecord_type
) < DECL_ALIGN (sfield
))
2814 TYPE_ALIGN (ctx
->srecord_type
) = DECL_ALIGN (sfield
);
2819 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2821 layout_type (ctx
->record_type
);
2822 fixup_child_record_type (ctx
);
2826 location_t loc
= gimple_location (ctx
->stmt
);
2827 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2828 /* Move VLA fields to the end. */
2829 p
= &TYPE_FIELDS (ctx
->record_type
);
2831 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2832 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2835 *p
= TREE_CHAIN (*p
);
2836 TREE_CHAIN (*q
) = NULL_TREE
;
2837 q
= &TREE_CHAIN (*q
);
2840 p
= &DECL_CHAIN (*p
);
2842 if (gimple_omp_task_taskloop_p (ctx
->stmt
))
2844 /* Move fields corresponding to first and second _looptemp_
2845 clause first. There are filled by GOMP_taskloop
2846 and thus need to be in specific positions. */
2847 tree c1
= gimple_omp_task_clauses (ctx
->stmt
);
2848 c1
= find_omp_clause (c1
, OMP_CLAUSE__LOOPTEMP_
);
2849 tree c2
= find_omp_clause (OMP_CLAUSE_CHAIN (c1
),
2850 OMP_CLAUSE__LOOPTEMP_
);
2851 tree f1
= lookup_field (OMP_CLAUSE_DECL (c1
), ctx
);
2852 tree f2
= lookup_field (OMP_CLAUSE_DECL (c2
), ctx
);
2853 p
= &TYPE_FIELDS (ctx
->record_type
);
2855 if (*p
== f1
|| *p
== f2
)
2856 *p
= DECL_CHAIN (*p
);
2858 p
= &DECL_CHAIN (*p
);
2859 DECL_CHAIN (f1
) = f2
;
2860 DECL_CHAIN (f2
) = TYPE_FIELDS (ctx
->record_type
);
2861 TYPE_FIELDS (ctx
->record_type
) = f1
;
2862 if (ctx
->srecord_type
)
2864 f1
= lookup_sfield (OMP_CLAUSE_DECL (c1
), ctx
);
2865 f2
= lookup_sfield (OMP_CLAUSE_DECL (c2
), ctx
);
2866 p
= &TYPE_FIELDS (ctx
->srecord_type
);
2868 if (*p
== f1
|| *p
== f2
)
2869 *p
= DECL_CHAIN (*p
);
2871 p
= &DECL_CHAIN (*p
);
2872 DECL_CHAIN (f1
) = f2
;
2873 DECL_CHAIN (f2
) = TYPE_FIELDS (ctx
->srecord_type
);
2874 TYPE_FIELDS (ctx
->srecord_type
) = f1
;
2877 layout_type (ctx
->record_type
);
2878 fixup_child_record_type (ctx
);
2879 if (ctx
->srecord_type
)
2880 layout_type (ctx
->srecord_type
);
2881 tree t
= fold_convert_loc (loc
, long_integer_type_node
,
2882 TYPE_SIZE_UNIT (ctx
->record_type
));
2883 gimple_omp_task_set_arg_size (ctx
->stmt
, t
);
2884 t
= build_int_cst (long_integer_type_node
,
2885 TYPE_ALIGN_UNIT (ctx
->record_type
));
2886 gimple_omp_task_set_arg_align (ctx
->stmt
, t
);
2890 /* Find the enclosing offload context. */
2892 static omp_context
*
2893 enclosing_target_ctx (omp_context
*ctx
)
2895 for (; ctx
; ctx
= ctx
->outer
)
2896 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
)
2902 /* Return true if ctx is part of an oacc kernels region. */
2905 ctx_in_oacc_kernels_region (omp_context
*ctx
)
2907 for (;ctx
!= NULL
; ctx
= ctx
->outer
)
2909 gimple
*stmt
= ctx
->stmt
;
2910 if (gimple_code (stmt
) == GIMPLE_OMP_TARGET
2911 && gimple_omp_target_kind (stmt
) == GF_OMP_TARGET_KIND_OACC_KERNELS
)
2918 /* Check the parallelism clauses inside a kernels regions.
2919 Until kernels handling moves to use the same loop indirection
2920 scheme as parallel, we need to do this checking early. */
2923 check_oacc_kernel_gwv (gomp_for
*stmt
, omp_context
*ctx
)
2925 bool checking
= true;
2926 unsigned outer_mask
= 0;
2927 unsigned this_mask
= 0;
2928 bool has_seq
= false, has_auto
= false;
2931 outer_mask
= check_oacc_kernel_gwv (NULL
, ctx
->outer
);
2935 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
)
2937 stmt
= as_a
<gomp_for
*> (ctx
->stmt
);
2940 for (tree c
= gimple_omp_for_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
2942 switch (OMP_CLAUSE_CODE (c
))
2944 case OMP_CLAUSE_GANG
:
2945 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_GANG
);
2947 case OMP_CLAUSE_WORKER
:
2948 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_WORKER
);
2950 case OMP_CLAUSE_VECTOR
:
2951 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_VECTOR
);
2953 case OMP_CLAUSE_SEQ
:
2956 case OMP_CLAUSE_AUTO
:
2966 if (has_seq
&& (this_mask
|| has_auto
))
2967 error_at (gimple_location (stmt
), "%<seq%> overrides other"
2968 " OpenACC loop specifiers");
2969 else if (has_auto
&& this_mask
)
2970 error_at (gimple_location (stmt
), "%<auto%> conflicts with other"
2971 " OpenACC loop specifiers");
2973 if (this_mask
& outer_mask
)
2974 error_at (gimple_location (stmt
), "inner loop uses same"
2975 " OpenACC parallelism as containing loop");
2978 return outer_mask
| this_mask
;
2981 /* Scan a GIMPLE_OMP_FOR. */
2984 scan_omp_for (gomp_for
*stmt
, omp_context
*outer_ctx
)
2988 tree clauses
= gimple_omp_for_clauses (stmt
);
2990 ctx
= new_omp_context (stmt
, outer_ctx
);
2992 if (is_gimple_omp_oacc (stmt
))
2994 omp_context
*tgt
= enclosing_target_ctx (outer_ctx
);
2996 if (!tgt
|| is_oacc_parallel (tgt
))
2997 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2999 char const *check
= NULL
;
3001 switch (OMP_CLAUSE_CODE (c
))
3003 case OMP_CLAUSE_GANG
:
3007 case OMP_CLAUSE_WORKER
:
3011 case OMP_CLAUSE_VECTOR
:
3019 if (check
&& OMP_CLAUSE_OPERAND (c
, 0))
3020 error_at (gimple_location (stmt
),
3021 "argument not permitted on %qs clause in"
3022 " OpenACC %<parallel%>", check
);
3025 if (tgt
&& is_oacc_kernels (tgt
))
3027 /* Strip out reductions, as they are not handled yet. */
3028 tree
*prev_ptr
= &clauses
;
3030 while (tree probe
= *prev_ptr
)
3032 tree
*next_ptr
= &OMP_CLAUSE_CHAIN (probe
);
3034 if (OMP_CLAUSE_CODE (probe
) == OMP_CLAUSE_REDUCTION
)
3035 *prev_ptr
= *next_ptr
;
3037 prev_ptr
= next_ptr
;
3040 gimple_omp_for_set_clauses (stmt
, clauses
);
3041 check_oacc_kernel_gwv (stmt
, ctx
);
3045 scan_sharing_clauses (clauses
, ctx
);
3047 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
3048 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
3050 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
3051 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
3052 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
3053 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
3055 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3058 /* Scan an OpenMP sections directive. */
3061 scan_omp_sections (gomp_sections
*stmt
, omp_context
*outer_ctx
)
3065 ctx
= new_omp_context (stmt
, outer_ctx
);
3066 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
3067 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3070 /* Scan an OpenMP single directive. */
3073 scan_omp_single (gomp_single
*stmt
, omp_context
*outer_ctx
)
3078 ctx
= new_omp_context (stmt
, outer_ctx
);
3079 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
3080 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
3081 name
= create_tmp_var_name (".omp_copy_s");
3082 name
= build_decl (gimple_location (stmt
),
3083 TYPE_DECL
, name
, ctx
->record_type
);
3084 TYPE_NAME (ctx
->record_type
) = name
;
3086 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
3087 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3089 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
3090 ctx
->record_type
= NULL
;
3092 layout_type (ctx
->record_type
);
3095 /* Return true if the CLAUSES of an omp target guarantee that the base pointers
3096 used in the corresponding offloaded function are restrict. */
3099 omp_target_base_pointers_restrict_p (tree clauses
)
3101 /* The analysis relies on the GOMP_MAP_FORCE_* mapping kinds, which are only
3103 if (flag_openacc
== 0)
3106 /* I. Basic example:
3110 unsigned int a[2], b[2];
3112 #pragma acc kernels \
3121 After gimplification, we have:
3123 #pragma omp target oacc_kernels \
3124 map(force_from:a [len: 8]) \
3125 map(force_from:b [len: 8])
3131 Because both mappings have the force prefix, we know that they will be
3132 allocated when calling the corresponding offloaded function, which means we
3133 can mark the base pointers for a and b in the offloaded function as
3137 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3139 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
)
3142 switch (OMP_CLAUSE_MAP_KIND (c
))
3144 case GOMP_MAP_FORCE_ALLOC
:
3145 case GOMP_MAP_FORCE_TO
:
3146 case GOMP_MAP_FORCE_FROM
:
3147 case GOMP_MAP_FORCE_TOFROM
:
3157 /* Scan a GIMPLE_OMP_TARGET. */
3160 scan_omp_target (gomp_target
*stmt
, omp_context
*outer_ctx
)
3164 bool offloaded
= is_gimple_omp_offloaded (stmt
);
3165 tree clauses
= gimple_omp_target_clauses (stmt
);
3167 ctx
= new_omp_context (stmt
, outer_ctx
);
3168 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
3169 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
3170 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
3171 name
= create_tmp_var_name (".omp_data_t");
3172 name
= build_decl (gimple_location (stmt
),
3173 TYPE_DECL
, name
, ctx
->record_type
);
3174 DECL_ARTIFICIAL (name
) = 1;
3175 DECL_NAMELESS (name
) = 1;
3176 TYPE_NAME (ctx
->record_type
) = name
;
3177 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
3179 bool base_pointers_restrict
= false;
3182 create_omp_child_function (ctx
, false);
3183 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
3185 base_pointers_restrict
= omp_target_base_pointers_restrict_p (clauses
);
3186 if (base_pointers_restrict
3187 && dump_file
&& (dump_flags
& TDF_DETAILS
))
3189 "Base pointers in offloaded function are restrict\n");
3192 scan_sharing_clauses (clauses
, ctx
, base_pointers_restrict
);
3193 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3195 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
3196 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
3199 TYPE_FIELDS (ctx
->record_type
)
3200 = nreverse (TYPE_FIELDS (ctx
->record_type
));
3203 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
3204 for (tree field
= TYPE_FIELDS (ctx
->record_type
);
3206 field
= DECL_CHAIN (field
))
3207 gcc_assert (DECL_ALIGN (field
) == align
);
3209 layout_type (ctx
->record_type
);
3211 fixup_child_record_type (ctx
);
3215 /* Scan an OpenMP teams directive. */
3218 scan_omp_teams (gomp_teams
*stmt
, omp_context
*outer_ctx
)
3220 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
3221 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
3222 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3225 /* Check nesting restrictions. */
3227 check_omp_nesting_restrictions (gimple
*stmt
, omp_context
*ctx
)
3231 if (ctx
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_GRID_BODY
)
3232 /* GRID_BODY is an artificial construct, nesting rules will be checked in
3233 the original copy of its contents. */
3236 /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
3237 inside an OpenACC CTX. */
3238 if (!(is_gimple_omp (stmt
)
3239 && is_gimple_omp_oacc (stmt
)))
3241 for (omp_context
*octx
= ctx
; octx
!= NULL
; octx
= octx
->outer
)
3242 if (is_gimple_omp (octx
->stmt
)
3243 && is_gimple_omp_oacc (octx
->stmt
)
3244 /* Except for atomic codes that we share with OpenMP. */
3245 && ! (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
3246 || gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
))
3248 error_at (gimple_location (stmt
),
3249 "non-OpenACC construct inside of OpenACC region");
3256 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3257 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3260 if (gimple_code (stmt
) == GIMPLE_OMP_ORDERED
)
3262 c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3263 if (find_omp_clause (c
, OMP_CLAUSE_SIMD
))
3265 if (find_omp_clause (c
, OMP_CLAUSE_THREADS
)
3266 && (ctx
->outer
== NULL
3267 || !gimple_omp_for_combined_into_p (ctx
->stmt
)
3268 || gimple_code (ctx
->outer
->stmt
) != GIMPLE_OMP_FOR
3269 || (gimple_omp_for_kind (ctx
->outer
->stmt
)
3270 != GF_OMP_FOR_KIND_FOR
)
3271 || !gimple_omp_for_combined_p (ctx
->outer
->stmt
)))
3273 error_at (gimple_location (stmt
),
3274 "%<ordered simd threads%> must be closely "
3275 "nested inside of %<for simd%> region");
3281 error_at (gimple_location (stmt
),
3282 "OpenMP constructs other than %<#pragma omp ordered simd%>"
3283 " may not be nested inside %<simd%> region");
3286 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3288 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
3289 || (gimple_omp_for_kind (stmt
)
3290 != GF_OMP_FOR_KIND_DISTRIBUTE
))
3291 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
3293 error_at (gimple_location (stmt
),
3294 "only %<distribute%> or %<parallel%> regions are "
3295 "allowed to be strictly nested inside %<teams%> "
3301 switch (gimple_code (stmt
))
3303 case GIMPLE_OMP_FOR
:
3304 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
3306 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
3308 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
3310 error_at (gimple_location (stmt
),
3311 "%<distribute%> region must be strictly nested "
3312 "inside %<teams%> construct");
3317 /* We split taskloop into task and nested taskloop in it. */
3318 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
3320 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
3325 switch (gimple_code (ctx
->stmt
))
3327 case GIMPLE_OMP_FOR
:
3328 ok
= (gimple_omp_for_kind (ctx
->stmt
)
3329 == GF_OMP_FOR_KIND_OACC_LOOP
);
3332 case GIMPLE_OMP_TARGET
:
3333 switch (gimple_omp_target_kind (ctx
->stmt
))
3335 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
3336 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
3347 else if (get_oacc_fn_attrib (current_function_decl
))
3351 error_at (gimple_location (stmt
),
3352 "OpenACC loop directive must be associated with"
3353 " an OpenACC compute region");
3359 if (is_gimple_call (stmt
)
3360 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3361 == BUILT_IN_GOMP_CANCEL
3362 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3363 == BUILT_IN_GOMP_CANCELLATION_POINT
))
3365 const char *bad
= NULL
;
3366 const char *kind
= NULL
;
3367 const char *construct
3368 = (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3369 == BUILT_IN_GOMP_CANCEL
)
3370 ? "#pragma omp cancel"
3371 : "#pragma omp cancellation point";
3374 error_at (gimple_location (stmt
), "orphaned %qs construct",
3378 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
3379 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
3383 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
3384 bad
= "#pragma omp parallel";
3385 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3386 == BUILT_IN_GOMP_CANCEL
3387 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3388 ctx
->cancellable
= true;
3392 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3393 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
3394 bad
= "#pragma omp for";
3395 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3396 == BUILT_IN_GOMP_CANCEL
3397 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3399 ctx
->cancellable
= true;
3400 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3402 warning_at (gimple_location (stmt
), 0,
3403 "%<#pragma omp cancel for%> inside "
3404 "%<nowait%> for construct");
3405 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3406 OMP_CLAUSE_ORDERED
))
3407 warning_at (gimple_location (stmt
), 0,
3408 "%<#pragma omp cancel for%> inside "
3409 "%<ordered%> for construct");
3414 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
3415 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
3416 bad
= "#pragma omp sections";
3417 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3418 == BUILT_IN_GOMP_CANCEL
3419 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3421 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
3423 ctx
->cancellable
= true;
3424 if (find_omp_clause (gimple_omp_sections_clauses
3427 warning_at (gimple_location (stmt
), 0,
3428 "%<#pragma omp cancel sections%> inside "
3429 "%<nowait%> sections construct");
3433 gcc_assert (ctx
->outer
3434 && gimple_code (ctx
->outer
->stmt
)
3435 == GIMPLE_OMP_SECTIONS
);
3436 ctx
->outer
->cancellable
= true;
3437 if (find_omp_clause (gimple_omp_sections_clauses
3440 warning_at (gimple_location (stmt
), 0,
3441 "%<#pragma omp cancel sections%> inside "
3442 "%<nowait%> sections construct");
3448 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
3449 bad
= "#pragma omp task";
3452 for (omp_context
*octx
= ctx
->outer
;
3453 octx
; octx
= octx
->outer
)
3455 switch (gimple_code (octx
->stmt
))
3457 case GIMPLE_OMP_TASKGROUP
:
3459 case GIMPLE_OMP_TARGET
:
3460 if (gimple_omp_target_kind (octx
->stmt
)
3461 != GF_OMP_TARGET_KIND_REGION
)
3464 case GIMPLE_OMP_PARALLEL
:
3465 case GIMPLE_OMP_TEAMS
:
3466 error_at (gimple_location (stmt
),
3467 "%<%s taskgroup%> construct not closely "
3468 "nested inside of %<taskgroup%> region",
3476 ctx
->cancellable
= true;
3481 error_at (gimple_location (stmt
), "invalid arguments");
3486 error_at (gimple_location (stmt
),
3487 "%<%s %s%> construct not closely nested inside of %qs",
3488 construct
, kind
, bad
);
3493 case GIMPLE_OMP_SECTIONS
:
3494 case GIMPLE_OMP_SINGLE
:
3495 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3496 switch (gimple_code (ctx
->stmt
))
3498 case GIMPLE_OMP_FOR
:
3499 if (gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
3500 && gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_TASKLOOP
)
3503 case GIMPLE_OMP_SECTIONS
:
3504 case GIMPLE_OMP_SINGLE
:
3505 case GIMPLE_OMP_ORDERED
:
3506 case GIMPLE_OMP_MASTER
:
3507 case GIMPLE_OMP_TASK
:
3508 case GIMPLE_OMP_CRITICAL
:
3509 if (is_gimple_call (stmt
))
3511 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3512 != BUILT_IN_GOMP_BARRIER
)
3514 error_at (gimple_location (stmt
),
3515 "barrier region may not be closely nested inside "
3516 "of work-sharing, %<critical%>, %<ordered%>, "
3517 "%<master%>, explicit %<task%> or %<taskloop%> "
3521 error_at (gimple_location (stmt
),
3522 "work-sharing region may not be closely nested inside "
3523 "of work-sharing, %<critical%>, %<ordered%>, "
3524 "%<master%>, explicit %<task%> or %<taskloop%> region");
3526 case GIMPLE_OMP_PARALLEL
:
3527 case GIMPLE_OMP_TEAMS
:
3529 case GIMPLE_OMP_TARGET
:
3530 if (gimple_omp_target_kind (ctx
->stmt
)
3531 == GF_OMP_TARGET_KIND_REGION
)
3538 case GIMPLE_OMP_MASTER
:
3539 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3540 switch (gimple_code (ctx
->stmt
))
3542 case GIMPLE_OMP_FOR
:
3543 if (gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
3544 && gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_TASKLOOP
)
3547 case GIMPLE_OMP_SECTIONS
:
3548 case GIMPLE_OMP_SINGLE
:
3549 case GIMPLE_OMP_TASK
:
3550 error_at (gimple_location (stmt
),
3551 "%<master%> region may not be closely nested inside "
3552 "of work-sharing, explicit %<task%> or %<taskloop%> "
3555 case GIMPLE_OMP_PARALLEL
:
3556 case GIMPLE_OMP_TEAMS
:
3558 case GIMPLE_OMP_TARGET
:
3559 if (gimple_omp_target_kind (ctx
->stmt
)
3560 == GF_OMP_TARGET_KIND_REGION
)
3567 case GIMPLE_OMP_TASK
:
3568 for (c
= gimple_omp_task_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
3569 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
3570 && (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
3571 || OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
))
3573 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3574 error_at (OMP_CLAUSE_LOCATION (c
),
3575 "%<depend(%s)%> is only allowed in %<omp ordered%>",
3576 kind
== OMP_CLAUSE_DEPEND_SOURCE
? "source" : "sink");
3580 case GIMPLE_OMP_ORDERED
:
3581 for (c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3582 c
; c
= OMP_CLAUSE_CHAIN (c
))
3584 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_DEPEND
)
3586 gcc_assert (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_THREADS
3587 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SIMD
);
3590 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3591 if (kind
== OMP_CLAUSE_DEPEND_SOURCE
3592 || kind
== OMP_CLAUSE_DEPEND_SINK
)
3595 /* Look for containing ordered(N) loop. */
3597 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3599 = find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3600 OMP_CLAUSE_ORDERED
)) == NULL_TREE
)
3602 error_at (OMP_CLAUSE_LOCATION (c
),
3603 "%<ordered%> construct with %<depend%> clause "
3604 "must be closely nested inside an %<ordered%> "
3608 else if (OMP_CLAUSE_ORDERED_EXPR (oclause
) == NULL_TREE
)
3610 error_at (OMP_CLAUSE_LOCATION (c
),
3611 "%<ordered%> construct with %<depend%> clause "
3612 "must be closely nested inside a loop with "
3613 "%<ordered%> clause with a parameter");
3619 error_at (OMP_CLAUSE_LOCATION (c
),
3620 "invalid depend kind in omp %<ordered%> %<depend%>");
3624 c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3625 if (find_omp_clause (c
, OMP_CLAUSE_SIMD
))
3627 /* ordered simd must be closely nested inside of simd region,
3628 and simd region must not encounter constructs other than
3629 ordered simd, therefore ordered simd may be either orphaned,
3630 or ctx->stmt must be simd. The latter case is handled already
3634 error_at (gimple_location (stmt
),
3635 "%<ordered%> %<simd%> must be closely nested inside "
3640 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3641 switch (gimple_code (ctx
->stmt
))
3643 case GIMPLE_OMP_CRITICAL
:
3644 case GIMPLE_OMP_TASK
:
3645 case GIMPLE_OMP_ORDERED
:
3646 ordered_in_taskloop
:
3647 error_at (gimple_location (stmt
),
3648 "%<ordered%> region may not be closely nested inside "
3649 "of %<critical%>, %<ordered%>, explicit %<task%> or "
3650 "%<taskloop%> region");
3652 case GIMPLE_OMP_FOR
:
3653 if (gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
3654 goto ordered_in_taskloop
;
3655 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3656 OMP_CLAUSE_ORDERED
) == NULL
)
3658 error_at (gimple_location (stmt
),
3659 "%<ordered%> region must be closely nested inside "
3660 "a loop region with an %<ordered%> clause");
3664 case GIMPLE_OMP_TARGET
:
3665 if (gimple_omp_target_kind (ctx
->stmt
)
3666 != GF_OMP_TARGET_KIND_REGION
)
3669 case GIMPLE_OMP_PARALLEL
:
3670 case GIMPLE_OMP_TEAMS
:
3671 error_at (gimple_location (stmt
),
3672 "%<ordered%> region must be closely nested inside "
3673 "a loop region with an %<ordered%> clause");
3679 case GIMPLE_OMP_CRITICAL
:
3682 = gimple_omp_critical_name (as_a
<gomp_critical
*> (stmt
));
3683 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3684 if (gomp_critical
*other_crit
3685 = dyn_cast
<gomp_critical
*> (ctx
->stmt
))
3686 if (this_stmt_name
== gimple_omp_critical_name (other_crit
))
3688 error_at (gimple_location (stmt
),
3689 "%<critical%> region may not be nested inside "
3690 "a %<critical%> region with the same name");
3695 case GIMPLE_OMP_TEAMS
:
3697 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
3698 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
3700 error_at (gimple_location (stmt
),
3701 "%<teams%> construct not closely nested inside of "
3702 "%<target%> construct");
3706 case GIMPLE_OMP_TARGET
:
3707 for (c
= gimple_omp_target_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
3708 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
3709 && (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
3710 || OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
))
3712 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3713 error_at (OMP_CLAUSE_LOCATION (c
),
3714 "%<depend(%s)%> is only allowed in %<omp ordered%>",
3715 kind
== OMP_CLAUSE_DEPEND_SOURCE
? "source" : "sink");
3718 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3720 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
)
3722 if (is_gimple_omp (stmt
)
3723 && is_gimple_omp_oacc (stmt
)
3724 && is_gimple_omp (ctx
->stmt
))
3726 error_at (gimple_location (stmt
),
3727 "OpenACC construct inside of non-OpenACC region");
3733 const char *stmt_name
, *ctx_stmt_name
;
3734 switch (gimple_omp_target_kind (stmt
))
3736 case GF_OMP_TARGET_KIND_REGION
: stmt_name
= "target"; break;
3737 case GF_OMP_TARGET_KIND_DATA
: stmt_name
= "target data"; break;
3738 case GF_OMP_TARGET_KIND_UPDATE
: stmt_name
= "target update"; break;
3739 case GF_OMP_TARGET_KIND_ENTER_DATA
:
3740 stmt_name
= "target enter data"; break;
3741 case GF_OMP_TARGET_KIND_EXIT_DATA
:
3742 stmt_name
= "target exit data"; break;
3743 case GF_OMP_TARGET_KIND_OACC_PARALLEL
: stmt_name
= "parallel"; break;
3744 case GF_OMP_TARGET_KIND_OACC_KERNELS
: stmt_name
= "kernels"; break;
3745 case GF_OMP_TARGET_KIND_OACC_DATA
: stmt_name
= "data"; break;
3746 case GF_OMP_TARGET_KIND_OACC_UPDATE
: stmt_name
= "update"; break;
3747 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
3748 stmt_name
= "enter/exit data"; break;
3749 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
: stmt_name
= "host_data";
3751 default: gcc_unreachable ();
3753 switch (gimple_omp_target_kind (ctx
->stmt
))
3755 case GF_OMP_TARGET_KIND_REGION
: ctx_stmt_name
= "target"; break;
3756 case GF_OMP_TARGET_KIND_DATA
: ctx_stmt_name
= "target data"; break;
3757 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
3758 ctx_stmt_name
= "parallel"; break;
3759 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
3760 ctx_stmt_name
= "kernels"; break;
3761 case GF_OMP_TARGET_KIND_OACC_DATA
: ctx_stmt_name
= "data"; break;
3762 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
3763 ctx_stmt_name
= "host_data"; break;
3764 default: gcc_unreachable ();
3767 /* OpenACC/OpenMP mismatch? */
3768 if (is_gimple_omp_oacc (stmt
)
3769 != is_gimple_omp_oacc (ctx
->stmt
))
3771 error_at (gimple_location (stmt
),
3772 "%s %qs construct inside of %s %qs region",
3773 (is_gimple_omp_oacc (stmt
)
3774 ? "OpenACC" : "OpenMP"), stmt_name
,
3775 (is_gimple_omp_oacc (ctx
->stmt
)
3776 ? "OpenACC" : "OpenMP"), ctx_stmt_name
);
3779 if (is_gimple_omp_offloaded (ctx
->stmt
))
3781 /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */
3782 if (is_gimple_omp_oacc (ctx
->stmt
))
3784 error_at (gimple_location (stmt
),
3785 "%qs construct inside of %qs region",
3786 stmt_name
, ctx_stmt_name
);
3791 warning_at (gimple_location (stmt
), 0,
3792 "%qs construct inside of %qs region",
3793 stmt_name
, ctx_stmt_name
);
3805 /* Helper function scan_omp.
3807 Callback for walk_tree or operators in walk_gimple_stmt used to
3808 scan for OMP directives in TP. */
3811 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
3813 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
3814 omp_context
*ctx
= (omp_context
*) wi
->info
;
3817 switch (TREE_CODE (t
))
3825 tree repl
= remap_decl (t
, &ctx
->cb
);
3826 gcc_checking_assert (TREE_CODE (repl
) != ERROR_MARK
);
3832 if (ctx
&& TYPE_P (t
))
3833 *tp
= remap_type (t
, &ctx
->cb
);
3834 else if (!DECL_P (t
))
3839 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
3840 if (tem
!= TREE_TYPE (t
))
3842 if (TREE_CODE (t
) == INTEGER_CST
)
3843 *tp
= wide_int_to_tree (tem
, t
);
3845 TREE_TYPE (t
) = tem
;
3855 /* Return true if FNDECL is a setjmp or a longjmp. */
3858 setjmp_or_longjmp_p (const_tree fndecl
)
3860 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
3861 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
3862 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
3865 tree declname
= DECL_NAME (fndecl
);
3868 const char *name
= IDENTIFIER_POINTER (declname
);
3869 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
3873 /* Helper function for scan_omp.
3875 Callback for walk_gimple_stmt used to scan for OMP directives in
3876 the current statement in GSI. */
3879 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
3880 struct walk_stmt_info
*wi
)
3882 gimple
*stmt
= gsi_stmt (*gsi
);
3883 omp_context
*ctx
= (omp_context
*) wi
->info
;
3885 if (gimple_has_location (stmt
))
3886 input_location
= gimple_location (stmt
);
3888 /* Check the nesting restrictions. */
3889 bool remove
= false;
3890 if (is_gimple_omp (stmt
))
3891 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3892 else if (is_gimple_call (stmt
))
3894 tree fndecl
= gimple_call_fndecl (stmt
);
3897 if (setjmp_or_longjmp_p (fndecl
)
3899 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3900 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3903 error_at (gimple_location (stmt
),
3904 "setjmp/longjmp inside simd construct");
3906 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
3907 switch (DECL_FUNCTION_CODE (fndecl
))
3909 case BUILT_IN_GOMP_BARRIER
:
3910 case BUILT_IN_GOMP_CANCEL
:
3911 case BUILT_IN_GOMP_CANCELLATION_POINT
:
3912 case BUILT_IN_GOMP_TASKYIELD
:
3913 case BUILT_IN_GOMP_TASKWAIT
:
3914 case BUILT_IN_GOMP_TASKGROUP_START
:
3915 case BUILT_IN_GOMP_TASKGROUP_END
:
3916 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3925 stmt
= gimple_build_nop ();
3926 gsi_replace (gsi
, stmt
, false);
3929 *handled_ops_p
= true;
3931 switch (gimple_code (stmt
))
3933 case GIMPLE_OMP_PARALLEL
:
3934 taskreg_nesting_level
++;
3935 scan_omp_parallel (gsi
, ctx
);
3936 taskreg_nesting_level
--;
3939 case GIMPLE_OMP_TASK
:
3940 taskreg_nesting_level
++;
3941 scan_omp_task (gsi
, ctx
);
3942 taskreg_nesting_level
--;
3945 case GIMPLE_OMP_FOR
:
3946 scan_omp_for (as_a
<gomp_for
*> (stmt
), ctx
);
3949 case GIMPLE_OMP_SECTIONS
:
3950 scan_omp_sections (as_a
<gomp_sections
*> (stmt
), ctx
);
3953 case GIMPLE_OMP_SINGLE
:
3954 scan_omp_single (as_a
<gomp_single
*> (stmt
), ctx
);
3957 case GIMPLE_OMP_SECTION
:
3958 case GIMPLE_OMP_MASTER
:
3959 case GIMPLE_OMP_TASKGROUP
:
3960 case GIMPLE_OMP_ORDERED
:
3961 case GIMPLE_OMP_CRITICAL
:
3962 case GIMPLE_OMP_GRID_BODY
:
3963 ctx
= new_omp_context (stmt
, ctx
);
3964 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3967 case GIMPLE_OMP_TARGET
:
3968 scan_omp_target (as_a
<gomp_target
*> (stmt
), ctx
);
3971 case GIMPLE_OMP_TEAMS
:
3972 scan_omp_teams (as_a
<gomp_teams
*> (stmt
), ctx
);
3979 *handled_ops_p
= false;
3981 for (var
= gimple_bind_vars (as_a
<gbind
*> (stmt
));
3983 var
= DECL_CHAIN (var
))
3984 insert_decl_map (&ctx
->cb
, var
, var
);
3988 *handled_ops_p
= false;
3996 /* Scan all the statements starting at the current statement. CTX
3997 contains context information about the OMP directives and
3998 clauses found during the scan. */
4001 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
4003 location_t saved_location
;
4004 struct walk_stmt_info wi
;
4006 memset (&wi
, 0, sizeof (wi
));
4008 wi
.want_locations
= true;
4010 saved_location
= input_location
;
4011 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
4012 input_location
= saved_location
;
4015 /* Re-gimplification and code generation routines. */
4017 /* Build a call to GOMP_barrier. */
4020 build_omp_barrier (tree lhs
)
4022 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
4023 : BUILT_IN_GOMP_BARRIER
);
4024 gcall
*g
= gimple_build_call (fndecl
, 0);
4026 gimple_call_set_lhs (g
, lhs
);
4030 /* If a context was created for STMT when it was scanned, return it. */
4032 static omp_context
*
4033 maybe_lookup_ctx (gimple
*stmt
)
4036 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
4037 return n
? (omp_context
*) n
->value
: NULL
;
4041 /* Find the mapping for DECL in CTX or the immediately enclosing
4042 context that has a mapping for DECL.
4044 If CTX is a nested parallel directive, we may have to use the decl
4045 mappings created in CTX's parent context. Suppose that we have the
4046 following parallel nesting (variable UIDs showed for clarity):
4049 #omp parallel shared(iD.1562) -> outer parallel
4050 iD.1562 = iD.1562 + 1;
4052 #omp parallel shared (iD.1562) -> inner parallel
4053 iD.1562 = iD.1562 - 1;
4055 Each parallel structure will create a distinct .omp_data_s structure
4056 for copying iD.1562 in/out of the directive:
4058 outer parallel .omp_data_s.1.i -> iD.1562
4059 inner parallel .omp_data_s.2.i -> iD.1562
4061 A shared variable mapping will produce a copy-out operation before
4062 the parallel directive and a copy-in operation after it. So, in
4063 this case we would have:
4066 .omp_data_o.1.i = iD.1562;
4067 #omp parallel shared(iD.1562) -> outer parallel
4068 .omp_data_i.1 = &.omp_data_o.1
4069 .omp_data_i.1->i = .omp_data_i.1->i + 1;
4071 .omp_data_o.2.i = iD.1562; -> **
4072 #omp parallel shared(iD.1562) -> inner parallel
4073 .omp_data_i.2 = &.omp_data_o.2
4074 .omp_data_i.2->i = .omp_data_i.2->i - 1;
4077 ** This is a problem. The symbol iD.1562 cannot be referenced
4078 inside the body of the outer parallel region. But since we are
4079 emitting this copy operation while expanding the inner parallel
4080 directive, we need to access the CTX structure of the outer
4081 parallel directive to get the correct mapping:
4083 .omp_data_o.2.i = .omp_data_i.1->i
4085 Since there may be other workshare or parallel directives enclosing
4086 the parallel directive, it may be necessary to walk up the context
4087 parent chain. This is not a problem in general because nested
4088 parallelism happens only rarely. */
4091 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
4096 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
4097 t
= maybe_lookup_decl (decl
, up
);
4099 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
4101 return t
? t
: decl
;
4105 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
4106 in outer contexts. */
4109 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
4114 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
4115 t
= maybe_lookup_decl (decl
, up
);
4117 return t
? t
: decl
;
4121 /* Construct the initialization value for reduction operation OP. */
4124 omp_reduction_init_op (location_t loc
, enum tree_code op
, tree type
)
4133 case TRUTH_ORIF_EXPR
:
4134 case TRUTH_XOR_EXPR
:
4136 return build_zero_cst (type
);
4139 case TRUTH_AND_EXPR
:
4140 case TRUTH_ANDIF_EXPR
:
4142 return fold_convert_loc (loc
, type
, integer_one_node
);
4145 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
4148 if (SCALAR_FLOAT_TYPE_P (type
))
4150 REAL_VALUE_TYPE max
, min
;
4151 if (HONOR_INFINITIES (type
))
4154 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
4157 real_maxval (&min
, 1, TYPE_MODE (type
));
4158 return build_real (type
, min
);
4160 else if (POINTER_TYPE_P (type
))
4163 = wi::min_value (TYPE_PRECISION (type
), TYPE_SIGN (type
));
4164 return wide_int_to_tree (type
, min
);
4168 gcc_assert (INTEGRAL_TYPE_P (type
));
4169 return TYPE_MIN_VALUE (type
);
4173 if (SCALAR_FLOAT_TYPE_P (type
))
4175 REAL_VALUE_TYPE max
;
4176 if (HONOR_INFINITIES (type
))
4179 real_maxval (&max
, 0, TYPE_MODE (type
));
4180 return build_real (type
, max
);
4182 else if (POINTER_TYPE_P (type
))
4185 = wi::max_value (TYPE_PRECISION (type
), TYPE_SIGN (type
));
4186 return wide_int_to_tree (type
, max
);
4190 gcc_assert (INTEGRAL_TYPE_P (type
));
4191 return TYPE_MAX_VALUE (type
);
4199 /* Construct the initialization value for reduction CLAUSE. */
4202 omp_reduction_init (tree clause
, tree type
)
4204 return omp_reduction_init_op (OMP_CLAUSE_LOCATION (clause
),
4205 OMP_CLAUSE_REDUCTION_CODE (clause
), type
);
4208 /* Return alignment to be assumed for var in CLAUSE, which should be
4209 OMP_CLAUSE_ALIGNED. */
4212 omp_clause_aligned_alignment (tree clause
)
4214 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
4215 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
4217 /* Otherwise return implementation defined alignment. */
4218 unsigned int al
= 1;
4219 machine_mode mode
, vmode
;
4220 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
4222 vs
= 1 << floor_log2 (vs
);
4223 static enum mode_class classes
[]
4224 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
4225 for (int i
= 0; i
< 4; i
+= 2)
4226 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
4228 mode
= GET_MODE_WIDER_MODE (mode
))
4230 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
4231 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
4234 && GET_MODE_SIZE (vmode
) < vs
4235 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
4236 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
4238 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
4239 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
4241 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
4242 / GET_MODE_SIZE (mode
));
4243 if (TYPE_MODE (type
) != vmode
)
4245 if (TYPE_ALIGN_UNIT (type
) > al
)
4246 al
= TYPE_ALIGN_UNIT (type
);
4248 return build_int_cst (integer_type_node
, al
);
4251 /* Return maximum possible vectorization factor for the target. */
4258 || !flag_tree_loop_optimize
4259 || (!flag_tree_loop_vectorize
4260 && (global_options_set
.x_flag_tree_loop_vectorize
4261 || global_options_set
.x_flag_tree_vectorize
)))
4264 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
4267 vs
= 1 << floor_log2 (vs
);
4270 machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
4271 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
4272 return GET_MODE_NUNITS (vqimode
);
4276 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
4280 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
4281 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
4285 max_vf
= omp_max_vf ();
4288 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
4289 OMP_CLAUSE_SAFELEN
);
4290 if (c
&& TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) != INTEGER_CST
)
4292 else if (c
&& compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
4294 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
4298 idx
= create_tmp_var (unsigned_type_node
);
4299 lane
= create_tmp_var (unsigned_type_node
);
4305 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
4306 tree avar
= create_tmp_var_raw (atype
);
4307 if (TREE_ADDRESSABLE (new_var
))
4308 TREE_ADDRESSABLE (avar
) = 1;
4309 DECL_ATTRIBUTES (avar
)
4310 = tree_cons (get_identifier ("omp simd array"), NULL
,
4311 DECL_ATTRIBUTES (avar
));
4312 gimple_add_tmp_var (avar
);
4313 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
4314 NULL_TREE
, NULL_TREE
);
4315 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
4316 NULL_TREE
, NULL_TREE
);
4317 if (DECL_P (new_var
))
4319 SET_DECL_VALUE_EXPR (new_var
, lvar
);
4320 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4325 /* Helper function of lower_rec_input_clauses. For a reference
4326 in simd reduction, add an underlying variable it will reference. */
4329 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
4331 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
4332 if (TREE_CONSTANT (z
))
4334 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)),
4335 get_name (new_vard
));
4336 gimple_add_tmp_var (z
);
4337 TREE_ADDRESSABLE (z
) = 1;
4338 z
= build_fold_addr_expr_loc (loc
, z
);
4339 gimplify_assign (new_vard
, z
, ilist
);
4343 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
4344 from the receiver (aka child) side and initializers for REFERENCE_TYPE
4345 private variables. Initialization statements go in ILIST, while calls
4346 to destructors go in DLIST. */
4349 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
4350 omp_context
*ctx
, struct omp_for_data
*fd
)
4352 tree c
, dtor
, copyin_seq
, x
, ptr
;
4353 bool copyin_by_ref
= false;
4354 bool lastprivate_firstprivate
= false;
4355 bool reduction_omp_orig_ref
= false;
4357 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4358 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
);
4360 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
4361 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
4362 gimple_seq llist
[2] = { NULL
, NULL
};
4366 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
4367 with data sharing clauses referencing variable sized vars. That
4368 is unnecessarily hard to support and very unlikely to result in
4369 vectorized code anyway. */
4371 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4372 switch (OMP_CLAUSE_CODE (c
))
4374 case OMP_CLAUSE_LINEAR
:
4375 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
4378 case OMP_CLAUSE_PRIVATE
:
4379 case OMP_CLAUSE_FIRSTPRIVATE
:
4380 case OMP_CLAUSE_LASTPRIVATE
:
4381 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
4384 case OMP_CLAUSE_REDUCTION
:
4385 if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
4386 || is_variable_sized (OMP_CLAUSE_DECL (c
)))
4393 /* Do all the fixed sized types in the first pass, and the variable sized
4394 types in the second pass. This makes sure that the scalar arguments to
4395 the variable sized types are processed before we use them in the
4396 variable sized operations. */
4397 for (pass
= 0; pass
< 2; ++pass
)
4399 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4401 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
4404 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4408 case OMP_CLAUSE_PRIVATE
:
4409 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
4412 case OMP_CLAUSE_SHARED
:
4413 /* Ignore shared directives in teams construct. */
4414 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
4416 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
4418 gcc_assert (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
)
4419 || is_global_var (OMP_CLAUSE_DECL (c
)));
4422 case OMP_CLAUSE_FIRSTPRIVATE
:
4423 case OMP_CLAUSE_COPYIN
:
4425 case OMP_CLAUSE_LINEAR
:
4426 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
)
4427 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
4428 lastprivate_firstprivate
= true;
4430 case OMP_CLAUSE_REDUCTION
:
4431 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
4432 reduction_omp_orig_ref
= true;
4434 case OMP_CLAUSE__LOOPTEMP_
:
4435 /* Handle _looptemp_ clauses only on parallel/task. */
4439 case OMP_CLAUSE_LASTPRIVATE
:
4440 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4442 lastprivate_firstprivate
= true;
4443 if (pass
!= 0 || is_taskloop_ctx (ctx
))
4446 /* Even without corresponding firstprivate, if
4447 decl is Fortran allocatable, it needs outer var
4450 && lang_hooks
.decls
.omp_private_outer_ref
4451 (OMP_CLAUSE_DECL (c
)))
4452 lastprivate_firstprivate
= true;
4454 case OMP_CLAUSE_ALIGNED
:
4457 var
= OMP_CLAUSE_DECL (c
);
4458 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
4459 && !is_global_var (var
))
4461 new_var
= maybe_lookup_decl (var
, ctx
);
4462 if (new_var
== NULL_TREE
)
4463 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
4464 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
4465 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
4466 omp_clause_aligned_alignment (c
));
4467 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4468 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
4469 gimplify_and_add (x
, ilist
);
4471 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
4472 && is_global_var (var
))
4474 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
4475 new_var
= lookup_decl (var
, ctx
);
4476 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
4477 t
= build_fold_addr_expr_loc (clause_loc
, t
);
4478 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
4479 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
4480 omp_clause_aligned_alignment (c
));
4481 t
= fold_convert_loc (clause_loc
, ptype
, t
);
4482 x
= create_tmp_var (ptype
);
4483 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
4484 gimplify_and_add (t
, ilist
);
4485 t
= build_simple_mem_ref_loc (clause_loc
, x
);
4486 SET_DECL_VALUE_EXPR (new_var
, t
);
4487 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4494 new_var
= var
= OMP_CLAUSE_DECL (c
);
4495 if (c_kind
== OMP_CLAUSE_REDUCTION
&& TREE_CODE (var
) == MEM_REF
)
4497 var
= TREE_OPERAND (var
, 0);
4498 if (TREE_CODE (var
) == POINTER_PLUS_EXPR
)
4499 var
= TREE_OPERAND (var
, 0);
4500 if (TREE_CODE (var
) == INDIRECT_REF
4501 || TREE_CODE (var
) == ADDR_EXPR
)
4502 var
= TREE_OPERAND (var
, 0);
4503 if (is_variable_sized (var
))
4505 gcc_assert (DECL_HAS_VALUE_EXPR_P (var
));
4506 var
= DECL_VALUE_EXPR (var
);
4507 gcc_assert (TREE_CODE (var
) == INDIRECT_REF
);
4508 var
= TREE_OPERAND (var
, 0);
4509 gcc_assert (DECL_P (var
));
4513 if (c_kind
!= OMP_CLAUSE_COPYIN
)
4514 new_var
= lookup_decl (var
, ctx
);
4516 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
4521 /* C/C++ array section reductions. */
4522 else if (c_kind
== OMP_CLAUSE_REDUCTION
4523 && var
!= OMP_CLAUSE_DECL (c
))
4528 tree bias
= TREE_OPERAND (OMP_CLAUSE_DECL (c
), 1);
4529 tree orig_var
= TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0);
4530 if (TREE_CODE (orig_var
) == POINTER_PLUS_EXPR
)
4532 tree b
= TREE_OPERAND (orig_var
, 1);
4533 b
= maybe_lookup_decl (b
, ctx
);
4536 b
= TREE_OPERAND (orig_var
, 1);
4537 b
= maybe_lookup_decl_in_outer_ctx (b
, ctx
);
4539 if (integer_zerop (bias
))
4543 bias
= fold_convert_loc (clause_loc
,
4544 TREE_TYPE (b
), bias
);
4545 bias
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
4546 TREE_TYPE (b
), b
, bias
);
4548 orig_var
= TREE_OPERAND (orig_var
, 0);
4550 if (TREE_CODE (orig_var
) == INDIRECT_REF
4551 || TREE_CODE (orig_var
) == ADDR_EXPR
)
4552 orig_var
= TREE_OPERAND (orig_var
, 0);
4553 tree d
= OMP_CLAUSE_DECL (c
);
4554 tree type
= TREE_TYPE (d
);
4555 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
4556 tree v
= TYPE_MAX_VALUE (TYPE_DOMAIN (type
));
4557 const char *name
= get_name (orig_var
);
4558 if (TREE_CONSTANT (v
))
4560 x
= create_tmp_var_raw (type
, name
);
4561 gimple_add_tmp_var (x
);
4562 TREE_ADDRESSABLE (x
) = 1;
4563 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4568 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4569 tree t
= maybe_lookup_decl (v
, ctx
);
4573 v
= maybe_lookup_decl_in_outer_ctx (v
, ctx
);
4574 gimplify_expr (&v
, ilist
, NULL
, is_gimple_val
, fb_rvalue
);
4575 t
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
4577 build_int_cst (TREE_TYPE (v
), 1));
4578 t
= fold_build2_loc (clause_loc
, MULT_EXPR
,
4580 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4581 tree al
= size_int (TYPE_ALIGN (TREE_TYPE (type
)));
4582 x
= build_call_expr_loc (clause_loc
, atmp
, 2, t
, al
);
4585 tree ptype
= build_pointer_type (TREE_TYPE (type
));
4586 x
= fold_convert_loc (clause_loc
, ptype
, x
);
4587 tree y
= create_tmp_var (ptype
, name
);
4588 gimplify_assign (y
, x
, ilist
);
4592 if (!integer_zerop (bias
))
4594 bias
= fold_convert_loc (clause_loc
, pointer_sized_int_node
,
4596 yb
= fold_convert_loc (clause_loc
, pointer_sized_int_node
,
4598 yb
= fold_build2_loc (clause_loc
, MINUS_EXPR
,
4599 pointer_sized_int_node
, yb
, bias
);
4600 x
= fold_convert_loc (clause_loc
, TREE_TYPE (x
), yb
);
4601 yb
= create_tmp_var (ptype
, name
);
4602 gimplify_assign (yb
, x
, ilist
);
4606 d
= TREE_OPERAND (d
, 0);
4607 if (TREE_CODE (d
) == POINTER_PLUS_EXPR
)
4608 d
= TREE_OPERAND (d
, 0);
4609 if (TREE_CODE (d
) == ADDR_EXPR
)
4611 if (orig_var
!= var
)
4613 gcc_assert (is_variable_sized (orig_var
));
4614 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
),
4616 gimplify_assign (new_var
, x
, ilist
);
4617 tree new_orig_var
= lookup_decl (orig_var
, ctx
);
4618 tree t
= build_fold_indirect_ref (new_var
);
4619 DECL_IGNORED_P (new_var
) = 0;
4620 TREE_THIS_NOTRAP (t
);
4621 SET_DECL_VALUE_EXPR (new_orig_var
, t
);
4622 DECL_HAS_VALUE_EXPR_P (new_orig_var
) = 1;
4626 x
= build2 (MEM_REF
, TREE_TYPE (new_var
), x
,
4627 build_int_cst (ptype
, 0));
4628 SET_DECL_VALUE_EXPR (new_var
, x
);
4629 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4634 gcc_assert (orig_var
== var
);
4635 if (TREE_CODE (d
) == INDIRECT_REF
)
4637 x
= create_tmp_var (ptype
, name
);
4638 TREE_ADDRESSABLE (x
) = 1;
4639 gimplify_assign (x
, yb
, ilist
);
4640 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4642 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4643 gimplify_assign (new_var
, x
, ilist
);
4645 tree y1
= create_tmp_var (ptype
, NULL
);
4646 gimplify_assign (y1
, y
, ilist
);
4647 tree i2
= NULL_TREE
, y2
= NULL_TREE
;
4648 tree body2
= NULL_TREE
, end2
= NULL_TREE
;
4649 tree y3
= NULL_TREE
, y4
= NULL_TREE
;
4650 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) || is_simd
)
4652 y2
= create_tmp_var (ptype
, NULL
);
4653 gimplify_assign (y2
, y
, ilist
);
4654 tree ref
= build_outer_var_ref (var
, ctx
);
4655 /* For ref build_outer_var_ref already performs this. */
4656 if (TREE_CODE (d
) == INDIRECT_REF
)
4657 gcc_assert (is_reference (var
));
4658 else if (TREE_CODE (d
) == ADDR_EXPR
)
4659 ref
= build_fold_addr_expr (ref
);
4660 else if (is_reference (var
))
4661 ref
= build_fold_addr_expr (ref
);
4662 ref
= fold_convert_loc (clause_loc
, ptype
, ref
);
4663 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
)
4664 && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
4666 y3
= create_tmp_var (ptype
, NULL
);
4667 gimplify_assign (y3
, unshare_expr (ref
), ilist
);
4671 y4
= create_tmp_var (ptype
, NULL
);
4672 gimplify_assign (y4
, ref
, dlist
);
4675 tree i
= create_tmp_var (TREE_TYPE (v
), NULL
);
4676 gimplify_assign (i
, build_int_cst (TREE_TYPE (v
), 0), ilist
);
4677 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
4678 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
4679 gimple_seq_add_stmt (ilist
, gimple_build_label (body
));
4682 i2
= create_tmp_var (TREE_TYPE (v
), NULL
);
4683 gimplify_assign (i2
, build_int_cst (TREE_TYPE (v
), 0), dlist
);
4684 body2
= create_artificial_label (UNKNOWN_LOCATION
);
4685 end2
= create_artificial_label (UNKNOWN_LOCATION
);
4686 gimple_seq_add_stmt (dlist
, gimple_build_label (body2
));
4688 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4690 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4691 tree decl_placeholder
4692 = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
);
4693 SET_DECL_VALUE_EXPR (decl_placeholder
,
4694 build_simple_mem_ref (y1
));
4695 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 1;
4696 SET_DECL_VALUE_EXPR (placeholder
,
4697 y3
? build_simple_mem_ref (y3
)
4699 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4700 x
= lang_hooks
.decls
.omp_clause_default_ctor
4701 (c
, build_simple_mem_ref (y1
),
4702 y3
? build_simple_mem_ref (y3
) : NULL_TREE
);
4704 gimplify_and_add (x
, ilist
);
4705 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4707 gimple_seq tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4708 lower_omp (&tseq
, ctx
);
4709 gimple_seq_add_seq (ilist
, tseq
);
4711 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4714 SET_DECL_VALUE_EXPR (decl_placeholder
,
4715 build_simple_mem_ref (y2
));
4716 SET_DECL_VALUE_EXPR (placeholder
,
4717 build_simple_mem_ref (y4
));
4718 gimple_seq tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4719 lower_omp (&tseq
, ctx
);
4720 gimple_seq_add_seq (dlist
, tseq
);
4721 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4723 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4724 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 0;
4725 x
= lang_hooks
.decls
.omp_clause_dtor
4726 (c
, build_simple_mem_ref (y2
));
4729 gimple_seq tseq
= NULL
;
4731 gimplify_stmt (&dtor
, &tseq
);
4732 gimple_seq_add_seq (dlist
, tseq
);
4737 x
= omp_reduction_init (c
, TREE_TYPE (type
));
4738 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4740 /* reduction(-:var) sums up the partial results, so it
4741 acts identically to reduction(+:var). */
4742 if (code
== MINUS_EXPR
)
4745 gimplify_assign (build_simple_mem_ref (y1
), x
, ilist
);
4748 x
= build2 (code
, TREE_TYPE (type
),
4749 build_simple_mem_ref (y4
),
4750 build_simple_mem_ref (y2
));
4751 gimplify_assign (build_simple_mem_ref (y4
), x
, dlist
);
4755 = gimple_build_assign (y1
, POINTER_PLUS_EXPR
, y1
,
4756 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4757 gimple_seq_add_stmt (ilist
, g
);
4760 g
= gimple_build_assign (y3
, POINTER_PLUS_EXPR
, y3
,
4761 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4762 gimple_seq_add_stmt (ilist
, g
);
4764 g
= gimple_build_assign (i
, PLUS_EXPR
, i
,
4765 build_int_cst (TREE_TYPE (i
), 1));
4766 gimple_seq_add_stmt (ilist
, g
);
4767 g
= gimple_build_cond (LE_EXPR
, i
, v
, body
, end
);
4768 gimple_seq_add_stmt (ilist
, g
);
4769 gimple_seq_add_stmt (ilist
, gimple_build_label (end
));
4772 g
= gimple_build_assign (y2
, POINTER_PLUS_EXPR
, y2
,
4773 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4774 gimple_seq_add_stmt (dlist
, g
);
4777 g
= gimple_build_assign
4778 (y4
, POINTER_PLUS_EXPR
, y4
,
4779 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4780 gimple_seq_add_stmt (dlist
, g
);
4782 g
= gimple_build_assign (i2
, PLUS_EXPR
, i2
,
4783 build_int_cst (TREE_TYPE (i2
), 1));
4784 gimple_seq_add_stmt (dlist
, g
);
4785 g
= gimple_build_cond (LE_EXPR
, i2
, v
, body2
, end2
);
4786 gimple_seq_add_stmt (dlist
, g
);
4787 gimple_seq_add_stmt (dlist
, gimple_build_label (end2
));
4791 else if (is_variable_sized (var
))
4793 /* For variable sized types, we need to allocate the
4794 actual storage here. Call alloca and store the
4795 result in the pointer decl that we created elsewhere. */
4799 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
4804 ptr
= DECL_VALUE_EXPR (new_var
);
4805 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
4806 ptr
= TREE_OPERAND (ptr
, 0);
4807 gcc_assert (DECL_P (ptr
));
4808 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
4810 /* void *tmp = __builtin_alloca */
4811 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4812 stmt
= gimple_build_call (atmp
, 2, x
,
4813 size_int (DECL_ALIGN (var
)));
4814 tmp
= create_tmp_var_raw (ptr_type_node
);
4815 gimple_add_tmp_var (tmp
);
4816 gimple_call_set_lhs (stmt
, tmp
);
4818 gimple_seq_add_stmt (ilist
, stmt
);
4820 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
4821 gimplify_assign (ptr
, x
, ilist
);
4824 else if (is_reference (var
) && !is_oacc_parallel (ctx
))
4826 /* For references that are being privatized for Fortran,
4827 allocate new backing storage for the new pointer
4828 variable. This allows us to avoid changing all the
4829 code that expects a pointer to something that expects
4830 a direct variable. */
4834 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
4835 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
4837 x
= build_receiver_ref (var
, false, ctx
);
4838 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4840 else if (TREE_CONSTANT (x
))
4842 /* For reduction in SIMD loop, defer adding the
4843 initialization of the reference, because if we decide
4844 to use SIMD array for it, the initilization could cause
4846 if (c_kind
== OMP_CLAUSE_REDUCTION
&& is_simd
)
4850 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
4852 gimple_add_tmp_var (x
);
4853 TREE_ADDRESSABLE (x
) = 1;
4854 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4860 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4861 tree rtype
= TREE_TYPE (TREE_TYPE (new_var
));
4862 tree al
= size_int (TYPE_ALIGN (rtype
));
4863 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
4868 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4869 gimplify_assign (new_var
, x
, ilist
);
4872 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4874 else if (c_kind
== OMP_CLAUSE_REDUCTION
4875 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4883 switch (OMP_CLAUSE_CODE (c
))
4885 case OMP_CLAUSE_SHARED
:
4886 /* Ignore shared directives in teams construct. */
4887 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
4889 /* Shared global vars are just accessed directly. */
4890 if (is_global_var (new_var
))
4892 /* For taskloop firstprivate/lastprivate, represented
4893 as firstprivate and shared clause on the task, new_var
4894 is the firstprivate var. */
4895 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
4897 /* Set up the DECL_VALUE_EXPR for shared variables now. This
4898 needs to be delayed until after fixup_child_record_type so
4899 that we get the correct type during the dereference. */
4900 by_ref
= use_pointer_for_field (var
, ctx
);
4901 x
= build_receiver_ref (var
, by_ref
, ctx
);
4902 SET_DECL_VALUE_EXPR (new_var
, x
);
4903 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4905 /* ??? If VAR is not passed by reference, and the variable
4906 hasn't been initialized yet, then we'll get a warning for
4907 the store into the omp_data_s structure. Ideally, we'd be
4908 able to notice this and not store anything at all, but
4909 we're generating code too early. Suppress the warning. */
4911 TREE_NO_WARNING (var
) = 1;
4914 case OMP_CLAUSE_LASTPRIVATE
:
4915 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4919 case OMP_CLAUSE_PRIVATE
:
4920 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
4921 x
= build_outer_var_ref (var
, ctx
);
4922 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4924 if (is_task_ctx (ctx
))
4925 x
= build_receiver_ref (var
, false, ctx
);
4927 x
= build_outer_var_ref (var
, ctx
);
4933 nx
= lang_hooks
.decls
.omp_clause_default_ctor
4934 (c
, unshare_expr (new_var
), x
);
4937 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
4938 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
4939 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
4940 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4941 idx
, lane
, ivar
, lvar
))
4944 x
= lang_hooks
.decls
.omp_clause_default_ctor
4945 (c
, unshare_expr (ivar
), x
);
4947 gimplify_and_add (x
, &llist
[0]);
4950 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4953 gimple_seq tseq
= NULL
;
4956 gimplify_stmt (&dtor
, &tseq
);
4957 gimple_seq_add_seq (&llist
[1], tseq
);
4964 gimplify_and_add (nx
, ilist
);
4968 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
4971 gimple_seq tseq
= NULL
;
4974 gimplify_stmt (&dtor
, &tseq
);
4975 gimple_seq_add_seq (dlist
, tseq
);
4979 case OMP_CLAUSE_LINEAR
:
4980 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
4981 goto do_firstprivate
;
4982 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
4985 x
= build_outer_var_ref (var
, ctx
);
4988 case OMP_CLAUSE_FIRSTPRIVATE
:
4989 if (is_task_ctx (ctx
))
4991 if (is_reference (var
) || is_variable_sized (var
))
4993 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
4995 || use_pointer_for_field (var
, NULL
))
4997 x
= build_receiver_ref (var
, false, ctx
);
4998 SET_DECL_VALUE_EXPR (new_var
, x
);
4999 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
5004 x
= build_outer_var_ref (var
, ctx
);
5007 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
5008 && gimple_omp_for_combined_into_p (ctx
->stmt
))
5010 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
5011 tree stept
= TREE_TYPE (t
);
5012 tree ct
= find_omp_clause (clauses
,
5013 OMP_CLAUSE__LOOPTEMP_
);
5015 tree l
= OMP_CLAUSE_DECL (ct
);
5016 tree n1
= fd
->loop
.n1
;
5017 tree step
= fd
->loop
.step
;
5018 tree itype
= TREE_TYPE (l
);
5019 if (POINTER_TYPE_P (itype
))
5020 itype
= signed_type_for (itype
);
5021 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
5022 if (TYPE_UNSIGNED (itype
)
5023 && fd
->loop
.cond_code
== GT_EXPR
)
5024 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5025 fold_build1 (NEGATE_EXPR
, itype
, l
),
5026 fold_build1 (NEGATE_EXPR
,
5029 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
5030 t
= fold_build2 (MULT_EXPR
, stept
,
5031 fold_convert (stept
, l
), t
);
5033 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
5035 x
= lang_hooks
.decls
.omp_clause_linear_ctor
5037 gimplify_and_add (x
, ilist
);
5041 if (POINTER_TYPE_P (TREE_TYPE (x
)))
5042 x
= fold_build2 (POINTER_PLUS_EXPR
,
5043 TREE_TYPE (x
), x
, t
);
5045 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
5048 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
5049 || TREE_ADDRESSABLE (new_var
))
5050 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
5051 idx
, lane
, ivar
, lvar
))
5053 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
5055 tree iv
= create_tmp_var (TREE_TYPE (new_var
));
5056 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
5057 gimplify_and_add (x
, ilist
);
5058 gimple_stmt_iterator gsi
5059 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
5061 = gimple_build_assign (unshare_expr (lvar
), iv
);
5062 gsi_insert_before_without_update (&gsi
, g
,
5064 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
5065 enum tree_code code
= PLUS_EXPR
;
5066 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
5067 code
= POINTER_PLUS_EXPR
;
5068 g
= gimple_build_assign (iv
, code
, iv
, t
);
5069 gsi_insert_before_without_update (&gsi
, g
,
5073 x
= lang_hooks
.decls
.omp_clause_copy_ctor
5074 (c
, unshare_expr (ivar
), x
);
5075 gimplify_and_add (x
, &llist
[0]);
5076 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
5079 gimple_seq tseq
= NULL
;
5082 gimplify_stmt (&dtor
, &tseq
);
5083 gimple_seq_add_seq (&llist
[1], tseq
);
5088 x
= lang_hooks
.decls
.omp_clause_copy_ctor
5089 (c
, unshare_expr (new_var
), x
);
5090 gimplify_and_add (x
, ilist
);
5093 case OMP_CLAUSE__LOOPTEMP_
:
5094 gcc_assert (is_taskreg_ctx (ctx
));
5095 x
= build_outer_var_ref (var
, ctx
);
5096 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
5097 gimplify_and_add (x
, ilist
);
5100 case OMP_CLAUSE_COPYIN
:
5101 by_ref
= use_pointer_for_field (var
, NULL
);
5102 x
= build_receiver_ref (var
, by_ref
, ctx
);
5103 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
5104 append_to_statement_list (x
, ©in_seq
);
5105 copyin_by_ref
|= by_ref
;
5108 case OMP_CLAUSE_REDUCTION
:
5109 /* OpenACC reductions are initialized using the
5110 GOACC_REDUCTION internal function. */
5111 if (is_gimple_omp_oacc (ctx
->stmt
))
5113 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5115 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
5117 x
= build_outer_var_ref (var
, ctx
);
5119 if (is_reference (var
)
5120 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
5122 x
= build_fold_addr_expr_loc (clause_loc
, x
);
5123 SET_DECL_VALUE_EXPR (placeholder
, x
);
5124 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
5125 tree new_vard
= new_var
;
5126 if (is_reference (var
))
5128 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
5129 new_vard
= TREE_OPERAND (new_var
, 0);
5130 gcc_assert (DECL_P (new_vard
));
5133 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
5134 idx
, lane
, ivar
, lvar
))
5136 if (new_vard
== new_var
)
5138 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
5139 SET_DECL_VALUE_EXPR (new_var
, ivar
);
5143 SET_DECL_VALUE_EXPR (new_vard
,
5144 build_fold_addr_expr (ivar
));
5145 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
5147 x
= lang_hooks
.decls
.omp_clause_default_ctor
5148 (c
, unshare_expr (ivar
),
5149 build_outer_var_ref (var
, ctx
));
5151 gimplify_and_add (x
, &llist
[0]);
5152 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
5154 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
5155 lower_omp (&tseq
, ctx
);
5156 gimple_seq_add_seq (&llist
[0], tseq
);
5158 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
5159 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
5160 lower_omp (&tseq
, ctx
);
5161 gimple_seq_add_seq (&llist
[1], tseq
);
5162 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5163 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
5164 if (new_vard
== new_var
)
5165 SET_DECL_VALUE_EXPR (new_var
, lvar
);
5167 SET_DECL_VALUE_EXPR (new_vard
,
5168 build_fold_addr_expr (lvar
));
5169 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
5174 gimplify_stmt (&dtor
, &tseq
);
5175 gimple_seq_add_seq (&llist
[1], tseq
);
5179 /* If this is a reference to constant size reduction var
5180 with placeholder, we haven't emitted the initializer
5181 for it because it is undesirable if SIMD arrays are used.
5182 But if they aren't used, we need to emit the deferred
5183 initialization now. */
5184 else if (is_reference (var
) && is_simd
)
5185 handle_simd_reference (clause_loc
, new_vard
, ilist
);
5186 x
= lang_hooks
.decls
.omp_clause_default_ctor
5187 (c
, unshare_expr (new_var
),
5188 build_outer_var_ref (var
, ctx
));
5190 gimplify_and_add (x
, ilist
);
5191 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
5193 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
5194 lower_omp (&tseq
, ctx
);
5195 gimple_seq_add_seq (ilist
, tseq
);
5197 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
5200 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
5201 lower_omp (&tseq
, ctx
);
5202 gimple_seq_add_seq (dlist
, tseq
);
5203 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5205 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
5210 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
5211 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
5212 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
5214 /* reduction(-:var) sums up the partial results, so it
5215 acts identically to reduction(+:var). */
5216 if (code
== MINUS_EXPR
)
5219 tree new_vard
= new_var
;
5220 if (is_simd
&& is_reference (var
))
5222 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
5223 new_vard
= TREE_OPERAND (new_var
, 0);
5224 gcc_assert (DECL_P (new_vard
));
5227 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
5228 idx
, lane
, ivar
, lvar
))
5230 tree ref
= build_outer_var_ref (var
, ctx
);
5232 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
5234 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
5235 ref
= build_outer_var_ref (var
, ctx
);
5236 gimplify_assign (ref
, x
, &llist
[1]);
5238 if (new_vard
!= new_var
)
5240 SET_DECL_VALUE_EXPR (new_vard
,
5241 build_fold_addr_expr (lvar
));
5242 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
5247 if (is_reference (var
) && is_simd
)
5248 handle_simd_reference (clause_loc
, new_vard
, ilist
);
5249 gimplify_assign (new_var
, x
, ilist
);
5252 tree ref
= build_outer_var_ref (var
, ctx
);
5254 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
5255 ref
= build_outer_var_ref (var
, ctx
);
5256 gimplify_assign (ref
, x
, dlist
);
5270 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
5271 /* Don't want uninit warnings on simduid, it is always uninitialized,
5272 but we use it not for the value, but for the DECL_UID only. */
5273 TREE_NO_WARNING (uid
) = 1;
5275 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
5276 gimple_call_set_lhs (g
, lane
);
5277 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
5278 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
5279 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
5280 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
5281 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
5282 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
5283 g
= gimple_build_assign (lane
, INTEGER_CST
,
5284 build_int_cst (unsigned_type_node
, 0));
5285 gimple_seq_add_stmt (ilist
, g
);
5286 for (int i
= 0; i
< 2; i
++)
5289 tree vf
= create_tmp_var (unsigned_type_node
);
5290 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
5291 gimple_call_set_lhs (g
, vf
);
5292 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
5293 gimple_seq_add_stmt (seq
, g
);
5294 tree t
= build_int_cst (unsigned_type_node
, 0);
5295 g
= gimple_build_assign (idx
, INTEGER_CST
, t
);
5296 gimple_seq_add_stmt (seq
, g
);
5297 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
5298 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
5299 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
5300 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
5301 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
5302 gimple_seq_add_seq (seq
, llist
[i
]);
5303 t
= build_int_cst (unsigned_type_node
, 1);
5304 g
= gimple_build_assign (idx
, PLUS_EXPR
, idx
, t
);
5305 gimple_seq_add_stmt (seq
, g
);
5306 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
5307 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
5308 gimple_seq_add_stmt (seq
, g
);
5309 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
5313 /* The copyin sequence is not to be executed by the main thread, since
5314 that would result in self-copies. Perhaps not visible to scalars,
5315 but it certainly is to C++ operator=. */
5318 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
5320 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
5321 build_int_cst (TREE_TYPE (x
), 0));
5322 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
5323 gimplify_and_add (x
, ilist
);
5326 /* If any copyin variable is passed by reference, we must ensure the
5327 master thread doesn't modify it before it is copied over in all
5328 threads. Similarly for variables in both firstprivate and
5329 lastprivate clauses we need to ensure the lastprivate copying
5330 happens after firstprivate copying in all threads. And similarly
5331 for UDRs if initializer expression refers to omp_orig. */
5332 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
5334 /* Don't add any barrier for #pragma omp simd or
5335 #pragma omp distribute. */
5336 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
5337 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
5338 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
5341 /* If max_vf is non-zero, then we can use only a vectorization factor
5342 up to the max_vf we chose. So stick it into the safelen clause. */
5345 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
5346 OMP_CLAUSE_SAFELEN
);
5348 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) == INTEGER_CST
5349 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
5352 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
5353 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
5355 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
5356 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
5362 /* Generate code to implement the LASTPRIVATE clauses. This is used for
5363 both parallel and workshare constructs. PREDICATE may be NULL if it's
5367 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
5370 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
5371 bool par_clauses
= false;
5372 tree simduid
= NULL
, lastlane
= NULL
;
5374 /* Early exit if there are no lastprivate or linear clauses. */
5375 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
5376 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
5377 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
5378 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
5380 if (clauses
== NULL
)
5382 /* If this was a workshare clause, see if it had been combined
5383 with its parallel. In that case, look for the clauses on the
5384 parallel statement itself. */
5385 if (is_parallel_ctx (ctx
))
5389 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
5392 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
5393 OMP_CLAUSE_LASTPRIVATE
);
5394 if (clauses
== NULL
)
5402 tree label_true
, arm1
, arm2
;
5404 label
= create_artificial_label (UNKNOWN_LOCATION
);
5405 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
5406 arm1
= TREE_OPERAND (predicate
, 0);
5407 arm2
= TREE_OPERAND (predicate
, 1);
5408 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
5409 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
5410 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
5412 gimple_seq_add_stmt (stmt_list
, stmt
);
5413 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
5416 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
5417 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
5419 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
5421 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
5424 for (c
= clauses
; c
;)
5427 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5429 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5430 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
5431 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
5433 var
= OMP_CLAUSE_DECL (c
);
5434 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5435 && OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
)
5436 && is_taskloop_ctx (ctx
))
5438 gcc_checking_assert (ctx
->outer
&& is_task_ctx (ctx
->outer
));
5439 new_var
= lookup_decl (var
, ctx
->outer
);
5442 new_var
= lookup_decl (var
, ctx
);
5444 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
5446 tree val
= DECL_VALUE_EXPR (new_var
);
5447 if (TREE_CODE (val
) == ARRAY_REF
5448 && VAR_P (TREE_OPERAND (val
, 0))
5449 && lookup_attribute ("omp simd array",
5450 DECL_ATTRIBUTES (TREE_OPERAND (val
,
5453 if (lastlane
== NULL
)
5455 lastlane
= create_tmp_var (unsigned_type_node
);
5457 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
5459 TREE_OPERAND (val
, 1));
5460 gimple_call_set_lhs (g
, lastlane
);
5461 gimple_seq_add_stmt (stmt_list
, g
);
5463 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
5464 TREE_OPERAND (val
, 0), lastlane
,
5465 NULL_TREE
, NULL_TREE
);
5469 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5470 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
5472 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
5473 gimple_seq_add_seq (stmt_list
,
5474 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
5475 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
5477 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
5478 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
5480 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
5481 gimple_seq_add_seq (stmt_list
,
5482 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
5483 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
5487 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5488 && OMP_CLAUSE_LASTPRIVATE_TASKLOOP_IV (c
))
5490 gcc_checking_assert (is_taskloop_ctx (ctx
));
5491 tree ovar
= maybe_lookup_decl_in_outer_ctx (var
,
5493 if (is_global_var (ovar
))
5497 x
= build_outer_var_ref (var
, ctx
, true);
5498 if (is_reference (var
))
5499 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5500 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
5501 gimplify_and_add (x
, stmt_list
);
5503 c
= OMP_CLAUSE_CHAIN (c
);
5504 if (c
== NULL
&& !par_clauses
)
5506 /* If this was a workshare clause, see if it had been combined
5507 with its parallel. In that case, continue looking for the
5508 clauses also on the parallel statement itself. */
5509 if (is_parallel_ctx (ctx
))
5513 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
5516 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
5517 OMP_CLAUSE_LASTPRIVATE
);
5523 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
5526 /* Lower the OpenACC reductions of CLAUSES for compute axis LEVEL
5527 (which might be a placeholder). INNER is true if this is an inner
5528 axis of a multi-axis loop. FORK and JOIN are (optional) fork and
5529 join markers. Generate the before-loop forking sequence in
5530 FORK_SEQ and the after-loop joining sequence to JOIN_SEQ. The
5531 general form of these sequences is
5533 GOACC_REDUCTION_SETUP
5535 GOACC_REDUCTION_INIT
5537 GOACC_REDUCTION_FINI
5539 GOACC_REDUCTION_TEARDOWN. */
5542 lower_oacc_reductions (location_t loc
, tree clauses
, tree level
, bool inner
,
5543 gcall
*fork
, gcall
*join
, gimple_seq
*fork_seq
,
5544 gimple_seq
*join_seq
, omp_context
*ctx
)
5546 gimple_seq before_fork
= NULL
;
5547 gimple_seq after_fork
= NULL
;
5548 gimple_seq before_join
= NULL
;
5549 gimple_seq after_join
= NULL
;
5550 tree init_code
= NULL_TREE
, fini_code
= NULL_TREE
,
5551 setup_code
= NULL_TREE
, teardown_code
= NULL_TREE
;
5552 unsigned offset
= 0;
5554 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5555 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
5557 tree orig
= OMP_CLAUSE_DECL (c
);
5558 tree var
= maybe_lookup_decl (orig
, ctx
);
5559 tree ref_to_res
= NULL_TREE
;
5560 tree incoming
, outgoing
;
5562 enum tree_code rcode
= OMP_CLAUSE_REDUCTION_CODE (c
);
5563 if (rcode
== MINUS_EXPR
)
5565 else if (rcode
== TRUTH_ANDIF_EXPR
)
5566 rcode
= BIT_AND_EXPR
;
5567 else if (rcode
== TRUTH_ORIF_EXPR
)
5568 rcode
= BIT_IOR_EXPR
;
5569 tree op
= build_int_cst (unsigned_type_node
, rcode
);
5573 gcc_assert (!is_reference (var
));
5575 incoming
= outgoing
= var
;
5579 /* See if an outer construct also reduces this variable. */
5580 omp_context
*outer
= ctx
;
5582 while (omp_context
*probe
= outer
->outer
)
5584 enum gimple_code type
= gimple_code (probe
->stmt
);
5589 case GIMPLE_OMP_FOR
:
5590 cls
= gimple_omp_for_clauses (probe
->stmt
);
5593 case GIMPLE_OMP_TARGET
:
5594 if (gimple_omp_target_kind (probe
->stmt
)
5595 != GF_OMP_TARGET_KIND_OACC_PARALLEL
)
5598 cls
= gimple_omp_target_clauses (probe
->stmt
);
5606 for (; cls
; cls
= OMP_CLAUSE_CHAIN (cls
))
5607 if (OMP_CLAUSE_CODE (cls
) == OMP_CLAUSE_REDUCTION
5608 && orig
== OMP_CLAUSE_DECL (cls
))
5609 goto has_outer_reduction
;
5613 /* This is the outermost construct with this reduction,
5614 see if there's a mapping for it. */
5615 if (gimple_code (outer
->stmt
) == GIMPLE_OMP_TARGET
5616 && maybe_lookup_field (orig
, outer
))
5618 ref_to_res
= build_receiver_ref (orig
, false, outer
);
5619 if (is_reference (orig
))
5620 ref_to_res
= build_simple_mem_ref (ref_to_res
);
5623 incoming
= omp_reduction_init_op (loc
, rcode
, TREE_TYPE (var
));
5626 incoming
= outgoing
= orig
;
5628 has_outer_reduction
:;
5632 ref_to_res
= integer_zero_node
;
5634 /* Determine position in reduction buffer, which may be used
5636 enum machine_mode mode
= TYPE_MODE (TREE_TYPE (var
));
5637 unsigned align
= GET_MODE_ALIGNMENT (mode
) / BITS_PER_UNIT
;
5638 offset
= (offset
+ align
- 1) & ~(align
- 1);
5639 tree off
= build_int_cst (sizetype
, offset
);
5640 offset
+= GET_MODE_SIZE (mode
);
5644 init_code
= build_int_cst (integer_type_node
,
5645 IFN_GOACC_REDUCTION_INIT
);
5646 fini_code
= build_int_cst (integer_type_node
,
5647 IFN_GOACC_REDUCTION_FINI
);
5648 setup_code
= build_int_cst (integer_type_node
,
5649 IFN_GOACC_REDUCTION_SETUP
);
5650 teardown_code
= build_int_cst (integer_type_node
,
5651 IFN_GOACC_REDUCTION_TEARDOWN
);
5655 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5656 TREE_TYPE (var
), 6, setup_code
,
5657 unshare_expr (ref_to_res
),
5658 incoming
, level
, op
, off
);
5660 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5661 TREE_TYPE (var
), 6, init_code
,
5662 unshare_expr (ref_to_res
),
5663 var
, level
, op
, off
);
5665 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5666 TREE_TYPE (var
), 6, fini_code
,
5667 unshare_expr (ref_to_res
),
5668 var
, level
, op
, off
);
5670 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5671 TREE_TYPE (var
), 6, teardown_code
,
5672 ref_to_res
, var
, level
, op
, off
);
5674 gimplify_assign (var
, setup_call
, &before_fork
);
5675 gimplify_assign (var
, init_call
, &after_fork
);
5676 gimplify_assign (var
, fini_call
, &before_join
);
5677 gimplify_assign (outgoing
, teardown_call
, &after_join
);
5680 /* Now stitch things together. */
5681 gimple_seq_add_seq (fork_seq
, before_fork
);
5683 gimple_seq_add_stmt (fork_seq
, fork
);
5684 gimple_seq_add_seq (fork_seq
, after_fork
);
5686 gimple_seq_add_seq (join_seq
, before_join
);
5688 gimple_seq_add_stmt (join_seq
, join
);
5689 gimple_seq_add_seq (join_seq
, after_join
);
5692 /* Generate code to implement the REDUCTION clauses. */
5695 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
5697 gimple_seq sub_seq
= NULL
;
5702 /* OpenACC loop reductions are handled elsewhere. */
5703 if (is_gimple_omp_oacc (ctx
->stmt
))
5706 /* SIMD reductions are handled in lower_rec_input_clauses. */
5707 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
5708 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
5711 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
5712 update in that case, otherwise use a lock. */
5713 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
5714 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
5716 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
)
5717 || TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
)
5719 /* Never use OMP_ATOMIC for array reductions or UDRs. */
5729 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5731 tree var
, ref
, new_var
, orig_var
;
5732 enum tree_code code
;
5733 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5735 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
5738 orig_var
= var
= OMP_CLAUSE_DECL (c
);
5739 if (TREE_CODE (var
) == MEM_REF
)
5741 var
= TREE_OPERAND (var
, 0);
5742 if (TREE_CODE (var
) == POINTER_PLUS_EXPR
)
5743 var
= TREE_OPERAND (var
, 0);
5744 if (TREE_CODE (var
) == INDIRECT_REF
5745 || TREE_CODE (var
) == ADDR_EXPR
)
5746 var
= TREE_OPERAND (var
, 0);
5748 if (is_variable_sized (var
))
5750 gcc_assert (DECL_HAS_VALUE_EXPR_P (var
));
5751 var
= DECL_VALUE_EXPR (var
);
5752 gcc_assert (TREE_CODE (var
) == INDIRECT_REF
);
5753 var
= TREE_OPERAND (var
, 0);
5754 gcc_assert (DECL_P (var
));
5757 new_var
= lookup_decl (var
, ctx
);
5758 if (var
== OMP_CLAUSE_DECL (c
) && is_reference (var
))
5759 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5760 ref
= build_outer_var_ref (var
, ctx
);
5761 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
5763 /* reduction(-:var) sums up the partial results, so it acts
5764 identically to reduction(+:var). */
5765 if (code
== MINUS_EXPR
)
5770 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
5772 addr
= save_expr (addr
);
5773 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
5774 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
5775 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
5776 gimplify_and_add (x
, stmt_seqp
);
5779 else if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
)
5781 tree d
= OMP_CLAUSE_DECL (c
);
5782 tree type
= TREE_TYPE (d
);
5783 tree v
= TYPE_MAX_VALUE (TYPE_DOMAIN (type
));
5784 tree i
= create_tmp_var (TREE_TYPE (v
), NULL
);
5785 tree ptype
= build_pointer_type (TREE_TYPE (type
));
5786 tree bias
= TREE_OPERAND (d
, 1);
5787 d
= TREE_OPERAND (d
, 0);
5788 if (TREE_CODE (d
) == POINTER_PLUS_EXPR
)
5790 tree b
= TREE_OPERAND (d
, 1);
5791 b
= maybe_lookup_decl (b
, ctx
);
5794 b
= TREE_OPERAND (d
, 1);
5795 b
= maybe_lookup_decl_in_outer_ctx (b
, ctx
);
5797 if (integer_zerop (bias
))
5801 bias
= fold_convert_loc (clause_loc
, TREE_TYPE (b
), bias
);
5802 bias
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
5803 TREE_TYPE (b
), b
, bias
);
5805 d
= TREE_OPERAND (d
, 0);
5807 /* For ref build_outer_var_ref already performs this, so
5808 only new_var needs a dereference. */
5809 if (TREE_CODE (d
) == INDIRECT_REF
)
5811 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5812 gcc_assert (is_reference (var
) && var
== orig_var
);
5814 else if (TREE_CODE (d
) == ADDR_EXPR
)
5816 if (orig_var
== var
)
5818 new_var
= build_fold_addr_expr (new_var
);
5819 ref
= build_fold_addr_expr (ref
);
5824 gcc_assert (orig_var
== var
);
5825 if (is_reference (var
))
5826 ref
= build_fold_addr_expr (ref
);
5830 tree t
= maybe_lookup_decl (v
, ctx
);
5834 v
= maybe_lookup_decl_in_outer_ctx (v
, ctx
);
5835 gimplify_expr (&v
, stmt_seqp
, NULL
, is_gimple_val
, fb_rvalue
);
5837 if (!integer_zerop (bias
))
5839 bias
= fold_convert_loc (clause_loc
, sizetype
, bias
);
5840 new_var
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
5841 TREE_TYPE (new_var
), new_var
,
5842 unshare_expr (bias
));
5843 ref
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
5844 TREE_TYPE (ref
), ref
, bias
);
5846 new_var
= fold_convert_loc (clause_loc
, ptype
, new_var
);
5847 ref
= fold_convert_loc (clause_loc
, ptype
, ref
);
5848 tree m
= create_tmp_var (ptype
, NULL
);
5849 gimplify_assign (m
, new_var
, stmt_seqp
);
5851 m
= create_tmp_var (ptype
, NULL
);
5852 gimplify_assign (m
, ref
, stmt_seqp
);
5854 gimplify_assign (i
, build_int_cst (TREE_TYPE (v
), 0), stmt_seqp
);
5855 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
5856 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
5857 gimple_seq_add_stmt (&sub_seq
, gimple_build_label (body
));
5858 tree priv
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5859 tree out
= build_simple_mem_ref_loc (clause_loc
, ref
);
5860 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5862 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
5863 tree decl_placeholder
5864 = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
);
5865 SET_DECL_VALUE_EXPR (placeholder
, out
);
5866 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
5867 SET_DECL_VALUE_EXPR (decl_placeholder
, priv
);
5868 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 1;
5869 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
5870 gimple_seq_add_seq (&sub_seq
,
5871 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
5872 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5873 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
5874 OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
) = NULL
;
5878 x
= build2 (code
, TREE_TYPE (out
), out
, priv
);
5879 out
= unshare_expr (out
);
5880 gimplify_assign (out
, x
, &sub_seq
);
5882 gimple
*g
= gimple_build_assign (new_var
, POINTER_PLUS_EXPR
, new_var
,
5883 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
5884 gimple_seq_add_stmt (&sub_seq
, g
);
5885 g
= gimple_build_assign (ref
, POINTER_PLUS_EXPR
, ref
,
5886 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
5887 gimple_seq_add_stmt (&sub_seq
, g
);
5888 g
= gimple_build_assign (i
, PLUS_EXPR
, i
,
5889 build_int_cst (TREE_TYPE (i
), 1));
5890 gimple_seq_add_stmt (&sub_seq
, g
);
5891 g
= gimple_build_cond (LE_EXPR
, i
, v
, body
, end
);
5892 gimple_seq_add_stmt (&sub_seq
, g
);
5893 gimple_seq_add_stmt (&sub_seq
, gimple_build_label (end
));
5895 else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5897 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
5899 if (is_reference (var
)
5900 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
5902 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
5903 SET_DECL_VALUE_EXPR (placeholder
, ref
);
5904 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
5905 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
5906 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
5907 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5908 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
5912 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
5913 ref
= build_outer_var_ref (var
, ctx
);
5914 gimplify_assign (ref
, x
, &sub_seq
);
5918 if (is_gimple_omp_oacc (ctx
->stmt
))
5921 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
5923 gimple_seq_add_stmt (stmt_seqp
, stmt
);
5925 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
5927 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
5929 gimple_seq_add_stmt (stmt_seqp
, stmt
);
5933 /* Generate code to implement the COPYPRIVATE clauses. */
5936 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
5941 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5943 tree var
, new_var
, ref
, x
;
5945 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5947 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
5950 var
= OMP_CLAUSE_DECL (c
);
5951 by_ref
= use_pointer_for_field (var
, NULL
);
5953 ref
= build_sender_ref (var
, ctx
);
5954 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
5957 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
5958 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
5960 gimplify_assign (ref
, x
, slist
);
5962 ref
= build_receiver_ref (var
, false, ctx
);
5965 ref
= fold_convert_loc (clause_loc
,
5966 build_pointer_type (TREE_TYPE (new_var
)),
5968 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
5970 if (is_reference (var
))
5972 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
5973 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
5974 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5976 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
5977 gimplify_and_add (x
, rlist
);
5982 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
5983 and REDUCTION from the sender (aka parent) side. */
5986 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
5990 int ignored_looptemp
= 0;
5991 bool is_taskloop
= false;
5993 /* For taskloop, ignore first two _looptemp_ clauses, those are initialized
5994 by GOMP_taskloop. */
5995 if (is_task_ctx (ctx
) && gimple_omp_task_taskloop_p (ctx
->stmt
))
5997 ignored_looptemp
= 2;
6001 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
6003 tree val
, ref
, x
, var
;
6004 bool by_ref
, do_in
= false, do_out
= false;
6005 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
6007 switch (OMP_CLAUSE_CODE (c
))
6009 case OMP_CLAUSE_PRIVATE
:
6010 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
6013 case OMP_CLAUSE_FIRSTPRIVATE
:
6014 case OMP_CLAUSE_COPYIN
:
6015 case OMP_CLAUSE_LASTPRIVATE
:
6016 case OMP_CLAUSE_REDUCTION
:
6018 case OMP_CLAUSE_SHARED
:
6019 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
6022 case OMP_CLAUSE__LOOPTEMP_
:
6023 if (ignored_looptemp
)
6033 val
= OMP_CLAUSE_DECL (c
);
6034 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
6035 && TREE_CODE (val
) == MEM_REF
)
6037 val
= TREE_OPERAND (val
, 0);
6038 if (TREE_CODE (val
) == POINTER_PLUS_EXPR
)
6039 val
= TREE_OPERAND (val
, 0);
6040 if (TREE_CODE (val
) == INDIRECT_REF
6041 || TREE_CODE (val
) == ADDR_EXPR
)
6042 val
= TREE_OPERAND (val
, 0);
6043 if (is_variable_sized (val
))
6047 /* For OMP_CLAUSE_SHARED_FIRSTPRIVATE, look beyond the
6048 outer taskloop region. */
6049 omp_context
*ctx_for_o
= ctx
;
6051 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
6052 && OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
6053 ctx_for_o
= ctx
->outer
;
6055 var
= lookup_decl_in_outer_ctx (val
, ctx_for_o
);
6057 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
6058 && is_global_var (var
))
6061 t
= omp_member_access_dummy_var (var
);
6064 var
= DECL_VALUE_EXPR (var
);
6065 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx_for_o
);
6067 var
= unshare_and_remap (var
, t
, o
);
6069 var
= unshare_expr (var
);
6072 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
)
6074 /* Handle taskloop firstprivate/lastprivate, where the
6075 lastprivate on GIMPLE_OMP_TASK is represented as
6076 OMP_CLAUSE_SHARED_FIRSTPRIVATE. */
6077 tree f
= lookup_sfield ((splay_tree_key
) &DECL_UID (val
), ctx
);
6078 x
= omp_build_component_ref (ctx
->sender_decl
, f
);
6079 if (use_pointer_for_field (val
, ctx
))
6080 var
= build_fold_addr_expr (var
);
6081 gimplify_assign (x
, var
, ilist
);
6082 DECL_ABSTRACT_ORIGIN (f
) = NULL
;
6086 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
6087 || val
== OMP_CLAUSE_DECL (c
))
6088 && is_variable_sized (val
))
6090 by_ref
= use_pointer_for_field (val
, NULL
);
6092 switch (OMP_CLAUSE_CODE (c
))
6094 case OMP_CLAUSE_PRIVATE
:
6095 case OMP_CLAUSE_FIRSTPRIVATE
:
6096 case OMP_CLAUSE_COPYIN
:
6097 case OMP_CLAUSE__LOOPTEMP_
:
6101 case OMP_CLAUSE_LASTPRIVATE
:
6102 if (by_ref
|| is_reference (val
))
6104 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
6111 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
6116 case OMP_CLAUSE_REDUCTION
:
6118 if (val
== OMP_CLAUSE_DECL (c
))
6119 do_out
= !(by_ref
|| is_reference (val
));
6121 by_ref
= TREE_CODE (TREE_TYPE (val
)) == ARRAY_TYPE
;
6130 ref
= build_sender_ref (val
, ctx
);
6131 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
6132 gimplify_assign (ref
, x
, ilist
);
6133 if (is_task_ctx (ctx
))
6134 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
6139 ref
= build_sender_ref (val
, ctx
);
6140 gimplify_assign (var
, ref
, olist
);
6145 /* Generate code to implement SHARED from the sender (aka parent)
6146 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
6147 list things that got automatically shared. */
6150 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
6152 tree var
, ovar
, nvar
, t
, f
, x
, record_type
;
6154 if (ctx
->record_type
== NULL
)
6157 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
6158 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
6160 ovar
= DECL_ABSTRACT_ORIGIN (f
);
6161 if (!ovar
|| TREE_CODE (ovar
) == FIELD_DECL
)
6164 nvar
= maybe_lookup_decl (ovar
, ctx
);
6165 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
6168 /* If CTX is a nested parallel directive. Find the immediately
6169 enclosing parallel or workshare construct that contains a
6170 mapping for OVAR. */
6171 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
6173 t
= omp_member_access_dummy_var (var
);
6176 var
= DECL_VALUE_EXPR (var
);
6177 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx
);
6179 var
= unshare_and_remap (var
, t
, o
);
6181 var
= unshare_expr (var
);
6184 if (use_pointer_for_field (ovar
, ctx
))
6186 x
= build_sender_ref (ovar
, ctx
);
6187 var
= build_fold_addr_expr (var
);
6188 gimplify_assign (x
, var
, ilist
);
6192 x
= build_sender_ref (ovar
, ctx
);
6193 gimplify_assign (x
, var
, ilist
);
6195 if (!TREE_READONLY (var
)
6196 /* We don't need to receive a new reference to a result
6197 or parm decl. In fact we may not store to it as we will
6198 invalidate any pending RSO and generate wrong gimple
6200 && !((TREE_CODE (var
) == RESULT_DECL
6201 || TREE_CODE (var
) == PARM_DECL
)
6202 && DECL_BY_REFERENCE (var
)))
6204 x
= build_sender_ref (ovar
, ctx
);
6205 gimplify_assign (var
, x
, olist
);
6211 /* Emit an OpenACC head marker call, encapulating the partitioning and
6212 other information that must be processed by the target compiler.
6213 Return the maximum number of dimensions the associated loop might
6214 be partitioned over. */
6217 lower_oacc_head_mark (location_t loc
, tree ddvar
, tree clauses
,
6218 gimple_seq
*seq
, omp_context
*ctx
)
6220 unsigned levels
= 0;
6222 tree gang_static
= NULL_TREE
;
6223 auto_vec
<tree
, 5> args
;
6225 args
.quick_push (build_int_cst
6226 (integer_type_node
, IFN_UNIQUE_OACC_HEAD_MARK
));
6227 args
.quick_push (ddvar
);
6228 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
6230 switch (OMP_CLAUSE_CODE (c
))
6232 case OMP_CLAUSE_GANG
:
6233 tag
|= OLF_DIM_GANG
;
6234 gang_static
= OMP_CLAUSE_GANG_STATIC_EXPR (c
);
6235 /* static:* is represented by -1, and we can ignore it, as
6236 scheduling is always static. */
6237 if (gang_static
&& integer_minus_onep (gang_static
))
6238 gang_static
= NULL_TREE
;
6242 case OMP_CLAUSE_WORKER
:
6243 tag
|= OLF_DIM_WORKER
;
6247 case OMP_CLAUSE_VECTOR
:
6248 tag
|= OLF_DIM_VECTOR
;
6252 case OMP_CLAUSE_SEQ
:
6256 case OMP_CLAUSE_AUTO
:
6260 case OMP_CLAUSE_INDEPENDENT
:
6261 tag
|= OLF_INDEPENDENT
;
6271 if (DECL_P (gang_static
))
6272 gang_static
= build_outer_var_ref (gang_static
, ctx
);
6273 tag
|= OLF_GANG_STATIC
;
6276 /* In a parallel region, loops are implicitly INDEPENDENT. */
6277 omp_context
*tgt
= enclosing_target_ctx (ctx
);
6278 if (!tgt
|| is_oacc_parallel (tgt
))
6279 tag
|= OLF_INDEPENDENT
;
6281 /* A loop lacking SEQ, GANG, WORKER and/or VECTOR is implicitly AUTO. */
6282 if (!(tag
& (((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1) << OLF_DIM_BASE
)
6286 /* Ensure at least one level. */
6290 args
.quick_push (build_int_cst (integer_type_node
, levels
));
6291 args
.quick_push (build_int_cst (integer_type_node
, tag
));
6293 args
.quick_push (gang_static
);
6295 gcall
*call
= gimple_build_call_internal_vec (IFN_UNIQUE
, args
);
6296 gimple_set_location (call
, loc
);
6297 gimple_set_lhs (call
, ddvar
);
6298 gimple_seq_add_stmt (seq
, call
);
6303 /* Emit an OpenACC lopp head or tail marker to SEQ. LEVEL is the
6304 partitioning level of the enclosed region. */
6307 lower_oacc_loop_marker (location_t loc
, tree ddvar
, bool head
,
6308 tree tofollow
, gimple_seq
*seq
)
6310 int marker_kind
= (head
? IFN_UNIQUE_OACC_HEAD_MARK
6311 : IFN_UNIQUE_OACC_TAIL_MARK
);
6312 tree marker
= build_int_cst (integer_type_node
, marker_kind
);
6313 int nargs
= 2 + (tofollow
!= NULL_TREE
);
6314 gcall
*call
= gimple_build_call_internal (IFN_UNIQUE
, nargs
,
6315 marker
, ddvar
, tofollow
);
6316 gimple_set_location (call
, loc
);
6317 gimple_set_lhs (call
, ddvar
);
6318 gimple_seq_add_stmt (seq
, call
);
6321 /* Generate the before and after OpenACC loop sequences. CLAUSES are
6322 the loop clauses, from which we extract reductions. Initialize
6326 lower_oacc_head_tail (location_t loc
, tree clauses
,
6327 gimple_seq
*head
, gimple_seq
*tail
, omp_context
*ctx
)
6330 tree ddvar
= create_tmp_var (integer_type_node
, ".data_dep");
6331 gimple_seq_add_stmt (head
, gimple_build_assign (ddvar
, integer_zero_node
));
6333 unsigned count
= lower_oacc_head_mark (loc
, ddvar
, clauses
, head
, ctx
);
6335 lower_oacc_loop_marker (loc
, ddvar
, false, integer_zero_node
, tail
);
6337 tree fork_kind
= build_int_cst (unsigned_type_node
, IFN_UNIQUE_OACC_FORK
);
6338 tree join_kind
= build_int_cst (unsigned_type_node
, IFN_UNIQUE_OACC_JOIN
);
6340 for (unsigned done
= 1; count
; count
--, done
++)
6342 gimple_seq fork_seq
= NULL
;
6343 gimple_seq join_seq
= NULL
;
6345 tree place
= build_int_cst (integer_type_node
, -1);
6346 gcall
*fork
= gimple_build_call_internal (IFN_UNIQUE
, 3,
6347 fork_kind
, ddvar
, place
);
6348 gimple_set_location (fork
, loc
);
6349 gimple_set_lhs (fork
, ddvar
);
6351 gcall
*join
= gimple_build_call_internal (IFN_UNIQUE
, 3,
6352 join_kind
, ddvar
, place
);
6353 gimple_set_location (join
, loc
);
6354 gimple_set_lhs (join
, ddvar
);
6356 /* Mark the beginning of this level sequence. */
6358 lower_oacc_loop_marker (loc
, ddvar
, true,
6359 build_int_cst (integer_type_node
, count
),
6361 lower_oacc_loop_marker (loc
, ddvar
, false,
6362 build_int_cst (integer_type_node
, done
),
6365 lower_oacc_reductions (loc
, clauses
, place
, inner
,
6366 fork
, join
, &fork_seq
, &join_seq
, ctx
);
6368 /* Append this level to head. */
6369 gimple_seq_add_seq (head
, fork_seq
);
6370 /* Prepend it to tail. */
6371 gimple_seq_add_seq (&join_seq
, *tail
);
6377 /* Mark the end of the sequence. */
6378 lower_oacc_loop_marker (loc
, ddvar
, true, NULL_TREE
, head
);
6379 lower_oacc_loop_marker (loc
, ddvar
, false, NULL_TREE
, tail
);
6382 /* A convenience function to build an empty GIMPLE_COND with just the
6386 gimple_build_cond_empty (tree cond
)
6388 enum tree_code pred_code
;
6391 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
6392 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
6395 /* Return true if a parallel REGION is within a declare target function or
6396 within a target region and is not a part of a gridified target. */
6399 parallel_needs_hsa_kernel_p (struct omp_region
*region
)
6401 bool indirect
= false;
6402 for (region
= region
->outer
; region
; region
= region
->outer
)
6404 if (region
->type
== GIMPLE_OMP_PARALLEL
)
6406 else if (region
->type
== GIMPLE_OMP_TARGET
)
6408 gomp_target
*tgt_stmt
6409 = as_a
<gomp_target
*> (last_stmt (region
->entry
));
6411 if (find_omp_clause (gimple_omp_target_clauses (tgt_stmt
),
6412 OMP_CLAUSE__GRIDDIM_
))
6419 if (lookup_attribute ("omp declare target",
6420 DECL_ATTRIBUTES (current_function_decl
)))
6426 static void expand_omp_build_assign (gimple_stmt_iterator
*, tree
, tree
,
6429 /* Build the function calls to GOMP_parallel_start etc to actually
6430 generate the parallel operation. REGION is the parallel region
6431 being expanded. BB is the block where to insert the code. WS_ARGS
6432 will be set if this is a call to a combined parallel+workshare
6433 construct, it contains the list of additional arguments needed by
6434 the workshare construct. */
6437 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
6438 gomp_parallel
*entry_stmt
,
6439 vec
<tree
, va_gc
> *ws_args
)
6441 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
6442 gimple_stmt_iterator gsi
;
6444 enum built_in_function start_ix
;
6446 location_t clause_loc
;
6447 vec
<tree
, va_gc
> *args
;
6449 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
6451 /* Determine what flavor of GOMP_parallel we will be
6453 start_ix
= BUILT_IN_GOMP_PARALLEL
;
6454 if (is_combined_parallel (region
))
6456 switch (region
->inner
->type
)
6458 case GIMPLE_OMP_FOR
:
6459 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
6460 switch (region
->inner
->sched_kind
)
6462 case OMP_CLAUSE_SCHEDULE_RUNTIME
:
6465 case OMP_CLAUSE_SCHEDULE_DYNAMIC
:
6466 case OMP_CLAUSE_SCHEDULE_GUIDED
:
6467 if (region
->inner
->sched_modifiers
6468 & OMP_CLAUSE_SCHEDULE_NONMONOTONIC
)
6470 start_ix2
= 3 + region
->inner
->sched_kind
;
6475 start_ix2
= region
->inner
->sched_kind
;
6478 start_ix2
+= (int) BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
;
6479 start_ix
= (enum built_in_function
) start_ix2
;
6481 case GIMPLE_OMP_SECTIONS
:
6482 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
6489 /* By default, the value of NUM_THREADS is zero (selected at run time)
6490 and there is no conditional. */
6492 val
= build_int_cst (unsigned_type_node
, 0);
6493 flags
= build_int_cst (unsigned_type_node
, 0);
6495 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
6497 cond
= OMP_CLAUSE_IF_EXPR (c
);
6499 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
6502 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
6503 clause_loc
= OMP_CLAUSE_LOCATION (c
);
6506 clause_loc
= gimple_location (entry_stmt
);
6508 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
6510 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
6512 /* Ensure 'val' is of the correct type. */
6513 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
6515 /* If we found the clause 'if (cond)', build either
6516 (cond != 0) or (cond ? val : 1u). */
6519 cond
= gimple_boolify (cond
);
6521 if (integer_zerop (val
))
6522 val
= fold_build2_loc (clause_loc
,
6523 EQ_EXPR
, unsigned_type_node
, cond
,
6524 build_int_cst (TREE_TYPE (cond
), 0));
6527 basic_block cond_bb
, then_bb
, else_bb
;
6528 edge e
, e_then
, e_else
;
6529 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
6531 tmp_var
= create_tmp_var (TREE_TYPE (val
));
6532 if (gimple_in_ssa_p (cfun
))
6534 tmp_then
= make_ssa_name (tmp_var
);
6535 tmp_else
= make_ssa_name (tmp_var
);
6536 tmp_join
= make_ssa_name (tmp_var
);
6545 e
= split_block_after_labels (bb
);
6550 then_bb
= create_empty_bb (cond_bb
);
6551 else_bb
= create_empty_bb (then_bb
);
6552 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
6553 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
6555 stmt
= gimple_build_cond_empty (cond
);
6556 gsi
= gsi_start_bb (cond_bb
);
6557 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6559 gsi
= gsi_start_bb (then_bb
);
6560 expand_omp_build_assign (&gsi
, tmp_then
, val
, true);
6562 gsi
= gsi_start_bb (else_bb
);
6563 expand_omp_build_assign (&gsi
, tmp_else
,
6564 build_int_cst (unsigned_type_node
, 1),
6567 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
6568 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
6569 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
6570 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
6571 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
6572 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
6574 if (gimple_in_ssa_p (cfun
))
6576 gphi
*phi
= create_phi_node (tmp_join
, bb
);
6577 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
6578 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
6584 gsi
= gsi_start_bb (bb
);
6585 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
6586 false, GSI_CONTINUE_LINKING
);
6589 gsi
= gsi_last_bb (bb
);
6590 t
= gimple_omp_parallel_data_arg (entry_stmt
);
6592 t1
= null_pointer_node
;
6594 t1
= build_fold_addr_expr (t
);
6595 tree child_fndecl
= gimple_omp_parallel_child_fn (entry_stmt
);
6596 t2
= build_fold_addr_expr (child_fndecl
);
6598 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
6599 args
->quick_push (t2
);
6600 args
->quick_push (t1
);
6601 args
->quick_push (val
);
6603 args
->splice (*ws_args
);
6604 args
->quick_push (flags
);
6606 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
6607 builtin_decl_explicit (start_ix
), args
);
6609 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6610 false, GSI_CONTINUE_LINKING
);
6612 if (hsa_gen_requested_p ()
6613 && parallel_needs_hsa_kernel_p (region
))
6615 cgraph_node
*child_cnode
= cgraph_node::get (child_fndecl
);
6616 hsa_register_kernel (child_cnode
);
6620 /* Insert a function call whose name is FUNC_NAME with the information from
6621 ENTRY_STMT into the basic_block BB. */
6624 expand_cilk_for_call (basic_block bb
, gomp_parallel
*entry_stmt
,
6625 vec
<tree
, va_gc
> *ws_args
)
6628 gimple_stmt_iterator gsi
;
6629 vec
<tree
, va_gc
> *args
;
6631 gcc_assert (vec_safe_length (ws_args
) == 2);
6632 tree func_name
= (*ws_args
)[0];
6633 tree grain
= (*ws_args
)[1];
6635 tree clauses
= gimple_omp_parallel_clauses (entry_stmt
);
6636 tree count
= find_omp_clause (clauses
, OMP_CLAUSE__CILK_FOR_COUNT_
);
6637 gcc_assert (count
!= NULL_TREE
);
6638 count
= OMP_CLAUSE_OPERAND (count
, 0);
6640 gsi
= gsi_last_bb (bb
);
6641 t
= gimple_omp_parallel_data_arg (entry_stmt
);
6643 t1
= null_pointer_node
;
6645 t1
= build_fold_addr_expr (t
);
6646 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
6648 vec_alloc (args
, 4);
6649 args
->quick_push (t2
);
6650 args
->quick_push (t1
);
6651 args
->quick_push (count
);
6652 args
->quick_push (grain
);
6653 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
, func_name
, args
);
6655 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, false,
6656 GSI_CONTINUE_LINKING
);
6659 /* Build the function call to GOMP_task to actually
6660 generate the task operation. BB is the block where to insert the code. */
6663 expand_task_call (struct omp_region
*region
, basic_block bb
,
6664 gomp_task
*entry_stmt
)
6667 gimple_stmt_iterator gsi
;
6668 location_t loc
= gimple_location (entry_stmt
);
6670 tree clauses
= gimple_omp_task_clauses (entry_stmt
);
6672 tree ifc
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
6673 tree untied
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
6674 tree mergeable
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
6675 tree depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
6676 tree finalc
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
6677 tree priority
= find_omp_clause (clauses
, OMP_CLAUSE_PRIORITY
);
6680 = (untied
? GOMP_TASK_FLAG_UNTIED
: 0)
6681 | (mergeable
? GOMP_TASK_FLAG_MERGEABLE
: 0)
6682 | (depend
? GOMP_TASK_FLAG_DEPEND
: 0);
6684 bool taskloop_p
= gimple_omp_task_taskloop_p (entry_stmt
);
6685 tree startvar
= NULL_TREE
, endvar
= NULL_TREE
, step
= NULL_TREE
;
6686 tree num_tasks
= NULL_TREE
;
6690 gimple
*g
= last_stmt (region
->outer
->entry
);
6691 gcc_assert (gimple_code (g
) == GIMPLE_OMP_FOR
6692 && gimple_omp_for_kind (g
) == GF_OMP_FOR_KIND_TASKLOOP
);
6693 struct omp_for_data fd
;
6694 extract_omp_for_data (as_a
<gomp_for
*> (g
), &fd
, NULL
);
6695 startvar
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6696 endvar
= find_omp_clause (OMP_CLAUSE_CHAIN (startvar
),
6697 OMP_CLAUSE__LOOPTEMP_
);
6698 startvar
= OMP_CLAUSE_DECL (startvar
);
6699 endvar
= OMP_CLAUSE_DECL (endvar
);
6700 step
= fold_convert_loc (loc
, fd
.iter_type
, fd
.loop
.step
);
6701 if (fd
.loop
.cond_code
== LT_EXPR
)
6702 iflags
|= GOMP_TASK_FLAG_UP
;
6703 tree tclauses
= gimple_omp_for_clauses (g
);
6704 num_tasks
= find_omp_clause (tclauses
, OMP_CLAUSE_NUM_TASKS
);
6706 num_tasks
= OMP_CLAUSE_NUM_TASKS_EXPR (num_tasks
);
6709 num_tasks
= find_omp_clause (tclauses
, OMP_CLAUSE_GRAINSIZE
);
6712 iflags
|= GOMP_TASK_FLAG_GRAINSIZE
;
6713 num_tasks
= OMP_CLAUSE_GRAINSIZE_EXPR (num_tasks
);
6716 num_tasks
= integer_zero_node
;
6718 num_tasks
= fold_convert_loc (loc
, long_integer_type_node
, num_tasks
);
6719 if (ifc
== NULL_TREE
)
6720 iflags
|= GOMP_TASK_FLAG_IF
;
6721 if (find_omp_clause (tclauses
, OMP_CLAUSE_NOGROUP
))
6722 iflags
|= GOMP_TASK_FLAG_NOGROUP
;
6723 ull
= fd
.iter_type
== long_long_unsigned_type_node
;
6726 iflags
|= GOMP_TASK_FLAG_PRIORITY
;
6728 tree flags
= build_int_cst (unsigned_type_node
, iflags
);
6730 tree cond
= boolean_true_node
;
6735 tree t
= gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc
));
6736 t
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, t
,
6737 build_int_cst (unsigned_type_node
,
6739 build_int_cst (unsigned_type_node
, 0));
6740 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
,
6744 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc
));
6749 tree t
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (finalc
));
6750 t
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, t
,
6751 build_int_cst (unsigned_type_node
,
6752 GOMP_TASK_FLAG_FINAL
),
6753 build_int_cst (unsigned_type_node
, 0));
6754 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, t
);
6757 depend
= OMP_CLAUSE_DECL (depend
);
6759 depend
= build_int_cst (ptr_type_node
, 0);
6761 priority
= fold_convert (integer_type_node
,
6762 OMP_CLAUSE_PRIORITY_EXPR (priority
));
6764 priority
= integer_zero_node
;
6766 gsi
= gsi_last_bb (bb
);
6767 tree t
= gimple_omp_task_data_arg (entry_stmt
);
6769 t2
= null_pointer_node
;
6771 t2
= build_fold_addr_expr_loc (loc
, t
);
6772 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
6773 t
= gimple_omp_task_copy_fn (entry_stmt
);
6775 t3
= null_pointer_node
;
6777 t3
= build_fold_addr_expr_loc (loc
, t
);
6780 t
= build_call_expr (ull
6781 ? builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP_ULL
)
6782 : builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP
),
6784 gimple_omp_task_arg_size (entry_stmt
),
6785 gimple_omp_task_arg_align (entry_stmt
), flags
,
6786 num_tasks
, priority
, startvar
, endvar
, step
);
6788 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
6790 gimple_omp_task_arg_size (entry_stmt
),
6791 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
6794 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6795 false, GSI_CONTINUE_LINKING
);
6799 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
6800 catch handler and return it. This prevents programs from violating the
6801 structured block semantics with throws. */
6804 maybe_catch_exception (gimple_seq body
)
6809 if (!flag_exceptions
)
6812 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
6813 decl
= lang_hooks
.eh_protect_cleanup_actions ();
6815 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
6817 g
= gimple_build_eh_must_not_throw (decl
);
6818 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
6821 return gimple_seq_alloc_with_stmt (g
);
6824 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
6827 vec2chain (vec
<tree
, va_gc
> *v
)
6829 tree chain
= NULL_TREE
, t
;
6832 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
6834 DECL_CHAIN (t
) = chain
;
6842 /* Remove barriers in REGION->EXIT's block. Note that this is only
6843 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
6844 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
6845 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
6849 remove_exit_barrier (struct omp_region
*region
)
6851 gimple_stmt_iterator gsi
;
6852 basic_block exit_bb
;
6856 int any_addressable_vars
= -1;
6858 exit_bb
= region
->exit
;
6860 /* If the parallel region doesn't return, we don't have REGION->EXIT
6865 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
6866 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
6867 statements that can appear in between are extremely limited -- no
6868 memory operations at all. Here, we allow nothing at all, so the
6869 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
6870 gsi
= gsi_last_bb (exit_bb
);
6871 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
6873 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
6876 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
6878 gsi
= gsi_last_bb (e
->src
);
6879 if (gsi_end_p (gsi
))
6881 stmt
= gsi_stmt (gsi
);
6882 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
6883 && !gimple_omp_return_nowait_p (stmt
))
6885 /* OpenMP 3.0 tasks unfortunately prevent this optimization
6886 in many cases. If there could be tasks queued, the barrier
6887 might be needed to let the tasks run before some local
6888 variable of the parallel that the task uses as shared
6889 runs out of scope. The task can be spawned either
6890 from within current function (this would be easy to check)
6891 or from some function it calls and gets passed an address
6892 of such a variable. */
6893 if (any_addressable_vars
< 0)
6895 gomp_parallel
*parallel_stmt
6896 = as_a
<gomp_parallel
*> (last_stmt (region
->entry
));
6897 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
6898 tree local_decls
, block
, decl
;
6901 any_addressable_vars
= 0;
6902 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
6903 if (TREE_ADDRESSABLE (decl
))
6905 any_addressable_vars
= 1;
6908 for (block
= gimple_block (stmt
);
6909 !any_addressable_vars
6911 && TREE_CODE (block
) == BLOCK
;
6912 block
= BLOCK_SUPERCONTEXT (block
))
6914 for (local_decls
= BLOCK_VARS (block
);
6916 local_decls
= DECL_CHAIN (local_decls
))
6917 if (TREE_ADDRESSABLE (local_decls
))
6919 any_addressable_vars
= 1;
6922 if (block
== gimple_block (parallel_stmt
))
6926 if (!any_addressable_vars
)
6927 gimple_omp_return_set_nowait (stmt
);
6933 remove_exit_barriers (struct omp_region
*region
)
6935 if (region
->type
== GIMPLE_OMP_PARALLEL
)
6936 remove_exit_barrier (region
);
6940 region
= region
->inner
;
6941 remove_exit_barriers (region
);
6942 while (region
->next
)
6944 region
= region
->next
;
6945 remove_exit_barriers (region
);
6950 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
6951 calls. These can't be declared as const functions, but
6952 within one parallel body they are constant, so they can be
6953 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
6954 which are declared const. Similarly for task body, except
6955 that in untied task omp_get_thread_num () can change at any task
6956 scheduling point. */
6959 optimize_omp_library_calls (gimple
*entry_stmt
)
6962 gimple_stmt_iterator gsi
;
6963 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
6964 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
6965 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
6966 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
6967 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
6968 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
6969 OMP_CLAUSE_UNTIED
) != NULL
);
6971 FOR_EACH_BB_FN (bb
, cfun
)
6972 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
6974 gimple
*call
= gsi_stmt (gsi
);
6977 if (is_gimple_call (call
)
6978 && (decl
= gimple_call_fndecl (call
))
6979 && DECL_EXTERNAL (decl
)
6980 && TREE_PUBLIC (decl
)
6981 && DECL_INITIAL (decl
) == NULL
)
6985 if (DECL_NAME (decl
) == thr_num_id
)
6987 /* In #pragma omp task untied omp_get_thread_num () can change
6988 during the execution of the task region. */
6991 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
6993 else if (DECL_NAME (decl
) == num_thr_id
)
6994 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
6998 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
6999 || gimple_call_num_args (call
) != 0)
7002 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
7005 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
7006 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
7007 TREE_TYPE (TREE_TYPE (built_in
))))
7010 gimple_call_set_fndecl (call
, built_in
);
7015 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
7019 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
7023 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
7024 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
7027 if (TREE_CODE (t
) == ADDR_EXPR
)
7028 recompute_tree_invariant_for_addr_expr (t
);
7030 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
7034 /* Prepend or append TO = FROM assignment before or after *GSI_P. */
7037 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
,
7040 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
7041 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
7042 !after
, after
? GSI_CONTINUE_LINKING
7044 gimple
*stmt
= gimple_build_assign (to
, from
);
7046 gsi_insert_after (gsi_p
, stmt
, GSI_CONTINUE_LINKING
);
7048 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
7049 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
7050 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
7052 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
7053 gimple_regimplify_operands (stmt
, &gsi
);
7057 /* Expand the OpenMP parallel or task directive starting at REGION. */
7060 expand_omp_taskreg (struct omp_region
*region
)
7062 basic_block entry_bb
, exit_bb
, new_bb
;
7063 struct function
*child_cfun
;
7064 tree child_fn
, block
, t
;
7065 gimple_stmt_iterator gsi
;
7066 gimple
*entry_stmt
, *stmt
;
7068 vec
<tree
, va_gc
> *ws_args
;
7070 entry_stmt
= last_stmt (region
->entry
);
7071 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
7072 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
7074 entry_bb
= region
->entry
;
7075 if (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
)
7076 exit_bb
= region
->cont
;
7078 exit_bb
= region
->exit
;
7082 && gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
7083 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt
),
7084 OMP_CLAUSE__CILK_FOR_COUNT_
) != NULL_TREE
);
7087 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for,
7088 and the inner statement contains the name of the built-in function
7090 ws_args
= region
->inner
->ws_args
;
7091 else if (is_combined_parallel (region
))
7092 ws_args
= region
->ws_args
;
7096 if (child_cfun
->cfg
)
7098 /* Due to inlining, it may happen that we have already outlined
7099 the region, in which case all we need to do is make the
7100 sub-graph unreachable and emit the parallel call. */
7101 edge entry_succ_e
, exit_succ_e
;
7103 entry_succ_e
= single_succ_edge (entry_bb
);
7105 gsi
= gsi_last_bb (entry_bb
);
7106 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
7107 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
7108 gsi_remove (&gsi
, true);
7113 exit_succ_e
= single_succ_edge (exit_bb
);
7114 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
7116 remove_edge_and_dominated_blocks (entry_succ_e
);
7120 unsigned srcidx
, dstidx
, num
;
7122 /* If the parallel region needs data sent from the parent
7123 function, then the very first statement (except possible
7124 tree profile counter updates) of the parallel body
7125 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
7126 &.OMP_DATA_O is passed as an argument to the child function,
7127 we need to replace it with the argument as seen by the child
7130 In most cases, this will end up being the identity assignment
7131 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
7132 a function call that has been inlined, the original PARM_DECL
7133 .OMP_DATA_I may have been converted into a different local
7134 variable. In which case, we need to keep the assignment. */
7135 if (gimple_omp_taskreg_data_arg (entry_stmt
))
7137 basic_block entry_succ_bb
7138 = single_succ_p (entry_bb
) ? single_succ (entry_bb
)
7139 : FALLTHRU_EDGE (entry_bb
)->dest
;
7141 gimple
*parcopy_stmt
= NULL
;
7143 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
7147 gcc_assert (!gsi_end_p (gsi
));
7148 stmt
= gsi_stmt (gsi
);
7149 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
7152 if (gimple_num_ops (stmt
) == 2)
7154 tree arg
= gimple_assign_rhs1 (stmt
);
7156 /* We're ignore the subcode because we're
7157 effectively doing a STRIP_NOPS. */
7159 if (TREE_CODE (arg
) == ADDR_EXPR
7160 && TREE_OPERAND (arg
, 0)
7161 == gimple_omp_taskreg_data_arg (entry_stmt
))
7163 parcopy_stmt
= stmt
;
7169 gcc_assert (parcopy_stmt
!= NULL
);
7170 arg
= DECL_ARGUMENTS (child_fn
);
7172 if (!gimple_in_ssa_p (cfun
))
7174 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
7175 gsi_remove (&gsi
, true);
7178 /* ?? Is setting the subcode really necessary ?? */
7179 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
7180 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
7185 tree lhs
= gimple_assign_lhs (parcopy_stmt
);
7186 gcc_assert (SSA_NAME_VAR (lhs
) == arg
);
7187 /* We'd like to set the rhs to the default def in the child_fn,
7188 but it's too early to create ssa names in the child_fn.
7189 Instead, we set the rhs to the parm. In
7190 move_sese_region_to_fn, we introduce a default def for the
7191 parm, map the parm to it's default def, and once we encounter
7192 this stmt, replace the parm with the default def. */
7193 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
7194 update_stmt (parcopy_stmt
);
7198 /* Declare local variables needed in CHILD_CFUN. */
7199 block
= DECL_INITIAL (child_fn
);
7200 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
7201 /* The gimplifier could record temporaries in parallel/task block
7202 rather than in containing function's local_decls chain,
7203 which would mean cgraph missed finalizing them. Do it now. */
7204 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
7205 if (TREE_CODE (t
) == VAR_DECL
7207 && !DECL_EXTERNAL (t
))
7208 varpool_node::finalize_decl (t
);
7209 DECL_SAVED_TREE (child_fn
) = NULL
;
7210 /* We'll create a CFG for child_fn, so no gimple body is needed. */
7211 gimple_set_body (child_fn
, NULL
);
7212 TREE_USED (block
) = 1;
7214 /* Reset DECL_CONTEXT on function arguments. */
7215 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
7216 DECL_CONTEXT (t
) = child_fn
;
7218 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
7219 so that it can be moved to the child function. */
7220 gsi
= gsi_last_bb (entry_bb
);
7221 stmt
= gsi_stmt (gsi
);
7222 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
7223 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
7224 e
= split_block (entry_bb
, stmt
);
7225 gsi_remove (&gsi
, true);
7228 if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
7229 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7232 e2
= make_edge (e
->src
, BRANCH_EDGE (entry_bb
)->dest
, EDGE_ABNORMAL
);
7233 gcc_assert (e2
->dest
== region
->exit
);
7234 remove_edge (BRANCH_EDGE (entry_bb
));
7235 set_immediate_dominator (CDI_DOMINATORS
, e2
->dest
, e
->src
);
7236 gsi
= gsi_last_bb (region
->exit
);
7237 gcc_assert (!gsi_end_p (gsi
)
7238 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
7239 gsi_remove (&gsi
, true);
7242 /* Convert GIMPLE_OMP_{RETURN,CONTINUE} into a RETURN_EXPR. */
7245 gsi
= gsi_last_bb (exit_bb
);
7246 gcc_assert (!gsi_end_p (gsi
)
7247 && (gimple_code (gsi_stmt (gsi
))
7248 == (e2
? GIMPLE_OMP_CONTINUE
: GIMPLE_OMP_RETURN
)));
7249 stmt
= gimple_build_return (NULL
);
7250 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
7251 gsi_remove (&gsi
, true);
7254 /* Move the parallel region into CHILD_CFUN. */
7256 if (gimple_in_ssa_p (cfun
))
7258 init_tree_ssa (child_cfun
);
7259 init_ssa_operands (child_cfun
);
7260 child_cfun
->gimple_df
->in_ssa_p
= true;
7264 block
= gimple_block (entry_stmt
);
7266 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
7268 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
7271 basic_block dest_bb
= e2
->dest
;
7273 make_edge (new_bb
, dest_bb
, EDGE_FALLTHRU
);
7275 set_immediate_dominator (CDI_DOMINATORS
, dest_bb
, new_bb
);
7277 /* When the OMP expansion process cannot guarantee an up-to-date
7278 loop tree arrange for the child function to fixup loops. */
7279 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
7280 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
7282 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
7283 num
= vec_safe_length (child_cfun
->local_decls
);
7284 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
7286 t
= (*child_cfun
->local_decls
)[srcidx
];
7287 if (DECL_CONTEXT (t
) == cfun
->decl
)
7289 if (srcidx
!= dstidx
)
7290 (*child_cfun
->local_decls
)[dstidx
] = t
;
7294 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
7296 /* Inform the callgraph about the new function. */
7297 child_cfun
->curr_properties
= cfun
->curr_properties
;
7298 child_cfun
->has_simduid_loops
|= cfun
->has_simduid_loops
;
7299 child_cfun
->has_force_vectorize_loops
|= cfun
->has_force_vectorize_loops
;
7300 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
7301 node
->parallelized_function
= 1;
7302 cgraph_node::add_new_function (child_fn
, true);
7304 bool need_asm
= DECL_ASSEMBLER_NAME_SET_P (current_function_decl
)
7305 && !DECL_ASSEMBLER_NAME_SET_P (child_fn
);
7307 /* Fix the callgraph edges for child_cfun. Those for cfun will be
7308 fixed in a following pass. */
7309 push_cfun (child_cfun
);
7311 assign_assembler_name_if_neeeded (child_fn
);
7314 optimize_omp_library_calls (entry_stmt
);
7315 cgraph_edge::rebuild_edges ();
7317 /* Some EH regions might become dead, see PR34608. If
7318 pass_cleanup_cfg isn't the first pass to happen with the
7319 new child, these dead EH edges might cause problems.
7320 Clean them up now. */
7321 if (flag_exceptions
)
7324 bool changed
= false;
7326 FOR_EACH_BB_FN (bb
, cfun
)
7327 changed
|= gimple_purge_dead_eh_edges (bb
);
7329 cleanup_tree_cfg ();
7331 if (gimple_in_ssa_p (cfun
))
7332 update_ssa (TODO_update_ssa
);
7333 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
7334 verify_loop_structure ();
7337 if (dump_file
&& !gimple_in_ssa_p (cfun
))
7339 omp_any_child_fn_dumped
= true;
7340 dump_function_header (dump_file
, child_fn
, dump_flags
);
7341 dump_function_to_file (child_fn
, dump_file
, dump_flags
);
7345 /* Emit a library call to launch the children threads. */
7347 expand_cilk_for_call (new_bb
,
7348 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
7349 else if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
7350 expand_parallel_call (region
, new_bb
,
7351 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
7353 expand_task_call (region
, new_bb
, as_a
<gomp_task
*> (entry_stmt
));
7354 if (gimple_in_ssa_p (cfun
))
7355 update_ssa (TODO_update_ssa_only_virtuals
);
7358 /* Information about members of an OpenACC collapsed loop nest. */
7360 struct oacc_collapse
7362 tree base
; /* Base value. */
7363 tree iters
; /* Number of steps. */
7364 tree step
; /* step size. */
7367 /* Helper for expand_oacc_for. Determine collapsed loop information.
7368 Fill in COUNTS array. Emit any initialization code before GSI.
7369 Return the calculated outer loop bound of BOUND_TYPE. */
7372 expand_oacc_collapse_init (const struct omp_for_data
*fd
,
7373 gimple_stmt_iterator
*gsi
,
7374 oacc_collapse
*counts
, tree bound_type
)
7376 tree total
= build_int_cst (bound_type
, 1);
7379 gcc_assert (integer_onep (fd
->loop
.step
));
7380 gcc_assert (integer_zerop (fd
->loop
.n1
));
7382 for (ix
= 0; ix
!= fd
->collapse
; ix
++)
7384 const omp_for_data_loop
*loop
= &fd
->loops
[ix
];
7386 tree iter_type
= TREE_TYPE (loop
->v
);
7387 tree diff_type
= iter_type
;
7388 tree plus_type
= iter_type
;
7390 gcc_assert (loop
->cond_code
== fd
->loop
.cond_code
);
7392 if (POINTER_TYPE_P (iter_type
))
7393 plus_type
= sizetype
;
7394 if (POINTER_TYPE_P (diff_type
) || TYPE_UNSIGNED (diff_type
))
7395 diff_type
= signed_type_for (diff_type
);
7399 tree s
= loop
->step
;
7400 bool up
= loop
->cond_code
== LT_EXPR
;
7401 tree dir
= build_int_cst (diff_type
, up
? +1 : -1);
7405 b
= force_gimple_operand_gsi (gsi
, b
, true, NULL_TREE
,
7406 true, GSI_SAME_STMT
);
7407 e
= force_gimple_operand_gsi (gsi
, e
, true, NULL_TREE
,
7408 true, GSI_SAME_STMT
);
7410 /* Convert the step, avoiding possible unsigned->signed overflow. */
7411 negating
= !up
&& TYPE_UNSIGNED (TREE_TYPE (s
));
7413 s
= fold_build1 (NEGATE_EXPR
, TREE_TYPE (s
), s
);
7414 s
= fold_convert (diff_type
, s
);
7416 s
= fold_build1 (NEGATE_EXPR
, diff_type
, s
);
7417 s
= force_gimple_operand_gsi (gsi
, s
, true, NULL_TREE
,
7418 true, GSI_SAME_STMT
);
7420 /* Determine the range, avoiding possible unsigned->signed overflow. */
7421 negating
= !up
&& TYPE_UNSIGNED (iter_type
);
7422 expr
= fold_build2 (MINUS_EXPR
, plus_type
,
7423 fold_convert (plus_type
, negating
? b
: e
),
7424 fold_convert (plus_type
, negating
? e
: b
));
7425 expr
= fold_convert (diff_type
, expr
);
7427 expr
= fold_build1 (NEGATE_EXPR
, diff_type
, expr
);
7428 tree range
= force_gimple_operand_gsi
7429 (gsi
, expr
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7431 /* Determine number of iterations. */
7432 expr
= fold_build2 (MINUS_EXPR
, diff_type
, range
, dir
);
7433 expr
= fold_build2 (PLUS_EXPR
, diff_type
, expr
, s
);
7434 expr
= fold_build2 (TRUNC_DIV_EXPR
, diff_type
, expr
, s
);
7436 tree iters
= force_gimple_operand_gsi (gsi
, expr
, true, NULL_TREE
,
7437 true, GSI_SAME_STMT
);
7439 counts
[ix
].base
= b
;
7440 counts
[ix
].iters
= iters
;
7441 counts
[ix
].step
= s
;
7443 total
= fold_build2 (MULT_EXPR
, bound_type
, total
,
7444 fold_convert (bound_type
, iters
));
7450 /* Emit initializers for collapsed loop members. IVAR is the outer
7451 loop iteration variable, from which collapsed loop iteration values
7452 are calculated. COUNTS array has been initialized by
7453 expand_oacc_collapse_inits. */
7456 expand_oacc_collapse_vars (const struct omp_for_data
*fd
,
7457 gimple_stmt_iterator
*gsi
,
7458 const oacc_collapse
*counts
, tree ivar
)
7460 tree ivar_type
= TREE_TYPE (ivar
);
7462 /* The most rapidly changing iteration variable is the innermost
7464 for (int ix
= fd
->collapse
; ix
--;)
7466 const omp_for_data_loop
*loop
= &fd
->loops
[ix
];
7467 const oacc_collapse
*collapse
= &counts
[ix
];
7468 tree iter_type
= TREE_TYPE (loop
->v
);
7469 tree diff_type
= TREE_TYPE (collapse
->step
);
7470 tree plus_type
= iter_type
;
7471 enum tree_code plus_code
= PLUS_EXPR
;
7474 if (POINTER_TYPE_P (iter_type
))
7476 plus_code
= POINTER_PLUS_EXPR
;
7477 plus_type
= sizetype
;
7480 expr
= fold_build2 (TRUNC_MOD_EXPR
, ivar_type
, ivar
,
7481 fold_convert (ivar_type
, collapse
->iters
));
7482 expr
= fold_build2 (MULT_EXPR
, diff_type
, fold_convert (diff_type
, expr
),
7484 expr
= fold_build2 (plus_code
, iter_type
, collapse
->base
,
7485 fold_convert (plus_type
, expr
));
7486 expr
= force_gimple_operand_gsi (gsi
, expr
, false, NULL_TREE
,
7487 true, GSI_SAME_STMT
);
7488 gassign
*ass
= gimple_build_assign (loop
->v
, expr
);
7489 gsi_insert_before (gsi
, ass
, GSI_SAME_STMT
);
7493 expr
= fold_build2 (TRUNC_DIV_EXPR
, ivar_type
, ivar
,
7494 fold_convert (ivar_type
, collapse
->iters
));
7495 ivar
= force_gimple_operand_gsi (gsi
, expr
, true, NULL_TREE
,
7496 true, GSI_SAME_STMT
);
7502 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
7503 of the combined collapse > 1 loop constructs, generate code like:
7504 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
7509 count3 = (adj + N32 - N31) / STEP3;
7510 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
7515 count2 = (adj + N22 - N21) / STEP2;
7516 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
7521 count1 = (adj + N12 - N11) / STEP1;
7522 count = count1 * count2 * count3;
7523 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
7525 and set ZERO_ITER_BB to that bb. If this isn't the outermost
7526 of the combined loop constructs, just initialize COUNTS array
7527 from the _looptemp_ clauses. */
7529 /* NOTE: It *could* be better to moosh all of the BBs together,
7530 creating one larger BB with all the computation and the unexpected
7531 jump at the end. I.e.
7533 bool zero3, zero2, zero1, zero;
7536 count3 = (N32 - N31) /[cl] STEP3;
7538 count2 = (N22 - N21) /[cl] STEP2;
7540 count1 = (N12 - N11) /[cl] STEP1;
7541 zero = zero3 || zero2 || zero1;
7542 count = count1 * count2 * count3;
7543 if (__builtin_expect(zero, false)) goto zero_iter_bb;
7545 After all, we expect the zero=false, and thus we expect to have to
7546 evaluate all of the comparison expressions, so short-circuiting
7547 oughtn't be a win. Since the condition isn't protecting a
7548 denominator, we're not concerned about divide-by-zero, so we can
7549 fully evaluate count even if a numerator turned out to be wrong.
7551 It seems like putting this all together would create much better
7552 scheduling opportunities, and less pressure on the chip's branch
7556 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
7557 basic_block
&entry_bb
, tree
*counts
,
7558 basic_block
&zero_iter1_bb
, int &first_zero_iter1
,
7559 basic_block
&zero_iter2_bb
, int &first_zero_iter2
,
7560 basic_block
&l2_dom_bb
)
7562 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
7566 /* Collapsed loops need work for expansion into SSA form. */
7567 gcc_assert (!gimple_in_ssa_p (cfun
));
7569 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
7570 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
7572 gcc_assert (fd
->ordered
== 0);
7573 /* First two _looptemp_ clauses are for istart/iend, counts[0]
7574 isn't supposed to be handled, as the inner loop doesn't
7576 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7577 OMP_CLAUSE__LOOPTEMP_
);
7578 gcc_assert (innerc
);
7579 for (i
= 0; i
< fd
->collapse
; i
++)
7581 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7582 OMP_CLAUSE__LOOPTEMP_
);
7583 gcc_assert (innerc
);
7585 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
7587 counts
[0] = NULL_TREE
;
7592 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
7594 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7595 counts
[i
] = NULL_TREE
;
7596 t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
7597 fold_convert (itype
, fd
->loops
[i
].n1
),
7598 fold_convert (itype
, fd
->loops
[i
].n2
));
7599 if (t
&& integer_zerop (t
))
7601 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
7602 counts
[i
] = build_int_cst (type
, 0);
7606 for (i
= 0; i
< (fd
->ordered
? fd
->ordered
: fd
->collapse
); i
++)
7608 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7610 if (i
>= fd
->collapse
&& counts
[i
])
7612 if ((SSA_VAR_P (fd
->loop
.n2
) || i
>= fd
->collapse
)
7613 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
7614 fold_convert (itype
, fd
->loops
[i
].n1
),
7615 fold_convert (itype
, fd
->loops
[i
].n2
)))
7616 == NULL_TREE
|| !integer_onep (t
)))
7620 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
7621 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
7622 true, GSI_SAME_STMT
);
7623 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
7624 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
7625 true, GSI_SAME_STMT
);
7626 cond_stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
7627 NULL_TREE
, NULL_TREE
);
7628 gsi_insert_before (gsi
, cond_stmt
, GSI_SAME_STMT
);
7629 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
7630 expand_omp_regimplify_p
, NULL
, NULL
)
7631 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
7632 expand_omp_regimplify_p
, NULL
, NULL
))
7634 *gsi
= gsi_for_stmt (cond_stmt
);
7635 gimple_regimplify_operands (cond_stmt
, gsi
);
7637 e
= split_block (entry_bb
, cond_stmt
);
7638 basic_block
&zero_iter_bb
7639 = i
< fd
->collapse
? zero_iter1_bb
: zero_iter2_bb
;
7640 int &first_zero_iter
7641 = i
< fd
->collapse
? first_zero_iter1
: first_zero_iter2
;
7642 if (zero_iter_bb
== NULL
)
7644 gassign
*assign_stmt
;
7645 first_zero_iter
= i
;
7646 zero_iter_bb
= create_empty_bb (entry_bb
);
7647 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
7648 *gsi
= gsi_after_labels (zero_iter_bb
);
7649 if (i
< fd
->collapse
)
7650 assign_stmt
= gimple_build_assign (fd
->loop
.n2
,
7651 build_zero_cst (type
));
7654 counts
[i
] = create_tmp_reg (type
, ".count");
7656 = gimple_build_assign (counts
[i
], build_zero_cst (type
));
7658 gsi_insert_before (gsi
, assign_stmt
, GSI_SAME_STMT
);
7659 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
7662 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
7663 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
7664 e
->flags
= EDGE_TRUE_VALUE
;
7665 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
7666 if (l2_dom_bb
== NULL
)
7667 l2_dom_bb
= entry_bb
;
7669 *gsi
= gsi_last_bb (entry_bb
);
7672 if (POINTER_TYPE_P (itype
))
7673 itype
= signed_type_for (itype
);
7674 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
7676 t
= fold_build2 (PLUS_EXPR
, itype
,
7677 fold_convert (itype
, fd
->loops
[i
].step
), t
);
7678 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
7679 fold_convert (itype
, fd
->loops
[i
].n2
));
7680 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
7681 fold_convert (itype
, fd
->loops
[i
].n1
));
7682 /* ?? We could probably use CEIL_DIV_EXPR instead of
7683 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
7684 generate the same code in the end because generically we
7685 don't know that the values involved must be negative for
7687 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
7688 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
7689 fold_build1 (NEGATE_EXPR
, itype
, t
),
7690 fold_build1 (NEGATE_EXPR
, itype
,
7691 fold_convert (itype
,
7692 fd
->loops
[i
].step
)));
7694 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
7695 fold_convert (itype
, fd
->loops
[i
].step
));
7696 t
= fold_convert (type
, t
);
7697 if (TREE_CODE (t
) == INTEGER_CST
)
7701 if (i
< fd
->collapse
|| i
!= first_zero_iter2
)
7702 counts
[i
] = create_tmp_reg (type
, ".count");
7703 expand_omp_build_assign (gsi
, counts
[i
], t
);
7705 if (SSA_VAR_P (fd
->loop
.n2
) && i
< fd
->collapse
)
7710 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
7711 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
7717 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
7719 V3 = N31 + (T % count3) * STEP3;
7721 V2 = N21 + (T % count2) * STEP2;
7723 V1 = N11 + T * STEP1;
7724 if this loop doesn't have an inner loop construct combined with it.
7725 If it does have an inner loop construct combined with it and the
7726 iteration count isn't known constant, store values from counts array
7727 into its _looptemp_ temporaries instead. */
7730 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
7731 tree
*counts
, gimple
*inner_stmt
, tree startvar
)
7734 if (gimple_omp_for_combined_p (fd
->for_stmt
))
7736 /* If fd->loop.n2 is constant, then no propagation of the counts
7737 is needed, they are constant. */
7738 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
7741 tree clauses
= gimple_code (inner_stmt
) != GIMPLE_OMP_FOR
7742 ? gimple_omp_taskreg_clauses (inner_stmt
)
7743 : gimple_omp_for_clauses (inner_stmt
);
7744 /* First two _looptemp_ clauses are for istart/iend, counts[0]
7745 isn't supposed to be handled, as the inner loop doesn't
7747 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
7748 gcc_assert (innerc
);
7749 for (i
= 0; i
< fd
->collapse
; i
++)
7751 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7752 OMP_CLAUSE__LOOPTEMP_
);
7753 gcc_assert (innerc
);
7756 tree tem
= OMP_CLAUSE_DECL (innerc
);
7757 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
7758 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
7759 false, GSI_CONTINUE_LINKING
);
7760 gassign
*stmt
= gimple_build_assign (tem
, t
);
7761 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7767 tree type
= TREE_TYPE (fd
->loop
.v
);
7768 tree tem
= create_tmp_reg (type
, ".tem");
7769 gassign
*stmt
= gimple_build_assign (tem
, startvar
);
7770 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7772 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
7774 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
7776 if (POINTER_TYPE_P (vtype
))
7777 itype
= signed_type_for (vtype
);
7779 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
7782 t
= fold_convert (itype
, t
);
7783 t
= fold_build2 (MULT_EXPR
, itype
, t
,
7784 fold_convert (itype
, fd
->loops
[i
].step
));
7785 if (POINTER_TYPE_P (vtype
))
7786 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
7788 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
7789 t
= force_gimple_operand_gsi (gsi
, t
,
7790 DECL_P (fd
->loops
[i
].v
)
7791 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
7793 GSI_CONTINUE_LINKING
);
7794 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
7795 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7798 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
7799 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
7800 false, GSI_CONTINUE_LINKING
);
7801 stmt
= gimple_build_assign (tem
, t
);
7802 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7808 /* Helper function for expand_omp_for_*. Generate code like:
7811 if (V3 cond3 N32) goto BODY_BB; else goto L11;
7815 if (V2 cond2 N22) goto BODY_BB; else goto L12;
7822 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
7823 basic_block body_bb
)
7825 basic_block last_bb
, bb
, collapse_bb
= NULL
;
7827 gimple_stmt_iterator gsi
;
7833 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
7835 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
7837 bb
= create_empty_bb (last_bb
);
7838 add_bb_to_loop (bb
, last_bb
->loop_father
);
7839 gsi
= gsi_start_bb (bb
);
7841 if (i
< fd
->collapse
- 1)
7843 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
7844 e
->probability
= REG_BR_PROB_BASE
/ 8;
7846 t
= fd
->loops
[i
+ 1].n1
;
7847 t
= force_gimple_operand_gsi (&gsi
, t
,
7848 DECL_P (fd
->loops
[i
+ 1].v
)
7849 && TREE_ADDRESSABLE (fd
->loops
[i
7852 GSI_CONTINUE_LINKING
);
7853 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
7854 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7859 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
7861 if (POINTER_TYPE_P (vtype
))
7862 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
7864 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
7865 t
= force_gimple_operand_gsi (&gsi
, t
,
7866 DECL_P (fd
->loops
[i
].v
)
7867 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
7868 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
7869 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
7870 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7874 t
= fd
->loops
[i
].n2
;
7875 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7876 false, GSI_CONTINUE_LINKING
);
7877 tree v
= fd
->loops
[i
].v
;
7878 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
7879 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
7880 false, GSI_CONTINUE_LINKING
);
7881 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
7882 stmt
= gimple_build_cond_empty (t
);
7883 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7884 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
7885 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7888 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
7896 /* Expand #pragma omp ordered depend(source). */
7899 expand_omp_ordered_source (gimple_stmt_iterator
*gsi
, struct omp_for_data
*fd
,
7900 tree
*counts
, location_t loc
)
7902 enum built_in_function source_ix
7903 = fd
->iter_type
== long_integer_type_node
7904 ? BUILT_IN_GOMP_DOACROSS_POST
: BUILT_IN_GOMP_DOACROSS_ULL_POST
;
7906 = gimple_build_call (builtin_decl_explicit (source_ix
), 1,
7907 build_fold_addr_expr (counts
[fd
->ordered
]));
7908 gimple_set_location (g
, loc
);
7909 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
7912 /* Expand a single depend from #pragma omp ordered depend(sink:...). */
7915 expand_omp_ordered_sink (gimple_stmt_iterator
*gsi
, struct omp_for_data
*fd
,
7916 tree
*counts
, tree c
, location_t loc
)
7918 auto_vec
<tree
, 10> args
;
7919 enum built_in_function sink_ix
7920 = fd
->iter_type
== long_integer_type_node
7921 ? BUILT_IN_GOMP_DOACROSS_WAIT
: BUILT_IN_GOMP_DOACROSS_ULL_WAIT
;
7922 tree t
, off
, coff
= NULL_TREE
, deps
= OMP_CLAUSE_DECL (c
), cond
= NULL_TREE
;
7924 gimple_stmt_iterator gsi2
= *gsi
;
7925 bool warned_step
= false;
7927 for (i
= 0; i
< fd
->ordered
; i
++)
7929 off
= TREE_PURPOSE (deps
);
7930 if (!integer_zerop (off
))
7932 gcc_assert (fd
->loops
[i
].cond_code
== LT_EXPR
7933 || fd
->loops
[i
].cond_code
== GT_EXPR
);
7934 bool forward
= fd
->loops
[i
].cond_code
== LT_EXPR
;
7935 if (forward
^ OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7936 warning_at (loc
, 0, "%<depend(sink)%> clause waiting for "
7937 "lexically later iteration");
7940 deps
= TREE_CHAIN (deps
);
7942 /* If all offsets corresponding to the collapsed loops are zero,
7943 this depend clause can be ignored. FIXME: but there is still a
7944 flush needed. We need to emit one __sync_synchronize () for it
7945 though (perhaps conditionally)? Solve this together with the
7946 conservative dependence folding optimization.
7947 if (i >= fd->collapse)
7950 deps
= OMP_CLAUSE_DECL (c
);
7952 edge e1
= split_block (gsi_bb (gsi2
), gsi_stmt (gsi2
));
7953 edge e2
= split_block_after_labels (e1
->dest
);
7955 *gsi
= gsi_after_labels (e1
->dest
);
7956 for (i
= 0; i
< fd
->ordered
; i
++)
7958 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7959 if (POINTER_TYPE_P (itype
))
7962 deps
= TREE_CHAIN (deps
);
7963 off
= TREE_PURPOSE (deps
);
7964 tree s
= fold_convert_loc (loc
, itype
, fd
->loops
[i
].step
);
7966 if (integer_zerop (off
))
7967 t
= boolean_true_node
;
7971 tree co
= fold_convert_loc (loc
, itype
, off
);
7972 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
7974 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7975 co
= fold_build1_loc (loc
, NEGATE_EXPR
, itype
, co
);
7976 a
= fold_build2_loc (loc
, POINTER_PLUS_EXPR
,
7977 TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].v
,
7980 else if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7981 a
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7982 fd
->loops
[i
].v
, co
);
7984 a
= fold_build2_loc (loc
, PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7985 fd
->loops
[i
].v
, co
);
7986 if (fd
->loops
[i
].cond_code
== LT_EXPR
)
7988 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7989 t
= fold_build2_loc (loc
, GE_EXPR
, boolean_type_node
, a
,
7992 t
= fold_build2_loc (loc
, LT_EXPR
, boolean_type_node
, a
,
7995 else if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7996 t
= fold_build2_loc (loc
, GT_EXPR
, boolean_type_node
, a
,
7999 t
= fold_build2_loc (loc
, LE_EXPR
, boolean_type_node
, a
,
8003 cond
= fold_build2_loc (loc
, BIT_AND_EXPR
, boolean_type_node
, cond
, t
);
8007 off
= fold_convert_loc (loc
, itype
, off
);
8009 if (fd
->loops
[i
].cond_code
== LT_EXPR
8010 ? !integer_onep (fd
->loops
[i
].step
)
8011 : !integer_minus_onep (fd
->loops
[i
].step
))
8013 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
8014 t
= fold_build2_loc (loc
, TRUNC_MOD_EXPR
, itype
, off
,
8015 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
8018 t
= fold_build2_loc (loc
, TRUNC_MOD_EXPR
, itype
, off
, s
);
8019 t
= fold_build2_loc (loc
, EQ_EXPR
, boolean_type_node
, t
,
8020 build_int_cst (itype
, 0));
8021 if (integer_zerop (t
) && !warned_step
)
8023 warning_at (loc
, 0, "%<depend(sink)%> refers to iteration never "
8024 "in the iteration space");
8027 cond
= fold_build2_loc (loc
, BIT_AND_EXPR
, boolean_type_node
,
8031 if (i
<= fd
->collapse
- 1 && fd
->collapse
> 1)
8037 t
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
8038 fd
->loops
[i
].v
, fd
->loops
[i
].n1
);
8039 t
= fold_convert_loc (loc
, fd
->iter_type
, t
);
8041 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
8042 off
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, off
,
8043 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
8046 off
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, off
, s
);
8047 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
8048 off
= fold_build1_loc (loc
, NEGATE_EXPR
, itype
, off
);
8049 off
= fold_convert_loc (loc
, fd
->iter_type
, off
);
8050 if (i
<= fd
->collapse
- 1 && fd
->collapse
> 1)
8053 off
= fold_build2_loc (loc
, PLUS_EXPR
, fd
->iter_type
, coff
,
8055 if (i
< fd
->collapse
- 1)
8057 coff
= fold_build2_loc (loc
, MULT_EXPR
, fd
->iter_type
, off
,
8062 off
= unshare_expr (off
);
8063 t
= fold_build2_loc (loc
, PLUS_EXPR
, fd
->iter_type
, t
, off
);
8064 t
= force_gimple_operand_gsi (gsi
, t
, true, NULL_TREE
,
8065 true, GSI_SAME_STMT
);
8068 gimple
*g
= gimple_build_call_vec (builtin_decl_explicit (sink_ix
), args
);
8069 gimple_set_location (g
, loc
);
8070 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
8072 *gsi
= gsi_last_bb (e1
->src
);
8073 cond
= unshare_expr (cond
);
8074 cond
= force_gimple_operand_gsi (gsi
, cond
, true, NULL_TREE
, false,
8075 GSI_CONTINUE_LINKING
);
8076 gsi_insert_after (gsi
, gimple_build_cond_empty (cond
), GSI_NEW_STMT
);
8077 edge e3
= make_edge (e1
->src
, e2
->dest
, EDGE_FALSE_VALUE
);
8078 e3
->probability
= REG_BR_PROB_BASE
/ 8;
8079 e1
->probability
= REG_BR_PROB_BASE
- e3
->probability
;
8080 e1
->flags
= EDGE_TRUE_VALUE
;
8081 set_immediate_dominator (CDI_DOMINATORS
, e2
->dest
, e1
->src
);
8083 *gsi
= gsi_after_labels (e2
->dest
);
8086 /* Expand all #pragma omp ordered depend(source) and
8087 #pragma omp ordered depend(sink:...) constructs in the current
8088 #pragma omp for ordered(n) region. */
8091 expand_omp_ordered_source_sink (struct omp_region
*region
,
8092 struct omp_for_data
*fd
, tree
*counts
,
8093 basic_block cont_bb
)
8095 struct omp_region
*inner
;
8097 for (i
= fd
->collapse
- 1; i
< fd
->ordered
; i
++)
8098 if (i
== fd
->collapse
- 1 && fd
->collapse
> 1)
8099 counts
[i
] = NULL_TREE
;
8100 else if (i
>= fd
->collapse
&& !cont_bb
)
8101 counts
[i
] = build_zero_cst (fd
->iter_type
);
8102 else if (!POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
))
8103 && integer_onep (fd
->loops
[i
].step
))
8104 counts
[i
] = NULL_TREE
;
8106 counts
[i
] = create_tmp_var (fd
->iter_type
, ".orditer");
8108 = build_array_type_nelts (fd
->iter_type
, fd
->ordered
- fd
->collapse
+ 1);
8109 counts
[fd
->ordered
] = create_tmp_var (atype
, ".orditera");
8110 TREE_ADDRESSABLE (counts
[fd
->ordered
]) = 1;
8112 for (inner
= region
->inner
; inner
; inner
= inner
->next
)
8113 if (inner
->type
== GIMPLE_OMP_ORDERED
)
8115 gomp_ordered
*ord_stmt
= inner
->ord_stmt
;
8116 gimple_stmt_iterator gsi
= gsi_for_stmt (ord_stmt
);
8117 location_t loc
= gimple_location (ord_stmt
);
8119 for (c
= gimple_omp_ordered_clauses (ord_stmt
);
8120 c
; c
= OMP_CLAUSE_CHAIN (c
))
8121 if (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
)
8124 expand_omp_ordered_source (&gsi
, fd
, counts
, loc
);
8125 for (c
= gimple_omp_ordered_clauses (ord_stmt
);
8126 c
; c
= OMP_CLAUSE_CHAIN (c
))
8127 if (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
)
8128 expand_omp_ordered_sink (&gsi
, fd
, counts
, c
, loc
);
8129 gsi_remove (&gsi
, true);
8133 /* Wrap the body into fd->ordered - fd->collapse loops that aren't
8137 expand_omp_for_ordered_loops (struct omp_for_data
*fd
, tree
*counts
,
8138 basic_block cont_bb
, basic_block body_bb
,
8139 bool ordered_lastprivate
)
8141 if (fd
->ordered
== fd
->collapse
)
8146 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
8147 for (int i
= fd
->collapse
; i
< fd
->ordered
; i
++)
8149 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8150 tree n1
= fold_convert (type
, fd
->loops
[i
].n1
);
8151 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, n1
);
8152 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8153 size_int (i
- fd
->collapse
+ 1),
8154 NULL_TREE
, NULL_TREE
);
8155 expand_omp_build_assign (&gsi
, aref
, build_zero_cst (fd
->iter_type
));
8160 for (int i
= fd
->ordered
- 1; i
>= fd
->collapse
; i
--)
8162 tree t
, type
= TREE_TYPE (fd
->loops
[i
].v
);
8163 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
8164 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
,
8165 fold_convert (type
, fd
->loops
[i
].n1
));
8167 expand_omp_build_assign (&gsi
, counts
[i
],
8168 build_zero_cst (fd
->iter_type
));
8169 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8170 size_int (i
- fd
->collapse
+ 1),
8171 NULL_TREE
, NULL_TREE
);
8172 expand_omp_build_assign (&gsi
, aref
, build_zero_cst (fd
->iter_type
));
8173 if (!gsi_end_p (gsi
))
8176 gsi
= gsi_last_bb (body_bb
);
8177 edge e1
= split_block (body_bb
, gsi_stmt (gsi
));
8178 basic_block new_body
= e1
->dest
;
8179 if (body_bb
== cont_bb
)
8182 basic_block new_header
;
8183 if (EDGE_COUNT (cont_bb
->preds
) > 0)
8185 gsi
= gsi_last_bb (cont_bb
);
8186 if (POINTER_TYPE_P (type
))
8187 t
= fold_build_pointer_plus (fd
->loops
[i
].v
,
8188 fold_convert (sizetype
,
8189 fd
->loops
[i
].step
));
8191 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loops
[i
].v
,
8192 fold_convert (type
, fd
->loops
[i
].step
));
8193 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
8196 t
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, counts
[i
],
8197 build_int_cst (fd
->iter_type
, 1));
8198 expand_omp_build_assign (&gsi
, counts
[i
], t
);
8203 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
8204 fd
->loops
[i
].v
, fd
->loops
[i
].n1
);
8205 t
= fold_convert (fd
->iter_type
, t
);
8206 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8207 true, GSI_SAME_STMT
);
8209 aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8210 size_int (i
- fd
->collapse
+ 1),
8211 NULL_TREE
, NULL_TREE
);
8212 expand_omp_build_assign (&gsi
, aref
, t
);
8214 e2
= split_block (cont_bb
, gsi_stmt (gsi
));
8215 new_header
= e2
->dest
;
8218 new_header
= cont_bb
;
8219 gsi
= gsi_after_labels (new_header
);
8220 tree v
= force_gimple_operand_gsi (&gsi
, fd
->loops
[i
].v
, true, NULL_TREE
,
8221 true, GSI_SAME_STMT
);
8223 = force_gimple_operand_gsi (&gsi
, fold_convert (type
, fd
->loops
[i
].n2
),
8224 true, NULL_TREE
, true, GSI_SAME_STMT
);
8225 t
= build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, n2
);
8226 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_NEW_STMT
);
8227 edge e3
= split_block (new_header
, gsi_stmt (gsi
));
8230 make_edge (body_bb
, new_header
, EDGE_FALLTHRU
);
8231 e3
->flags
= EDGE_FALSE_VALUE
;
8232 e3
->probability
= REG_BR_PROB_BASE
/ 8;
8233 e1
= make_edge (new_header
, new_body
, EDGE_TRUE_VALUE
);
8234 e1
->probability
= REG_BR_PROB_BASE
- e3
->probability
;
8236 set_immediate_dominator (CDI_DOMINATORS
, new_header
, body_bb
);
8237 set_immediate_dominator (CDI_DOMINATORS
, new_body
, new_header
);
8241 struct loop
*loop
= alloc_loop ();
8242 loop
->header
= new_header
;
8243 loop
->latch
= e2
->src
;
8244 add_loop (loop
, body_bb
->loop_father
);
8248 /* If there are any lastprivate clauses and it is possible some loops
8249 might have zero iterations, ensure all the decls are initialized,
8250 otherwise we could crash evaluating C++ class iterators with lastprivate
8252 bool need_inits
= false;
8253 for (int i
= fd
->collapse
; ordered_lastprivate
&& i
< fd
->ordered
; i
++)
8256 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8257 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
8258 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
,
8259 fold_convert (type
, fd
->loops
[i
].n1
));
8263 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8264 tree this_cond
= fold_build2 (fd
->loops
[i
].cond_code
,
8266 fold_convert (type
, fd
->loops
[i
].n1
),
8267 fold_convert (type
, fd
->loops
[i
].n2
));
8268 if (!integer_onep (this_cond
))
8276 /* A subroutine of expand_omp_for. Generate code for a parallel
8277 loop with any schedule. Given parameters:
8279 for (V = N1; V cond N2; V += STEP) BODY;
8281 where COND is "<" or ">", we generate pseudocode
8283 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
8284 if (more) goto L0; else goto L3;
8291 if (V cond iend) goto L1; else goto L2;
8293 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
8296 If this is a combined omp parallel loop, instead of the call to
8297 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
8298 If this is gimple_omp_for_combined_p loop, then instead of assigning
8299 V and iend in L0 we assign the first two _looptemp_ clause decls of the
8300 inner GIMPLE_OMP_FOR and V += STEP; and
8301 if (V cond iend) goto L1; else goto L2; are removed.
8303 For collapsed loops, given parameters:
8305 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
8306 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
8307 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
8310 we generate pseudocode
8312 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
8317 count3 = (adj + N32 - N31) / STEP3;
8318 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
8323 count2 = (adj + N22 - N21) / STEP2;
8324 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
8329 count1 = (adj + N12 - N11) / STEP1;
8330 count = count1 * count2 * count3;
8335 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
8336 if (more) goto L0; else goto L3;
8340 V3 = N31 + (T % count3) * STEP3;
8342 V2 = N21 + (T % count2) * STEP2;
8344 V1 = N11 + T * STEP1;
8349 if (V < iend) goto L10; else goto L2;
8352 if (V3 cond3 N32) goto L1; else goto L11;
8356 if (V2 cond2 N22) goto L1; else goto L12;
8362 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
8368 expand_omp_for_generic (struct omp_region
*region
,
8369 struct omp_for_data
*fd
,
8370 enum built_in_function start_fn
,
8371 enum built_in_function next_fn
,
8374 tree type
, istart0
, iend0
, iend
;
8375 tree t
, vmain
, vback
, bias
= NULL_TREE
;
8376 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
8377 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
8378 gimple_stmt_iterator gsi
;
8379 gassign
*assign_stmt
;
8380 bool in_combined_parallel
= is_combined_parallel (region
);
8381 bool broken_loop
= region
->cont
== NULL
;
8383 tree
*counts
= NULL
;
8385 bool ordered_lastprivate
= false;
8387 gcc_assert (!broken_loop
|| !in_combined_parallel
);
8388 gcc_assert (fd
->iter_type
== long_integer_type_node
8389 || !in_combined_parallel
);
8391 entry_bb
= region
->entry
;
8392 cont_bb
= region
->cont
;
8394 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
8395 gcc_assert (broken_loop
8396 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
8397 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
8398 l1_bb
= single_succ (l0_bb
);
8401 l2_bb
= create_empty_bb (cont_bb
);
8402 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
8403 || (single_succ_edge (BRANCH_EDGE (cont_bb
)->dest
)->dest
8405 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
8409 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
8410 exit_bb
= region
->exit
;
8412 gsi
= gsi_last_bb (entry_bb
);
8414 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
8416 && find_omp_clause (gimple_omp_for_clauses (gsi_stmt (gsi
)),
8417 OMP_CLAUSE_LASTPRIVATE
))
8418 ordered_lastprivate
= false;
8419 if (fd
->collapse
> 1 || fd
->ordered
)
8421 int first_zero_iter1
= -1, first_zero_iter2
= -1;
8422 basic_block zero_iter1_bb
= NULL
, zero_iter2_bb
= NULL
, l2_dom_bb
= NULL
;
8424 counts
= XALLOCAVEC (tree
, fd
->ordered
? fd
->ordered
+ 1 : fd
->collapse
);
8425 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
8426 zero_iter1_bb
, first_zero_iter1
,
8427 zero_iter2_bb
, first_zero_iter2
, l2_dom_bb
);
8431 /* Some counts[i] vars might be uninitialized if
8432 some loop has zero iterations. But the body shouldn't
8433 be executed in that case, so just avoid uninit warnings. */
8434 for (i
= first_zero_iter1
;
8435 i
< (fd
->ordered
? fd
->ordered
: fd
->collapse
); i
++)
8436 if (SSA_VAR_P (counts
[i
]))
8437 TREE_NO_WARNING (counts
[i
]) = 1;
8439 e
= split_block (entry_bb
, gsi_stmt (gsi
));
8441 make_edge (zero_iter1_bb
, entry_bb
, EDGE_FALLTHRU
);
8442 gsi
= gsi_last_bb (entry_bb
);
8443 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
8444 get_immediate_dominator (CDI_DOMINATORS
,
8449 /* Some counts[i] vars might be uninitialized if
8450 some loop has zero iterations. But the body shouldn't
8451 be executed in that case, so just avoid uninit warnings. */
8452 for (i
= first_zero_iter2
; i
< fd
->ordered
; i
++)
8453 if (SSA_VAR_P (counts
[i
]))
8454 TREE_NO_WARNING (counts
[i
]) = 1;
8456 make_edge (zero_iter2_bb
, entry_bb
, EDGE_FALLTHRU
);
8460 e
= split_block (entry_bb
, gsi_stmt (gsi
));
8462 make_edge (zero_iter2_bb
, entry_bb
, EDGE_FALLTHRU
);
8463 gsi
= gsi_last_bb (entry_bb
);
8464 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
8465 get_immediate_dominator
8466 (CDI_DOMINATORS
, zero_iter2_bb
));
8469 if (fd
->collapse
== 1)
8471 counts
[0] = fd
->loop
.n2
;
8472 fd
->loop
= fd
->loops
[0];
8476 type
= TREE_TYPE (fd
->loop
.v
);
8477 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
8478 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
8479 TREE_ADDRESSABLE (istart0
) = 1;
8480 TREE_ADDRESSABLE (iend0
) = 1;
8482 /* See if we need to bias by LLONG_MIN. */
8483 if (fd
->iter_type
== long_long_unsigned_type_node
8484 && TREE_CODE (type
) == INTEGER_TYPE
8485 && !TYPE_UNSIGNED (type
)
8486 && fd
->ordered
== 0)
8490 if (fd
->loop
.cond_code
== LT_EXPR
)
8493 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
8497 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
8500 if (TREE_CODE (n1
) != INTEGER_CST
8501 || TREE_CODE (n2
) != INTEGER_CST
8502 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
8503 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
8506 gimple_stmt_iterator gsif
= gsi
;
8509 tree arr
= NULL_TREE
;
8510 if (in_combined_parallel
)
8512 gcc_assert (fd
->ordered
== 0);
8513 /* In a combined parallel loop, emit a call to
8514 GOMP_loop_foo_next. */
8515 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
8516 build_fold_addr_expr (istart0
),
8517 build_fold_addr_expr (iend0
));
8521 tree t0
, t1
, t2
, t3
, t4
;
8522 /* If this is not a combined parallel loop, emit a call to
8523 GOMP_loop_foo_start in ENTRY_BB. */
8524 t4
= build_fold_addr_expr (iend0
);
8525 t3
= build_fold_addr_expr (istart0
);
8528 t0
= build_int_cst (unsigned_type_node
,
8529 fd
->ordered
- fd
->collapse
+ 1);
8530 arr
= create_tmp_var (build_array_type_nelts (fd
->iter_type
,
8532 - fd
->collapse
+ 1),
8534 DECL_NAMELESS (arr
) = 1;
8535 TREE_ADDRESSABLE (arr
) = 1;
8536 TREE_STATIC (arr
) = 1;
8537 vec
<constructor_elt
, va_gc
> *v
;
8538 vec_alloc (v
, fd
->ordered
- fd
->collapse
+ 1);
8541 for (idx
= 0; idx
< fd
->ordered
- fd
->collapse
+ 1; idx
++)
8544 if (idx
== 0 && fd
->collapse
> 1)
8547 c
= counts
[idx
+ fd
->collapse
- 1];
8548 tree purpose
= size_int (idx
);
8549 CONSTRUCTOR_APPEND_ELT (v
, purpose
, c
);
8550 if (TREE_CODE (c
) != INTEGER_CST
)
8551 TREE_STATIC (arr
) = 0;
8554 DECL_INITIAL (arr
) = build_constructor (TREE_TYPE (arr
), v
);
8555 if (!TREE_STATIC (arr
))
8556 force_gimple_operand_gsi (&gsi
, build1 (DECL_EXPR
,
8557 void_type_node
, arr
),
8558 true, NULL_TREE
, true, GSI_SAME_STMT
);
8559 t1
= build_fold_addr_expr (arr
);
8564 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
8567 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
8570 = find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
8571 OMP_CLAUSE__LOOPTEMP_
);
8572 gcc_assert (innerc
);
8573 t0
= OMP_CLAUSE_DECL (innerc
);
8574 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
8575 OMP_CLAUSE__LOOPTEMP_
);
8576 gcc_assert (innerc
);
8577 t1
= OMP_CLAUSE_DECL (innerc
);
8579 if (POINTER_TYPE_P (TREE_TYPE (t0
))
8580 && TYPE_PRECISION (TREE_TYPE (t0
))
8581 != TYPE_PRECISION (fd
->iter_type
))
8583 /* Avoid casting pointers to integer of a different size. */
8584 tree itype
= signed_type_for (type
);
8585 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
8586 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
8590 t1
= fold_convert (fd
->iter_type
, t1
);
8591 t0
= fold_convert (fd
->iter_type
, t0
);
8595 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
8596 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
8599 if (fd
->iter_type
== long_integer_type_node
|| fd
->ordered
)
8603 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
8604 t
= omp_adjust_chunk_size (t
, fd
->simd_schedule
);
8606 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8607 5, t0
, t1
, t
, t3
, t4
);
8609 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8610 6, t0
, t1
, t2
, t
, t3
, t4
);
8612 else if (fd
->ordered
)
8613 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8616 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8617 5, t0
, t1
, t2
, t3
, t4
);
8625 /* The GOMP_loop_ull_*start functions have additional boolean
8626 argument, true for < loops and false for > loops.
8627 In Fortran, the C bool type can be different from
8628 boolean_type_node. */
8629 bfn_decl
= builtin_decl_explicit (start_fn
);
8630 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
8631 t5
= build_int_cst (c_bool_type
,
8632 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
8635 tree bfn_decl
= builtin_decl_explicit (start_fn
);
8636 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
8637 t
= omp_adjust_chunk_size (t
, fd
->simd_schedule
);
8638 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
8641 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8642 6, t5
, t0
, t1
, t2
, t3
, t4
);
8645 if (TREE_TYPE (t
) != boolean_type_node
)
8646 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
8647 t
, build_int_cst (TREE_TYPE (t
), 0));
8648 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8649 true, GSI_SAME_STMT
);
8650 if (arr
&& !TREE_STATIC (arr
))
8652 tree clobber
= build_constructor (TREE_TYPE (arr
), NULL
);
8653 TREE_THIS_VOLATILE (clobber
) = 1;
8654 gsi_insert_before (&gsi
, gimple_build_assign (arr
, clobber
),
8657 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
8659 /* Remove the GIMPLE_OMP_FOR statement. */
8660 gsi_remove (&gsi
, true);
8662 if (gsi_end_p (gsif
))
8663 gsif
= gsi_after_labels (gsi_bb (gsif
));
8666 /* Iteration setup for sequential loop goes in L0_BB. */
8667 tree startvar
= fd
->loop
.v
;
8668 tree endvar
= NULL_TREE
;
8670 if (gimple_omp_for_combined_p (fd
->for_stmt
))
8672 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
8673 && gimple_omp_for_kind (inner_stmt
)
8674 == GF_OMP_FOR_KIND_SIMD
);
8675 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
8676 OMP_CLAUSE__LOOPTEMP_
);
8677 gcc_assert (innerc
);
8678 startvar
= OMP_CLAUSE_DECL (innerc
);
8679 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
8680 OMP_CLAUSE__LOOPTEMP_
);
8681 gcc_assert (innerc
);
8682 endvar
= OMP_CLAUSE_DECL (innerc
);
8685 gsi
= gsi_start_bb (l0_bb
);
8687 if (fd
->ordered
&& fd
->collapse
== 1)
8688 t
= fold_build2 (MULT_EXPR
, fd
->iter_type
, t
,
8689 fold_convert (fd
->iter_type
, fd
->loop
.step
));
8691 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
8692 if (fd
->ordered
&& fd
->collapse
== 1)
8694 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8695 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (startvar
),
8696 fd
->loop
.n1
, fold_convert (sizetype
, t
));
8699 t
= fold_convert (TREE_TYPE (startvar
), t
);
8700 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (startvar
),
8706 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8707 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
8708 t
= fold_convert (TREE_TYPE (startvar
), t
);
8710 t
= force_gimple_operand_gsi (&gsi
, t
,
8712 && TREE_ADDRESSABLE (startvar
),
8713 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
8714 assign_stmt
= gimple_build_assign (startvar
, t
);
8715 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8718 if (fd
->ordered
&& fd
->collapse
== 1)
8719 t
= fold_build2 (MULT_EXPR
, fd
->iter_type
, t
,
8720 fold_convert (fd
->iter_type
, fd
->loop
.step
));
8722 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
8723 if (fd
->ordered
&& fd
->collapse
== 1)
8725 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8726 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (startvar
),
8727 fd
->loop
.n1
, fold_convert (sizetype
, t
));
8730 t
= fold_convert (TREE_TYPE (startvar
), t
);
8731 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (startvar
),
8737 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8738 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
8739 t
= fold_convert (TREE_TYPE (startvar
), t
);
8741 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8742 false, GSI_CONTINUE_LINKING
);
8745 assign_stmt
= gimple_build_assign (endvar
, iend
);
8746 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8747 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
8748 assign_stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
8750 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, iend
);
8751 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8753 /* Handle linear clause adjustments. */
8754 tree itercnt
= NULL_TREE
;
8755 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
8756 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
8757 c
; c
= OMP_CLAUSE_CHAIN (c
))
8758 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
8759 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
8761 tree d
= OMP_CLAUSE_DECL (c
);
8762 bool is_ref
= is_reference (d
);
8763 tree t
= d
, a
, dest
;
8765 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
8766 tree type
= TREE_TYPE (t
);
8767 if (POINTER_TYPE_P (type
))
8769 dest
= unshare_expr (t
);
8770 tree v
= create_tmp_var (TREE_TYPE (t
), NULL
);
8771 expand_omp_build_assign (&gsif
, v
, t
);
8772 if (itercnt
== NULL_TREE
)
8775 tree n1
= fd
->loop
.n1
;
8776 if (POINTER_TYPE_P (TREE_TYPE (itercnt
)))
8779 = fold_convert (signed_type_for (TREE_TYPE (itercnt
)),
8781 n1
= fold_convert (TREE_TYPE (itercnt
), n1
);
8783 itercnt
= fold_build2 (MINUS_EXPR
, TREE_TYPE (itercnt
),
8785 itercnt
= fold_build2 (EXACT_DIV_EXPR
, TREE_TYPE (itercnt
),
8786 itercnt
, fd
->loop
.step
);
8787 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
8789 GSI_CONTINUE_LINKING
);
8791 a
= fold_build2 (MULT_EXPR
, type
,
8792 fold_convert (type
, itercnt
),
8793 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
8794 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
8795 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), v
, a
);
8796 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8797 false, GSI_CONTINUE_LINKING
);
8798 assign_stmt
= gimple_build_assign (dest
, t
);
8799 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8801 if (fd
->collapse
> 1)
8802 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
8806 /* Until now, counts array contained number of iterations or
8807 variable containing it for ith loop. From now on, we need
8808 those counts only for collapsed loops, and only for the 2nd
8809 till the last collapsed one. Move those one element earlier,
8810 we'll use counts[fd->collapse - 1] for the first source/sink
8811 iteration counter and so on and counts[fd->ordered]
8812 as the array holding the current counter values for
8814 if (fd
->collapse
> 1)
8815 memmove (counts
, counts
+ 1, (fd
->collapse
- 1) * sizeof (counts
[0]));
8819 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
8821 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8823 = fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
8824 fold_convert (type
, fd
->loops
[i
].n1
),
8825 fold_convert (type
, fd
->loops
[i
].n2
));
8826 if (!integer_onep (this_cond
))
8829 if (i
< fd
->ordered
)
8832 = create_empty_bb (EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
);
8833 add_bb_to_loop (cont_bb
, l1_bb
->loop_father
);
8834 gimple_stmt_iterator gsi
= gsi_after_labels (cont_bb
);
8835 gimple
*g
= gimple_build_omp_continue (fd
->loop
.v
, fd
->loop
.v
);
8836 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
8837 make_edge (cont_bb
, l3_bb
, EDGE_FALLTHRU
);
8838 make_edge (cont_bb
, l1_bb
, 0);
8839 l2_bb
= create_empty_bb (cont_bb
);
8840 broken_loop
= false;
8843 expand_omp_ordered_source_sink (region
, fd
, counts
, cont_bb
);
8844 cont_bb
= expand_omp_for_ordered_loops (fd
, counts
, cont_bb
, l1_bb
,
8845 ordered_lastprivate
);
8846 if (counts
[fd
->collapse
- 1])
8848 gcc_assert (fd
->collapse
== 1);
8849 gsi
= gsi_last_bb (l0_bb
);
8850 expand_omp_build_assign (&gsi
, counts
[fd
->collapse
- 1],
8852 gsi
= gsi_last_bb (cont_bb
);
8853 t
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, counts
[fd
->collapse
- 1],
8854 build_int_cst (fd
->iter_type
, 1));
8855 expand_omp_build_assign (&gsi
, counts
[fd
->collapse
- 1], t
);
8856 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8857 size_zero_node
, NULL_TREE
, NULL_TREE
);
8858 expand_omp_build_assign (&gsi
, aref
, counts
[fd
->collapse
- 1]);
8859 t
= counts
[fd
->collapse
- 1];
8861 else if (fd
->collapse
> 1)
8865 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[0].v
),
8866 fd
->loops
[0].v
, fd
->loops
[0].n1
);
8867 t
= fold_convert (fd
->iter_type
, t
);
8869 gsi
= gsi_last_bb (l0_bb
);
8870 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8871 size_zero_node
, NULL_TREE
, NULL_TREE
);
8872 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8873 false, GSI_CONTINUE_LINKING
);
8874 expand_omp_build_assign (&gsi
, aref
, t
, true);
8879 /* Code to control the increment and predicate for the sequential
8880 loop goes in the CONT_BB. */
8881 gsi
= gsi_last_bb (cont_bb
);
8882 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
8883 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
8884 vmain
= gimple_omp_continue_control_use (cont_stmt
);
8885 vback
= gimple_omp_continue_control_def (cont_stmt
);
8887 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
8889 if (POINTER_TYPE_P (type
))
8890 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
8892 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
8893 t
= force_gimple_operand_gsi (&gsi
, t
,
8895 && TREE_ADDRESSABLE (vback
),
8896 NULL_TREE
, true, GSI_SAME_STMT
);
8897 assign_stmt
= gimple_build_assign (vback
, t
);
8898 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
8900 if (fd
->ordered
&& counts
[fd
->collapse
- 1] == NULL_TREE
)
8902 if (fd
->collapse
> 1)
8906 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[0].v
),
8907 fd
->loops
[0].v
, fd
->loops
[0].n1
);
8908 t
= fold_convert (fd
->iter_type
, t
);
8910 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
,
8911 counts
[fd
->ordered
], size_zero_node
,
8912 NULL_TREE
, NULL_TREE
);
8913 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8914 true, GSI_SAME_STMT
);
8915 expand_omp_build_assign (&gsi
, aref
, t
);
8918 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
8919 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
8921 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
8922 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
8925 /* Remove GIMPLE_OMP_CONTINUE. */
8926 gsi_remove (&gsi
, true);
8928 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
8929 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
8931 /* Emit code to get the next parallel iteration in L2_BB. */
8932 gsi
= gsi_start_bb (l2_bb
);
8934 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
8935 build_fold_addr_expr (istart0
),
8936 build_fold_addr_expr (iend0
));
8937 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8938 false, GSI_CONTINUE_LINKING
);
8939 if (TREE_TYPE (t
) != boolean_type_node
)
8940 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
8941 t
, build_int_cst (TREE_TYPE (t
), 0));
8942 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
8943 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
8946 /* Add the loop cleanup function. */
8947 gsi
= gsi_last_bb (exit_bb
);
8948 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
8949 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
8950 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
8951 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
8953 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
8954 gcall
*call_stmt
= gimple_build_call (t
, 0);
8955 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
8956 gimple_call_set_lhs (call_stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
8957 gsi_insert_after (&gsi
, call_stmt
, GSI_SAME_STMT
);
8960 tree arr
= counts
[fd
->ordered
];
8961 tree clobber
= build_constructor (TREE_TYPE (arr
), NULL
);
8962 TREE_THIS_VOLATILE (clobber
) = 1;
8963 gsi_insert_after (&gsi
, gimple_build_assign (arr
, clobber
),
8966 gsi_remove (&gsi
, true);
8968 /* Connect the new blocks. */
8969 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
8970 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
8976 e
= find_edge (cont_bb
, l3_bb
);
8977 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
8979 phis
= phi_nodes (l3_bb
);
8980 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
8982 gimple
*phi
= gsi_stmt (gsi
);
8983 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
8984 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
8988 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
8989 e
= find_edge (cont_bb
, l1_bb
);
8992 e
= BRANCH_EDGE (cont_bb
);
8993 gcc_assert (single_succ (e
->dest
) == l1_bb
);
8995 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9000 else if (fd
->collapse
> 1)
9003 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
9006 e
->flags
= EDGE_TRUE_VALUE
;
9009 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
9010 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
9014 e
= find_edge (cont_bb
, l2_bb
);
9015 e
->flags
= EDGE_FALLTHRU
;
9017 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
9019 if (gimple_in_ssa_p (cfun
))
9021 /* Add phis to the outer loop that connect to the phis in the inner,
9022 original loop, and move the loop entry value of the inner phi to
9023 the loop entry value of the outer phi. */
9025 for (psi
= gsi_start_phis (l3_bb
); !gsi_end_p (psi
); gsi_next (&psi
))
9027 source_location locus
;
9029 gphi
*exit_phi
= psi
.phi ();
9031 edge l2_to_l3
= find_edge (l2_bb
, l3_bb
);
9032 tree exit_res
= PHI_ARG_DEF_FROM_EDGE (exit_phi
, l2_to_l3
);
9034 basic_block latch
= BRANCH_EDGE (cont_bb
)->dest
;
9035 edge latch_to_l1
= find_edge (latch
, l1_bb
);
9037 = find_phi_with_arg_on_edge (exit_res
, latch_to_l1
);
9039 tree t
= gimple_phi_result (exit_phi
);
9040 tree new_res
= copy_ssa_name (t
, NULL
);
9041 nphi
= create_phi_node (new_res
, l0_bb
);
9043 edge l0_to_l1
= find_edge (l0_bb
, l1_bb
);
9044 t
= PHI_ARG_DEF_FROM_EDGE (inner_phi
, l0_to_l1
);
9045 locus
= gimple_phi_arg_location_from_edge (inner_phi
, l0_to_l1
);
9046 edge entry_to_l0
= find_edge (entry_bb
, l0_bb
);
9047 add_phi_arg (nphi
, t
, entry_to_l0
, locus
);
9049 edge l2_to_l0
= find_edge (l2_bb
, l0_bb
);
9050 add_phi_arg (nphi
, exit_res
, l2_to_l0
, UNKNOWN_LOCATION
);
9052 add_phi_arg (inner_phi
, new_res
, l0_to_l1
, UNKNOWN_LOCATION
);
9056 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
9057 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
9058 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
9059 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
9060 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
9061 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
9062 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
9063 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
9065 /* We enter expand_omp_for_generic with a loop. This original loop may
9066 have its own loop struct, or it may be part of an outer loop struct
9067 (which may be the fake loop). */
9068 struct loop
*outer_loop
= entry_bb
->loop_father
;
9069 bool orig_loop_has_loop_struct
= l1_bb
->loop_father
!= outer_loop
;
9071 add_bb_to_loop (l2_bb
, outer_loop
);
9073 /* We've added a new loop around the original loop. Allocate the
9074 corresponding loop struct. */
9075 struct loop
*new_loop
= alloc_loop ();
9076 new_loop
->header
= l0_bb
;
9077 new_loop
->latch
= l2_bb
;
9078 add_loop (new_loop
, outer_loop
);
9080 /* Allocate a loop structure for the original loop unless we already
9082 if (!orig_loop_has_loop_struct
9083 && !gimple_omp_for_combined_p (fd
->for_stmt
))
9085 struct loop
*orig_loop
= alloc_loop ();
9086 orig_loop
->header
= l1_bb
;
9087 /* The loop may have multiple latches. */
9088 add_loop (orig_loop
, new_loop
);
9094 /* A subroutine of expand_omp_for. Generate code for a parallel
9095 loop with static schedule and no specified chunk size. Given
9098 for (V = N1; V cond N2; V += STEP) BODY;
9100 where COND is "<" or ">", we generate pseudocode
9102 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
9107 if ((__typeof (V)) -1 > 0 && cond is >)
9108 n = -(adj + N2 - N1) / -STEP;
9110 n = (adj + N2 - N1) / STEP;
9113 if (threadid < tt) goto L3; else goto L4;
9118 s0 = q * threadid + tt;
9121 if (s0 >= e0) goto L2; else goto L0;
9127 if (V cond e) goto L1;
9132 expand_omp_for_static_nochunk (struct omp_region
*region
,
9133 struct omp_for_data
*fd
,
9136 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
9137 tree type
, itype
, vmain
, vback
;
9138 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
9139 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
9141 gimple_stmt_iterator gsi
;
9143 bool broken_loop
= region
->cont
== NULL
;
9144 tree
*counts
= NULL
;
9147 itype
= type
= TREE_TYPE (fd
->loop
.v
);
9148 if (POINTER_TYPE_P (type
))
9149 itype
= signed_type_for (type
);
9151 entry_bb
= region
->entry
;
9152 cont_bb
= region
->cont
;
9153 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
9154 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
9155 gcc_assert (broken_loop
9156 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
9157 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
9158 body_bb
= single_succ (seq_start_bb
);
9161 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
9162 || single_succ (BRANCH_EDGE (cont_bb
)->dest
) == body_bb
);
9163 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
9165 exit_bb
= region
->exit
;
9167 /* Iteration space partitioning goes in ENTRY_BB. */
9168 gsi
= gsi_last_bb (entry_bb
);
9169 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9171 if (fd
->collapse
> 1)
9173 int first_zero_iter
= -1, dummy
= -1;
9174 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
9176 counts
= XALLOCAVEC (tree
, fd
->collapse
);
9177 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
9178 fin_bb
, first_zero_iter
,
9179 dummy_bb
, dummy
, l2_dom_bb
);
9182 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9183 t
= integer_one_node
;
9185 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
9186 fold_convert (type
, fd
->loop
.n1
),
9187 fold_convert (type
, fd
->loop
.n2
));
9188 if (fd
->collapse
== 1
9189 && TYPE_UNSIGNED (type
)
9190 && (t
== NULL_TREE
|| !integer_onep (t
)))
9192 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
9193 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
9194 true, GSI_SAME_STMT
);
9195 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
9196 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
9197 true, GSI_SAME_STMT
);
9198 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
9199 NULL_TREE
, NULL_TREE
);
9200 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
9201 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
9202 expand_omp_regimplify_p
, NULL
, NULL
)
9203 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
9204 expand_omp_regimplify_p
, NULL
, NULL
))
9206 gsi
= gsi_for_stmt (cond_stmt
);
9207 gimple_regimplify_operands (cond_stmt
, &gsi
);
9209 ep
= split_block (entry_bb
, cond_stmt
);
9210 ep
->flags
= EDGE_TRUE_VALUE
;
9211 entry_bb
= ep
->dest
;
9212 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
9213 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
9214 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
9215 if (gimple_in_ssa_p (cfun
))
9217 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
9218 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
9219 !gsi_end_p (gpi
); gsi_next (&gpi
))
9221 gphi
*phi
= gpi
.phi ();
9222 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
9223 ep
, UNKNOWN_LOCATION
);
9226 gsi
= gsi_last_bb (entry_bb
);
9229 switch (gimple_omp_for_kind (fd
->for_stmt
))
9231 case GF_OMP_FOR_KIND_FOR
:
9232 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
9233 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
9235 case GF_OMP_FOR_KIND_DISTRIBUTE
:
9236 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
9237 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
9242 nthreads
= build_call_expr (nthreads
, 0);
9243 nthreads
= fold_convert (itype
, nthreads
);
9244 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
9245 true, GSI_SAME_STMT
);
9246 threadid
= build_call_expr (threadid
, 0);
9247 threadid
= fold_convert (itype
, threadid
);
9248 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
9249 true, GSI_SAME_STMT
);
9253 step
= fd
->loop
.step
;
9254 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9256 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
9257 OMP_CLAUSE__LOOPTEMP_
);
9258 gcc_assert (innerc
);
9259 n1
= OMP_CLAUSE_DECL (innerc
);
9260 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9261 OMP_CLAUSE__LOOPTEMP_
);
9262 gcc_assert (innerc
);
9263 n2
= OMP_CLAUSE_DECL (innerc
);
9265 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
9266 true, NULL_TREE
, true, GSI_SAME_STMT
);
9267 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
9268 true, NULL_TREE
, true, GSI_SAME_STMT
);
9269 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
9270 true, NULL_TREE
, true, GSI_SAME_STMT
);
9272 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
9273 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
9274 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
9275 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
9276 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
9277 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
9278 fold_build1 (NEGATE_EXPR
, itype
, t
),
9279 fold_build1 (NEGATE_EXPR
, itype
, step
));
9281 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
9282 t
= fold_convert (itype
, t
);
9283 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
9285 q
= create_tmp_reg (itype
, "q");
9286 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
9287 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
9288 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
9290 tt
= create_tmp_reg (itype
, "tt");
9291 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
9292 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
9293 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
9295 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
9296 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
9297 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
9299 second_bb
= split_block (entry_bb
, cond_stmt
)->dest
;
9300 gsi
= gsi_last_bb (second_bb
);
9301 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9303 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
9305 gassign
*assign_stmt
9306 = gimple_build_assign (q
, PLUS_EXPR
, q
, build_int_cst (itype
, 1));
9307 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9309 third_bb
= split_block (second_bb
, assign_stmt
)->dest
;
9310 gsi
= gsi_last_bb (third_bb
);
9311 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9313 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
9314 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
9315 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
9317 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
9318 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
9320 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
9321 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
9323 /* Remove the GIMPLE_OMP_FOR statement. */
9324 gsi_remove (&gsi
, true);
9326 /* Setup code for sequential iteration goes in SEQ_START_BB. */
9327 gsi
= gsi_start_bb (seq_start_bb
);
9329 tree startvar
= fd
->loop
.v
;
9330 tree endvar
= NULL_TREE
;
9332 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9334 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
9335 ? gimple_omp_parallel_clauses (inner_stmt
)
9336 : gimple_omp_for_clauses (inner_stmt
);
9337 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
9338 gcc_assert (innerc
);
9339 startvar
= OMP_CLAUSE_DECL (innerc
);
9340 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9341 OMP_CLAUSE__LOOPTEMP_
);
9342 gcc_assert (innerc
);
9343 endvar
= OMP_CLAUSE_DECL (innerc
);
9344 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
9345 && gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
9348 for (i
= 1; i
< fd
->collapse
; i
++)
9350 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9351 OMP_CLAUSE__LOOPTEMP_
);
9352 gcc_assert (innerc
);
9354 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9355 OMP_CLAUSE__LOOPTEMP_
);
9358 /* If needed (distribute parallel for with lastprivate),
9359 propagate down the total number of iterations. */
9360 tree t
= fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (innerc
)),
9362 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, false,
9363 GSI_CONTINUE_LINKING
);
9364 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
9365 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9369 t
= fold_convert (itype
, s0
);
9370 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9371 if (POINTER_TYPE_P (type
))
9372 t
= fold_build_pointer_plus (n1
, t
);
9374 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9375 t
= fold_convert (TREE_TYPE (startvar
), t
);
9376 t
= force_gimple_operand_gsi (&gsi
, t
,
9378 && TREE_ADDRESSABLE (startvar
),
9379 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
9380 assign_stmt
= gimple_build_assign (startvar
, t
);
9381 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9383 t
= fold_convert (itype
, e0
);
9384 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9385 if (POINTER_TYPE_P (type
))
9386 t
= fold_build_pointer_plus (n1
, t
);
9388 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9389 t
= fold_convert (TREE_TYPE (startvar
), t
);
9390 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9391 false, GSI_CONTINUE_LINKING
);
9394 assign_stmt
= gimple_build_assign (endvar
, e
);
9395 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9396 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
9397 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
9399 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
9400 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9402 /* Handle linear clause adjustments. */
9403 tree itercnt
= NULL_TREE
;
9404 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
9405 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
9406 c
; c
= OMP_CLAUSE_CHAIN (c
))
9407 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
9408 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
9410 tree d
= OMP_CLAUSE_DECL (c
);
9411 bool is_ref
= is_reference (d
);
9412 tree t
= d
, a
, dest
;
9414 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
9415 if (itercnt
== NULL_TREE
)
9417 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9419 itercnt
= fold_build2 (MINUS_EXPR
, itype
,
9420 fold_convert (itype
, n1
),
9421 fold_convert (itype
, fd
->loop
.n1
));
9422 itercnt
= fold_build2 (EXACT_DIV_EXPR
, itype
, itercnt
, step
);
9423 itercnt
= fold_build2 (PLUS_EXPR
, itype
, itercnt
, s0
);
9424 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
9426 GSI_CONTINUE_LINKING
);
9431 tree type
= TREE_TYPE (t
);
9432 if (POINTER_TYPE_P (type
))
9434 a
= fold_build2 (MULT_EXPR
, type
,
9435 fold_convert (type
, itercnt
),
9436 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
9437 dest
= unshare_expr (t
);
9438 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
9439 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), t
, a
);
9440 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9441 false, GSI_CONTINUE_LINKING
);
9442 assign_stmt
= gimple_build_assign (dest
, t
);
9443 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9445 if (fd
->collapse
> 1)
9446 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
9450 /* The code controlling the sequential loop replaces the
9451 GIMPLE_OMP_CONTINUE. */
9452 gsi
= gsi_last_bb (cont_bb
);
9453 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
9454 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
9455 vmain
= gimple_omp_continue_control_use (cont_stmt
);
9456 vback
= gimple_omp_continue_control_def (cont_stmt
);
9458 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
9460 if (POINTER_TYPE_P (type
))
9461 t
= fold_build_pointer_plus (vmain
, step
);
9463 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
9464 t
= force_gimple_operand_gsi (&gsi
, t
,
9466 && TREE_ADDRESSABLE (vback
),
9467 NULL_TREE
, true, GSI_SAME_STMT
);
9468 assign_stmt
= gimple_build_assign (vback
, t
);
9469 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9471 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
9472 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
9474 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
9477 /* Remove the GIMPLE_OMP_CONTINUE statement. */
9478 gsi_remove (&gsi
, true);
9480 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
9481 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
9484 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
9485 gsi
= gsi_last_bb (exit_bb
);
9486 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
9488 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
9489 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
9491 gsi_remove (&gsi
, true);
9493 /* Connect all the blocks. */
9494 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
9495 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
9496 ep
= find_edge (entry_bb
, second_bb
);
9497 ep
->flags
= EDGE_TRUE_VALUE
;
9498 ep
->probability
= REG_BR_PROB_BASE
/ 4;
9499 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
9500 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
9504 ep
= find_edge (cont_bb
, body_bb
);
9507 ep
= BRANCH_EDGE (cont_bb
);
9508 gcc_assert (single_succ (ep
->dest
) == body_bb
);
9510 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9515 else if (fd
->collapse
> 1)
9518 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
9521 ep
->flags
= EDGE_TRUE_VALUE
;
9522 find_edge (cont_bb
, fin_bb
)->flags
9523 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
9526 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
9527 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
9528 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
9530 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
9531 recompute_dominator (CDI_DOMINATORS
, body_bb
));
9532 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
9533 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
9535 struct loop
*loop
= body_bb
->loop_father
;
9536 if (loop
!= entry_bb
->loop_father
)
9538 gcc_assert (loop
->header
== body_bb
);
9539 gcc_assert (broken_loop
9540 || loop
->latch
== region
->cont
9541 || single_pred (loop
->latch
) == region
->cont
);
9545 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
9547 loop
= alloc_loop ();
9548 loop
->header
= body_bb
;
9549 if (collapse_bb
== NULL
)
9550 loop
->latch
= cont_bb
;
9551 add_loop (loop
, body_bb
->loop_father
);
9555 /* Return phi in E->DEST with ARG on edge E. */
9558 find_phi_with_arg_on_edge (tree arg
, edge e
)
9560 basic_block bb
= e
->dest
;
9562 for (gphi_iterator gpi
= gsi_start_phis (bb
);
9566 gphi
*phi
= gpi
.phi ();
9567 if (PHI_ARG_DEF_FROM_EDGE (phi
, e
) == arg
)
9574 /* A subroutine of expand_omp_for. Generate code for a parallel
9575 loop with static schedule and a specified chunk size. Given
9578 for (V = N1; V cond N2; V += STEP) BODY;
9580 where COND is "<" or ">", we generate pseudocode
9582 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
9587 if ((__typeof (V)) -1 > 0 && cond is >)
9588 n = -(adj + N2 - N1) / -STEP;
9590 n = (adj + N2 - N1) / STEP;
9592 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
9593 here so that V is defined
9594 if the loop is not entered
9596 s0 = (trip * nthreads + threadid) * CHUNK;
9597 e0 = min(s0 + CHUNK, n);
9598 if (s0 < n) goto L1; else goto L4;
9605 if (V cond e) goto L2; else goto L3;
9613 expand_omp_for_static_chunk (struct omp_region
*region
,
9614 struct omp_for_data
*fd
, gimple
*inner_stmt
)
9616 tree n
, s0
, e0
, e
, t
;
9617 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
9618 tree type
, itype
, vmain
, vback
, vextra
;
9619 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
9620 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
9621 gimple_stmt_iterator gsi
;
9623 bool broken_loop
= region
->cont
== NULL
;
9624 tree
*counts
= NULL
;
9627 itype
= type
= TREE_TYPE (fd
->loop
.v
);
9628 if (POINTER_TYPE_P (type
))
9629 itype
= signed_type_for (type
);
9631 entry_bb
= region
->entry
;
9632 se
= split_block (entry_bb
, last_stmt (entry_bb
));
9634 iter_part_bb
= se
->dest
;
9635 cont_bb
= region
->cont
;
9636 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
9637 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
9638 gcc_assert (broken_loop
9639 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
9640 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
9641 body_bb
= single_succ (seq_start_bb
);
9644 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
9645 || single_succ (BRANCH_EDGE (cont_bb
)->dest
) == body_bb
);
9646 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
9647 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
9649 exit_bb
= region
->exit
;
9651 /* Trip and adjustment setup goes in ENTRY_BB. */
9652 gsi
= gsi_last_bb (entry_bb
);
9653 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9655 if (fd
->collapse
> 1)
9657 int first_zero_iter
= -1, dummy
= -1;
9658 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
9660 counts
= XALLOCAVEC (tree
, fd
->collapse
);
9661 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
9662 fin_bb
, first_zero_iter
,
9663 dummy_bb
, dummy
, l2_dom_bb
);
9666 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9667 t
= integer_one_node
;
9669 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
9670 fold_convert (type
, fd
->loop
.n1
),
9671 fold_convert (type
, fd
->loop
.n2
));
9672 if (fd
->collapse
== 1
9673 && TYPE_UNSIGNED (type
)
9674 && (t
== NULL_TREE
|| !integer_onep (t
)))
9676 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
9677 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
9678 true, GSI_SAME_STMT
);
9679 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
9680 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
9681 true, GSI_SAME_STMT
);
9682 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
9683 NULL_TREE
, NULL_TREE
);
9684 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
9685 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
9686 expand_omp_regimplify_p
, NULL
, NULL
)
9687 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
9688 expand_omp_regimplify_p
, NULL
, NULL
))
9690 gsi
= gsi_for_stmt (cond_stmt
);
9691 gimple_regimplify_operands (cond_stmt
, &gsi
);
9693 se
= split_block (entry_bb
, cond_stmt
);
9694 se
->flags
= EDGE_TRUE_VALUE
;
9695 entry_bb
= se
->dest
;
9696 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
9697 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
9698 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
9699 if (gimple_in_ssa_p (cfun
))
9701 int dest_idx
= find_edge (iter_part_bb
, fin_bb
)->dest_idx
;
9702 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
9703 !gsi_end_p (gpi
); gsi_next (&gpi
))
9705 gphi
*phi
= gpi
.phi ();
9706 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
9707 se
, UNKNOWN_LOCATION
);
9710 gsi
= gsi_last_bb (entry_bb
);
9713 switch (gimple_omp_for_kind (fd
->for_stmt
))
9715 case GF_OMP_FOR_KIND_FOR
:
9716 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
9717 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
9719 case GF_OMP_FOR_KIND_DISTRIBUTE
:
9720 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
9721 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
9726 nthreads
= build_call_expr (nthreads
, 0);
9727 nthreads
= fold_convert (itype
, nthreads
);
9728 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
9729 true, GSI_SAME_STMT
);
9730 threadid
= build_call_expr (threadid
, 0);
9731 threadid
= fold_convert (itype
, threadid
);
9732 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
9733 true, GSI_SAME_STMT
);
9737 step
= fd
->loop
.step
;
9738 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9740 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
9741 OMP_CLAUSE__LOOPTEMP_
);
9742 gcc_assert (innerc
);
9743 n1
= OMP_CLAUSE_DECL (innerc
);
9744 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9745 OMP_CLAUSE__LOOPTEMP_
);
9746 gcc_assert (innerc
);
9747 n2
= OMP_CLAUSE_DECL (innerc
);
9749 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
9750 true, NULL_TREE
, true, GSI_SAME_STMT
);
9751 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
9752 true, NULL_TREE
, true, GSI_SAME_STMT
);
9753 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
9754 true, NULL_TREE
, true, GSI_SAME_STMT
);
9755 tree chunk_size
= fold_convert (itype
, fd
->chunk_size
);
9756 chunk_size
= omp_adjust_chunk_size (chunk_size
, fd
->simd_schedule
);
9758 = force_gimple_operand_gsi (&gsi
, chunk_size
, true, NULL_TREE
, true,
9761 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
9762 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
9763 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
9764 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
9765 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
9766 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
9767 fold_build1 (NEGATE_EXPR
, itype
, t
),
9768 fold_build1 (NEGATE_EXPR
, itype
, step
));
9770 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
9771 t
= fold_convert (itype
, t
);
9772 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9773 true, GSI_SAME_STMT
);
9775 trip_var
= create_tmp_reg (itype
, ".trip");
9776 if (gimple_in_ssa_p (cfun
))
9778 trip_init
= make_ssa_name (trip_var
);
9779 trip_main
= make_ssa_name (trip_var
);
9780 trip_back
= make_ssa_name (trip_var
);
9784 trip_init
= trip_var
;
9785 trip_main
= trip_var
;
9786 trip_back
= trip_var
;
9789 gassign
*assign_stmt
9790 = gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
9791 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9793 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, chunk_size
);
9794 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9795 if (POINTER_TYPE_P (type
))
9796 t
= fold_build_pointer_plus (n1
, t
);
9798 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9799 vextra
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9800 true, GSI_SAME_STMT
);
9802 /* Remove the GIMPLE_OMP_FOR. */
9803 gsi_remove (&gsi
, true);
9805 gimple_stmt_iterator gsif
= gsi
;
9807 /* Iteration space partitioning goes in ITER_PART_BB. */
9808 gsi
= gsi_last_bb (iter_part_bb
);
9810 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
9811 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
9812 t
= fold_build2 (MULT_EXPR
, itype
, t
, chunk_size
);
9813 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9814 false, GSI_CONTINUE_LINKING
);
9816 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, chunk_size
);
9817 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
9818 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9819 false, GSI_CONTINUE_LINKING
);
9821 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
9822 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
9824 /* Setup code for sequential iteration goes in SEQ_START_BB. */
9825 gsi
= gsi_start_bb (seq_start_bb
);
9827 tree startvar
= fd
->loop
.v
;
9828 tree endvar
= NULL_TREE
;
9830 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9832 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
9833 ? gimple_omp_parallel_clauses (inner_stmt
)
9834 : gimple_omp_for_clauses (inner_stmt
);
9835 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
9836 gcc_assert (innerc
);
9837 startvar
= OMP_CLAUSE_DECL (innerc
);
9838 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9839 OMP_CLAUSE__LOOPTEMP_
);
9840 gcc_assert (innerc
);
9841 endvar
= OMP_CLAUSE_DECL (innerc
);
9842 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
9843 && gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
9846 for (i
= 1; i
< fd
->collapse
; i
++)
9848 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9849 OMP_CLAUSE__LOOPTEMP_
);
9850 gcc_assert (innerc
);
9852 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9853 OMP_CLAUSE__LOOPTEMP_
);
9856 /* If needed (distribute parallel for with lastprivate),
9857 propagate down the total number of iterations. */
9858 tree t
= fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (innerc
)),
9860 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, false,
9861 GSI_CONTINUE_LINKING
);
9862 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
9863 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9868 t
= fold_convert (itype
, s0
);
9869 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9870 if (POINTER_TYPE_P (type
))
9871 t
= fold_build_pointer_plus (n1
, t
);
9873 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9874 t
= fold_convert (TREE_TYPE (startvar
), t
);
9875 t
= force_gimple_operand_gsi (&gsi
, t
,
9877 && TREE_ADDRESSABLE (startvar
),
9878 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
9879 assign_stmt
= gimple_build_assign (startvar
, t
);
9880 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9882 t
= fold_convert (itype
, e0
);
9883 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9884 if (POINTER_TYPE_P (type
))
9885 t
= fold_build_pointer_plus (n1
, t
);
9887 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9888 t
= fold_convert (TREE_TYPE (startvar
), t
);
9889 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9890 false, GSI_CONTINUE_LINKING
);
9893 assign_stmt
= gimple_build_assign (endvar
, e
);
9894 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9895 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
9896 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
9898 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
9899 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9901 /* Handle linear clause adjustments. */
9902 tree itercnt
= NULL_TREE
, itercntbias
= NULL_TREE
;
9903 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
9904 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
9905 c
; c
= OMP_CLAUSE_CHAIN (c
))
9906 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
9907 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
9909 tree d
= OMP_CLAUSE_DECL (c
);
9910 bool is_ref
= is_reference (d
);
9911 tree t
= d
, a
, dest
;
9913 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
9914 tree type
= TREE_TYPE (t
);
9915 if (POINTER_TYPE_P (type
))
9917 dest
= unshare_expr (t
);
9918 tree v
= create_tmp_var (TREE_TYPE (t
), NULL
);
9919 expand_omp_build_assign (&gsif
, v
, t
);
9920 if (itercnt
== NULL_TREE
)
9922 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9925 = fold_build2 (MINUS_EXPR
, itype
, fold_convert (itype
, n1
),
9926 fold_convert (itype
, fd
->loop
.n1
));
9927 itercntbias
= fold_build2 (EXACT_DIV_EXPR
, itype
,
9930 = force_gimple_operand_gsi (&gsif
, itercntbias
, true,
9933 itercnt
= fold_build2 (PLUS_EXPR
, itype
, itercntbias
, s0
);
9934 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
9936 GSI_CONTINUE_LINKING
);
9941 a
= fold_build2 (MULT_EXPR
, type
,
9942 fold_convert (type
, itercnt
),
9943 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
9944 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
9945 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), v
, a
);
9946 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9947 false, GSI_CONTINUE_LINKING
);
9948 assign_stmt
= gimple_build_assign (dest
, t
);
9949 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9951 if (fd
->collapse
> 1)
9952 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
9956 /* The code controlling the sequential loop goes in CONT_BB,
9957 replacing the GIMPLE_OMP_CONTINUE. */
9958 gsi
= gsi_last_bb (cont_bb
);
9959 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
9960 vmain
= gimple_omp_continue_control_use (cont_stmt
);
9961 vback
= gimple_omp_continue_control_def (cont_stmt
);
9963 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
9965 if (POINTER_TYPE_P (type
))
9966 t
= fold_build_pointer_plus (vmain
, step
);
9968 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
9969 if (DECL_P (vback
) && TREE_ADDRESSABLE (vback
))
9970 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9971 true, GSI_SAME_STMT
);
9972 assign_stmt
= gimple_build_assign (vback
, t
);
9973 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9975 if (tree_int_cst_equal (fd
->chunk_size
, integer_one_node
))
9976 t
= build2 (EQ_EXPR
, boolean_type_node
,
9977 build_int_cst (itype
, 0),
9978 build_int_cst (itype
, 1));
9980 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
9981 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
9983 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
9986 /* Remove GIMPLE_OMP_CONTINUE. */
9987 gsi_remove (&gsi
, true);
9989 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
9990 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
9992 /* Trip update code goes into TRIP_UPDATE_BB. */
9993 gsi
= gsi_start_bb (trip_update_bb
);
9995 t
= build_int_cst (itype
, 1);
9996 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
9997 assign_stmt
= gimple_build_assign (trip_back
, t
);
9998 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10001 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
10002 gsi
= gsi_last_bb (exit_bb
);
10003 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
10005 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
10006 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
10008 gsi_remove (&gsi
, true);
10010 /* Connect the new blocks. */
10011 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
10012 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
10016 se
= find_edge (cont_bb
, body_bb
);
10019 se
= BRANCH_EDGE (cont_bb
);
10020 gcc_assert (single_succ (se
->dest
) == body_bb
);
10022 if (gimple_omp_for_combined_p (fd
->for_stmt
))
10027 else if (fd
->collapse
> 1)
10030 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
10033 se
->flags
= EDGE_TRUE_VALUE
;
10034 find_edge (cont_bb
, trip_update_bb
)->flags
10035 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
10037 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
10040 if (gimple_in_ssa_p (cfun
))
10048 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
10050 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
10051 remove arguments of the phi nodes in fin_bb. We need to create
10052 appropriate phi nodes in iter_part_bb instead. */
10053 se
= find_edge (iter_part_bb
, fin_bb
);
10054 re
= single_succ_edge (trip_update_bb
);
10055 vec
<edge_var_map
> *head
= redirect_edge_var_map_vector (re
);
10056 ene
= single_succ_edge (entry_bb
);
10058 psi
= gsi_start_phis (fin_bb
);
10059 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
10060 gsi_next (&psi
), ++i
)
10063 source_location locus
;
10066 t
= gimple_phi_result (phi
);
10067 gcc_assert (t
== redirect_edge_var_map_result (vm
));
10069 if (!single_pred_p (fin_bb
))
10070 t
= copy_ssa_name (t
, phi
);
10072 nphi
= create_phi_node (t
, iter_part_bb
);
10074 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
10075 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
10077 /* A special case -- fd->loop.v is not yet computed in
10078 iter_part_bb, we need to use vextra instead. */
10079 if (t
== fd
->loop
.v
)
10081 add_phi_arg (nphi
, t
, ene
, locus
);
10082 locus
= redirect_edge_var_map_location (vm
);
10083 tree back_arg
= redirect_edge_var_map_def (vm
);
10084 add_phi_arg (nphi
, back_arg
, re
, locus
);
10085 edge ce
= find_edge (cont_bb
, body_bb
);
10088 ce
= BRANCH_EDGE (cont_bb
);
10089 gcc_assert (single_succ (ce
->dest
) == body_bb
);
10090 ce
= single_succ_edge (ce
->dest
);
10092 gphi
*inner_loop_phi
= find_phi_with_arg_on_edge (back_arg
, ce
);
10093 gcc_assert (inner_loop_phi
!= NULL
);
10094 add_phi_arg (inner_loop_phi
, gimple_phi_result (nphi
),
10095 find_edge (seq_start_bb
, body_bb
), locus
);
10097 if (!single_pred_p (fin_bb
))
10098 add_phi_arg (phi
, gimple_phi_result (nphi
), se
, locus
);
10100 gcc_assert (gsi_end_p (psi
) && (head
== NULL
|| i
== head
->length ()));
10101 redirect_edge_var_map_clear (re
);
10102 if (single_pred_p (fin_bb
))
10105 psi
= gsi_start_phis (fin_bb
);
10106 if (gsi_end_p (psi
))
10108 remove_phi_node (&psi
, false);
10111 /* Make phi node for trip. */
10112 phi
= create_phi_node (trip_main
, iter_part_bb
);
10113 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
10115 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
10120 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
10121 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
10122 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
10123 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
10124 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
10125 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
10126 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
10127 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
10128 recompute_dominator (CDI_DOMINATORS
, body_bb
));
10132 struct loop
*loop
= body_bb
->loop_father
;
10133 struct loop
*trip_loop
= alloc_loop ();
10134 trip_loop
->header
= iter_part_bb
;
10135 trip_loop
->latch
= trip_update_bb
;
10136 add_loop (trip_loop
, iter_part_bb
->loop_father
);
10138 if (loop
!= entry_bb
->loop_father
)
10140 gcc_assert (loop
->header
== body_bb
);
10141 gcc_assert (loop
->latch
== region
->cont
10142 || single_pred (loop
->latch
) == region
->cont
);
10143 trip_loop
->inner
= loop
;
10147 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
10149 loop
= alloc_loop ();
10150 loop
->header
= body_bb
;
10151 if (collapse_bb
== NULL
)
10152 loop
->latch
= cont_bb
;
10153 add_loop (loop
, trip_loop
);
10158 /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop.
10160 for (V = N1; V cond N2; V += STEP) BODY;
10162 where COND is "<" or ">" or "!=", we generate pseudocode
10164 for (ind_var = low; ind_var < high; ind_var++)
10166 V = n1 + (ind_var * STEP)
10171 In the above pseudocode, low and high are function parameters of the
10172 child function. In the function below, we are inserting a temp.
10173 variable that will be making a call to two OMP functions that will not be
10174 found in the body of _Cilk_for (since OMP_FOR cannot be mixed
10175 with _Cilk_for). These functions are replaced with low and high
10176 by the function that handles taskreg. */
10180 expand_cilk_for (struct omp_region
*region
, struct omp_for_data
*fd
)
10182 bool broken_loop
= region
->cont
== NULL
;
10183 basic_block entry_bb
= region
->entry
;
10184 basic_block cont_bb
= region
->cont
;
10186 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10187 gcc_assert (broken_loop
10188 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
10189 basic_block l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10190 basic_block l1_bb
, l2_bb
;
10194 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
10195 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
10196 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
10197 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
10201 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
10202 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
10203 l2_bb
= single_succ (l1_bb
);
10205 basic_block exit_bb
= region
->exit
;
10206 basic_block l2_dom_bb
= NULL
;
10208 gimple_stmt_iterator gsi
= gsi_last_bb (entry_bb
);
10210 /* Below statements until the "tree high_val = ..." are pseudo statements
10211 used to pass information to be used by expand_omp_taskreg.
10212 low_val and high_val will be replaced by the __low and __high
10213 parameter from the child function.
10215 The call_exprs part is a place-holder, it is mainly used
10216 to distinctly identify to the top-level part that this is
10217 where we should put low and high (reasoning given in header
10221 = gimple_omp_parallel_child_fn (
10222 as_a
<gomp_parallel
*> (last_stmt (region
->outer
->entry
)));
10223 tree t
, low_val
= NULL_TREE
, high_val
= NULL_TREE
;
10224 for (t
= DECL_ARGUMENTS (child_fndecl
); t
; t
= TREE_CHAIN (t
))
10226 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__high"))
10228 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__low"))
10231 gcc_assert (low_val
&& high_val
);
10233 tree type
= TREE_TYPE (low_val
);
10234 tree ind_var
= create_tmp_reg (type
, "__cilk_ind_var");
10235 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
10237 /* Not needed in SSA form right now. */
10238 gcc_assert (!gimple_in_ssa_p (cfun
));
10239 if (l2_dom_bb
== NULL
)
10243 tree n2
= high_val
;
10245 gimple
*stmt
= gimple_build_assign (ind_var
, n1
);
10247 /* Replace the GIMPLE_OMP_FOR statement. */
10248 gsi_replace (&gsi
, stmt
, true);
10252 /* Code to control the increment goes in the CONT_BB. */
10253 gsi
= gsi_last_bb (cont_bb
);
10254 stmt
= gsi_stmt (gsi
);
10255 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
10256 stmt
= gimple_build_assign (ind_var
, PLUS_EXPR
, ind_var
,
10257 build_one_cst (type
));
10259 /* Replace GIMPLE_OMP_CONTINUE. */
10260 gsi_replace (&gsi
, stmt
, true);
10263 /* Emit the condition in L1_BB. */
10264 gsi
= gsi_after_labels (l1_bb
);
10265 t
= fold_build2 (MULT_EXPR
, TREE_TYPE (fd
->loop
.step
),
10266 fold_convert (TREE_TYPE (fd
->loop
.step
), ind_var
),
10268 if (POINTER_TYPE_P (TREE_TYPE (fd
->loop
.n1
)))
10269 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
10270 fd
->loop
.n1
, fold_convert (sizetype
, t
));
10272 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
10273 fd
->loop
.n1
, fold_convert (TREE_TYPE (fd
->loop
.n1
), t
));
10274 t
= fold_convert (TREE_TYPE (fd
->loop
.v
), t
);
10275 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
10277 /* The condition is always '<' since the runtime will fill in the low
10278 and high values. */
10279 stmt
= gimple_build_cond (LT_EXPR
, ind_var
, n2
, NULL_TREE
, NULL_TREE
);
10280 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
10282 /* Remove GIMPLE_OMP_RETURN. */
10283 gsi
= gsi_last_bb (exit_bb
);
10284 gsi_remove (&gsi
, true);
10286 /* Connect the new blocks. */
10287 remove_edge (FALLTHRU_EDGE (entry_bb
));
10292 remove_edge (BRANCH_EDGE (entry_bb
));
10293 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
10295 e
= BRANCH_EDGE (l1_bb
);
10296 ne
= FALLTHRU_EDGE (l1_bb
);
10297 e
->flags
= EDGE_TRUE_VALUE
;
10301 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
10303 ne
= single_succ_edge (l1_bb
);
10304 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
10307 ne
->flags
= EDGE_FALSE_VALUE
;
10308 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
10309 ne
->probability
= REG_BR_PROB_BASE
/ 8;
10311 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
10312 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
10313 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
10317 struct loop
*loop
= alloc_loop ();
10318 loop
->header
= l1_bb
;
10319 loop
->latch
= cont_bb
;
10320 add_loop (loop
, l1_bb
->loop_father
);
10321 loop
->safelen
= INT_MAX
;
10324 /* Pick the correct library function based on the precision of the
10325 induction variable type. */
10326 tree lib_fun
= NULL_TREE
;
10327 if (TYPE_PRECISION (type
) == 32)
10328 lib_fun
= cilk_for_32_fndecl
;
10329 else if (TYPE_PRECISION (type
) == 64)
10330 lib_fun
= cilk_for_64_fndecl
;
10332 gcc_unreachable ();
10334 gcc_assert (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_CILKFOR
);
10336 /* WS_ARGS contains the library function flavor to call:
10337 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the
10338 user-defined grain value. If the user does not define one, then zero
10339 is passed in by the parser. */
10340 vec_alloc (region
->ws_args
, 2);
10341 region
->ws_args
->quick_push (lib_fun
);
10342 region
->ws_args
->quick_push (fd
->chunk_size
);
10345 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
10346 loop. Given parameters:
10348 for (V = N1; V cond N2; V += STEP) BODY;
10350 where COND is "<" or ">", we generate pseudocode
10358 if (V cond N2) goto L0; else goto L2;
10361 For collapsed loops, given parameters:
10363 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
10364 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
10365 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
10368 we generate pseudocode
10374 count3 = (adj + N32 - N31) / STEP3;
10379 count2 = (adj + N22 - N21) / STEP2;
10384 count1 = (adj + N12 - N11) / STEP1;
10385 count = count1 * count2 * count3;
10395 V2 += (V3 cond3 N32) ? 0 : STEP2;
10396 V3 = (V3 cond3 N32) ? V3 : N31;
10397 V1 += (V2 cond2 N22) ? 0 : STEP1;
10398 V2 = (V2 cond2 N22) ? V2 : N21;
10400 if (V < count) goto L0; else goto L2;
10406 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
10409 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
10410 gimple_stmt_iterator gsi
;
10413 bool broken_loop
= region
->cont
== NULL
;
10415 tree
*counts
= NULL
;
10417 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10418 OMP_CLAUSE_SAFELEN
);
10419 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10420 OMP_CLAUSE__SIMDUID_
);
10423 type
= TREE_TYPE (fd
->loop
.v
);
10424 entry_bb
= region
->entry
;
10425 cont_bb
= region
->cont
;
10426 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10427 gcc_assert (broken_loop
10428 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
10429 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10432 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
10433 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
10434 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
10435 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
10439 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
10440 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
10441 l2_bb
= single_succ (l1_bb
);
10443 exit_bb
= region
->exit
;
10446 gsi
= gsi_last_bb (entry_bb
);
10448 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
10449 /* Not needed in SSA form right now. */
10450 gcc_assert (!gimple_in_ssa_p (cfun
));
10451 if (fd
->collapse
> 1)
10453 int first_zero_iter
= -1, dummy
= -1;
10454 basic_block zero_iter_bb
= l2_bb
, dummy_bb
= NULL
;
10456 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10457 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10458 zero_iter_bb
, first_zero_iter
,
10459 dummy_bb
, dummy
, l2_dom_bb
);
10461 if (l2_dom_bb
== NULL
)
10466 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
10468 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10469 OMP_CLAUSE__LOOPTEMP_
);
10470 gcc_assert (innerc
);
10471 n1
= OMP_CLAUSE_DECL (innerc
);
10472 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10473 OMP_CLAUSE__LOOPTEMP_
);
10474 gcc_assert (innerc
);
10475 n2
= OMP_CLAUSE_DECL (innerc
);
10476 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
10477 fold_convert (type
, n1
));
10478 if (fd
->collapse
> 1)
10481 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
10487 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
10488 fold_convert (type
, fd
->loop
.n1
));
10489 if (fd
->collapse
> 1)
10490 for (i
= 0; i
< fd
->collapse
; i
++)
10492 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
10493 if (POINTER_TYPE_P (itype
))
10494 itype
= signed_type_for (itype
);
10495 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
10496 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10500 /* Remove the GIMPLE_OMP_FOR statement. */
10501 gsi_remove (&gsi
, true);
10505 /* Code to control the increment goes in the CONT_BB. */
10506 gsi
= gsi_last_bb (cont_bb
);
10507 stmt
= gsi_stmt (gsi
);
10508 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
10510 if (POINTER_TYPE_P (type
))
10511 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
10513 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
10514 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
10516 if (fd
->collapse
> 1)
10518 i
= fd
->collapse
- 1;
10519 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
10521 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
10522 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
10526 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
10527 fd
->loops
[i
].step
);
10528 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
10529 fd
->loops
[i
].v
, t
);
10531 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10533 for (i
= fd
->collapse
- 1; i
> 0; i
--)
10535 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
10536 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
10537 if (POINTER_TYPE_P (itype2
))
10538 itype2
= signed_type_for (itype2
);
10539 t
= build3 (COND_EXPR
, itype2
,
10540 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
10542 fold_convert (itype
, fd
->loops
[i
].n2
)),
10543 build_int_cst (itype2
, 0),
10544 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
10545 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
10546 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
10548 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
10549 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
10551 t
= build3 (COND_EXPR
, itype
,
10552 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
10554 fold_convert (itype
, fd
->loops
[i
].n2
)),
10556 fold_convert (itype
, fd
->loops
[i
].n1
));
10557 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10561 /* Remove GIMPLE_OMP_CONTINUE. */
10562 gsi_remove (&gsi
, true);
10565 /* Emit the condition in L1_BB. */
10566 gsi
= gsi_start_bb (l1_bb
);
10568 t
= fold_convert (type
, n2
);
10569 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
10570 false, GSI_CONTINUE_LINKING
);
10571 tree v
= fd
->loop
.v
;
10572 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
10573 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
10574 false, GSI_CONTINUE_LINKING
);
10575 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, v
, t
);
10576 cond_stmt
= gimple_build_cond_empty (t
);
10577 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
10578 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
10580 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
10583 gsi
= gsi_for_stmt (cond_stmt
);
10584 gimple_regimplify_operands (cond_stmt
, &gsi
);
10587 /* Remove GIMPLE_OMP_RETURN. */
10588 gsi
= gsi_last_bb (exit_bb
);
10589 gsi_remove (&gsi
, true);
10591 /* Connect the new blocks. */
10592 remove_edge (FALLTHRU_EDGE (entry_bb
));
10596 remove_edge (BRANCH_EDGE (entry_bb
));
10597 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
10599 e
= BRANCH_EDGE (l1_bb
);
10600 ne
= FALLTHRU_EDGE (l1_bb
);
10601 e
->flags
= EDGE_TRUE_VALUE
;
10605 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
10607 ne
= single_succ_edge (l1_bb
);
10608 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
10611 ne
->flags
= EDGE_FALSE_VALUE
;
10612 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
10613 ne
->probability
= REG_BR_PROB_BASE
/ 8;
10615 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
10616 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
10617 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
10621 struct loop
*loop
= alloc_loop ();
10622 loop
->header
= l1_bb
;
10623 loop
->latch
= cont_bb
;
10624 add_loop (loop
, l1_bb
->loop_father
);
10625 if (safelen
== NULL_TREE
)
10626 loop
->safelen
= INT_MAX
;
10629 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
10630 if (TREE_CODE (safelen
) != INTEGER_CST
)
10632 else if (!tree_fits_uhwi_p (safelen
)
10633 || tree_to_uhwi (safelen
) > INT_MAX
)
10634 loop
->safelen
= INT_MAX
;
10636 loop
->safelen
= tree_to_uhwi (safelen
);
10637 if (loop
->safelen
== 1)
10642 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
10643 cfun
->has_simduid_loops
= true;
10645 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
10647 if ((flag_tree_loop_vectorize
10648 || (!global_options_set
.x_flag_tree_loop_vectorize
10649 && !global_options_set
.x_flag_tree_vectorize
))
10650 && flag_tree_loop_optimize
10651 && loop
->safelen
> 1)
10653 loop
->force_vectorize
= true;
10654 cfun
->has_force_vectorize_loops
= true;
10658 cfun
->has_simduid_loops
= true;
10661 /* Taskloop construct is represented after gimplification with
10662 two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
10663 in between them. This routine expands the outer GIMPLE_OMP_FOR,
10664 which should just compute all the needed loop temporaries
10665 for GIMPLE_OMP_TASK. */
10668 expand_omp_taskloop_for_outer (struct omp_region
*region
,
10669 struct omp_for_data
*fd
,
10670 gimple
*inner_stmt
)
10672 tree type
, bias
= NULL_TREE
;
10673 basic_block entry_bb
, cont_bb
, exit_bb
;
10674 gimple_stmt_iterator gsi
;
10675 gassign
*assign_stmt
;
10676 tree
*counts
= NULL
;
10679 gcc_assert (inner_stmt
);
10680 gcc_assert (region
->cont
);
10681 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_TASK
10682 && gimple_omp_task_taskloop_p (inner_stmt
));
10683 type
= TREE_TYPE (fd
->loop
.v
);
10685 /* See if we need to bias by LLONG_MIN. */
10686 if (fd
->iter_type
== long_long_unsigned_type_node
10687 && TREE_CODE (type
) == INTEGER_TYPE
10688 && !TYPE_UNSIGNED (type
))
10692 if (fd
->loop
.cond_code
== LT_EXPR
)
10695 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10699 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10702 if (TREE_CODE (n1
) != INTEGER_CST
10703 || TREE_CODE (n2
) != INTEGER_CST
10704 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
10705 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
10708 entry_bb
= region
->entry
;
10709 cont_bb
= region
->cont
;
10710 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10711 gcc_assert (BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
10712 exit_bb
= region
->exit
;
10714 gsi
= gsi_last_bb (entry_bb
);
10715 gimple
*for_stmt
= gsi_stmt (gsi
);
10716 gcc_assert (gimple_code (for_stmt
) == GIMPLE_OMP_FOR
);
10717 if (fd
->collapse
> 1)
10719 int first_zero_iter
= -1, dummy
= -1;
10720 basic_block zero_iter_bb
= NULL
, dummy_bb
= NULL
, l2_dom_bb
= NULL
;
10722 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10723 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10724 zero_iter_bb
, first_zero_iter
,
10725 dummy_bb
, dummy
, l2_dom_bb
);
10729 /* Some counts[i] vars might be uninitialized if
10730 some loop has zero iterations. But the body shouldn't
10731 be executed in that case, so just avoid uninit warnings. */
10732 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
10733 if (SSA_VAR_P (counts
[i
]))
10734 TREE_NO_WARNING (counts
[i
]) = 1;
10736 edge e
= split_block (entry_bb
, gsi_stmt (gsi
));
10737 entry_bb
= e
->dest
;
10738 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
10739 gsi
= gsi_last_bb (entry_bb
);
10740 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
10741 get_immediate_dominator (CDI_DOMINATORS
,
10749 if (POINTER_TYPE_P (TREE_TYPE (t0
))
10750 && TYPE_PRECISION (TREE_TYPE (t0
))
10751 != TYPE_PRECISION (fd
->iter_type
))
10753 /* Avoid casting pointers to integer of a different size. */
10754 tree itype
= signed_type_for (type
);
10755 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
10756 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
10760 t1
= fold_convert (fd
->iter_type
, t1
);
10761 t0
= fold_convert (fd
->iter_type
, t0
);
10765 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
10766 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
10769 tree innerc
= find_omp_clause (gimple_omp_task_clauses (inner_stmt
),
10770 OMP_CLAUSE__LOOPTEMP_
);
10771 gcc_assert (innerc
);
10772 tree startvar
= OMP_CLAUSE_DECL (innerc
);
10773 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
), OMP_CLAUSE__LOOPTEMP_
);
10774 gcc_assert (innerc
);
10775 tree endvar
= OMP_CLAUSE_DECL (innerc
);
10776 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
10778 gcc_assert (innerc
);
10779 for (i
= 1; i
< fd
->collapse
; i
++)
10781 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10782 OMP_CLAUSE__LOOPTEMP_
);
10783 gcc_assert (innerc
);
10785 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10786 OMP_CLAUSE__LOOPTEMP_
);
10789 /* If needed (inner taskloop has lastprivate clause), propagate
10790 down the total number of iterations. */
10791 tree t
= force_gimple_operand_gsi (&gsi
, fd
->loop
.n2
, false,
10793 GSI_CONTINUE_LINKING
);
10794 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
10795 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10799 t0
= force_gimple_operand_gsi (&gsi
, t0
, false, NULL_TREE
, false,
10800 GSI_CONTINUE_LINKING
);
10801 assign_stmt
= gimple_build_assign (startvar
, t0
);
10802 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10804 t1
= force_gimple_operand_gsi (&gsi
, t1
, false, NULL_TREE
, false,
10805 GSI_CONTINUE_LINKING
);
10806 assign_stmt
= gimple_build_assign (endvar
, t1
);
10807 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10808 if (fd
->collapse
> 1)
10809 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
10811 /* Remove the GIMPLE_OMP_FOR statement. */
10812 gsi
= gsi_for_stmt (for_stmt
);
10813 gsi_remove (&gsi
, true);
10815 gsi
= gsi_last_bb (cont_bb
);
10816 gsi_remove (&gsi
, true);
10818 gsi
= gsi_last_bb (exit_bb
);
10819 gsi_remove (&gsi
, true);
10821 FALLTHRU_EDGE (entry_bb
)->probability
= REG_BR_PROB_BASE
;
10822 remove_edge (BRANCH_EDGE (entry_bb
));
10823 FALLTHRU_EDGE (cont_bb
)->probability
= REG_BR_PROB_BASE
;
10824 remove_edge (BRANCH_EDGE (cont_bb
));
10825 set_immediate_dominator (CDI_DOMINATORS
, exit_bb
, cont_bb
);
10826 set_immediate_dominator (CDI_DOMINATORS
, region
->entry
,
10827 recompute_dominator (CDI_DOMINATORS
, region
->entry
));
10830 /* Taskloop construct is represented after gimplification with
10831 two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
10832 in between them. This routine expands the inner GIMPLE_OMP_FOR.
10833 GOMP_taskloop{,_ull} function arranges for each task to be given just
10834 a single range of iterations. */
10837 expand_omp_taskloop_for_inner (struct omp_region
*region
,
10838 struct omp_for_data
*fd
,
10839 gimple
*inner_stmt
)
10841 tree e
, t
, type
, itype
, vmain
, vback
, bias
= NULL_TREE
;
10842 basic_block entry_bb
, exit_bb
, body_bb
, cont_bb
, collapse_bb
= NULL
;
10843 basic_block fin_bb
;
10844 gimple_stmt_iterator gsi
;
10846 bool broken_loop
= region
->cont
== NULL
;
10847 tree
*counts
= NULL
;
10850 itype
= type
= TREE_TYPE (fd
->loop
.v
);
10851 if (POINTER_TYPE_P (type
))
10852 itype
= signed_type_for (type
);
10854 /* See if we need to bias by LLONG_MIN. */
10855 if (fd
->iter_type
== long_long_unsigned_type_node
10856 && TREE_CODE (type
) == INTEGER_TYPE
10857 && !TYPE_UNSIGNED (type
))
10861 if (fd
->loop
.cond_code
== LT_EXPR
)
10864 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10868 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10871 if (TREE_CODE (n1
) != INTEGER_CST
10872 || TREE_CODE (n2
) != INTEGER_CST
10873 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
10874 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
10877 entry_bb
= region
->entry
;
10878 cont_bb
= region
->cont
;
10879 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10880 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
10881 gcc_assert (broken_loop
10882 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
10883 body_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10886 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
10887 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
10889 exit_bb
= region
->exit
;
10891 /* Iteration space partitioning goes in ENTRY_BB. */
10892 gsi
= gsi_last_bb (entry_bb
);
10893 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
10895 if (fd
->collapse
> 1)
10897 int first_zero_iter
= -1, dummy
= -1;
10898 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
10900 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10901 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10902 fin_bb
, first_zero_iter
,
10903 dummy_bb
, dummy
, l2_dom_bb
);
10907 t
= integer_one_node
;
10909 step
= fd
->loop
.step
;
10910 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10911 OMP_CLAUSE__LOOPTEMP_
);
10912 gcc_assert (innerc
);
10913 n1
= OMP_CLAUSE_DECL (innerc
);
10914 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
), OMP_CLAUSE__LOOPTEMP_
);
10915 gcc_assert (innerc
);
10916 n2
= OMP_CLAUSE_DECL (innerc
);
10919 n1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, n1
, bias
);
10920 n2
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, n2
, bias
);
10922 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
10923 true, NULL_TREE
, true, GSI_SAME_STMT
);
10924 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
10925 true, NULL_TREE
, true, GSI_SAME_STMT
);
10926 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
10927 true, NULL_TREE
, true, GSI_SAME_STMT
);
10929 tree startvar
= fd
->loop
.v
;
10930 tree endvar
= NULL_TREE
;
10932 if (gimple_omp_for_combined_p (fd
->for_stmt
))
10934 tree clauses
= gimple_omp_for_clauses (inner_stmt
);
10935 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
10936 gcc_assert (innerc
);
10937 startvar
= OMP_CLAUSE_DECL (innerc
);
10938 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10939 OMP_CLAUSE__LOOPTEMP_
);
10940 gcc_assert (innerc
);
10941 endvar
= OMP_CLAUSE_DECL (innerc
);
10943 t
= fold_convert (TREE_TYPE (startvar
), n1
);
10944 t
= force_gimple_operand_gsi (&gsi
, t
,
10946 && TREE_ADDRESSABLE (startvar
),
10947 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
10948 gimple
*assign_stmt
= gimple_build_assign (startvar
, t
);
10949 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10951 t
= fold_convert (TREE_TYPE (startvar
), n2
);
10952 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
10953 false, GSI_CONTINUE_LINKING
);
10956 assign_stmt
= gimple_build_assign (endvar
, e
);
10957 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10958 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
10959 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
10961 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
10962 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10964 if (fd
->collapse
> 1)
10965 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
10969 /* The code controlling the sequential loop replaces the
10970 GIMPLE_OMP_CONTINUE. */
10971 gsi
= gsi_last_bb (cont_bb
);
10972 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
10973 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
10974 vmain
= gimple_omp_continue_control_use (cont_stmt
);
10975 vback
= gimple_omp_continue_control_def (cont_stmt
);
10977 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
10979 if (POINTER_TYPE_P (type
))
10980 t
= fold_build_pointer_plus (vmain
, step
);
10982 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
10983 t
= force_gimple_operand_gsi (&gsi
, t
,
10985 && TREE_ADDRESSABLE (vback
),
10986 NULL_TREE
, true, GSI_SAME_STMT
);
10987 assign_stmt
= gimple_build_assign (vback
, t
);
10988 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
10990 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
10991 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
10993 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
10996 /* Remove the GIMPLE_OMP_CONTINUE statement. */
10997 gsi_remove (&gsi
, true);
10999 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
11000 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
11003 /* Remove the GIMPLE_OMP_FOR statement. */
11004 gsi
= gsi_for_stmt (fd
->for_stmt
);
11005 gsi_remove (&gsi
, true);
11007 /* Remove the GIMPLE_OMP_RETURN statement. */
11008 gsi
= gsi_last_bb (exit_bb
);
11009 gsi_remove (&gsi
, true);
11011 FALLTHRU_EDGE (entry_bb
)->probability
= REG_BR_PROB_BASE
;
11013 remove_edge (BRANCH_EDGE (entry_bb
));
11016 remove_edge_and_dominated_blocks (BRANCH_EDGE (entry_bb
));
11017 region
->outer
->cont
= NULL
;
11020 /* Connect all the blocks. */
11023 ep
= find_edge (cont_bb
, body_bb
);
11024 if (gimple_omp_for_combined_p (fd
->for_stmt
))
11029 else if (fd
->collapse
> 1)
11032 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
11035 ep
->flags
= EDGE_TRUE_VALUE
;
11036 find_edge (cont_bb
, fin_bb
)->flags
11037 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
11040 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
11041 recompute_dominator (CDI_DOMINATORS
, body_bb
));
11043 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
11044 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
11046 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
11048 struct loop
*loop
= alloc_loop ();
11049 loop
->header
= body_bb
;
11050 if (collapse_bb
== NULL
)
11051 loop
->latch
= cont_bb
;
11052 add_loop (loop
, body_bb
->loop_father
);
11056 /* A subroutine of expand_omp_for. Generate code for an OpenACC
11057 partitioned loop. The lowering here is abstracted, in that the
11058 loop parameters are passed through internal functions, which are
11059 further lowered by oacc_device_lower, once we get to the target
11060 compiler. The loop is of the form:
11062 for (V = B; V LTGT E; V += S) {BODY}
11064 where LTGT is < or >. We may have a specified chunking size, CHUNKING
11065 (constant 0 for no chunking) and we will have a GWV partitioning
11066 mask, specifying dimensions over which the loop is to be
11067 partitioned (see note below). We generate code that looks like:
11069 <entry_bb> [incoming FALL->body, BRANCH->exit]
11070 typedef signedintify (typeof (V)) T; // underlying signed integral type
11073 T DIR = LTGT == '<' ? +1 : -1;
11074 T chunk_max = GOACC_LOOP_CHUNK (dir, range, S, CHUNK_SIZE, GWV);
11075 T step = GOACC_LOOP_STEP (dir, range, S, CHUNK_SIZE, GWV);
11077 <head_bb> [created by splitting end of entry_bb]
11078 T offset = GOACC_LOOP_OFFSET (dir, range, S, CHUNK_SIZE, GWV, chunk_no);
11079 T bound = GOACC_LOOP_BOUND (dir, range, S, CHUNK_SIZE, GWV, offset);
11080 if (!(offset LTGT bound)) goto bottom_bb;
11082 <body_bb> [incoming]
11086 <cont_bb> [incoming, may == body_bb FALL->exit_bb, BRANCH->body_bb]
11088 if (offset LTGT bound) goto body_bb; [*]
11090 <bottom_bb> [created by splitting start of exit_bb] insert BRANCH->head_bb
11092 if (chunk < chunk_max) goto head_bb;
11094 <exit_bb> [incoming]
11095 V = B + ((range -/+ 1) / S +/- 1) * S [*]
11097 [*] Needed if V live at end of loop
11099 Note: CHUNKING & GWV mask are specified explicitly here. This is a
11100 transition, and will be specified by a more general mechanism shortly.
11104 expand_oacc_for (struct omp_region
*region
, struct omp_for_data
*fd
)
11106 tree v
= fd
->loop
.v
;
11107 enum tree_code cond_code
= fd
->loop
.cond_code
;
11108 enum tree_code plus_code
= PLUS_EXPR
;
11110 tree chunk_size
= integer_minus_one_node
;
11111 tree gwv
= integer_zero_node
;
11112 tree iter_type
= TREE_TYPE (v
);
11113 tree diff_type
= iter_type
;
11114 tree plus_type
= iter_type
;
11115 struct oacc_collapse
*counts
= NULL
;
11117 gcc_checking_assert (gimple_omp_for_kind (fd
->for_stmt
)
11118 == GF_OMP_FOR_KIND_OACC_LOOP
);
11119 gcc_assert (!gimple_omp_for_combined_into_p (fd
->for_stmt
));
11120 gcc_assert (cond_code
== LT_EXPR
|| cond_code
== GT_EXPR
);
11122 if (POINTER_TYPE_P (iter_type
))
11124 plus_code
= POINTER_PLUS_EXPR
;
11125 plus_type
= sizetype
;
11127 if (POINTER_TYPE_P (diff_type
) || TYPE_UNSIGNED (diff_type
))
11128 diff_type
= signed_type_for (diff_type
);
11130 basic_block entry_bb
= region
->entry
; /* BB ending in OMP_FOR */
11131 basic_block exit_bb
= region
->exit
; /* BB ending in OMP_RETURN */
11132 basic_block cont_bb
= region
->cont
; /* BB ending in OMP_CONTINUE */
11133 basic_block bottom_bb
= NULL
;
11135 /* entry_bb has two sucessors; the branch edge is to the exit
11136 block, fallthrough edge to body. */
11137 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2
11138 && BRANCH_EDGE (entry_bb
)->dest
== exit_bb
);
11140 /* If cont_bb non-NULL, it has 2 successors. The branch successor is
11141 body_bb, or to a block whose only successor is the body_bb. Its
11142 fallthrough successor is the final block (same as the branch
11143 successor of the entry_bb). */
11146 basic_block body_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
11147 basic_block bed
= BRANCH_EDGE (cont_bb
)->dest
;
11149 gcc_assert (FALLTHRU_EDGE (cont_bb
)->dest
== exit_bb
);
11150 gcc_assert (bed
== body_bb
|| single_succ_edge (bed
)->dest
== body_bb
);
11153 gcc_assert (!gimple_in_ssa_p (cfun
));
11155 /* The exit block only has entry_bb and cont_bb as predecessors. */
11156 gcc_assert (EDGE_COUNT (exit_bb
->preds
) == 1 + (cont_bb
!= NULL
));
11159 tree chunk_max
= NULL_TREE
;
11160 tree bound
, offset
;
11161 tree step
= create_tmp_var (diff_type
, ".step");
11162 bool up
= cond_code
== LT_EXPR
;
11163 tree dir
= build_int_cst (diff_type
, up
? +1 : -1);
11164 bool chunking
= !gimple_in_ssa_p (cfun
);;
11167 /* SSA instances. */
11168 tree offset_incr
= NULL_TREE
;
11169 tree offset_init
= NULL_TREE
;
11171 gimple_stmt_iterator gsi
;
11177 edge split
, be
, fte
;
11179 /* Split the end of entry_bb to create head_bb. */
11180 split
= split_block (entry_bb
, last_stmt (entry_bb
));
11181 basic_block head_bb
= split
->dest
;
11182 entry_bb
= split
->src
;
11184 /* Chunk setup goes at end of entry_bb, replacing the omp_for. */
11185 gsi
= gsi_last_bb (entry_bb
);
11186 gomp_for
*for_stmt
= as_a
<gomp_for
*> (gsi_stmt (gsi
));
11187 loc
= gimple_location (for_stmt
);
11189 if (gimple_in_ssa_p (cfun
))
11191 offset_init
= gimple_omp_for_index (for_stmt
, 0);
11192 gcc_assert (integer_zerop (fd
->loop
.n1
));
11193 /* The SSA parallelizer does gang parallelism. */
11194 gwv
= build_int_cst (integer_type_node
, GOMP_DIM_MASK (GOMP_DIM_GANG
));
11197 if (fd
->collapse
> 1)
11199 counts
= XALLOCAVEC (struct oacc_collapse
, fd
->collapse
);
11200 tree total
= expand_oacc_collapse_init (fd
, &gsi
, counts
,
11201 TREE_TYPE (fd
->loop
.n2
));
11203 if (SSA_VAR_P (fd
->loop
.n2
))
11205 total
= force_gimple_operand_gsi (&gsi
, total
, false, NULL_TREE
,
11206 true, GSI_SAME_STMT
);
11207 ass
= gimple_build_assign (fd
->loop
.n2
, total
);
11208 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11213 tree b
= fd
->loop
.n1
;
11214 tree e
= fd
->loop
.n2
;
11215 tree s
= fd
->loop
.step
;
11217 b
= force_gimple_operand_gsi (&gsi
, b
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11218 e
= force_gimple_operand_gsi (&gsi
, e
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11220 /* Convert the step, avoiding possible unsigned->signed overflow. */
11221 negating
= !up
&& TYPE_UNSIGNED (TREE_TYPE (s
));
11223 s
= fold_build1 (NEGATE_EXPR
, TREE_TYPE (s
), s
);
11224 s
= fold_convert (diff_type
, s
);
11226 s
= fold_build1 (NEGATE_EXPR
, diff_type
, s
);
11227 s
= force_gimple_operand_gsi (&gsi
, s
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11230 chunk_size
= integer_zero_node
;
11231 expr
= fold_convert (diff_type
, chunk_size
);
11232 chunk_size
= force_gimple_operand_gsi (&gsi
, expr
, true,
11233 NULL_TREE
, true, GSI_SAME_STMT
);
11234 /* Determine the range, avoiding possible unsigned->signed overflow. */
11235 negating
= !up
&& TYPE_UNSIGNED (iter_type
);
11236 expr
= fold_build2 (MINUS_EXPR
, plus_type
,
11237 fold_convert (plus_type
, negating
? b
: e
),
11238 fold_convert (plus_type
, negating
? e
: b
));
11239 expr
= fold_convert (diff_type
, expr
);
11241 expr
= fold_build1 (NEGATE_EXPR
, diff_type
, expr
);
11242 tree range
= force_gimple_operand_gsi (&gsi
, expr
, true,
11243 NULL_TREE
, true, GSI_SAME_STMT
);
11245 chunk_no
= build_int_cst (diff_type
, 0);
11248 gcc_assert (!gimple_in_ssa_p (cfun
));
11251 chunk_max
= create_tmp_var (diff_type
, ".chunk_max");
11252 chunk_no
= create_tmp_var (diff_type
, ".chunk_no");
11254 ass
= gimple_build_assign (chunk_no
, expr
);
11255 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11257 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 6,
11258 build_int_cst (integer_type_node
,
11259 IFN_GOACC_LOOP_CHUNKS
),
11260 dir
, range
, s
, chunk_size
, gwv
);
11261 gimple_call_set_lhs (call
, chunk_max
);
11262 gimple_set_location (call
, loc
);
11263 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
11266 chunk_size
= chunk_no
;
11268 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 6,
11269 build_int_cst (integer_type_node
,
11270 IFN_GOACC_LOOP_STEP
),
11271 dir
, range
, s
, chunk_size
, gwv
);
11272 gimple_call_set_lhs (call
, step
);
11273 gimple_set_location (call
, loc
);
11274 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
11276 /* Remove the GIMPLE_OMP_FOR. */
11277 gsi_remove (&gsi
, true);
11279 /* Fixup edges from head_bb */
11280 be
= BRANCH_EDGE (head_bb
);
11281 fte
= FALLTHRU_EDGE (head_bb
);
11282 be
->flags
|= EDGE_FALSE_VALUE
;
11283 fte
->flags
^= EDGE_FALLTHRU
| EDGE_TRUE_VALUE
;
11285 basic_block body_bb
= fte
->dest
;
11287 if (gimple_in_ssa_p (cfun
))
11289 gsi
= gsi_last_bb (cont_bb
);
11290 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
11292 offset
= gimple_omp_continue_control_use (cont_stmt
);
11293 offset_incr
= gimple_omp_continue_control_def (cont_stmt
);
11297 offset
= create_tmp_var (diff_type
, ".offset");
11298 offset_init
= offset_incr
= offset
;
11300 bound
= create_tmp_var (TREE_TYPE (offset
), ".bound");
11302 /* Loop offset & bound go into head_bb. */
11303 gsi
= gsi_start_bb (head_bb
);
11305 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 7,
11306 build_int_cst (integer_type_node
,
11307 IFN_GOACC_LOOP_OFFSET
),
11309 chunk_size
, gwv
, chunk_no
);
11310 gimple_call_set_lhs (call
, offset_init
);
11311 gimple_set_location (call
, loc
);
11312 gsi_insert_after (&gsi
, call
, GSI_CONTINUE_LINKING
);
11314 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 7,
11315 build_int_cst (integer_type_node
,
11316 IFN_GOACC_LOOP_BOUND
),
11318 chunk_size
, gwv
, offset_init
);
11319 gimple_call_set_lhs (call
, bound
);
11320 gimple_set_location (call
, loc
);
11321 gsi_insert_after (&gsi
, call
, GSI_CONTINUE_LINKING
);
11323 expr
= build2 (cond_code
, boolean_type_node
, offset_init
, bound
);
11324 gsi_insert_after (&gsi
, gimple_build_cond_empty (expr
),
11325 GSI_CONTINUE_LINKING
);
11327 /* V assignment goes into body_bb. */
11328 if (!gimple_in_ssa_p (cfun
))
11330 gsi
= gsi_start_bb (body_bb
);
11332 expr
= build2 (plus_code
, iter_type
, b
,
11333 fold_convert (plus_type
, offset
));
11334 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
11335 true, GSI_SAME_STMT
);
11336 ass
= gimple_build_assign (v
, expr
);
11337 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11338 if (fd
->collapse
> 1)
11339 expand_oacc_collapse_vars (fd
, &gsi
, counts
, v
);
11342 /* Loop increment goes into cont_bb. If this is not a loop, we
11343 will have spawned threads as if it was, and each one will
11344 execute one iteration. The specification is not explicit about
11345 whether such constructs are ill-formed or not, and they can
11346 occur, especially when noreturn routines are involved. */
11349 gsi
= gsi_last_bb (cont_bb
);
11350 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
11351 loc
= gimple_location (cont_stmt
);
11353 /* Increment offset. */
11354 if (gimple_in_ssa_p (cfun
))
11355 expr
= build2 (plus_code
, iter_type
, offset
,
11356 fold_convert (plus_type
, step
));
11358 expr
= build2 (PLUS_EXPR
, diff_type
, offset
, step
);
11359 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
11360 true, GSI_SAME_STMT
);
11361 ass
= gimple_build_assign (offset_incr
, expr
);
11362 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11363 expr
= build2 (cond_code
, boolean_type_node
, offset_incr
, bound
);
11364 gsi_insert_before (&gsi
, gimple_build_cond_empty (expr
), GSI_SAME_STMT
);
11366 /* Remove the GIMPLE_OMP_CONTINUE. */
11367 gsi_remove (&gsi
, true);
11369 /* Fixup edges from cont_bb */
11370 be
= BRANCH_EDGE (cont_bb
);
11371 fte
= FALLTHRU_EDGE (cont_bb
);
11372 be
->flags
|= EDGE_TRUE_VALUE
;
11373 fte
->flags
^= EDGE_FALLTHRU
| EDGE_FALSE_VALUE
;
11377 /* Split the beginning of exit_bb to make bottom_bb. We
11378 need to insert a nop at the start, because splitting is
11379 after a stmt, not before. */
11380 gsi
= gsi_start_bb (exit_bb
);
11381 stmt
= gimple_build_nop ();
11382 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
11383 split
= split_block (exit_bb
, stmt
);
11384 bottom_bb
= split
->src
;
11385 exit_bb
= split
->dest
;
11386 gsi
= gsi_last_bb (bottom_bb
);
11388 /* Chunk increment and test goes into bottom_bb. */
11389 expr
= build2 (PLUS_EXPR
, diff_type
, chunk_no
,
11390 build_int_cst (diff_type
, 1));
11391 ass
= gimple_build_assign (chunk_no
, expr
);
11392 gsi_insert_after (&gsi
, ass
, GSI_CONTINUE_LINKING
);
11394 /* Chunk test at end of bottom_bb. */
11395 expr
= build2 (LT_EXPR
, boolean_type_node
, chunk_no
, chunk_max
);
11396 gsi_insert_after (&gsi
, gimple_build_cond_empty (expr
),
11397 GSI_CONTINUE_LINKING
);
11399 /* Fixup edges from bottom_bb. */
11400 split
->flags
^= EDGE_FALLTHRU
| EDGE_FALSE_VALUE
;
11401 make_edge (bottom_bb
, head_bb
, EDGE_TRUE_VALUE
);
11405 gsi
= gsi_last_bb (exit_bb
);
11406 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
11407 loc
= gimple_location (gsi_stmt (gsi
));
11409 if (!gimple_in_ssa_p (cfun
))
11411 /* Insert the final value of V, in case it is live. This is the
11412 value for the only thread that survives past the join. */
11413 expr
= fold_build2 (MINUS_EXPR
, diff_type
, range
, dir
);
11414 expr
= fold_build2 (PLUS_EXPR
, diff_type
, expr
, s
);
11415 expr
= fold_build2 (TRUNC_DIV_EXPR
, diff_type
, expr
, s
);
11416 expr
= fold_build2 (MULT_EXPR
, diff_type
, expr
, s
);
11417 expr
= build2 (plus_code
, iter_type
, b
, fold_convert (plus_type
, expr
));
11418 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
11419 true, GSI_SAME_STMT
);
11420 ass
= gimple_build_assign (v
, expr
);
11421 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11424 /* Remove the OMP_RETURN. */
11425 gsi_remove (&gsi
, true);
11429 /* We now have one or two nested loops. Update the loop
11431 struct loop
*parent
= entry_bb
->loop_father
;
11432 struct loop
*body
= body_bb
->loop_father
;
11436 struct loop
*chunk_loop
= alloc_loop ();
11437 chunk_loop
->header
= head_bb
;
11438 chunk_loop
->latch
= bottom_bb
;
11439 add_loop (chunk_loop
, parent
);
11440 parent
= chunk_loop
;
11442 else if (parent
!= body
)
11444 gcc_assert (body
->header
== body_bb
);
11445 gcc_assert (body
->latch
== cont_bb
11446 || single_pred (body
->latch
) == cont_bb
);
11452 struct loop
*body_loop
= alloc_loop ();
11453 body_loop
->header
= body_bb
;
11454 body_loop
->latch
= cont_bb
;
11455 add_loop (body_loop
, parent
);
11460 /* Expand the OMP loop defined by REGION. */
11463 expand_omp_for (struct omp_region
*region
, gimple
*inner_stmt
)
11465 struct omp_for_data fd
;
11466 struct omp_for_data_loop
*loops
;
11469 = (struct omp_for_data_loop
*)
11470 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
11471 * sizeof (struct omp_for_data_loop
));
11472 extract_omp_for_data (as_a
<gomp_for
*> (last_stmt (region
->entry
)),
11474 region
->sched_kind
= fd
.sched_kind
;
11475 region
->sched_modifiers
= fd
.sched_modifiers
;
11477 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
11478 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
11479 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
11482 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
11483 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
11484 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
11487 /* If there isn't a continue then this is a degerate case where
11488 the introduction of abnormal edges during lowering will prevent
11489 original loops from being detected. Fix that up. */
11490 loops_state_set (LOOPS_NEED_FIXUP
);
11492 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_SIMD
)
11493 expand_omp_simd (region
, &fd
);
11494 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
11495 expand_cilk_for (region
, &fd
);
11496 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
11498 gcc_assert (!inner_stmt
);
11499 expand_oacc_for (region
, &fd
);
11501 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
11503 if (gimple_omp_for_combined_into_p (fd
.for_stmt
))
11504 expand_omp_taskloop_for_inner (region
, &fd
, inner_stmt
);
11506 expand_omp_taskloop_for_outer (region
, &fd
, inner_stmt
);
11508 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
11509 && !fd
.have_ordered
)
11511 if (fd
.chunk_size
== NULL
)
11512 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
11514 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
11518 int fn_index
, start_ix
, next_ix
;
11520 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
11521 == GF_OMP_FOR_KIND_FOR
);
11522 if (fd
.chunk_size
== NULL
11523 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
11524 fd
.chunk_size
= integer_zero_node
;
11525 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
11526 switch (fd
.sched_kind
)
11528 case OMP_CLAUSE_SCHEDULE_RUNTIME
:
11531 case OMP_CLAUSE_SCHEDULE_DYNAMIC
:
11532 case OMP_CLAUSE_SCHEDULE_GUIDED
:
11533 if ((fd
.sched_modifiers
& OMP_CLAUSE_SCHEDULE_NONMONOTONIC
)
11535 && !fd
.have_ordered
)
11537 fn_index
= 3 + fd
.sched_kind
;
11542 fn_index
= fd
.sched_kind
;
11546 fn_index
+= fd
.have_ordered
* 6;
11548 start_ix
= ((int)BUILT_IN_GOMP_LOOP_DOACROSS_STATIC_START
) + fn_index
;
11550 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
11551 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
11552 if (fd
.iter_type
== long_long_unsigned_type_node
)
11554 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
11555 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
11556 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
11557 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
11559 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
11560 (enum built_in_function
) next_ix
, inner_stmt
);
11563 if (gimple_in_ssa_p (cfun
))
11564 update_ssa (TODO_update_ssa_only_virtuals
);
11568 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
11570 v = GOMP_sections_start (n);
11587 v = GOMP_sections_next ();
11592 If this is a combined parallel sections, replace the call to
11593 GOMP_sections_start with call to GOMP_sections_next. */
11596 expand_omp_sections (struct omp_region
*region
)
11598 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
11600 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
11601 gimple_stmt_iterator si
, switch_si
;
11602 gomp_sections
*sections_stmt
;
11604 gomp_continue
*cont
;
11607 struct omp_region
*inner
;
11609 bool exit_reachable
= region
->cont
!= NULL
;
11611 gcc_assert (region
->exit
!= NULL
);
11612 entry_bb
= region
->entry
;
11613 l0_bb
= single_succ (entry_bb
);
11614 l1_bb
= region
->cont
;
11615 l2_bb
= region
->exit
;
11616 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
11617 l2
= gimple_block_label (l2_bb
);
11620 /* This can happen if there are reductions. */
11621 len
= EDGE_COUNT (l0_bb
->succs
);
11622 gcc_assert (len
> 0);
11623 e
= EDGE_SUCC (l0_bb
, len
- 1);
11624 si
= gsi_last_bb (e
->dest
);
11627 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
11628 l2
= gimple_block_label (e
->dest
);
11630 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
11632 si
= gsi_last_bb (e
->dest
);
11634 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
11636 l2
= gimple_block_label (e
->dest
);
11641 if (exit_reachable
)
11642 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
11644 default_bb
= create_empty_bb (l0_bb
);
11646 /* We will build a switch() with enough cases for all the
11647 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
11648 and a default case to abort if something goes wrong. */
11649 len
= EDGE_COUNT (l0_bb
->succs
);
11651 /* Use vec::quick_push on label_vec throughout, since we know the size
11653 auto_vec
<tree
> label_vec (len
);
11655 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
11656 GIMPLE_OMP_SECTIONS statement. */
11657 si
= gsi_last_bb (entry_bb
);
11658 sections_stmt
= as_a
<gomp_sections
*> (gsi_stmt (si
));
11659 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
11660 vin
= gimple_omp_sections_control (sections_stmt
);
11661 if (!is_combined_parallel (region
))
11663 /* If we are not inside a combined parallel+sections region,
11664 call GOMP_sections_start. */
11665 t
= build_int_cst (unsigned_type_node
, len
- 1);
11666 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
11667 stmt
= gimple_build_call (u
, 1, t
);
11671 /* Otherwise, call GOMP_sections_next. */
11672 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
11673 stmt
= gimple_build_call (u
, 0);
11675 gimple_call_set_lhs (stmt
, vin
);
11676 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11677 gsi_remove (&si
, true);
11679 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
11681 switch_si
= gsi_last_bb (l0_bb
);
11682 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
11683 if (exit_reachable
)
11685 cont
= as_a
<gomp_continue
*> (last_stmt (l1_bb
));
11686 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
11687 vmain
= gimple_omp_continue_control_use (cont
);
11688 vnext
= gimple_omp_continue_control_def (cont
);
11696 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
11697 label_vec
.quick_push (t
);
11700 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
11701 for (inner
= region
->inner
, casei
= 1;
11703 inner
= inner
->next
, i
++, casei
++)
11705 basic_block s_entry_bb
, s_exit_bb
;
11707 /* Skip optional reduction region. */
11708 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
11715 s_entry_bb
= inner
->entry
;
11716 s_exit_bb
= inner
->exit
;
11718 t
= gimple_block_label (s_entry_bb
);
11719 u
= build_int_cst (unsigned_type_node
, casei
);
11720 u
= build_case_label (u
, NULL
, t
);
11721 label_vec
.quick_push (u
);
11723 si
= gsi_last_bb (s_entry_bb
);
11724 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
11725 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
11726 gsi_remove (&si
, true);
11727 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
11729 if (s_exit_bb
== NULL
)
11732 si
= gsi_last_bb (s_exit_bb
);
11733 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
11734 gsi_remove (&si
, true);
11736 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
11739 /* Error handling code goes in DEFAULT_BB. */
11740 t
= gimple_block_label (default_bb
);
11741 u
= build_case_label (NULL
, NULL
, t
);
11742 make_edge (l0_bb
, default_bb
, 0);
11743 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
11745 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
11746 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
11747 gsi_remove (&switch_si
, true);
11749 si
= gsi_start_bb (default_bb
);
11750 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
11751 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
11753 if (exit_reachable
)
11757 /* Code to get the next section goes in L1_BB. */
11758 si
= gsi_last_bb (l1_bb
);
11759 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
11761 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
11762 stmt
= gimple_build_call (bfn_decl
, 0);
11763 gimple_call_set_lhs (stmt
, vnext
);
11764 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11765 gsi_remove (&si
, true);
11767 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
11770 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
11771 si
= gsi_last_bb (l2_bb
);
11772 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
11773 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
11774 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
11775 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
11777 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
11778 stmt
= gimple_build_call (t
, 0);
11779 if (gimple_omp_return_lhs (gsi_stmt (si
)))
11780 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
11781 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11782 gsi_remove (&si
, true);
11784 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
11788 /* Expand code for an OpenMP single directive. We've already expanded
11789 much of the code, here we simply place the GOMP_barrier call. */
11792 expand_omp_single (struct omp_region
*region
)
11794 basic_block entry_bb
, exit_bb
;
11795 gimple_stmt_iterator si
;
11797 entry_bb
= region
->entry
;
11798 exit_bb
= region
->exit
;
11800 si
= gsi_last_bb (entry_bb
);
11801 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
11802 gsi_remove (&si
, true);
11803 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
11805 si
= gsi_last_bb (exit_bb
);
11806 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
11808 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
11809 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
11811 gsi_remove (&si
, true);
11812 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
11816 /* Generic expansion for OpenMP synchronization directives: master,
11817 ordered and critical. All we need to do here is remove the entry
11818 and exit markers for REGION. */
11821 expand_omp_synch (struct omp_region
*region
)
11823 basic_block entry_bb
, exit_bb
;
11824 gimple_stmt_iterator si
;
11826 entry_bb
= region
->entry
;
11827 exit_bb
= region
->exit
;
11829 si
= gsi_last_bb (entry_bb
);
11830 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
11831 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
11832 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
11833 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
11834 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
11835 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
11836 gsi_remove (&si
, true);
11837 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
11841 si
= gsi_last_bb (exit_bb
);
11842 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
11843 gsi_remove (&si
, true);
11844 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
11848 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
11849 operation as a normal volatile load. */
11852 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
11853 tree loaded_val
, int index
)
11855 enum built_in_function tmpbase
;
11856 gimple_stmt_iterator gsi
;
11857 basic_block store_bb
;
11860 tree decl
, call
, type
, itype
;
11862 gsi
= gsi_last_bb (load_bb
);
11863 stmt
= gsi_stmt (gsi
);
11864 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
11865 loc
= gimple_location (stmt
);
11867 /* ??? If the target does not implement atomic_load_optab[mode], and mode
11868 is smaller than word size, then expand_atomic_load assumes that the load
11869 is atomic. We could avoid the builtin entirely in this case. */
11871 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
11872 decl
= builtin_decl_explicit (tmpbase
);
11873 if (decl
== NULL_TREE
)
11876 type
= TREE_TYPE (loaded_val
);
11877 itype
= TREE_TYPE (TREE_TYPE (decl
));
11879 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
11880 build_int_cst (NULL
,
11881 gimple_omp_atomic_seq_cst_p (stmt
)
11883 : MEMMODEL_RELAXED
));
11884 if (!useless_type_conversion_p (type
, itype
))
11885 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
11886 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
11888 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11889 gsi_remove (&gsi
, true);
11891 store_bb
= single_succ (load_bb
);
11892 gsi
= gsi_last_bb (store_bb
);
11893 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
11894 gsi_remove (&gsi
, true);
11896 if (gimple_in_ssa_p (cfun
))
11897 update_ssa (TODO_update_ssa_no_phi
);
11902 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
11903 operation as a normal volatile store. */
11906 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
11907 tree loaded_val
, tree stored_val
, int index
)
11909 enum built_in_function tmpbase
;
11910 gimple_stmt_iterator gsi
;
11911 basic_block store_bb
= single_succ (load_bb
);
11914 tree decl
, call
, type
, itype
;
11915 machine_mode imode
;
11918 gsi
= gsi_last_bb (load_bb
);
11919 stmt
= gsi_stmt (gsi
);
11920 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
11922 /* If the load value is needed, then this isn't a store but an exchange. */
11923 exchange
= gimple_omp_atomic_need_value_p (stmt
);
11925 gsi
= gsi_last_bb (store_bb
);
11926 stmt
= gsi_stmt (gsi
);
11927 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
11928 loc
= gimple_location (stmt
);
11930 /* ??? If the target does not implement atomic_store_optab[mode], and mode
11931 is smaller than word size, then expand_atomic_store assumes that the store
11932 is atomic. We could avoid the builtin entirely in this case. */
11934 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
11935 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
11936 decl
= builtin_decl_explicit (tmpbase
);
11937 if (decl
== NULL_TREE
)
11940 type
= TREE_TYPE (stored_val
);
11942 /* Dig out the type of the function's second argument. */
11943 itype
= TREE_TYPE (decl
);
11944 itype
= TYPE_ARG_TYPES (itype
);
11945 itype
= TREE_CHAIN (itype
);
11946 itype
= TREE_VALUE (itype
);
11947 imode
= TYPE_MODE (itype
);
11949 if (exchange
&& !can_atomic_exchange_p (imode
, true))
11952 if (!useless_type_conversion_p (itype
, type
))
11953 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
11954 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
11955 build_int_cst (NULL
,
11956 gimple_omp_atomic_seq_cst_p (stmt
)
11958 : MEMMODEL_RELAXED
));
11961 if (!useless_type_conversion_p (type
, itype
))
11962 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
11963 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
11966 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11967 gsi_remove (&gsi
, true);
11969 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
11970 gsi
= gsi_last_bb (load_bb
);
11971 gsi_remove (&gsi
, true);
11973 if (gimple_in_ssa_p (cfun
))
11974 update_ssa (TODO_update_ssa_no_phi
);
11979 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
11980 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
11981 size of the data type, and thus usable to find the index of the builtin
11982 decl. Returns false if the expression is not of the proper form. */
11985 expand_omp_atomic_fetch_op (basic_block load_bb
,
11986 tree addr
, tree loaded_val
,
11987 tree stored_val
, int index
)
11989 enum built_in_function oldbase
, newbase
, tmpbase
;
11990 tree decl
, itype
, call
;
11992 basic_block store_bb
= single_succ (load_bb
);
11993 gimple_stmt_iterator gsi
;
11996 enum tree_code code
;
11997 bool need_old
, need_new
;
11998 machine_mode imode
;
12001 /* We expect to find the following sequences:
12004 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
12007 val = tmp OP something; (or: something OP tmp)
12008 GIMPLE_OMP_STORE (val)
12010 ???FIXME: Allow a more flexible sequence.
12011 Perhaps use data flow to pick the statements.
12015 gsi
= gsi_after_labels (store_bb
);
12016 stmt
= gsi_stmt (gsi
);
12017 loc
= gimple_location (stmt
);
12018 if (!is_gimple_assign (stmt
))
12021 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
12023 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
12024 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
12025 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
12026 gcc_checking_assert (!need_old
|| !need_new
);
12028 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
12031 /* Check for one of the supported fetch-op operations. */
12032 code
= gimple_assign_rhs_code (stmt
);
12036 case POINTER_PLUS_EXPR
:
12037 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
12038 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
12041 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
12042 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
12045 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
12046 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
12049 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
12050 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
12053 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
12054 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
12060 /* Make sure the expression is of the proper form. */
12061 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
12062 rhs
= gimple_assign_rhs2 (stmt
);
12063 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
12064 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
12065 rhs
= gimple_assign_rhs1 (stmt
);
12069 tmpbase
= ((enum built_in_function
)
12070 ((need_new
? newbase
: oldbase
) + index
+ 1));
12071 decl
= builtin_decl_explicit (tmpbase
);
12072 if (decl
== NULL_TREE
)
12074 itype
= TREE_TYPE (TREE_TYPE (decl
));
12075 imode
= TYPE_MODE (itype
);
12077 /* We could test all of the various optabs involved, but the fact of the
12078 matter is that (with the exception of i486 vs i586 and xadd) all targets
12079 that support any atomic operaton optab also implements compare-and-swap.
12080 Let optabs.c take care of expanding any compare-and-swap loop. */
12081 if (!can_compare_and_swap_p (imode
, true))
12084 gsi
= gsi_last_bb (load_bb
);
12085 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
12087 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
12088 It only requires that the operation happen atomically. Thus we can
12089 use the RELAXED memory model. */
12090 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
12091 fold_convert_loc (loc
, itype
, rhs
),
12092 build_int_cst (NULL
,
12093 seq_cst
? MEMMODEL_SEQ_CST
12094 : MEMMODEL_RELAXED
));
12096 if (need_old
|| need_new
)
12098 lhs
= need_old
? loaded_val
: stored_val
;
12099 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
12100 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
12103 call
= fold_convert_loc (loc
, void_type_node
, call
);
12104 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
12105 gsi_remove (&gsi
, true);
12107 gsi
= gsi_last_bb (store_bb
);
12108 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
12109 gsi_remove (&gsi
, true);
12110 gsi
= gsi_last_bb (store_bb
);
12111 stmt
= gsi_stmt (gsi
);
12112 gsi_remove (&gsi
, true);
12114 if (gimple_in_ssa_p (cfun
))
12116 release_defs (stmt
);
12117 update_ssa (TODO_update_ssa_no_phi
);
12123 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
12127 newval = rhs; // with oldval replacing *addr in rhs
12128 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
12129 if (oldval != newval)
12132 INDEX is log2 of the size of the data type, and thus usable to find the
12133 index of the builtin decl. */
12136 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
12137 tree addr
, tree loaded_val
, tree stored_val
,
12140 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
12141 tree type
, itype
, cmpxchg
, iaddr
;
12142 gimple_stmt_iterator si
;
12143 basic_block loop_header
= single_succ (load_bb
);
12144 gimple
*phi
, *stmt
;
12146 enum built_in_function fncode
;
12148 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
12149 order to use the RELAXED memory model effectively. */
12150 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
12152 cmpxchg
= builtin_decl_explicit (fncode
);
12153 if (cmpxchg
== NULL_TREE
)
12155 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
12156 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
12158 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
12161 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
12162 si
= gsi_last_bb (load_bb
);
12163 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
12165 /* For floating-point values, we'll need to view-convert them to integers
12166 so that we can perform the atomic compare and swap. Simplify the
12167 following code by always setting up the "i"ntegral variables. */
12168 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
12172 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
12175 = force_gimple_operand_gsi (&si
,
12176 fold_convert (TREE_TYPE (iaddr
), addr
),
12177 false, NULL_TREE
, true, GSI_SAME_STMT
);
12178 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
12179 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12180 loadedi
= create_tmp_var (itype
);
12181 if (gimple_in_ssa_p (cfun
))
12182 loadedi
= make_ssa_name (loadedi
);
12187 loadedi
= loaded_val
;
12190 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
12191 tree loaddecl
= builtin_decl_explicit (fncode
);
12194 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr
)),
12195 build_call_expr (loaddecl
, 2, iaddr
,
12196 build_int_cst (NULL_TREE
,
12197 MEMMODEL_RELAXED
)));
12199 initial
= build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)), iaddr
,
12200 build_int_cst (TREE_TYPE (iaddr
), 0));
12203 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
12206 /* Move the value to the LOADEDI temporary. */
12207 if (gimple_in_ssa_p (cfun
))
12209 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
12210 phi
= create_phi_node (loadedi
, loop_header
);
12211 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
12215 gsi_insert_before (&si
,
12216 gimple_build_assign (loadedi
, initial
),
12218 if (loadedi
!= loaded_val
)
12220 gimple_stmt_iterator gsi2
;
12223 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
12224 gsi2
= gsi_start_bb (loop_header
);
12225 if (gimple_in_ssa_p (cfun
))
12228 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
12229 true, GSI_SAME_STMT
);
12230 stmt
= gimple_build_assign (loaded_val
, x
);
12231 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
12235 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
12236 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
12237 true, GSI_SAME_STMT
);
12240 gsi_remove (&si
, true);
12242 si
= gsi_last_bb (store_bb
);
12243 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
12246 storedi
= stored_val
;
12249 force_gimple_operand_gsi (&si
,
12250 build1 (VIEW_CONVERT_EXPR
, itype
,
12251 stored_val
), true, NULL_TREE
, true,
12254 /* Build the compare&swap statement. */
12255 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
12256 new_storedi
= force_gimple_operand_gsi (&si
,
12257 fold_convert (TREE_TYPE (loadedi
),
12260 true, GSI_SAME_STMT
);
12262 if (gimple_in_ssa_p (cfun
))
12263 old_vali
= loadedi
;
12266 old_vali
= create_tmp_var (TREE_TYPE (loadedi
));
12267 stmt
= gimple_build_assign (old_vali
, loadedi
);
12268 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12270 stmt
= gimple_build_assign (loadedi
, new_storedi
);
12271 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12274 /* Note that we always perform the comparison as an integer, even for
12275 floating point. This allows the atomic operation to properly
12276 succeed even with NaNs and -0.0. */
12277 stmt
= gimple_build_cond_empty
12278 (build2 (NE_EXPR
, boolean_type_node
,
12279 new_storedi
, old_vali
));
12280 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12283 e
= single_succ_edge (store_bb
);
12284 e
->flags
&= ~EDGE_FALLTHRU
;
12285 e
->flags
|= EDGE_FALSE_VALUE
;
12287 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
12289 /* Copy the new value to loadedi (we already did that before the condition
12290 if we are not in SSA). */
12291 if (gimple_in_ssa_p (cfun
))
12293 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
12294 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
12297 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
12298 gsi_remove (&si
, true);
12300 struct loop
*loop
= alloc_loop ();
12301 loop
->header
= loop_header
;
12302 loop
->latch
= store_bb
;
12303 add_loop (loop
, loop_header
->loop_father
);
12305 if (gimple_in_ssa_p (cfun
))
12306 update_ssa (TODO_update_ssa_no_phi
);
12311 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
12313 GOMP_atomic_start ();
12315 GOMP_atomic_end ();
12317 The result is not globally atomic, but works so long as all parallel
12318 references are within #pragma omp atomic directives. According to
12319 responses received from omp@openmp.org, appears to be within spec.
12320 Which makes sense, since that's how several other compilers handle
12321 this situation as well.
12322 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
12323 expanding. STORED_VAL is the operand of the matching
12324 GIMPLE_OMP_ATOMIC_STORE.
12327 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
12328 loaded_val = *addr;
12331 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
12332 *addr = stored_val;
12336 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
12337 tree addr
, tree loaded_val
, tree stored_val
)
12339 gimple_stmt_iterator si
;
12343 si
= gsi_last_bb (load_bb
);
12344 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
12346 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
12347 t
= build_call_expr (t
, 0);
12348 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
12350 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
12351 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12352 gsi_remove (&si
, true);
12354 si
= gsi_last_bb (store_bb
);
12355 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
12357 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
12359 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12361 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
12362 t
= build_call_expr (t
, 0);
12363 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
12364 gsi_remove (&si
, true);
12366 if (gimple_in_ssa_p (cfun
))
12367 update_ssa (TODO_update_ssa_no_phi
);
12371 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
12372 using expand_omp_atomic_fetch_op. If it failed, we try to
12373 call expand_omp_atomic_pipeline, and if it fails too, the
12374 ultimate fallback is wrapping the operation in a mutex
12375 (expand_omp_atomic_mutex). REGION is the atomic region built
12376 by build_omp_regions_1(). */
12379 expand_omp_atomic (struct omp_region
*region
)
12381 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
12382 gomp_atomic_load
*load
= as_a
<gomp_atomic_load
*> (last_stmt (load_bb
));
12383 gomp_atomic_store
*store
= as_a
<gomp_atomic_store
*> (last_stmt (store_bb
));
12384 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
12385 tree addr
= gimple_omp_atomic_load_rhs (load
);
12386 tree stored_val
= gimple_omp_atomic_store_val (store
);
12387 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
12388 HOST_WIDE_INT index
;
12390 /* Make sure the type is one of the supported sizes. */
12391 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
12392 index
= exact_log2 (index
);
12393 if (index
>= 0 && index
<= 4)
12395 unsigned int align
= TYPE_ALIGN_UNIT (type
);
12397 /* __sync builtins require strict data alignment. */
12398 if (exact_log2 (align
) >= index
)
12401 if (loaded_val
== stored_val
12402 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
12403 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
12404 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
12405 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
12408 /* Atomic store. */
12409 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
12410 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
12411 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
12412 && store_bb
== single_succ (load_bb
)
12413 && first_stmt (store_bb
) == store
12414 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
12415 stored_val
, index
))
12418 /* When possible, use specialized atomic update functions. */
12419 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
12420 && store_bb
== single_succ (load_bb
)
12421 && expand_omp_atomic_fetch_op (load_bb
, addr
,
12422 loaded_val
, stored_val
, index
))
12425 /* If we don't have specialized __sync builtins, try and implement
12426 as a compare and swap loop. */
12427 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
12428 loaded_val
, stored_val
, index
))
12433 /* The ultimate fallback is wrapping the operation in a mutex. */
12434 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
12438 /* Encode an oacc launch argument. This matches the GOMP_LAUNCH_PACK
12439 macro on gomp-constants.h. We do not check for overflow. */
12442 oacc_launch_pack (unsigned code
, tree device
, unsigned op
)
12446 res
= build_int_cst (unsigned_type_node
, GOMP_LAUNCH_PACK (code
, 0, op
));
12449 device
= fold_build2 (LSHIFT_EXPR
, unsigned_type_node
,
12450 device
, build_int_cst (unsigned_type_node
,
12451 GOMP_LAUNCH_DEVICE_SHIFT
));
12452 res
= fold_build2 (BIT_IOR_EXPR
, unsigned_type_node
, res
, device
);
12457 /* Look for compute grid dimension clauses and convert to an attribute
12458 attached to FN. This permits the target-side code to (a) massage
12459 the dimensions, (b) emit that data and (c) optimize. Non-constant
12460 dimensions are pushed onto ARGS.
12462 The attribute value is a TREE_LIST. A set of dimensions is
12463 represented as a list of INTEGER_CST. Those that are runtime
12464 exprs are represented as an INTEGER_CST of zero.
12466 TOOO. Normally the attribute will just contain a single such list. If
12467 however it contains a list of lists, this will represent the use of
12468 device_type. Each member of the outer list is an assoc list of
12469 dimensions, keyed by the device type. The first entry will be the
12470 default. Well, that's the plan. */
12472 #define OACC_FN_ATTRIB "oacc function"
12474 /* Replace any existing oacc fn attribute with updated dimensions. */
12477 replace_oacc_fn_attrib (tree fn
, tree dims
)
12479 tree ident
= get_identifier (OACC_FN_ATTRIB
);
12480 tree attribs
= DECL_ATTRIBUTES (fn
);
12482 /* If we happen to be present as the first attrib, drop it. */
12483 if (attribs
&& TREE_PURPOSE (attribs
) == ident
)
12484 attribs
= TREE_CHAIN (attribs
);
12485 DECL_ATTRIBUTES (fn
) = tree_cons (ident
, dims
, attribs
);
12488 /* Scan CLAUSES for launch dimensions and attach them to the oacc
12489 function attribute. Push any that are non-constant onto the ARGS
12490 list, along with an appropriate GOMP_LAUNCH_DIM tag. */
12493 set_oacc_fn_attrib (tree fn
, tree clauses
, vec
<tree
> *args
)
12495 /* Must match GOMP_DIM ordering. */
12496 static const omp_clause_code ids
[]
12497 = { OMP_CLAUSE_NUM_GANGS
, OMP_CLAUSE_NUM_WORKERS
,
12498 OMP_CLAUSE_VECTOR_LENGTH
};
12500 tree dims
[GOMP_DIM_MAX
];
12501 tree attr
= NULL_TREE
;
12502 unsigned non_const
= 0;
12504 for (ix
= GOMP_DIM_MAX
; ix
--;)
12506 tree clause
= find_omp_clause (clauses
, ids
[ix
]);
12507 tree dim
= NULL_TREE
;
12510 dim
= OMP_CLAUSE_EXPR (clause
, ids
[ix
]);
12512 if (dim
&& TREE_CODE (dim
) != INTEGER_CST
)
12514 dim
= integer_zero_node
;
12515 non_const
|= GOMP_DIM_MASK (ix
);
12517 attr
= tree_cons (NULL_TREE
, dim
, attr
);
12520 replace_oacc_fn_attrib (fn
, attr
);
12524 /* Push a dynamic argument set. */
12525 args
->safe_push (oacc_launch_pack (GOMP_LAUNCH_DIM
,
12526 NULL_TREE
, non_const
));
12527 for (unsigned ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
12528 if (non_const
& GOMP_DIM_MASK (ix
))
12529 args
->safe_push (dims
[ix
]);
12533 /* Process the routine's dimension clauess to generate an attribute
12534 value. Issue diagnostics as appropriate. We default to SEQ
12535 (OpenACC 2.5 clarifies this). All dimensions have a size of zero
12536 (dynamic). TREE_PURPOSE is set to indicate whether that dimension
12537 can have a loop partitioned on it. non-zero indicates
12538 yes, zero indicates no. By construction once a non-zero has been
12539 reached, further inner dimensions must also be non-zero. We set
12540 TREE_VALUE to zero for the dimensions that may be partitioned and
12541 1 for the other ones -- if a loop is (erroneously) spawned at
12542 an outer level, we don't want to try and partition it. */
12545 build_oacc_routine_dims (tree clauses
)
12547 /* Must match GOMP_DIM ordering. */
12548 static const omp_clause_code ids
[] =
12549 {OMP_CLAUSE_GANG
, OMP_CLAUSE_WORKER
, OMP_CLAUSE_VECTOR
, OMP_CLAUSE_SEQ
};
12553 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
12554 for (ix
= GOMP_DIM_MAX
+ 1; ix
--;)
12555 if (OMP_CLAUSE_CODE (clauses
) == ids
[ix
])
12558 error_at (OMP_CLAUSE_LOCATION (clauses
),
12559 "multiple loop axes specified for routine");
12564 /* Default to SEQ. */
12566 level
= GOMP_DIM_MAX
;
12568 tree dims
= NULL_TREE
;
12570 for (ix
= GOMP_DIM_MAX
; ix
--;)
12571 dims
= tree_cons (build_int_cst (boolean_type_node
, ix
>= level
),
12572 build_int_cst (integer_type_node
, ix
< level
), dims
);
12577 /* Retrieve the oacc function attrib and return it. Non-oacc
12578 functions will return NULL. */
12581 get_oacc_fn_attrib (tree fn
)
12583 return lookup_attribute (OACC_FN_ATTRIB
, DECL_ATTRIBUTES (fn
));
12586 /* Extract an oacc execution dimension from FN. FN must be an
12587 offloaded function or routine that has already had its execution
12588 dimensions lowered to the target-specific values. */
12591 get_oacc_fn_dim_size (tree fn
, int axis
)
12593 tree attrs
= get_oacc_fn_attrib (fn
);
12595 gcc_assert (axis
< GOMP_DIM_MAX
);
12597 tree dims
= TREE_VALUE (attrs
);
12599 dims
= TREE_CHAIN (dims
);
12601 int size
= TREE_INT_CST_LOW (TREE_VALUE (dims
));
12606 /* Extract the dimension axis from an IFN_GOACC_DIM_POS or
12607 IFN_GOACC_DIM_SIZE call. */
12610 get_oacc_ifn_dim_arg (const gimple
*stmt
)
12612 gcc_checking_assert (gimple_call_internal_fn (stmt
) == IFN_GOACC_DIM_SIZE
12613 || gimple_call_internal_fn (stmt
) == IFN_GOACC_DIM_POS
);
12614 tree arg
= gimple_call_arg (stmt
, 0);
12615 HOST_WIDE_INT axis
= TREE_INT_CST_LOW (arg
);
12617 gcc_checking_assert (axis
>= 0 && axis
< GOMP_DIM_MAX
);
12621 /* Mark the loops inside the kernels region starting at REGION_ENTRY and ending
12625 mark_loops_in_oacc_kernels_region (basic_block region_entry
,
12626 basic_block region_exit
)
12628 struct loop
*outer
= region_entry
->loop_father
;
12629 gcc_assert (region_exit
== NULL
|| outer
== region_exit
->loop_father
);
12631 /* Don't parallelize the kernels region if it contains more than one outer
12633 unsigned int nr_outer_loops
= 0;
12634 struct loop
*single_outer
= NULL
;
12635 for (struct loop
*loop
= outer
->inner
; loop
!= NULL
; loop
= loop
->next
)
12637 gcc_assert (loop_outer (loop
) == outer
);
12639 if (!dominated_by_p (CDI_DOMINATORS
, loop
->header
, region_entry
))
12642 if (region_exit
!= NULL
12643 && dominated_by_p (CDI_DOMINATORS
, loop
->header
, region_exit
))
12647 single_outer
= loop
;
12649 if (nr_outer_loops
!= 1)
12652 for (struct loop
*loop
= single_outer
->inner
; loop
!= NULL
; loop
= loop
->inner
)
12656 /* Mark the loops in the region. */
12657 for (struct loop
*loop
= single_outer
; loop
!= NULL
; loop
= loop
->inner
)
12658 loop
->in_oacc_kernels_region
= true;
12661 /* Types used to pass grid and wortkgroup sizes to kernel invocation. */
12663 struct GTY(()) grid_launch_attributes_trees
12665 tree kernel_dim_array_type
;
12666 tree kernel_lattrs_dimnum_decl
;
12667 tree kernel_lattrs_grid_decl
;
12668 tree kernel_lattrs_group_decl
;
12669 tree kernel_launch_attributes_type
;
12672 static GTY(()) struct grid_launch_attributes_trees
*grid_attr_trees
;
12674 /* Create types used to pass kernel launch attributes to target. */
12677 grid_create_kernel_launch_attr_types (void)
12679 if (grid_attr_trees
)
12681 grid_attr_trees
= ggc_alloc
<grid_launch_attributes_trees
> ();
12683 tree dim_arr_index_type
12684 = build_index_type (build_int_cst (integer_type_node
, 2));
12685 grid_attr_trees
->kernel_dim_array_type
12686 = build_array_type (uint32_type_node
, dim_arr_index_type
);
12688 grid_attr_trees
->kernel_launch_attributes_type
= make_node (RECORD_TYPE
);
12689 grid_attr_trees
->kernel_lattrs_dimnum_decl
12690 = build_decl (BUILTINS_LOCATION
, FIELD_DECL
, get_identifier ("ndim"),
12692 DECL_CHAIN (grid_attr_trees
->kernel_lattrs_dimnum_decl
) = NULL_TREE
;
12694 grid_attr_trees
->kernel_lattrs_grid_decl
12695 = build_decl (BUILTINS_LOCATION
, FIELD_DECL
, get_identifier ("grid_size"),
12696 grid_attr_trees
->kernel_dim_array_type
);
12697 DECL_CHAIN (grid_attr_trees
->kernel_lattrs_grid_decl
)
12698 = grid_attr_trees
->kernel_lattrs_dimnum_decl
;
12699 grid_attr_trees
->kernel_lattrs_group_decl
12700 = build_decl (BUILTINS_LOCATION
, FIELD_DECL
, get_identifier ("group_size"),
12701 grid_attr_trees
->kernel_dim_array_type
);
12702 DECL_CHAIN (grid_attr_trees
->kernel_lattrs_group_decl
)
12703 = grid_attr_trees
->kernel_lattrs_grid_decl
;
12704 finish_builtin_struct (grid_attr_trees
->kernel_launch_attributes_type
,
12705 "__gomp_kernel_launch_attributes",
12706 grid_attr_trees
->kernel_lattrs_group_decl
, NULL_TREE
);
12709 /* Insert before the current statement in GSI a store of VALUE to INDEX of
12710 array (of type kernel_dim_array_type) FLD_DECL of RANGE_VAR. VALUE must be
12711 of type uint32_type_node. */
12714 grid_insert_store_range_dim (gimple_stmt_iterator
*gsi
, tree range_var
,
12715 tree fld_decl
, int index
, tree value
)
12717 tree ref
= build4 (ARRAY_REF
, uint32_type_node
,
12718 build3 (COMPONENT_REF
,
12719 grid_attr_trees
->kernel_dim_array_type
,
12720 range_var
, fld_decl
, NULL_TREE
),
12721 build_int_cst (integer_type_node
, index
),
12722 NULL_TREE
, NULL_TREE
);
12723 gsi_insert_before (gsi
, gimple_build_assign (ref
, value
), GSI_SAME_STMT
);
12726 /* Return a tree representation of a pointer to a structure with grid and
12727 work-group size information. Statements filling that information will be
12728 inserted before GSI, TGT_STMT is the target statement which has the
12729 necessary information in it. */
12732 grid_get_kernel_launch_attributes (gimple_stmt_iterator
*gsi
,
12733 gomp_target
*tgt_stmt
)
12735 grid_create_kernel_launch_attr_types ();
12736 tree u32_one
= build_one_cst (uint32_type_node
);
12737 tree lattrs
= create_tmp_var (grid_attr_trees
->kernel_launch_attributes_type
,
12738 "__kernel_launch_attrs");
12740 unsigned max_dim
= 0;
12741 for (tree clause
= gimple_omp_target_clauses (tgt_stmt
);
12743 clause
= OMP_CLAUSE_CHAIN (clause
))
12745 if (OMP_CLAUSE_CODE (clause
) != OMP_CLAUSE__GRIDDIM_
)
12748 unsigned dim
= OMP_CLAUSE__GRIDDIM__DIMENSION (clause
);
12749 max_dim
= MAX (dim
, max_dim
);
12751 grid_insert_store_range_dim (gsi
, lattrs
,
12752 grid_attr_trees
->kernel_lattrs_grid_decl
,
12753 dim
, OMP_CLAUSE__GRIDDIM__SIZE (clause
));
12754 grid_insert_store_range_dim (gsi
, lattrs
,
12755 grid_attr_trees
->kernel_lattrs_group_decl
,
12756 dim
, OMP_CLAUSE__GRIDDIM__GROUP (clause
));
12759 tree dimref
= build3 (COMPONENT_REF
, uint32_type_node
, lattrs
,
12760 grid_attr_trees
->kernel_lattrs_dimnum_decl
, NULL_TREE
);
12761 /* At this moment we cannot gridify a loop with a collapse clause. */
12762 /* TODO: Adjust when we support bigger collapse. */
12763 gcc_assert (max_dim
== 0);
12764 gsi_insert_before (gsi
, gimple_build_assign (dimref
, u32_one
), GSI_SAME_STMT
);
12765 TREE_ADDRESSABLE (lattrs
) = 1;
12766 return build_fold_addr_expr (lattrs
);
12769 /* Build target argument identifier from the DEVICE identifier, value
12770 identifier ID and whether the element also has a SUBSEQUENT_PARAM. */
12773 get_target_argument_identifier_1 (int device
, bool subseqent_param
, int id
)
12775 tree t
= build_int_cst (integer_type_node
, device
);
12776 if (subseqent_param
)
12777 t
= fold_build2 (BIT_IOR_EXPR
, integer_type_node
, t
,
12778 build_int_cst (integer_type_node
,
12779 GOMP_TARGET_ARG_SUBSEQUENT_PARAM
));
12780 t
= fold_build2 (BIT_IOR_EXPR
, integer_type_node
, t
,
12781 build_int_cst (integer_type_node
, id
));
12785 /* Like above but return it in type that can be directly stored as an element
12786 of the argument array. */
12789 get_target_argument_identifier (int device
, bool subseqent_param
, int id
)
12791 tree t
= get_target_argument_identifier_1 (device
, subseqent_param
, id
);
12792 return fold_convert (ptr_type_node
, t
);
12795 /* Return a target argument consisting of DEVICE identifier, value identifier
12796 ID, and the actual VALUE. */
12799 get_target_argument_value (gimple_stmt_iterator
*gsi
, int device
, int id
,
12802 tree t
= fold_build2 (LSHIFT_EXPR
, integer_type_node
,
12803 fold_convert (integer_type_node
, value
),
12804 build_int_cst (unsigned_type_node
,
12805 GOMP_TARGET_ARG_VALUE_SHIFT
));
12806 t
= fold_build2 (BIT_IOR_EXPR
, integer_type_node
, t
,
12807 get_target_argument_identifier_1 (device
, false, id
));
12808 t
= fold_convert (ptr_type_node
, t
);
12809 return force_gimple_operand_gsi (gsi
, t
, true, NULL
, true, GSI_SAME_STMT
);
12812 /* If VALUE is an integer constant greater than -2^15 and smaller than 2^15,
12813 push one argument to ARGS with both the DEVICE, ID and VALUE embedded in it,
12814 otherwise push an identifier (with DEVICE and ID) and the VALUE in two
12818 push_target_argument_according_to_value (gimple_stmt_iterator
*gsi
, int device
,
12819 int id
, tree value
, vec
<tree
> *args
)
12821 if (tree_fits_shwi_p (value
)
12822 && tree_to_shwi (value
) > -(1 << 15)
12823 && tree_to_shwi (value
) < (1 << 15))
12824 args
->quick_push (get_target_argument_value (gsi
, device
, id
, value
));
12827 args
->quick_push (get_target_argument_identifier (device
, true, id
));
12828 value
= fold_convert (ptr_type_node
, value
);
12829 value
= force_gimple_operand_gsi (gsi
, value
, true, NULL
, true,
12831 args
->quick_push (value
);
12835 /* Create an array of arguments that is then passed to GOMP_target. */
12838 get_target_arguments (gimple_stmt_iterator
*gsi
, gomp_target
*tgt_stmt
)
12840 auto_vec
<tree
, 6> args
;
12841 tree clauses
= gimple_omp_target_clauses (tgt_stmt
);
12842 tree t
, c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_TEAMS
);
12844 t
= OMP_CLAUSE_NUM_TEAMS_EXPR (c
);
12846 t
= integer_minus_one_node
;
12847 push_target_argument_according_to_value (gsi
, GOMP_TARGET_ARG_DEVICE_ALL
,
12848 GOMP_TARGET_ARG_NUM_TEAMS
, t
, &args
);
12850 c
= find_omp_clause (clauses
, OMP_CLAUSE_THREAD_LIMIT
);
12852 t
= OMP_CLAUSE_THREAD_LIMIT_EXPR (c
);
12854 t
= integer_minus_one_node
;
12855 push_target_argument_according_to_value (gsi
, GOMP_TARGET_ARG_DEVICE_ALL
,
12856 GOMP_TARGET_ARG_THREAD_LIMIT
, t
,
12859 /* Add HSA-specific grid sizes, if available. */
12860 if (find_omp_clause (gimple_omp_target_clauses (tgt_stmt
),
12861 OMP_CLAUSE__GRIDDIM_
))
12863 t
= get_target_argument_identifier (GOMP_DEVICE_HSA
, true,
12864 GOMP_TARGET_ARG_HSA_KERNEL_ATTRIBUTES
);
12865 args
.quick_push (t
);
12866 args
.quick_push (grid_get_kernel_launch_attributes (gsi
, tgt_stmt
));
12869 /* Produce more, perhaps device specific, arguments here. */
12871 tree argarray
= create_tmp_var (build_array_type_nelts (ptr_type_node
,
12872 args
.length () + 1),
12873 ".omp_target_args");
12874 for (unsigned i
= 0; i
< args
.length (); i
++)
12876 tree ref
= build4 (ARRAY_REF
, ptr_type_node
, argarray
,
12877 build_int_cst (integer_type_node
, i
),
12878 NULL_TREE
, NULL_TREE
);
12879 gsi_insert_before (gsi
, gimple_build_assign (ref
, args
[i
]),
12882 tree ref
= build4 (ARRAY_REF
, ptr_type_node
, argarray
,
12883 build_int_cst (integer_type_node
, args
.length ()),
12884 NULL_TREE
, NULL_TREE
);
12885 gsi_insert_before (gsi
, gimple_build_assign (ref
, null_pointer_node
),
12887 TREE_ADDRESSABLE (argarray
) = 1;
12888 return build_fold_addr_expr (argarray
);
12891 /* Expand the GIMPLE_OMP_TARGET starting at REGION. */
12894 expand_omp_target (struct omp_region
*region
)
12896 basic_block entry_bb
, exit_bb
, new_bb
;
12897 struct function
*child_cfun
;
12898 tree child_fn
, block
, t
;
12899 gimple_stmt_iterator gsi
;
12900 gomp_target
*entry_stmt
;
12903 bool offloaded
, data_region
;
12905 entry_stmt
= as_a
<gomp_target
*> (last_stmt (region
->entry
));
12906 new_bb
= region
->entry
;
12908 offloaded
= is_gimple_omp_offloaded (entry_stmt
);
12909 switch (gimple_omp_target_kind (entry_stmt
))
12911 case GF_OMP_TARGET_KIND_REGION
:
12912 case GF_OMP_TARGET_KIND_UPDATE
:
12913 case GF_OMP_TARGET_KIND_ENTER_DATA
:
12914 case GF_OMP_TARGET_KIND_EXIT_DATA
:
12915 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
12916 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
12917 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
12918 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
12919 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
12920 data_region
= false;
12922 case GF_OMP_TARGET_KIND_DATA
:
12923 case GF_OMP_TARGET_KIND_OACC_DATA
:
12924 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
12925 data_region
= true;
12928 gcc_unreachable ();
12931 child_fn
= NULL_TREE
;
12935 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
12936 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
12939 /* Supported by expand_omp_taskreg, but not here. */
12940 if (child_cfun
!= NULL
)
12941 gcc_checking_assert (!child_cfun
->cfg
);
12942 gcc_checking_assert (!gimple_in_ssa_p (cfun
));
12944 entry_bb
= region
->entry
;
12945 exit_bb
= region
->exit
;
12947 if (gimple_omp_target_kind (entry_stmt
) == GF_OMP_TARGET_KIND_OACC_KERNELS
)
12948 mark_loops_in_oacc_kernels_region (region
->entry
, region
->exit
);
12952 unsigned srcidx
, dstidx
, num
;
12954 /* If the offloading region needs data sent from the parent
12955 function, then the very first statement (except possible
12956 tree profile counter updates) of the offloading body
12957 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
12958 &.OMP_DATA_O is passed as an argument to the child function,
12959 we need to replace it with the argument as seen by the child
12962 In most cases, this will end up being the identity assignment
12963 .OMP_DATA_I = .OMP_DATA_I. However, if the offloading body had
12964 a function call that has been inlined, the original PARM_DECL
12965 .OMP_DATA_I may have been converted into a different local
12966 variable. In which case, we need to keep the assignment. */
12967 tree data_arg
= gimple_omp_target_data_arg (entry_stmt
);
12970 basic_block entry_succ_bb
= single_succ (entry_bb
);
12971 gimple_stmt_iterator gsi
;
12973 gimple
*tgtcopy_stmt
= NULL
;
12974 tree sender
= TREE_VEC_ELT (data_arg
, 0);
12976 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
12978 gcc_assert (!gsi_end_p (gsi
));
12979 stmt
= gsi_stmt (gsi
);
12980 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
12983 if (gimple_num_ops (stmt
) == 2)
12985 tree arg
= gimple_assign_rhs1 (stmt
);
12987 /* We're ignoring the subcode because we're
12988 effectively doing a STRIP_NOPS. */
12990 if (TREE_CODE (arg
) == ADDR_EXPR
12991 && TREE_OPERAND (arg
, 0) == sender
)
12993 tgtcopy_stmt
= stmt
;
12999 gcc_assert (tgtcopy_stmt
!= NULL
);
13000 arg
= DECL_ARGUMENTS (child_fn
);
13002 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
13003 gsi_remove (&gsi
, true);
13006 /* Declare local variables needed in CHILD_CFUN. */
13007 block
= DECL_INITIAL (child_fn
);
13008 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
13009 /* The gimplifier could record temporaries in the offloading block
13010 rather than in containing function's local_decls chain,
13011 which would mean cgraph missed finalizing them. Do it now. */
13012 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
13013 if (TREE_CODE (t
) == VAR_DECL
13015 && !DECL_EXTERNAL (t
))
13016 varpool_node::finalize_decl (t
);
13017 DECL_SAVED_TREE (child_fn
) = NULL
;
13018 /* We'll create a CFG for child_fn, so no gimple body is needed. */
13019 gimple_set_body (child_fn
, NULL
);
13020 TREE_USED (block
) = 1;
13022 /* Reset DECL_CONTEXT on function arguments. */
13023 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
13024 DECL_CONTEXT (t
) = child_fn
;
13026 /* Split ENTRY_BB at GIMPLE_*,
13027 so that it can be moved to the child function. */
13028 gsi
= gsi_last_bb (entry_bb
);
13029 stmt
= gsi_stmt (gsi
);
13031 && gimple_code (stmt
) == gimple_code (entry_stmt
));
13032 e
= split_block (entry_bb
, stmt
);
13033 gsi_remove (&gsi
, true);
13034 entry_bb
= e
->dest
;
13035 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
13037 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
13040 gsi
= gsi_last_bb (exit_bb
);
13041 gcc_assert (!gsi_end_p (gsi
)
13042 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
13043 stmt
= gimple_build_return (NULL
);
13044 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
13045 gsi_remove (&gsi
, true);
13048 /* Move the offloading region into CHILD_CFUN. */
13050 block
= gimple_block (entry_stmt
);
13052 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
13054 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
13055 /* When the OMP expansion process cannot guarantee an up-to-date
13056 loop tree arrange for the child function to fixup loops. */
13057 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
13058 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
13060 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
13061 num
= vec_safe_length (child_cfun
->local_decls
);
13062 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
13064 t
= (*child_cfun
->local_decls
)[srcidx
];
13065 if (DECL_CONTEXT (t
) == cfun
->decl
)
13067 if (srcidx
!= dstidx
)
13068 (*child_cfun
->local_decls
)[dstidx
] = t
;
13072 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
13074 /* Inform the callgraph about the new function. */
13075 child_cfun
->curr_properties
= cfun
->curr_properties
;
13076 child_cfun
->has_simduid_loops
|= cfun
->has_simduid_loops
;
13077 child_cfun
->has_force_vectorize_loops
|= cfun
->has_force_vectorize_loops
;
13078 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
13079 node
->parallelized_function
= 1;
13080 cgraph_node::add_new_function (child_fn
, true);
13082 /* Add the new function to the offload table. */
13083 if (ENABLE_OFFLOADING
)
13084 vec_safe_push (offload_funcs
, child_fn
);
13086 bool need_asm
= DECL_ASSEMBLER_NAME_SET_P (current_function_decl
)
13087 && !DECL_ASSEMBLER_NAME_SET_P (child_fn
);
13089 /* Fix the callgraph edges for child_cfun. Those for cfun will be
13090 fixed in a following pass. */
13091 push_cfun (child_cfun
);
13093 assign_assembler_name_if_neeeded (child_fn
);
13094 cgraph_edge::rebuild_edges ();
13096 /* Some EH regions might become dead, see PR34608. If
13097 pass_cleanup_cfg isn't the first pass to happen with the
13098 new child, these dead EH edges might cause problems.
13099 Clean them up now. */
13100 if (flag_exceptions
)
13103 bool changed
= false;
13105 FOR_EACH_BB_FN (bb
, cfun
)
13106 changed
|= gimple_purge_dead_eh_edges (bb
);
13108 cleanup_tree_cfg ();
13110 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
13111 verify_loop_structure ();
13114 if (dump_file
&& !gimple_in_ssa_p (cfun
))
13116 omp_any_child_fn_dumped
= true;
13117 dump_function_header (dump_file
, child_fn
, dump_flags
);
13118 dump_function_to_file (child_fn
, dump_file
, dump_flags
);
13122 /* Emit a library call to launch the offloading region, or do data
13124 tree t1
, t2
, t3
, t4
, device
, cond
, depend
, c
, clauses
;
13125 enum built_in_function start_ix
;
13126 location_t clause_loc
;
13127 unsigned int flags_i
= 0;
13129 switch (gimple_omp_target_kind (entry_stmt
))
13131 case GF_OMP_TARGET_KIND_REGION
:
13132 start_ix
= BUILT_IN_GOMP_TARGET
;
13134 case GF_OMP_TARGET_KIND_DATA
:
13135 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
13137 case GF_OMP_TARGET_KIND_UPDATE
:
13138 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
13140 case GF_OMP_TARGET_KIND_ENTER_DATA
:
13141 start_ix
= BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
;
13143 case GF_OMP_TARGET_KIND_EXIT_DATA
:
13144 start_ix
= BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
;
13145 flags_i
|= GOMP_TARGET_FLAG_EXIT_DATA
;
13147 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
13148 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
13149 start_ix
= BUILT_IN_GOACC_PARALLEL
;
13151 case GF_OMP_TARGET_KIND_OACC_DATA
:
13152 start_ix
= BUILT_IN_GOACC_DATA_START
;
13154 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
13155 start_ix
= BUILT_IN_GOACC_UPDATE
;
13157 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
13158 start_ix
= BUILT_IN_GOACC_ENTER_EXIT_DATA
;
13160 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
13161 start_ix
= BUILT_IN_GOACC_DECLARE
;
13163 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
13164 start_ix
= BUILT_IN_GOACC_HOST_DATA
;
13167 gcc_unreachable ();
13170 clauses
= gimple_omp_target_clauses (entry_stmt
);
13172 /* By default, the value of DEVICE is GOMP_DEVICE_ICV (let runtime
13173 library choose) and there is no conditional. */
13175 device
= build_int_cst (integer_type_node
, GOMP_DEVICE_ICV
);
13177 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
13179 cond
= OMP_CLAUSE_IF_EXPR (c
);
13181 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
13184 /* Even if we pass it to all library function calls, it is currently only
13185 defined/used for the OpenMP target ones. */
13186 gcc_checking_assert (start_ix
== BUILT_IN_GOMP_TARGET
13187 || start_ix
== BUILT_IN_GOMP_TARGET_DATA
13188 || start_ix
== BUILT_IN_GOMP_TARGET_UPDATE
13189 || start_ix
== BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
);
13191 device
= OMP_CLAUSE_DEVICE_ID (c
);
13192 clause_loc
= OMP_CLAUSE_LOCATION (c
);
13195 clause_loc
= gimple_location (entry_stmt
);
13197 c
= find_omp_clause (clauses
, OMP_CLAUSE_NOWAIT
);
13199 flags_i
|= GOMP_TARGET_FLAG_NOWAIT
;
13201 /* Ensure 'device' is of the correct type. */
13202 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
13204 /* If we found the clause 'if (cond)', build
13205 (cond ? device : GOMP_DEVICE_HOST_FALLBACK). */
13208 cond
= gimple_boolify (cond
);
13210 basic_block cond_bb
, then_bb
, else_bb
;
13214 tmp_var
= create_tmp_var (TREE_TYPE (device
));
13216 e
= split_block_after_labels (new_bb
);
13219 gsi
= gsi_last_bb (new_bb
);
13221 e
= split_block (new_bb
, gsi_stmt (gsi
));
13227 then_bb
= create_empty_bb (cond_bb
);
13228 else_bb
= create_empty_bb (then_bb
);
13229 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
13230 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
13232 stmt
= gimple_build_cond_empty (cond
);
13233 gsi
= gsi_last_bb (cond_bb
);
13234 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
13236 gsi
= gsi_start_bb (then_bb
);
13237 stmt
= gimple_build_assign (tmp_var
, device
);
13238 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
13240 gsi
= gsi_start_bb (else_bb
);
13241 stmt
= gimple_build_assign (tmp_var
,
13242 build_int_cst (integer_type_node
,
13243 GOMP_DEVICE_HOST_FALLBACK
));
13244 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
13246 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
13247 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
13248 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
13249 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
13250 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
13251 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
13256 gsi
= gsi_last_bb (new_bb
);
13257 t
= gimple_omp_target_data_arg (entry_stmt
);
13260 t1
= size_zero_node
;
13261 t2
= build_zero_cst (ptr_type_node
);
13267 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
13268 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
13269 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
13270 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
13271 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
13275 bool tagging
= false;
13276 /* The maximum number used by any start_ix, without varargs. */
13277 auto_vec
<tree
, 11> args
;
13278 args
.quick_push (device
);
13280 args
.quick_push (build_fold_addr_expr (child_fn
));
13281 args
.quick_push (t1
);
13282 args
.quick_push (t2
);
13283 args
.quick_push (t3
);
13284 args
.quick_push (t4
);
13287 case BUILT_IN_GOACC_DATA_START
:
13288 case BUILT_IN_GOACC_DECLARE
:
13289 case BUILT_IN_GOMP_TARGET_DATA
:
13290 case BUILT_IN_GOACC_HOST_DATA
:
13292 case BUILT_IN_GOMP_TARGET
:
13293 case BUILT_IN_GOMP_TARGET_UPDATE
:
13294 case BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
:
13295 args
.quick_push (build_int_cst (unsigned_type_node
, flags_i
));
13296 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
13298 depend
= OMP_CLAUSE_DECL (c
);
13300 depend
= build_int_cst (ptr_type_node
, 0);
13301 args
.quick_push (depend
);
13302 if (start_ix
== BUILT_IN_GOMP_TARGET
)
13303 args
.quick_push (get_target_arguments (&gsi
, entry_stmt
));
13305 case BUILT_IN_GOACC_PARALLEL
:
13307 set_oacc_fn_attrib (child_fn
, clauses
, &args
);
13311 case BUILT_IN_GOACC_ENTER_EXIT_DATA
:
13312 case BUILT_IN_GOACC_UPDATE
:
13314 tree t_async
= NULL_TREE
;
13316 /* If present, use the value specified by the respective
13317 clause, making sure that is of the correct type. */
13318 c
= find_omp_clause (clauses
, OMP_CLAUSE_ASYNC
);
13320 t_async
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
13322 OMP_CLAUSE_ASYNC_EXPR (c
));
13324 /* Default values for t_async. */
13325 t_async
= fold_convert_loc (gimple_location (entry_stmt
),
13327 build_int_cst (integer_type_node
,
13329 if (tagging
&& t_async
)
13331 unsigned HOST_WIDE_INT i_async
;
13333 if (TREE_CODE (t_async
) == INTEGER_CST
)
13335 /* See if we can pack the async arg in to the tag's
13337 i_async
= TREE_INT_CST_LOW (t_async
);
13339 if (i_async
< GOMP_LAUNCH_OP_MAX
)
13340 t_async
= NULL_TREE
;
13343 i_async
= GOMP_LAUNCH_OP_MAX
;
13344 args
.safe_push (oacc_launch_pack
13345 (GOMP_LAUNCH_ASYNC
, NULL_TREE
, i_async
));
13348 args
.safe_push (t_async
);
13350 /* Save the argument index, and ... */
13351 unsigned t_wait_idx
= args
.length ();
13352 unsigned num_waits
= 0;
13353 c
= find_omp_clause (clauses
, OMP_CLAUSE_WAIT
);
13355 /* ... push a placeholder. */
13356 args
.safe_push (integer_zero_node
);
13358 for (; c
; c
= OMP_CLAUSE_CHAIN (c
))
13359 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_WAIT
)
13361 args
.safe_push (fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
13363 OMP_CLAUSE_WAIT_EXPR (c
)));
13367 if (!tagging
|| num_waits
)
13371 /* Now that we know the number, update the placeholder. */
13373 len
= oacc_launch_pack (GOMP_LAUNCH_WAIT
, NULL_TREE
, num_waits
);
13375 len
= build_int_cst (integer_type_node
, num_waits
);
13376 len
= fold_convert_loc (gimple_location (entry_stmt
),
13377 unsigned_type_node
, len
);
13378 args
[t_wait_idx
] = len
;
13383 gcc_unreachable ();
13386 /* Push terminal marker - zero. */
13387 args
.safe_push (oacc_launch_pack (0, NULL_TREE
, 0));
13389 g
= gimple_build_call_vec (builtin_decl_explicit (start_ix
), args
);
13390 gimple_set_location (g
, gimple_location (entry_stmt
));
13391 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
13394 g
= gsi_stmt (gsi
);
13395 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
13396 gsi_remove (&gsi
, true);
13398 if (data_region
&& region
->exit
)
13400 gsi
= gsi_last_bb (region
->exit
);
13401 g
= gsi_stmt (gsi
);
13402 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
13403 gsi_remove (&gsi
, true);
13407 /* Expand KFOR loop as a GPGPU kernel, i.e. as a body only with iteration
13408 variable derived from the thread number. */
13411 grid_expand_omp_for_loop (struct omp_region
*kfor
)
13415 gimple_stmt_iterator gsi
;
13417 struct omp_for_data fd
;
13419 gomp_for
*for_stmt
= as_a
<gomp_for
*> (last_stmt (kfor
->entry
));
13420 gcc_checking_assert (gimple_omp_for_kind (for_stmt
)
13421 == GF_OMP_FOR_KIND_GRID_LOOP
);
13422 basic_block body_bb
= FALLTHRU_EDGE (kfor
->entry
)->dest
;
13424 gcc_assert (gimple_omp_for_collapse (for_stmt
) == 1);
13425 gcc_assert (kfor
->cont
);
13426 extract_omp_for_data (for_stmt
, &fd
, NULL
);
13428 itype
= type
= TREE_TYPE (fd
.loop
.v
);
13429 if (POINTER_TYPE_P (type
))
13430 itype
= signed_type_for (type
);
13432 gsi
= gsi_start_bb (body_bb
);
13435 step
= fd
.loop
.step
;
13436 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
13437 true, NULL_TREE
, true, GSI_SAME_STMT
);
13438 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
13439 true, NULL_TREE
, true, GSI_SAME_STMT
);
13440 threadid
= build_call_expr (builtin_decl_explicit
13441 (BUILT_IN_OMP_GET_THREAD_NUM
), 0);
13442 threadid
= fold_convert (itype
, threadid
);
13443 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
13444 true, GSI_SAME_STMT
);
13446 tree startvar
= fd
.loop
.v
;
13447 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, step
);
13448 if (POINTER_TYPE_P (type
))
13449 t
= fold_build_pointer_plus (n1
, t
);
13451 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
13452 t
= fold_convert (type
, t
);
13453 t
= force_gimple_operand_gsi (&gsi
, t
,
13455 && TREE_ADDRESSABLE (startvar
),
13456 NULL_TREE
, true, GSI_SAME_STMT
);
13457 gassign
*assign_stmt
= gimple_build_assign (startvar
, t
);
13458 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
13460 /* Remove the omp for statement */
13461 gsi
= gsi_last_bb (kfor
->entry
);
13462 gsi_remove (&gsi
, true);
13464 /* Remove the GIMPLE_OMP_CONTINUE statement. */
13465 gsi
= gsi_last_bb (kfor
->cont
);
13466 gcc_assert (!gsi_end_p (gsi
)
13467 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_CONTINUE
);
13468 gsi_remove (&gsi
, true);
13470 /* Replace the GIMPLE_OMP_RETURN with a real return. */
13471 gsi
= gsi_last_bb (kfor
->exit
);
13472 gcc_assert (!gsi_end_p (gsi
)
13473 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
13474 gsi_remove (&gsi
, true);
13476 /* Fixup the much simpler CFG. */
13477 remove_edge (find_edge (kfor
->cont
, body_bb
));
13479 if (kfor
->cont
!= body_bb
)
13480 set_immediate_dominator (CDI_DOMINATORS
, kfor
->cont
, body_bb
);
13481 set_immediate_dominator (CDI_DOMINATORS
, kfor
->exit
, kfor
->cont
);
13484 /* Structure passed to grid_remap_kernel_arg_accesses so that it can remap
13487 struct grid_arg_decl_map
13493 /* Invoked through walk_gimple_op, will remap all PARM_DECLs to the ones
13494 pertaining to kernel function. */
13497 grid_remap_kernel_arg_accesses (tree
*tp
, int *walk_subtrees
, void *data
)
13499 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
13500 struct grid_arg_decl_map
*adm
= (struct grid_arg_decl_map
*) wi
->info
;
13503 if (t
== adm
->old_arg
)
13504 *tp
= adm
->new_arg
;
13505 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
13509 static void expand_omp (struct omp_region
*region
);
13511 /* If TARGET region contains a kernel body for loop, remove its region from the
13512 TARGET and expand it in GPGPU kernel fashion. */
13515 grid_expand_target_grid_body (struct omp_region
*target
)
13517 if (!hsa_gen_requested_p ())
13520 gomp_target
*tgt_stmt
= as_a
<gomp_target
*> (last_stmt (target
->entry
));
13521 struct omp_region
**pp
;
13523 for (pp
= &target
->inner
; *pp
; pp
= &(*pp
)->next
)
13524 if ((*pp
)->type
== GIMPLE_OMP_GRID_BODY
)
13527 struct omp_region
*gpukernel
= *pp
;
13529 tree orig_child_fndecl
= gimple_omp_target_child_fn (tgt_stmt
);
13532 /* HSA cannot handle OACC stuff. */
13533 if (gimple_omp_target_kind (tgt_stmt
) != GF_OMP_TARGET_KIND_REGION
)
13535 gcc_checking_assert (orig_child_fndecl
);
13536 gcc_assert (!find_omp_clause (gimple_omp_target_clauses (tgt_stmt
),
13537 OMP_CLAUSE__GRIDDIM_
));
13538 cgraph_node
*n
= cgraph_node::get (orig_child_fndecl
);
13540 hsa_register_kernel (n
);
13544 gcc_assert (find_omp_clause (gimple_omp_target_clauses (tgt_stmt
),
13545 OMP_CLAUSE__GRIDDIM_
));
13546 tree inside_block
= gimple_block (first_stmt (single_succ (gpukernel
->entry
)));
13547 *pp
= gpukernel
->next
;
13548 for (pp
= &gpukernel
->inner
; *pp
; pp
= &(*pp
)->next
)
13549 if ((*pp
)->type
== GIMPLE_OMP_FOR
)
13552 struct omp_region
*kfor
= *pp
;
13554 gcc_assert (gimple_omp_for_kind (last_stmt ((kfor
)->entry
))
13555 == GF_OMP_FOR_KIND_GRID_LOOP
);
13558 expand_omp (kfor
->inner
);
13559 if (gpukernel
->inner
)
13560 expand_omp (gpukernel
->inner
);
13562 tree kern_fndecl
= copy_node (orig_child_fndecl
);
13563 DECL_NAME (kern_fndecl
) = clone_function_name (kern_fndecl
, "kernel");
13564 SET_DECL_ASSEMBLER_NAME (kern_fndecl
, DECL_NAME (kern_fndecl
));
13565 tree tgtblock
= gimple_block (tgt_stmt
);
13566 tree fniniblock
= make_node (BLOCK
);
13567 BLOCK_ABSTRACT_ORIGIN (fniniblock
) = tgtblock
;
13568 BLOCK_SOURCE_LOCATION (fniniblock
) = BLOCK_SOURCE_LOCATION (tgtblock
);
13569 BLOCK_SOURCE_END_LOCATION (fniniblock
) = BLOCK_SOURCE_END_LOCATION (tgtblock
);
13570 DECL_INITIAL (kern_fndecl
) = fniniblock
;
13571 push_struct_function (kern_fndecl
);
13572 cfun
->function_end_locus
= gimple_location (tgt_stmt
);
13575 tree old_parm_decl
= DECL_ARGUMENTS (kern_fndecl
);
13576 gcc_assert (!DECL_CHAIN (old_parm_decl
));
13577 tree new_parm_decl
= copy_node (DECL_ARGUMENTS (kern_fndecl
));
13578 DECL_CONTEXT (new_parm_decl
) = kern_fndecl
;
13579 DECL_ARGUMENTS (kern_fndecl
) = new_parm_decl
;
13580 struct function
*kern_cfun
= DECL_STRUCT_FUNCTION (kern_fndecl
);
13581 kern_cfun
->curr_properties
= cfun
->curr_properties
;
13583 remove_edge (BRANCH_EDGE (kfor
->entry
));
13584 grid_expand_omp_for_loop (kfor
);
13586 /* Remove the omp for statement */
13587 gimple_stmt_iterator gsi
= gsi_last_bb (gpukernel
->entry
);
13588 gsi_remove (&gsi
, true);
13589 /* Replace the GIMPLE_OMP_RETURN at the end of the kernel region with a real
13591 gsi
= gsi_last_bb (gpukernel
->exit
);
13592 gcc_assert (!gsi_end_p (gsi
)
13593 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
13594 gimple
*ret_stmt
= gimple_build_return (NULL
);
13595 gsi_insert_after (&gsi
, ret_stmt
, GSI_SAME_STMT
);
13596 gsi_remove (&gsi
, true);
13598 /* Statements in the first BB in the target construct have been produced by
13599 target lowering and must be copied inside the GPUKERNEL, with the two
13600 exceptions of the first OMP statement and the OMP_DATA assignment
13602 gsi
= gsi_start_bb (single_succ (gpukernel
->entry
));
13603 tree data_arg
= gimple_omp_target_data_arg (tgt_stmt
);
13604 tree sender
= data_arg
? TREE_VEC_ELT (data_arg
, 0) : NULL
;
13605 for (gimple_stmt_iterator tsi
= gsi_start_bb (single_succ (target
->entry
));
13606 !gsi_end_p (tsi
); gsi_next (&tsi
))
13608 gimple
*stmt
= gsi_stmt (tsi
);
13609 if (is_gimple_omp (stmt
))
13612 && is_gimple_assign (stmt
)
13613 && TREE_CODE (gimple_assign_rhs1 (stmt
)) == ADDR_EXPR
13614 && TREE_OPERAND (gimple_assign_rhs1 (stmt
), 0) == sender
)
13616 gimple
*copy
= gimple_copy (stmt
);
13617 gsi_insert_before (&gsi
, copy
, GSI_SAME_STMT
);
13618 gimple_set_block (copy
, fniniblock
);
13621 move_sese_region_to_fn (kern_cfun
, single_succ (gpukernel
->entry
),
13622 gpukernel
->exit
, inside_block
);
13624 cgraph_node
*kcn
= cgraph_node::get_create (kern_fndecl
);
13625 kcn
->mark_force_output ();
13626 cgraph_node
*orig_child
= cgraph_node::get (orig_child_fndecl
);
13628 hsa_register_kernel (kcn
, orig_child
);
13630 cgraph_node::add_new_function (kern_fndecl
, true);
13631 push_cfun (kern_cfun
);
13632 cgraph_edge::rebuild_edges ();
13634 /* Re-map any mention of the PARM_DECL of the original function to the
13635 PARM_DECL of the new one.
13637 TODO: It would be great if lowering produced references into the GPU
13638 kernel decl straight away and we did not have to do this. */
13639 struct grid_arg_decl_map adm
;
13640 adm
.old_arg
= old_parm_decl
;
13641 adm
.new_arg
= new_parm_decl
;
13643 FOR_EACH_BB_FN (bb
, kern_cfun
)
13645 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
13647 gimple
*stmt
= gsi_stmt (gsi
);
13648 struct walk_stmt_info wi
;
13649 memset (&wi
, 0, sizeof (wi
));
13651 walk_gimple_op (stmt
, grid_remap_kernel_arg_accesses
, &wi
);
13659 /* Expand the parallel region tree rooted at REGION. Expansion
13660 proceeds in depth-first order. Innermost regions are expanded
13661 first. This way, parallel regions that require a new function to
13662 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
13663 internal dependencies in their body. */
13666 expand_omp (struct omp_region
*region
)
13668 omp_any_child_fn_dumped
= false;
13671 location_t saved_location
;
13672 gimple
*inner_stmt
= NULL
;
13674 /* First, determine whether this is a combined parallel+workshare
13676 if (region
->type
== GIMPLE_OMP_PARALLEL
)
13677 determine_parallel_type (region
);
13678 else if (region
->type
== GIMPLE_OMP_TARGET
)
13679 grid_expand_target_grid_body (region
);
13681 if (region
->type
== GIMPLE_OMP_FOR
13682 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
13683 inner_stmt
= last_stmt (region
->inner
->entry
);
13686 expand_omp (region
->inner
);
13688 saved_location
= input_location
;
13689 if (gimple_has_location (last_stmt (region
->entry
)))
13690 input_location
= gimple_location (last_stmt (region
->entry
));
13692 switch (region
->type
)
13694 case GIMPLE_OMP_PARALLEL
:
13695 case GIMPLE_OMP_TASK
:
13696 expand_omp_taskreg (region
);
13699 case GIMPLE_OMP_FOR
:
13700 expand_omp_for (region
, inner_stmt
);
13703 case GIMPLE_OMP_SECTIONS
:
13704 expand_omp_sections (region
);
13707 case GIMPLE_OMP_SECTION
:
13708 /* Individual omp sections are handled together with their
13709 parent GIMPLE_OMP_SECTIONS region. */
13712 case GIMPLE_OMP_SINGLE
:
13713 expand_omp_single (region
);
13716 case GIMPLE_OMP_ORDERED
:
13718 gomp_ordered
*ord_stmt
13719 = as_a
<gomp_ordered
*> (last_stmt (region
->entry
));
13720 if (find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
13721 OMP_CLAUSE_DEPEND
))
13723 /* We'll expand these when expanding corresponding
13724 worksharing region with ordered(n) clause. */
13725 gcc_assert (region
->outer
13726 && region
->outer
->type
== GIMPLE_OMP_FOR
);
13727 region
->ord_stmt
= ord_stmt
;
13732 case GIMPLE_OMP_MASTER
:
13733 case GIMPLE_OMP_TASKGROUP
:
13734 case GIMPLE_OMP_CRITICAL
:
13735 case GIMPLE_OMP_TEAMS
:
13736 expand_omp_synch (region
);
13739 case GIMPLE_OMP_ATOMIC_LOAD
:
13740 expand_omp_atomic (region
);
13743 case GIMPLE_OMP_TARGET
:
13744 expand_omp_target (region
);
13748 gcc_unreachable ();
13751 input_location
= saved_location
;
13752 region
= region
->next
;
13754 if (omp_any_child_fn_dumped
)
13757 dump_function_header (dump_file
, current_function_decl
, dump_flags
);
13758 omp_any_child_fn_dumped
= false;
13763 /* Helper for build_omp_regions. Scan the dominator tree starting at
13764 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
13765 true, the function ends once a single tree is built (otherwise, whole
13766 forest of OMP constructs may be built). */
13769 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
13772 gimple_stmt_iterator gsi
;
13776 gsi
= gsi_last_bb (bb
);
13777 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
13779 struct omp_region
*region
;
13780 enum gimple_code code
;
13782 stmt
= gsi_stmt (gsi
);
13783 code
= gimple_code (stmt
);
13784 if (code
== GIMPLE_OMP_RETURN
)
13786 /* STMT is the return point out of region PARENT. Mark it
13787 as the exit point and make PARENT the immediately
13788 enclosing region. */
13789 gcc_assert (parent
);
13792 parent
= parent
->outer
;
13794 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
13796 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
13797 GIMPLE_OMP_RETURN, but matches with
13798 GIMPLE_OMP_ATOMIC_LOAD. */
13799 gcc_assert (parent
);
13800 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
13803 parent
= parent
->outer
;
13805 else if (code
== GIMPLE_OMP_CONTINUE
)
13807 gcc_assert (parent
);
13810 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
13812 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
13813 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
13817 region
= new_omp_region (bb
, code
, parent
);
13819 if (code
== GIMPLE_OMP_TARGET
)
13821 switch (gimple_omp_target_kind (stmt
))
13823 case GF_OMP_TARGET_KIND_REGION
:
13824 case GF_OMP_TARGET_KIND_DATA
:
13825 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
13826 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
13827 case GF_OMP_TARGET_KIND_OACC_DATA
:
13828 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
13830 case GF_OMP_TARGET_KIND_UPDATE
:
13831 case GF_OMP_TARGET_KIND_ENTER_DATA
:
13832 case GF_OMP_TARGET_KIND_EXIT_DATA
:
13833 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
13834 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
13835 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
13836 /* ..., other than for those stand-alone directives... */
13840 gcc_unreachable ();
13843 else if (code
== GIMPLE_OMP_ORDERED
13844 && find_omp_clause (gimple_omp_ordered_clauses
13845 (as_a
<gomp_ordered
*> (stmt
)),
13846 OMP_CLAUSE_DEPEND
))
13847 /* #pragma omp ordered depend is also just a stand-alone
13850 /* ..., this directive becomes the parent for a new region. */
13856 if (single_tree
&& !parent
)
13859 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
13861 son
= next_dom_son (CDI_DOMINATORS
, son
))
13862 build_omp_regions_1 (son
, parent
, single_tree
);
13865 /* Builds the tree of OMP regions rooted at ROOT, storing it to
13866 root_omp_region. */
13869 build_omp_regions_root (basic_block root
)
13871 gcc_assert (root_omp_region
== NULL
);
13872 build_omp_regions_1 (root
, NULL
, true);
13873 gcc_assert (root_omp_region
!= NULL
);
13876 /* Expands omp construct (and its subconstructs) starting in HEAD. */
13879 omp_expand_local (basic_block head
)
13881 build_omp_regions_root (head
);
13882 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
13884 fprintf (dump_file
, "\nOMP region tree\n\n");
13885 dump_omp_region (dump_file
, root_omp_region
, 0);
13886 fprintf (dump_file
, "\n");
13889 remove_exit_barriers (root_omp_region
);
13890 expand_omp (root_omp_region
);
13892 free_omp_regions ();
13895 /* Scan the CFG and build a tree of OMP regions. Return the root of
13896 the OMP region tree. */
13899 build_omp_regions (void)
13901 gcc_assert (root_omp_region
== NULL
);
13902 calculate_dominance_info (CDI_DOMINATORS
);
13903 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
13906 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
13908 static unsigned int
13909 execute_expand_omp (void)
13911 build_omp_regions ();
13913 if (!root_omp_region
)
13918 fprintf (dump_file
, "\nOMP region tree\n\n");
13919 dump_omp_region (dump_file
, root_omp_region
, 0);
13920 fprintf (dump_file
, "\n");
13923 remove_exit_barriers (root_omp_region
);
13925 expand_omp (root_omp_region
);
13927 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
13928 verify_loop_structure ();
13929 cleanup_tree_cfg ();
13931 free_omp_regions ();
13936 /* OMP expansion -- the default pass, run before creation of SSA form. */
13940 const pass_data pass_data_expand_omp
=
13942 GIMPLE_PASS
, /* type */
13943 "ompexp", /* name */
13944 OPTGROUP_NONE
, /* optinfo_flags */
13945 TV_NONE
, /* tv_id */
13946 PROP_gimple_any
, /* properties_required */
13947 PROP_gimple_eomp
, /* properties_provided */
13948 0, /* properties_destroyed */
13949 0, /* todo_flags_start */
13950 0, /* todo_flags_finish */
13953 class pass_expand_omp
: public gimple_opt_pass
13956 pass_expand_omp (gcc::context
*ctxt
)
13957 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
13960 /* opt_pass methods: */
13961 virtual unsigned int execute (function
*)
13963 bool gate
= ((flag_cilkplus
!= 0 || flag_openacc
!= 0 || flag_openmp
!= 0
13964 || flag_openmp_simd
!= 0)
13965 && !seen_error ());
13967 /* This pass always runs, to provide PROP_gimple_eomp.
13968 But often, there is nothing to do. */
13972 return execute_expand_omp ();
13975 }; // class pass_expand_omp
13977 } // anon namespace
13980 make_pass_expand_omp (gcc::context
*ctxt
)
13982 return new pass_expand_omp (ctxt
);
13987 const pass_data pass_data_expand_omp_ssa
=
13989 GIMPLE_PASS
, /* type */
13990 "ompexpssa", /* name */
13991 OPTGROUP_NONE
, /* optinfo_flags */
13992 TV_NONE
, /* tv_id */
13993 PROP_cfg
| PROP_ssa
, /* properties_required */
13994 PROP_gimple_eomp
, /* properties_provided */
13995 0, /* properties_destroyed */
13996 0, /* todo_flags_start */
13997 TODO_cleanup_cfg
| TODO_rebuild_alias
, /* todo_flags_finish */
14000 class pass_expand_omp_ssa
: public gimple_opt_pass
14003 pass_expand_omp_ssa (gcc::context
*ctxt
)
14004 : gimple_opt_pass (pass_data_expand_omp_ssa
, ctxt
)
14007 /* opt_pass methods: */
14008 virtual bool gate (function
*fun
)
14010 return !(fun
->curr_properties
& PROP_gimple_eomp
);
14012 virtual unsigned int execute (function
*) { return execute_expand_omp (); }
14013 opt_pass
* clone () { return new pass_expand_omp_ssa (m_ctxt
); }
14015 }; // class pass_expand_omp_ssa
14017 } // anon namespace
14020 make_pass_expand_omp_ssa (gcc::context
*ctxt
)
14022 return new pass_expand_omp_ssa (ctxt
);
14025 /* Routines to lower OMP directives into OMP-GIMPLE. */
14027 /* If ctx is a worksharing context inside of a cancellable parallel
14028 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
14029 and conditional branch to parallel's cancel_label to handle
14030 cancellation in the implicit barrier. */
14033 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
14035 gimple
*omp_return
= gimple_seq_last_stmt (*body
);
14036 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
14037 if (gimple_omp_return_nowait_p (omp_return
))
14040 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
14041 && ctx
->outer
->cancellable
)
14043 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
14044 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
14045 tree lhs
= create_tmp_var (c_bool_type
);
14046 gimple_omp_return_set_lhs (omp_return
, lhs
);
14047 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
14048 gimple
*g
= gimple_build_cond (NE_EXPR
, lhs
,
14049 fold_convert (c_bool_type
,
14050 boolean_false_node
),
14051 ctx
->outer
->cancel_label
, fallthru_label
);
14052 gimple_seq_add_stmt (body
, g
);
14053 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
14057 /* Lower the OpenMP sections directive in the current statement in GSI_P.
14058 CTX is the enclosing OMP context for the current statement. */
14061 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14063 tree block
, control
;
14064 gimple_stmt_iterator tgsi
;
14065 gomp_sections
*stmt
;
14067 gbind
*new_stmt
, *bind
;
14068 gimple_seq ilist
, dlist
, olist
, new_body
;
14070 stmt
= as_a
<gomp_sections
*> (gsi_stmt (*gsi_p
));
14072 push_gimplify_context ();
14076 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
14077 &ilist
, &dlist
, ctx
, NULL
);
14079 new_body
= gimple_omp_body (stmt
);
14080 gimple_omp_set_body (stmt
, NULL
);
14081 tgsi
= gsi_start (new_body
);
14082 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
14087 sec_start
= gsi_stmt (tgsi
);
14088 sctx
= maybe_lookup_ctx (sec_start
);
14091 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
14092 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
14093 GSI_CONTINUE_LINKING
);
14094 gimple_omp_set_body (sec_start
, NULL
);
14096 if (gsi_one_before_end_p (tgsi
))
14098 gimple_seq l
= NULL
;
14099 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
14101 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
14102 gimple_omp_section_set_last (sec_start
);
14105 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
14106 GSI_CONTINUE_LINKING
);
14109 block
= make_node (BLOCK
);
14110 bind
= gimple_build_bind (NULL
, new_body
, block
);
14113 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
14115 block
= make_node (BLOCK
);
14116 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
14117 gsi_replace (gsi_p
, new_stmt
, true);
14119 pop_gimplify_context (new_stmt
);
14120 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
14121 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
14122 if (BLOCK_VARS (block
))
14123 TREE_USED (block
) = 1;
14126 gimple_seq_add_seq (&new_body
, ilist
);
14127 gimple_seq_add_stmt (&new_body
, stmt
);
14128 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
14129 gimple_seq_add_stmt (&new_body
, bind
);
14131 control
= create_tmp_var (unsigned_type_node
, ".section");
14132 t
= gimple_build_omp_continue (control
, control
);
14133 gimple_omp_sections_set_control (stmt
, control
);
14134 gimple_seq_add_stmt (&new_body
, t
);
14136 gimple_seq_add_seq (&new_body
, olist
);
14137 if (ctx
->cancellable
)
14138 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
14139 gimple_seq_add_seq (&new_body
, dlist
);
14141 new_body
= maybe_catch_exception (new_body
);
14143 t
= gimple_build_omp_return
14144 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
14145 OMP_CLAUSE_NOWAIT
));
14146 gimple_seq_add_stmt (&new_body
, t
);
14147 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
14149 gimple_bind_set_body (new_stmt
, new_body
);
14153 /* A subroutine of lower_omp_single. Expand the simple form of
14154 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
14156 if (GOMP_single_start ())
14158 [ GOMP_barrier (); ] -> unless 'nowait' is present.
14160 FIXME. It may be better to delay expanding the logic of this until
14161 pass_expand_omp. The expanded logic may make the job more difficult
14162 to a synchronization analysis pass. */
14165 lower_omp_single_simple (gomp_single
*single_stmt
, gimple_seq
*pre_p
)
14167 location_t loc
= gimple_location (single_stmt
);
14168 tree tlabel
= create_artificial_label (loc
);
14169 tree flabel
= create_artificial_label (loc
);
14170 gimple
*call
, *cond
;
14173 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
14174 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)));
14175 call
= gimple_build_call (decl
, 0);
14176 gimple_call_set_lhs (call
, lhs
);
14177 gimple_seq_add_stmt (pre_p
, call
);
14179 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
14180 fold_convert_loc (loc
, TREE_TYPE (lhs
),
14181 boolean_true_node
),
14183 gimple_seq_add_stmt (pre_p
, cond
);
14184 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
14185 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
14186 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
14190 /* A subroutine of lower_omp_single. Expand the simple form of
14191 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
14193 #pragma omp single copyprivate (a, b, c)
14195 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
14198 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
14204 GOMP_single_copy_end (©out);
14215 FIXME. It may be better to delay expanding the logic of this until
14216 pass_expand_omp. The expanded logic may make the job more difficult
14217 to a synchronization analysis pass. */
14220 lower_omp_single_copy (gomp_single
*single_stmt
, gimple_seq
*pre_p
,
14223 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
14224 gimple_seq copyin_seq
;
14225 location_t loc
= gimple_location (single_stmt
);
14227 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
14229 ptr_type
= build_pointer_type (ctx
->record_type
);
14230 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
14232 l0
= create_artificial_label (loc
);
14233 l1
= create_artificial_label (loc
);
14234 l2
= create_artificial_label (loc
);
14236 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
14237 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
14238 t
= fold_convert_loc (loc
, ptr_type
, t
);
14239 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
14241 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
14242 build_int_cst (ptr_type
, 0));
14243 t
= build3 (COND_EXPR
, void_type_node
, t
,
14244 build_and_jump (&l0
), build_and_jump (&l1
));
14245 gimplify_and_add (t
, pre_p
);
14247 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
14249 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
14252 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
14255 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
14256 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
14257 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
14258 gimplify_and_add (t
, pre_p
);
14260 t
= build_and_jump (&l2
);
14261 gimplify_and_add (t
, pre_p
);
14263 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
14265 gimple_seq_add_seq (pre_p
, copyin_seq
);
14267 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
14271 /* Expand code for an OpenMP single directive. */
14274 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14278 gomp_single
*single_stmt
= as_a
<gomp_single
*> (gsi_stmt (*gsi_p
));
14280 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
14282 push_gimplify_context ();
14284 block
= make_node (BLOCK
);
14285 bind
= gimple_build_bind (NULL
, NULL
, block
);
14286 gsi_replace (gsi_p
, bind
, true);
14289 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
14290 &bind_body
, &dlist
, ctx
, NULL
);
14291 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
14293 gimple_seq_add_stmt (&bind_body
, single_stmt
);
14295 if (ctx
->record_type
)
14296 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
14298 lower_omp_single_simple (single_stmt
, &bind_body
);
14300 gimple_omp_set_body (single_stmt
, NULL
);
14302 gimple_seq_add_seq (&bind_body
, dlist
);
14304 bind_body
= maybe_catch_exception (bind_body
);
14306 t
= gimple_build_omp_return
14307 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
14308 OMP_CLAUSE_NOWAIT
));
14309 gimple_seq_add_stmt (&bind_body_tail
, t
);
14310 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
14311 if (ctx
->record_type
)
14313 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
14314 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
14315 TREE_THIS_VOLATILE (clobber
) = 1;
14316 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
14317 clobber
), GSI_SAME_STMT
);
14319 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
14320 gimple_bind_set_body (bind
, bind_body
);
14322 pop_gimplify_context (bind
);
14324 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14325 BLOCK_VARS (block
) = ctx
->block_vars
;
14326 if (BLOCK_VARS (block
))
14327 TREE_USED (block
) = 1;
14331 /* Expand code for an OpenMP master directive. */
14334 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14336 tree block
, lab
= NULL
, x
, bfn_decl
;
14337 gimple
*stmt
= gsi_stmt (*gsi_p
);
14339 location_t loc
= gimple_location (stmt
);
14342 push_gimplify_context ();
14344 block
= make_node (BLOCK
);
14345 bind
= gimple_build_bind (NULL
, NULL
, block
);
14346 gsi_replace (gsi_p
, bind
, true);
14347 gimple_bind_add_stmt (bind
, stmt
);
14349 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
14350 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
14351 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
14352 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
14354 gimplify_and_add (x
, &tseq
);
14355 gimple_bind_add_seq (bind
, tseq
);
14357 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14358 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
14359 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
14360 gimple_omp_set_body (stmt
, NULL
);
14362 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
14364 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
14366 pop_gimplify_context (bind
);
14368 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14369 BLOCK_VARS (block
) = ctx
->block_vars
;
14373 /* Expand code for an OpenMP taskgroup directive. */
14376 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14378 gimple
*stmt
= gsi_stmt (*gsi_p
);
14381 tree block
= make_node (BLOCK
);
14383 bind
= gimple_build_bind (NULL
, NULL
, block
);
14384 gsi_replace (gsi_p
, bind
, true);
14385 gimple_bind_add_stmt (bind
, stmt
);
14387 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
14389 gimple_bind_add_stmt (bind
, x
);
14391 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14392 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
14393 gimple_omp_set_body (stmt
, NULL
);
14395 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
14397 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14398 BLOCK_VARS (block
) = ctx
->block_vars
;
14402 /* Fold the OMP_ORDERED_CLAUSES for the OMP_ORDERED in STMT if possible. */
14405 lower_omp_ordered_clauses (gimple_stmt_iterator
*gsi_p
, gomp_ordered
*ord_stmt
,
14408 struct omp_for_data fd
;
14409 if (!ctx
->outer
|| gimple_code (ctx
->outer
->stmt
) != GIMPLE_OMP_FOR
)
14412 unsigned int len
= gimple_omp_for_collapse (ctx
->outer
->stmt
);
14413 struct omp_for_data_loop
*loops
= XALLOCAVEC (struct omp_for_data_loop
, len
);
14414 extract_omp_for_data (as_a
<gomp_for
*> (ctx
->outer
->stmt
), &fd
, loops
);
14418 tree
*list_p
= gimple_omp_ordered_clauses_ptr (ord_stmt
);
14419 tree c
= gimple_omp_ordered_clauses (ord_stmt
);
14420 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
14421 && OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
)
14423 /* Merge depend clauses from multiple adjacent
14424 #pragma omp ordered depend(sink:...) constructs
14425 into one #pragma omp ordered depend(sink:...), so that
14426 we can optimize them together. */
14427 gimple_stmt_iterator gsi
= *gsi_p
;
14429 while (!gsi_end_p (gsi
))
14431 gimple
*stmt
= gsi_stmt (gsi
);
14432 if (is_gimple_debug (stmt
)
14433 || gimple_code (stmt
) == GIMPLE_NOP
)
14438 if (gimple_code (stmt
) != GIMPLE_OMP_ORDERED
)
14440 gomp_ordered
*ord_stmt2
= as_a
<gomp_ordered
*> (stmt
);
14441 c
= gimple_omp_ordered_clauses (ord_stmt2
);
14443 || OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_DEPEND
14444 || OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_SINK
)
14447 list_p
= &OMP_CLAUSE_CHAIN (*list_p
);
14449 gsi_remove (&gsi
, true);
14453 /* Canonicalize sink dependence clauses into one folded clause if
14456 The basic algorithm is to create a sink vector whose first
14457 element is the GCD of all the first elements, and whose remaining
14458 elements are the minimum of the subsequent columns.
14460 We ignore dependence vectors whose first element is zero because
14461 such dependencies are known to be executed by the same thread.
14463 We take into account the direction of the loop, so a minimum
14464 becomes a maximum if the loop is iterating forwards. We also
14465 ignore sink clauses where the loop direction is unknown, or where
14466 the offsets are clearly invalid because they are not a multiple
14467 of the loop increment.
14471 #pragma omp for ordered(2)
14472 for (i=0; i < N; ++i)
14473 for (j=0; j < M; ++j)
14475 #pragma omp ordered \
14476 depend(sink:i-8,j-2) \
14477 depend(sink:i,j-1) \ // Completely ignored because i+0.
14478 depend(sink:i-4,j-3) \
14479 depend(sink:i-6,j-4)
14480 #pragma omp ordered depend(source)
14485 depend(sink:-gcd(8,4,6),-min(2,3,4))
14490 /* FIXME: Computing GCD's where the first element is zero is
14491 non-trivial in the presence of collapsed loops. Do this later. */
14492 if (fd
.collapse
> 1)
14495 wide_int
*folded_deps
= XALLOCAVEC (wide_int
, 2 * len
- 1);
14496 memset (folded_deps
, 0, sizeof (*folded_deps
) * (2 * len
- 1));
14497 tree folded_dep
= NULL_TREE
;
14498 /* TRUE if the first dimension's offset is negative. */
14499 bool neg_offset_p
= false;
14501 list_p
= gimple_omp_ordered_clauses_ptr (ord_stmt
);
14503 while ((c
= *list_p
) != NULL
)
14505 bool remove
= false;
14507 gcc_assert (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
);
14508 if (OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_SINK
)
14509 goto next_ordered_clause
;
14512 for (vec
= OMP_CLAUSE_DECL (c
), i
= 0;
14513 vec
&& TREE_CODE (vec
) == TREE_LIST
;
14514 vec
= TREE_CHAIN (vec
), ++i
)
14516 gcc_assert (i
< len
);
14518 /* extract_omp_for_data has canonicalized the condition. */
14519 gcc_assert (fd
.loops
[i
].cond_code
== LT_EXPR
14520 || fd
.loops
[i
].cond_code
== GT_EXPR
);
14521 bool forward
= fd
.loops
[i
].cond_code
== LT_EXPR
;
14522 bool maybe_lexically_later
= true;
14524 /* While the committee makes up its mind, bail if we have any
14525 non-constant steps. */
14526 if (TREE_CODE (fd
.loops
[i
].step
) != INTEGER_CST
)
14527 goto lower_omp_ordered_ret
;
14529 tree itype
= TREE_TYPE (TREE_VALUE (vec
));
14530 if (POINTER_TYPE_P (itype
))
14532 wide_int offset
= wide_int::from (TREE_PURPOSE (vec
),
14533 TYPE_PRECISION (itype
),
14534 TYPE_SIGN (itype
));
14536 /* Ignore invalid offsets that are not multiples of the step. */
14537 if (!wi::multiple_of_p
14538 (wi::abs (offset
), wi::abs ((wide_int
) fd
.loops
[i
].step
),
14541 warning_at (OMP_CLAUSE_LOCATION (c
), 0,
14542 "ignoring sink clause with offset that is not "
14543 "a multiple of the loop step");
14545 goto next_ordered_clause
;
14548 /* Calculate the first dimension. The first dimension of
14549 the folded dependency vector is the GCD of the first
14550 elements, while ignoring any first elements whose offset
14554 /* Ignore dependence vectors whose first dimension is 0. */
14558 goto next_ordered_clause
;
14562 if (!TYPE_UNSIGNED (itype
) && (forward
^ wi::neg_p (offset
)))
14564 error_at (OMP_CLAUSE_LOCATION (c
),
14565 "first offset must be in opposite direction "
14566 "of loop iterations");
14567 goto lower_omp_ordered_ret
;
14571 neg_offset_p
= forward
;
14572 /* Initialize the first time around. */
14573 if (folded_dep
== NULL_TREE
)
14576 folded_deps
[0] = offset
;
14579 folded_deps
[0] = wi::gcd (folded_deps
[0],
14583 /* Calculate minimum for the remaining dimensions. */
14586 folded_deps
[len
+ i
- 1] = offset
;
14587 if (folded_dep
== c
)
14588 folded_deps
[i
] = offset
;
14589 else if (maybe_lexically_later
14590 && !wi::eq_p (folded_deps
[i
], offset
))
14592 if (forward
^ wi::gts_p (folded_deps
[i
], offset
))
14596 for (j
= 1; j
<= i
; j
++)
14597 folded_deps
[j
] = folded_deps
[len
+ j
- 1];
14600 maybe_lexically_later
= false;
14604 gcc_assert (i
== len
);
14608 next_ordered_clause
:
14610 *list_p
= OMP_CLAUSE_CHAIN (c
);
14612 list_p
= &OMP_CLAUSE_CHAIN (c
);
14618 folded_deps
[0] = -folded_deps
[0];
14620 tree itype
= TREE_TYPE (TREE_VALUE (OMP_CLAUSE_DECL (folded_dep
)));
14621 if (POINTER_TYPE_P (itype
))
14624 TREE_PURPOSE (OMP_CLAUSE_DECL (folded_dep
))
14625 = wide_int_to_tree (itype
, folded_deps
[0]);
14626 OMP_CLAUSE_CHAIN (folded_dep
) = gimple_omp_ordered_clauses (ord_stmt
);
14627 *gimple_omp_ordered_clauses_ptr (ord_stmt
) = folded_dep
;
14630 lower_omp_ordered_ret
:
14632 /* Ordered without clauses is #pragma omp threads, while we want
14633 a nop instead if we remove all clauses. */
14634 if (gimple_omp_ordered_clauses (ord_stmt
) == NULL_TREE
)
14635 gsi_replace (gsi_p
, gimple_build_nop (), true);
14639 /* Expand code for an OpenMP ordered directive. */
14642 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14645 gimple
*stmt
= gsi_stmt (*gsi_p
);
14646 gomp_ordered
*ord_stmt
= as_a
<gomp_ordered
*> (stmt
);
14649 bool simd
= find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
14651 bool threads
= find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
14652 OMP_CLAUSE_THREADS
);
14654 if (find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
14655 OMP_CLAUSE_DEPEND
))
14657 /* FIXME: This is needs to be moved to the expansion to verify various
14658 conditions only testable on cfg with dominators computed, and also
14659 all the depend clauses to be merged still might need to be available
14660 for the runtime checks. */
14662 lower_omp_ordered_clauses (gsi_p
, ord_stmt
, ctx
);
14666 push_gimplify_context ();
14668 block
= make_node (BLOCK
);
14669 bind
= gimple_build_bind (NULL
, NULL
, block
);
14670 gsi_replace (gsi_p
, bind
, true);
14671 gimple_bind_add_stmt (bind
, stmt
);
14675 x
= gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_START
, 1,
14676 build_int_cst (NULL_TREE
, threads
));
14677 cfun
->has_simduid_loops
= true;
14680 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
14682 gimple_bind_add_stmt (bind
, x
);
14684 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14685 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
14686 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
14687 gimple_omp_set_body (stmt
, NULL
);
14690 x
= gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_END
, 1,
14691 build_int_cst (NULL_TREE
, threads
));
14693 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
),
14695 gimple_bind_add_stmt (bind
, x
);
14697 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
14699 pop_gimplify_context (bind
);
14701 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14702 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
14706 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
14707 substitution of a couple of function calls. But in the NAMED case,
14708 requires that languages coordinate a symbol name. It is therefore
14709 best put here in common code. */
14711 static GTY(()) hash_map
<tree
, tree
> *critical_name_mutexes
;
14714 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14717 tree name
, lock
, unlock
;
14718 gomp_critical
*stmt
= as_a
<gomp_critical
*> (gsi_stmt (*gsi_p
));
14720 location_t loc
= gimple_location (stmt
);
14723 name
= gimple_omp_critical_name (stmt
);
14728 if (!critical_name_mutexes
)
14729 critical_name_mutexes
= hash_map
<tree
, tree
>::create_ggc (10);
14731 tree
*n
= critical_name_mutexes
->get (name
);
14736 decl
= create_tmp_var_raw (ptr_type_node
);
14738 new_str
= ACONCAT ((".gomp_critical_user_",
14739 IDENTIFIER_POINTER (name
), NULL
));
14740 DECL_NAME (decl
) = get_identifier (new_str
);
14741 TREE_PUBLIC (decl
) = 1;
14742 TREE_STATIC (decl
) = 1;
14743 DECL_COMMON (decl
) = 1;
14744 DECL_ARTIFICIAL (decl
) = 1;
14745 DECL_IGNORED_P (decl
) = 1;
14747 varpool_node::finalize_decl (decl
);
14749 critical_name_mutexes
->put (name
, decl
);
14754 /* If '#pragma omp critical' is inside offloaded region or
14755 inside function marked as offloadable, the symbol must be
14756 marked as offloadable too. */
14758 if (cgraph_node::get (current_function_decl
)->offloadable
)
14759 varpool_node::get_create (decl
)->offloadable
= 1;
14761 for (octx
= ctx
->outer
; octx
; octx
= octx
->outer
)
14762 if (is_gimple_omp_offloaded (octx
->stmt
))
14764 varpool_node::get_create (decl
)->offloadable
= 1;
14768 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
14769 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
14771 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
14772 unlock
= build_call_expr_loc (loc
, unlock
, 1,
14773 build_fold_addr_expr_loc (loc
, decl
));
14777 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
14778 lock
= build_call_expr_loc (loc
, lock
, 0);
14780 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
14781 unlock
= build_call_expr_loc (loc
, unlock
, 0);
14784 push_gimplify_context ();
14786 block
= make_node (BLOCK
);
14787 bind
= gimple_build_bind (NULL
, NULL
, block
);
14788 gsi_replace (gsi_p
, bind
, true);
14789 gimple_bind_add_stmt (bind
, stmt
);
14791 tbody
= gimple_bind_body (bind
);
14792 gimplify_and_add (lock
, &tbody
);
14793 gimple_bind_set_body (bind
, tbody
);
14795 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14796 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
14797 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
14798 gimple_omp_set_body (stmt
, NULL
);
14800 tbody
= gimple_bind_body (bind
);
14801 gimplify_and_add (unlock
, &tbody
);
14802 gimple_bind_set_body (bind
, tbody
);
14804 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
14806 pop_gimplify_context (bind
);
14807 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14808 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
14812 /* A subroutine of lower_omp_for. Generate code to emit the predicate
14813 for a lastprivate clause. Given a loop control predicate of (V
14814 cond N2), we gate the clause on (!(V cond N2)). The lowered form
14815 is appended to *DLIST, iterator initialization is appended to
14819 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
14820 gimple_seq
*dlist
, struct omp_context
*ctx
)
14822 tree clauses
, cond
, vinit
;
14823 enum tree_code cond_code
;
14826 cond_code
= fd
->loop
.cond_code
;
14827 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
14829 /* When possible, use a strict equality expression. This can let VRP
14830 type optimizations deduce the value and remove a copy. */
14831 if (tree_fits_shwi_p (fd
->loop
.step
))
14833 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
14834 if (step
== 1 || step
== -1)
14835 cond_code
= EQ_EXPR
;
14838 tree n2
= fd
->loop
.n2
;
14839 if (fd
->collapse
> 1
14840 && TREE_CODE (n2
) != INTEGER_CST
14841 && gimple_omp_for_combined_into_p (fd
->for_stmt
))
14843 struct omp_context
*taskreg_ctx
= NULL
;
14844 if (gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_FOR
)
14846 gomp_for
*gfor
= as_a
<gomp_for
*> (ctx
->outer
->stmt
);
14847 if (gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_FOR
14848 || gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
14850 if (gimple_omp_for_combined_into_p (gfor
))
14852 gcc_assert (ctx
->outer
->outer
14853 && is_parallel_ctx (ctx
->outer
->outer
));
14854 taskreg_ctx
= ctx
->outer
->outer
;
14858 struct omp_for_data outer_fd
;
14859 extract_omp_for_data (gfor
, &outer_fd
, NULL
);
14860 n2
= fold_convert (TREE_TYPE (n2
), outer_fd
.loop
.n2
);
14863 else if (gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_TASKLOOP
)
14864 taskreg_ctx
= ctx
->outer
->outer
;
14866 else if (is_taskreg_ctx (ctx
->outer
))
14867 taskreg_ctx
= ctx
->outer
;
14872 = find_omp_clause (gimple_omp_taskreg_clauses (taskreg_ctx
->stmt
),
14873 OMP_CLAUSE__LOOPTEMP_
);
14874 gcc_assert (innerc
);
14875 for (i
= 0; i
< fd
->collapse
; i
++)
14877 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
14878 OMP_CLAUSE__LOOPTEMP_
);
14879 gcc_assert (innerc
);
14881 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
14882 OMP_CLAUSE__LOOPTEMP_
);
14884 n2
= fold_convert (TREE_TYPE (n2
),
14885 lookup_decl (OMP_CLAUSE_DECL (innerc
),
14889 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, n2
);
14891 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
14893 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
14894 if (!gimple_seq_empty_p (stmts
))
14896 gimple_seq_add_seq (&stmts
, *dlist
);
14899 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
14900 vinit
= fd
->loop
.n1
;
14901 if (cond_code
== EQ_EXPR
14902 && tree_fits_shwi_p (fd
->loop
.n2
)
14903 && ! integer_zerop (fd
->loop
.n2
))
14904 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
14906 vinit
= unshare_expr (vinit
);
14908 /* Initialize the iterator variable, so that threads that don't execute
14909 any iterations don't execute the lastprivate clauses by accident. */
14910 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
14915 /* Lower code for an OMP loop directive. */
14918 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14920 tree
*rhs_p
, block
;
14921 struct omp_for_data fd
, *fdp
= NULL
;
14922 gomp_for
*stmt
= as_a
<gomp_for
*> (gsi_stmt (*gsi_p
));
14924 gimple_seq omp_for_body
, body
, dlist
;
14925 gimple_seq oacc_head
= NULL
, oacc_tail
= NULL
;
14928 push_gimplify_context ();
14930 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
14932 block
= make_node (BLOCK
);
14933 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
14934 /* Replace at gsi right away, so that 'stmt' is no member
14935 of a sequence anymore as we're going to add to a different
14937 gsi_replace (gsi_p
, new_stmt
, true);
14939 /* Move declaration of temporaries in the loop body before we make
14941 omp_for_body
= gimple_omp_body (stmt
);
14942 if (!gimple_seq_empty_p (omp_for_body
)
14943 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
14946 = as_a
<gbind
*> (gimple_seq_first_stmt (omp_for_body
));
14947 tree vars
= gimple_bind_vars (inner_bind
);
14948 gimple_bind_append_vars (new_stmt
, vars
);
14949 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
14950 keep them on the inner_bind and it's block. */
14951 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
14952 if (gimple_bind_block (inner_bind
))
14953 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
14956 if (gimple_omp_for_combined_into_p (stmt
))
14958 extract_omp_for_data (stmt
, &fd
, NULL
);
14961 /* We need two temporaries with fd.loop.v type (istart/iend)
14962 and then (fd.collapse - 1) temporaries with the same
14963 type for count2 ... countN-1 vars if not constant. */
14965 tree type
= fd
.iter_type
;
14966 if (fd
.collapse
> 1
14967 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
14968 count
+= fd
.collapse
- 1;
14970 = (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
14971 || gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_TASKLOOP
);
14972 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
14973 tree clauses
= *pc
;
14976 = find_omp_clause (gimple_omp_taskreg_clauses (ctx
->outer
->stmt
),
14977 OMP_CLAUSE__LOOPTEMP_
);
14978 for (i
= 0; i
< count
; i
++)
14983 gcc_assert (outerc
);
14984 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
14985 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
14986 OMP_CLAUSE__LOOPTEMP_
);
14990 temp
= create_tmp_var (type
);
14991 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
14993 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
14994 OMP_CLAUSE_DECL (*pc
) = temp
;
14995 pc
= &OMP_CLAUSE_CHAIN (*pc
);
15000 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
15003 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
15005 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
15007 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
15009 /* Lower the header expressions. At this point, we can assume that
15010 the header is of the form:
15012 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
15014 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
15015 using the .omp_data_s mapping, if needed. */
15016 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
15018 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
15019 if (!is_gimple_min_invariant (*rhs_p
))
15020 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
15022 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
15023 if (!is_gimple_min_invariant (*rhs_p
))
15024 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
15026 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
15027 if (!is_gimple_min_invariant (*rhs_p
))
15028 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
15031 /* Once lowered, extract the bounds and clauses. */
15032 extract_omp_for_data (stmt
, &fd
, NULL
);
15034 if (is_gimple_omp_oacc (ctx
->stmt
)
15035 && !ctx_in_oacc_kernels_region (ctx
))
15036 lower_oacc_head_tail (gimple_location (stmt
),
15037 gimple_omp_for_clauses (stmt
),
15038 &oacc_head
, &oacc_tail
, ctx
);
15040 /* Add OpenACC partitioning and reduction markers just before the loop */
15042 gimple_seq_add_seq (&body
, oacc_head
);
15044 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
15046 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
15047 for (tree c
= gimple_omp_for_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
15048 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
15049 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
15051 OMP_CLAUSE_DECL (c
) = lookup_decl (OMP_CLAUSE_DECL (c
), ctx
);
15052 if (DECL_P (OMP_CLAUSE_LINEAR_STEP (c
)))
15053 OMP_CLAUSE_LINEAR_STEP (c
)
15054 = maybe_lookup_decl_in_outer_ctx (OMP_CLAUSE_LINEAR_STEP (c
),
15058 if (!gimple_omp_for_grid_phony (stmt
))
15059 gimple_seq_add_stmt (&body
, stmt
);
15060 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
15062 if (!gimple_omp_for_grid_phony (stmt
))
15063 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
15066 /* After the loop, add exit clauses. */
15067 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
15069 if (ctx
->cancellable
)
15070 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
15072 gimple_seq_add_seq (&body
, dlist
);
15074 body
= maybe_catch_exception (body
);
15076 if (!gimple_omp_for_grid_phony (stmt
))
15078 /* Region exit marker goes at the end of the loop body. */
15079 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
15080 maybe_add_implicit_barrier_cancel (ctx
, &body
);
15083 /* Add OpenACC joining and reduction markers just after the loop. */
15085 gimple_seq_add_seq (&body
, oacc_tail
);
15087 pop_gimplify_context (new_stmt
);
15089 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
15090 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
15091 if (BLOCK_VARS (block
))
15092 TREE_USED (block
) = 1;
15094 gimple_bind_set_body (new_stmt
, body
);
15095 gimple_omp_set_body (stmt
, NULL
);
15096 gimple_omp_for_set_pre_body (stmt
, NULL
);
15099 /* Callback for walk_stmts. Check if the current statement only contains
15100 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
15103 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
15104 bool *handled_ops_p
,
15105 struct walk_stmt_info
*wi
)
15107 int *info
= (int *) wi
->info
;
15108 gimple
*stmt
= gsi_stmt (*gsi_p
);
15110 *handled_ops_p
= true;
15111 switch (gimple_code (stmt
))
15115 case GIMPLE_OMP_FOR
:
15116 case GIMPLE_OMP_SECTIONS
:
15117 *info
= *info
== 0 ? 1 : -1;
15126 struct omp_taskcopy_context
15128 /* This field must be at the beginning, as we do "inheritance": Some
15129 callback functions for tree-inline.c (e.g., omp_copy_decl)
15130 receive a copy_body_data pointer that is up-casted to an
15131 omp_context pointer. */
15137 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
15139 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
15141 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
15142 return create_tmp_var (TREE_TYPE (var
));
15148 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
15150 tree name
, new_fields
= NULL
, type
, f
;
15152 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
15153 name
= DECL_NAME (TYPE_NAME (orig_type
));
15154 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
15155 TYPE_DECL
, name
, type
);
15156 TYPE_NAME (type
) = name
;
15158 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
15160 tree new_f
= copy_node (f
);
15161 DECL_CONTEXT (new_f
) = type
;
15162 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
15163 TREE_CHAIN (new_f
) = new_fields
;
15164 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
15165 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
15166 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
15168 new_fields
= new_f
;
15169 tcctx
->cb
.decl_map
->put (f
, new_f
);
15171 TYPE_FIELDS (type
) = nreverse (new_fields
);
15172 layout_type (type
);
15176 /* Create task copyfn. */
15179 create_task_copyfn (gomp_task
*task_stmt
, omp_context
*ctx
)
15181 struct function
*child_cfun
;
15182 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
15183 tree record_type
, srecord_type
, bind
, list
;
15184 bool record_needs_remap
= false, srecord_needs_remap
= false;
15186 struct omp_taskcopy_context tcctx
;
15187 location_t loc
= gimple_location (task_stmt
);
15189 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
15190 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
15191 gcc_assert (child_cfun
->cfg
== NULL
);
15192 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
15194 /* Reset DECL_CONTEXT on function arguments. */
15195 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
15196 DECL_CONTEXT (t
) = child_fn
;
15198 /* Populate the function. */
15199 push_gimplify_context ();
15200 push_cfun (child_cfun
);
15202 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
15203 TREE_SIDE_EFFECTS (bind
) = 1;
15205 DECL_SAVED_TREE (child_fn
) = bind
;
15206 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
15208 /* Remap src and dst argument types if needed. */
15209 record_type
= ctx
->record_type
;
15210 srecord_type
= ctx
->srecord_type
;
15211 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
15212 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
15214 record_needs_remap
= true;
15217 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
15218 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
15220 srecord_needs_remap
= true;
15224 if (record_needs_remap
|| srecord_needs_remap
)
15226 memset (&tcctx
, '\0', sizeof (tcctx
));
15227 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
15228 tcctx
.cb
.dst_fn
= child_fn
;
15229 tcctx
.cb
.src_node
= cgraph_node::get (tcctx
.cb
.src_fn
);
15230 gcc_checking_assert (tcctx
.cb
.src_node
);
15231 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
15232 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
15233 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
15234 tcctx
.cb
.eh_lp_nr
= 0;
15235 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
15236 tcctx
.cb
.decl_map
= new hash_map
<tree
, tree
>;
15239 if (record_needs_remap
)
15240 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
15241 if (srecord_needs_remap
)
15242 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
15245 tcctx
.cb
.decl_map
= NULL
;
15247 arg
= DECL_ARGUMENTS (child_fn
);
15248 TREE_TYPE (arg
) = build_pointer_type (record_type
);
15249 sarg
= DECL_CHAIN (arg
);
15250 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
15252 /* First pass: initialize temporaries used in record_type and srecord_type
15253 sizes and field offsets. */
15254 if (tcctx
.cb
.decl_map
)
15255 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
15256 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
15260 decl
= OMP_CLAUSE_DECL (c
);
15261 p
= tcctx
.cb
.decl_map
->get (decl
);
15264 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
15265 sf
= (tree
) n
->value
;
15266 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15267 src
= build_simple_mem_ref_loc (loc
, sarg
);
15268 src
= omp_build_component_ref (src
, sf
);
15269 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
15270 append_to_statement_list (t
, &list
);
15273 /* Second pass: copy shared var pointers and copy construct non-VLA
15274 firstprivate vars. */
15275 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
15276 switch (OMP_CLAUSE_CODE (c
))
15278 splay_tree_key key
;
15279 case OMP_CLAUSE_SHARED
:
15280 decl
= OMP_CLAUSE_DECL (c
);
15281 key
= (splay_tree_key
) decl
;
15282 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
15283 key
= (splay_tree_key
) &DECL_UID (decl
);
15284 n
= splay_tree_lookup (ctx
->field_map
, key
);
15287 f
= (tree
) n
->value
;
15288 if (tcctx
.cb
.decl_map
)
15289 f
= *tcctx
.cb
.decl_map
->get (f
);
15290 n
= splay_tree_lookup (ctx
->sfield_map
, key
);
15291 sf
= (tree
) n
->value
;
15292 if (tcctx
.cb
.decl_map
)
15293 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15294 src
= build_simple_mem_ref_loc (loc
, sarg
);
15295 src
= omp_build_component_ref (src
, sf
);
15296 dst
= build_simple_mem_ref_loc (loc
, arg
);
15297 dst
= omp_build_component_ref (dst
, f
);
15298 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
15299 append_to_statement_list (t
, &list
);
15301 case OMP_CLAUSE_FIRSTPRIVATE
:
15302 decl
= OMP_CLAUSE_DECL (c
);
15303 if (is_variable_sized (decl
))
15305 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
15308 f
= (tree
) n
->value
;
15309 if (tcctx
.cb
.decl_map
)
15310 f
= *tcctx
.cb
.decl_map
->get (f
);
15311 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
15314 sf
= (tree
) n
->value
;
15315 if (tcctx
.cb
.decl_map
)
15316 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15317 src
= build_simple_mem_ref_loc (loc
, sarg
);
15318 src
= omp_build_component_ref (src
, sf
);
15319 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
15320 src
= build_simple_mem_ref_loc (loc
, src
);
15324 dst
= build_simple_mem_ref_loc (loc
, arg
);
15325 dst
= omp_build_component_ref (dst
, f
);
15326 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
15327 append_to_statement_list (t
, &list
);
15329 case OMP_CLAUSE_PRIVATE
:
15330 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
15332 decl
= OMP_CLAUSE_DECL (c
);
15333 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
15334 f
= (tree
) n
->value
;
15335 if (tcctx
.cb
.decl_map
)
15336 f
= *tcctx
.cb
.decl_map
->get (f
);
15337 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
15340 sf
= (tree
) n
->value
;
15341 if (tcctx
.cb
.decl_map
)
15342 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15343 src
= build_simple_mem_ref_loc (loc
, sarg
);
15344 src
= omp_build_component_ref (src
, sf
);
15345 if (use_pointer_for_field (decl
, NULL
))
15346 src
= build_simple_mem_ref_loc (loc
, src
);
15350 dst
= build_simple_mem_ref_loc (loc
, arg
);
15351 dst
= omp_build_component_ref (dst
, f
);
15352 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
15353 append_to_statement_list (t
, &list
);
15359 /* Last pass: handle VLA firstprivates. */
15360 if (tcctx
.cb
.decl_map
)
15361 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
15362 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
15366 decl
= OMP_CLAUSE_DECL (c
);
15367 if (!is_variable_sized (decl
))
15369 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
15372 f
= (tree
) n
->value
;
15373 f
= *tcctx
.cb
.decl_map
->get (f
);
15374 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
15375 ind
= DECL_VALUE_EXPR (decl
);
15376 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
15377 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
15378 n
= splay_tree_lookup (ctx
->sfield_map
,
15379 (splay_tree_key
) TREE_OPERAND (ind
, 0));
15380 sf
= (tree
) n
->value
;
15381 sf
= *tcctx
.cb
.decl_map
->get (sf
);
15382 src
= build_simple_mem_ref_loc (loc
, sarg
);
15383 src
= omp_build_component_ref (src
, sf
);
15384 src
= build_simple_mem_ref_loc (loc
, src
);
15385 dst
= build_simple_mem_ref_loc (loc
, arg
);
15386 dst
= omp_build_component_ref (dst
, f
);
15387 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
15388 append_to_statement_list (t
, &list
);
15389 n
= splay_tree_lookup (ctx
->field_map
,
15390 (splay_tree_key
) TREE_OPERAND (ind
, 0));
15391 df
= (tree
) n
->value
;
15392 df
= *tcctx
.cb
.decl_map
->get (df
);
15393 ptr
= build_simple_mem_ref_loc (loc
, arg
);
15394 ptr
= omp_build_component_ref (ptr
, df
);
15395 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
15396 build_fold_addr_expr_loc (loc
, dst
));
15397 append_to_statement_list (t
, &list
);
15400 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
15401 append_to_statement_list (t
, &list
);
15403 if (tcctx
.cb
.decl_map
)
15404 delete tcctx
.cb
.decl_map
;
15405 pop_gimplify_context (NULL
);
15406 BIND_EXPR_BODY (bind
) = list
;
15411 lower_depend_clauses (tree
*pclauses
, gimple_seq
*iseq
, gimple_seq
*oseq
)
15415 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
15417 clauses
= find_omp_clause (*pclauses
, OMP_CLAUSE_DEPEND
);
15418 gcc_assert (clauses
);
15419 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15420 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
15421 switch (OMP_CLAUSE_DEPEND_KIND (c
))
15423 case OMP_CLAUSE_DEPEND_IN
:
15426 case OMP_CLAUSE_DEPEND_OUT
:
15427 case OMP_CLAUSE_DEPEND_INOUT
:
15430 case OMP_CLAUSE_DEPEND_SOURCE
:
15431 case OMP_CLAUSE_DEPEND_SINK
:
15434 gcc_unreachable ();
15436 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
15437 tree array
= create_tmp_var (type
);
15438 TREE_ADDRESSABLE (array
) = 1;
15439 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
15441 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
15442 gimple_seq_add_stmt (iseq
, g
);
15443 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
15445 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
15446 gimple_seq_add_stmt (iseq
, g
);
15447 for (i
= 0; i
< 2; i
++)
15449 if ((i
? n_in
: n_out
) == 0)
15451 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15452 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
15453 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
15455 tree t
= OMP_CLAUSE_DECL (c
);
15456 t
= fold_convert (ptr_type_node
, t
);
15457 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
15458 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
15459 NULL_TREE
, NULL_TREE
);
15460 g
= gimple_build_assign (r
, t
);
15461 gimple_seq_add_stmt (iseq
, g
);
15464 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
15465 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
15466 OMP_CLAUSE_CHAIN (c
) = *pclauses
;
15468 tree clobber
= build_constructor (type
, NULL
);
15469 TREE_THIS_VOLATILE (clobber
) = 1;
15470 g
= gimple_build_assign (array
, clobber
);
15471 gimple_seq_add_stmt (oseq
, g
);
15474 /* Lower the OpenMP parallel or task directive in the current statement
15475 in GSI_P. CTX holds context information for the directive. */
15478 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
15482 gimple
*stmt
= gsi_stmt (*gsi_p
);
15483 gbind
*par_bind
, *bind
, *dep_bind
= NULL
;
15484 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
15485 location_t loc
= gimple_location (stmt
);
15487 clauses
= gimple_omp_taskreg_clauses (stmt
);
15489 = as_a
<gbind
*> (gimple_seq_first_stmt (gimple_omp_body (stmt
)));
15490 par_body
= gimple_bind_body (par_bind
);
15491 child_fn
= ctx
->cb
.dst_fn
;
15492 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
15493 && !gimple_omp_parallel_combined_p (stmt
))
15495 struct walk_stmt_info wi
;
15498 memset (&wi
, 0, sizeof (wi
));
15500 wi
.val_only
= true;
15501 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
15503 gimple_omp_parallel_set_combined_p (stmt
, true);
15505 gimple_seq dep_ilist
= NULL
;
15506 gimple_seq dep_olist
= NULL
;
15507 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
15508 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
15510 push_gimplify_context ();
15511 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
15512 lower_depend_clauses (gimple_omp_task_clauses_ptr (stmt
),
15513 &dep_ilist
, &dep_olist
);
15516 if (ctx
->srecord_type
)
15517 create_task_copyfn (as_a
<gomp_task
*> (stmt
), ctx
);
15519 push_gimplify_context ();
15524 bool phony_construct
= gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
15525 && gimple_omp_parallel_grid_phony (as_a
<gomp_parallel
*> (stmt
));
15526 if (phony_construct
&& ctx
->record_type
)
15528 gcc_checking_assert (!ctx
->receiver_decl
);
15529 ctx
->receiver_decl
= create_tmp_var
15530 (build_reference_type (ctx
->record_type
), ".omp_rec");
15532 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
15533 lower_omp (&par_body
, ctx
);
15534 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
15535 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
15537 /* Declare all the variables created by mapping and the variables
15538 declared in the scope of the parallel body. */
15539 record_vars_into (ctx
->block_vars
, child_fn
);
15540 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
15542 if (ctx
->record_type
)
15545 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
15546 : ctx
->record_type
, ".omp_data_o");
15547 DECL_NAMELESS (ctx
->sender_decl
) = 1;
15548 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
15549 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
15554 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
15555 lower_send_shared_vars (&ilist
, &olist
, ctx
);
15557 if (ctx
->record_type
)
15559 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
15560 TREE_THIS_VOLATILE (clobber
) = 1;
15561 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
15565 /* Once all the expansions are done, sequence all the different
15566 fragments inside gimple_omp_body. */
15570 if (ctx
->record_type
)
15572 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
15573 /* fixup_child_record_type might have changed receiver_decl's type. */
15574 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
15575 gimple_seq_add_stmt (&new_body
,
15576 gimple_build_assign (ctx
->receiver_decl
, t
));
15579 gimple_seq_add_seq (&new_body
, par_ilist
);
15580 gimple_seq_add_seq (&new_body
, par_body
);
15581 gimple_seq_add_seq (&new_body
, par_rlist
);
15582 if (ctx
->cancellable
)
15583 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
15584 gimple_seq_add_seq (&new_body
, par_olist
);
15585 new_body
= maybe_catch_exception (new_body
);
15586 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
)
15587 gimple_seq_add_stmt (&new_body
,
15588 gimple_build_omp_continue (integer_zero_node
,
15589 integer_zero_node
));
15590 if (!phony_construct
)
15592 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
15593 gimple_omp_set_body (stmt
, new_body
);
15596 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
15597 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
15598 gimple_bind_add_seq (bind
, ilist
);
15599 if (!phony_construct
)
15600 gimple_bind_add_stmt (bind
, stmt
);
15602 gimple_bind_add_seq (bind
, new_body
);
15603 gimple_bind_add_seq (bind
, olist
);
15605 pop_gimplify_context (NULL
);
15609 gimple_bind_add_seq (dep_bind
, dep_ilist
);
15610 gimple_bind_add_stmt (dep_bind
, bind
);
15611 gimple_bind_add_seq (dep_bind
, dep_olist
);
15612 pop_gimplify_context (dep_bind
);
15616 /* Lower the GIMPLE_OMP_TARGET in the current statement
15617 in GSI_P. CTX holds context information for the directive. */
15620 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
15623 tree child_fn
, t
, c
;
15624 gomp_target
*stmt
= as_a
<gomp_target
*> (gsi_stmt (*gsi_p
));
15625 gbind
*tgt_bind
, *bind
, *dep_bind
= NULL
;
15626 gimple_seq tgt_body
, olist
, ilist
, fplist
, new_body
;
15627 location_t loc
= gimple_location (stmt
);
15628 bool offloaded
, data_region
;
15629 unsigned int map_cnt
= 0;
15630 bool has_depend
= false;
15632 offloaded
= is_gimple_omp_offloaded (stmt
);
15633 switch (gimple_omp_target_kind (stmt
))
15635 case GF_OMP_TARGET_KIND_REGION
:
15636 case GF_OMP_TARGET_KIND_UPDATE
:
15637 case GF_OMP_TARGET_KIND_ENTER_DATA
:
15638 case GF_OMP_TARGET_KIND_EXIT_DATA
:
15639 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
15640 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
15641 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
15642 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
15643 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
15644 data_region
= false;
15646 case GF_OMP_TARGET_KIND_DATA
:
15647 case GF_OMP_TARGET_KIND_OACC_DATA
:
15648 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
15649 data_region
= true;
15652 gcc_unreachable ();
15655 clauses
= gimple_omp_target_clauses (stmt
);
15657 gimple_seq dep_ilist
= NULL
;
15658 gimple_seq dep_olist
= NULL
;
15659 if (find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
15661 push_gimplify_context ();
15662 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
15663 lower_depend_clauses (gimple_omp_target_clauses_ptr (stmt
),
15664 &dep_ilist
, &dep_olist
);
15672 tgt_bind
= gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt
));
15673 tgt_body
= gimple_bind_body (tgt_bind
);
15675 else if (data_region
)
15676 tgt_body
= gimple_omp_body (stmt
);
15677 child_fn
= ctx
->cb
.dst_fn
;
15679 push_gimplify_context ();
15682 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15683 switch (OMP_CLAUSE_CODE (c
))
15689 case OMP_CLAUSE_MAP
:
15691 /* First check what we're prepared to handle in the following. */
15692 switch (OMP_CLAUSE_MAP_KIND (c
))
15694 case GOMP_MAP_ALLOC
:
15696 case GOMP_MAP_FROM
:
15697 case GOMP_MAP_TOFROM
:
15698 case GOMP_MAP_POINTER
:
15699 case GOMP_MAP_TO_PSET
:
15700 case GOMP_MAP_FORCE_DEALLOC
:
15701 case GOMP_MAP_RELEASE
:
15702 case GOMP_MAP_ALWAYS_TO
:
15703 case GOMP_MAP_ALWAYS_FROM
:
15704 case GOMP_MAP_ALWAYS_TOFROM
:
15705 case GOMP_MAP_FIRSTPRIVATE_POINTER
:
15706 case GOMP_MAP_FIRSTPRIVATE_REFERENCE
:
15707 case GOMP_MAP_STRUCT
:
15708 case GOMP_MAP_ALWAYS_POINTER
:
15710 case GOMP_MAP_FORCE_ALLOC
:
15711 case GOMP_MAP_FORCE_TO
:
15712 case GOMP_MAP_FORCE_FROM
:
15713 case GOMP_MAP_FORCE_TOFROM
:
15714 case GOMP_MAP_FORCE_PRESENT
:
15715 case GOMP_MAP_FORCE_DEVICEPTR
:
15716 case GOMP_MAP_DEVICE_RESIDENT
:
15717 case GOMP_MAP_LINK
:
15718 gcc_assert (is_gimple_omp_oacc (stmt
));
15721 gcc_unreachable ();
15725 case OMP_CLAUSE_TO
:
15726 case OMP_CLAUSE_FROM
:
15728 var
= OMP_CLAUSE_DECL (c
);
15731 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
15732 || (!OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
15733 && (OMP_CLAUSE_MAP_KIND (c
)
15734 != GOMP_MAP_FIRSTPRIVATE_POINTER
)))
15739 if (DECL_SIZE (var
)
15740 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
15742 tree var2
= DECL_VALUE_EXPR (var
);
15743 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
15744 var2
= TREE_OPERAND (var2
, 0);
15745 gcc_assert (DECL_P (var2
));
15750 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15751 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
15752 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
15754 if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15756 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
))
15757 && varpool_node::get_create (var
)->offloadable
)
15760 tree type
= build_pointer_type (TREE_TYPE (var
));
15761 tree new_var
= lookup_decl (var
, ctx
);
15762 x
= create_tmp_var_raw (type
, get_name (new_var
));
15763 gimple_add_tmp_var (x
);
15764 x
= build_simple_mem_ref (x
);
15765 SET_DECL_VALUE_EXPR (new_var
, x
);
15766 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15771 if (!maybe_lookup_field (var
, ctx
))
15776 x
= build_receiver_ref (var
, true, ctx
);
15777 tree new_var
= lookup_decl (var
, ctx
);
15779 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15780 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
15781 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
15782 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15783 x
= build_simple_mem_ref (x
);
15784 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
15786 gcc_assert (is_gimple_omp_oacc (ctx
->stmt
));
15787 if (is_reference (new_var
))
15789 /* Create a local object to hold the instance
15791 tree type
= TREE_TYPE (TREE_TYPE (new_var
));
15792 const char *id
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
15793 tree inst
= create_tmp_var (type
, id
);
15794 gimplify_assign (inst
, fold_indirect_ref (x
), &fplist
);
15795 x
= build_fold_addr_expr (inst
);
15797 gimplify_assign (new_var
, x
, &fplist
);
15799 else if (DECL_P (new_var
))
15801 SET_DECL_VALUE_EXPR (new_var
, x
);
15802 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15805 gcc_unreachable ();
15810 case OMP_CLAUSE_FIRSTPRIVATE
:
15811 if (is_oacc_parallel (ctx
))
15812 goto oacc_firstprivate
;
15814 var
= OMP_CLAUSE_DECL (c
);
15815 if (!is_reference (var
)
15816 && !is_gimple_reg_type (TREE_TYPE (var
)))
15818 tree new_var
= lookup_decl (var
, ctx
);
15819 if (is_variable_sized (var
))
15821 tree pvar
= DECL_VALUE_EXPR (var
);
15822 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15823 pvar
= TREE_OPERAND (pvar
, 0);
15824 gcc_assert (DECL_P (pvar
));
15825 tree new_pvar
= lookup_decl (pvar
, ctx
);
15826 x
= build_fold_indirect_ref (new_pvar
);
15827 TREE_THIS_NOTRAP (x
) = 1;
15830 x
= build_receiver_ref (var
, true, ctx
);
15831 SET_DECL_VALUE_EXPR (new_var
, x
);
15832 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15836 case OMP_CLAUSE_PRIVATE
:
15837 if (is_gimple_omp_oacc (ctx
->stmt
))
15839 var
= OMP_CLAUSE_DECL (c
);
15840 if (is_variable_sized (var
))
15842 tree new_var
= lookup_decl (var
, ctx
);
15843 tree pvar
= DECL_VALUE_EXPR (var
);
15844 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15845 pvar
= TREE_OPERAND (pvar
, 0);
15846 gcc_assert (DECL_P (pvar
));
15847 tree new_pvar
= lookup_decl (pvar
, ctx
);
15848 x
= build_fold_indirect_ref (new_pvar
);
15849 TREE_THIS_NOTRAP (x
) = 1;
15850 SET_DECL_VALUE_EXPR (new_var
, x
);
15851 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15855 case OMP_CLAUSE_USE_DEVICE_PTR
:
15856 case OMP_CLAUSE_IS_DEVICE_PTR
:
15857 var
= OMP_CLAUSE_DECL (c
);
15859 if (is_variable_sized (var
))
15861 tree new_var
= lookup_decl (var
, ctx
);
15862 tree pvar
= DECL_VALUE_EXPR (var
);
15863 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15864 pvar
= TREE_OPERAND (pvar
, 0);
15865 gcc_assert (DECL_P (pvar
));
15866 tree new_pvar
= lookup_decl (pvar
, ctx
);
15867 x
= build_fold_indirect_ref (new_pvar
);
15868 TREE_THIS_NOTRAP (x
) = 1;
15869 SET_DECL_VALUE_EXPR (new_var
, x
);
15870 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15872 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15874 tree new_var
= lookup_decl (var
, ctx
);
15875 tree type
= build_pointer_type (TREE_TYPE (var
));
15876 x
= create_tmp_var_raw (type
, get_name (new_var
));
15877 gimple_add_tmp_var (x
);
15878 x
= build_simple_mem_ref (x
);
15879 SET_DECL_VALUE_EXPR (new_var
, x
);
15880 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15887 target_nesting_level
++;
15888 lower_omp (&tgt_body
, ctx
);
15889 target_nesting_level
--;
15891 else if (data_region
)
15892 lower_omp (&tgt_body
, ctx
);
15896 /* Declare all the variables created by mapping and the variables
15897 declared in the scope of the target body. */
15898 record_vars_into (ctx
->block_vars
, child_fn
);
15899 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
15904 if (ctx
->record_type
)
15907 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
15908 DECL_NAMELESS (ctx
->sender_decl
) = 1;
15909 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
15910 t
= make_tree_vec (3);
15911 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
15912 TREE_VEC_ELT (t
, 1)
15913 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
15914 ".omp_data_sizes");
15915 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
15916 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
15917 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
15918 tree tkind_type
= short_unsigned_type_node
;
15919 int talign_shift
= 8;
15920 TREE_VEC_ELT (t
, 2)
15921 = create_tmp_var (build_array_type_nelts (tkind_type
, map_cnt
),
15922 ".omp_data_kinds");
15923 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
15924 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
15925 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
15926 gimple_omp_target_set_data_arg (stmt
, t
);
15928 vec
<constructor_elt
, va_gc
> *vsize
;
15929 vec
<constructor_elt
, va_gc
> *vkind
;
15930 vec_alloc (vsize
, map_cnt
);
15931 vec_alloc (vkind
, map_cnt
);
15932 unsigned int map_idx
= 0;
15934 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15935 switch (OMP_CLAUSE_CODE (c
))
15937 tree ovar
, nc
, s
, purpose
, var
, x
, type
;
15938 unsigned int talign
;
15943 case OMP_CLAUSE_MAP
:
15944 case OMP_CLAUSE_TO
:
15945 case OMP_CLAUSE_FROM
:
15946 oacc_firstprivate_map
:
15948 ovar
= OMP_CLAUSE_DECL (c
);
15949 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15950 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
15951 || (OMP_CLAUSE_MAP_KIND (c
)
15952 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
15954 if (!DECL_P (ovar
))
15956 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15957 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
15959 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
15960 == get_base_address (ovar
));
15961 nc
= OMP_CLAUSE_CHAIN (c
);
15962 ovar
= OMP_CLAUSE_DECL (nc
);
15966 tree x
= build_sender_ref (ovar
, ctx
);
15968 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
15969 gimplify_assign (x
, v
, &ilist
);
15975 if (DECL_SIZE (ovar
)
15976 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
15978 tree ovar2
= DECL_VALUE_EXPR (ovar
);
15979 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
15980 ovar2
= TREE_OPERAND (ovar2
, 0);
15981 gcc_assert (DECL_P (ovar2
));
15984 if (!maybe_lookup_field (ovar
, ctx
))
15988 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
15989 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
15990 talign
= DECL_ALIGN_UNIT (ovar
);
15993 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
15994 x
= build_sender_ref (ovar
, ctx
);
15996 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15997 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
15998 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
15999 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
16001 gcc_assert (offloaded
);
16003 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)));
16004 mark_addressable (avar
);
16005 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
16006 talign
= DECL_ALIGN_UNIT (avar
);
16007 avar
= build_fold_addr_expr (avar
);
16008 gimplify_assign (x
, avar
, &ilist
);
16010 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
16012 gcc_assert (is_gimple_omp_oacc (ctx
->stmt
));
16013 if (!is_reference (var
))
16014 var
= build_fold_addr_expr (var
);
16016 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
16017 gimplify_assign (x
, var
, &ilist
);
16019 else if (is_gimple_reg (var
))
16021 gcc_assert (offloaded
);
16022 tree avar
= create_tmp_var (TREE_TYPE (var
));
16023 mark_addressable (avar
);
16024 enum gomp_map_kind map_kind
= OMP_CLAUSE_MAP_KIND (c
);
16025 if (GOMP_MAP_COPY_TO_P (map_kind
)
16026 || map_kind
== GOMP_MAP_POINTER
16027 || map_kind
== GOMP_MAP_TO_PSET
16028 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
16029 gimplify_assign (avar
, var
, &ilist
);
16030 avar
= build_fold_addr_expr (avar
);
16031 gimplify_assign (x
, avar
, &ilist
);
16032 if ((GOMP_MAP_COPY_FROM_P (map_kind
)
16033 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
16034 && !TYPE_READONLY (TREE_TYPE (var
)))
16036 x
= unshare_expr (x
);
16037 x
= build_simple_mem_ref (x
);
16038 gimplify_assign (var
, x
, &olist
);
16043 var
= build_fold_addr_expr (var
);
16044 gimplify_assign (x
, var
, &ilist
);
16048 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
16050 gcc_checking_assert (is_gimple_omp_oacc (ctx
->stmt
));
16051 s
= TREE_TYPE (ovar
);
16052 if (TREE_CODE (s
) == REFERENCE_TYPE
)
16054 s
= TYPE_SIZE_UNIT (s
);
16057 s
= OMP_CLAUSE_SIZE (c
);
16058 if (s
== NULL_TREE
)
16059 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
16060 s
= fold_convert (size_type_node
, s
);
16061 purpose
= size_int (map_idx
++);
16062 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
16063 if (TREE_CODE (s
) != INTEGER_CST
)
16064 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
16066 unsigned HOST_WIDE_INT tkind
, tkind_zero
;
16067 switch (OMP_CLAUSE_CODE (c
))
16069 case OMP_CLAUSE_MAP
:
16070 tkind
= OMP_CLAUSE_MAP_KIND (c
);
16071 tkind_zero
= tkind
;
16072 if (OMP_CLAUSE_MAP_MAYBE_ZERO_LENGTH_ARRAY_SECTION (c
))
16075 case GOMP_MAP_ALLOC
:
16077 case GOMP_MAP_FROM
:
16078 case GOMP_MAP_TOFROM
:
16079 case GOMP_MAP_ALWAYS_TO
:
16080 case GOMP_MAP_ALWAYS_FROM
:
16081 case GOMP_MAP_ALWAYS_TOFROM
:
16082 case GOMP_MAP_RELEASE
:
16083 tkind_zero
= GOMP_MAP_ZERO_LEN_ARRAY_SECTION
;
16085 case GOMP_MAP_DELETE
:
16086 tkind_zero
= GOMP_MAP_DELETE_ZERO_LEN_ARRAY_SECTION
;
16090 if (tkind_zero
!= tkind
)
16092 if (integer_zerop (s
))
16093 tkind
= tkind_zero
;
16094 else if (integer_nonzerop (s
))
16095 tkind_zero
= tkind
;
16098 case OMP_CLAUSE_FIRSTPRIVATE
:
16099 gcc_checking_assert (is_gimple_omp_oacc (ctx
->stmt
));
16100 tkind
= GOMP_MAP_TO
;
16101 tkind_zero
= tkind
;
16103 case OMP_CLAUSE_TO
:
16104 tkind
= GOMP_MAP_TO
;
16105 tkind_zero
= tkind
;
16107 case OMP_CLAUSE_FROM
:
16108 tkind
= GOMP_MAP_FROM
;
16109 tkind_zero
= tkind
;
16112 gcc_unreachable ();
16114 gcc_checking_assert (tkind
16115 < (HOST_WIDE_INT_C (1U) << talign_shift
));
16116 gcc_checking_assert (tkind_zero
16117 < (HOST_WIDE_INT_C (1U) << talign_shift
));
16118 talign
= ceil_log2 (talign
);
16119 tkind
|= talign
<< talign_shift
;
16120 tkind_zero
|= talign
<< talign_shift
;
16121 gcc_checking_assert (tkind
16122 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
16123 gcc_checking_assert (tkind_zero
16124 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
16125 if (tkind
== tkind_zero
)
16126 x
= build_int_cstu (tkind_type
, tkind
);
16129 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 0;
16130 x
= build3 (COND_EXPR
, tkind_type
,
16131 fold_build2 (EQ_EXPR
, boolean_type_node
,
16132 unshare_expr (s
), size_zero_node
),
16133 build_int_cstu (tkind_type
, tkind_zero
),
16134 build_int_cstu (tkind_type
, tkind
));
16136 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
, x
);
16141 case OMP_CLAUSE_FIRSTPRIVATE
:
16142 if (is_oacc_parallel (ctx
))
16143 goto oacc_firstprivate_map
;
16144 ovar
= OMP_CLAUSE_DECL (c
);
16145 if (is_reference (ovar
))
16146 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
16148 talign
= DECL_ALIGN_UNIT (ovar
);
16149 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
16150 x
= build_sender_ref (ovar
, ctx
);
16151 tkind
= GOMP_MAP_FIRSTPRIVATE
;
16152 type
= TREE_TYPE (ovar
);
16153 if (is_reference (ovar
))
16154 type
= TREE_TYPE (type
);
16155 bool use_firstprivate_int
, force_addr
;
16156 use_firstprivate_int
= false;
16157 force_addr
= false;
16158 if ((INTEGRAL_TYPE_P (type
)
16159 && TYPE_PRECISION (type
) <= POINTER_SIZE
)
16160 || TREE_CODE (type
) == POINTER_TYPE
)
16161 use_firstprivate_int
= true;
16164 if (is_reference (var
))
16165 use_firstprivate_int
= false;
16166 else if (is_gimple_reg (var
))
16168 if (DECL_HAS_VALUE_EXPR_P (var
))
16170 tree v
= get_base_address (var
);
16171 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
16173 use_firstprivate_int
= false;
16177 switch (TREE_CODE (v
))
16181 use_firstprivate_int
= false;
16190 use_firstprivate_int
= false;
16192 if (use_firstprivate_int
)
16194 tkind
= GOMP_MAP_FIRSTPRIVATE_INT
;
16196 if (is_reference (var
))
16197 t
= build_simple_mem_ref (var
);
16198 if (TREE_CODE (type
) != POINTER_TYPE
)
16199 t
= fold_convert (pointer_sized_int_node
, t
);
16200 t
= fold_convert (TREE_TYPE (x
), t
);
16201 gimplify_assign (x
, t
, &ilist
);
16203 else if (is_reference (var
))
16204 gimplify_assign (x
, var
, &ilist
);
16205 else if (!force_addr
&& is_gimple_reg (var
))
16207 tree avar
= create_tmp_var (TREE_TYPE (var
));
16208 mark_addressable (avar
);
16209 gimplify_assign (avar
, var
, &ilist
);
16210 avar
= build_fold_addr_expr (avar
);
16211 gimplify_assign (x
, avar
, &ilist
);
16215 var
= build_fold_addr_expr (var
);
16216 gimplify_assign (x
, var
, &ilist
);
16218 if (tkind
== GOMP_MAP_FIRSTPRIVATE_INT
)
16220 else if (is_reference (var
))
16221 s
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
16223 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
16224 s
= fold_convert (size_type_node
, s
);
16225 purpose
= size_int (map_idx
++);
16226 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
16227 if (TREE_CODE (s
) != INTEGER_CST
)
16228 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
16230 gcc_checking_assert (tkind
16231 < (HOST_WIDE_INT_C (1U) << talign_shift
));
16232 talign
= ceil_log2 (talign
);
16233 tkind
|= talign
<< talign_shift
;
16234 gcc_checking_assert (tkind
16235 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
16236 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
16237 build_int_cstu (tkind_type
, tkind
));
16240 case OMP_CLAUSE_USE_DEVICE_PTR
:
16241 case OMP_CLAUSE_IS_DEVICE_PTR
:
16242 ovar
= OMP_CLAUSE_DECL (c
);
16243 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
16244 x
= build_sender_ref (ovar
, ctx
);
16245 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_USE_DEVICE_PTR
)
16246 tkind
= GOMP_MAP_USE_DEVICE_PTR
;
16248 tkind
= GOMP_MAP_FIRSTPRIVATE_INT
;
16249 type
= TREE_TYPE (ovar
);
16250 if (TREE_CODE (type
) == ARRAY_TYPE
)
16251 var
= build_fold_addr_expr (var
);
16254 if (is_reference (ovar
))
16256 type
= TREE_TYPE (type
);
16257 if (TREE_CODE (type
) != ARRAY_TYPE
)
16258 var
= build_simple_mem_ref (var
);
16259 var
= fold_convert (TREE_TYPE (x
), var
);
16262 gimplify_assign (x
, var
, &ilist
);
16264 purpose
= size_int (map_idx
++);
16265 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
16266 gcc_checking_assert (tkind
16267 < (HOST_WIDE_INT_C (1U) << talign_shift
));
16268 gcc_checking_assert (tkind
16269 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
16270 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
16271 build_int_cstu (tkind_type
, tkind
));
16275 gcc_assert (map_idx
== map_cnt
);
16277 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
16278 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
16279 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
16280 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
16281 for (int i
= 1; i
<= 2; i
++)
16282 if (!TREE_STATIC (TREE_VEC_ELT (t
, i
)))
16284 gimple_seq initlist
= NULL
;
16285 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
16286 TREE_VEC_ELT (t
, i
)),
16287 &initlist
, true, NULL_TREE
);
16288 gimple_seq_add_seq (&ilist
, initlist
);
16290 tree clobber
= build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, i
)),
16292 TREE_THIS_VOLATILE (clobber
) = 1;
16293 gimple_seq_add_stmt (&olist
,
16294 gimple_build_assign (TREE_VEC_ELT (t
, i
),
16298 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
16299 TREE_THIS_VOLATILE (clobber
) = 1;
16300 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
16304 /* Once all the expansions are done, sequence all the different
16305 fragments inside gimple_omp_body. */
16310 && ctx
->record_type
)
16312 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
16313 /* fixup_child_record_type might have changed receiver_decl's type. */
16314 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
16315 gimple_seq_add_stmt (&new_body
,
16316 gimple_build_assign (ctx
->receiver_decl
, t
));
16318 gimple_seq_add_seq (&new_body
, fplist
);
16320 if (offloaded
|| data_region
)
16322 tree prev
= NULL_TREE
;
16323 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
16324 switch (OMP_CLAUSE_CODE (c
))
16329 case OMP_CLAUSE_FIRSTPRIVATE
:
16330 if (is_gimple_omp_oacc (ctx
->stmt
))
16332 var
= OMP_CLAUSE_DECL (c
);
16333 if (is_reference (var
)
16334 || is_gimple_reg_type (TREE_TYPE (var
)))
16336 tree new_var
= lookup_decl (var
, ctx
);
16338 type
= TREE_TYPE (var
);
16339 if (is_reference (var
))
16340 type
= TREE_TYPE (type
);
16341 bool use_firstprivate_int
;
16342 use_firstprivate_int
= false;
16343 if ((INTEGRAL_TYPE_P (type
)
16344 && TYPE_PRECISION (type
) <= POINTER_SIZE
)
16345 || TREE_CODE (type
) == POINTER_TYPE
)
16346 use_firstprivate_int
= true;
16349 tree v
= lookup_decl_in_outer_ctx (var
, ctx
);
16350 if (is_reference (v
))
16351 use_firstprivate_int
= false;
16352 else if (is_gimple_reg (v
))
16354 if (DECL_HAS_VALUE_EXPR_P (v
))
16356 v
= get_base_address (v
);
16357 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
16358 use_firstprivate_int
= false;
16360 switch (TREE_CODE (v
))
16364 use_firstprivate_int
= false;
16372 use_firstprivate_int
= false;
16374 if (use_firstprivate_int
)
16376 x
= build_receiver_ref (var
, false, ctx
);
16377 if (TREE_CODE (type
) != POINTER_TYPE
)
16378 x
= fold_convert (pointer_sized_int_node
, x
);
16379 x
= fold_convert (type
, x
);
16380 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
16382 if (is_reference (var
))
16384 tree v
= create_tmp_var_raw (type
, get_name (var
));
16385 gimple_add_tmp_var (v
);
16386 TREE_ADDRESSABLE (v
) = 1;
16387 gimple_seq_add_stmt (&new_body
,
16388 gimple_build_assign (v
, x
));
16389 x
= build_fold_addr_expr (v
);
16391 gimple_seq_add_stmt (&new_body
,
16392 gimple_build_assign (new_var
, x
));
16396 x
= build_receiver_ref (var
, !is_reference (var
), ctx
);
16397 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
16399 gimple_seq_add_stmt (&new_body
,
16400 gimple_build_assign (new_var
, x
));
16403 else if (is_variable_sized (var
))
16405 tree pvar
= DECL_VALUE_EXPR (var
);
16406 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
16407 pvar
= TREE_OPERAND (pvar
, 0);
16408 gcc_assert (DECL_P (pvar
));
16409 tree new_var
= lookup_decl (pvar
, ctx
);
16410 x
= build_receiver_ref (var
, false, ctx
);
16411 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16412 gimple_seq_add_stmt (&new_body
,
16413 gimple_build_assign (new_var
, x
));
16416 case OMP_CLAUSE_PRIVATE
:
16417 if (is_gimple_omp_oacc (ctx
->stmt
))
16419 var
= OMP_CLAUSE_DECL (c
);
16420 if (is_reference (var
))
16422 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
16423 tree new_var
= lookup_decl (var
, ctx
);
16424 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
16425 if (TREE_CONSTANT (x
))
16427 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
16429 gimple_add_tmp_var (x
);
16430 TREE_ADDRESSABLE (x
) = 1;
16431 x
= build_fold_addr_expr_loc (clause_loc
, x
);
16436 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
16437 tree rtype
= TREE_TYPE (TREE_TYPE (new_var
));
16438 tree al
= size_int (TYPE_ALIGN (rtype
));
16439 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
16442 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
16443 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16444 gimple_seq_add_stmt (&new_body
,
16445 gimple_build_assign (new_var
, x
));
16448 case OMP_CLAUSE_USE_DEVICE_PTR
:
16449 case OMP_CLAUSE_IS_DEVICE_PTR
:
16450 var
= OMP_CLAUSE_DECL (c
);
16451 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_USE_DEVICE_PTR
)
16452 x
= build_sender_ref (var
, ctx
);
16454 x
= build_receiver_ref (var
, false, ctx
);
16455 if (is_variable_sized (var
))
16457 tree pvar
= DECL_VALUE_EXPR (var
);
16458 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
16459 pvar
= TREE_OPERAND (pvar
, 0);
16460 gcc_assert (DECL_P (pvar
));
16461 tree new_var
= lookup_decl (pvar
, ctx
);
16462 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16463 gimple_seq_add_stmt (&new_body
,
16464 gimple_build_assign (new_var
, x
));
16466 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
16468 tree new_var
= lookup_decl (var
, ctx
);
16469 new_var
= DECL_VALUE_EXPR (new_var
);
16470 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
16471 new_var
= TREE_OPERAND (new_var
, 0);
16472 gcc_assert (DECL_P (new_var
));
16473 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16474 gimple_seq_add_stmt (&new_body
,
16475 gimple_build_assign (new_var
, x
));
16479 tree type
= TREE_TYPE (var
);
16480 tree new_var
= lookup_decl (var
, ctx
);
16481 if (is_reference (var
))
16483 type
= TREE_TYPE (type
);
16484 if (TREE_CODE (type
) != ARRAY_TYPE
)
16486 tree v
= create_tmp_var_raw (type
, get_name (var
));
16487 gimple_add_tmp_var (v
);
16488 TREE_ADDRESSABLE (v
) = 1;
16489 x
= fold_convert (type
, x
);
16490 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
16492 gimple_seq_add_stmt (&new_body
,
16493 gimple_build_assign (v
, x
));
16494 x
= build_fold_addr_expr (v
);
16497 x
= fold_convert (TREE_TYPE (new_var
), x
);
16498 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16499 gimple_seq_add_stmt (&new_body
,
16500 gimple_build_assign (new_var
, x
));
16504 /* Handle GOMP_MAP_FIRSTPRIVATE_{POINTER,REFERENCE} in second pass,
16505 so that firstprivate vars holding OMP_CLAUSE_SIZE if needed
16506 are already handled. */
16507 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
16508 switch (OMP_CLAUSE_CODE (c
))
16513 case OMP_CLAUSE_MAP
:
16514 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
16515 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)
16517 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
16518 HOST_WIDE_INT offset
= 0;
16520 var
= OMP_CLAUSE_DECL (c
);
16522 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
16523 && is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
16525 && varpool_node::get_create (var
)->offloadable
)
16527 if (TREE_CODE (var
) == INDIRECT_REF
16528 && TREE_CODE (TREE_OPERAND (var
, 0)) == COMPONENT_REF
)
16529 var
= TREE_OPERAND (var
, 0);
16530 if (TREE_CODE (var
) == COMPONENT_REF
)
16532 var
= get_addr_base_and_unit_offset (var
, &offset
);
16533 gcc_assert (var
!= NULL_TREE
&& DECL_P (var
));
16535 else if (DECL_SIZE (var
)
16536 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
16538 tree var2
= DECL_VALUE_EXPR (var
);
16539 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
16540 var2
= TREE_OPERAND (var2
, 0);
16541 gcc_assert (DECL_P (var2
));
16544 tree new_var
= lookup_decl (var
, ctx
), x
;
16545 tree type
= TREE_TYPE (new_var
);
16547 if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == INDIRECT_REF
16548 && (TREE_CODE (TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0))
16551 type
= TREE_TYPE (TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0));
16553 new_var
= build2 (MEM_REF
, type
,
16554 build_fold_addr_expr (new_var
),
16555 build_int_cst (build_pointer_type (type
),
16558 else if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == COMPONENT_REF
)
16560 type
= TREE_TYPE (OMP_CLAUSE_DECL (c
));
16561 is_ref
= TREE_CODE (type
) == REFERENCE_TYPE
;
16562 new_var
= build2 (MEM_REF
, type
,
16563 build_fold_addr_expr (new_var
),
16564 build_int_cst (build_pointer_type (type
),
16568 is_ref
= is_reference (var
);
16569 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)
16571 bool ref_to_array
= false;
16574 type
= TREE_TYPE (type
);
16575 if (TREE_CODE (type
) == ARRAY_TYPE
)
16577 type
= build_pointer_type (type
);
16578 ref_to_array
= true;
16581 else if (TREE_CODE (type
) == ARRAY_TYPE
)
16583 tree decl2
= DECL_VALUE_EXPR (new_var
);
16584 gcc_assert (TREE_CODE (decl2
) == MEM_REF
);
16585 decl2
= TREE_OPERAND (decl2
, 0);
16586 gcc_assert (DECL_P (decl2
));
16588 type
= TREE_TYPE (new_var
);
16590 x
= build_receiver_ref (OMP_CLAUSE_DECL (prev
), false, ctx
);
16591 x
= fold_convert_loc (clause_loc
, type
, x
);
16592 if (!integer_zerop (OMP_CLAUSE_SIZE (c
)))
16594 tree bias
= OMP_CLAUSE_SIZE (c
);
16596 bias
= lookup_decl (bias
, ctx
);
16597 bias
= fold_convert_loc (clause_loc
, sizetype
, bias
);
16598 bias
= fold_build1_loc (clause_loc
, NEGATE_EXPR
, sizetype
,
16600 x
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
16601 TREE_TYPE (x
), x
, bias
);
16604 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
16605 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16606 if (is_ref
&& !ref_to_array
)
16608 tree t
= create_tmp_var_raw (type
, get_name (var
));
16609 gimple_add_tmp_var (t
);
16610 TREE_ADDRESSABLE (t
) = 1;
16611 gimple_seq_add_stmt (&new_body
,
16612 gimple_build_assign (t
, x
));
16613 x
= build_fold_addr_expr_loc (clause_loc
, t
);
16615 gimple_seq_add_stmt (&new_body
,
16616 gimple_build_assign (new_var
, x
));
16619 else if (OMP_CLAUSE_CHAIN (c
)
16620 && OMP_CLAUSE_CODE (OMP_CLAUSE_CHAIN (c
))
16622 && (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c
))
16623 == GOMP_MAP_FIRSTPRIVATE_POINTER
16624 || (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c
))
16625 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
16628 case OMP_CLAUSE_PRIVATE
:
16629 var
= OMP_CLAUSE_DECL (c
);
16630 if (is_variable_sized (var
))
16632 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
16633 tree new_var
= lookup_decl (var
, ctx
);
16634 tree pvar
= DECL_VALUE_EXPR (var
);
16635 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
16636 pvar
= TREE_OPERAND (pvar
, 0);
16637 gcc_assert (DECL_P (pvar
));
16638 tree new_pvar
= lookup_decl (pvar
, ctx
);
16639 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
16640 tree al
= size_int (DECL_ALIGN (var
));
16641 tree x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
16642 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
16643 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_pvar
), x
);
16644 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
16645 gimple_seq_add_stmt (&new_body
,
16646 gimple_build_assign (new_pvar
, x
));
16651 gimple_seq fork_seq
= NULL
;
16652 gimple_seq join_seq
= NULL
;
16654 if (is_oacc_parallel (ctx
))
16656 /* If there are reductions on the offloaded region itself, treat
16657 them as a dummy GANG loop. */
16658 tree level
= build_int_cst (integer_type_node
, GOMP_DIM_GANG
);
16660 lower_oacc_reductions (gimple_location (ctx
->stmt
), clauses
, level
,
16661 false, NULL
, NULL
, &fork_seq
, &join_seq
, ctx
);
16664 gimple_seq_add_seq (&new_body
, fork_seq
);
16665 gimple_seq_add_seq (&new_body
, tgt_body
);
16666 gimple_seq_add_seq (&new_body
, join_seq
);
16669 new_body
= maybe_catch_exception (new_body
);
16671 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
16672 gimple_omp_set_body (stmt
, new_body
);
16675 bind
= gimple_build_bind (NULL
, NULL
,
16676 tgt_bind
? gimple_bind_block (tgt_bind
)
16678 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
16679 gimple_bind_add_seq (bind
, ilist
);
16680 gimple_bind_add_stmt (bind
, stmt
);
16681 gimple_bind_add_seq (bind
, olist
);
16683 pop_gimplify_context (NULL
);
16687 gimple_bind_add_seq (dep_bind
, dep_ilist
);
16688 gimple_bind_add_stmt (dep_bind
, bind
);
16689 gimple_bind_add_seq (dep_bind
, dep_olist
);
16690 pop_gimplify_context (dep_bind
);
16694 /* Expand code for an OpenMP teams directive. */
16697 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
16699 gomp_teams
*teams_stmt
= as_a
<gomp_teams
*> (gsi_stmt (*gsi_p
));
16700 push_gimplify_context ();
16702 tree block
= make_node (BLOCK
);
16703 gbind
*bind
= gimple_build_bind (NULL
, NULL
, block
);
16704 gsi_replace (gsi_p
, bind
, true);
16705 gimple_seq bind_body
= NULL
;
16706 gimple_seq dlist
= NULL
;
16707 gimple_seq olist
= NULL
;
16709 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
16710 OMP_CLAUSE_NUM_TEAMS
);
16711 if (num_teams
== NULL_TREE
)
16712 num_teams
= build_int_cst (unsigned_type_node
, 0);
16715 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
16716 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
16717 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
16719 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
16720 OMP_CLAUSE_THREAD_LIMIT
);
16721 if (thread_limit
== NULL_TREE
)
16722 thread_limit
= build_int_cst (unsigned_type_node
, 0);
16725 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
16726 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
16727 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
16731 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
16732 &bind_body
, &dlist
, ctx
, NULL
);
16733 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
16734 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
16735 if (!gimple_omp_teams_grid_phony (teams_stmt
))
16737 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
16738 location_t loc
= gimple_location (teams_stmt
);
16739 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
16740 gimple
*call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
16741 gimple_set_location (call
, loc
);
16742 gimple_seq_add_stmt (&bind_body
, call
);
16745 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
16746 gimple_omp_set_body (teams_stmt
, NULL
);
16747 gimple_seq_add_seq (&bind_body
, olist
);
16748 gimple_seq_add_seq (&bind_body
, dlist
);
16749 if (!gimple_omp_teams_grid_phony (teams_stmt
))
16750 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
16751 gimple_bind_set_body (bind
, bind_body
);
16753 pop_gimplify_context (bind
);
16755 gimple_bind_append_vars (bind
, ctx
->block_vars
);
16756 BLOCK_VARS (block
) = ctx
->block_vars
;
16757 if (BLOCK_VARS (block
))
16758 TREE_USED (block
) = 1;
16761 /* Expand code within an artificial GIMPLE_OMP_GRID_BODY OMP construct. */
16764 lower_omp_grid_body (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
16766 gimple
*stmt
= gsi_stmt (*gsi_p
);
16767 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
16768 gimple_seq_add_stmt (gimple_omp_body_ptr (stmt
),
16769 gimple_build_omp_return (false));
16773 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
16774 regimplified. If DATA is non-NULL, lower_omp_1 is outside
16775 of OMP context, but with task_shared_vars set. */
16778 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
16783 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
16784 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
16787 if (task_shared_vars
16789 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
16792 /* If a global variable has been privatized, TREE_CONSTANT on
16793 ADDR_EXPR might be wrong. */
16794 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
16795 recompute_tree_invariant_for_addr_expr (t
);
16797 *walk_subtrees
= !IS_TYPE_OR_DECL_P (t
);
16801 /* Data to be communicated between lower_omp_regimplify_operands and
16802 lower_omp_regimplify_operands_p. */
16804 struct lower_omp_regimplify_operands_data
16810 /* Helper function for lower_omp_regimplify_operands. Find
16811 omp_member_access_dummy_var vars and adjust temporarily their
16812 DECL_VALUE_EXPRs if needed. */
16815 lower_omp_regimplify_operands_p (tree
*tp
, int *walk_subtrees
,
16818 tree t
= omp_member_access_dummy_var (*tp
);
16821 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
16822 lower_omp_regimplify_operands_data
*ldata
16823 = (lower_omp_regimplify_operands_data
*) wi
->info
;
16824 tree o
= maybe_lookup_decl (t
, ldata
->ctx
);
16827 ldata
->decls
->safe_push (DECL_VALUE_EXPR (*tp
));
16828 ldata
->decls
->safe_push (*tp
);
16829 tree v
= unshare_and_remap (DECL_VALUE_EXPR (*tp
), t
, o
);
16830 SET_DECL_VALUE_EXPR (*tp
, v
);
16833 *walk_subtrees
= !IS_TYPE_OR_DECL_P (*tp
);
16837 /* Wrapper around gimple_regimplify_operands that adjusts DECL_VALUE_EXPRs
16838 of omp_member_access_dummy_var vars during regimplification. */
16841 lower_omp_regimplify_operands (omp_context
*ctx
, gimple
*stmt
,
16842 gimple_stmt_iterator
*gsi_p
)
16844 auto_vec
<tree
, 10> decls
;
16847 struct walk_stmt_info wi
;
16848 memset (&wi
, '\0', sizeof (wi
));
16849 struct lower_omp_regimplify_operands_data data
;
16851 data
.decls
= &decls
;
16853 walk_gimple_op (stmt
, lower_omp_regimplify_operands_p
, &wi
);
16855 gimple_regimplify_operands (stmt
, gsi_p
);
16856 while (!decls
.is_empty ())
16858 tree t
= decls
.pop ();
16859 tree v
= decls
.pop ();
16860 SET_DECL_VALUE_EXPR (t
, v
);
16865 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
16867 gimple
*stmt
= gsi_stmt (*gsi_p
);
16868 struct walk_stmt_info wi
;
16871 if (gimple_has_location (stmt
))
16872 input_location
= gimple_location (stmt
);
16874 if (task_shared_vars
)
16875 memset (&wi
, '\0', sizeof (wi
));
16877 /* If we have issued syntax errors, avoid doing any heavy lifting.
16878 Just replace the OMP directives with a NOP to avoid
16879 confusing RTL expansion. */
16880 if (seen_error () && is_gimple_omp (stmt
))
16882 gsi_replace (gsi_p
, gimple_build_nop (), true);
16886 switch (gimple_code (stmt
))
16890 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
16891 if ((ctx
|| task_shared_vars
)
16892 && (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
16893 lower_omp_regimplify_p
,
16894 ctx
? NULL
: &wi
, NULL
)
16895 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
16896 lower_omp_regimplify_p
,
16897 ctx
? NULL
: &wi
, NULL
)))
16898 lower_omp_regimplify_operands (ctx
, cond_stmt
, gsi_p
);
16902 lower_omp (gimple_catch_handler_ptr (as_a
<gcatch
*> (stmt
)), ctx
);
16904 case GIMPLE_EH_FILTER
:
16905 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
16908 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
16909 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
16911 case GIMPLE_TRANSACTION
:
16912 lower_omp (gimple_transaction_body_ptr (
16913 as_a
<gtransaction
*> (stmt
)),
16917 lower_omp (gimple_bind_body_ptr (as_a
<gbind
*> (stmt
)), ctx
);
16919 case GIMPLE_OMP_PARALLEL
:
16920 case GIMPLE_OMP_TASK
:
16921 ctx
= maybe_lookup_ctx (stmt
);
16923 if (ctx
->cancellable
)
16924 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
16925 lower_omp_taskreg (gsi_p
, ctx
);
16927 case GIMPLE_OMP_FOR
:
16928 ctx
= maybe_lookup_ctx (stmt
);
16930 if (ctx
->cancellable
)
16931 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
16932 lower_omp_for (gsi_p
, ctx
);
16934 case GIMPLE_OMP_SECTIONS
:
16935 ctx
= maybe_lookup_ctx (stmt
);
16937 if (ctx
->cancellable
)
16938 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
16939 lower_omp_sections (gsi_p
, ctx
);
16941 case GIMPLE_OMP_SINGLE
:
16942 ctx
= maybe_lookup_ctx (stmt
);
16944 lower_omp_single (gsi_p
, ctx
);
16946 case GIMPLE_OMP_MASTER
:
16947 ctx
= maybe_lookup_ctx (stmt
);
16949 lower_omp_master (gsi_p
, ctx
);
16951 case GIMPLE_OMP_TASKGROUP
:
16952 ctx
= maybe_lookup_ctx (stmt
);
16954 lower_omp_taskgroup (gsi_p
, ctx
);
16956 case GIMPLE_OMP_ORDERED
:
16957 ctx
= maybe_lookup_ctx (stmt
);
16959 lower_omp_ordered (gsi_p
, ctx
);
16961 case GIMPLE_OMP_CRITICAL
:
16962 ctx
= maybe_lookup_ctx (stmt
);
16964 lower_omp_critical (gsi_p
, ctx
);
16966 case GIMPLE_OMP_ATOMIC_LOAD
:
16967 if ((ctx
|| task_shared_vars
)
16968 && walk_tree (gimple_omp_atomic_load_rhs_ptr (
16969 as_a
<gomp_atomic_load
*> (stmt
)),
16970 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
16971 lower_omp_regimplify_operands (ctx
, stmt
, gsi_p
);
16973 case GIMPLE_OMP_TARGET
:
16974 ctx
= maybe_lookup_ctx (stmt
);
16976 lower_omp_target (gsi_p
, ctx
);
16978 case GIMPLE_OMP_TEAMS
:
16979 ctx
= maybe_lookup_ctx (stmt
);
16981 lower_omp_teams (gsi_p
, ctx
);
16983 case GIMPLE_OMP_GRID_BODY
:
16984 ctx
= maybe_lookup_ctx (stmt
);
16986 lower_omp_grid_body (gsi_p
, ctx
);
16990 call_stmt
= as_a
<gcall
*> (stmt
);
16991 fndecl
= gimple_call_fndecl (call_stmt
);
16993 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
16994 switch (DECL_FUNCTION_CODE (fndecl
))
16996 case BUILT_IN_GOMP_BARRIER
:
17000 case BUILT_IN_GOMP_CANCEL
:
17001 case BUILT_IN_GOMP_CANCELLATION_POINT
:
17004 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
17005 cctx
= cctx
->outer
;
17006 gcc_assert (gimple_call_lhs (call_stmt
) == NULL_TREE
);
17007 if (!cctx
->cancellable
)
17009 if (DECL_FUNCTION_CODE (fndecl
)
17010 == BUILT_IN_GOMP_CANCELLATION_POINT
)
17012 stmt
= gimple_build_nop ();
17013 gsi_replace (gsi_p
, stmt
, false);
17017 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
17019 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
17020 gimple_call_set_fndecl (call_stmt
, fndecl
);
17021 gimple_call_set_fntype (call_stmt
, TREE_TYPE (fndecl
));
17024 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)));
17025 gimple_call_set_lhs (call_stmt
, lhs
);
17026 tree fallthru_label
;
17027 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
17029 g
= gimple_build_label (fallthru_label
);
17030 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
17031 g
= gimple_build_cond (NE_EXPR
, lhs
,
17032 fold_convert (TREE_TYPE (lhs
),
17033 boolean_false_node
),
17034 cctx
->cancel_label
, fallthru_label
);
17035 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
17042 if ((ctx
|| task_shared_vars
)
17043 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
17046 /* Just remove clobbers, this should happen only if we have
17047 "privatized" local addressable variables in SIMD regions,
17048 the clobber isn't needed in that case and gimplifying address
17049 of the ARRAY_REF into a pointer and creating MEM_REF based
17050 clobber would create worse code than we get with the clobber
17052 if (gimple_clobber_p (stmt
))
17054 gsi_replace (gsi_p
, gimple_build_nop (), true);
17057 lower_omp_regimplify_operands (ctx
, stmt
, gsi_p
);
17064 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
17066 location_t saved_location
= input_location
;
17067 gimple_stmt_iterator gsi
;
17068 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17069 lower_omp_1 (&gsi
, ctx
);
17070 /* During gimplification, we haven't folded statments inside offloading
17071 or taskreg regions (gimplify.c:maybe_fold_stmt); do that now. */
17072 if (target_nesting_level
|| taskreg_nesting_level
)
17073 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17075 input_location
= saved_location
;
17078 /* Returen true if STMT is an assignment of a register-type into a local
17082 grid_reg_assignment_to_local_var_p (gimple
*stmt
)
17084 gassign
*assign
= dyn_cast
<gassign
*> (stmt
);
17087 tree lhs
= gimple_assign_lhs (assign
);
17088 if (TREE_CODE (lhs
) != VAR_DECL
17089 || !is_gimple_reg_type (TREE_TYPE (lhs
))
17090 || is_global_var (lhs
))
17095 /* Return true if all statements in SEQ are assignments to local register-type
17099 grid_seq_only_contains_local_assignments (gimple_seq seq
)
17104 gimple_stmt_iterator gsi
;
17105 for (gsi
= gsi_start (seq
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17106 if (!grid_reg_assignment_to_local_var_p (gsi_stmt (gsi
)))
17111 /* Scan statements in SEQ and call itself recursively on any bind. If during
17112 whole search only assignments to register-type local variables and one
17113 single OMP statement is encountered, return true, otherwise return false.
17114 RET is where we store any OMP statement encountered. TARGET_LOC and NAME
17115 are used for dumping a note about a failure. */
17118 grid_find_single_omp_among_assignments_1 (gimple_seq seq
, location_t target_loc
,
17119 const char *name
, gimple
**ret
)
17121 gimple_stmt_iterator gsi
;
17122 for (gsi
= gsi_start (seq
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17124 gimple
*stmt
= gsi_stmt (gsi
);
17126 if (grid_reg_assignment_to_local_var_p (stmt
))
17128 if (gbind
*bind
= dyn_cast
<gbind
*> (stmt
))
17130 if (!grid_find_single_omp_among_assignments_1 (gimple_bind_body (bind
),
17131 target_loc
, name
, ret
))
17134 else if (is_gimple_omp (stmt
))
17138 if (dump_enabled_p ())
17139 dump_printf_loc (MSG_NOTE
, target_loc
,
17140 "Will not turn target construct into a simple "
17141 "GPGPU kernel because %s construct contains "
17142 "multiple OpenMP constructs\n", name
);
17149 if (dump_enabled_p ())
17150 dump_printf_loc (MSG_NOTE
, target_loc
,
17151 "Will not turn target construct into a simple "
17152 "GPGPU kernel because %s construct contains "
17153 "a complex statement\n", name
);
17160 /* Scan statements in SEQ and make sure that it and any binds in it contain
17161 only assignments to local register-type variables and one OMP construct. If
17162 so, return that construct, otherwise return NULL. If dumping is enabled and
17163 function fails, use TARGET_LOC and NAME to dump a note with the reason for
17167 grid_find_single_omp_among_assignments (gimple_seq seq
, location_t target_loc
,
17172 if (dump_enabled_p ())
17173 dump_printf_loc (MSG_NOTE
, target_loc
,
17174 "Will not turn target construct into a simple "
17175 "GPGPU kernel because %s construct has empty "
17181 gimple
*ret
= NULL
;
17182 if (grid_find_single_omp_among_assignments_1 (seq
, target_loc
, name
, &ret
))
17184 if (!ret
&& dump_enabled_p ())
17185 dump_printf_loc (MSG_NOTE
, target_loc
,
17186 "Will not turn target construct into a simple "
17187 "GPGPU kernel because %s construct does not contain"
17188 "any other OpenMP construct\n", name
);
17195 /* Walker function looking for statements there is no point gridifying (and for
17196 noreturn function calls which we cannot do). Return non-NULL if such a
17197 function is found. */
17200 grid_find_ungridifiable_statement (gimple_stmt_iterator
*gsi
,
17201 bool *handled_ops_p
,
17202 struct walk_stmt_info
*)
17204 *handled_ops_p
= false;
17205 gimple
*stmt
= gsi_stmt (*gsi
);
17206 switch (gimple_code (stmt
))
17209 if (gimple_call_noreturn_p (as_a
<gcall
*> (stmt
)))
17211 *handled_ops_p
= true;
17212 return error_mark_node
;
17216 /* We may reduce the following list if we find a way to implement the
17217 clauses, but now there is no point trying further. */
17218 case GIMPLE_OMP_CRITICAL
:
17219 case GIMPLE_OMP_TASKGROUP
:
17220 case GIMPLE_OMP_TASK
:
17221 case GIMPLE_OMP_SECTION
:
17222 case GIMPLE_OMP_SECTIONS
:
17223 case GIMPLE_OMP_SECTIONS_SWITCH
:
17224 case GIMPLE_OMP_TARGET
:
17225 case GIMPLE_OMP_ORDERED
:
17226 *handled_ops_p
= true;
17227 return error_mark_node
;
17236 /* If TARGET follows a pattern that can be turned into a gridified GPGPU
17237 kernel, return true, otherwise return false. In the case of success, also
17238 fill in GROUP_SIZE_P with the requested group size or NULL if there is
17242 grid_target_follows_gridifiable_pattern (gomp_target
*target
, tree
*group_size_p
)
17244 if (gimple_omp_target_kind (target
) != GF_OMP_TARGET_KIND_REGION
)
17247 location_t tloc
= gimple_location (target
);
17249 = grid_find_single_omp_among_assignments (gimple_omp_body (target
),
17253 gomp_teams
*teams
= dyn_cast
<gomp_teams
*> (stmt
);
17254 tree group_size
= NULL
;
17257 dump_printf_loc (MSG_NOTE
, tloc
,
17258 "Will not turn target construct into a simple "
17259 "GPGPU kernel because it does not have a sole teams "
17260 "construct in it.\n");
17264 tree clauses
= gimple_omp_teams_clauses (teams
);
17267 switch (OMP_CLAUSE_CODE (clauses
))
17269 case OMP_CLAUSE_NUM_TEAMS
:
17270 if (dump_enabled_p ())
17271 dump_printf_loc (MSG_NOTE
, tloc
,
17272 "Will not turn target construct into a "
17273 "gridified GPGPU kernel because we cannot "
17274 "handle num_teams clause of teams "
17278 case OMP_CLAUSE_REDUCTION
:
17279 if (dump_enabled_p ())
17280 dump_printf_loc (MSG_NOTE
, tloc
,
17281 "Will not turn target construct into a "
17282 "gridified GPGPU kernel because a reduction "
17283 "clause is present\n ");
17286 case OMP_CLAUSE_LASTPRIVATE
:
17287 if (dump_enabled_p ())
17288 dump_printf_loc (MSG_NOTE
, tloc
,
17289 "Will not turn target construct into a "
17290 "gridified GPGPU kernel because a lastprivate "
17291 "clause is present\n ");
17294 case OMP_CLAUSE_THREAD_LIMIT
:
17295 group_size
= OMP_CLAUSE_OPERAND (clauses
, 0);
17301 clauses
= OMP_CLAUSE_CHAIN (clauses
);
17304 stmt
= grid_find_single_omp_among_assignments (gimple_omp_body (teams
), tloc
,
17308 gomp_for
*dist
= dyn_cast
<gomp_for
*> (stmt
);
17311 dump_printf_loc (MSG_NOTE
, tloc
,
17312 "Will not turn target construct into a simple "
17313 "GPGPU kernel because the teams construct does not have "
17314 "a sole distribute construct in it.\n");
17318 gcc_assert (gimple_omp_for_kind (dist
) == GF_OMP_FOR_KIND_DISTRIBUTE
);
17319 if (!gimple_omp_for_combined_p (dist
))
17321 if (dump_enabled_p ())
17322 dump_printf_loc (MSG_NOTE
, tloc
,
17323 "Will not turn target construct into a gridified GPGPU "
17324 "kernel because we cannot handle a standalone "
17325 "distribute construct\n ");
17328 if (dist
->collapse
> 1)
17330 if (dump_enabled_p ())
17331 dump_printf_loc (MSG_NOTE
, tloc
,
17332 "Will not turn target construct into a gridified GPGPU "
17333 "kernel because the distribute construct contains "
17334 "collapse clause\n");
17337 struct omp_for_data fd
;
17338 extract_omp_for_data (dist
, &fd
, NULL
);
17341 if (group_size
&& !operand_equal_p (group_size
, fd
.chunk_size
, 0))
17343 if (dump_enabled_p ())
17344 dump_printf_loc (MSG_NOTE
, tloc
,
17345 "Will not turn target construct into a "
17346 "gridified GPGPU kernel because the teams "
17347 "thread limit is different from distribute "
17348 "schedule chunk\n");
17351 group_size
= fd
.chunk_size
;
17353 stmt
= grid_find_single_omp_among_assignments (gimple_omp_body (dist
), tloc
,
17355 gomp_parallel
*par
;
17356 if (!stmt
|| !(par
= dyn_cast
<gomp_parallel
*> (stmt
)))
17359 clauses
= gimple_omp_parallel_clauses (par
);
17362 switch (OMP_CLAUSE_CODE (clauses
))
17364 case OMP_CLAUSE_NUM_THREADS
:
17365 if (dump_enabled_p ())
17366 dump_printf_loc (MSG_NOTE
, tloc
,
17367 "Will not turn target construct into a gridified"
17368 "GPGPU kernel because there is a num_threads "
17369 "clause of the parallel construct\n");
17372 case OMP_CLAUSE_REDUCTION
:
17373 if (dump_enabled_p ())
17374 dump_printf_loc (MSG_NOTE
, tloc
,
17375 "Will not turn target construct into a "
17376 "gridified GPGPU kernel because a reduction "
17377 "clause is present\n ");
17380 case OMP_CLAUSE_LASTPRIVATE
:
17381 if (dump_enabled_p ())
17382 dump_printf_loc (MSG_NOTE
, tloc
,
17383 "Will not turn target construct into a "
17384 "gridified GPGPU kernel because a lastprivate "
17385 "clause is present\n ");
17391 clauses
= OMP_CLAUSE_CHAIN (clauses
);
17394 stmt
= grid_find_single_omp_among_assignments (gimple_omp_body (par
), tloc
,
17397 if (!stmt
|| !(gfor
= dyn_cast
<gomp_for
*> (stmt
)))
17400 if (gimple_omp_for_kind (gfor
) != GF_OMP_FOR_KIND_FOR
)
17402 if (dump_enabled_p ())
17403 dump_printf_loc (MSG_NOTE
, tloc
,
17404 "Will not turn target construct into a gridified GPGPU "
17405 "kernel because the inner loop is not a simple for "
17409 if (gfor
->collapse
> 1)
17411 if (dump_enabled_p ())
17412 dump_printf_loc (MSG_NOTE
, tloc
,
17413 "Will not turn target construct into a gridified GPGPU "
17414 "kernel because the inner loop contains collapse "
17419 if (!grid_seq_only_contains_local_assignments (gimple_omp_for_pre_body (gfor
)))
17421 if (dump_enabled_p ())
17422 dump_printf_loc (MSG_NOTE
, tloc
,
17423 "Will not turn target construct into a gridified GPGPU "
17424 "kernel because the inner loop pre_body contains"
17425 "a complex instruction\n");
17429 clauses
= gimple_omp_for_clauses (gfor
);
17432 switch (OMP_CLAUSE_CODE (clauses
))
17434 case OMP_CLAUSE_SCHEDULE
:
17435 if (OMP_CLAUSE_SCHEDULE_KIND (clauses
) != OMP_CLAUSE_SCHEDULE_AUTO
)
17437 if (dump_enabled_p ())
17438 dump_printf_loc (MSG_NOTE
, tloc
,
17439 "Will not turn target construct into a "
17440 "gridified GPGPU kernel because the inner "
17441 "loop has a non-automatic scheduling clause\n");
17446 case OMP_CLAUSE_REDUCTION
:
17447 if (dump_enabled_p ())
17448 dump_printf_loc (MSG_NOTE
, tloc
,
17449 "Will not turn target construct into a "
17450 "gridified GPGPU kernel because a reduction "
17451 "clause is present\n ");
17454 case OMP_CLAUSE_LASTPRIVATE
:
17455 if (dump_enabled_p ())
17456 dump_printf_loc (MSG_NOTE
, tloc
,
17457 "Will not turn target construct into a "
17458 "gridified GPGPU kernel because a lastprivate "
17459 "clause is present\n ");
17465 clauses
= OMP_CLAUSE_CHAIN (clauses
);
17468 struct walk_stmt_info wi
;
17469 memset (&wi
, 0, sizeof (wi
));
17470 if (gimple
*bad
= walk_gimple_seq (gimple_omp_body (gfor
),
17471 grid_find_ungridifiable_statement
,
17474 if (dump_enabled_p ())
17476 if (is_gimple_call (bad
))
17477 dump_printf_loc (MSG_NOTE
, tloc
,
17478 "Will not turn target construct into a gridified "
17479 " GPGPU kernel because the inner loop contains "
17480 "call to a noreturn function\n");
17482 dump_printf_loc (MSG_NOTE
, tloc
,
17483 "Will not turn target construct into a gridified "
17484 "GPGPU kernel because the inner loop contains "
17485 "statement %s which cannot be transformed\n",
17486 gimple_code_name
[(int) gimple_code (bad
)]);
17491 *group_size_p
= group_size
;
17495 /* Operand walker, used to remap pre-body declarations according to a hash map
17496 provided in DATA. */
17499 grid_remap_prebody_decls (tree
*tp
, int *walk_subtrees
, void *data
)
17503 if (DECL_P (t
) || TYPE_P (t
))
17504 *walk_subtrees
= 0;
17506 *walk_subtrees
= 1;
17508 if (TREE_CODE (t
) == VAR_DECL
)
17510 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
17511 hash_map
<tree
, tree
> *declmap
= (hash_map
<tree
, tree
> *) wi
->info
;
17512 tree
*repl
= declmap
->get (t
);
17519 /* Copy leading register-type assignments to local variables in SRC to just
17520 before DST, Creating temporaries, adjusting mapping of operands in WI and
17521 remapping operands as necessary. Add any new temporaries to TGT_BIND.
17522 Return the first statement that does not conform to
17523 grid_reg_assignment_to_local_var_p or NULL. */
17526 grid_copy_leading_local_assignments (gimple_seq src
, gimple_stmt_iterator
*dst
,
17527 gbind
*tgt_bind
, struct walk_stmt_info
*wi
)
17529 hash_map
<tree
, tree
> *declmap
= (hash_map
<tree
, tree
> *) wi
->info
;
17530 gimple_stmt_iterator gsi
;
17531 for (gsi
= gsi_start (src
); !gsi_end_p (gsi
); gsi_next (&gsi
))
17533 gimple
*stmt
= gsi_stmt (gsi
);
17534 if (gbind
*bind
= dyn_cast
<gbind
*> (stmt
))
17536 gimple
*r
= grid_copy_leading_local_assignments
17537 (gimple_bind_body (bind
), dst
, tgt_bind
, wi
);
17543 if (!grid_reg_assignment_to_local_var_p (stmt
))
17545 tree lhs
= gimple_assign_lhs (as_a
<gassign
*> (stmt
));
17546 tree repl
= copy_var_decl (lhs
, create_tmp_var_name (NULL
),
17548 DECL_CONTEXT (repl
) = current_function_decl
;
17549 gimple_bind_append_vars (tgt_bind
, repl
);
17551 declmap
->put (lhs
, repl
);
17552 gassign
*copy
= as_a
<gassign
*> (gimple_copy (stmt
));
17553 walk_gimple_op (copy
, grid_remap_prebody_decls
, wi
);
17554 gsi_insert_before (dst
, copy
, GSI_SAME_STMT
);
17559 /* Given freshly copied top level kernel SEQ, identify the individual OMP
17560 components, mark them as part of kernel and return the inner loop, and copy
17561 assignment leading to them just before DST, remapping them using WI and
17562 adding new temporaries to TGT_BIND. */
17565 grid_process_kernel_body_copy (gimple_seq seq
, gimple_stmt_iterator
*dst
,
17566 gbind
*tgt_bind
, struct walk_stmt_info
*wi
)
17568 gimple
*stmt
= grid_copy_leading_local_assignments (seq
, dst
, tgt_bind
, wi
);
17569 gomp_teams
*teams
= dyn_cast
<gomp_teams
*> (stmt
);
17570 gcc_assert (teams
);
17571 gimple_omp_teams_set_grid_phony (teams
, true);
17572 stmt
= grid_copy_leading_local_assignments (gimple_omp_body (teams
), dst
,
17574 gcc_checking_assert (stmt
);
17575 gomp_for
*dist
= dyn_cast
<gomp_for
*> (stmt
);
17577 gimple_seq prebody
= gimple_omp_for_pre_body (dist
);
17579 grid_copy_leading_local_assignments (prebody
, dst
, tgt_bind
, wi
);
17580 gimple_omp_for_set_grid_phony (dist
, true);
17581 stmt
= grid_copy_leading_local_assignments (gimple_omp_body (dist
), dst
,
17583 gcc_checking_assert (stmt
);
17585 gomp_parallel
*parallel
= as_a
<gomp_parallel
*> (stmt
);
17586 gimple_omp_parallel_set_grid_phony (parallel
, true);
17587 stmt
= grid_copy_leading_local_assignments (gimple_omp_body (parallel
), dst
,
17589 gomp_for
*inner_loop
= as_a
<gomp_for
*> (stmt
);
17590 gimple_omp_for_set_kind (inner_loop
, GF_OMP_FOR_KIND_GRID_LOOP
);
17591 prebody
= gimple_omp_for_pre_body (inner_loop
);
17593 grid_copy_leading_local_assignments (prebody
, dst
, tgt_bind
, wi
);
17598 /* If TARGET points to a GOMP_TARGET which follows a gridifiable pattern,
17599 create a GPU kernel for it. GSI must point to the same statement, TGT_BIND
17600 is the bind into which temporaries inserted before TARGET should be
17604 grid_attempt_target_gridification (gomp_target
*target
,
17605 gimple_stmt_iterator
*gsi
,
17609 if (!target
|| !grid_target_follows_gridifiable_pattern (target
, &group_size
))
17612 location_t loc
= gimple_location (target
);
17613 if (dump_enabled_p ())
17614 dump_printf_loc (MSG_OPTIMIZED_LOCATIONS
, loc
,
17615 "Target construct will be turned into a gridified GPGPU "
17618 /* Copy target body to a GPUKERNEL construct: */
17619 gimple_seq kernel_seq
= copy_gimple_seq_and_replace_locals
17620 (gimple_omp_body (target
));
17622 hash_map
<tree
, tree
> *declmap
= new hash_map
<tree
, tree
>;
17623 struct walk_stmt_info wi
;
17624 memset (&wi
, 0, sizeof (struct walk_stmt_info
));
17627 /* Copy assignments in between OMP statements before target, mark OMP
17628 statements within copy appropriatly. */
17629 gomp_for
*inner_loop
= grid_process_kernel_body_copy (kernel_seq
, gsi
,
17632 gbind
*old_bind
= as_a
<gbind
*> (gimple_seq_first (gimple_omp_body (target
)));
17633 gbind
*new_bind
= as_a
<gbind
*> (gimple_seq_first (kernel_seq
));
17634 tree new_block
= gimple_bind_block (new_bind
);
17635 tree enc_block
= BLOCK_SUPERCONTEXT (gimple_bind_block (old_bind
));
17636 BLOCK_CHAIN (new_block
) = BLOCK_SUBBLOCKS (enc_block
);
17637 BLOCK_SUBBLOCKS (enc_block
) = new_block
;
17638 BLOCK_SUPERCONTEXT (new_block
) = enc_block
;
17639 gimple
*gpukernel
= gimple_build_omp_grid_body (kernel_seq
);
17640 gimple_seq_add_stmt
17641 (gimple_bind_body_ptr (as_a
<gbind
*> (gimple_omp_body (target
))),
17644 walk_tree (&group_size
, grid_remap_prebody_decls
, &wi
, NULL
);
17645 push_gimplify_context ();
17646 size_t collapse
= gimple_omp_for_collapse (inner_loop
);
17647 for (size_t i
= 0; i
< collapse
; i
++)
17649 tree itype
, type
= TREE_TYPE (gimple_omp_for_index (inner_loop
, i
));
17650 if (POINTER_TYPE_P (type
))
17651 itype
= signed_type_for (type
);
17655 enum tree_code cond_code
= gimple_omp_for_cond (inner_loop
, i
);
17656 tree n1
= unshare_expr (gimple_omp_for_initial (inner_loop
, i
));
17657 walk_tree (&n1
, grid_remap_prebody_decls
, &wi
, NULL
);
17658 tree n2
= unshare_expr (gimple_omp_for_final (inner_loop
, i
));
17659 walk_tree (&n2
, grid_remap_prebody_decls
, &wi
, NULL
);
17660 adjust_for_condition (loc
, &cond_code
, &n2
);
17662 step
= get_omp_for_step_from_incr (loc
,
17663 gimple_omp_for_incr (inner_loop
, i
));
17664 gimple_seq tmpseq
= NULL
;
17665 n1
= fold_convert (itype
, n1
);
17666 n2
= fold_convert (itype
, n2
);
17667 tree t
= build_int_cst (itype
, (cond_code
== LT_EXPR
? -1 : 1));
17668 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
17669 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
17670 t
= fold_build2 (MINUS_EXPR
, itype
, t
, n1
);
17671 if (TYPE_UNSIGNED (itype
) && cond_code
== GT_EXPR
)
17672 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
17673 fold_build1 (NEGATE_EXPR
, itype
, t
),
17674 fold_build1 (NEGATE_EXPR
, itype
, step
));
17676 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
17677 tree gs
= fold_convert (uint32_type_node
, t
);
17678 gimplify_expr (&gs
, &tmpseq
, NULL
, is_gimple_val
, fb_rvalue
);
17679 if (!gimple_seq_empty_p (tmpseq
))
17680 gsi_insert_seq_before (gsi
, tmpseq
, GSI_SAME_STMT
);
17683 if (i
== 0 && group_size
)
17685 ws
= fold_convert (uint32_type_node
, group_size
);
17687 gimplify_expr (&ws
, &tmpseq
, NULL
, is_gimple_val
, fb_rvalue
);
17688 if (!gimple_seq_empty_p (tmpseq
))
17689 gsi_insert_seq_before (gsi
, tmpseq
, GSI_SAME_STMT
);
17692 ws
= build_zero_cst (uint32_type_node
);
17694 tree c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__GRIDDIM_
);
17695 OMP_CLAUSE__GRIDDIM__DIMENSION (c
) = i
;
17696 OMP_CLAUSE__GRIDDIM__SIZE (c
) = gs
;
17697 OMP_CLAUSE__GRIDDIM__GROUP (c
) = ws
;
17698 OMP_CLAUSE_CHAIN (c
) = gimple_omp_target_clauses (target
);
17699 gimple_omp_target_set_clauses (target
, c
);
17701 pop_gimplify_context (tgt_bind
);
17706 /* Walker function doing all the work for create_target_kernels. */
17709 grid_gridify_all_targets_stmt (gimple_stmt_iterator
*gsi
,
17710 bool *handled_ops_p
,
17711 struct walk_stmt_info
*incoming
)
17713 *handled_ops_p
= false;
17715 gimple
*stmt
= gsi_stmt (*gsi
);
17716 gomp_target
*target
= dyn_cast
<gomp_target
*> (stmt
);
17719 gbind
*tgt_bind
= (gbind
*) incoming
->info
;
17720 gcc_checking_assert (tgt_bind
);
17721 grid_attempt_target_gridification (target
, gsi
, tgt_bind
);
17724 gbind
*bind
= dyn_cast
<gbind
*> (stmt
);
17727 *handled_ops_p
= true;
17728 struct walk_stmt_info wi
;
17729 memset (&wi
, 0, sizeof (wi
));
17731 walk_gimple_seq_mod (gimple_bind_body_ptr (bind
),
17732 grid_gridify_all_targets_stmt
, NULL
, &wi
);
17737 /* Attempt to gridify all target constructs in BODY_P. All such targets will
17738 have their bodies duplicated, with the new copy being put into a
17739 gimple_omp_grid_body statement. All kernel-related construct within the
17740 grid_body will be marked with phony flags or kernel kinds. Moreover, some
17741 re-structuring is often needed, such as copying pre-bodies before the target
17742 construct so that kernel grid sizes can be computed. */
17745 grid_gridify_all_targets (gimple_seq
*body_p
)
17747 struct walk_stmt_info wi
;
17748 memset (&wi
, 0, sizeof (wi
));
17749 walk_gimple_seq_mod (body_p
, grid_gridify_all_targets_stmt
, NULL
, &wi
);
17753 /* Main entry point. */
17755 static unsigned int
17756 execute_lower_omp (void)
17762 /* This pass always runs, to provide PROP_gimple_lomp.
17763 But often, there is nothing to do. */
17764 if (flag_cilkplus
== 0 && flag_openacc
== 0 && flag_openmp
== 0
17765 && flag_openmp_simd
== 0)
17768 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
17769 delete_omp_context
);
17771 body
= gimple_body (current_function_decl
);
17773 if (hsa_gen_requested_p ())
17774 grid_gridify_all_targets (&body
);
17776 scan_omp (&body
, NULL
);
17777 gcc_assert (taskreg_nesting_level
== 0);
17778 FOR_EACH_VEC_ELT (taskreg_contexts
, i
, ctx
)
17779 finish_taskreg_scan (ctx
);
17780 taskreg_contexts
.release ();
17782 if (all_contexts
->root
)
17784 if (task_shared_vars
)
17785 push_gimplify_context ();
17786 lower_omp (&body
, NULL
);
17787 if (task_shared_vars
)
17788 pop_gimplify_context (NULL
);
17793 splay_tree_delete (all_contexts
);
17794 all_contexts
= NULL
;
17796 BITMAP_FREE (task_shared_vars
);
17802 const pass_data pass_data_lower_omp
=
17804 GIMPLE_PASS
, /* type */
17805 "omplower", /* name */
17806 OPTGROUP_NONE
, /* optinfo_flags */
17807 TV_NONE
, /* tv_id */
17808 PROP_gimple_any
, /* properties_required */
17809 PROP_gimple_lomp
, /* properties_provided */
17810 0, /* properties_destroyed */
17811 0, /* todo_flags_start */
17812 0, /* todo_flags_finish */
17815 class pass_lower_omp
: public gimple_opt_pass
17818 pass_lower_omp (gcc::context
*ctxt
)
17819 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
17822 /* opt_pass methods: */
17823 virtual unsigned int execute (function
*) { return execute_lower_omp (); }
17825 }; // class pass_lower_omp
17827 } // anon namespace
17830 make_pass_lower_omp (gcc::context
*ctxt
)
17832 return new pass_lower_omp (ctxt
);
17835 /* The following is a utility to diagnose structured block violations.
17836 It is not part of the "omplower" pass, as that's invoked too late. It
17837 should be invoked by the respective front ends after gimplification. */
17839 static splay_tree all_labels
;
17841 /* Check for mismatched contexts and generate an error if needed. Return
17842 true if an error is detected. */
17845 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
17846 gimple
*branch_ctx
, gimple
*label_ctx
)
17848 gcc_checking_assert (!branch_ctx
|| is_gimple_omp (branch_ctx
));
17849 gcc_checking_assert (!label_ctx
|| is_gimple_omp (label_ctx
));
17851 if (label_ctx
== branch_ctx
)
17854 const char* kind
= NULL
;
17859 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
17860 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
17862 && gimple_code (label_ctx
) == GIMPLE_OMP_FOR
17863 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
17864 kind
= "Cilk Plus";
17868 if ((branch_ctx
&& is_gimple_omp_oacc (branch_ctx
))
17869 || (label_ctx
&& is_gimple_omp_oacc (label_ctx
)))
17871 gcc_checking_assert (kind
== NULL
);
17877 gcc_checking_assert (flag_openmp
);
17882 Previously we kept track of the label's entire context in diagnose_sb_[12]
17883 so we could traverse it and issue a correct "exit" or "enter" error
17884 message upon a structured block violation.
17886 We built the context by building a list with tree_cons'ing, but there is
17887 no easy counterpart in gimple tuples. It seems like far too much work
17888 for issuing exit/enter error messages. If someone really misses the
17889 distinct error message... patches welcome.
17893 /* Try to avoid confusing the user by producing and error message
17894 with correct "exit" or "enter" verbiage. We prefer "exit"
17895 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
17896 if (branch_ctx
== NULL
)
17902 if (TREE_VALUE (label_ctx
) == branch_ctx
)
17907 label_ctx
= TREE_CHAIN (label_ctx
);
17912 error ("invalid exit from %s structured block", kind
);
17914 error ("invalid entry to %s structured block", kind
);
17917 /* If it's obvious we have an invalid entry, be specific about the error. */
17918 if (branch_ctx
== NULL
)
17919 error ("invalid entry to %s structured block", kind
);
17922 /* Otherwise, be vague and lazy, but efficient. */
17923 error ("invalid branch to/from %s structured block", kind
);
17926 gsi_replace (gsi_p
, gimple_build_nop (), false);
17930 /* Pass 1: Create a minimal tree of structured blocks, and record
17931 where each label is found. */
17934 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
17935 struct walk_stmt_info
*wi
)
17937 gimple
*context
= (gimple
*) wi
->info
;
17938 gimple
*inner_context
;
17939 gimple
*stmt
= gsi_stmt (*gsi_p
);
17941 *handled_ops_p
= true;
17943 switch (gimple_code (stmt
))
17947 case GIMPLE_OMP_PARALLEL
:
17948 case GIMPLE_OMP_TASK
:
17949 case GIMPLE_OMP_SECTIONS
:
17950 case GIMPLE_OMP_SINGLE
:
17951 case GIMPLE_OMP_SECTION
:
17952 case GIMPLE_OMP_MASTER
:
17953 case GIMPLE_OMP_ORDERED
:
17954 case GIMPLE_OMP_CRITICAL
:
17955 case GIMPLE_OMP_TARGET
:
17956 case GIMPLE_OMP_TEAMS
:
17957 case GIMPLE_OMP_TASKGROUP
:
17958 /* The minimal context here is just the current OMP construct. */
17959 inner_context
= stmt
;
17960 wi
->info
= inner_context
;
17961 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
17962 wi
->info
= context
;
17965 case GIMPLE_OMP_FOR
:
17966 inner_context
= stmt
;
17967 wi
->info
= inner_context
;
17968 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
17970 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
17971 diagnose_sb_1
, NULL
, wi
);
17972 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
17973 wi
->info
= context
;
17977 splay_tree_insert (all_labels
,
17978 (splay_tree_key
) gimple_label_label (
17979 as_a
<glabel
*> (stmt
)),
17980 (splay_tree_value
) context
);
17990 /* Pass 2: Check each branch and see if its context differs from that of
17991 the destination label's context. */
17994 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
17995 struct walk_stmt_info
*wi
)
17997 gimple
*context
= (gimple
*) wi
->info
;
17999 gimple
*stmt
= gsi_stmt (*gsi_p
);
18001 *handled_ops_p
= true;
18003 switch (gimple_code (stmt
))
18007 case GIMPLE_OMP_PARALLEL
:
18008 case GIMPLE_OMP_TASK
:
18009 case GIMPLE_OMP_SECTIONS
:
18010 case GIMPLE_OMP_SINGLE
:
18011 case GIMPLE_OMP_SECTION
:
18012 case GIMPLE_OMP_MASTER
:
18013 case GIMPLE_OMP_ORDERED
:
18014 case GIMPLE_OMP_CRITICAL
:
18015 case GIMPLE_OMP_TARGET
:
18016 case GIMPLE_OMP_TEAMS
:
18017 case GIMPLE_OMP_TASKGROUP
:
18019 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
18020 wi
->info
= context
;
18023 case GIMPLE_OMP_FOR
:
18025 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
18027 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
18028 diagnose_sb_2
, NULL
, wi
);
18029 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
18030 wi
->info
= context
;
18035 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
18036 tree lab
= gimple_cond_true_label (cond_stmt
);
18039 n
= splay_tree_lookup (all_labels
,
18040 (splay_tree_key
) lab
);
18041 diagnose_sb_0 (gsi_p
, context
,
18042 n
? (gimple
*) n
->value
: NULL
);
18044 lab
= gimple_cond_false_label (cond_stmt
);
18047 n
= splay_tree_lookup (all_labels
,
18048 (splay_tree_key
) lab
);
18049 diagnose_sb_0 (gsi_p
, context
,
18050 n
? (gimple
*) n
->value
: NULL
);
18057 tree lab
= gimple_goto_dest (stmt
);
18058 if (TREE_CODE (lab
) != LABEL_DECL
)
18061 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
18062 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
*) n
->value
: NULL
);
18066 case GIMPLE_SWITCH
:
18068 gswitch
*switch_stmt
= as_a
<gswitch
*> (stmt
);
18070 for (i
= 0; i
< gimple_switch_num_labels (switch_stmt
); ++i
)
18072 tree lab
= CASE_LABEL (gimple_switch_label (switch_stmt
, i
));
18073 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
18074 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
*) n
->value
))
18080 case GIMPLE_RETURN
:
18081 diagnose_sb_0 (gsi_p
, context
, NULL
);
18091 /* Called from tree-cfg.c::make_edges to create cfg edges for all relevant
18094 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
,
18097 gimple
*last
= last_stmt (bb
);
18098 enum gimple_code code
= gimple_code (last
);
18099 struct omp_region
*cur_region
= *region
;
18100 bool fallthru
= false;
18104 case GIMPLE_OMP_PARALLEL
:
18105 case GIMPLE_OMP_TASK
:
18106 case GIMPLE_OMP_FOR
:
18107 case GIMPLE_OMP_SINGLE
:
18108 case GIMPLE_OMP_TEAMS
:
18109 case GIMPLE_OMP_MASTER
:
18110 case GIMPLE_OMP_TASKGROUP
:
18111 case GIMPLE_OMP_CRITICAL
:
18112 case GIMPLE_OMP_SECTION
:
18113 case GIMPLE_OMP_GRID_BODY
:
18114 cur_region
= new_omp_region (bb
, code
, cur_region
);
18118 case GIMPLE_OMP_ORDERED
:
18119 cur_region
= new_omp_region (bb
, code
, cur_region
);
18121 if (find_omp_clause (gimple_omp_ordered_clauses
18122 (as_a
<gomp_ordered
*> (last
)),
18123 OMP_CLAUSE_DEPEND
))
18124 cur_region
= cur_region
->outer
;
18127 case GIMPLE_OMP_TARGET
:
18128 cur_region
= new_omp_region (bb
, code
, cur_region
);
18130 switch (gimple_omp_target_kind (last
))
18132 case GF_OMP_TARGET_KIND_REGION
:
18133 case GF_OMP_TARGET_KIND_DATA
:
18134 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
18135 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
18136 case GF_OMP_TARGET_KIND_OACC_DATA
:
18137 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
18139 case GF_OMP_TARGET_KIND_UPDATE
:
18140 case GF_OMP_TARGET_KIND_ENTER_DATA
:
18141 case GF_OMP_TARGET_KIND_EXIT_DATA
:
18142 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
18143 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
18144 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
18145 cur_region
= cur_region
->outer
;
18148 gcc_unreachable ();
18152 case GIMPLE_OMP_SECTIONS
:
18153 cur_region
= new_omp_region (bb
, code
, cur_region
);
18157 case GIMPLE_OMP_SECTIONS_SWITCH
:
18161 case GIMPLE_OMP_ATOMIC_LOAD
:
18162 case GIMPLE_OMP_ATOMIC_STORE
:
18166 case GIMPLE_OMP_RETURN
:
18167 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
18168 somewhere other than the next block. This will be
18170 cur_region
->exit
= bb
;
18171 if (cur_region
->type
== GIMPLE_OMP_TASK
)
18172 /* Add an edge corresponding to not scheduling the task
18174 make_edge (cur_region
->entry
, bb
, EDGE_ABNORMAL
);
18175 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
18176 cur_region
= cur_region
->outer
;
18179 case GIMPLE_OMP_CONTINUE
:
18180 cur_region
->cont
= bb
;
18181 switch (cur_region
->type
)
18183 case GIMPLE_OMP_FOR
:
18184 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
18185 succs edges as abnormal to prevent splitting
18187 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
18188 /* Make the loopback edge. */
18189 make_edge (bb
, single_succ (cur_region
->entry
),
18192 /* Create an edge from GIMPLE_OMP_FOR to exit, which
18193 corresponds to the case that the body of the loop
18194 is not executed at all. */
18195 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
18196 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
18200 case GIMPLE_OMP_SECTIONS
:
18201 /* Wire up the edges into and out of the nested sections. */
18203 basic_block switch_bb
= single_succ (cur_region
->entry
);
18205 struct omp_region
*i
;
18206 for (i
= cur_region
->inner
; i
; i
= i
->next
)
18208 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
18209 make_edge (switch_bb
, i
->entry
, 0);
18210 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
18213 /* Make the loopback edge to the block with
18214 GIMPLE_OMP_SECTIONS_SWITCH. */
18215 make_edge (bb
, switch_bb
, 0);
18217 /* Make the edge from the switch to exit. */
18218 make_edge (switch_bb
, bb
->next_bb
, 0);
18223 case GIMPLE_OMP_TASK
:
18228 gcc_unreachable ();
18233 gcc_unreachable ();
18236 if (*region
!= cur_region
)
18238 *region
= cur_region
;
18240 *region_idx
= cur_region
->entry
->index
;
18248 static unsigned int
18249 diagnose_omp_structured_block_errors (void)
18251 struct walk_stmt_info wi
;
18252 gimple_seq body
= gimple_body (current_function_decl
);
18254 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
18256 memset (&wi
, 0, sizeof (wi
));
18257 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
18259 memset (&wi
, 0, sizeof (wi
));
18260 wi
.want_locations
= true;
18261 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
18263 gimple_set_body (current_function_decl
, body
);
18265 splay_tree_delete (all_labels
);
18273 const pass_data pass_data_diagnose_omp_blocks
=
18275 GIMPLE_PASS
, /* type */
18276 "*diagnose_omp_blocks", /* name */
18277 OPTGROUP_NONE
, /* optinfo_flags */
18278 TV_NONE
, /* tv_id */
18279 PROP_gimple_any
, /* properties_required */
18280 0, /* properties_provided */
18281 0, /* properties_destroyed */
18282 0, /* todo_flags_start */
18283 0, /* todo_flags_finish */
18286 class pass_diagnose_omp_blocks
: public gimple_opt_pass
18289 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
18290 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
18293 /* opt_pass methods: */
18294 virtual bool gate (function
*)
18296 return flag_cilkplus
|| flag_openacc
|| flag_openmp
;
18298 virtual unsigned int execute (function
*)
18300 return diagnose_omp_structured_block_errors ();
18303 }; // class pass_diagnose_omp_blocks
18305 } // anon namespace
18308 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
18310 return new pass_diagnose_omp_blocks (ctxt
);
18313 /* SIMD clone supporting code. */
18315 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
18316 of arguments to reserve space for. */
18318 static struct cgraph_simd_clone
*
18319 simd_clone_struct_alloc (int nargs
)
18321 struct cgraph_simd_clone
*clone_info
;
18322 size_t len
= (sizeof (struct cgraph_simd_clone
)
18323 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
18324 clone_info
= (struct cgraph_simd_clone
*)
18325 ggc_internal_cleared_alloc (len
);
18329 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
18332 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
18333 struct cgraph_simd_clone
*from
)
18335 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
18336 + ((from
->nargs
- from
->inbranch
)
18337 * sizeof (struct cgraph_simd_clone_arg
))));
18340 /* Return vector of parameter types of function FNDECL. This uses
18341 TYPE_ARG_TYPES if available, otherwise falls back to types of
18342 DECL_ARGUMENTS types. */
18345 simd_clone_vector_of_formal_parm_types (tree fndecl
)
18347 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
18348 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
18349 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
18352 FOR_EACH_VEC_ELT (args
, i
, arg
)
18353 args
[i
] = TREE_TYPE (args
[i
]);
18357 /* Given a simd function in NODE, extract the simd specific
18358 information from the OMP clauses passed in CLAUSES, and return
18359 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
18360 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
18361 otherwise set to FALSE. */
18363 static struct cgraph_simd_clone
*
18364 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
18365 bool *inbranch_specified
)
18367 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
18370 *inbranch_specified
= false;
18372 n
= args
.length ();
18373 if (n
> 0 && args
.last () == void_type_node
)
18376 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
18377 be cloned have a distinctive artificial label in addition to "omp
18381 && lookup_attribute ("cilk simd function",
18382 DECL_ATTRIBUTES (node
->decl
)));
18384 /* Allocate one more than needed just in case this is an in-branch
18385 clone which will require a mask argument. */
18386 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
18387 clone_info
->nargs
= n
;
18388 clone_info
->cilk_elemental
= cilk_clone
;
18395 clauses
= TREE_VALUE (clauses
);
18396 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
18399 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
18401 switch (OMP_CLAUSE_CODE (t
))
18403 case OMP_CLAUSE_INBRANCH
:
18404 clone_info
->inbranch
= 1;
18405 *inbranch_specified
= true;
18407 case OMP_CLAUSE_NOTINBRANCH
:
18408 clone_info
->inbranch
= 0;
18409 *inbranch_specified
= true;
18411 case OMP_CLAUSE_SIMDLEN
:
18412 clone_info
->simdlen
18413 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
18415 case OMP_CLAUSE_LINEAR
:
18417 tree decl
= OMP_CLAUSE_DECL (t
);
18418 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
18419 int argno
= TREE_INT_CST_LOW (decl
);
18420 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
18422 enum cgraph_simd_clone_arg_type arg_type
;
18423 if (TREE_CODE (args
[argno
]) == REFERENCE_TYPE
)
18424 switch (OMP_CLAUSE_LINEAR_KIND (t
))
18426 case OMP_CLAUSE_LINEAR_REF
:
18428 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
;
18430 case OMP_CLAUSE_LINEAR_UVAL
:
18432 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
;
18434 case OMP_CLAUSE_LINEAR_VAL
:
18435 case OMP_CLAUSE_LINEAR_DEFAULT
:
18437 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
;
18440 gcc_unreachable ();
18443 arg_type
= SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
18444 clone_info
->args
[argno
].arg_type
= arg_type
;
18445 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
18446 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
18447 && clone_info
->args
[argno
].linear_step
< n
);
18451 if (POINTER_TYPE_P (args
[argno
]))
18452 step
= fold_convert (ssizetype
, step
);
18453 if (!tree_fits_shwi_p (step
))
18455 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
18456 "ignoring large linear step");
18460 else if (integer_zerop (step
))
18462 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
18463 "ignoring zero linear step");
18469 enum cgraph_simd_clone_arg_type arg_type
;
18470 if (TREE_CODE (args
[argno
]) == REFERENCE_TYPE
)
18471 switch (OMP_CLAUSE_LINEAR_KIND (t
))
18473 case OMP_CLAUSE_LINEAR_REF
:
18475 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
;
18477 case OMP_CLAUSE_LINEAR_UVAL
:
18479 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
;
18481 case OMP_CLAUSE_LINEAR_VAL
:
18482 case OMP_CLAUSE_LINEAR_DEFAULT
:
18484 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
;
18487 gcc_unreachable ();
18490 arg_type
= SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
18491 clone_info
->args
[argno
].arg_type
= arg_type
;
18492 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
18497 case OMP_CLAUSE_UNIFORM
:
18499 tree decl
= OMP_CLAUSE_DECL (t
);
18500 int argno
= tree_to_uhwi (decl
);
18501 clone_info
->args
[argno
].arg_type
18502 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
18505 case OMP_CLAUSE_ALIGNED
:
18507 tree decl
= OMP_CLAUSE_DECL (t
);
18508 int argno
= tree_to_uhwi (decl
);
18509 clone_info
->args
[argno
].alignment
18510 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
18521 /* Given a SIMD clone in NODE, calculate the characteristic data
18522 type and return the coresponding type. The characteristic data
18523 type is computed as described in the Intel Vector ABI. */
18526 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
18527 struct cgraph_simd_clone
*clone_info
)
18529 tree type
= integer_type_node
;
18530 tree fndecl
= node
->decl
;
18532 /* a) For non-void function, the characteristic data type is the
18534 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
18535 type
= TREE_TYPE (TREE_TYPE (fndecl
));
18537 /* b) If the function has any non-uniform, non-linear parameters,
18538 then the characteristic data type is the type of the first
18542 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
18543 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
18544 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
18552 /* c) If the characteristic data type determined by a) or b) above
18553 is struct, union, or class type which is pass-by-value (except
18554 for the type that maps to the built-in complex data type), the
18555 characteristic data type is int. */
18556 if (RECORD_OR_UNION_TYPE_P (type
)
18557 && !aggregate_value_p (type
, NULL
)
18558 && TREE_CODE (type
) != COMPLEX_TYPE
)
18559 return integer_type_node
;
18561 /* d) If none of the above three classes is applicable, the
18562 characteristic data type is int. */
18566 /* e) For Intel Xeon Phi native and offload compilation, if the
18567 resulting characteristic data type is 8-bit or 16-bit integer
18568 data type, the characteristic data type is int. */
18569 /* Well, we don't handle Xeon Phi yet. */
18573 simd_clone_mangle (struct cgraph_node
*node
,
18574 struct cgraph_simd_clone
*clone_info
)
18576 char vecsize_mangle
= clone_info
->vecsize_mangle
;
18577 char mask
= clone_info
->inbranch
? 'M' : 'N';
18578 unsigned int simdlen
= clone_info
->simdlen
;
18582 gcc_assert (vecsize_mangle
&& simdlen
);
18584 pp_string (&pp
, "_ZGV");
18585 pp_character (&pp
, vecsize_mangle
);
18586 pp_character (&pp
, mask
);
18587 pp_decimal_int (&pp
, simdlen
);
18589 for (n
= 0; n
< clone_info
->nargs
; ++n
)
18591 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
18593 switch (arg
.arg_type
)
18595 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
18596 pp_character (&pp
, 'u');
18598 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
18599 pp_character (&pp
, 'l');
18600 goto mangle_linear
;
18601 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
18602 pp_character (&pp
, 'R');
18603 goto mangle_linear
;
18604 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
18605 pp_character (&pp
, 'L');
18606 goto mangle_linear
;
18607 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
18608 pp_character (&pp
, 'U');
18609 goto mangle_linear
;
18611 gcc_assert (arg
.linear_step
!= 0);
18612 if (arg
.linear_step
> 1)
18613 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
18614 else if (arg
.linear_step
< 0)
18616 pp_character (&pp
, 'n');
18617 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
18621 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
18622 pp_string (&pp
, "ls");
18623 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
18625 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
18626 pp_string (&pp
, "Rs");
18627 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
18629 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
18630 pp_string (&pp
, "Ls");
18631 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
18633 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
18634 pp_string (&pp
, "Us");
18635 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
18638 pp_character (&pp
, 'v');
18642 pp_character (&pp
, 'a');
18643 pp_decimal_int (&pp
, arg
.alignment
);
18647 pp_underscore (&pp
);
18648 const char *str
= IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
));
18651 pp_string (&pp
, str
);
18652 str
= pp_formatted_text (&pp
);
18654 /* If there already is a SIMD clone with the same mangled name, don't
18655 add another one. This can happen e.g. for
18656 #pragma omp declare simd
18657 #pragma omp declare simd simdlen(8)
18658 int foo (int, int);
18659 if the simdlen is assumed to be 8 for the first one, etc. */
18660 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
18661 clone
= clone
->simdclone
->next_clone
)
18662 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone
->decl
)),
18666 return get_identifier (str
);
18669 /* Create a simd clone of OLD_NODE and return it. */
18671 static struct cgraph_node
*
18672 simd_clone_create (struct cgraph_node
*old_node
)
18674 struct cgraph_node
*new_node
;
18675 if (old_node
->definition
)
18677 if (!old_node
->has_gimple_body_p ())
18679 old_node
->get_body ();
18680 new_node
= old_node
->create_version_clone_with_body (vNULL
, NULL
, NULL
,
18686 tree old_decl
= old_node
->decl
;
18687 tree new_decl
= copy_node (old_node
->decl
);
18688 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
18689 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
18690 SET_DECL_RTL (new_decl
, NULL
);
18691 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
18692 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
18693 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
18694 if (old_node
->in_other_partition
)
18695 new_node
->in_other_partition
= 1;
18696 symtab
->call_cgraph_insertion_hooks (new_node
);
18698 if (new_node
== NULL
)
18701 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
18703 /* The function cgraph_function_versioning () will force the new
18704 symbol local. Undo this, and inherit external visability from
18706 new_node
->local
.local
= old_node
->local
.local
;
18707 new_node
->externally_visible
= old_node
->externally_visible
;
18712 /* Adjust the return type of the given function to its appropriate
18713 vector counterpart. Returns a simd array to be used throughout the
18714 function as a return value. */
18717 simd_clone_adjust_return_type (struct cgraph_node
*node
)
18719 tree fndecl
= node
->decl
;
18720 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
18721 unsigned int veclen
;
18724 /* Adjust the function return type. */
18725 if (orig_rettype
== void_type_node
)
18727 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
18728 t
= TREE_TYPE (TREE_TYPE (fndecl
));
18729 if (INTEGRAL_TYPE_P (t
) || POINTER_TYPE_P (t
))
18730 veclen
= node
->simdclone
->vecsize_int
;
18732 veclen
= node
->simdclone
->vecsize_float
;
18733 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (t
));
18734 if (veclen
> node
->simdclone
->simdlen
)
18735 veclen
= node
->simdclone
->simdlen
;
18736 if (POINTER_TYPE_P (t
))
18737 t
= pointer_sized_int_node
;
18738 if (veclen
== node
->simdclone
->simdlen
)
18739 t
= build_vector_type (t
, node
->simdclone
->simdlen
);
18742 t
= build_vector_type (t
, veclen
);
18743 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
18745 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
18746 if (!node
->definition
)
18749 t
= DECL_RESULT (fndecl
);
18750 /* Adjust the DECL_RESULT. */
18751 gcc_assert (TREE_TYPE (t
) != void_type_node
);
18752 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
18755 tree atype
= build_array_type_nelts (orig_rettype
,
18756 node
->simdclone
->simdlen
);
18757 if (veclen
!= node
->simdclone
->simdlen
)
18758 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
18760 /* Set up a SIMD array to use as the return value. */
18761 tree retval
= create_tmp_var_raw (atype
, "retval");
18762 gimple_add_tmp_var (retval
);
18766 /* Each vector argument has a corresponding array to be used locally
18767 as part of the eventual loop. Create such temporary array and
18770 PREFIX is the prefix to be used for the temporary.
18772 TYPE is the inner element type.
18774 SIMDLEN is the number of elements. */
18777 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
18779 tree atype
= build_array_type_nelts (type
, simdlen
);
18780 tree avar
= create_tmp_var_raw (atype
, prefix
);
18781 gimple_add_tmp_var (avar
);
18785 /* Modify the function argument types to their corresponding vector
18786 counterparts if appropriate. Also, create one array for each simd
18787 argument to be used locally when using the function arguments as
18790 NODE is the function whose arguments are to be adjusted.
18792 Returns an adjustment vector that will be filled describing how the
18793 argument types will be adjusted. */
18795 static ipa_parm_adjustment_vec
18796 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
18799 ipa_parm_adjustment_vec adjustments
;
18801 if (node
->definition
)
18802 args
= ipa_get_vector_of_formal_parms (node
->decl
);
18804 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
18805 adjustments
.create (args
.length ());
18806 unsigned i
, j
, veclen
;
18807 struct ipa_parm_adjustment adj
;
18808 for (i
= 0; i
< node
->simdclone
->nargs
; ++i
)
18810 memset (&adj
, 0, sizeof (adj
));
18811 tree parm
= args
[i
];
18812 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
18813 adj
.base_index
= i
;
18816 node
->simdclone
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
18817 node
->simdclone
->args
[i
].orig_type
= parm_type
;
18819 switch (node
->simdclone
->args
[i
].arg_type
)
18822 /* No adjustment necessary for scalar arguments. */
18823 adj
.op
= IPA_PARM_OP_COPY
;
18825 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
18826 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
18827 if (node
->definition
)
18828 node
->simdclone
->args
[i
].simd_array
18829 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
18830 TREE_TYPE (parm_type
),
18831 node
->simdclone
->simdlen
);
18832 adj
.op
= IPA_PARM_OP_COPY
;
18834 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
18835 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
18836 case SIMD_CLONE_ARG_TYPE_VECTOR
:
18837 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
18838 veclen
= node
->simdclone
->vecsize_int
;
18840 veclen
= node
->simdclone
->vecsize_float
;
18841 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
18842 if (veclen
> node
->simdclone
->simdlen
)
18843 veclen
= node
->simdclone
->simdlen
;
18844 adj
.arg_prefix
= "simd";
18845 if (POINTER_TYPE_P (parm_type
))
18846 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
18848 adj
.type
= build_vector_type (parm_type
, veclen
);
18849 node
->simdclone
->args
[i
].vector_type
= adj
.type
;
18850 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
18852 adjustments
.safe_push (adj
);
18855 memset (&adj
, 0, sizeof (adj
));
18856 adj
.op
= IPA_PARM_OP_NEW
;
18857 adj
.arg_prefix
= "simd";
18858 adj
.base_index
= i
;
18859 adj
.type
= node
->simdclone
->args
[i
].vector_type
;
18863 if (node
->definition
)
18864 node
->simdclone
->args
[i
].simd_array
18865 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
18866 parm_type
, node
->simdclone
->simdlen
);
18868 adjustments
.safe_push (adj
);
18871 if (node
->simdclone
->inbranch
)
18874 = simd_clone_compute_base_data_type (node
->simdclone
->origin
,
18877 memset (&adj
, 0, sizeof (adj
));
18878 adj
.op
= IPA_PARM_OP_NEW
;
18879 adj
.arg_prefix
= "mask";
18881 adj
.base_index
= i
;
18882 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
18883 veclen
= node
->simdclone
->vecsize_int
;
18885 veclen
= node
->simdclone
->vecsize_float
;
18886 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
18887 if (veclen
> node
->simdclone
->simdlen
)
18888 veclen
= node
->simdclone
->simdlen
;
18889 if (POINTER_TYPE_P (base_type
))
18890 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
18892 adj
.type
= build_vector_type (base_type
, veclen
);
18893 adjustments
.safe_push (adj
);
18895 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
18896 adjustments
.safe_push (adj
);
18898 /* We have previously allocated one extra entry for the mask. Use
18900 struct cgraph_simd_clone
*sc
= node
->simdclone
;
18902 if (node
->definition
)
18904 sc
->args
[i
].orig_arg
18905 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
18906 sc
->args
[i
].simd_array
18907 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
18909 sc
->args
[i
].orig_type
= base_type
;
18910 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
18913 if (node
->definition
)
18914 ipa_modify_formal_parameters (node
->decl
, adjustments
);
18917 tree new_arg_types
= NULL_TREE
, new_reversed
;
18918 bool last_parm_void
= false;
18919 if (args
.length () > 0 && args
.last () == void_type_node
)
18920 last_parm_void
= true;
18922 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
18923 j
= adjustments
.length ();
18924 for (i
= 0; i
< j
; i
++)
18926 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
18928 if (adj
->op
== IPA_PARM_OP_COPY
)
18929 ptype
= args
[adj
->base_index
];
18932 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
18934 new_reversed
= nreverse (new_arg_types
);
18935 if (last_parm_void
)
18938 TREE_CHAIN (new_arg_types
) = void_list_node
;
18940 new_reversed
= void_list_node
;
18943 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
18944 TYPE_ARG_TYPES (new_type
) = new_reversed
;
18945 TREE_TYPE (node
->decl
) = new_type
;
18947 adjustments
.release ();
18950 return adjustments
;
18953 /* Initialize and copy the function arguments in NODE to their
18954 corresponding local simd arrays. Returns a fresh gimple_seq with
18955 the instruction sequence generated. */
18958 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
18959 ipa_parm_adjustment_vec adjustments
)
18961 gimple_seq seq
= NULL
;
18962 unsigned i
= 0, j
= 0, k
;
18964 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
18966 arg
= DECL_CHAIN (arg
), i
++, j
++)
18968 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
18969 || POINTER_TYPE_P (TREE_TYPE (arg
)))
18972 node
->simdclone
->args
[i
].vector_arg
= arg
;
18974 tree array
= node
->simdclone
->args
[i
].simd_array
;
18975 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
18977 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
18978 tree ptr
= build_fold_addr_expr (array
);
18979 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
18980 build_int_cst (ptype
, 0));
18981 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
18982 gimplify_and_add (t
, &seq
);
18986 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
18987 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
18988 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
18990 tree ptr
= build_fold_addr_expr (array
);
18994 arg
= DECL_CHAIN (arg
);
18998 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
18999 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
19000 build_int_cst (ptype
, k
* elemsize
));
19001 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
19002 gimplify_and_add (t
, &seq
);
19009 /* Callback info for ipa_simd_modify_stmt_ops below. */
19011 struct modify_stmt_info
{
19012 ipa_parm_adjustment_vec adjustments
;
19014 /* True if the parent statement was modified by
19015 ipa_simd_modify_stmt_ops. */
19019 /* Callback for walk_gimple_op.
19021 Adjust operands from a given statement as specified in the
19022 adjustments vector in the callback data. */
19025 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
19027 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
19028 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
19029 tree
*orig_tp
= tp
;
19030 if (TREE_CODE (*tp
) == ADDR_EXPR
)
19031 tp
= &TREE_OPERAND (*tp
, 0);
19032 struct ipa_parm_adjustment
*cand
= NULL
;
19033 if (TREE_CODE (*tp
) == PARM_DECL
)
19034 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
19038 *walk_subtrees
= 0;
19041 tree repl
= NULL_TREE
;
19043 repl
= unshare_expr (cand
->new_decl
);
19048 *walk_subtrees
= 0;
19049 bool modified
= info
->modified
;
19050 info
->modified
= false;
19051 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
19052 if (!info
->modified
)
19054 info
->modified
= modified
;
19057 info
->modified
= modified
;
19066 repl
= build_fold_addr_expr (repl
);
19068 if (is_gimple_debug (info
->stmt
))
19070 tree vexpr
= make_node (DEBUG_EXPR_DECL
);
19071 stmt
= gimple_build_debug_source_bind (vexpr
, repl
, NULL
);
19072 DECL_ARTIFICIAL (vexpr
) = 1;
19073 TREE_TYPE (vexpr
) = TREE_TYPE (repl
);
19074 DECL_MODE (vexpr
) = TYPE_MODE (TREE_TYPE (repl
));
19079 stmt
= gimple_build_assign (make_ssa_name (TREE_TYPE (repl
)), repl
);
19080 repl
= gimple_assign_lhs (stmt
);
19082 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
19083 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
19086 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
19088 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
19094 info
->modified
= true;
19098 /* Traverse the function body and perform all modifications as
19099 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
19100 modified such that the replacement/reduction value will now be an
19101 offset into the corresponding simd_array.
19103 This function will replace all function argument uses with their
19104 corresponding simd array elements, and ajust the return values
19108 ipa_simd_modify_function_body (struct cgraph_node
*node
,
19109 ipa_parm_adjustment_vec adjustments
,
19110 tree retval_array
, tree iter
)
19113 unsigned int i
, j
, l
;
19115 /* Re-use the adjustments array, but this time use it to replace
19116 every function argument use to an offset into the corresponding
19118 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
19120 if (!node
->simdclone
->args
[i
].vector_arg
)
19123 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
19124 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
19125 adjustments
[j
].new_decl
19126 = build4 (ARRAY_REF
,
19128 node
->simdclone
->args
[i
].simd_array
,
19130 NULL_TREE
, NULL_TREE
);
19131 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
19132 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
19133 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
19136 l
= adjustments
.length ();
19137 for (i
= 1; i
< num_ssa_names
; i
++)
19139 tree name
= ssa_name (i
);
19141 && SSA_NAME_VAR (name
)
19142 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
19144 for (j
= 0; j
< l
; j
++)
19145 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
19146 && adjustments
[j
].new_decl
)
19149 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
19152 = copy_var_decl (adjustments
[j
].base
,
19153 DECL_NAME (adjustments
[j
].base
),
19154 TREE_TYPE (adjustments
[j
].base
));
19155 adjustments
[j
].new_ssa_base
= base_var
;
19158 base_var
= adjustments
[j
].new_ssa_base
;
19159 if (SSA_NAME_IS_DEFAULT_DEF (name
))
19161 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
19162 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
19163 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
19164 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
19165 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
19166 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
19167 gimple
*stmt
= gimple_build_assign (name
, new_decl
);
19168 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
19171 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
19176 struct modify_stmt_info info
;
19177 info
.adjustments
= adjustments
;
19179 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
19181 gimple_stmt_iterator gsi
;
19183 gsi
= gsi_start_bb (bb
);
19184 while (!gsi_end_p (gsi
))
19186 gimple
*stmt
= gsi_stmt (gsi
);
19188 struct walk_stmt_info wi
;
19190 memset (&wi
, 0, sizeof (wi
));
19191 info
.modified
= false;
19193 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
19195 if (greturn
*return_stmt
= dyn_cast
<greturn
*> (stmt
))
19197 tree retval
= gimple_return_retval (return_stmt
);
19200 gsi_remove (&gsi
, true);
19204 /* Replace `return foo' with `retval_array[iter] = foo'. */
19205 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
19206 retval_array
, iter
, NULL
, NULL
);
19207 stmt
= gimple_build_assign (ref
, retval
);
19208 gsi_replace (&gsi
, stmt
, true);
19209 info
.modified
= true;
19214 update_stmt (stmt
);
19215 if (maybe_clean_eh_stmt (stmt
))
19216 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
19223 /* Helper function of simd_clone_adjust, return linear step addend
19224 of Ith argument. */
19227 simd_clone_linear_addend (struct cgraph_node
*node
, unsigned int i
,
19228 tree addtype
, basic_block entry_bb
)
19230 tree ptype
= NULL_TREE
;
19231 switch (node
->simdclone
->args
[i
].arg_type
)
19233 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
19234 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
19235 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
19236 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
19237 return build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
19238 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
19239 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
19240 ptype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
19242 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
19243 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
19244 ptype
= TREE_TYPE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
));
19247 gcc_unreachable ();
19250 unsigned int idx
= node
->simdclone
->args
[i
].linear_step
;
19251 tree arg
= node
->simdclone
->args
[idx
].orig_arg
;
19252 gcc_assert (is_gimple_reg_type (TREE_TYPE (arg
)));
19253 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
19256 if (is_gimple_reg (arg
))
19257 ret
= get_or_create_ssa_default_def (cfun
, arg
);
19260 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (arg
)), arg
);
19261 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
19262 ret
= gimple_assign_lhs (g
);
19264 if (TREE_CODE (TREE_TYPE (arg
)) == REFERENCE_TYPE
)
19266 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (TREE_TYPE (arg
))),
19267 build_simple_mem_ref (ret
));
19268 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
19269 ret
= gimple_assign_lhs (g
);
19271 if (!useless_type_conversion_p (addtype
, TREE_TYPE (ret
)))
19273 g
= gimple_build_assign (make_ssa_name (addtype
), NOP_EXPR
, ret
);
19274 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
19275 ret
= gimple_assign_lhs (g
);
19277 if (POINTER_TYPE_P (ptype
))
19279 tree size
= TYPE_SIZE_UNIT (TREE_TYPE (ptype
));
19280 if (size
&& TREE_CODE (size
) == INTEGER_CST
)
19282 g
= gimple_build_assign (make_ssa_name (addtype
), MULT_EXPR
,
19283 ret
, fold_convert (addtype
, size
));
19284 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
19285 ret
= gimple_assign_lhs (g
);
19291 /* Adjust the argument types in NODE to their appropriate vector
19295 simd_clone_adjust (struct cgraph_node
*node
)
19297 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
19299 targetm
.simd_clone
.adjust (node
);
19301 tree retval
= simd_clone_adjust_return_type (node
);
19302 ipa_parm_adjustment_vec adjustments
19303 = simd_clone_adjust_argument_types (node
);
19305 push_gimplify_context ();
19307 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
19309 /* Adjust all uses of vector arguments accordingly. Adjust all
19310 return values accordingly. */
19311 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
19312 tree iter1
= make_ssa_name (iter
);
19313 tree iter2
= make_ssa_name (iter
);
19314 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
19316 /* Initialize the iteration variable. */
19317 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
19318 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
19319 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
19320 /* Insert the SIMD array and iv initialization at function
19322 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
19324 pop_gimplify_context (NULL
);
19326 /* Create a new BB right before the original exit BB, to hold the
19327 iteration increment and the condition/branch. */
19328 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
19329 basic_block incr_bb
= create_empty_bb (orig_exit
);
19330 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
19331 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
19332 flag. Set it now to be a FALLTHRU_EDGE. */
19333 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
19334 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
19335 for (unsigned i
= 0;
19336 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
19338 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
19339 redirect_edge_succ (e
, incr_bb
);
19341 edge e
= make_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
19342 e
->probability
= REG_BR_PROB_BASE
;
19343 gsi
= gsi_last_bb (incr_bb
);
19344 gimple
*g
= gimple_build_assign (iter2
, PLUS_EXPR
, iter1
,
19345 build_int_cst (unsigned_type_node
, 1));
19346 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
19348 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
19349 struct loop
*loop
= alloc_loop ();
19350 cfun
->has_force_vectorize_loops
= true;
19351 loop
->safelen
= node
->simdclone
->simdlen
;
19352 loop
->force_vectorize
= true;
19353 loop
->header
= body_bb
;
19355 /* Branch around the body if the mask applies. */
19356 if (node
->simdclone
->inbranch
)
19358 gimple_stmt_iterator gsi
= gsi_last_bb (loop
->header
);
19360 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
19361 tree mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)));
19362 tree aref
= build4 (ARRAY_REF
,
19363 TREE_TYPE (TREE_TYPE (mask_array
)),
19366 g
= gimple_build_assign (mask
, aref
);
19367 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
19368 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
19369 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
19371 aref
= build1 (VIEW_CONVERT_EXPR
,
19372 build_nonstandard_integer_type (bitsize
, 0), mask
);
19373 mask
= make_ssa_name (TREE_TYPE (aref
));
19374 g
= gimple_build_assign (mask
, aref
);
19375 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
19378 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
19380 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
19381 make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
19382 FALLTHRU_EDGE (loop
->header
)->flags
= EDGE_FALSE_VALUE
;
19385 /* Generate the condition. */
19386 g
= gimple_build_cond (LT_EXPR
,
19388 build_int_cst (unsigned_type_node
,
19389 node
->simdclone
->simdlen
),
19391 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
19392 e
= split_block (incr_bb
, gsi_stmt (gsi
));
19393 basic_block latch_bb
= e
->dest
;
19394 basic_block new_exit_bb
;
19395 new_exit_bb
= split_block_after_labels (latch_bb
)->dest
;
19396 loop
->latch
= latch_bb
;
19398 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
19400 make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
19401 /* The successor of incr_bb is already pointing to latch_bb; just
19403 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
19404 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
19406 gphi
*phi
= create_phi_node (iter1
, body_bb
);
19407 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
19408 edge latch_edge
= single_succ_edge (latch_bb
);
19409 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
19411 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
19413 /* Generate the new return. */
19414 gsi
= gsi_last_bb (new_exit_bb
);
19416 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
19417 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
19418 retval
= TREE_OPERAND (retval
, 0);
19421 retval
= build1 (VIEW_CONVERT_EXPR
,
19422 TREE_TYPE (TREE_TYPE (node
->decl
)),
19424 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
19425 false, GSI_CONTINUE_LINKING
);
19427 g
= gimple_build_return (retval
);
19428 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
19430 /* Handle aligned clauses by replacing default defs of the aligned
19431 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
19432 lhs. Handle linear by adding PHIs. */
19433 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
19434 if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
19435 && (TREE_ADDRESSABLE (node
->simdclone
->args
[i
].orig_arg
)
19436 || !is_gimple_reg_type
19437 (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
19439 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
19440 if (is_gimple_reg_type (TREE_TYPE (orig_arg
)))
19441 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
19444 iter1
= create_tmp_var_raw (TREE_TYPE (orig_arg
));
19445 gimple_add_tmp_var (iter1
);
19447 gsi
= gsi_after_labels (entry_bb
);
19448 g
= gimple_build_assign (iter1
, orig_arg
);
19449 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
19450 gsi
= gsi_after_labels (body_bb
);
19451 g
= gimple_build_assign (orig_arg
, iter1
);
19452 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
19454 else if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
19455 && DECL_BY_REFERENCE (node
->simdclone
->args
[i
].orig_arg
)
19456 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
19458 && TREE_ADDRESSABLE
19459 (TREE_TYPE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
19461 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
19462 tree def
= ssa_default_def (cfun
, orig_arg
);
19463 if (def
&& !has_zero_uses (def
))
19465 iter1
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg
)));
19466 gimple_add_tmp_var (iter1
);
19467 gsi
= gsi_after_labels (entry_bb
);
19468 g
= gimple_build_assign (iter1
, build_simple_mem_ref (def
));
19469 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
19470 gsi
= gsi_after_labels (body_bb
);
19471 g
= gimple_build_assign (build_simple_mem_ref (def
), iter1
);
19472 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
19475 else if (node
->simdclone
->args
[i
].alignment
19476 && node
->simdclone
->args
[i
].arg_type
19477 == SIMD_CLONE_ARG_TYPE_UNIFORM
19478 && (node
->simdclone
->args
[i
].alignment
19479 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
19480 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
19483 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
19484 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
19485 tree def
= ssa_default_def (cfun
, orig_arg
);
19486 if (def
&& !has_zero_uses (def
))
19488 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
19489 gimple_seq seq
= NULL
;
19490 bool need_cvt
= false;
19492 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
19494 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
19497 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
);
19498 gimple_call_set_lhs (g
, t
);
19499 gimple_seq_add_stmt_without_update (&seq
, g
);
19502 t
= make_ssa_name (orig_arg
);
19503 g
= gimple_build_assign (t
, NOP_EXPR
, gimple_call_lhs (g
));
19504 gimple_seq_add_stmt_without_update (&seq
, g
);
19506 gsi_insert_seq_on_edge_immediate
19507 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
19509 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
19510 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
19512 node
->create_edge (cgraph_node::get_create (fn
),
19513 call
, entry_bb
->count
, freq
);
19515 imm_use_iterator iter
;
19516 use_operand_p use_p
;
19518 tree repl
= gimple_get_lhs (g
);
19519 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
19520 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
19523 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
19524 SET_USE (use_p
, repl
);
19527 else if ((node
->simdclone
->args
[i
].arg_type
19528 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
19529 || (node
->simdclone
->args
[i
].arg_type
19530 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
)
19531 || (node
->simdclone
->args
[i
].arg_type
19532 == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
19533 || (node
->simdclone
->args
[i
].arg_type
19534 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
))
19536 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
19537 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
19538 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
19539 tree def
= NULL_TREE
;
19540 if (TREE_ADDRESSABLE (orig_arg
))
19542 def
= make_ssa_name (TREE_TYPE (orig_arg
));
19543 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
19544 iter2
= make_ssa_name (TREE_TYPE (orig_arg
));
19545 gsi
= gsi_after_labels (entry_bb
);
19546 g
= gimple_build_assign (def
, orig_arg
);
19547 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
19551 def
= ssa_default_def (cfun
, orig_arg
);
19552 if (!def
|| has_zero_uses (def
))
19556 iter1
= make_ssa_name (orig_arg
);
19557 iter2
= make_ssa_name (orig_arg
);
19562 phi
= create_phi_node (iter1
, body_bb
);
19563 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
19564 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
19565 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
19566 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
19567 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
19568 ? TREE_TYPE (orig_arg
) : sizetype
;
19569 tree addcst
= simd_clone_linear_addend (node
, i
, addtype
,
19571 gsi
= gsi_last_bb (incr_bb
);
19572 g
= gimple_build_assign (iter2
, code
, iter1
, addcst
);
19573 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
19575 imm_use_iterator iter
;
19576 use_operand_p use_p
;
19578 if (TREE_ADDRESSABLE (orig_arg
))
19580 gsi
= gsi_after_labels (body_bb
);
19581 g
= gimple_build_assign (orig_arg
, iter1
);
19582 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
19585 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
19586 if (use_stmt
== phi
)
19589 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
19590 SET_USE (use_p
, iter1
);
19593 else if (node
->simdclone
->args
[i
].arg_type
19594 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
19595 || (node
->simdclone
->args
[i
].arg_type
19596 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
))
19598 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
19599 tree def
= ssa_default_def (cfun
, orig_arg
);
19600 gcc_assert (!TREE_ADDRESSABLE (orig_arg
)
19601 && TREE_CODE (TREE_TYPE (orig_arg
)) == REFERENCE_TYPE
);
19602 if (def
&& !has_zero_uses (def
))
19604 tree rtype
= TREE_TYPE (TREE_TYPE (orig_arg
));
19605 iter1
= make_ssa_name (orig_arg
);
19606 iter2
= make_ssa_name (orig_arg
);
19607 tree iter3
= make_ssa_name (rtype
);
19608 tree iter4
= make_ssa_name (rtype
);
19609 tree iter5
= make_ssa_name (rtype
);
19610 gsi
= gsi_after_labels (entry_bb
);
19612 = gimple_build_assign (iter3
, build_simple_mem_ref (def
));
19613 gsi_insert_before (&gsi
, load
, GSI_NEW_STMT
);
19615 tree array
= node
->simdclone
->args
[i
].simd_array
;
19616 TREE_ADDRESSABLE (array
) = 1;
19617 tree ptr
= build_fold_addr_expr (array
);
19618 phi
= create_phi_node (iter1
, body_bb
);
19619 add_phi_arg (phi
, ptr
, preheader_edge
, UNKNOWN_LOCATION
);
19620 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
19621 g
= gimple_build_assign (iter2
, POINTER_PLUS_EXPR
, iter1
,
19622 TYPE_SIZE_UNIT (TREE_TYPE (iter3
)));
19623 gsi
= gsi_last_bb (incr_bb
);
19624 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
19626 phi
= create_phi_node (iter4
, body_bb
);
19627 add_phi_arg (phi
, iter3
, preheader_edge
, UNKNOWN_LOCATION
);
19628 add_phi_arg (phi
, iter5
, latch_edge
, UNKNOWN_LOCATION
);
19629 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (iter3
))
19630 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
19631 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (iter3
))
19632 ? TREE_TYPE (iter3
) : sizetype
;
19633 tree addcst
= simd_clone_linear_addend (node
, i
, addtype
,
19635 g
= gimple_build_assign (iter5
, code
, iter4
, addcst
);
19636 gsi
= gsi_last_bb (incr_bb
);
19637 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
19639 g
= gimple_build_assign (build_simple_mem_ref (iter1
), iter4
);
19640 gsi
= gsi_after_labels (body_bb
);
19641 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
19643 imm_use_iterator iter
;
19644 use_operand_p use_p
;
19646 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
19647 if (use_stmt
== load
)
19650 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
19651 SET_USE (use_p
, iter1
);
19653 if (!TYPE_READONLY (rtype
))
19655 tree v
= make_ssa_name (rtype
);
19656 tree aref
= build4 (ARRAY_REF
, rtype
, array
,
19657 size_zero_node
, NULL_TREE
,
19659 gsi
= gsi_after_labels (new_exit_bb
);
19660 g
= gimple_build_assign (v
, aref
);
19661 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
19662 g
= gimple_build_assign (build_simple_mem_ref (def
), v
);
19663 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
19668 calculate_dominance_info (CDI_DOMINATORS
);
19669 add_loop (loop
, loop
->header
->loop_father
);
19670 update_ssa (TODO_update_ssa
);
19675 /* If the function in NODE is tagged as an elemental SIMD function,
19676 create the appropriate SIMD clones. */
19679 expand_simd_clones (struct cgraph_node
*node
)
19681 tree attr
= lookup_attribute ("omp declare simd",
19682 DECL_ATTRIBUTES (node
->decl
));
19683 if (attr
== NULL_TREE
19684 || node
->global
.inlined_to
19685 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
19689 #pragma omp declare simd
19691 in C, there we don't know the argument types at all. */
19692 if (!node
->definition
19693 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
19696 /* Call this before creating clone_info, as it might ggc_collect. */
19697 if (node
->definition
&& node
->has_gimple_body_p ())
19702 /* Start with parsing the "omp declare simd" attribute(s). */
19703 bool inbranch_clause_specified
;
19704 struct cgraph_simd_clone
*clone_info
19705 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
19706 &inbranch_clause_specified
);
19707 if (clone_info
== NULL
)
19710 int orig_simdlen
= clone_info
->simdlen
;
19711 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
19712 /* The target can return 0 (no simd clones should be created),
19713 1 (just one ISA of simd clones should be created) or higher
19714 count of ISA variants. In that case, clone_info is initialized
19715 for the first ISA variant. */
19717 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
19722 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
19723 also create one inbranch and one !inbranch clone of it. */
19724 for (int i
= 0; i
< count
* 2; i
++)
19726 struct cgraph_simd_clone
*clone
= clone_info
;
19727 if (inbranch_clause_specified
&& (i
& 1) != 0)
19732 clone
= simd_clone_struct_alloc (clone_info
->nargs
19734 simd_clone_struct_copy (clone
, clone_info
);
19735 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
19736 and simd_clone_adjust_argument_types did to the first
19738 clone
->nargs
-= clone_info
->inbranch
;
19739 clone
->simdlen
= orig_simdlen
;
19740 /* And call the target hook again to get the right ISA. */
19741 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
19745 clone
->inbranch
= 1;
19748 /* simd_clone_mangle might fail if such a clone has been created
19750 tree id
= simd_clone_mangle (node
, clone
);
19751 if (id
== NULL_TREE
)
19754 /* Only when we are sure we want to create the clone actually
19755 clone the function (or definitions) or create another
19756 extern FUNCTION_DECL (for prototypes without definitions). */
19757 struct cgraph_node
*n
= simd_clone_create (node
);
19761 n
->simdclone
= clone
;
19762 clone
->origin
= node
;
19763 clone
->next_clone
= NULL
;
19764 if (node
->simd_clones
== NULL
)
19766 clone
->prev_clone
= n
;
19767 node
->simd_clones
= n
;
19771 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
19772 clone
->prev_clone
->simdclone
->next_clone
= n
;
19773 node
->simd_clones
->simdclone
->prev_clone
= n
;
19775 symtab
->change_decl_assembler_name (n
->decl
, id
);
19776 /* And finally adjust the return type, parameters and for
19777 definitions also function body. */
19778 if (node
->definition
)
19779 simd_clone_adjust (n
);
19782 simd_clone_adjust_return_type (n
);
19783 simd_clone_adjust_argument_types (n
);
19787 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
19790 /* Entry point for IPA simd clone creation pass. */
19792 static unsigned int
19793 ipa_omp_simd_clone (void)
19795 struct cgraph_node
*node
;
19796 FOR_EACH_FUNCTION (node
)
19797 expand_simd_clones (node
);
19803 const pass_data pass_data_omp_simd_clone
=
19805 SIMPLE_IPA_PASS
, /* type */
19806 "simdclone", /* name */
19807 OPTGROUP_NONE
, /* optinfo_flags */
19808 TV_NONE
, /* tv_id */
19809 ( PROP_ssa
| PROP_cfg
), /* properties_required */
19810 0, /* properties_provided */
19811 0, /* properties_destroyed */
19812 0, /* todo_flags_start */
19813 0, /* todo_flags_finish */
19816 class pass_omp_simd_clone
: public simple_ipa_opt_pass
19819 pass_omp_simd_clone(gcc::context
*ctxt
)
19820 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
19823 /* opt_pass methods: */
19824 virtual bool gate (function
*);
19825 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
19829 pass_omp_simd_clone::gate (function
*)
19831 return targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
;
19834 } // anon namespace
19836 simple_ipa_opt_pass
*
19837 make_pass_omp_simd_clone (gcc::context
*ctxt
)
19839 return new pass_omp_simd_clone (ctxt
);
19842 /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and
19843 adds their addresses and sizes to constructor-vector V_CTOR. */
19845 add_decls_addresses_to_decl_constructor (vec
<tree
, va_gc
> *v_decls
,
19846 vec
<constructor_elt
, va_gc
> *v_ctor
)
19848 unsigned len
= vec_safe_length (v_decls
);
19849 for (unsigned i
= 0; i
< len
; i
++)
19851 tree it
= (*v_decls
)[i
];
19852 bool is_var
= TREE_CODE (it
) == VAR_DECL
;
19855 #ifdef ACCEL_COMPILER
19856 && DECL_HAS_VALUE_EXPR_P (it
)
19858 && lookup_attribute ("omp declare target link", DECL_ATTRIBUTES (it
));
19860 tree size
= NULL_TREE
;
19862 size
= fold_convert (const_ptr_type_node
, DECL_SIZE_UNIT (it
));
19866 addr
= build_fold_addr_expr (it
);
19869 #ifdef ACCEL_COMPILER
19870 /* For "omp declare target link" vars add address of the pointer to
19871 the target table, instead of address of the var. */
19872 tree value_expr
= DECL_VALUE_EXPR (it
);
19873 tree link_ptr_decl
= TREE_OPERAND (value_expr
, 0);
19874 varpool_node::finalize_decl (link_ptr_decl
);
19875 addr
= build_fold_addr_expr (link_ptr_decl
);
19877 addr
= build_fold_addr_expr (it
);
19880 /* Most significant bit of the size marks "omp declare target link"
19881 vars in host and target tables. */
19882 unsigned HOST_WIDE_INT isize
= tree_to_uhwi (size
);
19883 isize
|= 1ULL << (int_size_in_bytes (const_ptr_type_node
)
19884 * BITS_PER_UNIT
- 1);
19885 size
= wide_int_to_tree (const_ptr_type_node
, isize
);
19888 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, addr
);
19890 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, size
);
19894 /* Create new symbols containing (address, size) pairs for global variables,
19895 marked with "omp declare target" attribute, as well as addresses for the
19896 functions, which are outlined offloading regions. */
19898 omp_finish_file (void)
19900 unsigned num_funcs
= vec_safe_length (offload_funcs
);
19901 unsigned num_vars
= vec_safe_length (offload_vars
);
19903 if (num_funcs
== 0 && num_vars
== 0)
19906 if (targetm_common
.have_named_sections
)
19908 vec
<constructor_elt
, va_gc
> *v_f
, *v_v
;
19909 vec_alloc (v_f
, num_funcs
);
19910 vec_alloc (v_v
, num_vars
* 2);
19912 add_decls_addresses_to_decl_constructor (offload_funcs
, v_f
);
19913 add_decls_addresses_to_decl_constructor (offload_vars
, v_v
);
19915 tree vars_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
19917 tree funcs_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
19919 TYPE_ALIGN (vars_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
19920 TYPE_ALIGN (funcs_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
19921 tree ctor_v
= build_constructor (vars_decl_type
, v_v
);
19922 tree ctor_f
= build_constructor (funcs_decl_type
, v_f
);
19923 TREE_CONSTANT (ctor_v
) = TREE_CONSTANT (ctor_f
) = 1;
19924 TREE_STATIC (ctor_v
) = TREE_STATIC (ctor_f
) = 1;
19925 tree funcs_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
19926 get_identifier (".offload_func_table"),
19928 tree vars_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
19929 get_identifier (".offload_var_table"),
19931 TREE_STATIC (funcs_decl
) = TREE_STATIC (vars_decl
) = 1;
19932 /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node),
19933 otherwise a joint table in a binary will contain padding between
19934 tables from multiple object files. */
19935 DECL_USER_ALIGN (funcs_decl
) = DECL_USER_ALIGN (vars_decl
) = 1;
19936 DECL_ALIGN (funcs_decl
) = TYPE_ALIGN (funcs_decl_type
);
19937 DECL_ALIGN (vars_decl
) = TYPE_ALIGN (vars_decl_type
);
19938 DECL_INITIAL (funcs_decl
) = ctor_f
;
19939 DECL_INITIAL (vars_decl
) = ctor_v
;
19940 set_decl_section_name (funcs_decl
, OFFLOAD_FUNC_TABLE_SECTION_NAME
);
19941 set_decl_section_name (vars_decl
, OFFLOAD_VAR_TABLE_SECTION_NAME
);
19943 varpool_node::finalize_decl (vars_decl
);
19944 varpool_node::finalize_decl (funcs_decl
);
19948 for (unsigned i
= 0; i
< num_funcs
; i
++)
19950 tree it
= (*offload_funcs
)[i
];
19951 targetm
.record_offload_symbol (it
);
19953 for (unsigned i
= 0; i
< num_vars
; i
++)
19955 tree it
= (*offload_vars
)[i
];
19956 targetm
.record_offload_symbol (it
);
19961 /* Find the number of threads (POS = false), or thread number (POS =
19962 true) for an OpenACC region partitioned as MASK. Setup code
19963 required for the calculation is added to SEQ. */
19966 oacc_thread_numbers (bool pos
, int mask
, gimple_seq
*seq
)
19968 tree res
= pos
? NULL_TREE
: build_int_cst (unsigned_type_node
, 1);
19971 /* Start at gang level, and examine relevant dimension indices. */
19972 for (ix
= GOMP_DIM_GANG
; ix
!= GOMP_DIM_MAX
; ix
++)
19973 if (GOMP_DIM_MASK (ix
) & mask
)
19975 tree arg
= build_int_cst (unsigned_type_node
, ix
);
19979 /* We had an outer index, so scale that by the size of
19981 tree n
= create_tmp_var (integer_type_node
);
19983 = gimple_build_call_internal (IFN_GOACC_DIM_SIZE
, 1, arg
);
19985 gimple_call_set_lhs (call
, n
);
19986 gimple_seq_add_stmt (seq
, call
);
19987 res
= fold_build2 (MULT_EXPR
, integer_type_node
, res
, n
);
19991 /* Determine index in this dimension. */
19992 tree id
= create_tmp_var (integer_type_node
);
19993 gimple
*call
= gimple_build_call_internal
19994 (IFN_GOACC_DIM_POS
, 1, arg
);
19996 gimple_call_set_lhs (call
, id
);
19997 gimple_seq_add_stmt (seq
, call
);
19999 res
= fold_build2 (PLUS_EXPR
, integer_type_node
, res
, id
);
20005 if (res
== NULL_TREE
)
20006 res
= integer_zero_node
;
20011 /* Transform IFN_GOACC_LOOP calls to actual code. See
20012 expand_oacc_for for where these are generated. At the vector
20013 level, we stride loops, such that each member of a warp will
20014 operate on adjacent iterations. At the worker and gang level,
20015 each gang/warp executes a set of contiguous iterations. Chunking
20016 can override this such that each iteration engine executes a
20017 contiguous chunk, and then moves on to stride to the next chunk. */
20020 oacc_xform_loop (gcall
*call
)
20022 gimple_stmt_iterator gsi
= gsi_for_stmt (call
);
20023 enum ifn_goacc_loop_kind code
20024 = (enum ifn_goacc_loop_kind
) TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
20025 tree dir
= gimple_call_arg (call
, 1);
20026 tree range
= gimple_call_arg (call
, 2);
20027 tree step
= gimple_call_arg (call
, 3);
20028 tree chunk_size
= NULL_TREE
;
20029 unsigned mask
= (unsigned) TREE_INT_CST_LOW (gimple_call_arg (call
, 5));
20030 tree lhs
= gimple_call_lhs (call
);
20031 tree type
= TREE_TYPE (lhs
);
20032 tree diff_type
= TREE_TYPE (range
);
20033 tree r
= NULL_TREE
;
20034 gimple_seq seq
= NULL
;
20035 bool chunking
= false, striding
= true;
20036 unsigned outer_mask
= mask
& (~mask
+ 1); // Outermost partitioning
20037 unsigned inner_mask
= mask
& ~outer_mask
; // Inner partitioning (if any)
20039 #ifdef ACCEL_COMPILER
20040 chunk_size
= gimple_call_arg (call
, 4);
20041 if (integer_minus_onep (chunk_size
) /* Force static allocation. */
20042 || integer_zerop (chunk_size
)) /* Default (also static). */
20044 /* If we're at the gang level, we want each to execute a
20045 contiguous run of iterations. Otherwise we want each element
20047 striding
= !(outer_mask
& GOMP_DIM_MASK (GOMP_DIM_GANG
));
20052 /* Chunk of size 1 is striding. */
20053 striding
= integer_onep (chunk_size
);
20054 chunking
= !striding
;
20058 /* striding=true, chunking=true
20060 striding=true, chunking=false
20062 striding=false,chunking=true
20063 -> chunks=ceil (range/(chunksize*threads*step))
20064 striding=false,chunking=false
20065 -> chunk_size=ceil(range/(threads*step)),chunks=1 */
20066 push_gimplify_context (true);
20070 default: gcc_unreachable ();
20072 case IFN_GOACC_LOOP_CHUNKS
:
20074 r
= build_int_cst (type
, 1);
20078 = (range - dir) / (chunks * step * num_threads) + dir */
20079 tree per
= oacc_thread_numbers (false, mask
, &seq
);
20080 per
= fold_convert (type
, per
);
20081 chunk_size
= fold_convert (type
, chunk_size
);
20082 per
= fold_build2 (MULT_EXPR
, type
, per
, chunk_size
);
20083 per
= fold_build2 (MULT_EXPR
, type
, per
, step
);
20084 r
= build2 (MINUS_EXPR
, type
, range
, dir
);
20085 r
= build2 (PLUS_EXPR
, type
, r
, per
);
20086 r
= build2 (TRUNC_DIV_EXPR
, type
, r
, per
);
20090 case IFN_GOACC_LOOP_STEP
:
20092 /* If striding, step by the entire compute volume, otherwise
20093 step by the inner volume. */
20094 unsigned volume
= striding
? mask
: inner_mask
;
20096 r
= oacc_thread_numbers (false, volume
, &seq
);
20097 r
= build2 (MULT_EXPR
, type
, fold_convert (type
, r
), step
);
20101 case IFN_GOACC_LOOP_OFFSET
:
20104 r
= oacc_thread_numbers (true, mask
, &seq
);
20105 r
= fold_convert (diff_type
, r
);
20109 tree inner_size
= oacc_thread_numbers (false, inner_mask
, &seq
);
20110 tree outer_size
= oacc_thread_numbers (false, outer_mask
, &seq
);
20111 tree volume
= fold_build2 (MULT_EXPR
, TREE_TYPE (inner_size
),
20112 inner_size
, outer_size
);
20114 volume
= fold_convert (diff_type
, volume
);
20116 chunk_size
= fold_convert (diff_type
, chunk_size
);
20119 tree per
= fold_build2 (MULT_EXPR
, diff_type
, volume
, step
);
20121 chunk_size
= build2 (MINUS_EXPR
, diff_type
, range
, dir
);
20122 chunk_size
= build2 (PLUS_EXPR
, diff_type
, chunk_size
, per
);
20123 chunk_size
= build2 (TRUNC_DIV_EXPR
, diff_type
, chunk_size
, per
);
20126 tree span
= build2 (MULT_EXPR
, diff_type
, chunk_size
,
20127 fold_convert (diff_type
, inner_size
));
20128 r
= oacc_thread_numbers (true, outer_mask
, &seq
);
20129 r
= fold_convert (diff_type
, r
);
20130 r
= build2 (MULT_EXPR
, diff_type
, r
, span
);
20132 tree inner
= oacc_thread_numbers (true, inner_mask
, &seq
);
20133 inner
= fold_convert (diff_type
, inner
);
20134 r
= fold_build2 (PLUS_EXPR
, diff_type
, r
, inner
);
20138 tree chunk
= fold_convert (diff_type
, gimple_call_arg (call
, 6));
20140 = fold_build2 (MULT_EXPR
, diff_type
, volume
, chunk_size
);
20141 per
= build2 (MULT_EXPR
, diff_type
, per
, chunk
);
20143 r
= build2 (PLUS_EXPR
, diff_type
, r
, per
);
20146 r
= fold_build2 (MULT_EXPR
, diff_type
, r
, step
);
20147 if (type
!= diff_type
)
20148 r
= fold_convert (type
, r
);
20151 case IFN_GOACC_LOOP_BOUND
:
20156 tree inner_size
= oacc_thread_numbers (false, inner_mask
, &seq
);
20157 tree outer_size
= oacc_thread_numbers (false, outer_mask
, &seq
);
20158 tree volume
= fold_build2 (MULT_EXPR
, TREE_TYPE (inner_size
),
20159 inner_size
, outer_size
);
20161 volume
= fold_convert (diff_type
, volume
);
20163 chunk_size
= fold_convert (diff_type
, chunk_size
);
20166 tree per
= fold_build2 (MULT_EXPR
, diff_type
, volume
, step
);
20168 chunk_size
= build2 (MINUS_EXPR
, diff_type
, range
, dir
);
20169 chunk_size
= build2 (PLUS_EXPR
, diff_type
, chunk_size
, per
);
20170 chunk_size
= build2 (TRUNC_DIV_EXPR
, diff_type
, chunk_size
, per
);
20173 tree span
= build2 (MULT_EXPR
, diff_type
, chunk_size
,
20174 fold_convert (diff_type
, inner_size
));
20176 r
= fold_build2 (MULT_EXPR
, diff_type
, span
, step
);
20178 tree offset
= gimple_call_arg (call
, 6);
20179 r
= build2 (PLUS_EXPR
, diff_type
, r
,
20180 fold_convert (diff_type
, offset
));
20181 r
= build2 (integer_onep (dir
) ? MIN_EXPR
: MAX_EXPR
,
20182 diff_type
, r
, range
);
20184 if (diff_type
!= type
)
20185 r
= fold_convert (type
, r
);
20189 gimplify_assign (lhs
, r
, &seq
);
20191 pop_gimplify_context (NULL
);
20193 gsi_replace_with_seq (&gsi
, seq
, true);
20196 /* Validate and update the dimensions for offloaded FN. ATTRS is the
20197 raw attribute. DIMS is an array of dimensions, which is returned.
20198 Returns the function level dimensionality -- the level at which an
20199 offload routine wishes to partition a loop. */
20202 oacc_validate_dims (tree fn
, tree attrs
, int *dims
)
20204 tree purpose
[GOMP_DIM_MAX
];
20206 tree pos
= TREE_VALUE (attrs
);
20209 /* Make sure the attribute creator attached the dimension
20213 for (ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
20215 purpose
[ix
] = TREE_PURPOSE (pos
);
20219 if (integer_zerop (purpose
[ix
]))
20221 else if (fn_level
< 0)
20225 tree val
= TREE_VALUE (pos
);
20226 dims
[ix
] = val
? TREE_INT_CST_LOW (val
) : -1;
20227 pos
= TREE_CHAIN (pos
);
20230 bool changed
= targetm
.goacc
.validate_dims (fn
, dims
, fn_level
);
20232 /* Default anything left to 1. */
20233 for (ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
20242 /* Replace the attribute with new values. */
20244 for (ix
= GOMP_DIM_MAX
; ix
--;)
20245 pos
= tree_cons (purpose
[ix
],
20246 build_int_cst (integer_type_node
, dims
[ix
]),
20248 replace_oacc_fn_attrib (fn
, pos
);
20254 /* Create an empty OpenACC loop structure at LOC. */
20257 new_oacc_loop_raw (oacc_loop
*parent
, location_t loc
)
20259 oacc_loop
*loop
= XCNEW (oacc_loop
);
20261 loop
->parent
= parent
;
20262 loop
->child
= loop
->sibling
= NULL
;
20266 loop
->sibling
= parent
->child
;
20267 parent
->child
= loop
;
20271 loop
->marker
= NULL
;
20272 memset (loop
->heads
, 0, sizeof (loop
->heads
));
20273 memset (loop
->tails
, 0, sizeof (loop
->tails
));
20274 loop
->routine
= NULL_TREE
;
20276 loop
->mask
= loop
->flags
= 0;
20277 loop
->chunk_size
= 0;
20278 loop
->head_end
= NULL
;
20283 /* Create an outermost, dummy OpenACC loop for offloaded function
20287 new_oacc_loop_outer (tree decl
)
20289 return new_oacc_loop_raw (NULL
, DECL_SOURCE_LOCATION (decl
));
20292 /* Start a new OpenACC loop structure beginning at head marker HEAD.
20293 Link into PARENT loop. Return the new loop. */
20296 new_oacc_loop (oacc_loop
*parent
, gcall
*marker
)
20298 oacc_loop
*loop
= new_oacc_loop_raw (parent
, gimple_location (marker
));
20300 loop
->marker
= marker
;
20302 /* TODO: This is where device_type flattening would occur for the loop
20305 loop
->flags
= TREE_INT_CST_LOW (gimple_call_arg (marker
, 3));
20307 tree chunk_size
= integer_zero_node
;
20308 if (loop
->flags
& OLF_GANG_STATIC
)
20309 chunk_size
= gimple_call_arg (marker
, 4);
20310 loop
->chunk_size
= chunk_size
;
20315 /* Create a dummy loop encompassing a call to a openACC routine.
20316 Extract the routine's partitioning requirements. */
20319 new_oacc_loop_routine (oacc_loop
*parent
, gcall
*call
, tree decl
, tree attrs
)
20321 oacc_loop
*loop
= new_oacc_loop_raw (parent
, gimple_location (call
));
20322 int dims
[GOMP_DIM_MAX
];
20323 int level
= oacc_validate_dims (decl
, attrs
, dims
);
20325 gcc_assert (level
>= 0);
20327 loop
->marker
= call
;
20328 loop
->routine
= decl
;
20329 loop
->mask
= ((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1)
20330 ^ (GOMP_DIM_MASK (level
) - 1));
20333 /* Finish off the current OpenACC loop ending at tail marker TAIL.
20334 Return the parent loop. */
20337 finish_oacc_loop (oacc_loop
*loop
)
20339 return loop
->parent
;
20342 /* Free all OpenACC loop structures within LOOP (inclusive). */
20345 free_oacc_loop (oacc_loop
*loop
)
20348 free_oacc_loop (loop
->sibling
);
20350 free_oacc_loop (loop
->child
);
20355 /* Dump out the OpenACC loop head or tail beginning at FROM. */
20358 dump_oacc_loop_part (FILE *file
, gcall
*from
, int depth
,
20359 const char *title
, int level
)
20361 enum ifn_unique_kind kind
20362 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (from
, 0));
20364 fprintf (file
, "%*s%s-%d:\n", depth
* 2, "", title
, level
);
20365 for (gimple_stmt_iterator gsi
= gsi_for_stmt (from
);;)
20367 gimple
*stmt
= gsi_stmt (gsi
);
20369 if (is_gimple_call (stmt
)
20370 && gimple_call_internal_p (stmt
)
20371 && gimple_call_internal_fn (stmt
) == IFN_UNIQUE
)
20373 enum ifn_unique_kind k
20374 = ((enum ifn_unique_kind
) TREE_INT_CST_LOW
20375 (gimple_call_arg (stmt
, 0)));
20377 if (k
== kind
&& stmt
!= from
)
20380 print_gimple_stmt (file
, stmt
, depth
* 2 + 2, 0);
20383 while (gsi_end_p (gsi
))
20384 gsi
= gsi_start_bb (single_succ (gsi_bb (gsi
)));
20388 /* Dump OpenACC loops LOOP, its siblings and its children. */
20391 dump_oacc_loop (FILE *file
, oacc_loop
*loop
, int depth
)
20395 fprintf (file
, "%*sLoop %x(%x) %s:%u\n", depth
* 2, "",
20396 loop
->flags
, loop
->mask
,
20397 LOCATION_FILE (loop
->loc
), LOCATION_LINE (loop
->loc
));
20400 print_gimple_stmt (file
, loop
->marker
, depth
* 2, 0);
20403 fprintf (file
, "%*sRoutine %s:%u:%s\n",
20404 depth
* 2, "", DECL_SOURCE_FILE (loop
->routine
),
20405 DECL_SOURCE_LINE (loop
->routine
),
20406 IDENTIFIER_POINTER (DECL_NAME (loop
->routine
)));
20408 for (ix
= GOMP_DIM_GANG
; ix
!= GOMP_DIM_MAX
; ix
++)
20409 if (loop
->heads
[ix
])
20410 dump_oacc_loop_part (file
, loop
->heads
[ix
], depth
, "Head", ix
);
20411 for (ix
= GOMP_DIM_MAX
; ix
--;)
20412 if (loop
->tails
[ix
])
20413 dump_oacc_loop_part (file
, loop
->tails
[ix
], depth
, "Tail", ix
);
20416 dump_oacc_loop (file
, loop
->child
, depth
+ 1);
20418 dump_oacc_loop (file
, loop
->sibling
, depth
);
20421 void debug_oacc_loop (oacc_loop
*);
20423 /* Dump loops to stderr. */
20425 DEBUG_FUNCTION
void
20426 debug_oacc_loop (oacc_loop
*loop
)
20428 dump_oacc_loop (stderr
, loop
, 0);
20431 /* DFS walk of basic blocks BB onwards, creating OpenACC loop
20432 structures as we go. By construction these loops are properly
20436 oacc_loop_discover_walk (oacc_loop
*loop
, basic_block bb
)
20441 if (bb
->flags
& BB_VISITED
)
20445 bb
->flags
|= BB_VISITED
;
20447 /* Scan for loop markers. */
20448 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
);
20451 gimple
*stmt
= gsi_stmt (gsi
);
20453 if (!is_gimple_call (stmt
))
20456 gcall
*call
= as_a
<gcall
*> (stmt
);
20458 /* If this is a routine, make a dummy loop for it. */
20459 if (tree decl
= gimple_call_fndecl (call
))
20460 if (tree attrs
= get_oacc_fn_attrib (decl
))
20462 gcc_assert (!marker
);
20463 new_oacc_loop_routine (loop
, call
, decl
, attrs
);
20466 if (!gimple_call_internal_p (call
))
20469 if (gimple_call_internal_fn (call
) != IFN_UNIQUE
)
20472 enum ifn_unique_kind kind
20473 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
20474 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
20475 || kind
== IFN_UNIQUE_OACC_TAIL_MARK
)
20477 if (gimple_call_num_args (call
) == 2)
20479 gcc_assert (marker
&& !remaining
);
20481 if (kind
== IFN_UNIQUE_OACC_TAIL_MARK
)
20482 loop
= finish_oacc_loop (loop
);
20484 loop
->head_end
= call
;
20488 int count
= TREE_INT_CST_LOW (gimple_call_arg (call
, 2));
20492 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
)
20493 loop
= new_oacc_loop (loop
, call
);
20496 gcc_assert (count
== remaining
);
20500 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
)
20501 loop
->heads
[marker
] = call
;
20503 loop
->tails
[remaining
] = call
;
20509 if (remaining
|| marker
)
20511 bb
= single_succ (bb
);
20512 gcc_assert (single_pred_p (bb
) && !(bb
->flags
& BB_VISITED
));
20516 /* Walk successor blocks. */
20520 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
20521 oacc_loop_discover_walk (loop
, e
->dest
);
20524 /* LOOP is the first sibling. Reverse the order in place and return
20525 the new first sibling. Recurse to child loops. */
20528 oacc_loop_sibling_nreverse (oacc_loop
*loop
)
20530 oacc_loop
*last
= NULL
;
20534 loop
->child
= oacc_loop_sibling_nreverse (loop
->child
);
20536 oacc_loop
*next
= loop
->sibling
;
20537 loop
->sibling
= last
;
20546 /* Discover the OpenACC loops marked up by HEAD and TAIL markers for
20547 the current function. */
20550 oacc_loop_discovery ()
20554 oacc_loop
*top
= new_oacc_loop_outer (current_function_decl
);
20555 oacc_loop_discover_walk (top
, ENTRY_BLOCK_PTR_FOR_FN (cfun
));
20557 /* The siblings were constructed in reverse order, reverse them so
20558 that diagnostics come out in an unsurprising order. */
20559 top
= oacc_loop_sibling_nreverse (top
);
20561 /* Reset the visited flags. */
20562 FOR_ALL_BB_FN (bb
, cfun
)
20563 bb
->flags
&= ~BB_VISITED
;
20568 /* Transform the abstract internal function markers starting at FROM
20569 to be for partitioning level LEVEL. Stop when we meet another HEAD
20573 oacc_loop_xform_head_tail (gcall
*from
, int level
)
20575 enum ifn_unique_kind kind
20576 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (from
, 0));
20577 tree replacement
= build_int_cst (unsigned_type_node
, level
);
20579 for (gimple_stmt_iterator gsi
= gsi_for_stmt (from
);;)
20581 gimple
*stmt
= gsi_stmt (gsi
);
20583 if (is_gimple_call (stmt
)
20584 && gimple_call_internal_p (stmt
)
20585 && gimple_call_internal_fn (stmt
) == IFN_UNIQUE
)
20587 enum ifn_unique_kind k
20588 = ((enum ifn_unique_kind
)
20589 TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0)));
20591 if (k
== IFN_UNIQUE_OACC_FORK
|| k
== IFN_UNIQUE_OACC_JOIN
)
20592 *gimple_call_arg_ptr (stmt
, 2) = replacement
;
20593 else if (k
== kind
&& stmt
!= from
)
20596 else if (is_gimple_call (stmt
)
20597 && gimple_call_internal_p (stmt
)
20598 && gimple_call_internal_fn (stmt
) == IFN_GOACC_REDUCTION
)
20599 *gimple_call_arg_ptr (stmt
, 3) = replacement
;
20602 while (gsi_end_p (gsi
))
20603 gsi
= gsi_start_bb (single_succ (gsi_bb (gsi
)));
20607 /* Transform the IFN_GOACC_LOOP internal functions by providing the
20608 determined partitioning mask and chunking argument. */
20611 oacc_loop_xform_loop (gcall
*end_marker
, tree mask_arg
, tree chunk_arg
)
20613 gimple_stmt_iterator gsi
= gsi_for_stmt (end_marker
);
20617 for (; !gsi_end_p (gsi
); gsi_next (&gsi
))
20619 gimple
*stmt
= gsi_stmt (gsi
);
20621 if (!is_gimple_call (stmt
))
20624 gcall
*call
= as_a
<gcall
*> (stmt
);
20626 if (!gimple_call_internal_p (call
))
20629 if (gimple_call_internal_fn (call
) != IFN_GOACC_LOOP
)
20632 *gimple_call_arg_ptr (call
, 5) = mask_arg
;
20633 *gimple_call_arg_ptr (call
, 4) = chunk_arg
;
20634 if (TREE_INT_CST_LOW (gimple_call_arg (call
, 0))
20635 == IFN_GOACC_LOOP_BOUND
)
20639 /* If we didn't see LOOP_BOUND, it should be in the single
20640 successor block. */
20641 basic_block bb
= single_succ (gsi_bb (gsi
));
20642 gsi
= gsi_start_bb (bb
);
20646 /* Process the discovered OpenACC loops, setting the correct
20647 partitioning level etc. */
20650 oacc_loop_process (oacc_loop
*loop
)
20653 oacc_loop_process (loop
->child
);
20655 if (loop
->mask
&& !loop
->routine
)
20658 unsigned mask
= loop
->mask
;
20659 unsigned dim
= GOMP_DIM_GANG
;
20660 tree mask_arg
= build_int_cst (unsigned_type_node
, mask
);
20661 tree chunk_arg
= loop
->chunk_size
;
20663 oacc_loop_xform_loop (loop
->head_end
, mask_arg
, chunk_arg
);
20665 for (ix
= 0; ix
!= GOMP_DIM_MAX
&& loop
->heads
[ix
]; ix
++)
20669 while (!(GOMP_DIM_MASK (dim
) & mask
))
20672 oacc_loop_xform_head_tail (loop
->heads
[ix
], dim
);
20673 oacc_loop_xform_head_tail (loop
->tails
[ix
], dim
);
20675 mask
^= GOMP_DIM_MASK (dim
);
20680 oacc_loop_process (loop
->sibling
);
20683 /* Walk the OpenACC loop heirarchy checking and assigning the
20684 programmer-specified partitionings. OUTER_MASK is the partitioning
20685 this loop is contained within. Return true if we contain an
20686 auto-partitionable loop. */
20689 oacc_loop_fixed_partitions (oacc_loop
*loop
, unsigned outer_mask
)
20691 unsigned this_mask
= loop
->mask
;
20692 bool has_auto
= false;
20695 #ifdef ACCEL_COMPILER
20696 /* When device_type is supported, we want the device compiler to be
20697 noisy, if the loop parameters are device_type-specific. */
20701 if (!loop
->routine
)
20703 bool auto_par
= (loop
->flags
& OLF_AUTO
) != 0;
20704 bool seq_par
= (loop
->flags
& OLF_SEQ
) != 0;
20706 this_mask
= ((loop
->flags
>> OLF_DIM_BASE
)
20707 & (GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1));
20709 if ((this_mask
!= 0) + auto_par
+ seq_par
> 1)
20712 error_at (loop
->loc
,
20714 ? "%<seq%> overrides other OpenACC loop specifiers"
20715 : "%<auto%> conflicts with other OpenACC loop specifiers");
20717 loop
->flags
&= ~OLF_AUTO
;
20721 ~((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1) << OLF_DIM_BASE
);
20725 if (auto_par
&& (loop
->flags
& OLF_INDEPENDENT
))
20729 if (this_mask
& outer_mask
)
20731 const oacc_loop
*outer
;
20732 for (outer
= loop
->parent
; outer
; outer
= outer
->parent
)
20733 if (outer
->mask
& this_mask
)
20740 error_at (loop
->loc
,
20741 "%s uses same OpenACC parallelism as containing loop",
20742 loop
->routine
? "routine call" : "inner loop");
20743 inform (outer
->loc
, "containing loop here");
20746 error_at (loop
->loc
,
20747 "%s uses OpenACC parallelism disallowed by containing routine",
20748 loop
->routine
? "routine call" : "loop");
20751 inform (DECL_SOURCE_LOCATION (loop
->routine
),
20752 "routine %qD declared here", loop
->routine
);
20754 this_mask
&= ~outer_mask
;
20758 unsigned outermost
= this_mask
& -this_mask
;
20760 if (outermost
&& outermost
<= outer_mask
)
20764 error_at (loop
->loc
,
20765 "incorrectly nested OpenACC loop parallelism");
20767 const oacc_loop
*outer
;
20768 for (outer
= loop
->parent
;
20769 outer
->flags
&& outer
->flags
< outermost
;
20770 outer
= outer
->parent
)
20772 inform (outer
->loc
, "containing loop here");
20775 this_mask
&= ~outermost
;
20779 loop
->mask
= this_mask
;
20782 && oacc_loop_fixed_partitions (loop
->child
, outer_mask
| this_mask
))
20786 && oacc_loop_fixed_partitions (loop
->sibling
, outer_mask
))
20792 /* Walk the OpenACC loop heirarchy to assign auto-partitioned loops.
20793 OUTER_MASK is the partitioning this loop is contained within.
20794 Return the cumulative partitioning used by this loop, siblings and
20798 oacc_loop_auto_partitions (oacc_loop
*loop
, unsigned outer_mask
)
20800 unsigned inner_mask
= 0;
20803 #ifdef ACCEL_COMPILER
20804 /* When device_type is supported, we want the device compiler to be
20805 noisy, if the loop parameters are device_type-specific. */
20810 inner_mask
|= oacc_loop_auto_partitions (loop
->child
,
20811 outer_mask
| loop
->mask
);
20813 if ((loop
->flags
& OLF_AUTO
) && (loop
->flags
& OLF_INDEPENDENT
))
20815 unsigned this_mask
= 0;
20817 /* Determine the outermost partitioning used within this loop. */
20818 this_mask
= inner_mask
| GOMP_DIM_MASK (GOMP_DIM_MAX
);
20819 this_mask
= (this_mask
& -this_mask
);
20821 /* Pick the partitioning just inside that one. */
20824 /* And avoid picking one use by an outer loop. */
20825 this_mask
&= ~outer_mask
;
20827 if (!this_mask
&& noisy
)
20828 warning_at (loop
->loc
, 0,
20829 "insufficient partitioning available to parallelize loop");
20831 loop
->mask
= this_mask
;
20833 inner_mask
|= loop
->mask
;
20836 inner_mask
|= oacc_loop_auto_partitions (loop
->sibling
, outer_mask
);
20841 /* Walk the OpenACC loop heirarchy to check and assign partitioning
20845 oacc_loop_partition (oacc_loop
*loop
, unsigned outer_mask
)
20847 if (oacc_loop_fixed_partitions (loop
, outer_mask
))
20848 oacc_loop_auto_partitions (loop
, outer_mask
);
20851 /* Default fork/join early expander. Delete the function calls if
20852 there is no RTL expander. */
20855 default_goacc_fork_join (gcall
*ARG_UNUSED (call
),
20856 const int *ARG_UNUSED (dims
), bool is_fork
)
20859 return targetm
.have_oacc_fork ();
20861 return targetm
.have_oacc_join ();
20864 /* Default goacc.reduction early expander.
20866 LHS-opt = IFN_REDUCTION (KIND, RES_PTR, VAR, LEVEL, OP, OFFSET)
20867 If RES_PTR is not integer-zerop:
20868 SETUP - emit 'LHS = *RES_PTR', LHS = NULL
20869 TEARDOWN - emit '*RES_PTR = VAR'
20871 emit 'LHS = VAR' */
20874 default_goacc_reduction (gcall
*call
)
20876 unsigned code
= (unsigned)TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
20877 gimple_stmt_iterator gsi
= gsi_for_stmt (call
);
20878 tree lhs
= gimple_call_lhs (call
);
20879 tree var
= gimple_call_arg (call
, 2);
20880 gimple_seq seq
= NULL
;
20882 if (code
== IFN_GOACC_REDUCTION_SETUP
20883 || code
== IFN_GOACC_REDUCTION_TEARDOWN
)
20885 /* Setup and Teardown need to copy from/to the receiver object,
20886 if there is one. */
20887 tree ref_to_res
= gimple_call_arg (call
, 1);
20889 if (!integer_zerop (ref_to_res
))
20891 tree dst
= build_simple_mem_ref (ref_to_res
);
20894 if (code
== IFN_GOACC_REDUCTION_SETUP
)
20900 gimple_seq_add_stmt (&seq
, gimple_build_assign (dst
, src
));
20904 /* Copy VAR to LHS, if there is an LHS. */
20906 gimple_seq_add_stmt (&seq
, gimple_build_assign (lhs
, var
));
20908 gsi_replace_with_seq (&gsi
, seq
, true);
20911 /* Main entry point for oacc transformations which run on the device
20912 compiler after LTO, so we know what the target device is at this
20913 point (including the host fallback). */
20915 static unsigned int
20916 execute_oacc_device_lower ()
20918 tree attrs
= get_oacc_fn_attrib (current_function_decl
);
20919 int dims
[GOMP_DIM_MAX
];
20922 /* Not an offloaded function. */
20925 int fn_level
= oacc_validate_dims (current_function_decl
, attrs
, dims
);
20927 /* Discover, partition and process the loops. */
20928 oacc_loop
*loops
= oacc_loop_discovery ();
20929 unsigned outer_mask
= fn_level
>= 0 ? GOMP_DIM_MASK (fn_level
) - 1 : 0;
20930 oacc_loop_partition (loops
, outer_mask
);
20931 oacc_loop_process (loops
);
20934 fprintf (dump_file
, "OpenACC loops\n");
20935 dump_oacc_loop (dump_file
, loops
, 0);
20936 fprintf (dump_file
, "\n");
20939 /* Offloaded targets may introduce new basic blocks, which require
20940 dominance information to update SSA. */
20941 calculate_dominance_info (CDI_DOMINATORS
);
20943 /* Now lower internal loop functions to target-specific code
20946 FOR_ALL_BB_FN (bb
, cfun
)
20947 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
);)
20949 gimple
*stmt
= gsi_stmt (gsi
);
20950 if (!is_gimple_call (stmt
))
20956 gcall
*call
= as_a
<gcall
*> (stmt
);
20957 if (!gimple_call_internal_p (call
))
20963 /* Rewind to allow rescan. */
20965 bool rescan
= false, remove
= false;
20966 enum internal_fn ifn_code
= gimple_call_internal_fn (call
);
20972 case IFN_GOACC_LOOP
:
20973 oacc_xform_loop (call
);
20977 case IFN_GOACC_REDUCTION
:
20978 /* Mark the function for SSA renaming. */
20979 mark_virtual_operands_for_renaming (cfun
);
20981 /* If the level is -1, this ended up being an unused
20982 axis. Handle as a default. */
20983 if (integer_minus_onep (gimple_call_arg (call
, 3)))
20984 default_goacc_reduction (call
);
20986 targetm
.goacc
.reduction (call
);
20992 enum ifn_unique_kind kind
20993 = ((enum ifn_unique_kind
)
20994 TREE_INT_CST_LOW (gimple_call_arg (call
, 0)));
20999 gcc_unreachable ();
21001 case IFN_UNIQUE_OACC_FORK
:
21002 case IFN_UNIQUE_OACC_JOIN
:
21003 if (integer_minus_onep (gimple_call_arg (call
, 2)))
21005 else if (!targetm
.goacc
.fork_join
21006 (call
, dims
, kind
== IFN_UNIQUE_OACC_FORK
))
21010 case IFN_UNIQUE_OACC_HEAD_MARK
:
21011 case IFN_UNIQUE_OACC_TAIL_MARK
:
21019 if (gsi_end_p (gsi
))
21020 /* We rewound past the beginning of the BB. */
21021 gsi
= gsi_start_bb (bb
);
21023 /* Undo the rewind. */
21028 if (gimple_vdef (call
))
21029 replace_uses_by (gimple_vdef (call
), gimple_vuse (call
));
21030 if (gimple_call_lhs (call
))
21032 /* Propagate the data dependency var. */
21033 gimple
*ass
= gimple_build_assign (gimple_call_lhs (call
),
21034 gimple_call_arg (call
, 1));
21035 gsi_replace (&gsi
, ass
, false);
21038 gsi_remove (&gsi
, true);
21041 /* If not rescanning, advance over the call. */
21045 free_oacc_loop (loops
);
21050 /* Default launch dimension validator. Force everything to 1. A
21051 backend that wants to provide larger dimensions must override this
21055 default_goacc_validate_dims (tree
ARG_UNUSED (decl
), int *dims
,
21056 int ARG_UNUSED (fn_level
))
21058 bool changed
= false;
21060 for (unsigned ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
21072 /* Default dimension bound is unknown on accelerator and 1 on host. */
21075 default_goacc_dim_limit (int ARG_UNUSED (axis
))
21077 #ifdef ACCEL_COMPILER
21086 const pass_data pass_data_oacc_device_lower
=
21088 GIMPLE_PASS
, /* type */
21089 "oaccdevlow", /* name */
21090 OPTGROUP_NONE
, /* optinfo_flags */
21091 TV_NONE
, /* tv_id */
21092 PROP_cfg
, /* properties_required */
21093 0 /* Possibly PROP_gimple_eomp. */, /* properties_provided */
21094 0, /* properties_destroyed */
21095 0, /* todo_flags_start */
21096 TODO_update_ssa
| TODO_cleanup_cfg
, /* todo_flags_finish */
21099 class pass_oacc_device_lower
: public gimple_opt_pass
21102 pass_oacc_device_lower (gcc::context
*ctxt
)
21103 : gimple_opt_pass (pass_data_oacc_device_lower
, ctxt
)
21106 /* opt_pass methods: */
21107 virtual unsigned int execute (function
*)
21109 bool gate
= flag_openacc
!= 0;
21114 return execute_oacc_device_lower ();
21117 }; // class pass_oacc_device_lower
21119 } // anon namespace
21122 make_pass_oacc_device_lower (gcc::context
*ctxt
)
21124 return new pass_oacc_device_lower (ctxt
);
21127 /* "omp declare target link" handling pass. */
21131 const pass_data pass_data_omp_target_link
=
21133 GIMPLE_PASS
, /* type */
21134 "omptargetlink", /* name */
21135 OPTGROUP_NONE
, /* optinfo_flags */
21136 TV_NONE
, /* tv_id */
21137 PROP_ssa
, /* properties_required */
21138 0, /* properties_provided */
21139 0, /* properties_destroyed */
21140 0, /* todo_flags_start */
21141 TODO_update_ssa
, /* todo_flags_finish */
21144 class pass_omp_target_link
: public gimple_opt_pass
21147 pass_omp_target_link (gcc::context
*ctxt
)
21148 : gimple_opt_pass (pass_data_omp_target_link
, ctxt
)
21151 /* opt_pass methods: */
21152 virtual bool gate (function
*fun
)
21154 #ifdef ACCEL_COMPILER
21155 tree attrs
= DECL_ATTRIBUTES (fun
->decl
);
21156 return lookup_attribute ("omp declare target", attrs
)
21157 || lookup_attribute ("omp target entrypoint", attrs
);
21164 virtual unsigned execute (function
*);
21167 /* Callback for walk_gimple_stmt used to scan for link var operands. */
21170 find_link_var_op (tree
*tp
, int *walk_subtrees
, void *)
21174 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
)
21175 && lookup_attribute ("omp declare target link", DECL_ATTRIBUTES (t
)))
21177 *walk_subtrees
= 0;
21185 pass_omp_target_link::execute (function
*fun
)
21188 FOR_EACH_BB_FN (bb
, fun
)
21190 gimple_stmt_iterator gsi
;
21191 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
21192 if (walk_gimple_stmt (&gsi
, NULL
, find_link_var_op
, NULL
))
21193 gimple_regimplify_operands (gsi_stmt (gsi
), &gsi
);
21199 } // anon namespace
21202 make_pass_omp_target_link (gcc::context
*ctxt
)
21204 return new pass_omp_target_link (ctxt
);
21207 #include "gt-omp-low.h"