1 /* Lowering pass for OpenMP directives. Converts OpenMP directives
2 into explicit calls to the runtime library (libgomp) and data
3 marshalling to implement data sharing and copying clauses.
4 Contributed by Diego Novillo <dnovillo@redhat.com>
6 Copyright (C) 2005-2013 Free Software Foundation, Inc.
8 This file is part of GCC.
10 GCC is free software; you can redistribute it and/or modify it under
11 the terms of the GNU General Public License as published by the Free
12 Software Foundation; either version 3, or (at your option) any later
15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
17 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
20 You should have received a copy of the GNU General Public License
21 along with GCC; see the file COPYING3. If not see
22 <http://www.gnu.org/licenses/>. */
26 #include "coretypes.h"
32 #include "gimple-iterator.h"
33 #include "gimplify-me.h"
34 #include "gimple-walk.h"
35 #include "tree-iterator.h"
36 #include "tree-inline.h"
37 #include "langhooks.h"
38 #include "diagnostic-core.h"
39 #include "gimple-ssa.h"
42 #include "tree-phinodes.h"
43 #include "ssa-iterators.h"
44 #include "tree-ssanames.h"
45 #include "tree-into-ssa.h"
51 #include "tree-pass.h"
54 #include "splay-tree.h"
59 #include "gimple-low.h"
60 #include "tree-cfgcleanup.h"
61 #include "tree-nested.h"
64 /* Lowering of OpenMP parallel and workshare constructs proceeds in two
65 phases. The first phase scans the function looking for OMP statements
66 and then for variables that must be replaced to satisfy data sharing
67 clauses. The second phase expands code for the constructs, as well as
68 re-gimplifying things when variables have been replaced with complex
71 Final code generation is done by pass_expand_omp. The flowgraph is
72 scanned for parallel regions which are then moved to a new
73 function, to be invoked by the thread library. */
75 /* Parallel region information. Every parallel and workshare
76 directive is enclosed between two markers, the OMP_* directive
77 and a corresponding OMP_RETURN statement. */
81 /* The enclosing region. */
82 struct omp_region
*outer
;
84 /* First child region. */
85 struct omp_region
*inner
;
87 /* Next peer region. */
88 struct omp_region
*next
;
90 /* Block containing the omp directive as its last stmt. */
93 /* Block containing the OMP_RETURN as its last stmt. */
96 /* Block containing the OMP_CONTINUE as its last stmt. */
99 /* If this is a combined parallel+workshare region, this is a list
100 of additional arguments needed by the combined parallel+workshare
102 vec
<tree
, va_gc
> *ws_args
;
104 /* The code for the omp directive of this region. */
105 enum gimple_code type
;
107 /* Schedule kind, only used for OMP_FOR type regions. */
108 enum omp_clause_schedule_kind sched_kind
;
110 /* True if this is a combined parallel+workshare region. */
111 bool is_combined_parallel
;
114 /* Context structure. Used to store information about each parallel
115 directive in the code. */
117 typedef struct omp_context
119 /* This field must be at the beginning, as we do "inheritance": Some
120 callback functions for tree-inline.c (e.g., omp_copy_decl)
121 receive a copy_body_data pointer that is up-casted to an
122 omp_context pointer. */
125 /* The tree of contexts corresponding to the encountered constructs. */
126 struct omp_context
*outer
;
129 /* Map variables to fields in a structure that allows communication
130 between sending and receiving threads. */
131 splay_tree field_map
;
136 /* These are used just by task contexts, if task firstprivate fn is
137 needed. srecord_type is used to communicate from the thread
138 that encountered the task construct to task firstprivate fn,
139 record_type is allocated by GOMP_task, initialized by task firstprivate
140 fn and passed to the task body fn. */
141 splay_tree sfield_map
;
144 /* A chain of variables to add to the top-level block surrounding the
145 construct. In the case of a parallel, this is in the child function. */
148 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
149 barriers should jump to during omplower pass. */
152 /* What to do with variables with implicitly determined sharing
154 enum omp_clause_default_kind default_kind
;
156 /* Nesting depth of this context. Used to beautify error messages re
157 invalid gotos. The outermost ctx is depth 1, with depth 0 being
158 reserved for the main body of the function. */
161 /* True if this parallel directive is nested within another. */
164 /* True if this construct can be cancelled. */
169 struct omp_for_data_loop
171 tree v
, n1
, n2
, step
;
172 enum tree_code cond_code
;
175 /* A structure describing the main elements of a parallel loop. */
179 struct omp_for_data_loop loop
;
184 bool have_nowait
, have_ordered
;
185 enum omp_clause_schedule_kind sched_kind
;
186 struct omp_for_data_loop
*loops
;
190 static splay_tree all_contexts
;
191 static int taskreg_nesting_level
;
192 static int target_nesting_level
;
193 static struct omp_region
*root_omp_region
;
194 static bitmap task_shared_vars
;
196 static void scan_omp (gimple_seq
*, omp_context
*);
197 static tree
scan_omp_1_op (tree
*, int *, void *);
199 #define WALK_SUBSTMTS \
203 case GIMPLE_EH_FILTER: \
204 case GIMPLE_TRANSACTION: \
205 /* The sub-statements for these should be walked. */ \
206 *handled_ops_p = false; \
209 /* Convenience function for calling scan_omp_1_op on tree operands. */
212 scan_omp_op (tree
*tp
, omp_context
*ctx
)
214 struct walk_stmt_info wi
;
216 memset (&wi
, 0, sizeof (wi
));
218 wi
.want_locations
= true;
220 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
223 static void lower_omp (gimple_seq
*, omp_context
*);
224 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
225 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
227 /* Find an OpenMP clause of type KIND within CLAUSES. */
230 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
232 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
233 if (OMP_CLAUSE_CODE (clauses
) == kind
)
239 /* Return true if CTX is for an omp parallel. */
242 is_parallel_ctx (omp_context
*ctx
)
244 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
248 /* Return true if CTX is for an omp task. */
251 is_task_ctx (omp_context
*ctx
)
253 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
257 /* Return true if CTX is for an omp parallel or omp task. */
260 is_taskreg_ctx (omp_context
*ctx
)
262 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
263 || gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
267 /* Return true if REGION is a combined parallel+workshare region. */
270 is_combined_parallel (struct omp_region
*region
)
272 return region
->is_combined_parallel
;
276 /* Extract the header elements of parallel loop FOR_STMT and store
280 extract_omp_for_data (gimple for_stmt
, struct omp_for_data
*fd
,
281 struct omp_for_data_loop
*loops
)
283 tree t
, var
, *collapse_iter
, *collapse_count
;
284 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
285 struct omp_for_data_loop
*loop
;
287 struct omp_for_data_loop dummy_loop
;
288 location_t loc
= gimple_location (for_stmt
);
289 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_KIND_SIMD
;
290 bool distribute
= gimple_omp_for_kind (for_stmt
)
291 == GF_OMP_FOR_KIND_DISTRIBUTE
;
293 fd
->for_stmt
= for_stmt
;
295 fd
->collapse
= gimple_omp_for_collapse (for_stmt
);
296 if (fd
->collapse
> 1)
299 fd
->loops
= &fd
->loop
;
301 fd
->have_nowait
= distribute
|| simd
;
302 fd
->have_ordered
= false;
303 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
304 fd
->chunk_size
= NULL_TREE
;
305 collapse_iter
= NULL
;
306 collapse_count
= NULL
;
308 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
309 switch (OMP_CLAUSE_CODE (t
))
311 case OMP_CLAUSE_NOWAIT
:
312 fd
->have_nowait
= true;
314 case OMP_CLAUSE_ORDERED
:
315 fd
->have_ordered
= true;
317 case OMP_CLAUSE_SCHEDULE
:
318 gcc_assert (!distribute
);
319 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_KIND (t
);
320 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
322 case OMP_CLAUSE_DIST_SCHEDULE
:
323 gcc_assert (distribute
);
324 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
326 case OMP_CLAUSE_COLLAPSE
:
327 if (fd
->collapse
> 1)
329 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
330 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
336 /* FIXME: for now map schedule(auto) to schedule(static).
337 There should be analysis to determine whether all iterations
338 are approximately the same amount of work (then schedule(static)
339 is best) or if it varies (then schedule(dynamic,N) is better). */
340 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
342 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
343 gcc_assert (fd
->chunk_size
== NULL
);
345 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
346 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
347 gcc_assert (fd
->chunk_size
== NULL
);
348 else if (fd
->chunk_size
== NULL
)
350 /* We only need to compute a default chunk size for ordered
351 static loops and dynamic loops. */
352 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
354 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
355 ? integer_zero_node
: integer_one_node
;
358 for (i
= 0; i
< fd
->collapse
; i
++)
360 if (fd
->collapse
== 1)
362 else if (loops
!= NULL
)
367 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
368 gcc_assert (SSA_VAR_P (loop
->v
));
369 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
370 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
371 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
372 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
374 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
375 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
376 switch (loop
->cond_code
)
382 gcc_assert (gimple_omp_for_kind (for_stmt
)
383 == GF_OMP_FOR_KIND_CILKSIMD
);
386 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
387 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, 1);
389 loop
->n2
= fold_build2_loc (loc
,
390 PLUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
391 build_int_cst (TREE_TYPE (loop
->n2
), 1));
392 loop
->cond_code
= LT_EXPR
;
395 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
396 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, -1);
398 loop
->n2
= fold_build2_loc (loc
,
399 MINUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
400 build_int_cst (TREE_TYPE (loop
->n2
), 1));
401 loop
->cond_code
= GT_EXPR
;
407 t
= gimple_omp_for_incr (for_stmt
, i
);
408 gcc_assert (TREE_OPERAND (t
, 0) == var
);
409 switch (TREE_CODE (t
))
412 loop
->step
= TREE_OPERAND (t
, 1);
414 case POINTER_PLUS_EXPR
:
415 loop
->step
= fold_convert (ssizetype
, TREE_OPERAND (t
, 1));
418 loop
->step
= TREE_OPERAND (t
, 1);
419 loop
->step
= fold_build1_loc (loc
,
420 NEGATE_EXPR
, TREE_TYPE (loop
->step
),
428 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
429 && !fd
->have_ordered
))
431 if (fd
->collapse
== 1)
432 iter_type
= TREE_TYPE (loop
->v
);
434 || TYPE_PRECISION (iter_type
)
435 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
437 = build_nonstandard_integer_type
438 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
440 else if (iter_type
!= long_long_unsigned_type_node
)
442 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
443 iter_type
= long_long_unsigned_type_node
;
444 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
445 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
446 >= TYPE_PRECISION (iter_type
))
450 if (loop
->cond_code
== LT_EXPR
)
451 n
= fold_build2_loc (loc
,
452 PLUS_EXPR
, TREE_TYPE (loop
->v
),
453 loop
->n2
, loop
->step
);
456 if (TREE_CODE (n
) != INTEGER_CST
457 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
458 iter_type
= long_long_unsigned_type_node
;
460 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
461 > TYPE_PRECISION (iter_type
))
465 if (loop
->cond_code
== LT_EXPR
)
468 n2
= fold_build2_loc (loc
,
469 PLUS_EXPR
, TREE_TYPE (loop
->v
),
470 loop
->n2
, loop
->step
);
474 n1
= fold_build2_loc (loc
,
475 MINUS_EXPR
, TREE_TYPE (loop
->v
),
476 loop
->n2
, loop
->step
);
479 if (TREE_CODE (n1
) != INTEGER_CST
480 || TREE_CODE (n2
) != INTEGER_CST
481 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
482 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
483 iter_type
= long_long_unsigned_type_node
;
487 if (collapse_count
&& *collapse_count
== NULL
)
489 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
490 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
491 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
492 if (t
&& integer_zerop (t
))
493 count
= build_zero_cst (long_long_unsigned_type_node
);
494 else if ((i
== 0 || count
!= NULL_TREE
)
495 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
496 && TREE_CONSTANT (loop
->n1
)
497 && TREE_CONSTANT (loop
->n2
)
498 && TREE_CODE (loop
->step
) == INTEGER_CST
)
500 tree itype
= TREE_TYPE (loop
->v
);
502 if (POINTER_TYPE_P (itype
))
503 itype
= signed_type_for (itype
);
504 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
505 t
= fold_build2_loc (loc
,
507 fold_convert_loc (loc
, itype
, loop
->step
), t
);
508 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
509 fold_convert_loc (loc
, itype
, loop
->n2
));
510 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
511 fold_convert_loc (loc
, itype
, loop
->n1
));
512 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
513 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
514 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
515 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
516 fold_convert_loc (loc
, itype
,
519 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
520 fold_convert_loc (loc
, itype
, loop
->step
));
521 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
522 if (count
!= NULL_TREE
)
523 count
= fold_build2_loc (loc
,
524 MULT_EXPR
, long_long_unsigned_type_node
,
528 if (TREE_CODE (count
) != INTEGER_CST
)
531 else if (count
&& !integer_zerop (count
))
538 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
539 || fd
->have_ordered
))
541 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
542 iter_type
= long_long_unsigned_type_node
;
544 iter_type
= long_integer_type_node
;
546 else if (collapse_iter
&& *collapse_iter
!= NULL
)
547 iter_type
= TREE_TYPE (*collapse_iter
);
548 fd
->iter_type
= iter_type
;
549 if (collapse_iter
&& *collapse_iter
== NULL
)
550 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
551 if (collapse_count
&& *collapse_count
== NULL
)
554 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
556 *collapse_count
= create_tmp_var (iter_type
, ".count");
559 if (fd
->collapse
> 1)
561 fd
->loop
.v
= *collapse_iter
;
562 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
563 fd
->loop
.n2
= *collapse_count
;
564 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
565 fd
->loop
.cond_code
= LT_EXPR
;
570 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
571 is the immediate dominator of PAR_ENTRY_BB, return true if there
572 are no data dependencies that would prevent expanding the parallel
573 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
575 When expanding a combined parallel+workshare region, the call to
576 the child function may need additional arguments in the case of
577 GIMPLE_OMP_FOR regions. In some cases, these arguments are
578 computed out of variables passed in from the parent to the child
579 via 'struct .omp_data_s'. For instance:
581 #pragma omp parallel for schedule (guided, i * 4)
586 # BLOCK 2 (PAR_ENTRY_BB)
588 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
590 # BLOCK 3 (WS_ENTRY_BB)
591 .omp_data_i = &.omp_data_o;
592 D.1667 = .omp_data_i->i;
594 #pragma omp for schedule (guided, D.1598)
596 When we outline the parallel region, the call to the child function
597 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
598 that value is computed *after* the call site. So, in principle we
599 cannot do the transformation.
601 To see whether the code in WS_ENTRY_BB blocks the combined
602 parallel+workshare call, we collect all the variables used in the
603 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
604 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
607 FIXME. If we had the SSA form built at this point, we could merely
608 hoist the code in block 3 into block 2 and be done with it. But at
609 this point we don't have dataflow information and though we could
610 hack something up here, it is really not worth the aggravation. */
613 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
615 struct omp_for_data fd
;
616 gimple ws_stmt
= last_stmt (ws_entry_bb
);
618 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
621 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
623 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
625 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
627 if (fd
.iter_type
!= long_integer_type_node
)
630 /* FIXME. We give up too easily here. If any of these arguments
631 are not constants, they will likely involve variables that have
632 been mapped into fields of .omp_data_s for sharing with the child
633 function. With appropriate data flow, it would be possible to
635 if (!is_gimple_min_invariant (fd
.loop
.n1
)
636 || !is_gimple_min_invariant (fd
.loop
.n2
)
637 || !is_gimple_min_invariant (fd
.loop
.step
)
638 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
645 /* Collect additional arguments needed to emit a combined
646 parallel+workshare call. WS_STMT is the workshare directive being
649 static vec
<tree
, va_gc
> *
650 get_ws_args_for (gimple par_stmt
, gimple ws_stmt
)
653 location_t loc
= gimple_location (ws_stmt
);
654 vec
<tree
, va_gc
> *ws_args
;
656 if (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
)
658 struct omp_for_data fd
;
661 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
665 if (gimple_omp_for_combined_into_p (ws_stmt
))
668 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
669 OMP_CLAUSE__LOOPTEMP_
);
671 n1
= OMP_CLAUSE_DECL (innerc
);
672 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
673 OMP_CLAUSE__LOOPTEMP_
);
675 n2
= OMP_CLAUSE_DECL (innerc
);
678 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
680 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
681 ws_args
->quick_push (t
);
683 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
684 ws_args
->quick_push (t
);
686 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
687 ws_args
->quick_push (t
);
691 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
692 ws_args
->quick_push (t
);
697 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
699 /* Number of sections is equal to the number of edges from the
700 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
701 the exit of the sections region. */
702 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
703 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
704 vec_alloc (ws_args
, 1);
705 ws_args
->quick_push (t
);
713 /* Discover whether REGION is a combined parallel+workshare region. */
716 determine_parallel_type (struct omp_region
*region
)
718 basic_block par_entry_bb
, par_exit_bb
;
719 basic_block ws_entry_bb
, ws_exit_bb
;
721 if (region
== NULL
|| region
->inner
== NULL
722 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
723 || region
->inner
->cont
== NULL
)
726 /* We only support parallel+for and parallel+sections. */
727 if (region
->type
!= GIMPLE_OMP_PARALLEL
728 || (region
->inner
->type
!= GIMPLE_OMP_FOR
729 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
732 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
733 WS_EXIT_BB -> PAR_EXIT_BB. */
734 par_entry_bb
= region
->entry
;
735 par_exit_bb
= region
->exit
;
736 ws_entry_bb
= region
->inner
->entry
;
737 ws_exit_bb
= region
->inner
->exit
;
739 if (single_succ (par_entry_bb
) == ws_entry_bb
740 && single_succ (ws_exit_bb
) == par_exit_bb
741 && workshare_safe_to_combine_p (ws_entry_bb
)
742 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
743 || (last_and_only_stmt (ws_entry_bb
)
744 && last_and_only_stmt (par_exit_bb
))))
746 gimple par_stmt
= last_stmt (par_entry_bb
);
747 gimple ws_stmt
= last_stmt (ws_entry_bb
);
749 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
751 /* If this is a combined parallel loop, we need to determine
752 whether or not to use the combined library calls. There
753 are two cases where we do not apply the transformation:
754 static loops and any kind of ordered loop. In the first
755 case, we already open code the loop so there is no need
756 to do anything else. In the latter case, the combined
757 parallel loop call would still need extra synchronization
758 to implement ordered semantics, so there would not be any
759 gain in using the combined call. */
760 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
761 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
763 || OMP_CLAUSE_SCHEDULE_KIND (c
) == OMP_CLAUSE_SCHEDULE_STATIC
764 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
766 region
->is_combined_parallel
= false;
767 region
->inner
->is_combined_parallel
= false;
772 region
->is_combined_parallel
= true;
773 region
->inner
->is_combined_parallel
= true;
774 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
779 /* Return true if EXPR is variable sized. */
782 is_variable_sized (const_tree expr
)
784 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
787 /* Return true if DECL is a reference type. */
790 is_reference (tree decl
)
792 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
795 /* Lookup variables in the decl or field splay trees. The "maybe" form
796 allows for the variable form to not have been entered, otherwise we
797 assert that the variable must have been entered. */
800 lookup_decl (tree var
, omp_context
*ctx
)
803 n
= (tree
*) pointer_map_contains (ctx
->cb
.decl_map
, var
);
808 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
811 n
= (tree
*) pointer_map_contains (ctx
->cb
.decl_map
, var
);
812 return n
? *n
: NULL_TREE
;
816 lookup_field (tree var
, omp_context
*ctx
)
819 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
820 return (tree
) n
->value
;
824 lookup_sfield (tree var
, omp_context
*ctx
)
827 n
= splay_tree_lookup (ctx
->sfield_map
828 ? ctx
->sfield_map
: ctx
->field_map
,
829 (splay_tree_key
) var
);
830 return (tree
) n
->value
;
834 maybe_lookup_field (tree var
, omp_context
*ctx
)
837 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
838 return n
? (tree
) n
->value
: NULL_TREE
;
841 /* Return true if DECL should be copied by pointer. SHARED_CTX is
842 the parallel context if DECL is to be shared. */
845 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
847 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
850 /* We can only use copy-in/copy-out semantics for shared variables
851 when we know the value is not accessible from an outer scope. */
854 /* ??? Trivially accessible from anywhere. But why would we even
855 be passing an address in this case? Should we simply assert
856 this to be false, or should we have a cleanup pass that removes
857 these from the list of mappings? */
858 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
861 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
862 without analyzing the expression whether or not its location
863 is accessible to anyone else. In the case of nested parallel
864 regions it certainly may be. */
865 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
868 /* Do not use copy-in/copy-out for variables that have their
870 if (TREE_ADDRESSABLE (decl
))
873 /* lower_send_shared_vars only uses copy-in, but not copy-out
875 if (TREE_READONLY (decl
)
876 || ((TREE_CODE (decl
) == RESULT_DECL
877 || TREE_CODE (decl
) == PARM_DECL
)
878 && DECL_BY_REFERENCE (decl
)))
881 /* Disallow copy-in/out in nested parallel if
882 decl is shared in outer parallel, otherwise
883 each thread could store the shared variable
884 in its own copy-in location, making the
885 variable no longer really shared. */
886 if (shared_ctx
->is_nested
)
890 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
891 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
898 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
899 c
; c
= OMP_CLAUSE_CHAIN (c
))
900 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
901 && OMP_CLAUSE_DECL (c
) == decl
)
905 goto maybe_mark_addressable_and_ret
;
909 /* For tasks avoid using copy-in/out. As tasks can be
910 deferred or executed in different thread, when GOMP_task
911 returns, the task hasn't necessarily terminated. */
912 if (is_task_ctx (shared_ctx
))
915 maybe_mark_addressable_and_ret
:
916 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
917 if (is_gimple_reg (outer
))
919 /* Taking address of OUTER in lower_send_shared_vars
920 might need regimplification of everything that uses the
922 if (!task_shared_vars
)
923 task_shared_vars
= BITMAP_ALLOC (NULL
);
924 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
925 TREE_ADDRESSABLE (outer
) = 1;
934 /* Construct a new automatic decl similar to VAR. */
937 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
939 tree copy
= copy_var_decl (var
, name
, type
);
941 DECL_CONTEXT (copy
) = current_function_decl
;
942 DECL_CHAIN (copy
) = ctx
->block_vars
;
943 ctx
->block_vars
= copy
;
949 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
951 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
954 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
957 omp_build_component_ref (tree obj
, tree field
)
959 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
960 if (TREE_THIS_VOLATILE (field
))
961 TREE_THIS_VOLATILE (ret
) |= 1;
962 if (TREE_READONLY (field
))
963 TREE_READONLY (ret
) |= 1;
967 /* Build tree nodes to access the field for VAR on the receiver side. */
970 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
972 tree x
, field
= lookup_field (var
, ctx
);
974 /* If the receiver record type was remapped in the child function,
975 remap the field into the new record type. */
976 x
= maybe_lookup_field (field
, ctx
);
980 x
= build_simple_mem_ref (ctx
->receiver_decl
);
981 x
= omp_build_component_ref (x
, field
);
983 x
= build_simple_mem_ref (x
);
988 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
989 of a parallel, this is a component reference; for workshare constructs
990 this is some variable. */
993 build_outer_var_ref (tree var
, omp_context
*ctx
)
997 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
999 else if (is_variable_sized (var
))
1001 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1002 x
= build_outer_var_ref (x
, ctx
);
1003 x
= build_simple_mem_ref (x
);
1005 else if (is_taskreg_ctx (ctx
))
1007 bool by_ref
= use_pointer_for_field (var
, NULL
);
1008 x
= build_receiver_ref (var
, by_ref
, ctx
);
1010 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1011 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
)
1013 /* #pragma omp simd isn't a worksharing construct, and can reference even
1014 private vars in its linear etc. clauses. */
1016 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1017 x
= lookup_decl (var
, ctx
->outer
);
1018 else if (ctx
->outer
)
1019 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1023 else if (ctx
->outer
)
1024 x
= lookup_decl (var
, ctx
->outer
);
1025 else if (is_reference (var
))
1026 /* This can happen with orphaned constructs. If var is reference, it is
1027 possible it is shared and as such valid. */
1032 if (is_reference (var
))
1033 x
= build_simple_mem_ref (x
);
1038 /* Build tree nodes to access the field for VAR on the sender side. */
1041 build_sender_ref (tree var
, omp_context
*ctx
)
1043 tree field
= lookup_sfield (var
, ctx
);
1044 return omp_build_component_ref (ctx
->sender_decl
, field
);
1047 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1050 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
)
1052 tree field
, type
, sfield
= NULL_TREE
;
1054 gcc_assert ((mask
& 1) == 0
1055 || !splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
));
1056 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1057 || !splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) var
));
1059 type
= TREE_TYPE (var
);
1062 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1063 type
= build_pointer_type (build_pointer_type (type
));
1066 type
= build_pointer_type (type
);
1067 else if ((mask
& 3) == 1 && is_reference (var
))
1068 type
= TREE_TYPE (type
);
1070 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1071 FIELD_DECL
, DECL_NAME (var
), type
);
1073 /* Remember what variable this field was created for. This does have a
1074 side effect of making dwarf2out ignore this member, so for helpful
1075 debugging we clear it later in delete_omp_context. */
1076 DECL_ABSTRACT_ORIGIN (field
) = var
;
1077 if (type
== TREE_TYPE (var
))
1079 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1080 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1081 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1084 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1086 if ((mask
& 3) == 3)
1088 insert_field_into_struct (ctx
->record_type
, field
);
1089 if (ctx
->srecord_type
)
1091 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1092 FIELD_DECL
, DECL_NAME (var
), type
);
1093 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1094 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1095 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1096 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1097 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1102 if (ctx
->srecord_type
== NULL_TREE
)
1106 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1107 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1108 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1110 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1111 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1112 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1113 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1114 splay_tree_insert (ctx
->sfield_map
,
1115 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1116 (splay_tree_value
) sfield
);
1120 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1121 : ctx
->srecord_type
, field
);
1125 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) var
,
1126 (splay_tree_value
) field
);
1127 if ((mask
& 2) && ctx
->sfield_map
)
1128 splay_tree_insert (ctx
->sfield_map
, (splay_tree_key
) var
,
1129 (splay_tree_value
) sfield
);
1133 install_var_local (tree var
, omp_context
*ctx
)
1135 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1136 insert_decl_map (&ctx
->cb
, var
, new_var
);
1140 /* Adjust the replacement for DECL in CTX for the new context. This means
1141 copying the DECL_VALUE_EXPR, and fixing up the type. */
1144 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1146 tree new_decl
, size
;
1148 new_decl
= lookup_decl (decl
, ctx
);
1150 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1152 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1153 && DECL_HAS_VALUE_EXPR_P (decl
))
1155 tree ve
= DECL_VALUE_EXPR (decl
);
1156 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1157 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1158 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1161 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1163 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1164 if (size
== error_mark_node
)
1165 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1166 DECL_SIZE (new_decl
) = size
;
1168 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1169 if (size
== error_mark_node
)
1170 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1171 DECL_SIZE_UNIT (new_decl
) = size
;
1175 /* The callback for remap_decl. Search all containing contexts for a
1176 mapping of the variable; this avoids having to duplicate the splay
1177 tree ahead of time. We know a mapping doesn't already exist in the
1178 given context. Create new mappings to implement default semantics. */
1181 omp_copy_decl (tree var
, copy_body_data
*cb
)
1183 omp_context
*ctx
= (omp_context
*) cb
;
1186 if (TREE_CODE (var
) == LABEL_DECL
)
1188 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1189 DECL_CONTEXT (new_var
) = current_function_decl
;
1190 insert_decl_map (&ctx
->cb
, var
, new_var
);
1194 while (!is_taskreg_ctx (ctx
))
1199 new_var
= maybe_lookup_decl (var
, ctx
);
1204 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1207 return error_mark_node
;
1211 /* Return the parallel region associated with STMT. */
1213 /* Debugging dumps for parallel regions. */
1214 void dump_omp_region (FILE *, struct omp_region
*, int);
1215 void debug_omp_region (struct omp_region
*);
1216 void debug_all_omp_regions (void);
1218 /* Dump the parallel region tree rooted at REGION. */
1221 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1223 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1224 gimple_code_name
[region
->type
]);
1227 dump_omp_region (file
, region
->inner
, indent
+ 4);
1231 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1232 region
->cont
->index
);
1236 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1237 region
->exit
->index
);
1239 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1242 dump_omp_region (file
, region
->next
, indent
);
1246 debug_omp_region (struct omp_region
*region
)
1248 dump_omp_region (stderr
, region
, 0);
1252 debug_all_omp_regions (void)
1254 dump_omp_region (stderr
, root_omp_region
, 0);
1258 /* Create a new parallel region starting at STMT inside region PARENT. */
1260 static struct omp_region
*
1261 new_omp_region (basic_block bb
, enum gimple_code type
,
1262 struct omp_region
*parent
)
1264 struct omp_region
*region
= XCNEW (struct omp_region
);
1266 region
->outer
= parent
;
1268 region
->type
= type
;
1272 /* This is a nested region. Add it to the list of inner
1273 regions in PARENT. */
1274 region
->next
= parent
->inner
;
1275 parent
->inner
= region
;
1279 /* This is a toplevel region. Add it to the list of toplevel
1280 regions in ROOT_OMP_REGION. */
1281 region
->next
= root_omp_region
;
1282 root_omp_region
= region
;
1288 /* Release the memory associated with the region tree rooted at REGION. */
1291 free_omp_region_1 (struct omp_region
*region
)
1293 struct omp_region
*i
, *n
;
1295 for (i
= region
->inner
; i
; i
= n
)
1298 free_omp_region_1 (i
);
1304 /* Release the memory for the entire omp region tree. */
1307 free_omp_regions (void)
1309 struct omp_region
*r
, *n
;
1310 for (r
= root_omp_region
; r
; r
= n
)
1313 free_omp_region_1 (r
);
1315 root_omp_region
= NULL
;
1319 /* Create a new context, with OUTER_CTX being the surrounding context. */
1321 static omp_context
*
1322 new_omp_context (gimple stmt
, omp_context
*outer_ctx
)
1324 omp_context
*ctx
= XCNEW (omp_context
);
1326 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1327 (splay_tree_value
) ctx
);
1332 ctx
->outer
= outer_ctx
;
1333 ctx
->cb
= outer_ctx
->cb
;
1334 ctx
->cb
.block
= NULL
;
1335 ctx
->depth
= outer_ctx
->depth
+ 1;
1339 ctx
->cb
.src_fn
= current_function_decl
;
1340 ctx
->cb
.dst_fn
= current_function_decl
;
1341 ctx
->cb
.src_node
= cgraph_get_node (current_function_decl
);
1342 gcc_checking_assert (ctx
->cb
.src_node
);
1343 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1344 ctx
->cb
.src_cfun
= cfun
;
1345 ctx
->cb
.copy_decl
= omp_copy_decl
;
1346 ctx
->cb
.eh_lp_nr
= 0;
1347 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1351 ctx
->cb
.decl_map
= pointer_map_create ();
1356 static gimple_seq
maybe_catch_exception (gimple_seq
);
1358 /* Finalize task copyfn. */
1361 finalize_task_copyfn (gimple task_stmt
)
1363 struct function
*child_cfun
;
1365 gimple_seq seq
= NULL
, new_seq
;
1368 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1369 if (child_fn
== NULL_TREE
)
1372 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1373 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1375 push_cfun (child_cfun
);
1376 bind
= gimplify_body (child_fn
, false);
1377 gimple_seq_add_stmt (&seq
, bind
);
1378 new_seq
= maybe_catch_exception (seq
);
1381 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1383 gimple_seq_add_stmt (&seq
, bind
);
1385 gimple_set_body (child_fn
, seq
);
1388 /* Inform the callgraph about the new function. */
1389 cgraph_add_new_function (child_fn
, false);
1392 /* Destroy a omp_context data structures. Called through the splay tree
1393 value delete callback. */
1396 delete_omp_context (splay_tree_value value
)
1398 omp_context
*ctx
= (omp_context
*) value
;
1400 pointer_map_destroy (ctx
->cb
.decl_map
);
1403 splay_tree_delete (ctx
->field_map
);
1404 if (ctx
->sfield_map
)
1405 splay_tree_delete (ctx
->sfield_map
);
1407 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1408 it produces corrupt debug information. */
1409 if (ctx
->record_type
)
1412 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1413 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1415 if (ctx
->srecord_type
)
1418 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1419 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1422 if (is_task_ctx (ctx
))
1423 finalize_task_copyfn (ctx
->stmt
);
1428 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1432 fixup_child_record_type (omp_context
*ctx
)
1434 tree f
, type
= ctx
->record_type
;
1436 /* ??? It isn't sufficient to just call remap_type here, because
1437 variably_modified_type_p doesn't work the way we expect for
1438 record types. Testing each field for whether it needs remapping
1439 and creating a new record by hand works, however. */
1440 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1441 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1445 tree name
, new_fields
= NULL
;
1447 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1448 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1449 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1450 TYPE_DECL
, name
, type
);
1451 TYPE_NAME (type
) = name
;
1453 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1455 tree new_f
= copy_node (f
);
1456 DECL_CONTEXT (new_f
) = type
;
1457 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1458 DECL_CHAIN (new_f
) = new_fields
;
1459 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1460 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1462 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1466 /* Arrange to be able to look up the receiver field
1467 given the sender field. */
1468 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1469 (splay_tree_value
) new_f
);
1471 TYPE_FIELDS (type
) = nreverse (new_fields
);
1475 TREE_TYPE (ctx
->receiver_decl
) = build_pointer_type (type
);
1478 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1479 specified by CLAUSES. */
1482 scan_sharing_clauses (tree clauses
, omp_context
*ctx
)
1485 bool scan_array_reductions
= false;
1487 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1491 switch (OMP_CLAUSE_CODE (c
))
1493 case OMP_CLAUSE_PRIVATE
:
1494 decl
= OMP_CLAUSE_DECL (c
);
1495 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1497 else if (!is_variable_sized (decl
))
1498 install_var_local (decl
, ctx
);
1501 case OMP_CLAUSE_SHARED
:
1502 /* Ignore shared directives in teams construct. */
1503 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1505 gcc_assert (is_taskreg_ctx (ctx
));
1506 decl
= OMP_CLAUSE_DECL (c
);
1507 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1508 || !is_variable_sized (decl
));
1509 /* Global variables don't need to be copied,
1510 the receiver side will use them directly. */
1511 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1513 by_ref
= use_pointer_for_field (decl
, ctx
);
1514 if (! TREE_READONLY (decl
)
1515 || TREE_ADDRESSABLE (decl
)
1517 || is_reference (decl
))
1519 install_var_field (decl
, by_ref
, 3, ctx
);
1520 install_var_local (decl
, ctx
);
1523 /* We don't need to copy const scalar vars back. */
1524 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1527 case OMP_CLAUSE_LASTPRIVATE
:
1528 /* Let the corresponding firstprivate clause create
1530 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1534 case OMP_CLAUSE_FIRSTPRIVATE
:
1535 case OMP_CLAUSE_REDUCTION
:
1536 case OMP_CLAUSE_LINEAR
:
1537 decl
= OMP_CLAUSE_DECL (c
);
1539 if (is_variable_sized (decl
))
1541 if (is_task_ctx (ctx
))
1542 install_var_field (decl
, false, 1, ctx
);
1545 else if (is_taskreg_ctx (ctx
))
1548 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1549 by_ref
= use_pointer_for_field (decl
, NULL
);
1551 if (is_task_ctx (ctx
)
1552 && (global
|| by_ref
|| is_reference (decl
)))
1554 install_var_field (decl
, false, 1, ctx
);
1556 install_var_field (decl
, by_ref
, 2, ctx
);
1559 install_var_field (decl
, by_ref
, 3, ctx
);
1561 install_var_local (decl
, ctx
);
1564 case OMP_CLAUSE__LOOPTEMP_
:
1565 gcc_assert (is_parallel_ctx (ctx
));
1566 decl
= OMP_CLAUSE_DECL (c
);
1567 install_var_field (decl
, false, 3, ctx
);
1568 install_var_local (decl
, ctx
);
1571 case OMP_CLAUSE_COPYPRIVATE
:
1572 case OMP_CLAUSE_COPYIN
:
1573 decl
= OMP_CLAUSE_DECL (c
);
1574 by_ref
= use_pointer_for_field (decl
, NULL
);
1575 install_var_field (decl
, by_ref
, 3, ctx
);
1578 case OMP_CLAUSE_DEFAULT
:
1579 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
1582 case OMP_CLAUSE_FINAL
:
1584 case OMP_CLAUSE_NUM_THREADS
:
1585 case OMP_CLAUSE_NUM_TEAMS
:
1586 case OMP_CLAUSE_THREAD_LIMIT
:
1587 case OMP_CLAUSE_DEVICE
:
1588 case OMP_CLAUSE_SCHEDULE
:
1589 case OMP_CLAUSE_DIST_SCHEDULE
:
1590 case OMP_CLAUSE_DEPEND
:
1592 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
1596 case OMP_CLAUSE_FROM
:
1597 case OMP_CLAUSE_MAP
:
1599 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
1600 decl
= OMP_CLAUSE_DECL (c
);
1601 /* Global variables with "omp declare target" attribute
1602 don't need to be copied, the receiver side will use them
1604 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1606 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1607 && lookup_attribute ("omp declare target",
1608 DECL_ATTRIBUTES (decl
)))
1610 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1611 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
)
1613 /* Ignore OMP_CLAUSE_MAP_POINTER kind for arrays in
1614 #pragma omp target data, there is nothing to map for
1616 if (gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_DATA
1617 && !POINTER_TYPE_P (TREE_TYPE (decl
)))
1622 if (DECL_SIZE (decl
)
1623 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1625 tree decl2
= DECL_VALUE_EXPR (decl
);
1626 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1627 decl2
= TREE_OPERAND (decl2
, 0);
1628 gcc_assert (DECL_P (decl2
));
1629 install_var_field (decl2
, true, 3, ctx
);
1630 install_var_local (decl2
, ctx
);
1631 install_var_local (decl
, ctx
);
1635 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1636 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1637 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
1638 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1639 install_var_field (decl
, true, 7, ctx
);
1641 install_var_field (decl
, true, 3, ctx
);
1642 if (gimple_omp_target_kind (ctx
->stmt
)
1643 == GF_OMP_TARGET_KIND_REGION
)
1644 install_var_local (decl
, ctx
);
1649 tree base
= get_base_address (decl
);
1650 tree nc
= OMP_CLAUSE_CHAIN (c
);
1653 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
1654 && OMP_CLAUSE_DECL (nc
) == base
1655 && OMP_CLAUSE_MAP_KIND (nc
) == OMP_CLAUSE_MAP_POINTER
1656 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
1658 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
1659 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
1663 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
1664 (splay_tree_key
) decl
));
1666 = build_decl (OMP_CLAUSE_LOCATION (c
),
1667 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
1668 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
1669 insert_field_into_struct (ctx
->record_type
, field
);
1670 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
1671 (splay_tree_value
) field
);
1676 case OMP_CLAUSE_NOWAIT
:
1677 case OMP_CLAUSE_ORDERED
:
1678 case OMP_CLAUSE_COLLAPSE
:
1679 case OMP_CLAUSE_UNTIED
:
1680 case OMP_CLAUSE_MERGEABLE
:
1681 case OMP_CLAUSE_PROC_BIND
:
1682 case OMP_CLAUSE_SAFELEN
:
1685 case OMP_CLAUSE_ALIGNED
:
1686 decl
= OMP_CLAUSE_DECL (c
);
1687 if (is_global_var (decl
)
1688 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1689 install_var_local (decl
, ctx
);
1697 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1699 switch (OMP_CLAUSE_CODE (c
))
1701 case OMP_CLAUSE_LASTPRIVATE
:
1702 /* Let the corresponding firstprivate clause create
1704 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1705 scan_array_reductions
= true;
1706 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1710 case OMP_CLAUSE_PRIVATE
:
1711 case OMP_CLAUSE_FIRSTPRIVATE
:
1712 case OMP_CLAUSE_REDUCTION
:
1713 case OMP_CLAUSE_LINEAR
:
1714 decl
= OMP_CLAUSE_DECL (c
);
1715 if (is_variable_sized (decl
))
1716 install_var_local (decl
, ctx
);
1717 fixup_remapped_decl (decl
, ctx
,
1718 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
1719 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
1720 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1721 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1722 scan_array_reductions
= true;
1725 case OMP_CLAUSE_SHARED
:
1726 /* Ignore shared directives in teams construct. */
1727 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1729 decl
= OMP_CLAUSE_DECL (c
);
1730 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1731 fixup_remapped_decl (decl
, ctx
, false);
1734 case OMP_CLAUSE_MAP
:
1735 if (gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_DATA
)
1737 decl
= OMP_CLAUSE_DECL (c
);
1739 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1740 && lookup_attribute ("omp declare target",
1741 DECL_ATTRIBUTES (decl
)))
1745 if (OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1746 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
1747 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
1749 tree new_decl
= lookup_decl (decl
, ctx
);
1750 TREE_TYPE (new_decl
)
1751 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1753 else if (DECL_SIZE (decl
)
1754 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1756 tree decl2
= DECL_VALUE_EXPR (decl
);
1757 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1758 decl2
= TREE_OPERAND (decl2
, 0);
1759 gcc_assert (DECL_P (decl2
));
1760 fixup_remapped_decl (decl2
, ctx
, false);
1761 fixup_remapped_decl (decl
, ctx
, true);
1764 fixup_remapped_decl (decl
, ctx
, false);
1768 case OMP_CLAUSE_COPYPRIVATE
:
1769 case OMP_CLAUSE_COPYIN
:
1770 case OMP_CLAUSE_DEFAULT
:
1772 case OMP_CLAUSE_NUM_THREADS
:
1773 case OMP_CLAUSE_NUM_TEAMS
:
1774 case OMP_CLAUSE_THREAD_LIMIT
:
1775 case OMP_CLAUSE_DEVICE
:
1776 case OMP_CLAUSE_SCHEDULE
:
1777 case OMP_CLAUSE_DIST_SCHEDULE
:
1778 case OMP_CLAUSE_NOWAIT
:
1779 case OMP_CLAUSE_ORDERED
:
1780 case OMP_CLAUSE_COLLAPSE
:
1781 case OMP_CLAUSE_UNTIED
:
1782 case OMP_CLAUSE_FINAL
:
1783 case OMP_CLAUSE_MERGEABLE
:
1784 case OMP_CLAUSE_PROC_BIND
:
1785 case OMP_CLAUSE_SAFELEN
:
1786 case OMP_CLAUSE_ALIGNED
:
1787 case OMP_CLAUSE_DEPEND
:
1788 case OMP_CLAUSE__LOOPTEMP_
:
1790 case OMP_CLAUSE_FROM
:
1798 if (scan_array_reductions
)
1799 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1800 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1801 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1803 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
1804 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
1806 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
1807 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1808 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
1811 /* Create a new name for omp child function. Returns an identifier. */
1813 static GTY(()) unsigned int tmp_ompfn_id_num
;
1816 create_omp_child_function_name (bool task_copy
)
1818 return (clone_function_name (current_function_decl
,
1819 task_copy
? "_omp_cpyfn" : "_omp_fn"));
1822 /* Build a decl for the omp child function. It'll not contain a body
1823 yet, just the bare decl. */
1826 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
1828 tree decl
, type
, name
, t
;
1830 name
= create_omp_child_function_name (task_copy
);
1832 type
= build_function_type_list (void_type_node
, ptr_type_node
,
1833 ptr_type_node
, NULL_TREE
);
1835 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
1837 decl
= build_decl (gimple_location (ctx
->stmt
),
1838 FUNCTION_DECL
, name
, type
);
1841 ctx
->cb
.dst_fn
= decl
;
1843 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
1845 TREE_STATIC (decl
) = 1;
1846 TREE_USED (decl
) = 1;
1847 DECL_ARTIFICIAL (decl
) = 1;
1848 DECL_NAMELESS (decl
) = 1;
1849 DECL_IGNORED_P (decl
) = 0;
1850 TREE_PUBLIC (decl
) = 0;
1851 DECL_UNINLINABLE (decl
) = 1;
1852 DECL_EXTERNAL (decl
) = 0;
1853 DECL_CONTEXT (decl
) = NULL_TREE
;
1854 DECL_INITIAL (decl
) = make_node (BLOCK
);
1855 bool target_p
= false;
1856 if (lookup_attribute ("omp declare target",
1857 DECL_ATTRIBUTES (current_function_decl
)))
1862 for (octx
= ctx
; octx
; octx
= octx
->outer
)
1863 if (gimple_code (octx
->stmt
) == GIMPLE_OMP_TARGET
1864 && gimple_omp_target_kind (octx
->stmt
)
1865 == GF_OMP_TARGET_KIND_REGION
)
1872 DECL_ATTRIBUTES (decl
)
1873 = tree_cons (get_identifier ("omp declare target"),
1874 NULL_TREE
, DECL_ATTRIBUTES (decl
));
1876 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1877 RESULT_DECL
, NULL_TREE
, void_type_node
);
1878 DECL_ARTIFICIAL (t
) = 1;
1879 DECL_IGNORED_P (t
) = 1;
1880 DECL_CONTEXT (t
) = decl
;
1881 DECL_RESULT (decl
) = t
;
1883 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1884 PARM_DECL
, get_identifier (".omp_data_i"), ptr_type_node
);
1885 DECL_ARTIFICIAL (t
) = 1;
1886 DECL_NAMELESS (t
) = 1;
1887 DECL_ARG_TYPE (t
) = ptr_type_node
;
1888 DECL_CONTEXT (t
) = current_function_decl
;
1890 DECL_ARGUMENTS (decl
) = t
;
1892 ctx
->receiver_decl
= t
;
1895 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1896 PARM_DECL
, get_identifier (".omp_data_o"),
1898 DECL_ARTIFICIAL (t
) = 1;
1899 DECL_NAMELESS (t
) = 1;
1900 DECL_ARG_TYPE (t
) = ptr_type_node
;
1901 DECL_CONTEXT (t
) = current_function_decl
;
1903 TREE_ADDRESSABLE (t
) = 1;
1904 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
1905 DECL_ARGUMENTS (decl
) = t
;
1908 /* Allocate memory for the function structure. The call to
1909 allocate_struct_function clobbers CFUN, so we need to restore
1911 push_struct_function (decl
);
1912 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
1916 /* Callback for walk_gimple_seq. Check if combined parallel
1917 contains gimple_omp_for_combined_into_p OMP_FOR. */
1920 find_combined_for (gimple_stmt_iterator
*gsi_p
,
1921 bool *handled_ops_p
,
1922 struct walk_stmt_info
*wi
)
1924 gimple stmt
= gsi_stmt (*gsi_p
);
1926 *handled_ops_p
= true;
1927 switch (gimple_code (stmt
))
1931 case GIMPLE_OMP_FOR
:
1932 if (gimple_omp_for_combined_into_p (stmt
)
1933 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
1936 return integer_zero_node
;
1945 /* Scan an OpenMP parallel directive. */
1948 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
1952 gimple stmt
= gsi_stmt (*gsi
);
1954 /* Ignore parallel directives with empty bodies, unless there
1955 are copyin clauses. */
1957 && empty_body_p (gimple_omp_body (stmt
))
1958 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
1959 OMP_CLAUSE_COPYIN
) == NULL
)
1961 gsi_replace (gsi
, gimple_build_nop (), false);
1965 if (gimple_omp_parallel_combined_p (stmt
))
1968 struct walk_stmt_info wi
;
1970 memset (&wi
, 0, sizeof (wi
));
1972 walk_gimple_seq (gimple_omp_body (stmt
),
1973 find_combined_for
, NULL
, &wi
);
1974 for_stmt
= (gimple
) wi
.info
;
1977 struct omp_for_data fd
;
1978 extract_omp_for_data (for_stmt
, &fd
, NULL
);
1979 /* We need two temporaries with fd.loop.v type (istart/iend)
1980 and then (fd.collapse - 1) temporaries with the same
1981 type for count2 ... countN-1 vars if not constant. */
1982 size_t count
= 2, i
;
1983 tree type
= fd
.iter_type
;
1985 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
1986 count
+= fd
.collapse
- 1;
1987 for (i
= 0; i
< count
; i
++)
1989 tree temp
= create_tmp_var (type
, NULL
);
1990 tree c
= build_omp_clause (UNKNOWN_LOCATION
,
1991 OMP_CLAUSE__LOOPTEMP_
);
1992 OMP_CLAUSE_DECL (c
) = temp
;
1993 OMP_CLAUSE_CHAIN (c
) = gimple_omp_parallel_clauses (stmt
);
1994 gimple_omp_parallel_set_clauses (stmt
, c
);
1999 ctx
= new_omp_context (stmt
, outer_ctx
);
2000 if (taskreg_nesting_level
> 1)
2001 ctx
->is_nested
= true;
2002 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2003 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2004 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2005 name
= create_tmp_var_name (".omp_data_s");
2006 name
= build_decl (gimple_location (stmt
),
2007 TYPE_DECL
, name
, ctx
->record_type
);
2008 DECL_ARTIFICIAL (name
) = 1;
2009 DECL_NAMELESS (name
) = 1;
2010 TYPE_NAME (ctx
->record_type
) = name
;
2011 create_omp_child_function (ctx
, false);
2012 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2014 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2015 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2017 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2018 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2021 layout_type (ctx
->record_type
);
2022 fixup_child_record_type (ctx
);
2026 /* Scan an OpenMP task directive. */
2029 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2033 gimple stmt
= gsi_stmt (*gsi
);
2034 location_t loc
= gimple_location (stmt
);
2036 /* Ignore task directives with empty bodies. */
2038 && empty_body_p (gimple_omp_body (stmt
)))
2040 gsi_replace (gsi
, gimple_build_nop (), false);
2044 ctx
= new_omp_context (stmt
, outer_ctx
);
2045 if (taskreg_nesting_level
> 1)
2046 ctx
->is_nested
= true;
2047 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2048 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2049 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2050 name
= create_tmp_var_name (".omp_data_s");
2051 name
= build_decl (gimple_location (stmt
),
2052 TYPE_DECL
, name
, ctx
->record_type
);
2053 DECL_ARTIFICIAL (name
) = 1;
2054 DECL_NAMELESS (name
) = 1;
2055 TYPE_NAME (ctx
->record_type
) = name
;
2056 create_omp_child_function (ctx
, false);
2057 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2059 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2061 if (ctx
->srecord_type
)
2063 name
= create_tmp_var_name (".omp_data_a");
2064 name
= build_decl (gimple_location (stmt
),
2065 TYPE_DECL
, name
, ctx
->srecord_type
);
2066 DECL_ARTIFICIAL (name
) = 1;
2067 DECL_NAMELESS (name
) = 1;
2068 TYPE_NAME (ctx
->srecord_type
) = name
;
2069 create_omp_child_function (ctx
, true);
2072 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2074 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2076 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2077 t
= build_int_cst (long_integer_type_node
, 0);
2078 gimple_omp_task_set_arg_size (stmt
, t
);
2079 t
= build_int_cst (long_integer_type_node
, 1);
2080 gimple_omp_task_set_arg_align (stmt
, t
);
2084 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2085 /* Move VLA fields to the end. */
2086 p
= &TYPE_FIELDS (ctx
->record_type
);
2088 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2089 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2092 *p
= TREE_CHAIN (*p
);
2093 TREE_CHAIN (*q
) = NULL_TREE
;
2094 q
= &TREE_CHAIN (*q
);
2097 p
= &DECL_CHAIN (*p
);
2099 layout_type (ctx
->record_type
);
2100 fixup_child_record_type (ctx
);
2101 if (ctx
->srecord_type
)
2102 layout_type (ctx
->srecord_type
);
2103 t
= fold_convert_loc (loc
, long_integer_type_node
,
2104 TYPE_SIZE_UNIT (ctx
->record_type
));
2105 gimple_omp_task_set_arg_size (stmt
, t
);
2106 t
= build_int_cst (long_integer_type_node
,
2107 TYPE_ALIGN_UNIT (ctx
->record_type
));
2108 gimple_omp_task_set_arg_align (stmt
, t
);
2113 /* Scan an OpenMP loop directive. */
2116 scan_omp_for (gimple stmt
, omp_context
*outer_ctx
)
2121 ctx
= new_omp_context (stmt
, outer_ctx
);
2123 scan_sharing_clauses (gimple_omp_for_clauses (stmt
), ctx
);
2125 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
2126 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
2128 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
2129 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
2130 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
2131 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
2133 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2136 /* Scan an OpenMP sections directive. */
2139 scan_omp_sections (gimple stmt
, omp_context
*outer_ctx
)
2143 ctx
= new_omp_context (stmt
, outer_ctx
);
2144 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
2145 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2148 /* Scan an OpenMP single directive. */
2151 scan_omp_single (gimple stmt
, omp_context
*outer_ctx
)
2156 ctx
= new_omp_context (stmt
, outer_ctx
);
2157 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2158 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2159 name
= create_tmp_var_name (".omp_copy_s");
2160 name
= build_decl (gimple_location (stmt
),
2161 TYPE_DECL
, name
, ctx
->record_type
);
2162 TYPE_NAME (ctx
->record_type
) = name
;
2164 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
2165 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2167 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2168 ctx
->record_type
= NULL
;
2170 layout_type (ctx
->record_type
);
2173 /* Scan an OpenMP target{, data, update} directive. */
2176 scan_omp_target (gimple stmt
, omp_context
*outer_ctx
)
2180 int kind
= gimple_omp_target_kind (stmt
);
2182 ctx
= new_omp_context (stmt
, outer_ctx
);
2183 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2184 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2185 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2186 name
= create_tmp_var_name (".omp_data_t");
2187 name
= build_decl (gimple_location (stmt
),
2188 TYPE_DECL
, name
, ctx
->record_type
);
2189 DECL_ARTIFICIAL (name
) = 1;
2190 DECL_NAMELESS (name
) = 1;
2191 TYPE_NAME (ctx
->record_type
) = name
;
2192 if (kind
== GF_OMP_TARGET_KIND_REGION
)
2194 create_omp_child_function (ctx
, false);
2195 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2198 scan_sharing_clauses (gimple_omp_target_clauses (stmt
), ctx
);
2199 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2201 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2202 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2205 TYPE_FIELDS (ctx
->record_type
)
2206 = nreverse (TYPE_FIELDS (ctx
->record_type
));
2207 #ifdef ENABLE_CHECKING
2209 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
2210 for (field
= TYPE_FIELDS (ctx
->record_type
);
2212 field
= DECL_CHAIN (field
))
2213 gcc_assert (DECL_ALIGN (field
) == align
);
2215 layout_type (ctx
->record_type
);
2216 if (kind
== GF_OMP_TARGET_KIND_REGION
)
2217 fixup_child_record_type (ctx
);
2221 /* Scan an OpenMP teams directive. */
2224 scan_omp_teams (gimple stmt
, omp_context
*outer_ctx
)
2226 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
2227 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
2228 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2231 /* Check OpenMP nesting restrictions. */
2233 check_omp_nesting_restrictions (gimple stmt
, omp_context
*ctx
)
2237 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2238 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
)
2240 error_at (gimple_location (stmt
),
2241 "OpenMP constructs may not be nested inside simd region");
2244 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2246 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
2247 || (gimple_omp_for_kind (stmt
)
2248 != GF_OMP_FOR_KIND_DISTRIBUTE
))
2249 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
2251 error_at (gimple_location (stmt
),
2252 "only distribute or parallel constructs are allowed to "
2253 "be closely nested inside teams construct");
2258 switch (gimple_code (stmt
))
2260 case GIMPLE_OMP_FOR
:
2261 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_KIND_SIMD
)
2263 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
2265 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
2267 error_at (gimple_location (stmt
),
2268 "distribute construct must be closely nested inside "
2276 if (is_gimple_call (stmt
)
2277 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2278 == BUILT_IN_GOMP_CANCEL
2279 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2280 == BUILT_IN_GOMP_CANCELLATION_POINT
))
2282 const char *bad
= NULL
;
2283 const char *kind
= NULL
;
2286 error_at (gimple_location (stmt
), "orphaned %qs construct",
2287 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2288 == BUILT_IN_GOMP_CANCEL
2289 ? "#pragma omp cancel"
2290 : "#pragma omp cancellation point");
2293 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
2294 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
2298 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
2299 bad
= "#pragma omp parallel";
2300 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2301 == BUILT_IN_GOMP_CANCEL
2302 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2303 ctx
->cancellable
= true;
2307 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
2308 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
2309 bad
= "#pragma omp for";
2310 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2311 == BUILT_IN_GOMP_CANCEL
2312 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2314 ctx
->cancellable
= true;
2315 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2317 warning_at (gimple_location (stmt
), 0,
2318 "%<#pragma omp cancel for%> inside "
2319 "%<nowait%> for construct");
2320 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2321 OMP_CLAUSE_ORDERED
))
2322 warning_at (gimple_location (stmt
), 0,
2323 "%<#pragma omp cancel for%> inside "
2324 "%<ordered%> for construct");
2329 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
2330 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
2331 bad
= "#pragma omp sections";
2332 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2333 == BUILT_IN_GOMP_CANCEL
2334 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2336 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
2338 ctx
->cancellable
= true;
2339 if (find_omp_clause (gimple_omp_sections_clauses
2342 warning_at (gimple_location (stmt
), 0,
2343 "%<#pragma omp cancel sections%> inside "
2344 "%<nowait%> sections construct");
2348 gcc_assert (ctx
->outer
2349 && gimple_code (ctx
->outer
->stmt
)
2350 == GIMPLE_OMP_SECTIONS
);
2351 ctx
->outer
->cancellable
= true;
2352 if (find_omp_clause (gimple_omp_sections_clauses
2355 warning_at (gimple_location (stmt
), 0,
2356 "%<#pragma omp cancel sections%> inside "
2357 "%<nowait%> sections construct");
2363 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
2364 bad
= "#pragma omp task";
2366 ctx
->cancellable
= true;
2370 error_at (gimple_location (stmt
), "invalid arguments");
2375 error_at (gimple_location (stmt
),
2376 "%<%s %s%> construct not closely nested inside of %qs",
2377 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2378 == BUILT_IN_GOMP_CANCEL
2379 ? "#pragma omp cancel"
2380 : "#pragma omp cancellation point", kind
, bad
);
2385 case GIMPLE_OMP_SECTIONS
:
2386 case GIMPLE_OMP_SINGLE
:
2387 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2388 switch (gimple_code (ctx
->stmt
))
2390 case GIMPLE_OMP_FOR
:
2391 case GIMPLE_OMP_SECTIONS
:
2392 case GIMPLE_OMP_SINGLE
:
2393 case GIMPLE_OMP_ORDERED
:
2394 case GIMPLE_OMP_MASTER
:
2395 case GIMPLE_OMP_TASK
:
2396 case GIMPLE_OMP_CRITICAL
:
2397 if (is_gimple_call (stmt
))
2399 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2400 != BUILT_IN_GOMP_BARRIER
)
2402 error_at (gimple_location (stmt
),
2403 "barrier region may not be closely nested inside "
2404 "of work-sharing, critical, ordered, master or "
2405 "explicit task region");
2408 error_at (gimple_location (stmt
),
2409 "work-sharing region may not be closely nested inside "
2410 "of work-sharing, critical, ordered, master or explicit "
2413 case GIMPLE_OMP_PARALLEL
:
2419 case GIMPLE_OMP_MASTER
:
2420 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2421 switch (gimple_code (ctx
->stmt
))
2423 case GIMPLE_OMP_FOR
:
2424 case GIMPLE_OMP_SECTIONS
:
2425 case GIMPLE_OMP_SINGLE
:
2426 case GIMPLE_OMP_TASK
:
2427 error_at (gimple_location (stmt
),
2428 "master region may not be closely nested inside "
2429 "of work-sharing or explicit task region");
2431 case GIMPLE_OMP_PARALLEL
:
2437 case GIMPLE_OMP_ORDERED
:
2438 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2439 switch (gimple_code (ctx
->stmt
))
2441 case GIMPLE_OMP_CRITICAL
:
2442 case GIMPLE_OMP_TASK
:
2443 error_at (gimple_location (stmt
),
2444 "ordered region may not be closely nested inside "
2445 "of critical or explicit task region");
2447 case GIMPLE_OMP_FOR
:
2448 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2449 OMP_CLAUSE_ORDERED
) == NULL
)
2451 error_at (gimple_location (stmt
),
2452 "ordered region must be closely nested inside "
2453 "a loop region with an ordered clause");
2457 case GIMPLE_OMP_PARALLEL
:
2458 error_at (gimple_location (stmt
),
2459 "ordered region must be closely nested inside "
2460 "a loop region with an ordered clause");
2466 case GIMPLE_OMP_CRITICAL
:
2467 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2468 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_CRITICAL
2469 && (gimple_omp_critical_name (stmt
)
2470 == gimple_omp_critical_name (ctx
->stmt
)))
2472 error_at (gimple_location (stmt
),
2473 "critical region may not be nested inside a critical "
2474 "region with the same name");
2478 case GIMPLE_OMP_TEAMS
:
2480 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
2481 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
2483 error_at (gimple_location (stmt
),
2484 "teams construct not closely nested inside of target "
2496 /* Helper function scan_omp.
2498 Callback for walk_tree or operators in walk_gimple_stmt used to
2499 scan for OpenMP directives in TP. */
2502 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
2504 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
2505 omp_context
*ctx
= (omp_context
*) wi
->info
;
2508 switch (TREE_CODE (t
))
2515 *tp
= remap_decl (t
, &ctx
->cb
);
2519 if (ctx
&& TYPE_P (t
))
2520 *tp
= remap_type (t
, &ctx
->cb
);
2521 else if (!DECL_P (t
))
2526 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
2527 if (tem
!= TREE_TYPE (t
))
2529 if (TREE_CODE (t
) == INTEGER_CST
)
2530 *tp
= build_int_cst_wide (tem
,
2531 TREE_INT_CST_LOW (t
),
2532 TREE_INT_CST_HIGH (t
));
2534 TREE_TYPE (t
) = tem
;
2544 /* Return true if FNDECL is a setjmp or a longjmp. */
2547 setjmp_or_longjmp_p (const_tree fndecl
)
2549 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
2550 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
2551 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
2554 tree declname
= DECL_NAME (fndecl
);
2557 const char *name
= IDENTIFIER_POINTER (declname
);
2558 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
2562 /* Helper function for scan_omp.
2564 Callback for walk_gimple_stmt used to scan for OpenMP directives in
2565 the current statement in GSI. */
2568 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
2569 struct walk_stmt_info
*wi
)
2571 gimple stmt
= gsi_stmt (*gsi
);
2572 omp_context
*ctx
= (omp_context
*) wi
->info
;
2574 if (gimple_has_location (stmt
))
2575 input_location
= gimple_location (stmt
);
2577 /* Check the OpenMP nesting restrictions. */
2578 bool remove
= false;
2579 if (is_gimple_omp (stmt
))
2580 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
2581 else if (is_gimple_call (stmt
))
2583 tree fndecl
= gimple_call_fndecl (stmt
);
2586 if (setjmp_or_longjmp_p (fndecl
)
2588 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2589 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
)
2592 error_at (gimple_location (stmt
),
2593 "setjmp/longjmp inside simd construct");
2595 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
2596 switch (DECL_FUNCTION_CODE (fndecl
))
2598 case BUILT_IN_GOMP_BARRIER
:
2599 case BUILT_IN_GOMP_CANCEL
:
2600 case BUILT_IN_GOMP_CANCELLATION_POINT
:
2601 case BUILT_IN_GOMP_TASKYIELD
:
2602 case BUILT_IN_GOMP_TASKWAIT
:
2603 case BUILT_IN_GOMP_TASKGROUP_START
:
2604 case BUILT_IN_GOMP_TASKGROUP_END
:
2605 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
2614 stmt
= gimple_build_nop ();
2615 gsi_replace (gsi
, stmt
, false);
2618 *handled_ops_p
= true;
2620 switch (gimple_code (stmt
))
2622 case GIMPLE_OMP_PARALLEL
:
2623 taskreg_nesting_level
++;
2624 scan_omp_parallel (gsi
, ctx
);
2625 taskreg_nesting_level
--;
2628 case GIMPLE_OMP_TASK
:
2629 taskreg_nesting_level
++;
2630 scan_omp_task (gsi
, ctx
);
2631 taskreg_nesting_level
--;
2634 case GIMPLE_OMP_FOR
:
2635 scan_omp_for (stmt
, ctx
);
2638 case GIMPLE_OMP_SECTIONS
:
2639 scan_omp_sections (stmt
, ctx
);
2642 case GIMPLE_OMP_SINGLE
:
2643 scan_omp_single (stmt
, ctx
);
2646 case GIMPLE_OMP_SECTION
:
2647 case GIMPLE_OMP_MASTER
:
2648 case GIMPLE_OMP_TASKGROUP
:
2649 case GIMPLE_OMP_ORDERED
:
2650 case GIMPLE_OMP_CRITICAL
:
2651 ctx
= new_omp_context (stmt
, ctx
);
2652 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2655 case GIMPLE_OMP_TARGET
:
2656 scan_omp_target (stmt
, ctx
);
2659 case GIMPLE_OMP_TEAMS
:
2660 scan_omp_teams (stmt
, ctx
);
2667 *handled_ops_p
= false;
2669 for (var
= gimple_bind_vars (stmt
); var
; var
= DECL_CHAIN (var
))
2670 insert_decl_map (&ctx
->cb
, var
, var
);
2674 *handled_ops_p
= false;
2682 /* Scan all the statements starting at the current statement. CTX
2683 contains context information about the OpenMP directives and
2684 clauses found during the scan. */
2687 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
2689 location_t saved_location
;
2690 struct walk_stmt_info wi
;
2692 memset (&wi
, 0, sizeof (wi
));
2694 wi
.want_locations
= true;
2696 saved_location
= input_location
;
2697 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
2698 input_location
= saved_location
;
2701 /* Re-gimplification and code generation routines. */
2703 /* Build a call to GOMP_barrier. */
2706 build_omp_barrier (tree lhs
)
2708 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
2709 : BUILT_IN_GOMP_BARRIER
);
2710 gimple g
= gimple_build_call (fndecl
, 0);
2712 gimple_call_set_lhs (g
, lhs
);
2716 /* If a context was created for STMT when it was scanned, return it. */
2718 static omp_context
*
2719 maybe_lookup_ctx (gimple stmt
)
2722 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
2723 return n
? (omp_context
*) n
->value
: NULL
;
2727 /* Find the mapping for DECL in CTX or the immediately enclosing
2728 context that has a mapping for DECL.
2730 If CTX is a nested parallel directive, we may have to use the decl
2731 mappings created in CTX's parent context. Suppose that we have the
2732 following parallel nesting (variable UIDs showed for clarity):
2735 #omp parallel shared(iD.1562) -> outer parallel
2736 iD.1562 = iD.1562 + 1;
2738 #omp parallel shared (iD.1562) -> inner parallel
2739 iD.1562 = iD.1562 - 1;
2741 Each parallel structure will create a distinct .omp_data_s structure
2742 for copying iD.1562 in/out of the directive:
2744 outer parallel .omp_data_s.1.i -> iD.1562
2745 inner parallel .omp_data_s.2.i -> iD.1562
2747 A shared variable mapping will produce a copy-out operation before
2748 the parallel directive and a copy-in operation after it. So, in
2749 this case we would have:
2752 .omp_data_o.1.i = iD.1562;
2753 #omp parallel shared(iD.1562) -> outer parallel
2754 .omp_data_i.1 = &.omp_data_o.1
2755 .omp_data_i.1->i = .omp_data_i.1->i + 1;
2757 .omp_data_o.2.i = iD.1562; -> **
2758 #omp parallel shared(iD.1562) -> inner parallel
2759 .omp_data_i.2 = &.omp_data_o.2
2760 .omp_data_i.2->i = .omp_data_i.2->i - 1;
2763 ** This is a problem. The symbol iD.1562 cannot be referenced
2764 inside the body of the outer parallel region. But since we are
2765 emitting this copy operation while expanding the inner parallel
2766 directive, we need to access the CTX structure of the outer
2767 parallel directive to get the correct mapping:
2769 .omp_data_o.2.i = .omp_data_i.1->i
2771 Since there may be other workshare or parallel directives enclosing
2772 the parallel directive, it may be necessary to walk up the context
2773 parent chain. This is not a problem in general because nested
2774 parallelism happens only rarely. */
2777 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
2782 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
2783 t
= maybe_lookup_decl (decl
, up
);
2785 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
2787 return t
? t
: decl
;
2791 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
2792 in outer contexts. */
2795 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
2800 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
2801 t
= maybe_lookup_decl (decl
, up
);
2803 return t
? t
: decl
;
2807 /* Construct the initialization value for reduction CLAUSE. */
2810 omp_reduction_init (tree clause
, tree type
)
2812 location_t loc
= OMP_CLAUSE_LOCATION (clause
);
2813 switch (OMP_CLAUSE_REDUCTION_CODE (clause
))
2820 case TRUTH_ORIF_EXPR
:
2821 case TRUTH_XOR_EXPR
:
2823 return build_zero_cst (type
);
2826 case TRUTH_AND_EXPR
:
2827 case TRUTH_ANDIF_EXPR
:
2829 return fold_convert_loc (loc
, type
, integer_one_node
);
2832 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
2835 if (SCALAR_FLOAT_TYPE_P (type
))
2837 REAL_VALUE_TYPE max
, min
;
2838 if (HONOR_INFINITIES (TYPE_MODE (type
)))
2841 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
2844 real_maxval (&min
, 1, TYPE_MODE (type
));
2845 return build_real (type
, min
);
2849 gcc_assert (INTEGRAL_TYPE_P (type
));
2850 return TYPE_MIN_VALUE (type
);
2854 if (SCALAR_FLOAT_TYPE_P (type
))
2856 REAL_VALUE_TYPE max
;
2857 if (HONOR_INFINITIES (TYPE_MODE (type
)))
2860 real_maxval (&max
, 0, TYPE_MODE (type
));
2861 return build_real (type
, max
);
2865 gcc_assert (INTEGRAL_TYPE_P (type
));
2866 return TYPE_MAX_VALUE (type
);
2874 /* Return alignment to be assumed for var in CLAUSE, which should be
2875 OMP_CLAUSE_ALIGNED. */
2878 omp_clause_aligned_alignment (tree clause
)
2880 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
2881 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
2883 /* Otherwise return implementation defined alignment. */
2884 unsigned int al
= 1;
2885 enum machine_mode mode
, vmode
;
2886 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
2888 vs
= 1 << floor_log2 (vs
);
2889 static enum mode_class classes
[]
2890 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
2891 for (int i
= 0; i
< 4; i
+= 2)
2892 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
2894 mode
= GET_MODE_WIDER_MODE (mode
))
2896 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
2897 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
2900 && GET_MODE_SIZE (vmode
) < vs
2901 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
2902 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
2904 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
2905 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
2907 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
2908 / GET_MODE_SIZE (mode
));
2909 if (TYPE_MODE (type
) != vmode
)
2911 if (TYPE_ALIGN_UNIT (type
) > al
)
2912 al
= TYPE_ALIGN_UNIT (type
);
2914 return build_int_cst (integer_type_node
, al
);
2917 /* Return maximum possible vectorization factor for the target. */
2924 || (!flag_tree_loop_vectorize
2925 && (global_options_set
.x_flag_tree_loop_vectorize
2926 || global_options_set
.x_flag_tree_vectorize
)))
2929 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
2932 vs
= 1 << floor_log2 (vs
);
2935 enum machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
2936 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
2937 return GET_MODE_NUNITS (vqimode
);
2941 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
2945 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
2946 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
2950 max_vf
= omp_max_vf ();
2953 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2954 OMP_CLAUSE_SAFELEN
);
2956 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
), max_vf
) == -1)
2957 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
2961 idx
= create_tmp_var (unsigned_type_node
, NULL
);
2962 lane
= create_tmp_var (unsigned_type_node
, NULL
);
2968 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
2969 tree avar
= create_tmp_var_raw (atype
, NULL
);
2970 if (TREE_ADDRESSABLE (new_var
))
2971 TREE_ADDRESSABLE (avar
) = 1;
2972 DECL_ATTRIBUTES (avar
)
2973 = tree_cons (get_identifier ("omp simd array"), NULL
,
2974 DECL_ATTRIBUTES (avar
));
2975 gimple_add_tmp_var (avar
);
2976 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
2977 NULL_TREE
, NULL_TREE
);
2978 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
2979 NULL_TREE
, NULL_TREE
);
2980 if (DECL_P (new_var
))
2982 SET_DECL_VALUE_EXPR (new_var
, lvar
);
2983 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
2988 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
2989 from the receiver (aka child) side and initializers for REFERENCE_TYPE
2990 private variables. Initialization statements go in ILIST, while calls
2991 to destructors go in DLIST. */
2994 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
2995 omp_context
*ctx
, struct omp_for_data
*fd
)
2997 tree c
, dtor
, copyin_seq
, x
, ptr
;
2998 bool copyin_by_ref
= false;
2999 bool lastprivate_firstprivate
= false;
3000 bool reduction_omp_orig_ref
= false;
3002 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3003 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
);
3005 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
3006 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
3007 gimple_seq llist
[2] = { NULL
, NULL
};
3011 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
3012 with data sharing clauses referencing variable sized vars. That
3013 is unnecessarily hard to support and very unlikely to result in
3014 vectorized code anyway. */
3016 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3017 switch (OMP_CLAUSE_CODE (c
))
3019 case OMP_CLAUSE_REDUCTION
:
3020 case OMP_CLAUSE_PRIVATE
:
3021 case OMP_CLAUSE_FIRSTPRIVATE
:
3022 case OMP_CLAUSE_LASTPRIVATE
:
3023 case OMP_CLAUSE_LINEAR
:
3024 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
3031 /* Do all the fixed sized types in the first pass, and the variable sized
3032 types in the second pass. This makes sure that the scalar arguments to
3033 the variable sized types are processed before we use them in the
3034 variable sized operations. */
3035 for (pass
= 0; pass
< 2; ++pass
)
3037 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3039 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
3042 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3046 case OMP_CLAUSE_PRIVATE
:
3047 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
3050 case OMP_CLAUSE_SHARED
:
3051 /* Ignore shared directives in teams construct. */
3052 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3054 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
3056 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c
)));
3059 case OMP_CLAUSE_FIRSTPRIVATE
:
3060 case OMP_CLAUSE_COPYIN
:
3061 case OMP_CLAUSE_LINEAR
:
3063 case OMP_CLAUSE_REDUCTION
:
3064 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
3065 reduction_omp_orig_ref
= true;
3067 case OMP_CLAUSE__LOOPTEMP_
:
3068 /* Handle _looptemp_ clauses only on parallel. */
3072 case OMP_CLAUSE_LASTPRIVATE
:
3073 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3075 lastprivate_firstprivate
= true;
3080 case OMP_CLAUSE_ALIGNED
:
3083 var
= OMP_CLAUSE_DECL (c
);
3084 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
3085 && !is_global_var (var
))
3087 new_var
= maybe_lookup_decl (var
, ctx
);
3088 if (new_var
== NULL_TREE
)
3089 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3090 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3091 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
3092 omp_clause_aligned_alignment (c
));
3093 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3094 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3095 gimplify_and_add (x
, ilist
);
3097 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
3098 && is_global_var (var
))
3100 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
3101 new_var
= lookup_decl (var
, ctx
);
3102 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3103 t
= build_fold_addr_expr_loc (clause_loc
, t
);
3104 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3105 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
3106 omp_clause_aligned_alignment (c
));
3107 t
= fold_convert_loc (clause_loc
, ptype
, t
);
3108 x
= create_tmp_var (ptype
, NULL
);
3109 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
3110 gimplify_and_add (t
, ilist
);
3111 t
= build_simple_mem_ref_loc (clause_loc
, x
);
3112 SET_DECL_VALUE_EXPR (new_var
, t
);
3113 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3120 new_var
= var
= OMP_CLAUSE_DECL (c
);
3121 if (c_kind
!= OMP_CLAUSE_COPYIN
)
3122 new_var
= lookup_decl (var
, ctx
);
3124 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
3129 else if (is_variable_sized (var
))
3131 /* For variable sized types, we need to allocate the
3132 actual storage here. Call alloca and store the
3133 result in the pointer decl that we created elsewhere. */
3137 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
3142 ptr
= DECL_VALUE_EXPR (new_var
);
3143 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
3144 ptr
= TREE_OPERAND (ptr
, 0);
3145 gcc_assert (DECL_P (ptr
));
3146 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
3148 /* void *tmp = __builtin_alloca */
3149 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3150 stmt
= gimple_build_call (atmp
, 1, x
);
3151 tmp
= create_tmp_var_raw (ptr_type_node
, NULL
);
3152 gimple_add_tmp_var (tmp
);
3153 gimple_call_set_lhs (stmt
, tmp
);
3155 gimple_seq_add_stmt (ilist
, stmt
);
3157 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
3158 gimplify_assign (ptr
, x
, ilist
);
3161 else if (is_reference (var
))
3163 /* For references that are being privatized for Fortran,
3164 allocate new backing storage for the new pointer
3165 variable. This allows us to avoid changing all the
3166 code that expects a pointer to something that expects
3167 a direct variable. */
3171 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
3172 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
3174 x
= build_receiver_ref (var
, false, ctx
);
3175 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3177 else if (TREE_CONSTANT (x
))
3179 const char *name
= NULL
;
3180 if (DECL_NAME (var
))
3181 name
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
3183 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
3185 gimple_add_tmp_var (x
);
3186 TREE_ADDRESSABLE (x
) = 1;
3187 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3191 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3192 x
= build_call_expr_loc (clause_loc
, atmp
, 1, x
);
3195 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3196 gimplify_assign (new_var
, x
, ilist
);
3198 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3200 else if (c_kind
== OMP_CLAUSE_REDUCTION
3201 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3209 switch (OMP_CLAUSE_CODE (c
))
3211 case OMP_CLAUSE_SHARED
:
3212 /* Ignore shared directives in teams construct. */
3213 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3215 /* Shared global vars are just accessed directly. */
3216 if (is_global_var (new_var
))
3218 /* Set up the DECL_VALUE_EXPR for shared variables now. This
3219 needs to be delayed until after fixup_child_record_type so
3220 that we get the correct type during the dereference. */
3221 by_ref
= use_pointer_for_field (var
, ctx
);
3222 x
= build_receiver_ref (var
, by_ref
, ctx
);
3223 SET_DECL_VALUE_EXPR (new_var
, x
);
3224 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3226 /* ??? If VAR is not passed by reference, and the variable
3227 hasn't been initialized yet, then we'll get a warning for
3228 the store into the omp_data_s structure. Ideally, we'd be
3229 able to notice this and not store anything at all, but
3230 we're generating code too early. Suppress the warning. */
3232 TREE_NO_WARNING (var
) = 1;
3235 case OMP_CLAUSE_LASTPRIVATE
:
3236 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3240 case OMP_CLAUSE_PRIVATE
:
3241 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
3242 x
= build_outer_var_ref (var
, ctx
);
3243 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
3245 if (is_task_ctx (ctx
))
3246 x
= build_receiver_ref (var
, false, ctx
);
3248 x
= build_outer_var_ref (var
, ctx
);
3254 nx
= lang_hooks
.decls
.omp_clause_default_ctor (c
, new_var
, x
);
3257 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3258 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
3259 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
3260 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3261 idx
, lane
, ivar
, lvar
))
3264 x
= lang_hooks
.decls
.omp_clause_default_ctor
3265 (c
, unshare_expr (ivar
), x
);
3267 gimplify_and_add (x
, &llist
[0]);
3270 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3273 gimple_seq tseq
= NULL
;
3276 gimplify_stmt (&dtor
, &tseq
);
3277 gimple_seq_add_seq (&llist
[1], tseq
);
3284 gimplify_and_add (nx
, ilist
);
3288 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3291 gimple_seq tseq
= NULL
;
3294 gimplify_stmt (&dtor
, &tseq
);
3295 gimple_seq_add_seq (dlist
, tseq
);
3299 case OMP_CLAUSE_LINEAR
:
3300 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
3301 goto do_firstprivate
;
3302 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
3305 x
= build_outer_var_ref (var
, ctx
);
3308 case OMP_CLAUSE_FIRSTPRIVATE
:
3309 if (is_task_ctx (ctx
))
3311 if (is_reference (var
) || is_variable_sized (var
))
3313 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
3315 || use_pointer_for_field (var
, NULL
))
3317 x
= build_receiver_ref (var
, false, ctx
);
3318 SET_DECL_VALUE_EXPR (new_var
, x
);
3319 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3324 x
= build_outer_var_ref (var
, ctx
);
3327 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3328 && gimple_omp_for_combined_into_p (ctx
->stmt
))
3330 tree stept
= POINTER_TYPE_P (TREE_TYPE (x
))
3331 ? sizetype
: TREE_TYPE (x
);
3332 tree t
= fold_convert (stept
,
3333 OMP_CLAUSE_LINEAR_STEP (c
));
3334 tree c
= find_omp_clause (clauses
,
3335 OMP_CLAUSE__LOOPTEMP_
);
3337 tree l
= OMP_CLAUSE_DECL (c
);
3338 if (fd
->collapse
== 1)
3340 tree n1
= fd
->loop
.n1
;
3341 tree step
= fd
->loop
.step
;
3342 tree itype
= TREE_TYPE (l
);
3343 if (POINTER_TYPE_P (itype
))
3344 itype
= signed_type_for (itype
);
3345 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
3346 if (TYPE_UNSIGNED (itype
)
3347 && fd
->loop
.cond_code
== GT_EXPR
)
3348 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
3349 fold_build1 (NEGATE_EXPR
,
3351 fold_build1 (NEGATE_EXPR
,
3354 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
3356 t
= fold_build2 (MULT_EXPR
, stept
,
3357 fold_convert (stept
, l
), t
);
3358 if (POINTER_TYPE_P (TREE_TYPE (x
)))
3359 x
= fold_build2 (POINTER_PLUS_EXPR
,
3360 TREE_TYPE (x
), x
, t
);
3362 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
3365 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
3366 || TREE_ADDRESSABLE (new_var
))
3367 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3368 idx
, lane
, ivar
, lvar
))
3370 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
3372 tree iv
= create_tmp_var (TREE_TYPE (new_var
), NULL
);
3373 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
3374 gimplify_and_add (x
, ilist
);
3375 gimple_stmt_iterator gsi
3376 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3378 = gimple_build_assign (unshare_expr (lvar
), iv
);
3379 gsi_insert_before_without_update (&gsi
, g
,
3381 tree stept
= POINTER_TYPE_P (TREE_TYPE (x
))
3382 ? sizetype
: TREE_TYPE (x
);
3383 tree t
= fold_convert (stept
,
3384 OMP_CLAUSE_LINEAR_STEP (c
));
3385 enum tree_code code
= PLUS_EXPR
;
3386 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
3387 code
= POINTER_PLUS_EXPR
;
3388 g
= gimple_build_assign_with_ops (code
, iv
, iv
, t
);
3389 gsi_insert_before_without_update (&gsi
, g
,
3393 x
= lang_hooks
.decls
.omp_clause_copy_ctor
3394 (c
, unshare_expr (ivar
), x
);
3395 gimplify_and_add (x
, &llist
[0]);
3396 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3399 gimple_seq tseq
= NULL
;
3402 gimplify_stmt (&dtor
, &tseq
);
3403 gimple_seq_add_seq (&llist
[1], tseq
);
3408 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, new_var
, x
);
3409 gimplify_and_add (x
, ilist
);
3412 case OMP_CLAUSE__LOOPTEMP_
:
3413 gcc_assert (is_parallel_ctx (ctx
));
3414 x
= build_outer_var_ref (var
, ctx
);
3415 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3416 gimplify_and_add (x
, ilist
);
3419 case OMP_CLAUSE_COPYIN
:
3420 by_ref
= use_pointer_for_field (var
, NULL
);
3421 x
= build_receiver_ref (var
, by_ref
, ctx
);
3422 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
3423 append_to_statement_list (x
, ©in_seq
);
3424 copyin_by_ref
|= by_ref
;
3427 case OMP_CLAUSE_REDUCTION
:
3428 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3430 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
3432 x
= build_outer_var_ref (var
, ctx
);
3434 if (is_reference (var
)
3435 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
3437 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3438 SET_DECL_VALUE_EXPR (placeholder
, x
);
3439 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
3440 tree new_vard
= new_var
;
3441 if (is_reference (var
))
3443 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
3444 new_vard
= TREE_OPERAND (new_var
, 0);
3445 gcc_assert (DECL_P (new_vard
));
3448 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3449 idx
, lane
, ivar
, lvar
))
3451 if (new_vard
== new_var
)
3453 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
3454 SET_DECL_VALUE_EXPR (new_var
, ivar
);
3458 SET_DECL_VALUE_EXPR (new_vard
,
3459 build_fold_addr_expr (ivar
));
3460 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
3462 x
= lang_hooks
.decls
.omp_clause_default_ctor
3463 (c
, unshare_expr (ivar
),
3464 build_outer_var_ref (var
, ctx
));
3466 gimplify_and_add (x
, &llist
[0]);
3467 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
3469 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
3470 lower_omp (&tseq
, ctx
);
3471 gimple_seq_add_seq (&llist
[0], tseq
);
3473 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
3474 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
3475 lower_omp (&tseq
, ctx
);
3476 gimple_seq_add_seq (&llist
[1], tseq
);
3477 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3478 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
3479 if (new_vard
== new_var
)
3480 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3482 SET_DECL_VALUE_EXPR (new_vard
,
3483 build_fold_addr_expr (lvar
));
3484 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3489 gimplify_stmt (&dtor
, &tseq
);
3490 gimple_seq_add_seq (&llist
[1], tseq
);
3494 x
= lang_hooks
.decls
.omp_clause_default_ctor
3495 (c
, new_var
, unshare_expr (x
));
3497 gimplify_and_add (x
, ilist
);
3498 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
3500 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
3501 lower_omp (&tseq
, ctx
);
3502 gimple_seq_add_seq (ilist
, tseq
);
3504 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
3507 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
3508 lower_omp (&tseq
, ctx
);
3509 gimple_seq_add_seq (dlist
, tseq
);
3510 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3512 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
3517 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
3518 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
3520 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3521 idx
, lane
, ivar
, lvar
))
3523 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
3524 tree ref
= build_outer_var_ref (var
, ctx
);
3526 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
3528 /* reduction(-:var) sums up the partial results, so it
3529 acts identically to reduction(+:var). */
3530 if (code
== MINUS_EXPR
)
3533 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
3534 ref
= build_outer_var_ref (var
, ctx
);
3535 gimplify_assign (ref
, x
, &llist
[1]);
3539 gimplify_assign (new_var
, x
, ilist
);
3541 gimplify_assign (build_outer_var_ref (var
, ctx
),
3555 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
3556 /* Don't want uninit warnings on simduid, it is always uninitialized,
3557 but we use it not for the value, but for the DECL_UID only. */
3558 TREE_NO_WARNING (uid
) = 1;
3560 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
3561 gimple_call_set_lhs (g
, lane
);
3562 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3563 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
3564 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
3565 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
3566 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
3567 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
3568 g
= gimple_build_assign_with_ops (INTEGER_CST
, lane
,
3569 build_int_cst (unsigned_type_node
, 0),
3571 gimple_seq_add_stmt (ilist
, g
);
3572 for (int i
= 0; i
< 2; i
++)
3575 tree vf
= create_tmp_var (unsigned_type_node
, NULL
);
3576 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
3577 gimple_call_set_lhs (g
, vf
);
3578 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
3579 gimple_seq_add_stmt (seq
, g
);
3580 tree t
= build_int_cst (unsigned_type_node
, 0);
3581 g
= gimple_build_assign_with_ops (INTEGER_CST
, idx
, t
, NULL_TREE
);
3582 gimple_seq_add_stmt (seq
, g
);
3583 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
3584 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
3585 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
3586 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
3587 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
3588 gimple_seq_add_seq (seq
, llist
[i
]);
3589 t
= build_int_cst (unsigned_type_node
, 1);
3590 g
= gimple_build_assign_with_ops (PLUS_EXPR
, idx
, idx
, t
);
3591 gimple_seq_add_stmt (seq
, g
);
3592 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
3593 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
3594 gimple_seq_add_stmt (seq
, g
);
3595 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
3599 /* The copyin sequence is not to be executed by the main thread, since
3600 that would result in self-copies. Perhaps not visible to scalars,
3601 but it certainly is to C++ operator=. */
3604 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
3606 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
3607 build_int_cst (TREE_TYPE (x
), 0));
3608 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
3609 gimplify_and_add (x
, ilist
);
3612 /* If any copyin variable is passed by reference, we must ensure the
3613 master thread doesn't modify it before it is copied over in all
3614 threads. Similarly for variables in both firstprivate and
3615 lastprivate clauses we need to ensure the lastprivate copying
3616 happens after firstprivate copying in all threads. And similarly
3617 for UDRs if initializer expression refers to omp_orig. */
3618 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
3620 /* Don't add any barrier for #pragma omp simd or
3621 #pragma omp distribute. */
3622 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3623 || gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_FOR
)
3624 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
3627 /* If max_vf is non-zero, then we can use only a vectorization factor
3628 up to the max_vf we chose. So stick it into the safelen clause. */
3631 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3632 OMP_CLAUSE_SAFELEN
);
3634 || compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3637 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
3638 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
3640 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
3641 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
3647 /* Generate code to implement the LASTPRIVATE clauses. This is used for
3648 both parallel and workshare constructs. PREDICATE may be NULL if it's
3652 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
3655 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
3656 bool par_clauses
= false;
3657 tree simduid
= NULL
, lastlane
= NULL
;
3659 /* Early exit if there are no lastprivate or linear clauses. */
3660 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
3661 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
3662 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
3663 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
3665 if (clauses
== NULL
)
3667 /* If this was a workshare clause, see if it had been combined
3668 with its parallel. In that case, look for the clauses on the
3669 parallel statement itself. */
3670 if (is_parallel_ctx (ctx
))
3674 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
3677 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
3678 OMP_CLAUSE_LASTPRIVATE
);
3679 if (clauses
== NULL
)
3687 tree label_true
, arm1
, arm2
;
3689 label
= create_artificial_label (UNKNOWN_LOCATION
);
3690 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
3691 arm1
= TREE_OPERAND (predicate
, 0);
3692 arm2
= TREE_OPERAND (predicate
, 1);
3693 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
3694 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
3695 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
3697 gimple_seq_add_stmt (stmt_list
, stmt
);
3698 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
3701 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3702 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
)
3704 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
3706 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
3709 for (c
= clauses
; c
;)
3712 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3714 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
3715 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3716 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
3718 var
= OMP_CLAUSE_DECL (c
);
3719 new_var
= lookup_decl (var
, ctx
);
3721 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
3723 tree val
= DECL_VALUE_EXPR (new_var
);
3724 if (TREE_CODE (val
) == ARRAY_REF
3725 && VAR_P (TREE_OPERAND (val
, 0))
3726 && lookup_attribute ("omp simd array",
3727 DECL_ATTRIBUTES (TREE_OPERAND (val
,
3730 if (lastlane
== NULL
)
3732 lastlane
= create_tmp_var (unsigned_type_node
, NULL
);
3734 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
3736 TREE_OPERAND (val
, 1));
3737 gimple_call_set_lhs (g
, lastlane
);
3738 gimple_seq_add_stmt (stmt_list
, g
);
3740 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
3741 TREE_OPERAND (val
, 0), lastlane
,
3742 NULL_TREE
, NULL_TREE
);
3746 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
3747 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
3749 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
3750 gimple_seq_add_seq (stmt_list
,
3751 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
3752 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
3755 x
= build_outer_var_ref (var
, ctx
);
3756 if (is_reference (var
))
3757 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3758 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
3759 gimplify_and_add (x
, stmt_list
);
3761 c
= OMP_CLAUSE_CHAIN (c
);
3762 if (c
== NULL
&& !par_clauses
)
3764 /* If this was a workshare clause, see if it had been combined
3765 with its parallel. In that case, continue looking for the
3766 clauses also on the parallel statement itself. */
3767 if (is_parallel_ctx (ctx
))
3771 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
3774 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
3775 OMP_CLAUSE_LASTPRIVATE
);
3781 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
3785 /* Generate code to implement the REDUCTION clauses. */
3788 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
3790 gimple_seq sub_seq
= NULL
;
3795 /* SIMD reductions are handled in lower_rec_input_clauses. */
3796 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3797 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_KIND_SIMD
)
3800 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
3801 update in that case, otherwise use a lock. */
3802 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
3803 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
3805 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3807 /* Never use OMP_ATOMIC for array reductions or UDRs. */
3817 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3819 tree var
, ref
, new_var
;
3820 enum tree_code code
;
3821 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3823 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
3826 var
= OMP_CLAUSE_DECL (c
);
3827 new_var
= lookup_decl (var
, ctx
);
3828 if (is_reference (var
))
3829 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3830 ref
= build_outer_var_ref (var
, ctx
);
3831 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
3833 /* reduction(-:var) sums up the partial results, so it acts
3834 identically to reduction(+:var). */
3835 if (code
== MINUS_EXPR
)
3840 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
3842 addr
= save_expr (addr
);
3843 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
3844 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
3845 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
3846 gimplify_and_add (x
, stmt_seqp
);
3850 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3852 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
3854 if (is_reference (var
)
3855 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
3857 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
3858 SET_DECL_VALUE_EXPR (placeholder
, ref
);
3859 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
3860 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
3861 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
3862 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3863 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
3867 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
3868 ref
= build_outer_var_ref (var
, ctx
);
3869 gimplify_assign (ref
, x
, &sub_seq
);
3873 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
3875 gimple_seq_add_stmt (stmt_seqp
, stmt
);
3877 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
3879 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
3881 gimple_seq_add_stmt (stmt_seqp
, stmt
);
3885 /* Generate code to implement the COPYPRIVATE clauses. */
3888 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
3893 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3895 tree var
, new_var
, ref
, x
;
3897 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3899 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
3902 var
= OMP_CLAUSE_DECL (c
);
3903 by_ref
= use_pointer_for_field (var
, NULL
);
3905 ref
= build_sender_ref (var
, ctx
);
3906 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
3909 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
3910 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
3912 gimplify_assign (ref
, x
, slist
);
3914 ref
= build_receiver_ref (var
, false, ctx
);
3917 ref
= fold_convert_loc (clause_loc
,
3918 build_pointer_type (TREE_TYPE (new_var
)),
3920 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
3922 if (is_reference (var
))
3924 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
3925 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
3926 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3928 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
3929 gimplify_and_add (x
, rlist
);
3934 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
3935 and REDUCTION from the sender (aka parent) side. */
3938 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
3943 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3945 tree val
, ref
, x
, var
;
3946 bool by_ref
, do_in
= false, do_out
= false;
3947 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3949 switch (OMP_CLAUSE_CODE (c
))
3951 case OMP_CLAUSE_PRIVATE
:
3952 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
3955 case OMP_CLAUSE_FIRSTPRIVATE
:
3956 case OMP_CLAUSE_COPYIN
:
3957 case OMP_CLAUSE_LASTPRIVATE
:
3958 case OMP_CLAUSE_REDUCTION
:
3959 case OMP_CLAUSE__LOOPTEMP_
:
3965 val
= OMP_CLAUSE_DECL (c
);
3966 var
= lookup_decl_in_outer_ctx (val
, ctx
);
3968 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
3969 && is_global_var (var
))
3971 if (is_variable_sized (val
))
3973 by_ref
= use_pointer_for_field (val
, NULL
);
3975 switch (OMP_CLAUSE_CODE (c
))
3977 case OMP_CLAUSE_PRIVATE
:
3978 case OMP_CLAUSE_FIRSTPRIVATE
:
3979 case OMP_CLAUSE_COPYIN
:
3980 case OMP_CLAUSE__LOOPTEMP_
:
3984 case OMP_CLAUSE_LASTPRIVATE
:
3985 if (by_ref
|| is_reference (val
))
3987 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3994 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
3999 case OMP_CLAUSE_REDUCTION
:
4001 do_out
= !(by_ref
|| is_reference (val
));
4010 ref
= build_sender_ref (val
, ctx
);
4011 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
4012 gimplify_assign (ref
, x
, ilist
);
4013 if (is_task_ctx (ctx
))
4014 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
4019 ref
= build_sender_ref (val
, ctx
);
4020 gimplify_assign (var
, ref
, olist
);
4025 /* Generate code to implement SHARED from the sender (aka parent)
4026 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
4027 list things that got automatically shared. */
4030 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
4032 tree var
, ovar
, nvar
, f
, x
, record_type
;
4034 if (ctx
->record_type
== NULL
)
4037 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
4038 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
4040 ovar
= DECL_ABSTRACT_ORIGIN (f
);
4041 nvar
= maybe_lookup_decl (ovar
, ctx
);
4042 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
4045 /* If CTX is a nested parallel directive. Find the immediately
4046 enclosing parallel or workshare construct that contains a
4047 mapping for OVAR. */
4048 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
4050 if (use_pointer_for_field (ovar
, ctx
))
4052 x
= build_sender_ref (ovar
, ctx
);
4053 var
= build_fold_addr_expr (var
);
4054 gimplify_assign (x
, var
, ilist
);
4058 x
= build_sender_ref (ovar
, ctx
);
4059 gimplify_assign (x
, var
, ilist
);
4061 if (!TREE_READONLY (var
)
4062 /* We don't need to receive a new reference to a result
4063 or parm decl. In fact we may not store to it as we will
4064 invalidate any pending RSO and generate wrong gimple
4066 && !((TREE_CODE (var
) == RESULT_DECL
4067 || TREE_CODE (var
) == PARM_DECL
)
4068 && DECL_BY_REFERENCE (var
)))
4070 x
= build_sender_ref (ovar
, ctx
);
4071 gimplify_assign (var
, x
, olist
);
4078 /* A convenience function to build an empty GIMPLE_COND with just the
4082 gimple_build_cond_empty (tree cond
)
4084 enum tree_code pred_code
;
4087 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
4088 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
4092 /* Build the function calls to GOMP_parallel_start etc to actually
4093 generate the parallel operation. REGION is the parallel region
4094 being expanded. BB is the block where to insert the code. WS_ARGS
4095 will be set if this is a call to a combined parallel+workshare
4096 construct, it contains the list of additional arguments needed by
4097 the workshare construct. */
4100 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
4101 gimple entry_stmt
, vec
<tree
, va_gc
> *ws_args
)
4103 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
4104 gimple_stmt_iterator gsi
;
4106 enum built_in_function start_ix
;
4108 location_t clause_loc
;
4109 vec
<tree
, va_gc
> *args
;
4111 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
4113 /* Determine what flavor of GOMP_parallel we will be
4115 start_ix
= BUILT_IN_GOMP_PARALLEL
;
4116 if (is_combined_parallel (region
))
4118 switch (region
->inner
->type
)
4120 case GIMPLE_OMP_FOR
:
4121 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
4122 start_ix2
= ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
4123 + (region
->inner
->sched_kind
4124 == OMP_CLAUSE_SCHEDULE_RUNTIME
4125 ? 3 : region
->inner
->sched_kind
));
4126 start_ix
= (enum built_in_function
)start_ix2
;
4128 case GIMPLE_OMP_SECTIONS
:
4129 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
4136 /* By default, the value of NUM_THREADS is zero (selected at run time)
4137 and there is no conditional. */
4139 val
= build_int_cst (unsigned_type_node
, 0);
4140 flags
= build_int_cst (unsigned_type_node
, 0);
4142 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4144 cond
= OMP_CLAUSE_IF_EXPR (c
);
4146 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
4149 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
4150 clause_loc
= OMP_CLAUSE_LOCATION (c
);
4153 clause_loc
= gimple_location (entry_stmt
);
4155 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
4157 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
4159 /* Ensure 'val' is of the correct type. */
4160 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
4162 /* If we found the clause 'if (cond)', build either
4163 (cond != 0) or (cond ? val : 1u). */
4166 gimple_stmt_iterator gsi
;
4168 cond
= gimple_boolify (cond
);
4170 if (integer_zerop (val
))
4171 val
= fold_build2_loc (clause_loc
,
4172 EQ_EXPR
, unsigned_type_node
, cond
,
4173 build_int_cst (TREE_TYPE (cond
), 0));
4176 basic_block cond_bb
, then_bb
, else_bb
;
4177 edge e
, e_then
, e_else
;
4178 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
4180 tmp_var
= create_tmp_var (TREE_TYPE (val
), NULL
);
4181 if (gimple_in_ssa_p (cfun
))
4183 tmp_then
= make_ssa_name (tmp_var
, NULL
);
4184 tmp_else
= make_ssa_name (tmp_var
, NULL
);
4185 tmp_join
= make_ssa_name (tmp_var
, NULL
);
4194 e
= split_block (bb
, NULL
);
4199 then_bb
= create_empty_bb (cond_bb
);
4200 else_bb
= create_empty_bb (then_bb
);
4201 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
4202 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
4204 stmt
= gimple_build_cond_empty (cond
);
4205 gsi
= gsi_start_bb (cond_bb
);
4206 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4208 gsi
= gsi_start_bb (then_bb
);
4209 stmt
= gimple_build_assign (tmp_then
, val
);
4210 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4212 gsi
= gsi_start_bb (else_bb
);
4213 stmt
= gimple_build_assign
4214 (tmp_else
, build_int_cst (unsigned_type_node
, 1));
4215 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4217 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
4218 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
4221 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
4222 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
4224 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
4225 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
4227 if (gimple_in_ssa_p (cfun
))
4229 gimple phi
= create_phi_node (tmp_join
, bb
);
4230 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
4231 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
4237 gsi
= gsi_start_bb (bb
);
4238 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
4239 false, GSI_CONTINUE_LINKING
);
4242 gsi
= gsi_last_bb (bb
);
4243 t
= gimple_omp_parallel_data_arg (entry_stmt
);
4245 t1
= null_pointer_node
;
4247 t1
= build_fold_addr_expr (t
);
4248 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
4250 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
4251 args
->quick_push (t2
);
4252 args
->quick_push (t1
);
4253 args
->quick_push (val
);
4255 args
->splice (*ws_args
);
4256 args
->quick_push (flags
);
4258 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
4259 builtin_decl_explicit (start_ix
), args
);
4261 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4262 false, GSI_CONTINUE_LINKING
);
4266 /* Build the function call to GOMP_task to actually
4267 generate the task operation. BB is the block where to insert the code. */
4270 expand_task_call (basic_block bb
, gimple entry_stmt
)
4272 tree t
, t1
, t2
, t3
, flags
, cond
, c
, c2
, clauses
, depend
;
4273 gimple_stmt_iterator gsi
;
4274 location_t loc
= gimple_location (entry_stmt
);
4276 clauses
= gimple_omp_task_clauses (entry_stmt
);
4278 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4280 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (c
));
4282 cond
= boolean_true_node
;
4284 c
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
4285 c2
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
4286 depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
4287 flags
= build_int_cst (unsigned_type_node
,
4288 (c
? 1 : 0) + (c2
? 4 : 0) + (depend
? 8 : 0));
4290 c
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
4293 c
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c
));
4294 c
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, c
,
4295 build_int_cst (unsigned_type_node
, 2),
4296 build_int_cst (unsigned_type_node
, 0));
4297 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, c
);
4300 depend
= OMP_CLAUSE_DECL (depend
);
4302 depend
= build_int_cst (ptr_type_node
, 0);
4304 gsi
= gsi_last_bb (bb
);
4305 t
= gimple_omp_task_data_arg (entry_stmt
);
4307 t2
= null_pointer_node
;
4309 t2
= build_fold_addr_expr_loc (loc
, t
);
4310 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
4311 t
= gimple_omp_task_copy_fn (entry_stmt
);
4313 t3
= null_pointer_node
;
4315 t3
= build_fold_addr_expr_loc (loc
, t
);
4317 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
4319 gimple_omp_task_arg_size (entry_stmt
),
4320 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
4323 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4324 false, GSI_CONTINUE_LINKING
);
4328 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
4329 catch handler and return it. This prevents programs from violating the
4330 structured block semantics with throws. */
4333 maybe_catch_exception (gimple_seq body
)
4338 if (!flag_exceptions
)
4341 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
4342 decl
= lang_hooks
.eh_protect_cleanup_actions ();
4344 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
4346 g
= gimple_build_eh_must_not_throw (decl
);
4347 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
4350 return gimple_seq_alloc_with_stmt (g
);
4353 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
4356 vec2chain (vec
<tree
, va_gc
> *v
)
4358 tree chain
= NULL_TREE
, t
;
4361 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
4363 DECL_CHAIN (t
) = chain
;
4371 /* Remove barriers in REGION->EXIT's block. Note that this is only
4372 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
4373 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
4374 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
4378 remove_exit_barrier (struct omp_region
*region
)
4380 gimple_stmt_iterator gsi
;
4381 basic_block exit_bb
;
4385 int any_addressable_vars
= -1;
4387 exit_bb
= region
->exit
;
4389 /* If the parallel region doesn't return, we don't have REGION->EXIT
4394 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
4395 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
4396 statements that can appear in between are extremely limited -- no
4397 memory operations at all. Here, we allow nothing at all, so the
4398 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
4399 gsi
= gsi_last_bb (exit_bb
);
4400 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
4402 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
4405 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
4407 gsi
= gsi_last_bb (e
->src
);
4408 if (gsi_end_p (gsi
))
4410 stmt
= gsi_stmt (gsi
);
4411 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
4412 && !gimple_omp_return_nowait_p (stmt
))
4414 /* OpenMP 3.0 tasks unfortunately prevent this optimization
4415 in many cases. If there could be tasks queued, the barrier
4416 might be needed to let the tasks run before some local
4417 variable of the parallel that the task uses as shared
4418 runs out of scope. The task can be spawned either
4419 from within current function (this would be easy to check)
4420 or from some function it calls and gets passed an address
4421 of such a variable. */
4422 if (any_addressable_vars
< 0)
4424 gimple parallel_stmt
= last_stmt (region
->entry
);
4425 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
4426 tree local_decls
, block
, decl
;
4429 any_addressable_vars
= 0;
4430 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
4431 if (TREE_ADDRESSABLE (decl
))
4433 any_addressable_vars
= 1;
4436 for (block
= gimple_block (stmt
);
4437 !any_addressable_vars
4439 && TREE_CODE (block
) == BLOCK
;
4440 block
= BLOCK_SUPERCONTEXT (block
))
4442 for (local_decls
= BLOCK_VARS (block
);
4444 local_decls
= DECL_CHAIN (local_decls
))
4445 if (TREE_ADDRESSABLE (local_decls
))
4447 any_addressable_vars
= 1;
4450 if (block
== gimple_block (parallel_stmt
))
4454 if (!any_addressable_vars
)
4455 gimple_omp_return_set_nowait (stmt
);
4461 remove_exit_barriers (struct omp_region
*region
)
4463 if (region
->type
== GIMPLE_OMP_PARALLEL
)
4464 remove_exit_barrier (region
);
4468 region
= region
->inner
;
4469 remove_exit_barriers (region
);
4470 while (region
->next
)
4472 region
= region
->next
;
4473 remove_exit_barriers (region
);
4478 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
4479 calls. These can't be declared as const functions, but
4480 within one parallel body they are constant, so they can be
4481 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
4482 which are declared const. Similarly for task body, except
4483 that in untied task omp_get_thread_num () can change at any task
4484 scheduling point. */
4487 optimize_omp_library_calls (gimple entry_stmt
)
4490 gimple_stmt_iterator gsi
;
4491 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4492 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
4493 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
4494 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
4495 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
4496 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
4497 OMP_CLAUSE_UNTIED
) != NULL
);
4500 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4502 gimple call
= gsi_stmt (gsi
);
4505 if (is_gimple_call (call
)
4506 && (decl
= gimple_call_fndecl (call
))
4507 && DECL_EXTERNAL (decl
)
4508 && TREE_PUBLIC (decl
)
4509 && DECL_INITIAL (decl
) == NULL
)
4513 if (DECL_NAME (decl
) == thr_num_id
)
4515 /* In #pragma omp task untied omp_get_thread_num () can change
4516 during the execution of the task region. */
4519 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4521 else if (DECL_NAME (decl
) == num_thr_id
)
4522 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
4526 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
4527 || gimple_call_num_args (call
) != 0)
4530 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
4533 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
4534 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
4535 TREE_TYPE (TREE_TYPE (built_in
))))
4538 gimple_call_set_fndecl (call
, built_in
);
4543 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
4547 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
4551 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
4552 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
4555 if (TREE_CODE (t
) == ADDR_EXPR
)
4556 recompute_tree_invariant_for_addr_expr (t
);
4558 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
4562 /* Prepend TO = FROM assignment before *GSI_P. */
4565 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
)
4567 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
4568 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
4569 true, GSI_SAME_STMT
);
4570 gimple stmt
= gimple_build_assign (to
, from
);
4571 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
4572 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
4573 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
4575 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
4576 gimple_regimplify_operands (stmt
, &gsi
);
4580 /* Expand the OpenMP parallel or task directive starting at REGION. */
4583 expand_omp_taskreg (struct omp_region
*region
)
4585 basic_block entry_bb
, exit_bb
, new_bb
;
4586 struct function
*child_cfun
;
4587 tree child_fn
, block
, t
;
4588 gimple_stmt_iterator gsi
;
4589 gimple entry_stmt
, stmt
;
4591 vec
<tree
, va_gc
> *ws_args
;
4593 entry_stmt
= last_stmt (region
->entry
);
4594 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
4595 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
4597 entry_bb
= region
->entry
;
4598 exit_bb
= region
->exit
;
4600 if (is_combined_parallel (region
))
4601 ws_args
= region
->ws_args
;
4605 if (child_cfun
->cfg
)
4607 /* Due to inlining, it may happen that we have already outlined
4608 the region, in which case all we need to do is make the
4609 sub-graph unreachable and emit the parallel call. */
4610 edge entry_succ_e
, exit_succ_e
;
4611 gimple_stmt_iterator gsi
;
4613 entry_succ_e
= single_succ_edge (entry_bb
);
4615 gsi
= gsi_last_bb (entry_bb
);
4616 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
4617 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
4618 gsi_remove (&gsi
, true);
4623 exit_succ_e
= single_succ_edge (exit_bb
);
4624 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
4626 remove_edge_and_dominated_blocks (entry_succ_e
);
4630 unsigned srcidx
, dstidx
, num
;
4632 /* If the parallel region needs data sent from the parent
4633 function, then the very first statement (except possible
4634 tree profile counter updates) of the parallel body
4635 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
4636 &.OMP_DATA_O is passed as an argument to the child function,
4637 we need to replace it with the argument as seen by the child
4640 In most cases, this will end up being the identity assignment
4641 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
4642 a function call that has been inlined, the original PARM_DECL
4643 .OMP_DATA_I may have been converted into a different local
4644 variable. In which case, we need to keep the assignment. */
4645 if (gimple_omp_taskreg_data_arg (entry_stmt
))
4647 basic_block entry_succ_bb
= single_succ (entry_bb
);
4648 gimple_stmt_iterator gsi
;
4650 gimple parcopy_stmt
= NULL
;
4652 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
4656 gcc_assert (!gsi_end_p (gsi
));
4657 stmt
= gsi_stmt (gsi
);
4658 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
4661 if (gimple_num_ops (stmt
) == 2)
4663 tree arg
= gimple_assign_rhs1 (stmt
);
4665 /* We're ignore the subcode because we're
4666 effectively doing a STRIP_NOPS. */
4668 if (TREE_CODE (arg
) == ADDR_EXPR
4669 && TREE_OPERAND (arg
, 0)
4670 == gimple_omp_taskreg_data_arg (entry_stmt
))
4672 parcopy_stmt
= stmt
;
4678 gcc_assert (parcopy_stmt
!= NULL
);
4679 arg
= DECL_ARGUMENTS (child_fn
);
4681 if (!gimple_in_ssa_p (cfun
))
4683 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
4684 gsi_remove (&gsi
, true);
4687 /* ?? Is setting the subcode really necessary ?? */
4688 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
4689 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
4694 /* If we are in ssa form, we must load the value from the default
4695 definition of the argument. That should not be defined now,
4696 since the argument is not used uninitialized. */
4697 gcc_assert (ssa_default_def (cfun
, arg
) == NULL
);
4698 narg
= make_ssa_name (arg
, gimple_build_nop ());
4699 set_ssa_default_def (cfun
, arg
, narg
);
4700 /* ?? Is setting the subcode really necessary ?? */
4701 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (narg
));
4702 gimple_assign_set_rhs1 (parcopy_stmt
, narg
);
4703 update_stmt (parcopy_stmt
);
4707 /* Declare local variables needed in CHILD_CFUN. */
4708 block
= DECL_INITIAL (child_fn
);
4709 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
4710 /* The gimplifier could record temporaries in parallel/task block
4711 rather than in containing function's local_decls chain,
4712 which would mean cgraph missed finalizing them. Do it now. */
4713 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
4714 if (TREE_CODE (t
) == VAR_DECL
4716 && !DECL_EXTERNAL (t
))
4717 varpool_finalize_decl (t
);
4718 DECL_SAVED_TREE (child_fn
) = NULL
;
4719 /* We'll create a CFG for child_fn, so no gimple body is needed. */
4720 gimple_set_body (child_fn
, NULL
);
4721 TREE_USED (block
) = 1;
4723 /* Reset DECL_CONTEXT on function arguments. */
4724 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
4725 DECL_CONTEXT (t
) = child_fn
;
4727 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
4728 so that it can be moved to the child function. */
4729 gsi
= gsi_last_bb (entry_bb
);
4730 stmt
= gsi_stmt (gsi
);
4731 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
4732 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
4733 gsi_remove (&gsi
, true);
4734 e
= split_block (entry_bb
, stmt
);
4736 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
4738 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
4741 gsi
= gsi_last_bb (exit_bb
);
4742 gcc_assert (!gsi_end_p (gsi
)
4743 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
4744 stmt
= gimple_build_return (NULL
);
4745 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
4746 gsi_remove (&gsi
, true);
4749 /* Move the parallel region into CHILD_CFUN. */
4751 if (gimple_in_ssa_p (cfun
))
4753 init_tree_ssa (child_cfun
);
4754 init_ssa_operands (child_cfun
);
4755 child_cfun
->gimple_df
->in_ssa_p
= true;
4759 block
= gimple_block (entry_stmt
);
4761 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
4763 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
4764 /* When the OMP expansion process cannot guarantee an up-to-date
4765 loop tree arrange for the child function to fixup loops. */
4766 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
4767 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
4769 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
4770 num
= vec_safe_length (child_cfun
->local_decls
);
4771 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
4773 t
= (*child_cfun
->local_decls
)[srcidx
];
4774 if (DECL_CONTEXT (t
) == cfun
->decl
)
4776 if (srcidx
!= dstidx
)
4777 (*child_cfun
->local_decls
)[dstidx
] = t
;
4781 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
4783 /* Inform the callgraph about the new function. */
4784 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
4785 cgraph_add_new_function (child_fn
, true);
4787 /* Fix the callgraph edges for child_cfun. Those for cfun will be
4788 fixed in a following pass. */
4789 push_cfun (child_cfun
);
4791 optimize_omp_library_calls (entry_stmt
);
4792 rebuild_cgraph_edges ();
4794 /* Some EH regions might become dead, see PR34608. If
4795 pass_cleanup_cfg isn't the first pass to happen with the
4796 new child, these dead EH edges might cause problems.
4797 Clean them up now. */
4798 if (flag_exceptions
)
4801 bool changed
= false;
4804 changed
|= gimple_purge_dead_eh_edges (bb
);
4806 cleanup_tree_cfg ();
4808 if (gimple_in_ssa_p (cfun
))
4809 update_ssa (TODO_update_ssa
);
4813 /* Emit a library call to launch the children threads. */
4814 if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
4815 expand_parallel_call (region
, new_bb
, entry_stmt
, ws_args
);
4817 expand_task_call (new_bb
, entry_stmt
);
4818 if (gimple_in_ssa_p (cfun
))
4819 update_ssa (TODO_update_ssa_only_virtuals
);
4823 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
4824 of the combined collapse > 1 loop constructs, generate code like:
4825 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
4830 count3 = (adj + N32 - N31) / STEP3;
4831 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
4836 count2 = (adj + N22 - N21) / STEP2;
4837 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
4842 count1 = (adj + N12 - N11) / STEP1;
4843 count = count1 * count2 * count3;
4844 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
4846 and set ZERO_ITER_BB to that bb. If this isn't the outermost
4847 of the combined loop constructs, just initialize COUNTS array
4848 from the _looptemp_ clauses. */
4850 /* NOTE: It *could* be better to moosh all of the BBs together,
4851 creating one larger BB with all the computation and the unexpected
4852 jump at the end. I.e.
4854 bool zero3, zero2, zero1, zero;
4857 count3 = (N32 - N31) /[cl] STEP3;
4859 count2 = (N22 - N21) /[cl] STEP2;
4861 count1 = (N12 - N11) /[cl] STEP1;
4862 zero = zero3 || zero2 || zero1;
4863 count = count1 * count2 * count3;
4864 if (__builtin_expect(zero, false)) goto zero_iter_bb;
4866 After all, we expect the zero=false, and thus we expect to have to
4867 evaluate all of the comparison expressions, so short-circuiting
4868 oughtn't be a win. Since the condition isn't protecting a
4869 denominator, we're not concerned about divide-by-zero, so we can
4870 fully evaluate count even if a numerator turned out to be wrong.
4872 It seems like putting this all together would create much better
4873 scheduling opportunities, and less pressure on the chip's branch
4877 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
4878 basic_block
&entry_bb
, tree
*counts
,
4879 basic_block
&zero_iter_bb
, int &first_zero_iter
,
4880 basic_block
&l2_dom_bb
)
4882 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
4887 /* Collapsed loops need work for expansion into SSA form. */
4888 gcc_assert (!gimple_in_ssa_p (cfun
));
4890 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
4891 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
4893 /* First two _looptemp_ clauses are for istart/iend, counts[0]
4894 isn't supposed to be handled, as the inner loop doesn't
4896 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
4897 OMP_CLAUSE__LOOPTEMP_
);
4898 gcc_assert (innerc
);
4899 for (i
= 0; i
< fd
->collapse
; i
++)
4901 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
4902 OMP_CLAUSE__LOOPTEMP_
);
4903 gcc_assert (innerc
);
4905 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
4907 counts
[0] = NULL_TREE
;
4912 for (i
= 0; i
< fd
->collapse
; i
++)
4914 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
4916 if (SSA_VAR_P (fd
->loop
.n2
)
4917 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
4918 fold_convert (itype
, fd
->loops
[i
].n1
),
4919 fold_convert (itype
, fd
->loops
[i
].n2
)))
4920 == NULL_TREE
|| !integer_onep (t
)))
4923 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
4924 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
4925 true, GSI_SAME_STMT
);
4926 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
4927 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
4928 true, GSI_SAME_STMT
);
4929 stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
4930 NULL_TREE
, NULL_TREE
);
4931 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
4932 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
4933 expand_omp_regimplify_p
, NULL
, NULL
)
4934 || walk_tree (gimple_cond_rhs_ptr (stmt
),
4935 expand_omp_regimplify_p
, NULL
, NULL
))
4937 *gsi
= gsi_for_stmt (stmt
);
4938 gimple_regimplify_operands (stmt
, gsi
);
4940 e
= split_block (entry_bb
, stmt
);
4941 if (zero_iter_bb
== NULL
)
4943 first_zero_iter
= i
;
4944 zero_iter_bb
= create_empty_bb (entry_bb
);
4946 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
4947 *gsi
= gsi_after_labels (zero_iter_bb
);
4948 stmt
= gimple_build_assign (fd
->loop
.n2
,
4949 build_zero_cst (type
));
4950 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
4951 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
4954 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
4955 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
4956 e
->flags
= EDGE_TRUE_VALUE
;
4957 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
4958 if (l2_dom_bb
== NULL
)
4959 l2_dom_bb
= entry_bb
;
4961 *gsi
= gsi_last_bb (entry_bb
);
4964 if (POINTER_TYPE_P (itype
))
4965 itype
= signed_type_for (itype
);
4966 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
4968 t
= fold_build2 (PLUS_EXPR
, itype
,
4969 fold_convert (itype
, fd
->loops
[i
].step
), t
);
4970 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
4971 fold_convert (itype
, fd
->loops
[i
].n2
));
4972 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
4973 fold_convert (itype
, fd
->loops
[i
].n1
));
4974 /* ?? We could probably use CEIL_DIV_EXPR instead of
4975 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
4976 generate the same code in the end because generically we
4977 don't know that the values involved must be negative for
4979 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
4980 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
4981 fold_build1 (NEGATE_EXPR
, itype
, t
),
4982 fold_build1 (NEGATE_EXPR
, itype
,
4983 fold_convert (itype
,
4984 fd
->loops
[i
].step
)));
4986 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
4987 fold_convert (itype
, fd
->loops
[i
].step
));
4988 t
= fold_convert (type
, t
);
4989 if (TREE_CODE (t
) == INTEGER_CST
)
4993 counts
[i
] = create_tmp_reg (type
, ".count");
4994 expand_omp_build_assign (gsi
, counts
[i
], t
);
4996 if (SSA_VAR_P (fd
->loop
.n2
))
5001 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
5002 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
5008 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
5010 V3 = N31 + (T % count3) * STEP3;
5012 V2 = N21 + (T % count2) * STEP2;
5014 V1 = N11 + T * STEP1;
5015 if this loop doesn't have an inner loop construct combined with it.
5016 If it does have an inner loop construct combined with it and the
5017 iteration count isn't known constant, store values from counts array
5018 into its _looptemp_ temporaries instead. */
5021 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5022 tree
*counts
, gimple inner_stmt
, tree startvar
)
5025 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5027 /* If fd->loop.n2 is constant, then no propagation of the counts
5028 is needed, they are constant. */
5029 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
5032 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
5033 ? gimple_omp_parallel_clauses (inner_stmt
)
5034 : gimple_omp_for_clauses (inner_stmt
);
5035 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5036 isn't supposed to be handled, as the inner loop doesn't
5038 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
5039 gcc_assert (innerc
);
5040 for (i
= 0; i
< fd
->collapse
; i
++)
5042 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5043 OMP_CLAUSE__LOOPTEMP_
);
5044 gcc_assert (innerc
);
5047 tree tem
= OMP_CLAUSE_DECL (innerc
);
5048 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
5049 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5050 false, GSI_CONTINUE_LINKING
);
5051 gimple stmt
= gimple_build_assign (tem
, t
);
5052 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5058 tree type
= TREE_TYPE (fd
->loop
.v
);
5059 tree tem
= create_tmp_reg (type
, ".tem");
5060 gimple stmt
= gimple_build_assign (tem
, startvar
);
5061 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5063 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5065 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
5067 if (POINTER_TYPE_P (vtype
))
5068 itype
= signed_type_for (vtype
);
5070 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
5073 t
= fold_convert (itype
, t
);
5074 t
= fold_build2 (MULT_EXPR
, itype
, t
,
5075 fold_convert (itype
, fd
->loops
[i
].step
));
5076 if (POINTER_TYPE_P (vtype
))
5077 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
5079 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
5080 t
= force_gimple_operand_gsi (gsi
, t
,
5081 DECL_P (fd
->loops
[i
].v
)
5082 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5084 GSI_CONTINUE_LINKING
);
5085 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5086 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5089 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
5090 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5091 false, GSI_CONTINUE_LINKING
);
5092 stmt
= gimple_build_assign (tem
, t
);
5093 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5099 /* Helper function for expand_omp_for_*. Generate code like:
5102 if (V3 cond3 N32) goto BODY_BB; else goto L11;
5106 if (V2 cond2 N22) goto BODY_BB; else goto L12;
5113 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
5114 basic_block body_bb
)
5116 basic_block last_bb
, bb
, collapse_bb
= NULL
;
5118 gimple_stmt_iterator gsi
;
5124 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5126 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
5128 bb
= create_empty_bb (last_bb
);
5130 add_bb_to_loop (bb
, last_bb
->loop_father
);
5131 gsi
= gsi_start_bb (bb
);
5133 if (i
< fd
->collapse
- 1)
5135 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
5136 e
->probability
= REG_BR_PROB_BASE
/ 8;
5138 t
= fd
->loops
[i
+ 1].n1
;
5139 t
= force_gimple_operand_gsi (&gsi
, t
,
5140 DECL_P (fd
->loops
[i
+ 1].v
)
5141 && TREE_ADDRESSABLE (fd
->loops
[i
5144 GSI_CONTINUE_LINKING
);
5145 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
5146 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5151 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
5153 if (POINTER_TYPE_P (vtype
))
5154 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5156 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5157 t
= force_gimple_operand_gsi (&gsi
, t
,
5158 DECL_P (fd
->loops
[i
].v
)
5159 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5160 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5161 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5162 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5166 t
= fd
->loops
[i
].n2
;
5167 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5168 false, GSI_CONTINUE_LINKING
);
5169 tree v
= fd
->loops
[i
].v
;
5170 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
5171 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
5172 false, GSI_CONTINUE_LINKING
);
5173 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
5174 stmt
= gimple_build_cond_empty (t
);
5175 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5176 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
5177 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5180 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
5188 /* A subroutine of expand_omp_for. Generate code for a parallel
5189 loop with any schedule. Given parameters:
5191 for (V = N1; V cond N2; V += STEP) BODY;
5193 where COND is "<" or ">", we generate pseudocode
5195 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
5196 if (more) goto L0; else goto L3;
5203 if (V cond iend) goto L1; else goto L2;
5205 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5208 If this is a combined omp parallel loop, instead of the call to
5209 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
5210 If this is gimple_omp_for_combined_p loop, then instead of assigning
5211 V and iend in L0 we assign the first two _looptemp_ clause decls of the
5212 inner GIMPLE_OMP_FOR and V += STEP; and
5213 if (V cond iend) goto L1; else goto L2; are removed.
5215 For collapsed loops, given parameters:
5217 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
5218 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
5219 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
5222 we generate pseudocode
5224 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
5229 count3 = (adj + N32 - N31) / STEP3;
5230 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
5235 count2 = (adj + N22 - N21) / STEP2;
5236 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
5241 count1 = (adj + N12 - N11) / STEP1;
5242 count = count1 * count2 * count3;
5247 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
5248 if (more) goto L0; else goto L3;
5252 V3 = N31 + (T % count3) * STEP3;
5254 V2 = N21 + (T % count2) * STEP2;
5256 V1 = N11 + T * STEP1;
5261 if (V < iend) goto L10; else goto L2;
5264 if (V3 cond3 N32) goto L1; else goto L11;
5268 if (V2 cond2 N22) goto L1; else goto L12;
5274 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5280 expand_omp_for_generic (struct omp_region
*region
,
5281 struct omp_for_data
*fd
,
5282 enum built_in_function start_fn
,
5283 enum built_in_function next_fn
,
5286 tree type
, istart0
, iend0
, iend
;
5287 tree t
, vmain
, vback
, bias
= NULL_TREE
;
5288 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
5289 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
5290 gimple_stmt_iterator gsi
;
5292 bool in_combined_parallel
= is_combined_parallel (region
);
5293 bool broken_loop
= region
->cont
== NULL
;
5295 tree
*counts
= NULL
;
5298 gcc_assert (!broken_loop
|| !in_combined_parallel
);
5299 gcc_assert (fd
->iter_type
== long_integer_type_node
5300 || !in_combined_parallel
);
5302 type
= TREE_TYPE (fd
->loop
.v
);
5303 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
5304 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
5305 TREE_ADDRESSABLE (istart0
) = 1;
5306 TREE_ADDRESSABLE (iend0
) = 1;
5308 /* See if we need to bias by LLONG_MIN. */
5309 if (fd
->iter_type
== long_long_unsigned_type_node
5310 && TREE_CODE (type
) == INTEGER_TYPE
5311 && !TYPE_UNSIGNED (type
))
5315 if (fd
->loop
.cond_code
== LT_EXPR
)
5318 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5322 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5325 if (TREE_CODE (n1
) != INTEGER_CST
5326 || TREE_CODE (n2
) != INTEGER_CST
5327 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
5328 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
5331 entry_bb
= region
->entry
;
5332 cont_bb
= region
->cont
;
5334 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
5335 gcc_assert (broken_loop
5336 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
5337 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
5338 l1_bb
= single_succ (l0_bb
);
5341 l2_bb
= create_empty_bb (cont_bb
);
5342 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
);
5343 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
5347 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
5348 exit_bb
= region
->exit
;
5350 gsi
= gsi_last_bb (entry_bb
);
5352 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5353 if (fd
->collapse
> 1)
5355 int first_zero_iter
= -1;
5356 basic_block zero_iter_bb
= NULL
, l2_dom_bb
= NULL
;
5358 counts
= XALLOCAVEC (tree
, fd
->collapse
);
5359 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
5360 zero_iter_bb
, first_zero_iter
,
5365 /* Some counts[i] vars might be uninitialized if
5366 some loop has zero iterations. But the body shouldn't
5367 be executed in that case, so just avoid uninit warnings. */
5368 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
5369 if (SSA_VAR_P (counts
[i
]))
5370 TREE_NO_WARNING (counts
[i
]) = 1;
5372 e
= split_block (entry_bb
, gsi_stmt (gsi
));
5374 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
5375 gsi
= gsi_last_bb (entry_bb
);
5376 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
5377 get_immediate_dominator (CDI_DOMINATORS
,
5381 if (in_combined_parallel
)
5383 /* In a combined parallel loop, emit a call to
5384 GOMP_loop_foo_next. */
5385 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
5386 build_fold_addr_expr (istart0
),
5387 build_fold_addr_expr (iend0
));
5391 tree t0
, t1
, t2
, t3
, t4
;
5392 /* If this is not a combined parallel loop, emit a call to
5393 GOMP_loop_foo_start in ENTRY_BB. */
5394 t4
= build_fold_addr_expr (iend0
);
5395 t3
= build_fold_addr_expr (istart0
);
5396 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
5399 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5401 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5402 OMP_CLAUSE__LOOPTEMP_
);
5403 gcc_assert (innerc
);
5404 t0
= OMP_CLAUSE_DECL (innerc
);
5405 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5406 OMP_CLAUSE__LOOPTEMP_
);
5407 gcc_assert (innerc
);
5408 t1
= OMP_CLAUSE_DECL (innerc
);
5410 if (POINTER_TYPE_P (TREE_TYPE (t0
))
5411 && TYPE_PRECISION (TREE_TYPE (t0
))
5412 != TYPE_PRECISION (fd
->iter_type
))
5414 /* Avoid casting pointers to integer of a different size. */
5415 tree itype
= signed_type_for (type
);
5416 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
5417 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
5421 t1
= fold_convert (fd
->iter_type
, t1
);
5422 t0
= fold_convert (fd
->iter_type
, t0
);
5426 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
5427 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
5429 if (fd
->iter_type
== long_integer_type_node
)
5433 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
5434 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5435 6, t0
, t1
, t2
, t
, t3
, t4
);
5438 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5439 5, t0
, t1
, t2
, t3
, t4
);
5447 /* The GOMP_loop_ull_*start functions have additional boolean
5448 argument, true for < loops and false for > loops.
5449 In Fortran, the C bool type can be different from
5450 boolean_type_node. */
5451 bfn_decl
= builtin_decl_explicit (start_fn
);
5452 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
5453 t5
= build_int_cst (c_bool_type
,
5454 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
5457 tree bfn_decl
= builtin_decl_explicit (start_fn
);
5458 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
5459 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
5462 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5463 6, t5
, t0
, t1
, t2
, t3
, t4
);
5466 if (TREE_TYPE (t
) != boolean_type_node
)
5467 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
5468 t
, build_int_cst (TREE_TYPE (t
), 0));
5469 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5470 true, GSI_SAME_STMT
);
5471 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
5473 /* Remove the GIMPLE_OMP_FOR statement. */
5474 gsi_remove (&gsi
, true);
5476 /* Iteration setup for sequential loop goes in L0_BB. */
5477 tree startvar
= fd
->loop
.v
;
5478 tree endvar
= NULL_TREE
;
5480 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5482 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
5483 && gimple_omp_for_kind (inner_stmt
)
5484 == GF_OMP_FOR_KIND_SIMD
);
5485 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
5486 OMP_CLAUSE__LOOPTEMP_
);
5487 gcc_assert (innerc
);
5488 startvar
= OMP_CLAUSE_DECL (innerc
);
5489 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5490 OMP_CLAUSE__LOOPTEMP_
);
5491 gcc_assert (innerc
);
5492 endvar
= OMP_CLAUSE_DECL (innerc
);
5495 gsi
= gsi_start_bb (l0_bb
);
5498 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
5499 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
5500 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
5501 t
= fold_convert (TREE_TYPE (startvar
), t
);
5502 t
= force_gimple_operand_gsi (&gsi
, t
,
5504 && TREE_ADDRESSABLE (startvar
),
5505 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5506 stmt
= gimple_build_assign (startvar
, t
);
5507 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5511 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
5512 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
5513 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
5514 t
= fold_convert (TREE_TYPE (startvar
), t
);
5515 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5516 false, GSI_CONTINUE_LINKING
);
5519 stmt
= gimple_build_assign (endvar
, iend
);
5520 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5522 if (fd
->collapse
> 1)
5523 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
5527 /* Code to control the increment and predicate for the sequential
5528 loop goes in the CONT_BB. */
5529 gsi
= gsi_last_bb (cont_bb
);
5530 stmt
= gsi_stmt (gsi
);
5531 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
5532 vmain
= gimple_omp_continue_control_use (stmt
);
5533 vback
= gimple_omp_continue_control_def (stmt
);
5535 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5537 if (POINTER_TYPE_P (type
))
5538 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
5540 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
5541 t
= force_gimple_operand_gsi (&gsi
, t
,
5543 && TREE_ADDRESSABLE (vback
),
5544 NULL_TREE
, true, GSI_SAME_STMT
);
5545 stmt
= gimple_build_assign (vback
, t
);
5546 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5548 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
5549 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
5551 stmt
= gimple_build_cond_empty (t
);
5552 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5555 /* Remove GIMPLE_OMP_CONTINUE. */
5556 gsi_remove (&gsi
, true);
5558 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
5559 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
5561 /* Emit code to get the next parallel iteration in L2_BB. */
5562 gsi
= gsi_start_bb (l2_bb
);
5564 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
5565 build_fold_addr_expr (istart0
),
5566 build_fold_addr_expr (iend0
));
5567 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5568 false, GSI_CONTINUE_LINKING
);
5569 if (TREE_TYPE (t
) != boolean_type_node
)
5570 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
5571 t
, build_int_cst (TREE_TYPE (t
), 0));
5572 stmt
= gimple_build_cond_empty (t
);
5573 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5576 /* Add the loop cleanup function. */
5577 gsi
= gsi_last_bb (exit_bb
);
5578 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
5579 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
5580 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
5581 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
5583 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
5584 stmt
= gimple_build_call (t
, 0);
5585 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
5586 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
5587 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
5588 gsi_remove (&gsi
, true);
5590 /* Connect the new blocks. */
5591 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
5592 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
5598 e
= find_edge (cont_bb
, l3_bb
);
5599 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
5601 phis
= phi_nodes (l3_bb
);
5602 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5604 gimple phi
= gsi_stmt (gsi
);
5605 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
5606 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
5610 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
5612 add_bb_to_loop (l2_bb
, cont_bb
->loop_father
);
5613 e
= find_edge (cont_bb
, l1_bb
);
5614 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5619 else if (fd
->collapse
> 1)
5622 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
5625 e
->flags
= EDGE_TRUE_VALUE
;
5628 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5629 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
5633 e
= find_edge (cont_bb
, l2_bb
);
5634 e
->flags
= EDGE_FALLTHRU
;
5636 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
5638 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
5639 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
5640 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
5641 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
5642 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
5643 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
5644 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
5645 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
5647 struct loop
*outer_loop
= alloc_loop ();
5648 outer_loop
->header
= l0_bb
;
5649 outer_loop
->latch
= l2_bb
;
5650 add_loop (outer_loop
, l0_bb
->loop_father
);
5652 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5654 struct loop
*loop
= alloc_loop ();
5655 loop
->header
= l1_bb
;
5656 /* The loop may have multiple latches. */
5657 add_loop (loop
, outer_loop
);
5663 /* A subroutine of expand_omp_for. Generate code for a parallel
5664 loop with static schedule and no specified chunk size. Given
5667 for (V = N1; V cond N2; V += STEP) BODY;
5669 where COND is "<" or ">", we generate pseudocode
5671 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
5676 if ((__typeof (V)) -1 > 0 && cond is >)
5677 n = -(adj + N2 - N1) / -STEP;
5679 n = (adj + N2 - N1) / STEP;
5682 if (threadid < tt) goto L3; else goto L4;
5687 s0 = q * threadid + tt;
5690 if (s0 >= e0) goto L2; else goto L0;
5696 if (V cond e) goto L1;
5701 expand_omp_for_static_nochunk (struct omp_region
*region
,
5702 struct omp_for_data
*fd
,
5705 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
5706 tree type
, itype
, vmain
, vback
;
5707 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
5708 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
5710 gimple_stmt_iterator gsi
;
5713 enum built_in_function get_num_threads
= BUILT_IN_OMP_GET_NUM_THREADS
;
5714 enum built_in_function get_thread_num
= BUILT_IN_OMP_GET_THREAD_NUM
;
5715 bool broken_loop
= region
->cont
== NULL
;
5716 tree
*counts
= NULL
;
5719 itype
= type
= TREE_TYPE (fd
->loop
.v
);
5720 if (POINTER_TYPE_P (type
))
5721 itype
= signed_type_for (type
);
5723 entry_bb
= region
->entry
;
5724 cont_bb
= region
->cont
;
5725 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
5726 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
5727 gcc_assert (broken_loop
5728 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
5729 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
5730 body_bb
= single_succ (seq_start_bb
);
5733 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
5734 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
5736 exit_bb
= region
->exit
;
5738 /* Iteration space partitioning goes in ENTRY_BB. */
5739 gsi
= gsi_last_bb (entry_bb
);
5740 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5742 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
5744 get_num_threads
= BUILT_IN_OMP_GET_NUM_TEAMS
;
5745 get_thread_num
= BUILT_IN_OMP_GET_TEAM_NUM
;
5748 if (fd
->collapse
> 1)
5750 int first_zero_iter
= -1;
5751 basic_block l2_dom_bb
= NULL
;
5753 counts
= XALLOCAVEC (tree
, fd
->collapse
);
5754 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
5755 fin_bb
, first_zero_iter
,
5759 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5760 t
= integer_one_node
;
5762 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
5763 fold_convert (type
, fd
->loop
.n1
),
5764 fold_convert (type
, fd
->loop
.n2
));
5765 if (fd
->collapse
== 1
5766 && TYPE_UNSIGNED (type
)
5767 && (t
== NULL_TREE
|| !integer_onep (t
)))
5769 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
5770 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
5771 true, GSI_SAME_STMT
);
5772 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
5773 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
5774 true, GSI_SAME_STMT
);
5775 stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
5776 NULL_TREE
, NULL_TREE
);
5777 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5778 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
5779 expand_omp_regimplify_p
, NULL
, NULL
)
5780 || walk_tree (gimple_cond_rhs_ptr (stmt
),
5781 expand_omp_regimplify_p
, NULL
, NULL
))
5783 gsi
= gsi_for_stmt (stmt
);
5784 gimple_regimplify_operands (stmt
, &gsi
);
5786 ep
= split_block (entry_bb
, stmt
);
5787 ep
->flags
= EDGE_TRUE_VALUE
;
5788 entry_bb
= ep
->dest
;
5789 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
5790 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
5791 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
5792 if (gimple_in_ssa_p (cfun
))
5794 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
5795 for (gsi
= gsi_start_phis (fin_bb
);
5796 !gsi_end_p (gsi
); gsi_next (&gsi
))
5798 gimple phi
= gsi_stmt (gsi
);
5799 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
5800 ep
, UNKNOWN_LOCATION
);
5803 gsi
= gsi_last_bb (entry_bb
);
5806 t
= build_call_expr (builtin_decl_explicit (get_num_threads
), 0);
5807 t
= fold_convert (itype
, t
);
5808 nthreads
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5809 true, GSI_SAME_STMT
);
5811 t
= build_call_expr (builtin_decl_explicit (get_thread_num
), 0);
5812 t
= fold_convert (itype
, t
);
5813 threadid
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5814 true, GSI_SAME_STMT
);
5818 step
= fd
->loop
.step
;
5819 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5821 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5822 OMP_CLAUSE__LOOPTEMP_
);
5823 gcc_assert (innerc
);
5824 n1
= OMP_CLAUSE_DECL (innerc
);
5825 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5826 OMP_CLAUSE__LOOPTEMP_
);
5827 gcc_assert (innerc
);
5828 n2
= OMP_CLAUSE_DECL (innerc
);
5830 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
5831 true, NULL_TREE
, true, GSI_SAME_STMT
);
5832 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
5833 true, NULL_TREE
, true, GSI_SAME_STMT
);
5834 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
5835 true, NULL_TREE
, true, GSI_SAME_STMT
);
5837 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
5838 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
5839 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
5840 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
5841 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
5842 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5843 fold_build1 (NEGATE_EXPR
, itype
, t
),
5844 fold_build1 (NEGATE_EXPR
, itype
, step
));
5846 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
5847 t
= fold_convert (itype
, t
);
5848 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
5850 q
= create_tmp_reg (itype
, "q");
5851 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
5852 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
5853 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
5855 tt
= create_tmp_reg (itype
, "tt");
5856 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
5857 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
5858 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
5860 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
5861 stmt
= gimple_build_cond_empty (t
);
5862 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5864 second_bb
= split_block (entry_bb
, stmt
)->dest
;
5865 gsi
= gsi_last_bb (second_bb
);
5866 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5868 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
5870 stmt
= gimple_build_assign_with_ops (PLUS_EXPR
, q
, q
,
5871 build_int_cst (itype
, 1));
5872 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5874 third_bb
= split_block (second_bb
, stmt
)->dest
;
5875 gsi
= gsi_last_bb (third_bb
);
5876 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5878 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
5879 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
5880 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
5882 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
5883 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
5885 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
5886 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
5888 /* Remove the GIMPLE_OMP_FOR statement. */
5889 gsi_remove (&gsi
, true);
5891 /* Setup code for sequential iteration goes in SEQ_START_BB. */
5892 gsi
= gsi_start_bb (seq_start_bb
);
5894 tree startvar
= fd
->loop
.v
;
5895 tree endvar
= NULL_TREE
;
5897 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5899 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
5900 ? gimple_omp_parallel_clauses (inner_stmt
)
5901 : gimple_omp_for_clauses (inner_stmt
);
5902 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
5903 gcc_assert (innerc
);
5904 startvar
= OMP_CLAUSE_DECL (innerc
);
5905 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5906 OMP_CLAUSE__LOOPTEMP_
);
5907 gcc_assert (innerc
);
5908 endvar
= OMP_CLAUSE_DECL (innerc
);
5910 t
= fold_convert (itype
, s0
);
5911 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
5912 if (POINTER_TYPE_P (type
))
5913 t
= fold_build_pointer_plus (n1
, t
);
5915 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
5916 t
= fold_convert (TREE_TYPE (startvar
), t
);
5917 t
= force_gimple_operand_gsi (&gsi
, t
,
5919 && TREE_ADDRESSABLE (startvar
),
5920 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5921 stmt
= gimple_build_assign (startvar
, t
);
5922 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5924 t
= fold_convert (itype
, e0
);
5925 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
5926 if (POINTER_TYPE_P (type
))
5927 t
= fold_build_pointer_plus (n1
, t
);
5929 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
5930 t
= fold_convert (TREE_TYPE (startvar
), t
);
5931 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5932 false, GSI_CONTINUE_LINKING
);
5935 stmt
= gimple_build_assign (endvar
, e
);
5936 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5938 if (fd
->collapse
> 1)
5939 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
5943 /* The code controlling the sequential loop replaces the
5944 GIMPLE_OMP_CONTINUE. */
5945 gsi
= gsi_last_bb (cont_bb
);
5946 stmt
= gsi_stmt (gsi
);
5947 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
5948 vmain
= gimple_omp_continue_control_use (stmt
);
5949 vback
= gimple_omp_continue_control_def (stmt
);
5951 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5953 if (POINTER_TYPE_P (type
))
5954 t
= fold_build_pointer_plus (vmain
, step
);
5956 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
5957 t
= force_gimple_operand_gsi (&gsi
, t
,
5959 && TREE_ADDRESSABLE (vback
),
5960 NULL_TREE
, true, GSI_SAME_STMT
);
5961 stmt
= gimple_build_assign (vback
, t
);
5962 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5964 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
5965 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
5967 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
5970 /* Remove the GIMPLE_OMP_CONTINUE statement. */
5971 gsi_remove (&gsi
, true);
5973 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
5974 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
5977 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
5978 gsi
= gsi_last_bb (exit_bb
);
5979 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
5981 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
5982 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
5984 gsi_remove (&gsi
, true);
5986 /* Connect all the blocks. */
5987 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
5988 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
5989 ep
= find_edge (entry_bb
, second_bb
);
5990 ep
->flags
= EDGE_TRUE_VALUE
;
5991 ep
->probability
= REG_BR_PROB_BASE
/ 4;
5992 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
5993 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
5997 ep
= find_edge (cont_bb
, body_bb
);
5998 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6003 else if (fd
->collapse
> 1)
6006 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6009 ep
->flags
= EDGE_TRUE_VALUE
;
6010 find_edge (cont_bb
, fin_bb
)->flags
6011 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6014 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
6015 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
6016 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
6018 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6019 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6020 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6021 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6023 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
6025 struct loop
*loop
= alloc_loop ();
6026 loop
->header
= body_bb
;
6027 if (collapse_bb
== NULL
)
6028 loop
->latch
= cont_bb
;
6029 add_loop (loop
, body_bb
->loop_father
);
6034 /* A subroutine of expand_omp_for. Generate code for a parallel
6035 loop with static schedule and a specified chunk size. Given
6038 for (V = N1; V cond N2; V += STEP) BODY;
6040 where COND is "<" or ">", we generate pseudocode
6042 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6047 if ((__typeof (V)) -1 > 0 && cond is >)
6048 n = -(adj + N2 - N1) / -STEP;
6050 n = (adj + N2 - N1) / STEP;
6052 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
6053 here so that V is defined
6054 if the loop is not entered
6056 s0 = (trip * nthreads + threadid) * CHUNK;
6057 e0 = min(s0 + CHUNK, n);
6058 if (s0 < n) goto L1; else goto L4;
6065 if (V cond e) goto L2; else goto L3;
6073 expand_omp_for_static_chunk (struct omp_region
*region
,
6074 struct omp_for_data
*fd
, gimple inner_stmt
)
6076 tree n
, s0
, e0
, e
, t
;
6077 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
6078 tree type
, itype
, v_main
, v_back
, v_extra
;
6079 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
6080 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
6081 gimple_stmt_iterator si
;
6084 enum built_in_function get_num_threads
= BUILT_IN_OMP_GET_NUM_THREADS
;
6085 enum built_in_function get_thread_num
= BUILT_IN_OMP_GET_THREAD_NUM
;
6086 bool broken_loop
= region
->cont
== NULL
;
6087 tree
*counts
= NULL
;
6090 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6091 if (POINTER_TYPE_P (type
))
6092 itype
= signed_type_for (type
);
6094 entry_bb
= region
->entry
;
6095 se
= split_block (entry_bb
, last_stmt (entry_bb
));
6097 iter_part_bb
= se
->dest
;
6098 cont_bb
= region
->cont
;
6099 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
6100 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
6101 gcc_assert (broken_loop
6102 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
6103 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
6104 body_bb
= single_succ (seq_start_bb
);
6107 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6108 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6109 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
6111 exit_bb
= region
->exit
;
6113 /* Trip and adjustment setup goes in ENTRY_BB. */
6114 si
= gsi_last_bb (entry_bb
);
6115 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_FOR
);
6117 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
6119 get_num_threads
= BUILT_IN_OMP_GET_NUM_TEAMS
;
6120 get_thread_num
= BUILT_IN_OMP_GET_TEAM_NUM
;
6123 if (fd
->collapse
> 1)
6125 int first_zero_iter
= -1;
6126 basic_block l2_dom_bb
= NULL
;
6128 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6129 expand_omp_for_init_counts (fd
, &si
, entry_bb
, counts
,
6130 fin_bb
, first_zero_iter
,
6134 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6135 t
= integer_one_node
;
6137 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6138 fold_convert (type
, fd
->loop
.n1
),
6139 fold_convert (type
, fd
->loop
.n2
));
6140 if (fd
->collapse
== 1
6141 && TYPE_UNSIGNED (type
)
6142 && (t
== NULL_TREE
|| !integer_onep (t
)))
6144 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6145 n1
= force_gimple_operand_gsi (&si
, n1
, true, NULL_TREE
,
6146 true, GSI_SAME_STMT
);
6147 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6148 n2
= force_gimple_operand_gsi (&si
, n2
, true, NULL_TREE
,
6149 true, GSI_SAME_STMT
);
6150 stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6151 NULL_TREE
, NULL_TREE
);
6152 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
6153 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
6154 expand_omp_regimplify_p
, NULL
, NULL
)
6155 || walk_tree (gimple_cond_rhs_ptr (stmt
),
6156 expand_omp_regimplify_p
, NULL
, NULL
))
6158 si
= gsi_for_stmt (stmt
);
6159 gimple_regimplify_operands (stmt
, &si
);
6161 se
= split_block (entry_bb
, stmt
);
6162 se
->flags
= EDGE_TRUE_VALUE
;
6163 entry_bb
= se
->dest
;
6164 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6165 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6166 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6167 if (gimple_in_ssa_p (cfun
))
6169 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6170 for (si
= gsi_start_phis (fin_bb
);
6171 !gsi_end_p (si
); gsi_next (&si
))
6173 gimple phi
= gsi_stmt (si
);
6174 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6175 se
, UNKNOWN_LOCATION
);
6178 si
= gsi_last_bb (entry_bb
);
6181 t
= build_call_expr (builtin_decl_explicit (get_num_threads
), 0);
6182 t
= fold_convert (itype
, t
);
6183 nthreads
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6184 true, GSI_SAME_STMT
);
6186 t
= build_call_expr (builtin_decl_explicit (get_thread_num
), 0);
6187 t
= fold_convert (itype
, t
);
6188 threadid
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6189 true, GSI_SAME_STMT
);
6193 step
= fd
->loop
.step
;
6194 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6196 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6197 OMP_CLAUSE__LOOPTEMP_
);
6198 gcc_assert (innerc
);
6199 n1
= OMP_CLAUSE_DECL (innerc
);
6200 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6201 OMP_CLAUSE__LOOPTEMP_
);
6202 gcc_assert (innerc
);
6203 n2
= OMP_CLAUSE_DECL (innerc
);
6205 n1
= force_gimple_operand_gsi (&si
, fold_convert (type
, n1
),
6206 true, NULL_TREE
, true, GSI_SAME_STMT
);
6207 n2
= force_gimple_operand_gsi (&si
, fold_convert (itype
, n2
),
6208 true, NULL_TREE
, true, GSI_SAME_STMT
);
6209 step
= force_gimple_operand_gsi (&si
, fold_convert (itype
, step
),
6210 true, NULL_TREE
, true, GSI_SAME_STMT
);
6212 = force_gimple_operand_gsi (&si
, fold_convert (itype
, fd
->chunk_size
),
6213 true, NULL_TREE
, true, GSI_SAME_STMT
);
6215 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
6216 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
6217 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
6218 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
6219 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
6220 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6221 fold_build1 (NEGATE_EXPR
, itype
, t
),
6222 fold_build1 (NEGATE_EXPR
, itype
, step
));
6224 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
6225 t
= fold_convert (itype
, t
);
6226 n
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6227 true, GSI_SAME_STMT
);
6229 trip_var
= create_tmp_reg (itype
, ".trip");
6230 if (gimple_in_ssa_p (cfun
))
6232 trip_init
= make_ssa_name (trip_var
, NULL
);
6233 trip_main
= make_ssa_name (trip_var
, NULL
);
6234 trip_back
= make_ssa_name (trip_var
, NULL
);
6238 trip_init
= trip_var
;
6239 trip_main
= trip_var
;
6240 trip_back
= trip_var
;
6243 stmt
= gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
6244 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
6246 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, fd
->chunk_size
);
6247 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6248 if (POINTER_TYPE_P (type
))
6249 t
= fold_build_pointer_plus (n1
, t
);
6251 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6252 v_extra
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6253 true, GSI_SAME_STMT
);
6255 /* Remove the GIMPLE_OMP_FOR. */
6256 gsi_remove (&si
, true);
6258 /* Iteration space partitioning goes in ITER_PART_BB. */
6259 si
= gsi_last_bb (iter_part_bb
);
6261 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
6262 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
6263 t
= fold_build2 (MULT_EXPR
, itype
, t
, fd
->chunk_size
);
6264 s0
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6265 false, GSI_CONTINUE_LINKING
);
6267 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, fd
->chunk_size
);
6268 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
6269 e0
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6270 false, GSI_CONTINUE_LINKING
);
6272 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
6273 gsi_insert_after (&si
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
6275 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6276 si
= gsi_start_bb (seq_start_bb
);
6278 tree startvar
= fd
->loop
.v
;
6279 tree endvar
= NULL_TREE
;
6281 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6283 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6284 ? gimple_omp_parallel_clauses (inner_stmt
)
6285 : gimple_omp_for_clauses (inner_stmt
);
6286 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6287 gcc_assert (innerc
);
6288 startvar
= OMP_CLAUSE_DECL (innerc
);
6289 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6290 OMP_CLAUSE__LOOPTEMP_
);
6291 gcc_assert (innerc
);
6292 endvar
= OMP_CLAUSE_DECL (innerc
);
6295 t
= fold_convert (itype
, s0
);
6296 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6297 if (POINTER_TYPE_P (type
))
6298 t
= fold_build_pointer_plus (n1
, t
);
6300 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6301 t
= fold_convert (TREE_TYPE (startvar
), t
);
6302 t
= force_gimple_operand_gsi (&si
, t
,
6304 && TREE_ADDRESSABLE (startvar
),
6305 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6306 stmt
= gimple_build_assign (startvar
, t
);
6307 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
6309 t
= fold_convert (itype
, e0
);
6310 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6311 if (POINTER_TYPE_P (type
))
6312 t
= fold_build_pointer_plus (n1
, t
);
6314 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6315 t
= fold_convert (TREE_TYPE (startvar
), t
);
6316 e
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6317 false, GSI_CONTINUE_LINKING
);
6320 stmt
= gimple_build_assign (endvar
, e
);
6321 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
6323 if (fd
->collapse
> 1)
6324 expand_omp_for_init_vars (fd
, &si
, counts
, inner_stmt
, startvar
);
6328 /* The code controlling the sequential loop goes in CONT_BB,
6329 replacing the GIMPLE_OMP_CONTINUE. */
6330 si
= gsi_last_bb (cont_bb
);
6331 stmt
= gsi_stmt (si
);
6332 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6333 v_main
= gimple_omp_continue_control_use (stmt
);
6334 v_back
= gimple_omp_continue_control_def (stmt
);
6336 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6338 if (POINTER_TYPE_P (type
))
6339 t
= fold_build_pointer_plus (v_main
, step
);
6341 t
= fold_build2 (PLUS_EXPR
, type
, v_main
, step
);
6342 if (DECL_P (v_back
) && TREE_ADDRESSABLE (v_back
))
6343 t
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6344 true, GSI_SAME_STMT
);
6345 stmt
= gimple_build_assign (v_back
, t
);
6346 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
6348 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6349 DECL_P (v_back
) && TREE_ADDRESSABLE (v_back
)
6351 gsi_insert_before (&si
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6354 /* Remove GIMPLE_OMP_CONTINUE. */
6355 gsi_remove (&si
, true);
6357 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6358 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6360 /* Trip update code goes into TRIP_UPDATE_BB. */
6361 si
= gsi_start_bb (trip_update_bb
);
6363 t
= build_int_cst (itype
, 1);
6364 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
6365 stmt
= gimple_build_assign (trip_back
, t
);
6366 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
6369 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6370 si
= gsi_last_bb (exit_bb
);
6371 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
6373 t
= gimple_omp_return_lhs (gsi_stmt (si
));
6374 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
6376 gsi_remove (&si
, true);
6378 /* Connect the new blocks. */
6379 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
6380 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
6384 se
= find_edge (cont_bb
, body_bb
);
6385 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6390 else if (fd
->collapse
> 1)
6393 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6396 se
->flags
= EDGE_TRUE_VALUE
;
6397 find_edge (cont_bb
, trip_update_bb
)->flags
6398 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6400 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
6403 if (gimple_in_ssa_p (cfun
))
6405 gimple_stmt_iterator psi
;
6408 edge_var_map_vector
*head
;
6412 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
6414 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
6415 remove arguments of the phi nodes in fin_bb. We need to create
6416 appropriate phi nodes in iter_part_bb instead. */
6417 se
= single_pred_edge (fin_bb
);
6418 re
= single_succ_edge (trip_update_bb
);
6419 head
= redirect_edge_var_map_vector (re
);
6420 ene
= single_succ_edge (entry_bb
);
6422 psi
= gsi_start_phis (fin_bb
);
6423 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
6424 gsi_next (&psi
), ++i
)
6427 source_location locus
;
6429 phi
= gsi_stmt (psi
);
6430 t
= gimple_phi_result (phi
);
6431 gcc_assert (t
== redirect_edge_var_map_result (vm
));
6432 nphi
= create_phi_node (t
, iter_part_bb
);
6434 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
6435 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
6437 /* A special case -- fd->loop.v is not yet computed in
6438 iter_part_bb, we need to use v_extra instead. */
6439 if (t
== fd
->loop
.v
)
6441 add_phi_arg (nphi
, t
, ene
, locus
);
6442 locus
= redirect_edge_var_map_location (vm
);
6443 add_phi_arg (nphi
, redirect_edge_var_map_def (vm
), re
, locus
);
6445 gcc_assert (!gsi_end_p (psi
) && i
== head
->length ());
6446 redirect_edge_var_map_clear (re
);
6449 psi
= gsi_start_phis (fin_bb
);
6450 if (gsi_end_p (psi
))
6452 remove_phi_node (&psi
, false);
6455 /* Make phi node for trip. */
6456 phi
= create_phi_node (trip_main
, iter_part_bb
);
6457 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
6459 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
6464 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
6465 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
6466 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
6467 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6468 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6469 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
6470 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
6471 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6472 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6476 struct loop
*trip_loop
= alloc_loop ();
6477 trip_loop
->header
= iter_part_bb
;
6478 trip_loop
->latch
= trip_update_bb
;
6479 add_loop (trip_loop
, iter_part_bb
->loop_father
);
6481 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6483 struct loop
*loop
= alloc_loop ();
6484 loop
->header
= body_bb
;
6485 loop
->latch
= cont_bb
;
6486 add_loop (loop
, trip_loop
);
6492 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
6493 loop. Given parameters:
6495 for (V = N1; V cond N2; V += STEP) BODY;
6497 where COND is "<" or ">", we generate pseudocode
6505 if (V cond N2) goto L0; else goto L2;
6508 For collapsed loops, given parameters:
6510 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
6511 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
6512 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
6515 we generate pseudocode
6521 count3 = (adj + N32 - N31) / STEP3;
6526 count2 = (adj + N22 - N21) / STEP2;
6531 count1 = (adj + N12 - N11) / STEP1;
6532 count = count1 * count2 * count3;
6542 V2 += (V3 cond3 N32) ? 0 : STEP2;
6543 V3 = (V3 cond3 N32) ? V3 : N31;
6544 V1 += (V2 cond2 N22) ? 0 : STEP1;
6545 V2 = (V2 cond2 N22) ? V2 : N21;
6547 if (V < count) goto L0; else goto L2;
6553 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
6556 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
6557 gimple_stmt_iterator gsi
;
6559 bool broken_loop
= region
->cont
== NULL
;
6561 tree
*counts
= NULL
;
6563 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6564 OMP_CLAUSE_SAFELEN
);
6565 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6566 OMP_CLAUSE__SIMDUID_
);
6569 type
= TREE_TYPE (fd
->loop
.v
);
6570 entry_bb
= region
->entry
;
6571 cont_bb
= region
->cont
;
6572 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6573 gcc_assert (broken_loop
6574 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
6575 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
6578 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
6579 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6580 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
6581 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6585 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
6586 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
6587 l2_bb
= single_succ (l1_bb
);
6589 exit_bb
= region
->exit
;
6592 gsi
= gsi_last_bb (entry_bb
);
6594 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6595 /* Not needed in SSA form right now. */
6596 gcc_assert (!gimple_in_ssa_p (cfun
));
6597 if (fd
->collapse
> 1)
6599 int first_zero_iter
= -1;
6600 basic_block zero_iter_bb
= l2_bb
;
6602 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6603 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6604 zero_iter_bb
, first_zero_iter
,
6607 if (l2_dom_bb
== NULL
)
6612 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6614 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6615 OMP_CLAUSE__LOOPTEMP_
);
6616 gcc_assert (innerc
);
6617 n1
= OMP_CLAUSE_DECL (innerc
);
6618 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6619 OMP_CLAUSE__LOOPTEMP_
);
6620 gcc_assert (innerc
);
6621 n2
= OMP_CLAUSE_DECL (innerc
);
6622 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
6623 fold_convert (type
, n1
));
6624 if (fd
->collapse
> 1)
6627 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
6633 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
6634 fold_convert (type
, fd
->loop
.n1
));
6635 if (fd
->collapse
> 1)
6636 for (i
= 0; i
< fd
->collapse
; i
++)
6638 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
6639 if (POINTER_TYPE_P (itype
))
6640 itype
= signed_type_for (itype
);
6641 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
6642 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6646 /* Remove the GIMPLE_OMP_FOR statement. */
6647 gsi_remove (&gsi
, true);
6651 /* Code to control the increment goes in the CONT_BB. */
6652 gsi
= gsi_last_bb (cont_bb
);
6653 stmt
= gsi_stmt (gsi
);
6654 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6656 if (POINTER_TYPE_P (type
))
6657 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
6659 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
6660 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
6662 if (fd
->collapse
> 1)
6664 i
= fd
->collapse
- 1;
6665 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
6667 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
6668 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
6672 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
6674 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
6677 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6679 for (i
= fd
->collapse
- 1; i
> 0; i
--)
6681 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
6682 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
6683 if (POINTER_TYPE_P (itype2
))
6684 itype2
= signed_type_for (itype2
);
6685 t
= build3 (COND_EXPR
, itype2
,
6686 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
6688 fold_convert (itype
, fd
->loops
[i
].n2
)),
6689 build_int_cst (itype2
, 0),
6690 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
6691 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
6692 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
6694 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
6695 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
6697 t
= build3 (COND_EXPR
, itype
,
6698 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
6700 fold_convert (itype
, fd
->loops
[i
].n2
)),
6702 fold_convert (itype
, fd
->loops
[i
].n1
));
6703 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6707 /* Remove GIMPLE_OMP_CONTINUE. */
6708 gsi_remove (&gsi
, true);
6711 /* Emit the condition in L1_BB. */
6712 gsi
= gsi_start_bb (l1_bb
);
6714 t
= fold_convert (type
, n2
);
6715 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6716 false, GSI_CONTINUE_LINKING
);
6717 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, fd
->loop
.v
, t
);
6718 stmt
= gimple_build_cond_empty (t
);
6719 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6720 if (walk_tree (gimple_cond_lhs_ptr (stmt
), expand_omp_regimplify_p
,
6722 || walk_tree (gimple_cond_rhs_ptr (stmt
), expand_omp_regimplify_p
,
6725 gsi
= gsi_for_stmt (stmt
);
6726 gimple_regimplify_operands (stmt
, &gsi
);
6729 /* Remove GIMPLE_OMP_RETURN. */
6730 gsi
= gsi_last_bb (exit_bb
);
6731 gsi_remove (&gsi
, true);
6733 /* Connect the new blocks. */
6734 remove_edge (FALLTHRU_EDGE (entry_bb
));
6738 remove_edge (BRANCH_EDGE (entry_bb
));
6739 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
6741 e
= BRANCH_EDGE (l1_bb
);
6742 ne
= FALLTHRU_EDGE (l1_bb
);
6743 e
->flags
= EDGE_TRUE_VALUE
;
6747 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
6749 ne
= single_succ_edge (l1_bb
);
6750 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
6753 ne
->flags
= EDGE_FALSE_VALUE
;
6754 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
6755 ne
->probability
= REG_BR_PROB_BASE
/ 8;
6757 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
6758 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
6759 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
6763 struct loop
*loop
= alloc_loop ();
6764 loop
->header
= l1_bb
;
6765 loop
->latch
= e
->dest
;
6766 add_loop (loop
, l1_bb
->loop_father
);
6767 if (safelen
== NULL_TREE
)
6768 loop
->safelen
= INT_MAX
;
6771 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
6772 if (!tree_fits_uhwi_p (safelen
)
6773 || tree_to_uhwi (safelen
) > INT_MAX
)
6774 loop
->safelen
= INT_MAX
;
6776 loop
->safelen
= tree_to_uhwi (safelen
);
6777 if (loop
->safelen
== 1)
6782 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
6783 cfun
->has_simduid_loops
= true;
6785 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
6787 if ((flag_tree_loop_vectorize
6788 || (!global_options_set
.x_flag_tree_loop_vectorize
6789 && !global_options_set
.x_flag_tree_vectorize
))
6790 && loop
->safelen
> 1)
6792 loop
->force_vect
= true;
6793 cfun
->has_force_vect_loops
= true;
6799 /* Expand the OpenMP loop defined by REGION. */
6802 expand_omp_for (struct omp_region
*region
, gimple inner_stmt
)
6804 struct omp_for_data fd
;
6805 struct omp_for_data_loop
*loops
;
6808 = (struct omp_for_data_loop
*)
6809 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
6810 * sizeof (struct omp_for_data_loop
));
6811 extract_omp_for_data (last_stmt (region
->entry
), &fd
, loops
);
6812 region
->sched_kind
= fd
.sched_kind
;
6814 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
6815 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
6816 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
6819 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
6820 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
6821 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
6824 /* If there isn't a continue then this is a degerate case where
6825 the introduction of abnormal edges during lowering will prevent
6826 original loops from being detected. Fix that up. */
6827 loops_state_set (LOOPS_NEED_FIXUP
);
6829 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_KIND_SIMD
)
6830 expand_omp_simd (region
, &fd
);
6831 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
6832 && !fd
.have_ordered
)
6834 if (fd
.chunk_size
== NULL
)
6835 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
6837 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
6841 int fn_index
, start_ix
, next_ix
;
6843 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
6844 == GF_OMP_FOR_KIND_FOR
);
6845 if (fd
.chunk_size
== NULL
6846 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
6847 fd
.chunk_size
= integer_zero_node
;
6848 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
6849 fn_index
= (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
6850 ? 3 : fd
.sched_kind
;
6851 fn_index
+= fd
.have_ordered
* 4;
6852 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
6853 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
6854 if (fd
.iter_type
== long_long_unsigned_type_node
)
6856 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
6857 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
6858 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
6859 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
6861 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
6862 (enum built_in_function
) next_ix
, inner_stmt
);
6865 if (gimple_in_ssa_p (cfun
))
6866 update_ssa (TODO_update_ssa_only_virtuals
);
6870 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
6872 v = GOMP_sections_start (n);
6889 v = GOMP_sections_next ();
6894 If this is a combined parallel sections, replace the call to
6895 GOMP_sections_start with call to GOMP_sections_next. */
6898 expand_omp_sections (struct omp_region
*region
)
6900 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
6901 vec
<tree
> label_vec
;
6903 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
6904 gimple_stmt_iterator si
, switch_si
;
6905 gimple sections_stmt
, stmt
, cont
;
6908 struct omp_region
*inner
;
6910 bool exit_reachable
= region
->cont
!= NULL
;
6912 gcc_assert (region
->exit
!= NULL
);
6913 entry_bb
= region
->entry
;
6914 l0_bb
= single_succ (entry_bb
);
6915 l1_bb
= region
->cont
;
6916 l2_bb
= region
->exit
;
6917 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
6918 l2
= gimple_block_label (l2_bb
);
6921 /* This can happen if there are reductions. */
6922 len
= EDGE_COUNT (l0_bb
->succs
);
6923 gcc_assert (len
> 0);
6924 e
= EDGE_SUCC (l0_bb
, len
- 1);
6925 si
= gsi_last_bb (e
->dest
);
6928 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
6929 l2
= gimple_block_label (e
->dest
);
6931 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
6933 si
= gsi_last_bb (e
->dest
);
6935 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
6937 l2
= gimple_block_label (e
->dest
);
6943 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
6945 default_bb
= create_empty_bb (l0_bb
);
6947 /* We will build a switch() with enough cases for all the
6948 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
6949 and a default case to abort if something goes wrong. */
6950 len
= EDGE_COUNT (l0_bb
->succs
);
6952 /* Use vec::quick_push on label_vec throughout, since we know the size
6954 label_vec
.create (len
);
6956 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
6957 GIMPLE_OMP_SECTIONS statement. */
6958 si
= gsi_last_bb (entry_bb
);
6959 sections_stmt
= gsi_stmt (si
);
6960 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
6961 vin
= gimple_omp_sections_control (sections_stmt
);
6962 if (!is_combined_parallel (region
))
6964 /* If we are not inside a combined parallel+sections region,
6965 call GOMP_sections_start. */
6966 t
= build_int_cst (unsigned_type_node
, len
- 1);
6967 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
6968 stmt
= gimple_build_call (u
, 1, t
);
6972 /* Otherwise, call GOMP_sections_next. */
6973 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
6974 stmt
= gimple_build_call (u
, 0);
6976 gimple_call_set_lhs (stmt
, vin
);
6977 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
6978 gsi_remove (&si
, true);
6980 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
6982 switch_si
= gsi_last_bb (l0_bb
);
6983 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
6986 cont
= last_stmt (l1_bb
);
6987 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
6988 vmain
= gimple_omp_continue_control_use (cont
);
6989 vnext
= gimple_omp_continue_control_def (cont
);
6997 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
6998 label_vec
.quick_push (t
);
7001 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
7002 for (inner
= region
->inner
, casei
= 1;
7004 inner
= inner
->next
, i
++, casei
++)
7006 basic_block s_entry_bb
, s_exit_bb
;
7008 /* Skip optional reduction region. */
7009 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
7016 s_entry_bb
= inner
->entry
;
7017 s_exit_bb
= inner
->exit
;
7019 t
= gimple_block_label (s_entry_bb
);
7020 u
= build_int_cst (unsigned_type_node
, casei
);
7021 u
= build_case_label (u
, NULL
, t
);
7022 label_vec
.quick_push (u
);
7024 si
= gsi_last_bb (s_entry_bb
);
7025 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
7026 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
7027 gsi_remove (&si
, true);
7028 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
7030 if (s_exit_bb
== NULL
)
7033 si
= gsi_last_bb (s_exit_bb
);
7034 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
7035 gsi_remove (&si
, true);
7037 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
7040 /* Error handling code goes in DEFAULT_BB. */
7041 t
= gimple_block_label (default_bb
);
7042 u
= build_case_label (NULL
, NULL
, t
);
7043 make_edge (l0_bb
, default_bb
, 0);
7045 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
7047 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
7048 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
7049 gsi_remove (&switch_si
, true);
7050 label_vec
.release ();
7052 si
= gsi_start_bb (default_bb
);
7053 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
7054 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
7060 /* Code to get the next section goes in L1_BB. */
7061 si
= gsi_last_bb (l1_bb
);
7062 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
7064 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
7065 stmt
= gimple_build_call (bfn_decl
, 0);
7066 gimple_call_set_lhs (stmt
, vnext
);
7067 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7068 gsi_remove (&si
, true);
7070 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
7073 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
7074 si
= gsi_last_bb (l2_bb
);
7075 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
7076 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
7077 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
7078 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
7080 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
7081 stmt
= gimple_build_call (t
, 0);
7082 if (gimple_omp_return_lhs (gsi_stmt (si
)))
7083 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
7084 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7085 gsi_remove (&si
, true);
7087 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
7091 /* Expand code for an OpenMP single directive. We've already expanded
7092 much of the code, here we simply place the GOMP_barrier call. */
7095 expand_omp_single (struct omp_region
*region
)
7097 basic_block entry_bb
, exit_bb
;
7098 gimple_stmt_iterator si
;
7100 entry_bb
= region
->entry
;
7101 exit_bb
= region
->exit
;
7103 si
= gsi_last_bb (entry_bb
);
7104 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
7105 gsi_remove (&si
, true);
7106 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7108 si
= gsi_last_bb (exit_bb
);
7109 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
7111 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
7112 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
7114 gsi_remove (&si
, true);
7115 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
7119 /* Generic expansion for OpenMP synchronization directives: master,
7120 ordered and critical. All we need to do here is remove the entry
7121 and exit markers for REGION. */
7124 expand_omp_synch (struct omp_region
*region
)
7126 basic_block entry_bb
, exit_bb
;
7127 gimple_stmt_iterator si
;
7129 entry_bb
= region
->entry
;
7130 exit_bb
= region
->exit
;
7132 si
= gsi_last_bb (entry_bb
);
7133 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
7134 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
7135 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
7136 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
7137 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
7138 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
7139 gsi_remove (&si
, true);
7140 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7144 si
= gsi_last_bb (exit_bb
);
7145 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
7146 gsi_remove (&si
, true);
7147 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
7151 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7152 operation as a normal volatile load. */
7155 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
7156 tree loaded_val
, int index
)
7158 enum built_in_function tmpbase
;
7159 gimple_stmt_iterator gsi
;
7160 basic_block store_bb
;
7163 tree decl
, call
, type
, itype
;
7165 gsi
= gsi_last_bb (load_bb
);
7166 stmt
= gsi_stmt (gsi
);
7167 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7168 loc
= gimple_location (stmt
);
7170 /* ??? If the target does not implement atomic_load_optab[mode], and mode
7171 is smaller than word size, then expand_atomic_load assumes that the load
7172 is atomic. We could avoid the builtin entirely in this case. */
7174 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
7175 decl
= builtin_decl_explicit (tmpbase
);
7176 if (decl
== NULL_TREE
)
7179 type
= TREE_TYPE (loaded_val
);
7180 itype
= TREE_TYPE (TREE_TYPE (decl
));
7182 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
7183 build_int_cst (NULL
,
7184 gimple_omp_atomic_seq_cst_p (stmt
)
7186 : MEMMODEL_RELAXED
));
7187 if (!useless_type_conversion_p (type
, itype
))
7188 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7189 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7191 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7192 gsi_remove (&gsi
, true);
7194 store_bb
= single_succ (load_bb
);
7195 gsi
= gsi_last_bb (store_bb
);
7196 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7197 gsi_remove (&gsi
, true);
7199 if (gimple_in_ssa_p (cfun
))
7200 update_ssa (TODO_update_ssa_no_phi
);
7205 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7206 operation as a normal volatile store. */
7209 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
7210 tree loaded_val
, tree stored_val
, int index
)
7212 enum built_in_function tmpbase
;
7213 gimple_stmt_iterator gsi
;
7214 basic_block store_bb
= single_succ (load_bb
);
7217 tree decl
, call
, type
, itype
;
7218 enum machine_mode imode
;
7221 gsi
= gsi_last_bb (load_bb
);
7222 stmt
= gsi_stmt (gsi
);
7223 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7225 /* If the load value is needed, then this isn't a store but an exchange. */
7226 exchange
= gimple_omp_atomic_need_value_p (stmt
);
7228 gsi
= gsi_last_bb (store_bb
);
7229 stmt
= gsi_stmt (gsi
);
7230 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
7231 loc
= gimple_location (stmt
);
7233 /* ??? If the target does not implement atomic_store_optab[mode], and mode
7234 is smaller than word size, then expand_atomic_store assumes that the store
7235 is atomic. We could avoid the builtin entirely in this case. */
7237 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
7238 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
7239 decl
= builtin_decl_explicit (tmpbase
);
7240 if (decl
== NULL_TREE
)
7243 type
= TREE_TYPE (stored_val
);
7245 /* Dig out the type of the function's second argument. */
7246 itype
= TREE_TYPE (decl
);
7247 itype
= TYPE_ARG_TYPES (itype
);
7248 itype
= TREE_CHAIN (itype
);
7249 itype
= TREE_VALUE (itype
);
7250 imode
= TYPE_MODE (itype
);
7252 if (exchange
&& !can_atomic_exchange_p (imode
, true))
7255 if (!useless_type_conversion_p (itype
, type
))
7256 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
7257 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
7258 build_int_cst (NULL
,
7259 gimple_omp_atomic_seq_cst_p (stmt
)
7261 : MEMMODEL_RELAXED
));
7264 if (!useless_type_conversion_p (type
, itype
))
7265 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7266 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7269 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7270 gsi_remove (&gsi
, true);
7272 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
7273 gsi
= gsi_last_bb (load_bb
);
7274 gsi_remove (&gsi
, true);
7276 if (gimple_in_ssa_p (cfun
))
7277 update_ssa (TODO_update_ssa_no_phi
);
7282 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7283 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
7284 size of the data type, and thus usable to find the index of the builtin
7285 decl. Returns false if the expression is not of the proper form. */
7288 expand_omp_atomic_fetch_op (basic_block load_bb
,
7289 tree addr
, tree loaded_val
,
7290 tree stored_val
, int index
)
7292 enum built_in_function oldbase
, newbase
, tmpbase
;
7293 tree decl
, itype
, call
;
7295 basic_block store_bb
= single_succ (load_bb
);
7296 gimple_stmt_iterator gsi
;
7299 enum tree_code code
;
7300 bool need_old
, need_new
;
7301 enum machine_mode imode
;
7304 /* We expect to find the following sequences:
7307 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
7310 val = tmp OP something; (or: something OP tmp)
7311 GIMPLE_OMP_STORE (val)
7313 ???FIXME: Allow a more flexible sequence.
7314 Perhaps use data flow to pick the statements.
7318 gsi
= gsi_after_labels (store_bb
);
7319 stmt
= gsi_stmt (gsi
);
7320 loc
= gimple_location (stmt
);
7321 if (!is_gimple_assign (stmt
))
7324 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
7326 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
7327 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
7328 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
7329 gcc_checking_assert (!need_old
|| !need_new
);
7331 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
7334 /* Check for one of the supported fetch-op operations. */
7335 code
= gimple_assign_rhs_code (stmt
);
7339 case POINTER_PLUS_EXPR
:
7340 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
7341 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
7344 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
7345 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
7348 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
7349 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
7352 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
7353 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
7356 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
7357 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
7363 /* Make sure the expression is of the proper form. */
7364 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
7365 rhs
= gimple_assign_rhs2 (stmt
);
7366 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
7367 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
7368 rhs
= gimple_assign_rhs1 (stmt
);
7372 tmpbase
= ((enum built_in_function
)
7373 ((need_new
? newbase
: oldbase
) + index
+ 1));
7374 decl
= builtin_decl_explicit (tmpbase
);
7375 if (decl
== NULL_TREE
)
7377 itype
= TREE_TYPE (TREE_TYPE (decl
));
7378 imode
= TYPE_MODE (itype
);
7380 /* We could test all of the various optabs involved, but the fact of the
7381 matter is that (with the exception of i486 vs i586 and xadd) all targets
7382 that support any atomic operaton optab also implements compare-and-swap.
7383 Let optabs.c take care of expanding any compare-and-swap loop. */
7384 if (!can_compare_and_swap_p (imode
, true))
7387 gsi
= gsi_last_bb (load_bb
);
7388 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7390 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
7391 It only requires that the operation happen atomically. Thus we can
7392 use the RELAXED memory model. */
7393 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
7394 fold_convert_loc (loc
, itype
, rhs
),
7395 build_int_cst (NULL
,
7396 seq_cst
? MEMMODEL_SEQ_CST
7397 : MEMMODEL_RELAXED
));
7399 if (need_old
|| need_new
)
7401 lhs
= need_old
? loaded_val
: stored_val
;
7402 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
7403 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
7406 call
= fold_convert_loc (loc
, void_type_node
, call
);
7407 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7408 gsi_remove (&gsi
, true);
7410 gsi
= gsi_last_bb (store_bb
);
7411 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7412 gsi_remove (&gsi
, true);
7413 gsi
= gsi_last_bb (store_bb
);
7414 gsi_remove (&gsi
, true);
7416 if (gimple_in_ssa_p (cfun
))
7417 update_ssa (TODO_update_ssa_no_phi
);
7422 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7426 newval = rhs; // with oldval replacing *addr in rhs
7427 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
7428 if (oldval != newval)
7431 INDEX is log2 of the size of the data type, and thus usable to find the
7432 index of the builtin decl. */
7435 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
7436 tree addr
, tree loaded_val
, tree stored_val
,
7439 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
7440 tree type
, itype
, cmpxchg
, iaddr
;
7441 gimple_stmt_iterator si
;
7442 basic_block loop_header
= single_succ (load_bb
);
7445 enum built_in_function fncode
;
7447 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
7448 order to use the RELAXED memory model effectively. */
7449 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
7451 cmpxchg
= builtin_decl_explicit (fncode
);
7452 if (cmpxchg
== NULL_TREE
)
7454 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
7455 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
7457 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
7460 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
7461 si
= gsi_last_bb (load_bb
);
7462 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7464 /* For floating-point values, we'll need to view-convert them to integers
7465 so that we can perform the atomic compare and swap. Simplify the
7466 following code by always setting up the "i"ntegral variables. */
7467 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
7471 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
7474 = force_gimple_operand_gsi (&si
,
7475 fold_convert (TREE_TYPE (iaddr
), addr
),
7476 false, NULL_TREE
, true, GSI_SAME_STMT
);
7477 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
7478 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7479 loadedi
= create_tmp_var (itype
, NULL
);
7480 if (gimple_in_ssa_p (cfun
))
7481 loadedi
= make_ssa_name (loadedi
, NULL
);
7486 loadedi
= loaded_val
;
7490 = force_gimple_operand_gsi (&si
,
7491 build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)),
7493 build_int_cst (TREE_TYPE (iaddr
), 0)),
7494 true, NULL_TREE
, true, GSI_SAME_STMT
);
7496 /* Move the value to the LOADEDI temporary. */
7497 if (gimple_in_ssa_p (cfun
))
7499 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
7500 phi
= create_phi_node (loadedi
, loop_header
);
7501 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
7505 gsi_insert_before (&si
,
7506 gimple_build_assign (loadedi
, initial
),
7508 if (loadedi
!= loaded_val
)
7510 gimple_stmt_iterator gsi2
;
7513 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
7514 gsi2
= gsi_start_bb (loop_header
);
7515 if (gimple_in_ssa_p (cfun
))
7518 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
7519 true, GSI_SAME_STMT
);
7520 stmt
= gimple_build_assign (loaded_val
, x
);
7521 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
7525 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
7526 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
7527 true, GSI_SAME_STMT
);
7530 gsi_remove (&si
, true);
7532 si
= gsi_last_bb (store_bb
);
7533 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
7536 storedi
= stored_val
;
7539 force_gimple_operand_gsi (&si
,
7540 build1 (VIEW_CONVERT_EXPR
, itype
,
7541 stored_val
), true, NULL_TREE
, true,
7544 /* Build the compare&swap statement. */
7545 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
7546 new_storedi
= force_gimple_operand_gsi (&si
,
7547 fold_convert (TREE_TYPE (loadedi
),
7550 true, GSI_SAME_STMT
);
7552 if (gimple_in_ssa_p (cfun
))
7556 old_vali
= create_tmp_var (TREE_TYPE (loadedi
), NULL
);
7557 stmt
= gimple_build_assign (old_vali
, loadedi
);
7558 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7560 stmt
= gimple_build_assign (loadedi
, new_storedi
);
7561 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7564 /* Note that we always perform the comparison as an integer, even for
7565 floating point. This allows the atomic operation to properly
7566 succeed even with NaNs and -0.0. */
7567 stmt
= gimple_build_cond_empty
7568 (build2 (NE_EXPR
, boolean_type_node
,
7569 new_storedi
, old_vali
));
7570 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7573 e
= single_succ_edge (store_bb
);
7574 e
->flags
&= ~EDGE_FALLTHRU
;
7575 e
->flags
|= EDGE_FALSE_VALUE
;
7577 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
7579 /* Copy the new value to loadedi (we already did that before the condition
7580 if we are not in SSA). */
7581 if (gimple_in_ssa_p (cfun
))
7583 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
7584 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
7587 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
7588 gsi_remove (&si
, true);
7590 struct loop
*loop
= alloc_loop ();
7591 loop
->header
= loop_header
;
7592 loop
->latch
= store_bb
;
7593 add_loop (loop
, loop_header
->loop_father
);
7595 if (gimple_in_ssa_p (cfun
))
7596 update_ssa (TODO_update_ssa_no_phi
);
7601 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7603 GOMP_atomic_start ();
7607 The result is not globally atomic, but works so long as all parallel
7608 references are within #pragma omp atomic directives. According to
7609 responses received from omp@openmp.org, appears to be within spec.
7610 Which makes sense, since that's how several other compilers handle
7611 this situation as well.
7612 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
7613 expanding. STORED_VAL is the operand of the matching
7614 GIMPLE_OMP_ATOMIC_STORE.
7617 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
7621 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
7626 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
7627 tree addr
, tree loaded_val
, tree stored_val
)
7629 gimple_stmt_iterator si
;
7633 si
= gsi_last_bb (load_bb
);
7634 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7636 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
7637 t
= build_call_expr (t
, 0);
7638 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7640 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
7641 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7642 gsi_remove (&si
, true);
7644 si
= gsi_last_bb (store_bb
);
7645 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
7647 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
7649 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7651 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
7652 t
= build_call_expr (t
, 0);
7653 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7654 gsi_remove (&si
, true);
7656 if (gimple_in_ssa_p (cfun
))
7657 update_ssa (TODO_update_ssa_no_phi
);
7661 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
7662 using expand_omp_atomic_fetch_op. If it failed, we try to
7663 call expand_omp_atomic_pipeline, and if it fails too, the
7664 ultimate fallback is wrapping the operation in a mutex
7665 (expand_omp_atomic_mutex). REGION is the atomic region built
7666 by build_omp_regions_1(). */
7669 expand_omp_atomic (struct omp_region
*region
)
7671 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
7672 gimple load
= last_stmt (load_bb
), store
= last_stmt (store_bb
);
7673 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
7674 tree addr
= gimple_omp_atomic_load_rhs (load
);
7675 tree stored_val
= gimple_omp_atomic_store_val (store
);
7676 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
7677 HOST_WIDE_INT index
;
7679 /* Make sure the type is one of the supported sizes. */
7680 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
7681 index
= exact_log2 (index
);
7682 if (index
>= 0 && index
<= 4)
7684 unsigned int align
= TYPE_ALIGN_UNIT (type
);
7686 /* __sync builtins require strict data alignment. */
7687 if (exact_log2 (align
) >= index
)
7690 if (loaded_val
== stored_val
7691 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
7692 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
7693 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
7694 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
7698 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
7699 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
7700 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
7701 && store_bb
== single_succ (load_bb
)
7702 && first_stmt (store_bb
) == store
7703 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
7707 /* When possible, use specialized atomic update functions. */
7708 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
7709 && store_bb
== single_succ (load_bb
)
7710 && expand_omp_atomic_fetch_op (load_bb
, addr
,
7711 loaded_val
, stored_val
, index
))
7714 /* If we don't have specialized __sync builtins, try and implement
7715 as a compare and swap loop. */
7716 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
7717 loaded_val
, stored_val
, index
))
7722 /* The ultimate fallback is wrapping the operation in a mutex. */
7723 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
7727 /* Expand the OpenMP target{, data, update} directive starting at REGION. */
7730 expand_omp_target (struct omp_region
*region
)
7732 basic_block entry_bb
, exit_bb
, new_bb
;
7733 struct function
*child_cfun
= NULL
;
7734 tree child_fn
= NULL_TREE
, block
, t
;
7735 gimple_stmt_iterator gsi
;
7736 gimple entry_stmt
, stmt
;
7739 entry_stmt
= last_stmt (region
->entry
);
7740 new_bb
= region
->entry
;
7741 int kind
= gimple_omp_target_kind (entry_stmt
);
7742 if (kind
== GF_OMP_TARGET_KIND_REGION
)
7744 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
7745 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
7748 entry_bb
= region
->entry
;
7749 exit_bb
= region
->exit
;
7751 if (kind
== GF_OMP_TARGET_KIND_REGION
)
7753 unsigned srcidx
, dstidx
, num
;
7755 /* If the target region needs data sent from the parent
7756 function, then the very first statement (except possible
7757 tree profile counter updates) of the parallel body
7758 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
7759 &.OMP_DATA_O is passed as an argument to the child function,
7760 we need to replace it with the argument as seen by the child
7763 In most cases, this will end up being the identity assignment
7764 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
7765 a function call that has been inlined, the original PARM_DECL
7766 .OMP_DATA_I may have been converted into a different local
7767 variable. In which case, we need to keep the assignment. */
7768 if (gimple_omp_target_data_arg (entry_stmt
))
7770 basic_block entry_succ_bb
= single_succ (entry_bb
);
7771 gimple_stmt_iterator gsi
;
7773 gimple tgtcopy_stmt
= NULL
;
7775 = TREE_VEC_ELT (gimple_omp_target_data_arg (entry_stmt
), 0);
7777 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
7779 gcc_assert (!gsi_end_p (gsi
));
7780 stmt
= gsi_stmt (gsi
);
7781 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
7784 if (gimple_num_ops (stmt
) == 2)
7786 tree arg
= gimple_assign_rhs1 (stmt
);
7788 /* We're ignoring the subcode because we're
7789 effectively doing a STRIP_NOPS. */
7791 if (TREE_CODE (arg
) == ADDR_EXPR
7792 && TREE_OPERAND (arg
, 0) == sender
)
7794 tgtcopy_stmt
= stmt
;
7800 gcc_assert (tgtcopy_stmt
!= NULL
);
7801 arg
= DECL_ARGUMENTS (child_fn
);
7803 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
7804 gsi_remove (&gsi
, true);
7807 /* Declare local variables needed in CHILD_CFUN. */
7808 block
= DECL_INITIAL (child_fn
);
7809 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
7810 /* The gimplifier could record temporaries in target block
7811 rather than in containing function's local_decls chain,
7812 which would mean cgraph missed finalizing them. Do it now. */
7813 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
7814 if (TREE_CODE (t
) == VAR_DECL
7816 && !DECL_EXTERNAL (t
))
7817 varpool_finalize_decl (t
);
7818 DECL_SAVED_TREE (child_fn
) = NULL
;
7819 /* We'll create a CFG for child_fn, so no gimple body is needed. */
7820 gimple_set_body (child_fn
, NULL
);
7821 TREE_USED (block
) = 1;
7823 /* Reset DECL_CONTEXT on function arguments. */
7824 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
7825 DECL_CONTEXT (t
) = child_fn
;
7827 /* Split ENTRY_BB at GIMPLE_OMP_TARGET,
7828 so that it can be moved to the child function. */
7829 gsi
= gsi_last_bb (entry_bb
);
7830 stmt
= gsi_stmt (gsi
);
7831 gcc_assert (stmt
&& gimple_code (stmt
) == GIMPLE_OMP_TARGET
7832 && gimple_omp_target_kind (stmt
)
7833 == GF_OMP_TARGET_KIND_REGION
);
7834 gsi_remove (&gsi
, true);
7835 e
= split_block (entry_bb
, stmt
);
7837 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7839 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
7842 gsi
= gsi_last_bb (exit_bb
);
7843 gcc_assert (!gsi_end_p (gsi
)
7844 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
7845 stmt
= gimple_build_return (NULL
);
7846 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
7847 gsi_remove (&gsi
, true);
7850 /* Move the target region into CHILD_CFUN. */
7852 block
= gimple_block (entry_stmt
);
7854 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
7856 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
7857 /* When the OMP expansion process cannot guarantee an up-to-date
7858 loop tree arrange for the child function to fixup loops. */
7859 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
7860 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
7862 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
7863 num
= vec_safe_length (child_cfun
->local_decls
);
7864 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
7866 t
= (*child_cfun
->local_decls
)[srcidx
];
7867 if (DECL_CONTEXT (t
) == cfun
->decl
)
7869 if (srcidx
!= dstidx
)
7870 (*child_cfun
->local_decls
)[dstidx
] = t
;
7874 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
7876 /* Inform the callgraph about the new function. */
7877 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
7878 cgraph_add_new_function (child_fn
, true);
7880 /* Fix the callgraph edges for child_cfun. Those for cfun will be
7881 fixed in a following pass. */
7882 push_cfun (child_cfun
);
7883 rebuild_cgraph_edges ();
7885 /* Some EH regions might become dead, see PR34608. If
7886 pass_cleanup_cfg isn't the first pass to happen with the
7887 new child, these dead EH edges might cause problems.
7888 Clean them up now. */
7889 if (flag_exceptions
)
7892 bool changed
= false;
7895 changed
|= gimple_purge_dead_eh_edges (bb
);
7897 cleanup_tree_cfg ();
7902 /* Emit a library call to launch the target region, or do data
7904 tree t1
, t2
, t3
, t4
, device
, cond
, c
, clauses
;
7905 enum built_in_function start_ix
;
7906 location_t clause_loc
;
7908 clauses
= gimple_omp_target_clauses (entry_stmt
);
7910 if (kind
== GF_OMP_TARGET_KIND_REGION
)
7911 start_ix
= BUILT_IN_GOMP_TARGET
;
7912 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
7913 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
7915 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
7917 /* By default, the value of DEVICE is -1 (let runtime library choose)
7918 and there is no conditional. */
7920 device
= build_int_cst (integer_type_node
, -1);
7922 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
7924 cond
= OMP_CLAUSE_IF_EXPR (c
);
7926 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
7929 device
= OMP_CLAUSE_DEVICE_ID (c
);
7930 clause_loc
= OMP_CLAUSE_LOCATION (c
);
7933 clause_loc
= gimple_location (entry_stmt
);
7935 /* Ensure 'device' is of the correct type. */
7936 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
7938 /* If we found the clause 'if (cond)', build
7939 (cond ? device : -2). */
7942 cond
= gimple_boolify (cond
);
7944 basic_block cond_bb
, then_bb
, else_bb
;
7948 tmp_var
= create_tmp_var (TREE_TYPE (device
), NULL
);
7949 if (kind
!= GF_OMP_TARGET_KIND_REGION
)
7951 gsi
= gsi_last_bb (new_bb
);
7953 e
= split_block (new_bb
, gsi_stmt (gsi
));
7956 e
= split_block (new_bb
, NULL
);
7961 then_bb
= create_empty_bb (cond_bb
);
7962 else_bb
= create_empty_bb (then_bb
);
7963 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
7964 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
7966 stmt
= gimple_build_cond_empty (cond
);
7967 gsi
= gsi_last_bb (cond_bb
);
7968 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7970 gsi
= gsi_start_bb (then_bb
);
7971 stmt
= gimple_build_assign (tmp_var
, device
);
7972 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7974 gsi
= gsi_start_bb (else_bb
);
7975 stmt
= gimple_build_assign (tmp_var
,
7976 build_int_cst (integer_type_node
, -2));
7977 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7979 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
7980 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
7983 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
7984 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
7986 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
7987 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
7992 gsi
= gsi_last_bb (new_bb
);
7993 t
= gimple_omp_target_data_arg (entry_stmt
);
7996 t1
= size_zero_node
;
7997 t2
= build_zero_cst (ptr_type_node
);
8003 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
8004 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
8005 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
8006 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
8007 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
8011 /* FIXME: This will be address of
8012 extern char __OPENMP_TARGET__[] __attribute__((visibility ("hidden")))
8013 symbol, as soon as the linker plugin is able to create it for us. */
8014 tree openmp_target
= build_zero_cst (ptr_type_node
);
8015 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8017 tree fnaddr
= build_fold_addr_expr (child_fn
);
8018 g
= gimple_build_call (builtin_decl_explicit (start_ix
), 7,
8019 device
, fnaddr
, openmp_target
, t1
, t2
, t3
, t4
);
8022 g
= gimple_build_call (builtin_decl_explicit (start_ix
), 6,
8023 device
, openmp_target
, t1
, t2
, t3
, t4
);
8024 gimple_set_location (g
, gimple_location (entry_stmt
));
8025 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
8026 if (kind
!= GF_OMP_TARGET_KIND_REGION
)
8029 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
8030 gsi_remove (&gsi
, true);
8032 if (kind
== GF_OMP_TARGET_KIND_DATA
&& region
->exit
)
8034 gsi
= gsi_last_bb (region
->exit
);
8036 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
8037 gsi_remove (&gsi
, true);
8042 /* Expand the parallel region tree rooted at REGION. Expansion
8043 proceeds in depth-first order. Innermost regions are expanded
8044 first. This way, parallel regions that require a new function to
8045 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
8046 internal dependencies in their body. */
8049 expand_omp (struct omp_region
*region
)
8053 location_t saved_location
;
8054 gimple inner_stmt
= NULL
;
8056 /* First, determine whether this is a combined parallel+workshare
8058 if (region
->type
== GIMPLE_OMP_PARALLEL
)
8059 determine_parallel_type (region
);
8061 if (region
->type
== GIMPLE_OMP_FOR
8062 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
8063 inner_stmt
= last_stmt (region
->inner
->entry
);
8066 expand_omp (region
->inner
);
8068 saved_location
= input_location
;
8069 if (gimple_has_location (last_stmt (region
->entry
)))
8070 input_location
= gimple_location (last_stmt (region
->entry
));
8072 switch (region
->type
)
8074 case GIMPLE_OMP_PARALLEL
:
8075 case GIMPLE_OMP_TASK
:
8076 expand_omp_taskreg (region
);
8079 case GIMPLE_OMP_FOR
:
8080 expand_omp_for (region
, inner_stmt
);
8083 case GIMPLE_OMP_SECTIONS
:
8084 expand_omp_sections (region
);
8087 case GIMPLE_OMP_SECTION
:
8088 /* Individual omp sections are handled together with their
8089 parent GIMPLE_OMP_SECTIONS region. */
8092 case GIMPLE_OMP_SINGLE
:
8093 expand_omp_single (region
);
8096 case GIMPLE_OMP_MASTER
:
8097 case GIMPLE_OMP_TASKGROUP
:
8098 case GIMPLE_OMP_ORDERED
:
8099 case GIMPLE_OMP_CRITICAL
:
8100 case GIMPLE_OMP_TEAMS
:
8101 expand_omp_synch (region
);
8104 case GIMPLE_OMP_ATOMIC_LOAD
:
8105 expand_omp_atomic (region
);
8108 case GIMPLE_OMP_TARGET
:
8109 expand_omp_target (region
);
8116 input_location
= saved_location
;
8117 region
= region
->next
;
8122 /* Helper for build_omp_regions. Scan the dominator tree starting at
8123 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
8124 true, the function ends once a single tree is built (otherwise, whole
8125 forest of OMP constructs may be built). */
8128 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
8131 gimple_stmt_iterator gsi
;
8135 gsi
= gsi_last_bb (bb
);
8136 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
8138 struct omp_region
*region
;
8139 enum gimple_code code
;
8141 stmt
= gsi_stmt (gsi
);
8142 code
= gimple_code (stmt
);
8143 if (code
== GIMPLE_OMP_RETURN
)
8145 /* STMT is the return point out of region PARENT. Mark it
8146 as the exit point and make PARENT the immediately
8147 enclosing region. */
8148 gcc_assert (parent
);
8151 parent
= parent
->outer
;
8153 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
8155 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
8156 GIMPLE_OMP_RETURN, but matches with
8157 GIMPLE_OMP_ATOMIC_LOAD. */
8158 gcc_assert (parent
);
8159 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
8162 parent
= parent
->outer
;
8165 else if (code
== GIMPLE_OMP_CONTINUE
)
8167 gcc_assert (parent
);
8170 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
8172 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
8173 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
8176 else if (code
== GIMPLE_OMP_TARGET
8177 && gimple_omp_target_kind (stmt
) == GF_OMP_TARGET_KIND_UPDATE
)
8178 new_omp_region (bb
, code
, parent
);
8181 /* Otherwise, this directive becomes the parent for a new
8183 region
= new_omp_region (bb
, code
, parent
);
8188 if (single_tree
&& !parent
)
8191 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
8193 son
= next_dom_son (CDI_DOMINATORS
, son
))
8194 build_omp_regions_1 (son
, parent
, single_tree
);
8197 /* Builds the tree of OMP regions rooted at ROOT, storing it to
8201 build_omp_regions_root (basic_block root
)
8203 gcc_assert (root_omp_region
== NULL
);
8204 build_omp_regions_1 (root
, NULL
, true);
8205 gcc_assert (root_omp_region
!= NULL
);
8208 /* Expands omp construct (and its subconstructs) starting in HEAD. */
8211 omp_expand_local (basic_block head
)
8213 build_omp_regions_root (head
);
8214 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
8216 fprintf (dump_file
, "\nOMP region tree\n\n");
8217 dump_omp_region (dump_file
, root_omp_region
, 0);
8218 fprintf (dump_file
, "\n");
8221 remove_exit_barriers (root_omp_region
);
8222 expand_omp (root_omp_region
);
8224 free_omp_regions ();
8227 /* Scan the CFG and build a tree of OMP regions. Return the root of
8228 the OMP region tree. */
8231 build_omp_regions (void)
8233 gcc_assert (root_omp_region
== NULL
);
8234 calculate_dominance_info (CDI_DOMINATORS
);
8235 build_omp_regions_1 (ENTRY_BLOCK_PTR
, NULL
, false);
8238 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
8241 execute_expand_omp (void)
8243 build_omp_regions ();
8245 if (!root_omp_region
)
8250 fprintf (dump_file
, "\nOMP region tree\n\n");
8251 dump_omp_region (dump_file
, root_omp_region
, 0);
8252 fprintf (dump_file
, "\n");
8255 remove_exit_barriers (root_omp_region
);
8257 expand_omp (root_omp_region
);
8259 cleanup_tree_cfg ();
8261 free_omp_regions ();
8266 /* OMP expansion -- the default pass, run before creation of SSA form. */
8269 gate_expand_omp (void)
8271 return ((flag_openmp
!= 0 || flag_openmp_simd
!= 0
8272 || flag_enable_cilkplus
!= 0) && !seen_error ());
8277 const pass_data pass_data_expand_omp
=
8279 GIMPLE_PASS
, /* type */
8280 "ompexp", /* name */
8281 OPTGROUP_NONE
, /* optinfo_flags */
8282 true, /* has_gate */
8283 true, /* has_execute */
8284 TV_NONE
, /* tv_id */
8285 PROP_gimple_any
, /* properties_required */
8286 0, /* properties_provided */
8287 0, /* properties_destroyed */
8288 0, /* todo_flags_start */
8289 0, /* todo_flags_finish */
8292 class pass_expand_omp
: public gimple_opt_pass
8295 pass_expand_omp (gcc::context
*ctxt
)
8296 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
8299 /* opt_pass methods: */
8300 bool gate () { return gate_expand_omp (); }
8301 unsigned int execute () { return execute_expand_omp (); }
8303 }; // class pass_expand_omp
8308 make_pass_expand_omp (gcc::context
*ctxt
)
8310 return new pass_expand_omp (ctxt
);
8313 /* Routines to lower OpenMP directives into OMP-GIMPLE. */
8315 /* If ctx is a worksharing context inside of a cancellable parallel
8316 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
8317 and conditional branch to parallel's cancel_label to handle
8318 cancellation in the implicit barrier. */
8321 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
8323 gimple omp_return
= gimple_seq_last_stmt (*body
);
8324 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
8325 if (gimple_omp_return_nowait_p (omp_return
))
8328 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
8329 && ctx
->outer
->cancellable
)
8331 tree lhs
= create_tmp_var (boolean_type_node
, NULL
);
8332 gimple_omp_return_set_lhs (omp_return
, lhs
);
8333 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
8334 gimple g
= gimple_build_cond (NE_EXPR
, lhs
, boolean_false_node
,
8335 ctx
->outer
->cancel_label
, fallthru_label
);
8336 gimple_seq_add_stmt (body
, g
);
8337 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
8341 /* Lower the OpenMP sections directive in the current statement in GSI_P.
8342 CTX is the enclosing OMP context for the current statement. */
8345 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8347 tree block
, control
;
8348 gimple_stmt_iterator tgsi
;
8349 gimple stmt
, new_stmt
, bind
, t
;
8350 gimple_seq ilist
, dlist
, olist
, new_body
;
8351 struct gimplify_ctx gctx
;
8353 stmt
= gsi_stmt (*gsi_p
);
8355 push_gimplify_context (&gctx
);
8359 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
8360 &ilist
, &dlist
, ctx
, NULL
);
8362 new_body
= gimple_omp_body (stmt
);
8363 gimple_omp_set_body (stmt
, NULL
);
8364 tgsi
= gsi_start (new_body
);
8365 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
8370 sec_start
= gsi_stmt (tgsi
);
8371 sctx
= maybe_lookup_ctx (sec_start
);
8374 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
8375 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
8376 GSI_CONTINUE_LINKING
);
8377 gimple_omp_set_body (sec_start
, NULL
);
8379 if (gsi_one_before_end_p (tgsi
))
8381 gimple_seq l
= NULL
;
8382 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
8384 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
8385 gimple_omp_section_set_last (sec_start
);
8388 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
8389 GSI_CONTINUE_LINKING
);
8392 block
= make_node (BLOCK
);
8393 bind
= gimple_build_bind (NULL
, new_body
, block
);
8396 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
8398 block
= make_node (BLOCK
);
8399 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
8400 gsi_replace (gsi_p
, new_stmt
, true);
8402 pop_gimplify_context (new_stmt
);
8403 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
8404 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8405 if (BLOCK_VARS (block
))
8406 TREE_USED (block
) = 1;
8409 gimple_seq_add_seq (&new_body
, ilist
);
8410 gimple_seq_add_stmt (&new_body
, stmt
);
8411 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
8412 gimple_seq_add_stmt (&new_body
, bind
);
8414 control
= create_tmp_var (unsigned_type_node
, ".section");
8415 t
= gimple_build_omp_continue (control
, control
);
8416 gimple_omp_sections_set_control (stmt
, control
);
8417 gimple_seq_add_stmt (&new_body
, t
);
8419 gimple_seq_add_seq (&new_body
, olist
);
8420 if (ctx
->cancellable
)
8421 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
8422 gimple_seq_add_seq (&new_body
, dlist
);
8424 new_body
= maybe_catch_exception (new_body
);
8426 t
= gimple_build_omp_return
8427 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
8428 OMP_CLAUSE_NOWAIT
));
8429 gimple_seq_add_stmt (&new_body
, t
);
8430 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
8432 gimple_bind_set_body (new_stmt
, new_body
);
8436 /* A subroutine of lower_omp_single. Expand the simple form of
8437 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
8439 if (GOMP_single_start ())
8441 [ GOMP_barrier (); ] -> unless 'nowait' is present.
8443 FIXME. It may be better to delay expanding the logic of this until
8444 pass_expand_omp. The expanded logic may make the job more difficult
8445 to a synchronization analysis pass. */
8448 lower_omp_single_simple (gimple single_stmt
, gimple_seq
*pre_p
)
8450 location_t loc
= gimple_location (single_stmt
);
8451 tree tlabel
= create_artificial_label (loc
);
8452 tree flabel
= create_artificial_label (loc
);
8456 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
8457 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)), NULL
);
8458 call
= gimple_build_call (decl
, 0);
8459 gimple_call_set_lhs (call
, lhs
);
8460 gimple_seq_add_stmt (pre_p
, call
);
8462 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
8463 fold_convert_loc (loc
, TREE_TYPE (lhs
),
8466 gimple_seq_add_stmt (pre_p
, cond
);
8467 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
8468 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
8469 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
8473 /* A subroutine of lower_omp_single. Expand the simple form of
8474 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
8476 #pragma omp single copyprivate (a, b, c)
8478 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
8481 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
8487 GOMP_single_copy_end (©out);
8498 FIXME. It may be better to delay expanding the logic of this until
8499 pass_expand_omp. The expanded logic may make the job more difficult
8500 to a synchronization analysis pass. */
8503 lower_omp_single_copy (gimple single_stmt
, gimple_seq
*pre_p
, omp_context
*ctx
)
8505 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
8506 gimple_seq copyin_seq
;
8507 location_t loc
= gimple_location (single_stmt
);
8509 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
8511 ptr_type
= build_pointer_type (ctx
->record_type
);
8512 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
8514 l0
= create_artificial_label (loc
);
8515 l1
= create_artificial_label (loc
);
8516 l2
= create_artificial_label (loc
);
8518 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
8519 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
8520 t
= fold_convert_loc (loc
, ptr_type
, t
);
8521 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
8523 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
8524 build_int_cst (ptr_type
, 0));
8525 t
= build3 (COND_EXPR
, void_type_node
, t
,
8526 build_and_jump (&l0
), build_and_jump (&l1
));
8527 gimplify_and_add (t
, pre_p
);
8529 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
8531 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
8534 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
8537 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
8538 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
8539 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
8540 gimplify_and_add (t
, pre_p
);
8542 t
= build_and_jump (&l2
);
8543 gimplify_and_add (t
, pre_p
);
8545 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
8547 gimple_seq_add_seq (pre_p
, copyin_seq
);
8549 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
8553 /* Expand code for an OpenMP single directive. */
8556 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8559 gimple t
, bind
, single_stmt
= gsi_stmt (*gsi_p
);
8560 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
8561 struct gimplify_ctx gctx
;
8563 push_gimplify_context (&gctx
);
8565 block
= make_node (BLOCK
);
8566 bind
= gimple_build_bind (NULL
, NULL
, block
);
8567 gsi_replace (gsi_p
, bind
, true);
8570 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
8571 &bind_body
, &dlist
, ctx
, NULL
);
8572 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
8574 gimple_seq_add_stmt (&bind_body
, single_stmt
);
8576 if (ctx
->record_type
)
8577 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
8579 lower_omp_single_simple (single_stmt
, &bind_body
);
8581 gimple_omp_set_body (single_stmt
, NULL
);
8583 gimple_seq_add_seq (&bind_body
, dlist
);
8585 bind_body
= maybe_catch_exception (bind_body
);
8587 t
= gimple_build_omp_return
8588 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
8589 OMP_CLAUSE_NOWAIT
));
8590 gimple_seq_add_stmt (&bind_body_tail
, t
);
8591 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
8592 if (ctx
->record_type
)
8594 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
8595 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
8596 TREE_THIS_VOLATILE (clobber
) = 1;
8597 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
8598 clobber
), GSI_SAME_STMT
);
8600 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
8601 gimple_bind_set_body (bind
, bind_body
);
8603 pop_gimplify_context (bind
);
8605 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8606 BLOCK_VARS (block
) = ctx
->block_vars
;
8607 if (BLOCK_VARS (block
))
8608 TREE_USED (block
) = 1;
8612 /* Expand code for an OpenMP master directive. */
8615 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8617 tree block
, lab
= NULL
, x
, bfn_decl
;
8618 gimple stmt
= gsi_stmt (*gsi_p
), bind
;
8619 location_t loc
= gimple_location (stmt
);
8621 struct gimplify_ctx gctx
;
8623 push_gimplify_context (&gctx
);
8625 block
= make_node (BLOCK
);
8626 bind
= gimple_build_bind (NULL
, NULL
, block
);
8627 gsi_replace (gsi_p
, bind
, true);
8628 gimple_bind_add_stmt (bind
, stmt
);
8630 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
8631 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
8632 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
8633 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
8635 gimplify_and_add (x
, &tseq
);
8636 gimple_bind_add_seq (bind
, tseq
);
8638 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8639 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
8640 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8641 gimple_omp_set_body (stmt
, NULL
);
8643 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
8645 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8647 pop_gimplify_context (bind
);
8649 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8650 BLOCK_VARS (block
) = ctx
->block_vars
;
8654 /* Expand code for an OpenMP taskgroup directive. */
8657 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8659 gimple stmt
= gsi_stmt (*gsi_p
), bind
, x
;
8660 tree block
= make_node (BLOCK
);
8662 bind
= gimple_build_bind (NULL
, NULL
, block
);
8663 gsi_replace (gsi_p
, bind
, true);
8664 gimple_bind_add_stmt (bind
, stmt
);
8666 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
8668 gimple_bind_add_stmt (bind
, x
);
8670 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8671 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8672 gimple_omp_set_body (stmt
, NULL
);
8674 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8676 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8677 BLOCK_VARS (block
) = ctx
->block_vars
;
8681 /* Expand code for an OpenMP ordered directive. */
8684 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8687 gimple stmt
= gsi_stmt (*gsi_p
), bind
, x
;
8688 struct gimplify_ctx gctx
;
8690 push_gimplify_context (&gctx
);
8692 block
= make_node (BLOCK
);
8693 bind
= gimple_build_bind (NULL
, NULL
, block
);
8694 gsi_replace (gsi_p
, bind
, true);
8695 gimple_bind_add_stmt (bind
, stmt
);
8697 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
8699 gimple_bind_add_stmt (bind
, x
);
8701 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8702 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
8703 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8704 gimple_omp_set_body (stmt
, NULL
);
8706 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
), 0);
8707 gimple_bind_add_stmt (bind
, x
);
8709 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8711 pop_gimplify_context (bind
);
8713 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8714 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8718 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
8719 substitution of a couple of function calls. But in the NAMED case,
8720 requires that languages coordinate a symbol name. It is therefore
8721 best put here in common code. */
8723 static GTY((param1_is (tree
), param2_is (tree
)))
8724 splay_tree critical_name_mutexes
;
8727 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8730 tree name
, lock
, unlock
;
8731 gimple stmt
= gsi_stmt (*gsi_p
), bind
;
8732 location_t loc
= gimple_location (stmt
);
8734 struct gimplify_ctx gctx
;
8736 name
= gimple_omp_critical_name (stmt
);
8742 if (!critical_name_mutexes
)
8743 critical_name_mutexes
8744 = splay_tree_new_ggc (splay_tree_compare_pointers
,
8745 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_s
,
8746 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_node_s
);
8748 n
= splay_tree_lookup (critical_name_mutexes
, (splay_tree_key
) name
);
8753 decl
= create_tmp_var_raw (ptr_type_node
, NULL
);
8755 new_str
= ACONCAT ((".gomp_critical_user_",
8756 IDENTIFIER_POINTER (name
), NULL
));
8757 DECL_NAME (decl
) = get_identifier (new_str
);
8758 TREE_PUBLIC (decl
) = 1;
8759 TREE_STATIC (decl
) = 1;
8760 DECL_COMMON (decl
) = 1;
8761 DECL_ARTIFICIAL (decl
) = 1;
8762 DECL_IGNORED_P (decl
) = 1;
8763 varpool_finalize_decl (decl
);
8765 splay_tree_insert (critical_name_mutexes
, (splay_tree_key
) name
,
8766 (splay_tree_value
) decl
);
8769 decl
= (tree
) n
->value
;
8771 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
8772 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
8774 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
8775 unlock
= build_call_expr_loc (loc
, unlock
, 1,
8776 build_fold_addr_expr_loc (loc
, decl
));
8780 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
8781 lock
= build_call_expr_loc (loc
, lock
, 0);
8783 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
8784 unlock
= build_call_expr_loc (loc
, unlock
, 0);
8787 push_gimplify_context (&gctx
);
8789 block
= make_node (BLOCK
);
8790 bind
= gimple_build_bind (NULL
, NULL
, block
);
8791 gsi_replace (gsi_p
, bind
, true);
8792 gimple_bind_add_stmt (bind
, stmt
);
8794 tbody
= gimple_bind_body (bind
);
8795 gimplify_and_add (lock
, &tbody
);
8796 gimple_bind_set_body (bind
, tbody
);
8798 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8799 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
8800 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8801 gimple_omp_set_body (stmt
, NULL
);
8803 tbody
= gimple_bind_body (bind
);
8804 gimplify_and_add (unlock
, &tbody
);
8805 gimple_bind_set_body (bind
, tbody
);
8807 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8809 pop_gimplify_context (bind
);
8810 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8811 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8815 /* A subroutine of lower_omp_for. Generate code to emit the predicate
8816 for a lastprivate clause. Given a loop control predicate of (V
8817 cond N2), we gate the clause on (!(V cond N2)). The lowered form
8818 is appended to *DLIST, iterator initialization is appended to
8822 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
8823 gimple_seq
*dlist
, struct omp_context
*ctx
)
8825 tree clauses
, cond
, vinit
;
8826 enum tree_code cond_code
;
8829 cond_code
= fd
->loop
.cond_code
;
8830 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
8832 /* When possible, use a strict equality expression. This can let VRP
8833 type optimizations deduce the value and remove a copy. */
8834 if (tree_fits_shwi_p (fd
->loop
.step
))
8836 HOST_WIDE_INT step
= TREE_INT_CST_LOW (fd
->loop
.step
);
8837 if (step
== 1 || step
== -1)
8838 cond_code
= EQ_EXPR
;
8841 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, fd
->loop
.n2
);
8843 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
8845 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
8846 if (!gimple_seq_empty_p (stmts
))
8848 gimple_seq_add_seq (&stmts
, *dlist
);
8851 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
8852 vinit
= fd
->loop
.n1
;
8853 if (cond_code
== EQ_EXPR
8854 && tree_fits_shwi_p (fd
->loop
.n2
)
8855 && ! integer_zerop (fd
->loop
.n2
))
8856 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
8858 vinit
= unshare_expr (vinit
);
8860 /* Initialize the iterator variable, so that threads that don't execute
8861 any iterations don't execute the lastprivate clauses by accident. */
8862 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
8867 /* Lower code for an OpenMP loop directive. */
8870 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8873 struct omp_for_data fd
, *fdp
= NULL
;
8874 gimple stmt
= gsi_stmt (*gsi_p
), new_stmt
;
8875 gimple_seq omp_for_body
, body
, dlist
;
8877 struct gimplify_ctx gctx
;
8879 push_gimplify_context (&gctx
);
8881 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
8883 block
= make_node (BLOCK
);
8884 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
8885 /* Replace at gsi right away, so that 'stmt' is no member
8886 of a sequence anymore as we're going to add to to a different
8888 gsi_replace (gsi_p
, new_stmt
, true);
8890 /* Move declaration of temporaries in the loop body before we make
8892 omp_for_body
= gimple_omp_body (stmt
);
8893 if (!gimple_seq_empty_p (omp_for_body
)
8894 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
8896 tree vars
= gimple_bind_vars (gimple_seq_first_stmt (omp_for_body
));
8897 gimple_bind_append_vars (new_stmt
, vars
);
8900 if (gimple_omp_for_combined_into_p (stmt
))
8902 extract_omp_for_data (stmt
, &fd
, NULL
);
8905 /* We need two temporaries with fd.loop.v type (istart/iend)
8906 and then (fd.collapse - 1) temporaries with the same
8907 type for count2 ... countN-1 vars if not constant. */
8909 tree type
= fd
.iter_type
;
8911 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
8912 count
+= fd
.collapse
- 1;
8913 bool parallel_for
= gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
;
8914 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
8918 = find_omp_clause (gimple_omp_parallel_clauses (ctx
->outer
->stmt
),
8919 OMP_CLAUSE__LOOPTEMP_
);
8920 for (i
= 0; i
< count
; i
++)
8925 gcc_assert (outerc
);
8926 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
8927 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
8928 OMP_CLAUSE__LOOPTEMP_
);
8931 temp
= create_tmp_var (type
, NULL
);
8932 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
8933 OMP_CLAUSE_DECL (*pc
) = temp
;
8934 pc
= &OMP_CLAUSE_CHAIN (*pc
);
8939 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
8942 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
8944 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
8946 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8948 /* Lower the header expressions. At this point, we can assume that
8949 the header is of the form:
8951 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
8953 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
8954 using the .omp_data_s mapping, if needed. */
8955 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
8957 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
8958 if (!is_gimple_min_invariant (*rhs_p
))
8959 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
8961 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
8962 if (!is_gimple_min_invariant (*rhs_p
))
8963 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
8965 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
8966 if (!is_gimple_min_invariant (*rhs_p
))
8967 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
8970 /* Once lowered, extract the bounds and clauses. */
8971 extract_omp_for_data (stmt
, &fd
, NULL
);
8973 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
8975 gimple_seq_add_stmt (&body
, stmt
);
8976 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
8978 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
8981 /* After the loop, add exit clauses. */
8982 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
8984 if (ctx
->cancellable
)
8985 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
8987 gimple_seq_add_seq (&body
, dlist
);
8989 body
= maybe_catch_exception (body
);
8991 /* Region exit marker goes at the end of the loop body. */
8992 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
8993 maybe_add_implicit_barrier_cancel (ctx
, &body
);
8994 pop_gimplify_context (new_stmt
);
8996 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
8997 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
8998 if (BLOCK_VARS (block
))
8999 TREE_USED (block
) = 1;
9001 gimple_bind_set_body (new_stmt
, body
);
9002 gimple_omp_set_body (stmt
, NULL
);
9003 gimple_omp_for_set_pre_body (stmt
, NULL
);
9006 /* Callback for walk_stmts. Check if the current statement only contains
9007 GIMPLE_OMP_FOR or GIMPLE_OMP_PARALLEL. */
9010 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
9011 bool *handled_ops_p
,
9012 struct walk_stmt_info
*wi
)
9014 int *info
= (int *) wi
->info
;
9015 gimple stmt
= gsi_stmt (*gsi_p
);
9017 *handled_ops_p
= true;
9018 switch (gimple_code (stmt
))
9022 case GIMPLE_OMP_FOR
:
9023 case GIMPLE_OMP_SECTIONS
:
9024 *info
= *info
== 0 ? 1 : -1;
9033 struct omp_taskcopy_context
9035 /* This field must be at the beginning, as we do "inheritance": Some
9036 callback functions for tree-inline.c (e.g., omp_copy_decl)
9037 receive a copy_body_data pointer that is up-casted to an
9038 omp_context pointer. */
9044 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
9046 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
9048 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
9049 return create_tmp_var (TREE_TYPE (var
), NULL
);
9055 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
9057 tree name
, new_fields
= NULL
, type
, f
;
9059 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
9060 name
= DECL_NAME (TYPE_NAME (orig_type
));
9061 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
9062 TYPE_DECL
, name
, type
);
9063 TYPE_NAME (type
) = name
;
9065 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
9067 tree new_f
= copy_node (f
);
9068 DECL_CONTEXT (new_f
) = type
;
9069 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
9070 TREE_CHAIN (new_f
) = new_fields
;
9071 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
9072 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
9073 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
9076 *pointer_map_insert (tcctx
->cb
.decl_map
, f
) = new_f
;
9078 TYPE_FIELDS (type
) = nreverse (new_fields
);
9083 /* Create task copyfn. */
9086 create_task_copyfn (gimple task_stmt
, omp_context
*ctx
)
9088 struct function
*child_cfun
;
9089 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
9090 tree record_type
, srecord_type
, bind
, list
;
9091 bool record_needs_remap
= false, srecord_needs_remap
= false;
9093 struct omp_taskcopy_context tcctx
;
9094 struct gimplify_ctx gctx
;
9095 location_t loc
= gimple_location (task_stmt
);
9097 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
9098 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
9099 gcc_assert (child_cfun
->cfg
== NULL
);
9100 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
9102 /* Reset DECL_CONTEXT on function arguments. */
9103 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
9104 DECL_CONTEXT (t
) = child_fn
;
9106 /* Populate the function. */
9107 push_gimplify_context (&gctx
);
9108 push_cfun (child_cfun
);
9110 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
9111 TREE_SIDE_EFFECTS (bind
) = 1;
9113 DECL_SAVED_TREE (child_fn
) = bind
;
9114 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
9116 /* Remap src and dst argument types if needed. */
9117 record_type
= ctx
->record_type
;
9118 srecord_type
= ctx
->srecord_type
;
9119 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
9120 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
9122 record_needs_remap
= true;
9125 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
9126 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
9128 srecord_needs_remap
= true;
9132 if (record_needs_remap
|| srecord_needs_remap
)
9134 memset (&tcctx
, '\0', sizeof (tcctx
));
9135 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
9136 tcctx
.cb
.dst_fn
= child_fn
;
9137 tcctx
.cb
.src_node
= cgraph_get_node (tcctx
.cb
.src_fn
);
9138 gcc_checking_assert (tcctx
.cb
.src_node
);
9139 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
9140 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
9141 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
9142 tcctx
.cb
.eh_lp_nr
= 0;
9143 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
9144 tcctx
.cb
.decl_map
= pointer_map_create ();
9147 if (record_needs_remap
)
9148 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
9149 if (srecord_needs_remap
)
9150 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
9153 tcctx
.cb
.decl_map
= NULL
;
9155 arg
= DECL_ARGUMENTS (child_fn
);
9156 TREE_TYPE (arg
) = build_pointer_type (record_type
);
9157 sarg
= DECL_CHAIN (arg
);
9158 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
9160 /* First pass: initialize temporaries used in record_type and srecord_type
9161 sizes and field offsets. */
9162 if (tcctx
.cb
.decl_map
)
9163 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9164 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
9168 decl
= OMP_CLAUSE_DECL (c
);
9169 p
= (tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, decl
);
9172 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9173 sf
= (tree
) n
->value
;
9174 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9175 src
= build_simple_mem_ref_loc (loc
, sarg
);
9176 src
= omp_build_component_ref (src
, sf
);
9177 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
9178 append_to_statement_list (t
, &list
);
9181 /* Second pass: copy shared var pointers and copy construct non-VLA
9182 firstprivate vars. */
9183 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9184 switch (OMP_CLAUSE_CODE (c
))
9186 case OMP_CLAUSE_SHARED
:
9187 decl
= OMP_CLAUSE_DECL (c
);
9188 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9191 f
= (tree
) n
->value
;
9192 if (tcctx
.cb
.decl_map
)
9193 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9194 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9195 sf
= (tree
) n
->value
;
9196 if (tcctx
.cb
.decl_map
)
9197 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9198 src
= build_simple_mem_ref_loc (loc
, sarg
);
9199 src
= omp_build_component_ref (src
, sf
);
9200 dst
= build_simple_mem_ref_loc (loc
, arg
);
9201 dst
= omp_build_component_ref (dst
, f
);
9202 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
9203 append_to_statement_list (t
, &list
);
9205 case OMP_CLAUSE_FIRSTPRIVATE
:
9206 decl
= OMP_CLAUSE_DECL (c
);
9207 if (is_variable_sized (decl
))
9209 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9212 f
= (tree
) n
->value
;
9213 if (tcctx
.cb
.decl_map
)
9214 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9215 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9218 sf
= (tree
) n
->value
;
9219 if (tcctx
.cb
.decl_map
)
9220 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9221 src
= build_simple_mem_ref_loc (loc
, sarg
);
9222 src
= omp_build_component_ref (src
, sf
);
9223 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
9224 src
= build_simple_mem_ref_loc (loc
, src
);
9228 dst
= build_simple_mem_ref_loc (loc
, arg
);
9229 dst
= omp_build_component_ref (dst
, f
);
9230 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
9231 append_to_statement_list (t
, &list
);
9233 case OMP_CLAUSE_PRIVATE
:
9234 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
9236 decl
= OMP_CLAUSE_DECL (c
);
9237 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9238 f
= (tree
) n
->value
;
9239 if (tcctx
.cb
.decl_map
)
9240 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9241 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9244 sf
= (tree
) n
->value
;
9245 if (tcctx
.cb
.decl_map
)
9246 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9247 src
= build_simple_mem_ref_loc (loc
, sarg
);
9248 src
= omp_build_component_ref (src
, sf
);
9249 if (use_pointer_for_field (decl
, NULL
))
9250 src
= build_simple_mem_ref_loc (loc
, src
);
9254 dst
= build_simple_mem_ref_loc (loc
, arg
);
9255 dst
= omp_build_component_ref (dst
, f
);
9256 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
9257 append_to_statement_list (t
, &list
);
9263 /* Last pass: handle VLA firstprivates. */
9264 if (tcctx
.cb
.decl_map
)
9265 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9266 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
9270 decl
= OMP_CLAUSE_DECL (c
);
9271 if (!is_variable_sized (decl
))
9273 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9276 f
= (tree
) n
->value
;
9277 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9278 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
9279 ind
= DECL_VALUE_EXPR (decl
);
9280 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
9281 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
9282 n
= splay_tree_lookup (ctx
->sfield_map
,
9283 (splay_tree_key
) TREE_OPERAND (ind
, 0));
9284 sf
= (tree
) n
->value
;
9285 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9286 src
= build_simple_mem_ref_loc (loc
, sarg
);
9287 src
= omp_build_component_ref (src
, sf
);
9288 src
= build_simple_mem_ref_loc (loc
, src
);
9289 dst
= build_simple_mem_ref_loc (loc
, arg
);
9290 dst
= omp_build_component_ref (dst
, f
);
9291 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
9292 append_to_statement_list (t
, &list
);
9293 n
= splay_tree_lookup (ctx
->field_map
,
9294 (splay_tree_key
) TREE_OPERAND (ind
, 0));
9295 df
= (tree
) n
->value
;
9296 df
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, df
);
9297 ptr
= build_simple_mem_ref_loc (loc
, arg
);
9298 ptr
= omp_build_component_ref (ptr
, df
);
9299 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
9300 build_fold_addr_expr_loc (loc
, dst
));
9301 append_to_statement_list (t
, &list
);
9304 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
9305 append_to_statement_list (t
, &list
);
9307 if (tcctx
.cb
.decl_map
)
9308 pointer_map_destroy (tcctx
.cb
.decl_map
);
9309 pop_gimplify_context (NULL
);
9310 BIND_EXPR_BODY (bind
) = list
;
9315 lower_depend_clauses (gimple stmt
, gimple_seq
*iseq
, gimple_seq
*oseq
)
9319 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
9321 clauses
= find_omp_clause (gimple_omp_task_clauses (stmt
),
9323 gcc_assert (clauses
);
9324 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9325 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
9326 switch (OMP_CLAUSE_DEPEND_KIND (c
))
9328 case OMP_CLAUSE_DEPEND_IN
:
9331 case OMP_CLAUSE_DEPEND_OUT
:
9332 case OMP_CLAUSE_DEPEND_INOUT
:
9338 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
9339 tree array
= create_tmp_var (type
, NULL
);
9340 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
9342 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
9343 gimple_seq_add_stmt (iseq
, g
);
9344 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
9346 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
9347 gimple_seq_add_stmt (iseq
, g
);
9348 for (i
= 0; i
< 2; i
++)
9350 if ((i
? n_in
: n_out
) == 0)
9352 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9353 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
9354 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
9356 tree t
= OMP_CLAUSE_DECL (c
);
9357 t
= fold_convert (ptr_type_node
, t
);
9358 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
9359 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
9360 NULL_TREE
, NULL_TREE
);
9361 g
= gimple_build_assign (r
, t
);
9362 gimple_seq_add_stmt (iseq
, g
);
9365 tree
*p
= gimple_omp_task_clauses_ptr (stmt
);
9366 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
9367 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
9368 OMP_CLAUSE_CHAIN (c
) = *p
;
9370 tree clobber
= build_constructor (type
, NULL
);
9371 TREE_THIS_VOLATILE (clobber
) = 1;
9372 g
= gimple_build_assign (array
, clobber
);
9373 gimple_seq_add_stmt (oseq
, g
);
9376 /* Lower the OpenMP parallel or task directive in the current statement
9377 in GSI_P. CTX holds context information for the directive. */
9380 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9384 gimple stmt
= gsi_stmt (*gsi_p
);
9385 gimple par_bind
, bind
, dep_bind
= NULL
;
9386 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
9387 struct gimplify_ctx gctx
, dep_gctx
;
9388 location_t loc
= gimple_location (stmt
);
9390 clauses
= gimple_omp_taskreg_clauses (stmt
);
9391 par_bind
= gimple_seq_first_stmt (gimple_omp_body (stmt
));
9392 par_body
= gimple_bind_body (par_bind
);
9393 child_fn
= ctx
->cb
.dst_fn
;
9394 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
9395 && !gimple_omp_parallel_combined_p (stmt
))
9397 struct walk_stmt_info wi
;
9400 memset (&wi
, 0, sizeof (wi
));
9403 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
9405 gimple_omp_parallel_set_combined_p (stmt
, true);
9407 gimple_seq dep_ilist
= NULL
;
9408 gimple_seq dep_olist
= NULL
;
9409 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
9410 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
9412 push_gimplify_context (&dep_gctx
);
9413 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
9414 lower_depend_clauses (stmt
, &dep_ilist
, &dep_olist
);
9417 if (ctx
->srecord_type
)
9418 create_task_copyfn (stmt
, ctx
);
9420 push_gimplify_context (&gctx
);
9425 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
9426 lower_omp (&par_body
, ctx
);
9427 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
9428 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
9430 /* Declare all the variables created by mapping and the variables
9431 declared in the scope of the parallel body. */
9432 record_vars_into (ctx
->block_vars
, child_fn
);
9433 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
9435 if (ctx
->record_type
)
9438 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
9439 : ctx
->record_type
, ".omp_data_o");
9440 DECL_NAMELESS (ctx
->sender_decl
) = 1;
9441 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
9442 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
9447 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
9448 lower_send_shared_vars (&ilist
, &olist
, ctx
);
9450 if (ctx
->record_type
)
9452 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
9453 TREE_THIS_VOLATILE (clobber
) = 1;
9454 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
9458 /* Once all the expansions are done, sequence all the different
9459 fragments inside gimple_omp_body. */
9463 if (ctx
->record_type
)
9465 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
9466 /* fixup_child_record_type might have changed receiver_decl's type. */
9467 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
9468 gimple_seq_add_stmt (&new_body
,
9469 gimple_build_assign (ctx
->receiver_decl
, t
));
9472 gimple_seq_add_seq (&new_body
, par_ilist
);
9473 gimple_seq_add_seq (&new_body
, par_body
);
9474 gimple_seq_add_seq (&new_body
, par_rlist
);
9475 if (ctx
->cancellable
)
9476 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
9477 gimple_seq_add_seq (&new_body
, par_olist
);
9478 new_body
= maybe_catch_exception (new_body
);
9479 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
9480 gimple_omp_set_body (stmt
, new_body
);
9482 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
9483 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
9484 gimple_bind_add_seq (bind
, ilist
);
9485 gimple_bind_add_stmt (bind
, stmt
);
9486 gimple_bind_add_seq (bind
, olist
);
9488 pop_gimplify_context (NULL
);
9492 gimple_bind_add_seq (dep_bind
, dep_ilist
);
9493 gimple_bind_add_stmt (dep_bind
, bind
);
9494 gimple_bind_add_seq (dep_bind
, dep_olist
);
9495 pop_gimplify_context (dep_bind
);
9499 /* Lower the OpenMP target directive in the current statement
9500 in GSI_P. CTX holds context information for the directive. */
9503 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9506 tree child_fn
, t
, c
;
9507 gimple stmt
= gsi_stmt (*gsi_p
);
9508 gimple tgt_bind
= NULL
, bind
;
9509 gimple_seq tgt_body
= NULL
, olist
, ilist
, new_body
;
9510 struct gimplify_ctx gctx
;
9511 location_t loc
= gimple_location (stmt
);
9512 int kind
= gimple_omp_target_kind (stmt
);
9513 unsigned int map_cnt
= 0;
9515 clauses
= gimple_omp_target_clauses (stmt
);
9516 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9518 tgt_bind
= gimple_seq_first_stmt (gimple_omp_body (stmt
));
9519 tgt_body
= gimple_bind_body (tgt_bind
);
9521 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
9522 tgt_body
= gimple_omp_body (stmt
);
9523 child_fn
= ctx
->cb
.dst_fn
;
9525 push_gimplify_context (&gctx
);
9527 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9528 switch (OMP_CLAUSE_CODE (c
))
9534 case OMP_CLAUSE_MAP
:
9536 case OMP_CLAUSE_FROM
:
9537 var
= OMP_CLAUSE_DECL (c
);
9540 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
9541 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
9547 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
9549 tree var2
= DECL_VALUE_EXPR (var
);
9550 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
9551 var2
= TREE_OPERAND (var2
, 0);
9552 gcc_assert (DECL_P (var2
));
9556 if (!maybe_lookup_field (var
, ctx
))
9559 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9561 x
= build_receiver_ref (var
, true, ctx
);
9562 tree new_var
= lookup_decl (var
, ctx
);
9563 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9564 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
9565 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
9566 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
9567 x
= build_simple_mem_ref (x
);
9568 SET_DECL_VALUE_EXPR (new_var
, x
);
9569 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
9574 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9576 target_nesting_level
++;
9577 lower_omp (&tgt_body
, ctx
);
9578 target_nesting_level
--;
9580 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
9581 lower_omp (&tgt_body
, ctx
);
9583 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9585 /* Declare all the variables created by mapping and the variables
9586 declared in the scope of the target body. */
9587 record_vars_into (ctx
->block_vars
, child_fn
);
9588 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
9593 if (ctx
->record_type
)
9596 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
9597 DECL_NAMELESS (ctx
->sender_decl
) = 1;
9598 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
9599 t
= make_tree_vec (3);
9600 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
9602 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
9604 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
9605 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
9606 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
9608 = create_tmp_var (build_array_type_nelts (unsigned_char_type_node
,
9611 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
9612 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
9613 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
9614 gimple_omp_target_set_data_arg (stmt
, t
);
9616 vec
<constructor_elt
, va_gc
> *vsize
;
9617 vec
<constructor_elt
, va_gc
> *vkind
;
9618 vec_alloc (vsize
, map_cnt
);
9619 vec_alloc (vkind
, map_cnt
);
9620 unsigned int map_idx
= 0;
9622 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9623 switch (OMP_CLAUSE_CODE (c
))
9629 case OMP_CLAUSE_MAP
:
9631 case OMP_CLAUSE_FROM
:
9633 ovar
= OMP_CLAUSE_DECL (c
);
9636 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9637 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
9639 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
9640 == get_base_address (ovar
));
9641 nc
= OMP_CLAUSE_CHAIN (c
);
9642 ovar
= OMP_CLAUSE_DECL (nc
);
9646 tree x
= build_sender_ref (ovar
, ctx
);
9648 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
9649 gimplify_assign (x
, v
, &ilist
);
9655 if (DECL_SIZE (ovar
)
9656 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
9658 tree ovar2
= DECL_VALUE_EXPR (ovar
);
9659 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
9660 ovar2
= TREE_OPERAND (ovar2
, 0);
9661 gcc_assert (DECL_P (ovar2
));
9664 if (!maybe_lookup_field (ovar
, ctx
))
9670 tree var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
9671 tree x
= build_sender_ref (ovar
, ctx
);
9672 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9673 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
9674 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
9675 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
9677 gcc_assert (kind
== GF_OMP_TARGET_KIND_REGION
);
9679 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)), NULL
);
9680 mark_addressable (avar
);
9681 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
9682 avar
= build_fold_addr_expr (avar
);
9683 gimplify_assign (x
, avar
, &ilist
);
9685 else if (is_gimple_reg (var
))
9687 gcc_assert (kind
== GF_OMP_TARGET_KIND_REGION
);
9688 tree avar
= create_tmp_var (TREE_TYPE (var
), NULL
);
9689 mark_addressable (avar
);
9690 if (OMP_CLAUSE_MAP_KIND (c
) != OMP_CLAUSE_MAP_ALLOC
9691 && OMP_CLAUSE_MAP_KIND (c
) != OMP_CLAUSE_MAP_FROM
)
9692 gimplify_assign (avar
, var
, &ilist
);
9693 avar
= build_fold_addr_expr (avar
);
9694 gimplify_assign (x
, avar
, &ilist
);
9695 if ((OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_FROM
9696 || OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_TOFROM
)
9697 && !TYPE_READONLY (TREE_TYPE (var
)))
9699 x
= build_sender_ref (ovar
, ctx
);
9700 x
= build_simple_mem_ref (x
);
9701 gimplify_assign (var
, x
, &olist
);
9706 var
= build_fold_addr_expr (var
);
9707 gimplify_assign (x
, var
, &ilist
);
9710 tree s
= OMP_CLAUSE_SIZE (c
);
9712 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
9713 s
= fold_convert (size_type_node
, s
);
9714 tree purpose
= size_int (map_idx
++);
9715 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
9716 if (TREE_CODE (s
) != INTEGER_CST
)
9717 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
9719 unsigned char tkind
= 0;
9720 switch (OMP_CLAUSE_CODE (c
))
9722 case OMP_CLAUSE_MAP
:
9723 tkind
= OMP_CLAUSE_MAP_KIND (c
);
9726 tkind
= OMP_CLAUSE_MAP_TO
;
9728 case OMP_CLAUSE_FROM
:
9729 tkind
= OMP_CLAUSE_MAP_FROM
;
9734 unsigned int talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
9735 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
9736 talign
= DECL_ALIGN_UNIT (ovar
);
9737 talign
= ceil_log2 (talign
);
9738 tkind
|= talign
<< 3;
9739 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
9740 build_int_cst (unsigned_char_type_node
,
9746 gcc_assert (map_idx
== map_cnt
);
9748 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
9749 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
9750 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
9751 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
9752 if (!TREE_STATIC (TREE_VEC_ELT (t
, 1)))
9754 gimple_seq initlist
= NULL
;
9755 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
9756 TREE_VEC_ELT (t
, 1)),
9757 &initlist
, true, NULL_TREE
);
9758 gimple_seq_add_seq (&ilist
, initlist
);
9761 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
9762 TREE_THIS_VOLATILE (clobber
) = 1;
9763 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
9767 /* Once all the expansions are done, sequence all the different
9768 fragments inside gimple_omp_body. */
9772 if (ctx
->record_type
&& kind
== GF_OMP_TARGET_KIND_REGION
)
9774 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
9775 /* fixup_child_record_type might have changed receiver_decl's type. */
9776 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
9777 gimple_seq_add_stmt (&new_body
,
9778 gimple_build_assign (ctx
->receiver_decl
, t
));
9781 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9783 gimple_seq_add_seq (&new_body
, tgt_body
);
9784 new_body
= maybe_catch_exception (new_body
);
9786 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
9787 new_body
= tgt_body
;
9788 if (kind
!= GF_OMP_TARGET_KIND_UPDATE
)
9790 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
9791 gimple_omp_set_body (stmt
, new_body
);
9794 bind
= gimple_build_bind (NULL
, NULL
,
9795 tgt_bind
? gimple_bind_block (tgt_bind
)
9797 gsi_replace (gsi_p
, bind
, true);
9798 gimple_bind_add_seq (bind
, ilist
);
9799 gimple_bind_add_stmt (bind
, stmt
);
9800 gimple_bind_add_seq (bind
, olist
);
9802 pop_gimplify_context (NULL
);
9805 /* Expand code for an OpenMP teams directive. */
9808 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9810 gimple teams_stmt
= gsi_stmt (*gsi_p
);
9811 struct gimplify_ctx gctx
;
9812 push_gimplify_context (&gctx
);
9814 tree block
= make_node (BLOCK
);
9815 gimple bind
= gimple_build_bind (NULL
, NULL
, block
);
9816 gsi_replace (gsi_p
, bind
, true);
9817 gimple_seq bind_body
= NULL
;
9818 gimple_seq dlist
= NULL
;
9819 gimple_seq olist
= NULL
;
9821 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
9822 OMP_CLAUSE_NUM_TEAMS
);
9823 if (num_teams
== NULL_TREE
)
9824 num_teams
= build_int_cst (unsigned_type_node
, 0);
9827 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
9828 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
9829 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
9831 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
9832 OMP_CLAUSE_THREAD_LIMIT
);
9833 if (thread_limit
== NULL_TREE
)
9834 thread_limit
= build_int_cst (unsigned_type_node
, 0);
9837 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
9838 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
9839 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
9843 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
9844 &bind_body
, &dlist
, ctx
, NULL
);
9845 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
9846 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
9847 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
9849 location_t loc
= gimple_location (teams_stmt
);
9850 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
9851 gimple call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
9852 gimple_set_location (call
, loc
);
9853 gimple_seq_add_stmt (&bind_body
, call
);
9855 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
9856 gimple_omp_set_body (teams_stmt
, NULL
);
9857 gimple_seq_add_seq (&bind_body
, olist
);
9858 gimple_seq_add_seq (&bind_body
, dlist
);
9859 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
9860 gimple_bind_set_body (bind
, bind_body
);
9862 pop_gimplify_context (bind
);
9864 gimple_bind_append_vars (bind
, ctx
->block_vars
);
9865 BLOCK_VARS (block
) = ctx
->block_vars
;
9866 if (BLOCK_VARS (block
))
9867 TREE_USED (block
) = 1;
9871 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
9872 regimplified. If DATA is non-NULL, lower_omp_1 is outside
9873 of OpenMP context, but with task_shared_vars set. */
9876 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
9881 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
9882 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
9885 if (task_shared_vars
9887 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
9890 /* If a global variable has been privatized, TREE_CONSTANT on
9891 ADDR_EXPR might be wrong. */
9892 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
9893 recompute_tree_invariant_for_addr_expr (t
);
9895 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
9900 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9902 gimple stmt
= gsi_stmt (*gsi_p
);
9903 struct walk_stmt_info wi
;
9905 if (gimple_has_location (stmt
))
9906 input_location
= gimple_location (stmt
);
9908 if (task_shared_vars
)
9909 memset (&wi
, '\0', sizeof (wi
));
9911 /* If we have issued syntax errors, avoid doing any heavy lifting.
9912 Just replace the OpenMP directives with a NOP to avoid
9913 confusing RTL expansion. */
9914 if (seen_error () && is_gimple_omp (stmt
))
9916 gsi_replace (gsi_p
, gimple_build_nop (), true);
9920 switch (gimple_code (stmt
))
9923 if ((ctx
|| task_shared_vars
)
9924 && (walk_tree (gimple_cond_lhs_ptr (stmt
), lower_omp_regimplify_p
,
9925 ctx
? NULL
: &wi
, NULL
)
9926 || walk_tree (gimple_cond_rhs_ptr (stmt
), lower_omp_regimplify_p
,
9927 ctx
? NULL
: &wi
, NULL
)))
9928 gimple_regimplify_operands (stmt
, gsi_p
);
9931 lower_omp (gimple_catch_handler_ptr (stmt
), ctx
);
9933 case GIMPLE_EH_FILTER
:
9934 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
9937 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
9938 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
9940 case GIMPLE_TRANSACTION
:
9941 lower_omp (gimple_transaction_body_ptr (stmt
), ctx
);
9944 lower_omp (gimple_bind_body_ptr (stmt
), ctx
);
9946 case GIMPLE_OMP_PARALLEL
:
9947 case GIMPLE_OMP_TASK
:
9948 ctx
= maybe_lookup_ctx (stmt
);
9950 if (ctx
->cancellable
)
9951 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
9952 lower_omp_taskreg (gsi_p
, ctx
);
9954 case GIMPLE_OMP_FOR
:
9955 ctx
= maybe_lookup_ctx (stmt
);
9957 if (ctx
->cancellable
)
9958 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
9959 lower_omp_for (gsi_p
, ctx
);
9961 case GIMPLE_OMP_SECTIONS
:
9962 ctx
= maybe_lookup_ctx (stmt
);
9964 if (ctx
->cancellable
)
9965 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
9966 lower_omp_sections (gsi_p
, ctx
);
9968 case GIMPLE_OMP_SINGLE
:
9969 ctx
= maybe_lookup_ctx (stmt
);
9971 lower_omp_single (gsi_p
, ctx
);
9973 case GIMPLE_OMP_MASTER
:
9974 ctx
= maybe_lookup_ctx (stmt
);
9976 lower_omp_master (gsi_p
, ctx
);
9978 case GIMPLE_OMP_TASKGROUP
:
9979 ctx
= maybe_lookup_ctx (stmt
);
9981 lower_omp_taskgroup (gsi_p
, ctx
);
9983 case GIMPLE_OMP_ORDERED
:
9984 ctx
= maybe_lookup_ctx (stmt
);
9986 lower_omp_ordered (gsi_p
, ctx
);
9988 case GIMPLE_OMP_CRITICAL
:
9989 ctx
= maybe_lookup_ctx (stmt
);
9991 lower_omp_critical (gsi_p
, ctx
);
9993 case GIMPLE_OMP_ATOMIC_LOAD
:
9994 if ((ctx
|| task_shared_vars
)
9995 && walk_tree (gimple_omp_atomic_load_rhs_ptr (stmt
),
9996 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
9997 gimple_regimplify_operands (stmt
, gsi_p
);
9999 case GIMPLE_OMP_TARGET
:
10000 ctx
= maybe_lookup_ctx (stmt
);
10002 lower_omp_target (gsi_p
, ctx
);
10004 case GIMPLE_OMP_TEAMS
:
10005 ctx
= maybe_lookup_ctx (stmt
);
10007 lower_omp_teams (gsi_p
, ctx
);
10011 fndecl
= gimple_call_fndecl (stmt
);
10013 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
10014 switch (DECL_FUNCTION_CODE (fndecl
))
10016 case BUILT_IN_GOMP_BARRIER
:
10020 case BUILT_IN_GOMP_CANCEL
:
10021 case BUILT_IN_GOMP_CANCELLATION_POINT
:
10024 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
10025 cctx
= cctx
->outer
;
10026 gcc_assert (gimple_call_lhs (stmt
) == NULL_TREE
);
10027 if (!cctx
->cancellable
)
10029 if (DECL_FUNCTION_CODE (fndecl
)
10030 == BUILT_IN_GOMP_CANCELLATION_POINT
)
10032 stmt
= gimple_build_nop ();
10033 gsi_replace (gsi_p
, stmt
, false);
10038 lhs
= create_tmp_var (boolean_type_node
, NULL
);
10039 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
10041 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
10042 gimple_call_set_fndecl (stmt
, fndecl
);
10043 gimple_call_set_fntype (stmt
, TREE_TYPE (fndecl
));
10045 gimple_call_set_lhs (stmt
, lhs
);
10046 tree fallthru_label
;
10047 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
10049 g
= gimple_build_label (fallthru_label
);
10050 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
10051 g
= gimple_build_cond (NE_EXPR
, lhs
, boolean_false_node
,
10052 cctx
->cancel_label
, fallthru_label
);
10053 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
10060 if ((ctx
|| task_shared_vars
)
10061 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
10063 gimple_regimplify_operands (stmt
, gsi_p
);
10069 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
10071 location_t saved_location
= input_location
;
10072 gimple_stmt_iterator gsi
;
10073 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
10074 lower_omp_1 (&gsi
, ctx
);
10075 /* Inside target region we haven't called fold_stmt during gimplification,
10076 because it can break code by adding decl references that weren't in the
10077 source. Call fold_stmt now. */
10078 if (target_nesting_level
)
10079 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
10081 input_location
= saved_location
;
10084 /* Main entry point. */
10086 static unsigned int
10087 execute_lower_omp (void)
10091 /* This pass always runs, to provide PROP_gimple_lomp.
10092 But there is nothing to do unless -fopenmp is given. */
10093 if (flag_openmp
== 0 && flag_openmp_simd
== 0 && flag_enable_cilkplus
== 0)
10096 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
10097 delete_omp_context
);
10099 body
= gimple_body (current_function_decl
);
10100 scan_omp (&body
, NULL
);
10101 gcc_assert (taskreg_nesting_level
== 0);
10103 if (all_contexts
->root
)
10105 struct gimplify_ctx gctx
;
10107 if (task_shared_vars
)
10108 push_gimplify_context (&gctx
);
10109 lower_omp (&body
, NULL
);
10110 if (task_shared_vars
)
10111 pop_gimplify_context (NULL
);
10116 splay_tree_delete (all_contexts
);
10117 all_contexts
= NULL
;
10119 BITMAP_FREE (task_shared_vars
);
10125 const pass_data pass_data_lower_omp
=
10127 GIMPLE_PASS
, /* type */
10128 "omplower", /* name */
10129 OPTGROUP_NONE
, /* optinfo_flags */
10130 false, /* has_gate */
10131 true, /* has_execute */
10132 TV_NONE
, /* tv_id */
10133 PROP_gimple_any
, /* properties_required */
10134 PROP_gimple_lomp
, /* properties_provided */
10135 0, /* properties_destroyed */
10136 0, /* todo_flags_start */
10137 0, /* todo_flags_finish */
10140 class pass_lower_omp
: public gimple_opt_pass
10143 pass_lower_omp (gcc::context
*ctxt
)
10144 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
10147 /* opt_pass methods: */
10148 unsigned int execute () { return execute_lower_omp (); }
10150 }; // class pass_lower_omp
10152 } // anon namespace
10155 make_pass_lower_omp (gcc::context
*ctxt
)
10157 return new pass_lower_omp (ctxt
);
10160 /* The following is a utility to diagnose OpenMP structured block violations.
10161 It is not part of the "omplower" pass, as that's invoked too late. It
10162 should be invoked by the respective front ends after gimplification. */
10164 static splay_tree all_labels
;
10166 /* Check for mismatched contexts and generate an error if needed. Return
10167 true if an error is detected. */
10170 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
10171 gimple branch_ctx
, gimple label_ctx
)
10173 if (label_ctx
== branch_ctx
)
10178 Previously we kept track of the label's entire context in diagnose_sb_[12]
10179 so we could traverse it and issue a correct "exit" or "enter" error
10180 message upon a structured block violation.
10182 We built the context by building a list with tree_cons'ing, but there is
10183 no easy counterpart in gimple tuples. It seems like far too much work
10184 for issuing exit/enter error messages. If someone really misses the
10185 distinct error message... patches welcome.
10189 /* Try to avoid confusing the user by producing and error message
10190 with correct "exit" or "enter" verbiage. We prefer "exit"
10191 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
10192 if (branch_ctx
== NULL
)
10198 if (TREE_VALUE (label_ctx
) == branch_ctx
)
10203 label_ctx
= TREE_CHAIN (label_ctx
);
10208 error ("invalid exit from OpenMP structured block");
10210 error ("invalid entry to OpenMP structured block");
10213 bool cilkplus_block
= false;
10214 if (flag_enable_cilkplus
)
10217 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
10218 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
10219 || (gimple_code (label_ctx
) == GIMPLE_OMP_FOR
10220 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
10221 cilkplus_block
= true;
10224 /* If it's obvious we have an invalid entry, be specific about the error. */
10225 if (branch_ctx
== NULL
)
10227 if (cilkplus_block
)
10228 error ("invalid entry to Cilk Plus structured block");
10230 error ("invalid entry to OpenMP structured block");
10234 /* Otherwise, be vague and lazy, but efficient. */
10235 if (cilkplus_block
)
10236 error ("invalid branch to/from a Cilk Plus structured block");
10238 error ("invalid branch to/from an OpenMP structured block");
10241 gsi_replace (gsi_p
, gimple_build_nop (), false);
10245 /* Pass 1: Create a minimal tree of OpenMP structured blocks, and record
10246 where each label is found. */
10249 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
10250 struct walk_stmt_info
*wi
)
10252 gimple context
= (gimple
) wi
->info
;
10253 gimple inner_context
;
10254 gimple stmt
= gsi_stmt (*gsi_p
);
10256 *handled_ops_p
= true;
10258 switch (gimple_code (stmt
))
10262 case GIMPLE_OMP_PARALLEL
:
10263 case GIMPLE_OMP_TASK
:
10264 case GIMPLE_OMP_SECTIONS
:
10265 case GIMPLE_OMP_SINGLE
:
10266 case GIMPLE_OMP_SECTION
:
10267 case GIMPLE_OMP_MASTER
:
10268 case GIMPLE_OMP_ORDERED
:
10269 case GIMPLE_OMP_CRITICAL
:
10270 case GIMPLE_OMP_TARGET
:
10271 case GIMPLE_OMP_TEAMS
:
10272 case GIMPLE_OMP_TASKGROUP
:
10273 /* The minimal context here is just the current OMP construct. */
10274 inner_context
= stmt
;
10275 wi
->info
= inner_context
;
10276 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
10277 wi
->info
= context
;
10280 case GIMPLE_OMP_FOR
:
10281 inner_context
= stmt
;
10282 wi
->info
= inner_context
;
10283 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10285 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
10286 diagnose_sb_1
, NULL
, wi
);
10287 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
10288 wi
->info
= context
;
10292 splay_tree_insert (all_labels
, (splay_tree_key
) gimple_label_label (stmt
),
10293 (splay_tree_value
) context
);
10303 /* Pass 2: Check each branch and see if its context differs from that of
10304 the destination label's context. */
10307 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
10308 struct walk_stmt_info
*wi
)
10310 gimple context
= (gimple
) wi
->info
;
10312 gimple stmt
= gsi_stmt (*gsi_p
);
10314 *handled_ops_p
= true;
10316 switch (gimple_code (stmt
))
10320 case GIMPLE_OMP_PARALLEL
:
10321 case GIMPLE_OMP_TASK
:
10322 case GIMPLE_OMP_SECTIONS
:
10323 case GIMPLE_OMP_SINGLE
:
10324 case GIMPLE_OMP_SECTION
:
10325 case GIMPLE_OMP_MASTER
:
10326 case GIMPLE_OMP_ORDERED
:
10327 case GIMPLE_OMP_CRITICAL
:
10328 case GIMPLE_OMP_TARGET
:
10329 case GIMPLE_OMP_TEAMS
:
10330 case GIMPLE_OMP_TASKGROUP
:
10332 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
10333 wi
->info
= context
;
10336 case GIMPLE_OMP_FOR
:
10338 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10340 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
10341 diagnose_sb_2
, NULL
, wi
);
10342 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
10343 wi
->info
= context
;
10348 tree lab
= gimple_cond_true_label (stmt
);
10351 n
= splay_tree_lookup (all_labels
,
10352 (splay_tree_key
) lab
);
10353 diagnose_sb_0 (gsi_p
, context
,
10354 n
? (gimple
) n
->value
: NULL
);
10356 lab
= gimple_cond_false_label (stmt
);
10359 n
= splay_tree_lookup (all_labels
,
10360 (splay_tree_key
) lab
);
10361 diagnose_sb_0 (gsi_p
, context
,
10362 n
? (gimple
) n
->value
: NULL
);
10369 tree lab
= gimple_goto_dest (stmt
);
10370 if (TREE_CODE (lab
) != LABEL_DECL
)
10373 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
10374 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
) n
->value
: NULL
);
10378 case GIMPLE_SWITCH
:
10381 for (i
= 0; i
< gimple_switch_num_labels (stmt
); ++i
)
10383 tree lab
= CASE_LABEL (gimple_switch_label (stmt
, i
));
10384 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
10385 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
) n
->value
))
10391 case GIMPLE_RETURN
:
10392 diagnose_sb_0 (gsi_p
, context
, NULL
);
10402 /* Called from tree-cfg.c::make_edges to create cfg edges for all GIMPLE_OMP
10405 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
)
10407 gimple last
= last_stmt (bb
);
10408 enum gimple_code code
= gimple_code (last
);
10409 struct omp_region
*cur_region
= *region
;
10410 bool fallthru
= false;
10414 case GIMPLE_OMP_PARALLEL
:
10415 case GIMPLE_OMP_TASK
:
10416 case GIMPLE_OMP_FOR
:
10417 case GIMPLE_OMP_SINGLE
:
10418 case GIMPLE_OMP_TEAMS
:
10419 case GIMPLE_OMP_MASTER
:
10420 case GIMPLE_OMP_TASKGROUP
:
10421 case GIMPLE_OMP_ORDERED
:
10422 case GIMPLE_OMP_CRITICAL
:
10423 case GIMPLE_OMP_SECTION
:
10424 cur_region
= new_omp_region (bb
, code
, cur_region
);
10428 case GIMPLE_OMP_TARGET
:
10429 cur_region
= new_omp_region (bb
, code
, cur_region
);
10431 if (gimple_omp_target_kind (last
) == GF_OMP_TARGET_KIND_UPDATE
)
10432 cur_region
= cur_region
->outer
;
10435 case GIMPLE_OMP_SECTIONS
:
10436 cur_region
= new_omp_region (bb
, code
, cur_region
);
10440 case GIMPLE_OMP_SECTIONS_SWITCH
:
10444 case GIMPLE_OMP_ATOMIC_LOAD
:
10445 case GIMPLE_OMP_ATOMIC_STORE
:
10449 case GIMPLE_OMP_RETURN
:
10450 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
10451 somewhere other than the next block. This will be
10453 cur_region
->exit
= bb
;
10454 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
10455 cur_region
= cur_region
->outer
;
10458 case GIMPLE_OMP_CONTINUE
:
10459 cur_region
->cont
= bb
;
10460 switch (cur_region
->type
)
10462 case GIMPLE_OMP_FOR
:
10463 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
10464 succs edges as abnormal to prevent splitting
10466 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
10467 /* Make the loopback edge. */
10468 make_edge (bb
, single_succ (cur_region
->entry
),
10471 /* Create an edge from GIMPLE_OMP_FOR to exit, which
10472 corresponds to the case that the body of the loop
10473 is not executed at all. */
10474 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
10475 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
10479 case GIMPLE_OMP_SECTIONS
:
10480 /* Wire up the edges into and out of the nested sections. */
10482 basic_block switch_bb
= single_succ (cur_region
->entry
);
10484 struct omp_region
*i
;
10485 for (i
= cur_region
->inner
; i
; i
= i
->next
)
10487 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
10488 make_edge (switch_bb
, i
->entry
, 0);
10489 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
10492 /* Make the loopback edge to the block with
10493 GIMPLE_OMP_SECTIONS_SWITCH. */
10494 make_edge (bb
, switch_bb
, 0);
10496 /* Make the edge from the switch to exit. */
10497 make_edge (switch_bb
, bb
->next_bb
, 0);
10503 gcc_unreachable ();
10508 gcc_unreachable ();
10511 if (*region
!= cur_region
)
10512 *region
= cur_region
;
10517 static unsigned int
10518 diagnose_omp_structured_block_errors (void)
10520 struct walk_stmt_info wi
;
10521 gimple_seq body
= gimple_body (current_function_decl
);
10523 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
10525 memset (&wi
, 0, sizeof (wi
));
10526 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
10528 memset (&wi
, 0, sizeof (wi
));
10529 wi
.want_locations
= true;
10530 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
10532 gimple_set_body (current_function_decl
, body
);
10534 splay_tree_delete (all_labels
);
10541 gate_diagnose_omp_blocks (void)
10543 return flag_openmp
|| flag_enable_cilkplus
;
10548 const pass_data pass_data_diagnose_omp_blocks
=
10550 GIMPLE_PASS
, /* type */
10551 "*diagnose_omp_blocks", /* name */
10552 OPTGROUP_NONE
, /* optinfo_flags */
10553 true, /* has_gate */
10554 true, /* has_execute */
10555 TV_NONE
, /* tv_id */
10556 PROP_gimple_any
, /* properties_required */
10557 0, /* properties_provided */
10558 0, /* properties_destroyed */
10559 0, /* todo_flags_start */
10560 0, /* todo_flags_finish */
10563 class pass_diagnose_omp_blocks
: public gimple_opt_pass
10566 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
10567 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
10570 /* opt_pass methods: */
10571 bool gate () { return gate_diagnose_omp_blocks (); }
10572 unsigned int execute () {
10573 return diagnose_omp_structured_block_errors ();
10576 }; // class pass_diagnose_omp_blocks
10578 } // anon namespace
10581 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
10583 return new pass_diagnose_omp_blocks (ctxt
);
10586 #include "gt-omp-low.h"