1 /* Lowering pass for OpenMP directives. Converts OpenMP directives
2 into explicit calls to the runtime library (libgomp) and data
3 marshalling to implement data sharing and copying clauses.
4 Contributed by Diego Novillo <dnovillo@redhat.com>
6 Copyright (C) 2005-2013 Free Software Foundation, Inc.
8 This file is part of GCC.
10 GCC is free software; you can redistribute it and/or modify it under
11 the terms of the GNU General Public License as published by the Free
12 Software Foundation; either version 3, or (at your option) any later
15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
17 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
20 You should have received a copy of the GNU General Public License
21 along with GCC; see the file COPYING3. If not see
22 <http://www.gnu.org/licenses/>. */
26 #include "coretypes.h"
31 #include "gimple-iterator.h"
32 #include "gimple-walk.h"
33 #include "tree-iterator.h"
34 #include "tree-inline.h"
35 #include "langhooks.h"
36 #include "diagnostic-core.h"
37 #include "gimple-ssa.h"
40 #include "tree-phinodes.h"
41 #include "ssa-iterators.h"
42 #include "tree-ssanames.h"
43 #include "tree-into-ssa.h"
49 #include "tree-pass.h"
52 #include "splay-tree.h"
57 #include "gimple-low.h"
58 #include "tree-cfgcleanup.h"
61 /* Lowering of OpenMP parallel and workshare constructs proceeds in two
62 phases. The first phase scans the function looking for OMP statements
63 and then for variables that must be replaced to satisfy data sharing
64 clauses. The second phase expands code for the constructs, as well as
65 re-gimplifying things when variables have been replaced with complex
68 Final code generation is done by pass_expand_omp. The flowgraph is
69 scanned for parallel regions which are then moved to a new
70 function, to be invoked by the thread library. */
72 /* Parallel region information. Every parallel and workshare
73 directive is enclosed between two markers, the OMP_* directive
74 and a corresponding OMP_RETURN statement. */
78 /* The enclosing region. */
79 struct omp_region
*outer
;
81 /* First child region. */
82 struct omp_region
*inner
;
84 /* Next peer region. */
85 struct omp_region
*next
;
87 /* Block containing the omp directive as its last stmt. */
90 /* Block containing the OMP_RETURN as its last stmt. */
93 /* Block containing the OMP_CONTINUE as its last stmt. */
96 /* If this is a combined parallel+workshare region, this is a list
97 of additional arguments needed by the combined parallel+workshare
99 vec
<tree
, va_gc
> *ws_args
;
101 /* The code for the omp directive of this region. */
102 enum gimple_code type
;
104 /* Schedule kind, only used for OMP_FOR type regions. */
105 enum omp_clause_schedule_kind sched_kind
;
107 /* True if this is a combined parallel+workshare region. */
108 bool is_combined_parallel
;
111 /* Context structure. Used to store information about each parallel
112 directive in the code. */
114 typedef struct omp_context
116 /* This field must be at the beginning, as we do "inheritance": Some
117 callback functions for tree-inline.c (e.g., omp_copy_decl)
118 receive a copy_body_data pointer that is up-casted to an
119 omp_context pointer. */
122 /* The tree of contexts corresponding to the encountered constructs. */
123 struct omp_context
*outer
;
126 /* Map variables to fields in a structure that allows communication
127 between sending and receiving threads. */
128 splay_tree field_map
;
133 /* These are used just by task contexts, if task firstprivate fn is
134 needed. srecord_type is used to communicate from the thread
135 that encountered the task construct to task firstprivate fn,
136 record_type is allocated by GOMP_task, initialized by task firstprivate
137 fn and passed to the task body fn. */
138 splay_tree sfield_map
;
141 /* A chain of variables to add to the top-level block surrounding the
142 construct. In the case of a parallel, this is in the child function. */
145 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
146 barriers should jump to during omplower pass. */
149 /* What to do with variables with implicitly determined sharing
151 enum omp_clause_default_kind default_kind
;
153 /* Nesting depth of this context. Used to beautify error messages re
154 invalid gotos. The outermost ctx is depth 1, with depth 0 being
155 reserved for the main body of the function. */
158 /* True if this parallel directive is nested within another. */
161 /* True if this construct can be cancelled. */
166 struct omp_for_data_loop
168 tree v
, n1
, n2
, step
;
169 enum tree_code cond_code
;
172 /* A structure describing the main elements of a parallel loop. */
176 struct omp_for_data_loop loop
;
181 bool have_nowait
, have_ordered
;
182 enum omp_clause_schedule_kind sched_kind
;
183 struct omp_for_data_loop
*loops
;
187 static splay_tree all_contexts
;
188 static int taskreg_nesting_level
;
189 static int target_nesting_level
;
190 static struct omp_region
*root_omp_region
;
191 static bitmap task_shared_vars
;
193 static void scan_omp (gimple_seq
*, omp_context
*);
194 static tree
scan_omp_1_op (tree
*, int *, void *);
196 #define WALK_SUBSTMTS \
200 case GIMPLE_EH_FILTER: \
201 case GIMPLE_TRANSACTION: \
202 /* The sub-statements for these should be walked. */ \
203 *handled_ops_p = false; \
206 /* Convenience function for calling scan_omp_1_op on tree operands. */
209 scan_omp_op (tree
*tp
, omp_context
*ctx
)
211 struct walk_stmt_info wi
;
213 memset (&wi
, 0, sizeof (wi
));
215 wi
.want_locations
= true;
217 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
220 static void lower_omp (gimple_seq
*, omp_context
*);
221 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
222 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
224 /* Find an OpenMP clause of type KIND within CLAUSES. */
227 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
229 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
230 if (OMP_CLAUSE_CODE (clauses
) == kind
)
236 /* Return true if CTX is for an omp parallel. */
239 is_parallel_ctx (omp_context
*ctx
)
241 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
245 /* Return true if CTX is for an omp task. */
248 is_task_ctx (omp_context
*ctx
)
250 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
254 /* Return true if CTX is for an omp parallel or omp task. */
257 is_taskreg_ctx (omp_context
*ctx
)
259 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
260 || gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
264 /* Return true if REGION is a combined parallel+workshare region. */
267 is_combined_parallel (struct omp_region
*region
)
269 return region
->is_combined_parallel
;
273 /* Extract the header elements of parallel loop FOR_STMT and store
277 extract_omp_for_data (gimple for_stmt
, struct omp_for_data
*fd
,
278 struct omp_for_data_loop
*loops
)
280 tree t
, var
, *collapse_iter
, *collapse_count
;
281 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
282 struct omp_for_data_loop
*loop
;
284 struct omp_for_data_loop dummy_loop
;
285 location_t loc
= gimple_location (for_stmt
);
286 bool simd
= gimple_omp_for_kind (for_stmt
) == GF_OMP_FOR_KIND_SIMD
;
287 bool distribute
= gimple_omp_for_kind (for_stmt
)
288 == GF_OMP_FOR_KIND_DISTRIBUTE
;
290 fd
->for_stmt
= for_stmt
;
292 fd
->collapse
= gimple_omp_for_collapse (for_stmt
);
293 if (fd
->collapse
> 1)
296 fd
->loops
= &fd
->loop
;
298 fd
->have_nowait
= distribute
|| simd
;
299 fd
->have_ordered
= false;
300 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
301 fd
->chunk_size
= NULL_TREE
;
302 collapse_iter
= NULL
;
303 collapse_count
= NULL
;
305 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
306 switch (OMP_CLAUSE_CODE (t
))
308 case OMP_CLAUSE_NOWAIT
:
309 fd
->have_nowait
= true;
311 case OMP_CLAUSE_ORDERED
:
312 fd
->have_ordered
= true;
314 case OMP_CLAUSE_SCHEDULE
:
315 gcc_assert (!distribute
);
316 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_KIND (t
);
317 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
319 case OMP_CLAUSE_DIST_SCHEDULE
:
320 gcc_assert (distribute
);
321 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
323 case OMP_CLAUSE_COLLAPSE
:
324 if (fd
->collapse
> 1)
326 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
327 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
333 /* FIXME: for now map schedule(auto) to schedule(static).
334 There should be analysis to determine whether all iterations
335 are approximately the same amount of work (then schedule(static)
336 is best) or if it varies (then schedule(dynamic,N) is better). */
337 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
339 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
340 gcc_assert (fd
->chunk_size
== NULL
);
342 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
343 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
344 gcc_assert (fd
->chunk_size
== NULL
);
345 else if (fd
->chunk_size
== NULL
)
347 /* We only need to compute a default chunk size for ordered
348 static loops and dynamic loops. */
349 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
351 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
352 ? integer_zero_node
: integer_one_node
;
355 for (i
= 0; i
< fd
->collapse
; i
++)
357 if (fd
->collapse
== 1)
359 else if (loops
!= NULL
)
364 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
365 gcc_assert (SSA_VAR_P (loop
->v
));
366 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
367 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
368 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
369 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
371 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
372 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
373 switch (loop
->cond_code
)
379 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
380 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, 1);
382 loop
->n2
= fold_build2_loc (loc
,
383 PLUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
384 build_int_cst (TREE_TYPE (loop
->n2
), 1));
385 loop
->cond_code
= LT_EXPR
;
388 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
389 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, -1);
391 loop
->n2
= fold_build2_loc (loc
,
392 MINUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
393 build_int_cst (TREE_TYPE (loop
->n2
), 1));
394 loop
->cond_code
= GT_EXPR
;
400 t
= gimple_omp_for_incr (for_stmt
, i
);
401 gcc_assert (TREE_OPERAND (t
, 0) == var
);
402 switch (TREE_CODE (t
))
405 loop
->step
= TREE_OPERAND (t
, 1);
407 case POINTER_PLUS_EXPR
:
408 loop
->step
= fold_convert (ssizetype
, TREE_OPERAND (t
, 1));
411 loop
->step
= TREE_OPERAND (t
, 1);
412 loop
->step
= fold_build1_loc (loc
,
413 NEGATE_EXPR
, TREE_TYPE (loop
->step
),
421 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
422 && !fd
->have_ordered
))
424 if (fd
->collapse
== 1)
425 iter_type
= TREE_TYPE (loop
->v
);
427 || TYPE_PRECISION (iter_type
)
428 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
430 = build_nonstandard_integer_type
431 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
433 else if (iter_type
!= long_long_unsigned_type_node
)
435 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
436 iter_type
= long_long_unsigned_type_node
;
437 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
438 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
439 >= TYPE_PRECISION (iter_type
))
443 if (loop
->cond_code
== LT_EXPR
)
444 n
= fold_build2_loc (loc
,
445 PLUS_EXPR
, TREE_TYPE (loop
->v
),
446 loop
->n2
, loop
->step
);
449 if (TREE_CODE (n
) != INTEGER_CST
450 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
451 iter_type
= long_long_unsigned_type_node
;
453 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
454 > TYPE_PRECISION (iter_type
))
458 if (loop
->cond_code
== LT_EXPR
)
461 n2
= fold_build2_loc (loc
,
462 PLUS_EXPR
, TREE_TYPE (loop
->v
),
463 loop
->n2
, loop
->step
);
467 n1
= fold_build2_loc (loc
,
468 MINUS_EXPR
, TREE_TYPE (loop
->v
),
469 loop
->n2
, loop
->step
);
472 if (TREE_CODE (n1
) != INTEGER_CST
473 || TREE_CODE (n2
) != INTEGER_CST
474 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
475 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
476 iter_type
= long_long_unsigned_type_node
;
480 if (collapse_count
&& *collapse_count
== NULL
)
482 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
483 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
484 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
485 if (t
&& integer_zerop (t
))
486 count
= build_zero_cst (long_long_unsigned_type_node
);
487 else if ((i
== 0 || count
!= NULL_TREE
)
488 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
489 && TREE_CONSTANT (loop
->n1
)
490 && TREE_CONSTANT (loop
->n2
)
491 && TREE_CODE (loop
->step
) == INTEGER_CST
)
493 tree itype
= TREE_TYPE (loop
->v
);
495 if (POINTER_TYPE_P (itype
))
496 itype
= signed_type_for (itype
);
497 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
498 t
= fold_build2_loc (loc
,
500 fold_convert_loc (loc
, itype
, loop
->step
), t
);
501 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
502 fold_convert_loc (loc
, itype
, loop
->n2
));
503 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
504 fold_convert_loc (loc
, itype
, loop
->n1
));
505 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
506 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
507 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
508 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
509 fold_convert_loc (loc
, itype
,
512 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
513 fold_convert_loc (loc
, itype
, loop
->step
));
514 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
515 if (count
!= NULL_TREE
)
516 count
= fold_build2_loc (loc
,
517 MULT_EXPR
, long_long_unsigned_type_node
,
521 if (TREE_CODE (count
) != INTEGER_CST
)
524 else if (count
&& !integer_zerop (count
))
531 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
532 || fd
->have_ordered
))
534 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
535 iter_type
= long_long_unsigned_type_node
;
537 iter_type
= long_integer_type_node
;
539 else if (collapse_iter
&& *collapse_iter
!= NULL
)
540 iter_type
= TREE_TYPE (*collapse_iter
);
541 fd
->iter_type
= iter_type
;
542 if (collapse_iter
&& *collapse_iter
== NULL
)
543 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
544 if (collapse_count
&& *collapse_count
== NULL
)
547 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
549 *collapse_count
= create_tmp_var (iter_type
, ".count");
552 if (fd
->collapse
> 1)
554 fd
->loop
.v
= *collapse_iter
;
555 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
556 fd
->loop
.n2
= *collapse_count
;
557 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
558 fd
->loop
.cond_code
= LT_EXPR
;
563 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
564 is the immediate dominator of PAR_ENTRY_BB, return true if there
565 are no data dependencies that would prevent expanding the parallel
566 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
568 When expanding a combined parallel+workshare region, the call to
569 the child function may need additional arguments in the case of
570 GIMPLE_OMP_FOR regions. In some cases, these arguments are
571 computed out of variables passed in from the parent to the child
572 via 'struct .omp_data_s'. For instance:
574 #pragma omp parallel for schedule (guided, i * 4)
579 # BLOCK 2 (PAR_ENTRY_BB)
581 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
583 # BLOCK 3 (WS_ENTRY_BB)
584 .omp_data_i = &.omp_data_o;
585 D.1667 = .omp_data_i->i;
587 #pragma omp for schedule (guided, D.1598)
589 When we outline the parallel region, the call to the child function
590 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
591 that value is computed *after* the call site. So, in principle we
592 cannot do the transformation.
594 To see whether the code in WS_ENTRY_BB blocks the combined
595 parallel+workshare call, we collect all the variables used in the
596 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
597 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
600 FIXME. If we had the SSA form built at this point, we could merely
601 hoist the code in block 3 into block 2 and be done with it. But at
602 this point we don't have dataflow information and though we could
603 hack something up here, it is really not worth the aggravation. */
606 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
608 struct omp_for_data fd
;
609 gimple ws_stmt
= last_stmt (ws_entry_bb
);
611 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
614 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
616 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
618 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
620 if (fd
.iter_type
!= long_integer_type_node
)
623 /* FIXME. We give up too easily here. If any of these arguments
624 are not constants, they will likely involve variables that have
625 been mapped into fields of .omp_data_s for sharing with the child
626 function. With appropriate data flow, it would be possible to
628 if (!is_gimple_min_invariant (fd
.loop
.n1
)
629 || !is_gimple_min_invariant (fd
.loop
.n2
)
630 || !is_gimple_min_invariant (fd
.loop
.step
)
631 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
638 /* Collect additional arguments needed to emit a combined
639 parallel+workshare call. WS_STMT is the workshare directive being
642 static vec
<tree
, va_gc
> *
643 get_ws_args_for (gimple par_stmt
, gimple ws_stmt
)
646 location_t loc
= gimple_location (ws_stmt
);
647 vec
<tree
, va_gc
> *ws_args
;
649 if (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
)
651 struct omp_for_data fd
;
654 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
658 if (gimple_omp_for_combined_into_p (ws_stmt
))
661 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
662 OMP_CLAUSE__LOOPTEMP_
);
664 n1
= OMP_CLAUSE_DECL (innerc
);
665 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
666 OMP_CLAUSE__LOOPTEMP_
);
668 n2
= OMP_CLAUSE_DECL (innerc
);
671 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
673 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
674 ws_args
->quick_push (t
);
676 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
677 ws_args
->quick_push (t
);
679 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
680 ws_args
->quick_push (t
);
684 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
685 ws_args
->quick_push (t
);
690 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
692 /* Number of sections is equal to the number of edges from the
693 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
694 the exit of the sections region. */
695 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
696 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
697 vec_alloc (ws_args
, 1);
698 ws_args
->quick_push (t
);
706 /* Discover whether REGION is a combined parallel+workshare region. */
709 determine_parallel_type (struct omp_region
*region
)
711 basic_block par_entry_bb
, par_exit_bb
;
712 basic_block ws_entry_bb
, ws_exit_bb
;
714 if (region
== NULL
|| region
->inner
== NULL
715 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
716 || region
->inner
->cont
== NULL
)
719 /* We only support parallel+for and parallel+sections. */
720 if (region
->type
!= GIMPLE_OMP_PARALLEL
721 || (region
->inner
->type
!= GIMPLE_OMP_FOR
722 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
725 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
726 WS_EXIT_BB -> PAR_EXIT_BB. */
727 par_entry_bb
= region
->entry
;
728 par_exit_bb
= region
->exit
;
729 ws_entry_bb
= region
->inner
->entry
;
730 ws_exit_bb
= region
->inner
->exit
;
732 if (single_succ (par_entry_bb
) == ws_entry_bb
733 && single_succ (ws_exit_bb
) == par_exit_bb
734 && workshare_safe_to_combine_p (ws_entry_bb
)
735 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
736 || (last_and_only_stmt (ws_entry_bb
)
737 && last_and_only_stmt (par_exit_bb
))))
739 gimple par_stmt
= last_stmt (par_entry_bb
);
740 gimple ws_stmt
= last_stmt (ws_entry_bb
);
742 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
744 /* If this is a combined parallel loop, we need to determine
745 whether or not to use the combined library calls. There
746 are two cases where we do not apply the transformation:
747 static loops and any kind of ordered loop. In the first
748 case, we already open code the loop so there is no need
749 to do anything else. In the latter case, the combined
750 parallel loop call would still need extra synchronization
751 to implement ordered semantics, so there would not be any
752 gain in using the combined call. */
753 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
754 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
756 || OMP_CLAUSE_SCHEDULE_KIND (c
) == OMP_CLAUSE_SCHEDULE_STATIC
757 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
759 region
->is_combined_parallel
= false;
760 region
->inner
->is_combined_parallel
= false;
765 region
->is_combined_parallel
= true;
766 region
->inner
->is_combined_parallel
= true;
767 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
772 /* Return true if EXPR is variable sized. */
775 is_variable_sized (const_tree expr
)
777 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
780 /* Return true if DECL is a reference type. */
783 is_reference (tree decl
)
785 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
788 /* Lookup variables in the decl or field splay trees. The "maybe" form
789 allows for the variable form to not have been entered, otherwise we
790 assert that the variable must have been entered. */
793 lookup_decl (tree var
, omp_context
*ctx
)
796 n
= (tree
*) pointer_map_contains (ctx
->cb
.decl_map
, var
);
801 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
804 n
= (tree
*) pointer_map_contains (ctx
->cb
.decl_map
, var
);
805 return n
? *n
: NULL_TREE
;
809 lookup_field (tree var
, omp_context
*ctx
)
812 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
813 return (tree
) n
->value
;
817 lookup_sfield (tree var
, omp_context
*ctx
)
820 n
= splay_tree_lookup (ctx
->sfield_map
821 ? ctx
->sfield_map
: ctx
->field_map
,
822 (splay_tree_key
) var
);
823 return (tree
) n
->value
;
827 maybe_lookup_field (tree var
, omp_context
*ctx
)
830 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
831 return n
? (tree
) n
->value
: NULL_TREE
;
834 /* Return true if DECL should be copied by pointer. SHARED_CTX is
835 the parallel context if DECL is to be shared. */
838 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
840 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
843 /* We can only use copy-in/copy-out semantics for shared variables
844 when we know the value is not accessible from an outer scope. */
847 /* ??? Trivially accessible from anywhere. But why would we even
848 be passing an address in this case? Should we simply assert
849 this to be false, or should we have a cleanup pass that removes
850 these from the list of mappings? */
851 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
854 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
855 without analyzing the expression whether or not its location
856 is accessible to anyone else. In the case of nested parallel
857 regions it certainly may be. */
858 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
861 /* Do not use copy-in/copy-out for variables that have their
863 if (TREE_ADDRESSABLE (decl
))
866 /* lower_send_shared_vars only uses copy-in, but not copy-out
868 if (TREE_READONLY (decl
)
869 || ((TREE_CODE (decl
) == RESULT_DECL
870 || TREE_CODE (decl
) == PARM_DECL
)
871 && DECL_BY_REFERENCE (decl
)))
874 /* Disallow copy-in/out in nested parallel if
875 decl is shared in outer parallel, otherwise
876 each thread could store the shared variable
877 in its own copy-in location, making the
878 variable no longer really shared. */
879 if (shared_ctx
->is_nested
)
883 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
884 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
891 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
892 c
; c
= OMP_CLAUSE_CHAIN (c
))
893 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
894 && OMP_CLAUSE_DECL (c
) == decl
)
898 goto maybe_mark_addressable_and_ret
;
902 /* For tasks avoid using copy-in/out. As tasks can be
903 deferred or executed in different thread, when GOMP_task
904 returns, the task hasn't necessarily terminated. */
905 if (is_task_ctx (shared_ctx
))
908 maybe_mark_addressable_and_ret
:
909 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
910 if (is_gimple_reg (outer
))
912 /* Taking address of OUTER in lower_send_shared_vars
913 might need regimplification of everything that uses the
915 if (!task_shared_vars
)
916 task_shared_vars
= BITMAP_ALLOC (NULL
);
917 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
918 TREE_ADDRESSABLE (outer
) = 1;
927 /* Construct a new automatic decl similar to VAR. */
930 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
932 tree copy
= copy_var_decl (var
, name
, type
);
934 DECL_CONTEXT (copy
) = current_function_decl
;
935 DECL_CHAIN (copy
) = ctx
->block_vars
;
936 ctx
->block_vars
= copy
;
942 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
944 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
947 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
950 omp_build_component_ref (tree obj
, tree field
)
952 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
953 if (TREE_THIS_VOLATILE (field
))
954 TREE_THIS_VOLATILE (ret
) |= 1;
955 if (TREE_READONLY (field
))
956 TREE_READONLY (ret
) |= 1;
960 /* Build tree nodes to access the field for VAR on the receiver side. */
963 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
965 tree x
, field
= lookup_field (var
, ctx
);
967 /* If the receiver record type was remapped in the child function,
968 remap the field into the new record type. */
969 x
= maybe_lookup_field (field
, ctx
);
973 x
= build_simple_mem_ref (ctx
->receiver_decl
);
974 x
= omp_build_component_ref (x
, field
);
976 x
= build_simple_mem_ref (x
);
981 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
982 of a parallel, this is a component reference; for workshare constructs
983 this is some variable. */
986 build_outer_var_ref (tree var
, omp_context
*ctx
)
990 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
992 else if (is_variable_sized (var
))
994 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
995 x
= build_outer_var_ref (x
, ctx
);
996 x
= build_simple_mem_ref (x
);
998 else if (is_taskreg_ctx (ctx
))
1000 bool by_ref
= use_pointer_for_field (var
, NULL
);
1001 x
= build_receiver_ref (var
, by_ref
, ctx
);
1003 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1004 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_SIMD
)
1006 /* #pragma omp simd isn't a worksharing construct, and can reference even
1007 private vars in its linear etc. clauses. */
1009 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1010 x
= lookup_decl (var
, ctx
->outer
);
1011 else if (ctx
->outer
)
1012 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1016 else if (ctx
->outer
)
1017 x
= lookup_decl (var
, ctx
->outer
);
1018 else if (is_reference (var
))
1019 /* This can happen with orphaned constructs. If var is reference, it is
1020 possible it is shared and as such valid. */
1025 if (is_reference (var
))
1026 x
= build_simple_mem_ref (x
);
1031 /* Build tree nodes to access the field for VAR on the sender side. */
1034 build_sender_ref (tree var
, omp_context
*ctx
)
1036 tree field
= lookup_sfield (var
, ctx
);
1037 return omp_build_component_ref (ctx
->sender_decl
, field
);
1040 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1043 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
)
1045 tree field
, type
, sfield
= NULL_TREE
;
1047 gcc_assert ((mask
& 1) == 0
1048 || !splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
));
1049 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1050 || !splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) var
));
1052 type
= TREE_TYPE (var
);
1055 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1056 type
= build_pointer_type (build_pointer_type (type
));
1059 type
= build_pointer_type (type
);
1060 else if ((mask
& 3) == 1 && is_reference (var
))
1061 type
= TREE_TYPE (type
);
1063 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1064 FIELD_DECL
, DECL_NAME (var
), type
);
1066 /* Remember what variable this field was created for. This does have a
1067 side effect of making dwarf2out ignore this member, so for helpful
1068 debugging we clear it later in delete_omp_context. */
1069 DECL_ABSTRACT_ORIGIN (field
) = var
;
1070 if (type
== TREE_TYPE (var
))
1072 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1073 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1074 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1077 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1079 if ((mask
& 3) == 3)
1081 insert_field_into_struct (ctx
->record_type
, field
);
1082 if (ctx
->srecord_type
)
1084 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1085 FIELD_DECL
, DECL_NAME (var
), type
);
1086 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1087 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1088 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1089 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1090 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1095 if (ctx
->srecord_type
== NULL_TREE
)
1099 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1100 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1101 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1103 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1104 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1105 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1106 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1107 splay_tree_insert (ctx
->sfield_map
,
1108 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1109 (splay_tree_value
) sfield
);
1113 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1114 : ctx
->srecord_type
, field
);
1118 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) var
,
1119 (splay_tree_value
) field
);
1120 if ((mask
& 2) && ctx
->sfield_map
)
1121 splay_tree_insert (ctx
->sfield_map
, (splay_tree_key
) var
,
1122 (splay_tree_value
) sfield
);
1126 install_var_local (tree var
, omp_context
*ctx
)
1128 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1129 insert_decl_map (&ctx
->cb
, var
, new_var
);
1133 /* Adjust the replacement for DECL in CTX for the new context. This means
1134 copying the DECL_VALUE_EXPR, and fixing up the type. */
1137 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1139 tree new_decl
, size
;
1141 new_decl
= lookup_decl (decl
, ctx
);
1143 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1145 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1146 && DECL_HAS_VALUE_EXPR_P (decl
))
1148 tree ve
= DECL_VALUE_EXPR (decl
);
1149 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1150 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1151 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1154 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1156 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1157 if (size
== error_mark_node
)
1158 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1159 DECL_SIZE (new_decl
) = size
;
1161 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1162 if (size
== error_mark_node
)
1163 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1164 DECL_SIZE_UNIT (new_decl
) = size
;
1168 /* The callback for remap_decl. Search all containing contexts for a
1169 mapping of the variable; this avoids having to duplicate the splay
1170 tree ahead of time. We know a mapping doesn't already exist in the
1171 given context. Create new mappings to implement default semantics. */
1174 omp_copy_decl (tree var
, copy_body_data
*cb
)
1176 omp_context
*ctx
= (omp_context
*) cb
;
1179 if (TREE_CODE (var
) == LABEL_DECL
)
1181 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1182 DECL_CONTEXT (new_var
) = current_function_decl
;
1183 insert_decl_map (&ctx
->cb
, var
, new_var
);
1187 while (!is_taskreg_ctx (ctx
))
1192 new_var
= maybe_lookup_decl (var
, ctx
);
1197 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1200 return error_mark_node
;
1204 /* Return the parallel region associated with STMT. */
1206 /* Debugging dumps for parallel regions. */
1207 void dump_omp_region (FILE *, struct omp_region
*, int);
1208 void debug_omp_region (struct omp_region
*);
1209 void debug_all_omp_regions (void);
1211 /* Dump the parallel region tree rooted at REGION. */
1214 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1216 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1217 gimple_code_name
[region
->type
]);
1220 dump_omp_region (file
, region
->inner
, indent
+ 4);
1224 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1225 region
->cont
->index
);
1229 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1230 region
->exit
->index
);
1232 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1235 dump_omp_region (file
, region
->next
, indent
);
1239 debug_omp_region (struct omp_region
*region
)
1241 dump_omp_region (stderr
, region
, 0);
1245 debug_all_omp_regions (void)
1247 dump_omp_region (stderr
, root_omp_region
, 0);
1251 /* Create a new parallel region starting at STMT inside region PARENT. */
1253 static struct omp_region
*
1254 new_omp_region (basic_block bb
, enum gimple_code type
,
1255 struct omp_region
*parent
)
1257 struct omp_region
*region
= XCNEW (struct omp_region
);
1259 region
->outer
= parent
;
1261 region
->type
= type
;
1265 /* This is a nested region. Add it to the list of inner
1266 regions in PARENT. */
1267 region
->next
= parent
->inner
;
1268 parent
->inner
= region
;
1272 /* This is a toplevel region. Add it to the list of toplevel
1273 regions in ROOT_OMP_REGION. */
1274 region
->next
= root_omp_region
;
1275 root_omp_region
= region
;
1281 /* Release the memory associated with the region tree rooted at REGION. */
1284 free_omp_region_1 (struct omp_region
*region
)
1286 struct omp_region
*i
, *n
;
1288 for (i
= region
->inner
; i
; i
= n
)
1291 free_omp_region_1 (i
);
1297 /* Release the memory for the entire omp region tree. */
1300 free_omp_regions (void)
1302 struct omp_region
*r
, *n
;
1303 for (r
= root_omp_region
; r
; r
= n
)
1306 free_omp_region_1 (r
);
1308 root_omp_region
= NULL
;
1312 /* Create a new context, with OUTER_CTX being the surrounding context. */
1314 static omp_context
*
1315 new_omp_context (gimple stmt
, omp_context
*outer_ctx
)
1317 omp_context
*ctx
= XCNEW (omp_context
);
1319 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1320 (splay_tree_value
) ctx
);
1325 ctx
->outer
= outer_ctx
;
1326 ctx
->cb
= outer_ctx
->cb
;
1327 ctx
->cb
.block
= NULL
;
1328 ctx
->depth
= outer_ctx
->depth
+ 1;
1332 ctx
->cb
.src_fn
= current_function_decl
;
1333 ctx
->cb
.dst_fn
= current_function_decl
;
1334 ctx
->cb
.src_node
= cgraph_get_node (current_function_decl
);
1335 gcc_checking_assert (ctx
->cb
.src_node
);
1336 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1337 ctx
->cb
.src_cfun
= cfun
;
1338 ctx
->cb
.copy_decl
= omp_copy_decl
;
1339 ctx
->cb
.eh_lp_nr
= 0;
1340 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1344 ctx
->cb
.decl_map
= pointer_map_create ();
1349 static gimple_seq
maybe_catch_exception (gimple_seq
);
1351 /* Finalize task copyfn. */
1354 finalize_task_copyfn (gimple task_stmt
)
1356 struct function
*child_cfun
;
1358 gimple_seq seq
= NULL
, new_seq
;
1361 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1362 if (child_fn
== NULL_TREE
)
1365 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1366 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1368 push_cfun (child_cfun
);
1369 bind
= gimplify_body (child_fn
, false);
1370 gimple_seq_add_stmt (&seq
, bind
);
1371 new_seq
= maybe_catch_exception (seq
);
1374 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1376 gimple_seq_add_stmt (&seq
, bind
);
1378 gimple_set_body (child_fn
, seq
);
1381 /* Inform the callgraph about the new function. */
1382 cgraph_add_new_function (child_fn
, false);
1385 /* Destroy a omp_context data structures. Called through the splay tree
1386 value delete callback. */
1389 delete_omp_context (splay_tree_value value
)
1391 omp_context
*ctx
= (omp_context
*) value
;
1393 pointer_map_destroy (ctx
->cb
.decl_map
);
1396 splay_tree_delete (ctx
->field_map
);
1397 if (ctx
->sfield_map
)
1398 splay_tree_delete (ctx
->sfield_map
);
1400 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1401 it produces corrupt debug information. */
1402 if (ctx
->record_type
)
1405 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1406 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1408 if (ctx
->srecord_type
)
1411 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1412 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1415 if (is_task_ctx (ctx
))
1416 finalize_task_copyfn (ctx
->stmt
);
1421 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1425 fixup_child_record_type (omp_context
*ctx
)
1427 tree f
, type
= ctx
->record_type
;
1429 /* ??? It isn't sufficient to just call remap_type here, because
1430 variably_modified_type_p doesn't work the way we expect for
1431 record types. Testing each field for whether it needs remapping
1432 and creating a new record by hand works, however. */
1433 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1434 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1438 tree name
, new_fields
= NULL
;
1440 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1441 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1442 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1443 TYPE_DECL
, name
, type
);
1444 TYPE_NAME (type
) = name
;
1446 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1448 tree new_f
= copy_node (f
);
1449 DECL_CONTEXT (new_f
) = type
;
1450 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1451 DECL_CHAIN (new_f
) = new_fields
;
1452 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1453 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1455 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1459 /* Arrange to be able to look up the receiver field
1460 given the sender field. */
1461 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1462 (splay_tree_value
) new_f
);
1464 TYPE_FIELDS (type
) = nreverse (new_fields
);
1468 TREE_TYPE (ctx
->receiver_decl
) = build_pointer_type (type
);
1471 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1472 specified by CLAUSES. */
1475 scan_sharing_clauses (tree clauses
, omp_context
*ctx
)
1478 bool scan_array_reductions
= false;
1480 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1484 switch (OMP_CLAUSE_CODE (c
))
1486 case OMP_CLAUSE_PRIVATE
:
1487 decl
= OMP_CLAUSE_DECL (c
);
1488 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1490 else if (!is_variable_sized (decl
))
1491 install_var_local (decl
, ctx
);
1494 case OMP_CLAUSE_SHARED
:
1495 /* Ignore shared directives in teams construct. */
1496 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1498 gcc_assert (is_taskreg_ctx (ctx
));
1499 decl
= OMP_CLAUSE_DECL (c
);
1500 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1501 || !is_variable_sized (decl
));
1502 /* Global variables don't need to be copied,
1503 the receiver side will use them directly. */
1504 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1506 by_ref
= use_pointer_for_field (decl
, ctx
);
1507 if (! TREE_READONLY (decl
)
1508 || TREE_ADDRESSABLE (decl
)
1510 || is_reference (decl
))
1512 install_var_field (decl
, by_ref
, 3, ctx
);
1513 install_var_local (decl
, ctx
);
1516 /* We don't need to copy const scalar vars back. */
1517 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1520 case OMP_CLAUSE_LASTPRIVATE
:
1521 /* Let the corresponding firstprivate clause create
1523 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1527 case OMP_CLAUSE_FIRSTPRIVATE
:
1528 case OMP_CLAUSE_REDUCTION
:
1529 case OMP_CLAUSE_LINEAR
:
1530 decl
= OMP_CLAUSE_DECL (c
);
1532 if (is_variable_sized (decl
))
1534 if (is_task_ctx (ctx
))
1535 install_var_field (decl
, false, 1, ctx
);
1538 else if (is_taskreg_ctx (ctx
))
1541 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1542 by_ref
= use_pointer_for_field (decl
, NULL
);
1544 if (is_task_ctx (ctx
)
1545 && (global
|| by_ref
|| is_reference (decl
)))
1547 install_var_field (decl
, false, 1, ctx
);
1549 install_var_field (decl
, by_ref
, 2, ctx
);
1552 install_var_field (decl
, by_ref
, 3, ctx
);
1554 install_var_local (decl
, ctx
);
1557 case OMP_CLAUSE__LOOPTEMP_
:
1558 gcc_assert (is_parallel_ctx (ctx
));
1559 decl
= OMP_CLAUSE_DECL (c
);
1560 install_var_field (decl
, false, 3, ctx
);
1561 install_var_local (decl
, ctx
);
1564 case OMP_CLAUSE_COPYPRIVATE
:
1565 case OMP_CLAUSE_COPYIN
:
1566 decl
= OMP_CLAUSE_DECL (c
);
1567 by_ref
= use_pointer_for_field (decl
, NULL
);
1568 install_var_field (decl
, by_ref
, 3, ctx
);
1571 case OMP_CLAUSE_DEFAULT
:
1572 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
1575 case OMP_CLAUSE_FINAL
:
1577 case OMP_CLAUSE_NUM_THREADS
:
1578 case OMP_CLAUSE_NUM_TEAMS
:
1579 case OMP_CLAUSE_THREAD_LIMIT
:
1580 case OMP_CLAUSE_DEVICE
:
1581 case OMP_CLAUSE_SCHEDULE
:
1582 case OMP_CLAUSE_DIST_SCHEDULE
:
1583 case OMP_CLAUSE_DEPEND
:
1585 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
1589 case OMP_CLAUSE_FROM
:
1590 case OMP_CLAUSE_MAP
:
1592 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
1593 decl
= OMP_CLAUSE_DECL (c
);
1594 /* Global variables with "omp declare target" attribute
1595 don't need to be copied, the receiver side will use them
1597 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1599 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1600 && lookup_attribute ("omp declare target",
1601 DECL_ATTRIBUTES (decl
)))
1603 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1604 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
)
1606 /* Ignore OMP_CLAUSE_MAP_POINTER kind for arrays in
1607 #pragma omp target data, there is nothing to map for
1609 if (gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_DATA
1610 && !POINTER_TYPE_P (TREE_TYPE (decl
)))
1615 if (DECL_SIZE (decl
)
1616 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1618 tree decl2
= DECL_VALUE_EXPR (decl
);
1619 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1620 decl2
= TREE_OPERAND (decl2
, 0);
1621 gcc_assert (DECL_P (decl2
));
1622 install_var_field (decl2
, true, 3, ctx
);
1623 install_var_local (decl2
, ctx
);
1624 install_var_local (decl
, ctx
);
1628 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1629 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1630 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
1631 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1632 install_var_field (decl
, true, 7, ctx
);
1634 install_var_field (decl
, true, 3, ctx
);
1635 if (gimple_omp_target_kind (ctx
->stmt
)
1636 == GF_OMP_TARGET_KIND_REGION
)
1637 install_var_local (decl
, ctx
);
1642 tree base
= get_base_address (decl
);
1643 tree nc
= OMP_CLAUSE_CHAIN (c
);
1646 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
1647 && OMP_CLAUSE_DECL (nc
) == base
1648 && OMP_CLAUSE_MAP_KIND (nc
) == OMP_CLAUSE_MAP_POINTER
1649 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
1651 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
1652 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
1656 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
1657 (splay_tree_key
) decl
));
1659 = build_decl (OMP_CLAUSE_LOCATION (c
),
1660 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
1661 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
1662 insert_field_into_struct (ctx
->record_type
, field
);
1663 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
1664 (splay_tree_value
) field
);
1669 case OMP_CLAUSE_NOWAIT
:
1670 case OMP_CLAUSE_ORDERED
:
1671 case OMP_CLAUSE_COLLAPSE
:
1672 case OMP_CLAUSE_UNTIED
:
1673 case OMP_CLAUSE_MERGEABLE
:
1674 case OMP_CLAUSE_PROC_BIND
:
1675 case OMP_CLAUSE_SAFELEN
:
1678 case OMP_CLAUSE_ALIGNED
:
1679 decl
= OMP_CLAUSE_DECL (c
);
1680 if (is_global_var (decl
)
1681 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1682 install_var_local (decl
, ctx
);
1690 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1692 switch (OMP_CLAUSE_CODE (c
))
1694 case OMP_CLAUSE_LASTPRIVATE
:
1695 /* Let the corresponding firstprivate clause create
1697 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1698 scan_array_reductions
= true;
1699 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1703 case OMP_CLAUSE_PRIVATE
:
1704 case OMP_CLAUSE_FIRSTPRIVATE
:
1705 case OMP_CLAUSE_REDUCTION
:
1706 case OMP_CLAUSE_LINEAR
:
1707 decl
= OMP_CLAUSE_DECL (c
);
1708 if (is_variable_sized (decl
))
1709 install_var_local (decl
, ctx
);
1710 fixup_remapped_decl (decl
, ctx
,
1711 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
1712 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
1713 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1714 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1715 scan_array_reductions
= true;
1718 case OMP_CLAUSE_SHARED
:
1719 /* Ignore shared directives in teams construct. */
1720 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1722 decl
= OMP_CLAUSE_DECL (c
);
1723 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1724 fixup_remapped_decl (decl
, ctx
, false);
1727 case OMP_CLAUSE_MAP
:
1728 if (gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_DATA
)
1730 decl
= OMP_CLAUSE_DECL (c
);
1732 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1733 && lookup_attribute ("omp declare target",
1734 DECL_ATTRIBUTES (decl
)))
1738 if (OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1739 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
1740 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
1742 tree new_decl
= lookup_decl (decl
, ctx
);
1743 TREE_TYPE (new_decl
)
1744 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1746 else if (DECL_SIZE (decl
)
1747 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1749 tree decl2
= DECL_VALUE_EXPR (decl
);
1750 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1751 decl2
= TREE_OPERAND (decl2
, 0);
1752 gcc_assert (DECL_P (decl2
));
1753 fixup_remapped_decl (decl2
, ctx
, false);
1754 fixup_remapped_decl (decl
, ctx
, true);
1757 fixup_remapped_decl (decl
, ctx
, false);
1761 case OMP_CLAUSE_COPYPRIVATE
:
1762 case OMP_CLAUSE_COPYIN
:
1763 case OMP_CLAUSE_DEFAULT
:
1765 case OMP_CLAUSE_NUM_THREADS
:
1766 case OMP_CLAUSE_NUM_TEAMS
:
1767 case OMP_CLAUSE_THREAD_LIMIT
:
1768 case OMP_CLAUSE_DEVICE
:
1769 case OMP_CLAUSE_SCHEDULE
:
1770 case OMP_CLAUSE_DIST_SCHEDULE
:
1771 case OMP_CLAUSE_NOWAIT
:
1772 case OMP_CLAUSE_ORDERED
:
1773 case OMP_CLAUSE_COLLAPSE
:
1774 case OMP_CLAUSE_UNTIED
:
1775 case OMP_CLAUSE_FINAL
:
1776 case OMP_CLAUSE_MERGEABLE
:
1777 case OMP_CLAUSE_PROC_BIND
:
1778 case OMP_CLAUSE_SAFELEN
:
1779 case OMP_CLAUSE_ALIGNED
:
1780 case OMP_CLAUSE_DEPEND
:
1781 case OMP_CLAUSE__LOOPTEMP_
:
1783 case OMP_CLAUSE_FROM
:
1791 if (scan_array_reductions
)
1792 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1793 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1794 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1796 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
1797 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
1799 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
1800 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1801 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
1804 /* Create a new name for omp child function. Returns an identifier. */
1806 static GTY(()) unsigned int tmp_ompfn_id_num
;
1809 create_omp_child_function_name (bool task_copy
)
1811 return (clone_function_name (current_function_decl
,
1812 task_copy
? "_omp_cpyfn" : "_omp_fn"));
1815 /* Build a decl for the omp child function. It'll not contain a body
1816 yet, just the bare decl. */
1819 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
1821 tree decl
, type
, name
, t
;
1823 name
= create_omp_child_function_name (task_copy
);
1825 type
= build_function_type_list (void_type_node
, ptr_type_node
,
1826 ptr_type_node
, NULL_TREE
);
1828 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
1830 decl
= build_decl (gimple_location (ctx
->stmt
),
1831 FUNCTION_DECL
, name
, type
);
1834 ctx
->cb
.dst_fn
= decl
;
1836 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
1838 TREE_STATIC (decl
) = 1;
1839 TREE_USED (decl
) = 1;
1840 DECL_ARTIFICIAL (decl
) = 1;
1841 DECL_NAMELESS (decl
) = 1;
1842 DECL_IGNORED_P (decl
) = 0;
1843 TREE_PUBLIC (decl
) = 0;
1844 DECL_UNINLINABLE (decl
) = 1;
1845 DECL_EXTERNAL (decl
) = 0;
1846 DECL_CONTEXT (decl
) = NULL_TREE
;
1847 DECL_INITIAL (decl
) = make_node (BLOCK
);
1848 bool target_p
= false;
1849 if (lookup_attribute ("omp declare target",
1850 DECL_ATTRIBUTES (current_function_decl
)))
1855 for (octx
= ctx
; octx
; octx
= octx
->outer
)
1856 if (gimple_code (octx
->stmt
) == GIMPLE_OMP_TARGET
1857 && gimple_omp_target_kind (octx
->stmt
)
1858 == GF_OMP_TARGET_KIND_REGION
)
1865 DECL_ATTRIBUTES (decl
)
1866 = tree_cons (get_identifier ("omp declare target"),
1867 NULL_TREE
, DECL_ATTRIBUTES (decl
));
1869 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1870 RESULT_DECL
, NULL_TREE
, void_type_node
);
1871 DECL_ARTIFICIAL (t
) = 1;
1872 DECL_IGNORED_P (t
) = 1;
1873 DECL_CONTEXT (t
) = decl
;
1874 DECL_RESULT (decl
) = t
;
1876 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1877 PARM_DECL
, get_identifier (".omp_data_i"), ptr_type_node
);
1878 DECL_ARTIFICIAL (t
) = 1;
1879 DECL_NAMELESS (t
) = 1;
1880 DECL_ARG_TYPE (t
) = ptr_type_node
;
1881 DECL_CONTEXT (t
) = current_function_decl
;
1883 DECL_ARGUMENTS (decl
) = t
;
1885 ctx
->receiver_decl
= t
;
1888 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1889 PARM_DECL
, get_identifier (".omp_data_o"),
1891 DECL_ARTIFICIAL (t
) = 1;
1892 DECL_NAMELESS (t
) = 1;
1893 DECL_ARG_TYPE (t
) = ptr_type_node
;
1894 DECL_CONTEXT (t
) = current_function_decl
;
1896 TREE_ADDRESSABLE (t
) = 1;
1897 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
1898 DECL_ARGUMENTS (decl
) = t
;
1901 /* Allocate memory for the function structure. The call to
1902 allocate_struct_function clobbers CFUN, so we need to restore
1904 push_struct_function (decl
);
1905 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
1909 /* Callback for walk_gimple_seq. Check if combined parallel
1910 contains gimple_omp_for_combined_into_p OMP_FOR. */
1913 find_combined_for (gimple_stmt_iterator
*gsi_p
,
1914 bool *handled_ops_p
,
1915 struct walk_stmt_info
*wi
)
1917 gimple stmt
= gsi_stmt (*gsi_p
);
1919 *handled_ops_p
= true;
1920 switch (gimple_code (stmt
))
1924 case GIMPLE_OMP_FOR
:
1925 if (gimple_omp_for_combined_into_p (stmt
)
1926 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
1929 return integer_zero_node
;
1938 /* Scan an OpenMP parallel directive. */
1941 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
1945 gimple stmt
= gsi_stmt (*gsi
);
1947 /* Ignore parallel directives with empty bodies, unless there
1948 are copyin clauses. */
1950 && empty_body_p (gimple_omp_body (stmt
))
1951 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
1952 OMP_CLAUSE_COPYIN
) == NULL
)
1954 gsi_replace (gsi
, gimple_build_nop (), false);
1958 if (gimple_omp_parallel_combined_p (stmt
))
1961 struct walk_stmt_info wi
;
1963 memset (&wi
, 0, sizeof (wi
));
1965 walk_gimple_seq (gimple_omp_body (stmt
),
1966 find_combined_for
, NULL
, &wi
);
1967 for_stmt
= (gimple
) wi
.info
;
1970 struct omp_for_data fd
;
1971 extract_omp_for_data (for_stmt
, &fd
, NULL
);
1972 /* We need two temporaries with fd.loop.v type (istart/iend)
1973 and then (fd.collapse - 1) temporaries with the same
1974 type for count2 ... countN-1 vars if not constant. */
1975 size_t count
= 2, i
;
1976 tree type
= fd
.iter_type
;
1978 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
1979 count
+= fd
.collapse
- 1;
1980 for (i
= 0; i
< count
; i
++)
1982 tree temp
= create_tmp_var (type
, NULL
);
1983 tree c
= build_omp_clause (UNKNOWN_LOCATION
,
1984 OMP_CLAUSE__LOOPTEMP_
);
1985 OMP_CLAUSE_DECL (c
) = temp
;
1986 OMP_CLAUSE_CHAIN (c
) = gimple_omp_parallel_clauses (stmt
);
1987 gimple_omp_parallel_set_clauses (stmt
, c
);
1992 ctx
= new_omp_context (stmt
, outer_ctx
);
1993 if (taskreg_nesting_level
> 1)
1994 ctx
->is_nested
= true;
1995 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1996 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
1997 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1998 name
= create_tmp_var_name (".omp_data_s");
1999 name
= build_decl (gimple_location (stmt
),
2000 TYPE_DECL
, name
, ctx
->record_type
);
2001 DECL_ARTIFICIAL (name
) = 1;
2002 DECL_NAMELESS (name
) = 1;
2003 TYPE_NAME (ctx
->record_type
) = name
;
2004 create_omp_child_function (ctx
, false);
2005 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2007 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2008 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2010 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2011 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2014 layout_type (ctx
->record_type
);
2015 fixup_child_record_type (ctx
);
2019 /* Scan an OpenMP task directive. */
2022 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2026 gimple stmt
= gsi_stmt (*gsi
);
2027 location_t loc
= gimple_location (stmt
);
2029 /* Ignore task directives with empty bodies. */
2031 && empty_body_p (gimple_omp_body (stmt
)))
2033 gsi_replace (gsi
, gimple_build_nop (), false);
2037 ctx
= new_omp_context (stmt
, outer_ctx
);
2038 if (taskreg_nesting_level
> 1)
2039 ctx
->is_nested
= true;
2040 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2041 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2042 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2043 name
= create_tmp_var_name (".omp_data_s");
2044 name
= build_decl (gimple_location (stmt
),
2045 TYPE_DECL
, name
, ctx
->record_type
);
2046 DECL_ARTIFICIAL (name
) = 1;
2047 DECL_NAMELESS (name
) = 1;
2048 TYPE_NAME (ctx
->record_type
) = name
;
2049 create_omp_child_function (ctx
, false);
2050 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2052 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2054 if (ctx
->srecord_type
)
2056 name
= create_tmp_var_name (".omp_data_a");
2057 name
= build_decl (gimple_location (stmt
),
2058 TYPE_DECL
, name
, ctx
->srecord_type
);
2059 DECL_ARTIFICIAL (name
) = 1;
2060 DECL_NAMELESS (name
) = 1;
2061 TYPE_NAME (ctx
->srecord_type
) = name
;
2062 create_omp_child_function (ctx
, true);
2065 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2067 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2069 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2070 t
= build_int_cst (long_integer_type_node
, 0);
2071 gimple_omp_task_set_arg_size (stmt
, t
);
2072 t
= build_int_cst (long_integer_type_node
, 1);
2073 gimple_omp_task_set_arg_align (stmt
, t
);
2077 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2078 /* Move VLA fields to the end. */
2079 p
= &TYPE_FIELDS (ctx
->record_type
);
2081 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2082 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2085 *p
= TREE_CHAIN (*p
);
2086 TREE_CHAIN (*q
) = NULL_TREE
;
2087 q
= &TREE_CHAIN (*q
);
2090 p
= &DECL_CHAIN (*p
);
2092 layout_type (ctx
->record_type
);
2093 fixup_child_record_type (ctx
);
2094 if (ctx
->srecord_type
)
2095 layout_type (ctx
->srecord_type
);
2096 t
= fold_convert_loc (loc
, long_integer_type_node
,
2097 TYPE_SIZE_UNIT (ctx
->record_type
));
2098 gimple_omp_task_set_arg_size (stmt
, t
);
2099 t
= build_int_cst (long_integer_type_node
,
2100 TYPE_ALIGN_UNIT (ctx
->record_type
));
2101 gimple_omp_task_set_arg_align (stmt
, t
);
2106 /* Scan an OpenMP loop directive. */
2109 scan_omp_for (gimple stmt
, omp_context
*outer_ctx
)
2114 ctx
= new_omp_context (stmt
, outer_ctx
);
2116 scan_sharing_clauses (gimple_omp_for_clauses (stmt
), ctx
);
2118 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
2119 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
2121 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
2122 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
2123 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
2124 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
2126 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2129 /* Scan an OpenMP sections directive. */
2132 scan_omp_sections (gimple stmt
, omp_context
*outer_ctx
)
2136 ctx
= new_omp_context (stmt
, outer_ctx
);
2137 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
2138 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2141 /* Scan an OpenMP single directive. */
2144 scan_omp_single (gimple stmt
, omp_context
*outer_ctx
)
2149 ctx
= new_omp_context (stmt
, outer_ctx
);
2150 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2151 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2152 name
= create_tmp_var_name (".omp_copy_s");
2153 name
= build_decl (gimple_location (stmt
),
2154 TYPE_DECL
, name
, ctx
->record_type
);
2155 TYPE_NAME (ctx
->record_type
) = name
;
2157 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
2158 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2160 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2161 ctx
->record_type
= NULL
;
2163 layout_type (ctx
->record_type
);
2166 /* Scan an OpenMP target{, data, update} directive. */
2169 scan_omp_target (gimple stmt
, omp_context
*outer_ctx
)
2173 int kind
= gimple_omp_target_kind (stmt
);
2175 ctx
= new_omp_context (stmt
, outer_ctx
);
2176 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2177 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2178 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2179 name
= create_tmp_var_name (".omp_data_t");
2180 name
= build_decl (gimple_location (stmt
),
2181 TYPE_DECL
, name
, ctx
->record_type
);
2182 DECL_ARTIFICIAL (name
) = 1;
2183 DECL_NAMELESS (name
) = 1;
2184 TYPE_NAME (ctx
->record_type
) = name
;
2185 if (kind
== GF_OMP_TARGET_KIND_REGION
)
2187 create_omp_child_function (ctx
, false);
2188 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2191 scan_sharing_clauses (gimple_omp_target_clauses (stmt
), ctx
);
2192 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2194 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2195 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2198 TYPE_FIELDS (ctx
->record_type
)
2199 = nreverse (TYPE_FIELDS (ctx
->record_type
));
2200 #ifdef ENABLE_CHECKING
2202 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
2203 for (field
= TYPE_FIELDS (ctx
->record_type
);
2205 field
= DECL_CHAIN (field
))
2206 gcc_assert (DECL_ALIGN (field
) == align
);
2208 layout_type (ctx
->record_type
);
2209 if (kind
== GF_OMP_TARGET_KIND_REGION
)
2210 fixup_child_record_type (ctx
);
2214 /* Scan an OpenMP teams directive. */
2217 scan_omp_teams (gimple stmt
, omp_context
*outer_ctx
)
2219 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
2220 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
2221 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2224 /* Check OpenMP nesting restrictions. */
2226 check_omp_nesting_restrictions (gimple stmt
, omp_context
*ctx
)
2230 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2231 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_SIMD
)
2233 error_at (gimple_location (stmt
),
2234 "OpenMP constructs may not be nested inside simd region");
2237 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2239 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
2240 || (gimple_omp_for_kind (stmt
)
2241 != GF_OMP_FOR_KIND_DISTRIBUTE
))
2242 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
2244 error_at (gimple_location (stmt
),
2245 "only distribute or parallel constructs are allowed to "
2246 "be closely nested inside teams construct");
2251 switch (gimple_code (stmt
))
2253 case GIMPLE_OMP_FOR
:
2254 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_SIMD
)
2256 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
2258 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
2260 error_at (gimple_location (stmt
),
2261 "distribute construct must be closely nested inside "
2269 if (is_gimple_call (stmt
)
2270 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2271 == BUILT_IN_GOMP_CANCEL
2272 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2273 == BUILT_IN_GOMP_CANCELLATION_POINT
))
2275 const char *bad
= NULL
;
2276 const char *kind
= NULL
;
2279 error_at (gimple_location (stmt
), "orphaned %qs construct",
2280 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2281 == BUILT_IN_GOMP_CANCEL
2282 ? "#pragma omp cancel"
2283 : "#pragma omp cancellation point");
2286 switch (host_integerp (gimple_call_arg (stmt
, 0), 0)
2287 ? tree_low_cst (gimple_call_arg (stmt
, 0), 0)
2291 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
2292 bad
= "#pragma omp parallel";
2293 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2294 == BUILT_IN_GOMP_CANCEL
2295 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2296 ctx
->cancellable
= true;
2300 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
2301 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
2302 bad
= "#pragma omp for";
2303 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2304 == BUILT_IN_GOMP_CANCEL
2305 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2307 ctx
->cancellable
= true;
2308 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2310 warning_at (gimple_location (stmt
), 0,
2311 "%<#pragma omp cancel for%> inside "
2312 "%<nowait%> for construct");
2313 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2314 OMP_CLAUSE_ORDERED
))
2315 warning_at (gimple_location (stmt
), 0,
2316 "%<#pragma omp cancel for%> inside "
2317 "%<ordered%> for construct");
2322 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
2323 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
2324 bad
= "#pragma omp sections";
2325 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2326 == BUILT_IN_GOMP_CANCEL
2327 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2329 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
2331 ctx
->cancellable
= true;
2332 if (find_omp_clause (gimple_omp_sections_clauses
2335 warning_at (gimple_location (stmt
), 0,
2336 "%<#pragma omp cancel sections%> inside "
2337 "%<nowait%> sections construct");
2341 gcc_assert (ctx
->outer
2342 && gimple_code (ctx
->outer
->stmt
)
2343 == GIMPLE_OMP_SECTIONS
);
2344 ctx
->outer
->cancellable
= true;
2345 if (find_omp_clause (gimple_omp_sections_clauses
2348 warning_at (gimple_location (stmt
), 0,
2349 "%<#pragma omp cancel sections%> inside "
2350 "%<nowait%> sections construct");
2356 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
2357 bad
= "#pragma omp task";
2359 ctx
->cancellable
= true;
2363 error_at (gimple_location (stmt
), "invalid arguments");
2368 error_at (gimple_location (stmt
),
2369 "%<%s %s%> construct not closely nested inside of %qs",
2370 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2371 == BUILT_IN_GOMP_CANCEL
2372 ? "#pragma omp cancel"
2373 : "#pragma omp cancellation point", kind
, bad
);
2378 case GIMPLE_OMP_SECTIONS
:
2379 case GIMPLE_OMP_SINGLE
:
2380 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2381 switch (gimple_code (ctx
->stmt
))
2383 case GIMPLE_OMP_FOR
:
2384 case GIMPLE_OMP_SECTIONS
:
2385 case GIMPLE_OMP_SINGLE
:
2386 case GIMPLE_OMP_ORDERED
:
2387 case GIMPLE_OMP_MASTER
:
2388 case GIMPLE_OMP_TASK
:
2389 case GIMPLE_OMP_CRITICAL
:
2390 if (is_gimple_call (stmt
))
2392 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2393 != BUILT_IN_GOMP_BARRIER
)
2395 error_at (gimple_location (stmt
),
2396 "barrier region may not be closely nested inside "
2397 "of work-sharing, critical, ordered, master or "
2398 "explicit task region");
2401 error_at (gimple_location (stmt
),
2402 "work-sharing region may not be closely nested inside "
2403 "of work-sharing, critical, ordered, master or explicit "
2406 case GIMPLE_OMP_PARALLEL
:
2412 case GIMPLE_OMP_MASTER
:
2413 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2414 switch (gimple_code (ctx
->stmt
))
2416 case GIMPLE_OMP_FOR
:
2417 case GIMPLE_OMP_SECTIONS
:
2418 case GIMPLE_OMP_SINGLE
:
2419 case GIMPLE_OMP_TASK
:
2420 error_at (gimple_location (stmt
),
2421 "master region may not be closely nested inside "
2422 "of work-sharing or explicit task region");
2424 case GIMPLE_OMP_PARALLEL
:
2430 case GIMPLE_OMP_ORDERED
:
2431 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2432 switch (gimple_code (ctx
->stmt
))
2434 case GIMPLE_OMP_CRITICAL
:
2435 case GIMPLE_OMP_TASK
:
2436 error_at (gimple_location (stmt
),
2437 "ordered region may not be closely nested inside "
2438 "of critical or explicit task region");
2440 case GIMPLE_OMP_FOR
:
2441 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2442 OMP_CLAUSE_ORDERED
) == NULL
)
2444 error_at (gimple_location (stmt
),
2445 "ordered region must be closely nested inside "
2446 "a loop region with an ordered clause");
2450 case GIMPLE_OMP_PARALLEL
:
2451 error_at (gimple_location (stmt
),
2452 "ordered region must be closely nested inside "
2453 "a loop region with an ordered clause");
2459 case GIMPLE_OMP_CRITICAL
:
2460 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2461 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_CRITICAL
2462 && (gimple_omp_critical_name (stmt
)
2463 == gimple_omp_critical_name (ctx
->stmt
)))
2465 error_at (gimple_location (stmt
),
2466 "critical region may not be nested inside a critical "
2467 "region with the same name");
2471 case GIMPLE_OMP_TEAMS
:
2473 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
2474 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
2476 error_at (gimple_location (stmt
),
2477 "teams construct not closely nested inside of target "
2489 /* Helper function scan_omp.
2491 Callback for walk_tree or operators in walk_gimple_stmt used to
2492 scan for OpenMP directives in TP. */
2495 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
2497 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
2498 omp_context
*ctx
= (omp_context
*) wi
->info
;
2501 switch (TREE_CODE (t
))
2508 *tp
= remap_decl (t
, &ctx
->cb
);
2512 if (ctx
&& TYPE_P (t
))
2513 *tp
= remap_type (t
, &ctx
->cb
);
2514 else if (!DECL_P (t
))
2519 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
2520 if (tem
!= TREE_TYPE (t
))
2522 if (TREE_CODE (t
) == INTEGER_CST
)
2523 *tp
= build_int_cst_wide (tem
,
2524 TREE_INT_CST_LOW (t
),
2525 TREE_INT_CST_HIGH (t
));
2527 TREE_TYPE (t
) = tem
;
2538 /* Helper function for scan_omp.
2540 Callback for walk_gimple_stmt used to scan for OpenMP directives in
2541 the current statement in GSI. */
2544 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
2545 struct walk_stmt_info
*wi
)
2547 gimple stmt
= gsi_stmt (*gsi
);
2548 omp_context
*ctx
= (omp_context
*) wi
->info
;
2550 if (gimple_has_location (stmt
))
2551 input_location
= gimple_location (stmt
);
2553 /* Check the OpenMP nesting restrictions. */
2554 bool remove
= false;
2555 if (is_gimple_omp (stmt
))
2556 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
2557 else if (is_gimple_call (stmt
))
2559 tree fndecl
= gimple_call_fndecl (stmt
);
2561 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
2562 switch (DECL_FUNCTION_CODE (fndecl
))
2564 case BUILT_IN_GOMP_BARRIER
:
2565 case BUILT_IN_GOMP_CANCEL
:
2566 case BUILT_IN_GOMP_CANCELLATION_POINT
:
2567 case BUILT_IN_GOMP_TASKYIELD
:
2568 case BUILT_IN_GOMP_TASKWAIT
:
2569 case BUILT_IN_GOMP_TASKGROUP_START
:
2570 case BUILT_IN_GOMP_TASKGROUP_END
:
2571 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
2579 stmt
= gimple_build_nop ();
2580 gsi_replace (gsi
, stmt
, false);
2583 *handled_ops_p
= true;
2585 switch (gimple_code (stmt
))
2587 case GIMPLE_OMP_PARALLEL
:
2588 taskreg_nesting_level
++;
2589 scan_omp_parallel (gsi
, ctx
);
2590 taskreg_nesting_level
--;
2593 case GIMPLE_OMP_TASK
:
2594 taskreg_nesting_level
++;
2595 scan_omp_task (gsi
, ctx
);
2596 taskreg_nesting_level
--;
2599 case GIMPLE_OMP_FOR
:
2600 scan_omp_for (stmt
, ctx
);
2603 case GIMPLE_OMP_SECTIONS
:
2604 scan_omp_sections (stmt
, ctx
);
2607 case GIMPLE_OMP_SINGLE
:
2608 scan_omp_single (stmt
, ctx
);
2611 case GIMPLE_OMP_SECTION
:
2612 case GIMPLE_OMP_MASTER
:
2613 case GIMPLE_OMP_TASKGROUP
:
2614 case GIMPLE_OMP_ORDERED
:
2615 case GIMPLE_OMP_CRITICAL
:
2616 ctx
= new_omp_context (stmt
, ctx
);
2617 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2620 case GIMPLE_OMP_TARGET
:
2621 scan_omp_target (stmt
, ctx
);
2624 case GIMPLE_OMP_TEAMS
:
2625 scan_omp_teams (stmt
, ctx
);
2632 *handled_ops_p
= false;
2634 for (var
= gimple_bind_vars (stmt
); var
; var
= DECL_CHAIN (var
))
2635 insert_decl_map (&ctx
->cb
, var
, var
);
2639 *handled_ops_p
= false;
2647 /* Scan all the statements starting at the current statement. CTX
2648 contains context information about the OpenMP directives and
2649 clauses found during the scan. */
2652 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
2654 location_t saved_location
;
2655 struct walk_stmt_info wi
;
2657 memset (&wi
, 0, sizeof (wi
));
2659 wi
.want_locations
= true;
2661 saved_location
= input_location
;
2662 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
2663 input_location
= saved_location
;
2666 /* Re-gimplification and code generation routines. */
2668 /* Build a call to GOMP_barrier. */
2671 build_omp_barrier (tree lhs
)
2673 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
2674 : BUILT_IN_GOMP_BARRIER
);
2675 gimple g
= gimple_build_call (fndecl
, 0);
2677 gimple_call_set_lhs (g
, lhs
);
2681 /* If a context was created for STMT when it was scanned, return it. */
2683 static omp_context
*
2684 maybe_lookup_ctx (gimple stmt
)
2687 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
2688 return n
? (omp_context
*) n
->value
: NULL
;
2692 /* Find the mapping for DECL in CTX or the immediately enclosing
2693 context that has a mapping for DECL.
2695 If CTX is a nested parallel directive, we may have to use the decl
2696 mappings created in CTX's parent context. Suppose that we have the
2697 following parallel nesting (variable UIDs showed for clarity):
2700 #omp parallel shared(iD.1562) -> outer parallel
2701 iD.1562 = iD.1562 + 1;
2703 #omp parallel shared (iD.1562) -> inner parallel
2704 iD.1562 = iD.1562 - 1;
2706 Each parallel structure will create a distinct .omp_data_s structure
2707 for copying iD.1562 in/out of the directive:
2709 outer parallel .omp_data_s.1.i -> iD.1562
2710 inner parallel .omp_data_s.2.i -> iD.1562
2712 A shared variable mapping will produce a copy-out operation before
2713 the parallel directive and a copy-in operation after it. So, in
2714 this case we would have:
2717 .omp_data_o.1.i = iD.1562;
2718 #omp parallel shared(iD.1562) -> outer parallel
2719 .omp_data_i.1 = &.omp_data_o.1
2720 .omp_data_i.1->i = .omp_data_i.1->i + 1;
2722 .omp_data_o.2.i = iD.1562; -> **
2723 #omp parallel shared(iD.1562) -> inner parallel
2724 .omp_data_i.2 = &.omp_data_o.2
2725 .omp_data_i.2->i = .omp_data_i.2->i - 1;
2728 ** This is a problem. The symbol iD.1562 cannot be referenced
2729 inside the body of the outer parallel region. But since we are
2730 emitting this copy operation while expanding the inner parallel
2731 directive, we need to access the CTX structure of the outer
2732 parallel directive to get the correct mapping:
2734 .omp_data_o.2.i = .omp_data_i.1->i
2736 Since there may be other workshare or parallel directives enclosing
2737 the parallel directive, it may be necessary to walk up the context
2738 parent chain. This is not a problem in general because nested
2739 parallelism happens only rarely. */
2742 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
2747 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
2748 t
= maybe_lookup_decl (decl
, up
);
2750 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
2752 return t
? t
: decl
;
2756 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
2757 in outer contexts. */
2760 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
2765 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
2766 t
= maybe_lookup_decl (decl
, up
);
2768 return t
? t
: decl
;
2772 /* Construct the initialization value for reduction CLAUSE. */
2775 omp_reduction_init (tree clause
, tree type
)
2777 location_t loc
= OMP_CLAUSE_LOCATION (clause
);
2778 switch (OMP_CLAUSE_REDUCTION_CODE (clause
))
2785 case TRUTH_ORIF_EXPR
:
2786 case TRUTH_XOR_EXPR
:
2788 return build_zero_cst (type
);
2791 case TRUTH_AND_EXPR
:
2792 case TRUTH_ANDIF_EXPR
:
2794 return fold_convert_loc (loc
, type
, integer_one_node
);
2797 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
2800 if (SCALAR_FLOAT_TYPE_P (type
))
2802 REAL_VALUE_TYPE max
, min
;
2803 if (HONOR_INFINITIES (TYPE_MODE (type
)))
2806 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
2809 real_maxval (&min
, 1, TYPE_MODE (type
));
2810 return build_real (type
, min
);
2814 gcc_assert (INTEGRAL_TYPE_P (type
));
2815 return TYPE_MIN_VALUE (type
);
2819 if (SCALAR_FLOAT_TYPE_P (type
))
2821 REAL_VALUE_TYPE max
;
2822 if (HONOR_INFINITIES (TYPE_MODE (type
)))
2825 real_maxval (&max
, 0, TYPE_MODE (type
));
2826 return build_real (type
, max
);
2830 gcc_assert (INTEGRAL_TYPE_P (type
));
2831 return TYPE_MAX_VALUE (type
);
2839 /* Return alignment to be assumed for var in CLAUSE, which should be
2840 OMP_CLAUSE_ALIGNED. */
2843 omp_clause_aligned_alignment (tree clause
)
2845 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
2846 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
2848 /* Otherwise return implementation defined alignment. */
2849 unsigned int al
= 1;
2850 enum machine_mode mode
, vmode
;
2851 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
2853 vs
= 1 << floor_log2 (vs
);
2854 static enum mode_class classes
[]
2855 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
2856 for (int i
= 0; i
< 4; i
+= 2)
2857 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
2859 mode
= GET_MODE_WIDER_MODE (mode
))
2861 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
2862 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
2865 && GET_MODE_SIZE (vmode
) < vs
2866 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
2867 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
2869 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
2870 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
2872 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
2873 / GET_MODE_SIZE (mode
));
2874 if (TYPE_MODE (type
) != vmode
)
2876 if (TYPE_ALIGN_UNIT (type
) > al
)
2877 al
= TYPE_ALIGN_UNIT (type
);
2879 return build_int_cst (integer_type_node
, al
);
2882 /* Return maximum possible vectorization factor for the target. */
2889 || (!flag_tree_loop_vectorize
2890 && (global_options_set
.x_flag_tree_loop_vectorize
2891 || global_options_set
.x_flag_tree_vectorize
)))
2894 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
2897 vs
= 1 << floor_log2 (vs
);
2900 enum machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
2901 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
2902 return GET_MODE_NUNITS (vqimode
);
2906 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
2910 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
2911 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
2915 max_vf
= omp_max_vf ();
2918 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2919 OMP_CLAUSE_SAFELEN
);
2921 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
), max_vf
) == -1)
2922 max_vf
= tree_low_cst (OMP_CLAUSE_SAFELEN_EXPR (c
), 0);
2926 idx
= create_tmp_var (unsigned_type_node
, NULL
);
2927 lane
= create_tmp_var (unsigned_type_node
, NULL
);
2933 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
2934 tree avar
= create_tmp_var_raw (atype
, NULL
);
2935 if (TREE_ADDRESSABLE (new_var
))
2936 TREE_ADDRESSABLE (avar
) = 1;
2937 DECL_ATTRIBUTES (avar
)
2938 = tree_cons (get_identifier ("omp simd array"), NULL
,
2939 DECL_ATTRIBUTES (avar
));
2940 gimple_add_tmp_var (avar
);
2941 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
2942 NULL_TREE
, NULL_TREE
);
2943 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
2944 NULL_TREE
, NULL_TREE
);
2945 if (DECL_P (new_var
))
2947 SET_DECL_VALUE_EXPR (new_var
, lvar
);
2948 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
2953 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
2954 from the receiver (aka child) side and initializers for REFERENCE_TYPE
2955 private variables. Initialization statements go in ILIST, while calls
2956 to destructors go in DLIST. */
2959 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
2960 omp_context
*ctx
, struct omp_for_data
*fd
)
2962 tree c
, dtor
, copyin_seq
, x
, ptr
;
2963 bool copyin_by_ref
= false;
2964 bool lastprivate_firstprivate
= false;
2965 bool reduction_omp_orig_ref
= false;
2967 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2968 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_SIMD
);
2970 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
2971 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
2972 gimple_seq llist
[2] = { NULL
, NULL
};
2976 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
2977 with data sharing clauses referencing variable sized vars. That
2978 is unnecessarily hard to support and very unlikely to result in
2979 vectorized code anyway. */
2981 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2982 switch (OMP_CLAUSE_CODE (c
))
2984 case OMP_CLAUSE_REDUCTION
:
2985 case OMP_CLAUSE_PRIVATE
:
2986 case OMP_CLAUSE_FIRSTPRIVATE
:
2987 case OMP_CLAUSE_LASTPRIVATE
:
2988 case OMP_CLAUSE_LINEAR
:
2989 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
2996 /* Do all the fixed sized types in the first pass, and the variable sized
2997 types in the second pass. This makes sure that the scalar arguments to
2998 the variable sized types are processed before we use them in the
2999 variable sized operations. */
3000 for (pass
= 0; pass
< 2; ++pass
)
3002 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3004 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
3007 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3011 case OMP_CLAUSE_PRIVATE
:
3012 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
3015 case OMP_CLAUSE_SHARED
:
3016 /* Ignore shared directives in teams construct. */
3017 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3019 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
3021 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c
)));
3024 case OMP_CLAUSE_FIRSTPRIVATE
:
3025 case OMP_CLAUSE_COPYIN
:
3026 case OMP_CLAUSE_LINEAR
:
3028 case OMP_CLAUSE_REDUCTION
:
3029 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
3030 reduction_omp_orig_ref
= true;
3032 case OMP_CLAUSE__LOOPTEMP_
:
3033 /* Handle _looptemp_ clauses only on parallel. */
3037 case OMP_CLAUSE_LASTPRIVATE
:
3038 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3040 lastprivate_firstprivate
= true;
3045 case OMP_CLAUSE_ALIGNED
:
3048 var
= OMP_CLAUSE_DECL (c
);
3049 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
3050 && !is_global_var (var
))
3052 new_var
= maybe_lookup_decl (var
, ctx
);
3053 if (new_var
== NULL_TREE
)
3054 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3055 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3056 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
3057 omp_clause_aligned_alignment (c
));
3058 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3059 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3060 gimplify_and_add (x
, ilist
);
3062 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
3063 && is_global_var (var
))
3065 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
3066 new_var
= lookup_decl (var
, ctx
);
3067 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3068 t
= build_fold_addr_expr_loc (clause_loc
, t
);
3069 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3070 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
3071 omp_clause_aligned_alignment (c
));
3072 t
= fold_convert_loc (clause_loc
, ptype
, t
);
3073 x
= create_tmp_var (ptype
, NULL
);
3074 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
3075 gimplify_and_add (t
, ilist
);
3076 t
= build_simple_mem_ref_loc (clause_loc
, x
);
3077 SET_DECL_VALUE_EXPR (new_var
, t
);
3078 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3085 new_var
= var
= OMP_CLAUSE_DECL (c
);
3086 if (c_kind
!= OMP_CLAUSE_COPYIN
)
3087 new_var
= lookup_decl (var
, ctx
);
3089 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
3094 else if (is_variable_sized (var
))
3096 /* For variable sized types, we need to allocate the
3097 actual storage here. Call alloca and store the
3098 result in the pointer decl that we created elsewhere. */
3102 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
3107 ptr
= DECL_VALUE_EXPR (new_var
);
3108 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
3109 ptr
= TREE_OPERAND (ptr
, 0);
3110 gcc_assert (DECL_P (ptr
));
3111 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
3113 /* void *tmp = __builtin_alloca */
3114 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3115 stmt
= gimple_build_call (atmp
, 1, x
);
3116 tmp
= create_tmp_var_raw (ptr_type_node
, NULL
);
3117 gimple_add_tmp_var (tmp
);
3118 gimple_call_set_lhs (stmt
, tmp
);
3120 gimple_seq_add_stmt (ilist
, stmt
);
3122 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
3123 gimplify_assign (ptr
, x
, ilist
);
3126 else if (is_reference (var
))
3128 /* For references that are being privatized for Fortran,
3129 allocate new backing storage for the new pointer
3130 variable. This allows us to avoid changing all the
3131 code that expects a pointer to something that expects
3132 a direct variable. */
3136 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
3137 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
3139 x
= build_receiver_ref (var
, false, ctx
);
3140 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3142 else if (TREE_CONSTANT (x
))
3144 const char *name
= NULL
;
3145 if (DECL_NAME (var
))
3146 name
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
3148 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
3150 gimple_add_tmp_var (x
);
3151 TREE_ADDRESSABLE (x
) = 1;
3152 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3156 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3157 x
= build_call_expr_loc (clause_loc
, atmp
, 1, x
);
3160 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3161 gimplify_assign (new_var
, x
, ilist
);
3163 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3165 else if (c_kind
== OMP_CLAUSE_REDUCTION
3166 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3174 switch (OMP_CLAUSE_CODE (c
))
3176 case OMP_CLAUSE_SHARED
:
3177 /* Ignore shared directives in teams construct. */
3178 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3180 /* Shared global vars are just accessed directly. */
3181 if (is_global_var (new_var
))
3183 /* Set up the DECL_VALUE_EXPR for shared variables now. This
3184 needs to be delayed until after fixup_child_record_type so
3185 that we get the correct type during the dereference. */
3186 by_ref
= use_pointer_for_field (var
, ctx
);
3187 x
= build_receiver_ref (var
, by_ref
, ctx
);
3188 SET_DECL_VALUE_EXPR (new_var
, x
);
3189 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3191 /* ??? If VAR is not passed by reference, and the variable
3192 hasn't been initialized yet, then we'll get a warning for
3193 the store into the omp_data_s structure. Ideally, we'd be
3194 able to notice this and not store anything at all, but
3195 we're generating code too early. Suppress the warning. */
3197 TREE_NO_WARNING (var
) = 1;
3200 case OMP_CLAUSE_LASTPRIVATE
:
3201 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3205 case OMP_CLAUSE_PRIVATE
:
3206 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
3207 x
= build_outer_var_ref (var
, ctx
);
3208 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
3210 if (is_task_ctx (ctx
))
3211 x
= build_receiver_ref (var
, false, ctx
);
3213 x
= build_outer_var_ref (var
, ctx
);
3219 nx
= lang_hooks
.decls
.omp_clause_default_ctor (c
, new_var
, x
);
3222 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3223 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
3224 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
3225 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3226 idx
, lane
, ivar
, lvar
))
3229 x
= lang_hooks
.decls
.omp_clause_default_ctor
3230 (c
, unshare_expr (ivar
), x
);
3232 gimplify_and_add (x
, &llist
[0]);
3235 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3238 gimple_seq tseq
= NULL
;
3241 gimplify_stmt (&dtor
, &tseq
);
3242 gimple_seq_add_seq (&llist
[1], tseq
);
3249 gimplify_and_add (nx
, ilist
);
3253 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3256 gimple_seq tseq
= NULL
;
3259 gimplify_stmt (&dtor
, &tseq
);
3260 gimple_seq_add_seq (dlist
, tseq
);
3264 case OMP_CLAUSE_LINEAR
:
3265 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
3266 goto do_firstprivate
;
3267 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
3270 x
= build_outer_var_ref (var
, ctx
);
3273 case OMP_CLAUSE_FIRSTPRIVATE
:
3274 if (is_task_ctx (ctx
))
3276 if (is_reference (var
) || is_variable_sized (var
))
3278 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
3280 || use_pointer_for_field (var
, NULL
))
3282 x
= build_receiver_ref (var
, false, ctx
);
3283 SET_DECL_VALUE_EXPR (new_var
, x
);
3284 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3289 x
= build_outer_var_ref (var
, ctx
);
3292 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3293 && gimple_omp_for_combined_into_p (ctx
->stmt
))
3295 tree stept
= POINTER_TYPE_P (TREE_TYPE (x
))
3296 ? sizetype
: TREE_TYPE (x
);
3297 tree t
= fold_convert (stept
,
3298 OMP_CLAUSE_LINEAR_STEP (c
));
3299 tree c
= find_omp_clause (clauses
,
3300 OMP_CLAUSE__LOOPTEMP_
);
3302 tree l
= OMP_CLAUSE_DECL (c
);
3303 if (fd
->collapse
== 1)
3305 tree n1
= fd
->loop
.n1
;
3306 tree step
= fd
->loop
.step
;
3307 tree itype
= TREE_TYPE (l
);
3308 if (POINTER_TYPE_P (itype
))
3309 itype
= signed_type_for (itype
);
3310 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
3311 if (TYPE_UNSIGNED (itype
)
3312 && fd
->loop
.cond_code
== GT_EXPR
)
3313 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
3314 fold_build1 (NEGATE_EXPR
,
3316 fold_build1 (NEGATE_EXPR
,
3319 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
3321 t
= fold_build2 (MULT_EXPR
, stept
,
3322 fold_convert (stept
, l
), t
);
3323 if (POINTER_TYPE_P (TREE_TYPE (x
)))
3324 x
= fold_build2 (POINTER_PLUS_EXPR
,
3325 TREE_TYPE (x
), x
, t
);
3327 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
3330 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
3331 || TREE_ADDRESSABLE (new_var
))
3332 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3333 idx
, lane
, ivar
, lvar
))
3335 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
3337 tree iv
= create_tmp_var (TREE_TYPE (new_var
), NULL
);
3338 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
3339 gimplify_and_add (x
, ilist
);
3340 gimple_stmt_iterator gsi
3341 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3343 = gimple_build_assign (unshare_expr (lvar
), iv
);
3344 gsi_insert_before_without_update (&gsi
, g
,
3346 tree stept
= POINTER_TYPE_P (TREE_TYPE (x
))
3347 ? sizetype
: TREE_TYPE (x
);
3348 tree t
= fold_convert (stept
,
3349 OMP_CLAUSE_LINEAR_STEP (c
));
3350 enum tree_code code
= PLUS_EXPR
;
3351 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
3352 code
= POINTER_PLUS_EXPR
;
3353 g
= gimple_build_assign_with_ops (code
, iv
, iv
, t
);
3354 gsi_insert_before_without_update (&gsi
, g
,
3358 x
= lang_hooks
.decls
.omp_clause_copy_ctor
3359 (c
, unshare_expr (ivar
), x
);
3360 gimplify_and_add (x
, &llist
[0]);
3361 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3364 gimple_seq tseq
= NULL
;
3367 gimplify_stmt (&dtor
, &tseq
);
3368 gimple_seq_add_seq (&llist
[1], tseq
);
3373 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, new_var
, x
);
3374 gimplify_and_add (x
, ilist
);
3377 case OMP_CLAUSE__LOOPTEMP_
:
3378 gcc_assert (is_parallel_ctx (ctx
));
3379 x
= build_outer_var_ref (var
, ctx
);
3380 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3381 gimplify_and_add (x
, ilist
);
3384 case OMP_CLAUSE_COPYIN
:
3385 by_ref
= use_pointer_for_field (var
, NULL
);
3386 x
= build_receiver_ref (var
, by_ref
, ctx
);
3387 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
3388 append_to_statement_list (x
, ©in_seq
);
3389 copyin_by_ref
|= by_ref
;
3392 case OMP_CLAUSE_REDUCTION
:
3393 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3395 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
3397 x
= build_outer_var_ref (var
, ctx
);
3399 if (is_reference (var
)
3400 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
3402 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3403 SET_DECL_VALUE_EXPR (placeholder
, x
);
3404 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
3405 tree new_vard
= new_var
;
3406 if (is_reference (var
))
3408 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
3409 new_vard
= TREE_OPERAND (new_var
, 0);
3410 gcc_assert (DECL_P (new_vard
));
3413 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3414 idx
, lane
, ivar
, lvar
))
3416 if (new_vard
== new_var
)
3418 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
3419 SET_DECL_VALUE_EXPR (new_var
, ivar
);
3423 SET_DECL_VALUE_EXPR (new_vard
,
3424 build_fold_addr_expr (ivar
));
3425 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
3427 x
= lang_hooks
.decls
.omp_clause_default_ctor
3428 (c
, unshare_expr (ivar
),
3429 build_outer_var_ref (var
, ctx
));
3431 gimplify_and_add (x
, &llist
[0]);
3432 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
3434 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
3435 lower_omp (&tseq
, ctx
);
3436 gimple_seq_add_seq (&llist
[0], tseq
);
3438 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
3439 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
3440 lower_omp (&tseq
, ctx
);
3441 gimple_seq_add_seq (&llist
[1], tseq
);
3442 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3443 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
3444 if (new_vard
== new_var
)
3445 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3447 SET_DECL_VALUE_EXPR (new_vard
,
3448 build_fold_addr_expr (lvar
));
3449 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3454 gimplify_stmt (&dtor
, &tseq
);
3455 gimple_seq_add_seq (&llist
[1], tseq
);
3459 x
= lang_hooks
.decls
.omp_clause_default_ctor
3460 (c
, new_var
, unshare_expr (x
));
3462 gimplify_and_add (x
, ilist
);
3463 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
3465 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
3466 lower_omp (&tseq
, ctx
);
3467 gimple_seq_add_seq (ilist
, tseq
);
3469 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
3472 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
3473 lower_omp (&tseq
, ctx
);
3474 gimple_seq_add_seq (dlist
, tseq
);
3475 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3477 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
3482 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
3483 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
3485 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3486 idx
, lane
, ivar
, lvar
))
3488 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
3489 tree ref
= build_outer_var_ref (var
, ctx
);
3491 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
3493 /* reduction(-:var) sums up the partial results, so it
3494 acts identically to reduction(+:var). */
3495 if (code
== MINUS_EXPR
)
3498 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
3499 ref
= build_outer_var_ref (var
, ctx
);
3500 gimplify_assign (ref
, x
, &llist
[1]);
3504 gimplify_assign (new_var
, x
, ilist
);
3506 gimplify_assign (build_outer_var_ref (var
, ctx
),
3520 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
3521 /* Don't want uninit warnings on simduid, it is always uninitialized,
3522 but we use it not for the value, but for the DECL_UID only. */
3523 TREE_NO_WARNING (uid
) = 1;
3525 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
3526 gimple_call_set_lhs (g
, lane
);
3527 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3528 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
3529 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
3530 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
3531 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
3532 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
3533 g
= gimple_build_assign_with_ops (INTEGER_CST
, lane
,
3534 build_int_cst (unsigned_type_node
, 0),
3536 gimple_seq_add_stmt (ilist
, g
);
3537 for (int i
= 0; i
< 2; i
++)
3540 tree vf
= create_tmp_var (unsigned_type_node
, NULL
);
3541 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
3542 gimple_call_set_lhs (g
, vf
);
3543 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
3544 gimple_seq_add_stmt (seq
, g
);
3545 tree t
= build_int_cst (unsigned_type_node
, 0);
3546 g
= gimple_build_assign_with_ops (INTEGER_CST
, idx
, t
, NULL_TREE
);
3547 gimple_seq_add_stmt (seq
, g
);
3548 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
3549 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
3550 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
3551 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
3552 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
3553 gimple_seq_add_seq (seq
, llist
[i
]);
3554 t
= build_int_cst (unsigned_type_node
, 1);
3555 g
= gimple_build_assign_with_ops (PLUS_EXPR
, idx
, idx
, t
);
3556 gimple_seq_add_stmt (seq
, g
);
3557 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
3558 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
3559 gimple_seq_add_stmt (seq
, g
);
3560 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
3564 /* The copyin sequence is not to be executed by the main thread, since
3565 that would result in self-copies. Perhaps not visible to scalars,
3566 but it certainly is to C++ operator=. */
3569 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
3571 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
3572 build_int_cst (TREE_TYPE (x
), 0));
3573 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
3574 gimplify_and_add (x
, ilist
);
3577 /* If any copyin variable is passed by reference, we must ensure the
3578 master thread doesn't modify it before it is copied over in all
3579 threads. Similarly for variables in both firstprivate and
3580 lastprivate clauses we need to ensure the lastprivate copying
3581 happens after firstprivate copying in all threads. And similarly
3582 for UDRs if initializer expression refers to omp_orig. */
3583 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
3585 /* Don't add any barrier for #pragma omp simd or
3586 #pragma omp distribute. */
3587 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3588 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
3589 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
3592 /* If max_vf is non-zero, then we can use only a vectorization factor
3593 up to the max_vf we chose. So stick it into the safelen clause. */
3596 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3597 OMP_CLAUSE_SAFELEN
);
3599 || compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3602 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
3603 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
3605 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
3606 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
3612 /* Generate code to implement the LASTPRIVATE clauses. This is used for
3613 both parallel and workshare constructs. PREDICATE may be NULL if it's
3617 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
3620 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
3621 bool par_clauses
= false;
3622 tree simduid
= NULL
, lastlane
= NULL
;
3624 /* Early exit if there are no lastprivate or linear clauses. */
3625 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
3626 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
3627 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
3628 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
3630 if (clauses
== NULL
)
3632 /* If this was a workshare clause, see if it had been combined
3633 with its parallel. In that case, look for the clauses on the
3634 parallel statement itself. */
3635 if (is_parallel_ctx (ctx
))
3639 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
3642 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
3643 OMP_CLAUSE_LASTPRIVATE
);
3644 if (clauses
== NULL
)
3652 tree label_true
, arm1
, arm2
;
3654 label
= create_artificial_label (UNKNOWN_LOCATION
);
3655 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
3656 arm1
= TREE_OPERAND (predicate
, 0);
3657 arm2
= TREE_OPERAND (predicate
, 1);
3658 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
3659 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
3660 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
3662 gimple_seq_add_stmt (stmt_list
, stmt
);
3663 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
3666 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3667 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_SIMD
)
3669 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
3671 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
3674 for (c
= clauses
; c
;)
3677 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3679 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
3680 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3681 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
3683 var
= OMP_CLAUSE_DECL (c
);
3684 new_var
= lookup_decl (var
, ctx
);
3686 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
3688 tree val
= DECL_VALUE_EXPR (new_var
);
3689 if (TREE_CODE (val
) == ARRAY_REF
3690 && VAR_P (TREE_OPERAND (val
, 0))
3691 && lookup_attribute ("omp simd array",
3692 DECL_ATTRIBUTES (TREE_OPERAND (val
,
3695 if (lastlane
== NULL
)
3697 lastlane
= create_tmp_var (unsigned_type_node
, NULL
);
3699 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
3701 TREE_OPERAND (val
, 1));
3702 gimple_call_set_lhs (g
, lastlane
);
3703 gimple_seq_add_stmt (stmt_list
, g
);
3705 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
3706 TREE_OPERAND (val
, 0), lastlane
,
3707 NULL_TREE
, NULL_TREE
);
3711 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
3712 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
3714 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
3715 gimple_seq_add_seq (stmt_list
,
3716 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
3717 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
3720 x
= build_outer_var_ref (var
, ctx
);
3721 if (is_reference (var
))
3722 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3723 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
3724 gimplify_and_add (x
, stmt_list
);
3726 c
= OMP_CLAUSE_CHAIN (c
);
3727 if (c
== NULL
&& !par_clauses
)
3729 /* If this was a workshare clause, see if it had been combined
3730 with its parallel. In that case, continue looking for the
3731 clauses also on the parallel statement itself. */
3732 if (is_parallel_ctx (ctx
))
3736 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
3739 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
3740 OMP_CLAUSE_LASTPRIVATE
);
3746 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
3750 /* Generate code to implement the REDUCTION clauses. */
3753 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
3755 gimple_seq sub_seq
= NULL
;
3760 /* SIMD reductions are handled in lower_rec_input_clauses. */
3761 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3762 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_SIMD
)
3765 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
3766 update in that case, otherwise use a lock. */
3767 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
3768 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
3770 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3772 /* Never use OMP_ATOMIC for array reductions or UDRs. */
3782 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3784 tree var
, ref
, new_var
;
3785 enum tree_code code
;
3786 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3788 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
3791 var
= OMP_CLAUSE_DECL (c
);
3792 new_var
= lookup_decl (var
, ctx
);
3793 if (is_reference (var
))
3794 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3795 ref
= build_outer_var_ref (var
, ctx
);
3796 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
3798 /* reduction(-:var) sums up the partial results, so it acts
3799 identically to reduction(+:var). */
3800 if (code
== MINUS_EXPR
)
3805 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
3807 addr
= save_expr (addr
);
3808 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
3809 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
3810 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
3811 gimplify_and_add (x
, stmt_seqp
);
3815 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3817 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
3819 if (is_reference (var
)
3820 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
3822 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
3823 SET_DECL_VALUE_EXPR (placeholder
, ref
);
3824 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
3825 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
3826 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
3827 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3828 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
3832 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
3833 ref
= build_outer_var_ref (var
, ctx
);
3834 gimplify_assign (ref
, x
, &sub_seq
);
3838 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
3840 gimple_seq_add_stmt (stmt_seqp
, stmt
);
3842 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
3844 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
3846 gimple_seq_add_stmt (stmt_seqp
, stmt
);
3850 /* Generate code to implement the COPYPRIVATE clauses. */
3853 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
3858 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3860 tree var
, new_var
, ref
, x
;
3862 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3864 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
3867 var
= OMP_CLAUSE_DECL (c
);
3868 by_ref
= use_pointer_for_field (var
, NULL
);
3870 ref
= build_sender_ref (var
, ctx
);
3871 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
3874 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
3875 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
3877 gimplify_assign (ref
, x
, slist
);
3879 ref
= build_receiver_ref (var
, false, ctx
);
3882 ref
= fold_convert_loc (clause_loc
,
3883 build_pointer_type (TREE_TYPE (new_var
)),
3885 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
3887 if (is_reference (var
))
3889 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
3890 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
3891 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3893 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
3894 gimplify_and_add (x
, rlist
);
3899 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
3900 and REDUCTION from the sender (aka parent) side. */
3903 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
3908 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3910 tree val
, ref
, x
, var
;
3911 bool by_ref
, do_in
= false, do_out
= false;
3912 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3914 switch (OMP_CLAUSE_CODE (c
))
3916 case OMP_CLAUSE_PRIVATE
:
3917 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
3920 case OMP_CLAUSE_FIRSTPRIVATE
:
3921 case OMP_CLAUSE_COPYIN
:
3922 case OMP_CLAUSE_LASTPRIVATE
:
3923 case OMP_CLAUSE_REDUCTION
:
3924 case OMP_CLAUSE__LOOPTEMP_
:
3930 val
= OMP_CLAUSE_DECL (c
);
3931 var
= lookup_decl_in_outer_ctx (val
, ctx
);
3933 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
3934 && is_global_var (var
))
3936 if (is_variable_sized (val
))
3938 by_ref
= use_pointer_for_field (val
, NULL
);
3940 switch (OMP_CLAUSE_CODE (c
))
3942 case OMP_CLAUSE_PRIVATE
:
3943 case OMP_CLAUSE_FIRSTPRIVATE
:
3944 case OMP_CLAUSE_COPYIN
:
3945 case OMP_CLAUSE__LOOPTEMP_
:
3949 case OMP_CLAUSE_LASTPRIVATE
:
3950 if (by_ref
|| is_reference (val
))
3952 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3959 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
3964 case OMP_CLAUSE_REDUCTION
:
3966 do_out
= !(by_ref
|| is_reference (val
));
3975 ref
= build_sender_ref (val
, ctx
);
3976 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
3977 gimplify_assign (ref
, x
, ilist
);
3978 if (is_task_ctx (ctx
))
3979 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
3984 ref
= build_sender_ref (val
, ctx
);
3985 gimplify_assign (var
, ref
, olist
);
3990 /* Generate code to implement SHARED from the sender (aka parent)
3991 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
3992 list things that got automatically shared. */
3995 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
3997 tree var
, ovar
, nvar
, f
, x
, record_type
;
3999 if (ctx
->record_type
== NULL
)
4002 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
4003 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
4005 ovar
= DECL_ABSTRACT_ORIGIN (f
);
4006 nvar
= maybe_lookup_decl (ovar
, ctx
);
4007 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
4010 /* If CTX is a nested parallel directive. Find the immediately
4011 enclosing parallel or workshare construct that contains a
4012 mapping for OVAR. */
4013 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
4015 if (use_pointer_for_field (ovar
, ctx
))
4017 x
= build_sender_ref (ovar
, ctx
);
4018 var
= build_fold_addr_expr (var
);
4019 gimplify_assign (x
, var
, ilist
);
4023 x
= build_sender_ref (ovar
, ctx
);
4024 gimplify_assign (x
, var
, ilist
);
4026 if (!TREE_READONLY (var
)
4027 /* We don't need to receive a new reference to a result
4028 or parm decl. In fact we may not store to it as we will
4029 invalidate any pending RSO and generate wrong gimple
4031 && !((TREE_CODE (var
) == RESULT_DECL
4032 || TREE_CODE (var
) == PARM_DECL
)
4033 && DECL_BY_REFERENCE (var
)))
4035 x
= build_sender_ref (ovar
, ctx
);
4036 gimplify_assign (var
, x
, olist
);
4043 /* A convenience function to build an empty GIMPLE_COND with just the
4047 gimple_build_cond_empty (tree cond
)
4049 enum tree_code pred_code
;
4052 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
4053 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
4057 /* Build the function calls to GOMP_parallel_start etc to actually
4058 generate the parallel operation. REGION is the parallel region
4059 being expanded. BB is the block where to insert the code. WS_ARGS
4060 will be set if this is a call to a combined parallel+workshare
4061 construct, it contains the list of additional arguments needed by
4062 the workshare construct. */
4065 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
4066 gimple entry_stmt
, vec
<tree
, va_gc
> *ws_args
)
4068 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
4069 gimple_stmt_iterator gsi
;
4071 enum built_in_function start_ix
;
4073 location_t clause_loc
;
4074 vec
<tree
, va_gc
> *args
;
4076 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
4078 /* Determine what flavor of GOMP_parallel we will be
4080 start_ix
= BUILT_IN_GOMP_PARALLEL
;
4081 if (is_combined_parallel (region
))
4083 switch (region
->inner
->type
)
4085 case GIMPLE_OMP_FOR
:
4086 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
4087 start_ix2
= ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
4088 + (region
->inner
->sched_kind
4089 == OMP_CLAUSE_SCHEDULE_RUNTIME
4090 ? 3 : region
->inner
->sched_kind
));
4091 start_ix
= (enum built_in_function
)start_ix2
;
4093 case GIMPLE_OMP_SECTIONS
:
4094 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
4101 /* By default, the value of NUM_THREADS is zero (selected at run time)
4102 and there is no conditional. */
4104 val
= build_int_cst (unsigned_type_node
, 0);
4105 flags
= build_int_cst (unsigned_type_node
, 0);
4107 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4109 cond
= OMP_CLAUSE_IF_EXPR (c
);
4111 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
4114 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
4115 clause_loc
= OMP_CLAUSE_LOCATION (c
);
4118 clause_loc
= gimple_location (entry_stmt
);
4120 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
4122 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
4124 /* Ensure 'val' is of the correct type. */
4125 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
4127 /* If we found the clause 'if (cond)', build either
4128 (cond != 0) or (cond ? val : 1u). */
4131 gimple_stmt_iterator gsi
;
4133 cond
= gimple_boolify (cond
);
4135 if (integer_zerop (val
))
4136 val
= fold_build2_loc (clause_loc
,
4137 EQ_EXPR
, unsigned_type_node
, cond
,
4138 build_int_cst (TREE_TYPE (cond
), 0));
4141 basic_block cond_bb
, then_bb
, else_bb
;
4142 edge e
, e_then
, e_else
;
4143 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
4145 tmp_var
= create_tmp_var (TREE_TYPE (val
), NULL
);
4146 if (gimple_in_ssa_p (cfun
))
4148 tmp_then
= make_ssa_name (tmp_var
, NULL
);
4149 tmp_else
= make_ssa_name (tmp_var
, NULL
);
4150 tmp_join
= make_ssa_name (tmp_var
, NULL
);
4159 e
= split_block (bb
, NULL
);
4164 then_bb
= create_empty_bb (cond_bb
);
4165 else_bb
= create_empty_bb (then_bb
);
4166 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
4167 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
4169 stmt
= gimple_build_cond_empty (cond
);
4170 gsi
= gsi_start_bb (cond_bb
);
4171 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4173 gsi
= gsi_start_bb (then_bb
);
4174 stmt
= gimple_build_assign (tmp_then
, val
);
4175 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4177 gsi
= gsi_start_bb (else_bb
);
4178 stmt
= gimple_build_assign
4179 (tmp_else
, build_int_cst (unsigned_type_node
, 1));
4180 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4182 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
4183 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
4186 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
4187 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
4189 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
4190 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
4192 if (gimple_in_ssa_p (cfun
))
4194 gimple phi
= create_phi_node (tmp_join
, bb
);
4195 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
4196 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
4202 gsi
= gsi_start_bb (bb
);
4203 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
4204 false, GSI_CONTINUE_LINKING
);
4207 gsi
= gsi_last_bb (bb
);
4208 t
= gimple_omp_parallel_data_arg (entry_stmt
);
4210 t1
= null_pointer_node
;
4212 t1
= build_fold_addr_expr (t
);
4213 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
4215 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
4216 args
->quick_push (t2
);
4217 args
->quick_push (t1
);
4218 args
->quick_push (val
);
4220 args
->splice (*ws_args
);
4221 args
->quick_push (flags
);
4223 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
4224 builtin_decl_explicit (start_ix
), args
);
4226 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4227 false, GSI_CONTINUE_LINKING
);
4231 /* Build the function call to GOMP_task to actually
4232 generate the task operation. BB is the block where to insert the code. */
4235 expand_task_call (basic_block bb
, gimple entry_stmt
)
4237 tree t
, t1
, t2
, t3
, flags
, cond
, c
, c2
, clauses
, depend
;
4238 gimple_stmt_iterator gsi
;
4239 location_t loc
= gimple_location (entry_stmt
);
4241 clauses
= gimple_omp_task_clauses (entry_stmt
);
4243 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4245 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (c
));
4247 cond
= boolean_true_node
;
4249 c
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
4250 c2
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
4251 depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
4252 flags
= build_int_cst (unsigned_type_node
,
4253 (c
? 1 : 0) + (c2
? 4 : 0) + (depend
? 8 : 0));
4255 c
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
4258 c
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c
));
4259 c
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, c
,
4260 build_int_cst (unsigned_type_node
, 2),
4261 build_int_cst (unsigned_type_node
, 0));
4262 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, c
);
4265 depend
= OMP_CLAUSE_DECL (depend
);
4267 depend
= build_int_cst (ptr_type_node
, 0);
4269 gsi
= gsi_last_bb (bb
);
4270 t
= gimple_omp_task_data_arg (entry_stmt
);
4272 t2
= null_pointer_node
;
4274 t2
= build_fold_addr_expr_loc (loc
, t
);
4275 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
4276 t
= gimple_omp_task_copy_fn (entry_stmt
);
4278 t3
= null_pointer_node
;
4280 t3
= build_fold_addr_expr_loc (loc
, t
);
4282 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
4284 gimple_omp_task_arg_size (entry_stmt
),
4285 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
4288 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4289 false, GSI_CONTINUE_LINKING
);
4293 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
4294 catch handler and return it. This prevents programs from violating the
4295 structured block semantics with throws. */
4298 maybe_catch_exception (gimple_seq body
)
4303 if (!flag_exceptions
)
4306 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
4307 decl
= lang_hooks
.eh_protect_cleanup_actions ();
4309 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
4311 g
= gimple_build_eh_must_not_throw (decl
);
4312 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
4315 return gimple_seq_alloc_with_stmt (g
);
4318 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
4321 vec2chain (vec
<tree
, va_gc
> *v
)
4323 tree chain
= NULL_TREE
, t
;
4326 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
4328 DECL_CHAIN (t
) = chain
;
4336 /* Remove barriers in REGION->EXIT's block. Note that this is only
4337 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
4338 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
4339 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
4343 remove_exit_barrier (struct omp_region
*region
)
4345 gimple_stmt_iterator gsi
;
4346 basic_block exit_bb
;
4350 int any_addressable_vars
= -1;
4352 exit_bb
= region
->exit
;
4354 /* If the parallel region doesn't return, we don't have REGION->EXIT
4359 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
4360 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
4361 statements that can appear in between are extremely limited -- no
4362 memory operations at all. Here, we allow nothing at all, so the
4363 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
4364 gsi
= gsi_last_bb (exit_bb
);
4365 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
4367 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
4370 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
4372 gsi
= gsi_last_bb (e
->src
);
4373 if (gsi_end_p (gsi
))
4375 stmt
= gsi_stmt (gsi
);
4376 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
4377 && !gimple_omp_return_nowait_p (stmt
))
4379 /* OpenMP 3.0 tasks unfortunately prevent this optimization
4380 in many cases. If there could be tasks queued, the barrier
4381 might be needed to let the tasks run before some local
4382 variable of the parallel that the task uses as shared
4383 runs out of scope. The task can be spawned either
4384 from within current function (this would be easy to check)
4385 or from some function it calls and gets passed an address
4386 of such a variable. */
4387 if (any_addressable_vars
< 0)
4389 gimple parallel_stmt
= last_stmt (region
->entry
);
4390 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
4391 tree local_decls
, block
, decl
;
4394 any_addressable_vars
= 0;
4395 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
4396 if (TREE_ADDRESSABLE (decl
))
4398 any_addressable_vars
= 1;
4401 for (block
= gimple_block (stmt
);
4402 !any_addressable_vars
4404 && TREE_CODE (block
) == BLOCK
;
4405 block
= BLOCK_SUPERCONTEXT (block
))
4407 for (local_decls
= BLOCK_VARS (block
);
4409 local_decls
= DECL_CHAIN (local_decls
))
4410 if (TREE_ADDRESSABLE (local_decls
))
4412 any_addressable_vars
= 1;
4415 if (block
== gimple_block (parallel_stmt
))
4419 if (!any_addressable_vars
)
4420 gimple_omp_return_set_nowait (stmt
);
4426 remove_exit_barriers (struct omp_region
*region
)
4428 if (region
->type
== GIMPLE_OMP_PARALLEL
)
4429 remove_exit_barrier (region
);
4433 region
= region
->inner
;
4434 remove_exit_barriers (region
);
4435 while (region
->next
)
4437 region
= region
->next
;
4438 remove_exit_barriers (region
);
4443 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
4444 calls. These can't be declared as const functions, but
4445 within one parallel body they are constant, so they can be
4446 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
4447 which are declared const. Similarly for task body, except
4448 that in untied task omp_get_thread_num () can change at any task
4449 scheduling point. */
4452 optimize_omp_library_calls (gimple entry_stmt
)
4455 gimple_stmt_iterator gsi
;
4456 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4457 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
4458 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
4459 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
4460 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
4461 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
4462 OMP_CLAUSE_UNTIED
) != NULL
);
4465 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4467 gimple call
= gsi_stmt (gsi
);
4470 if (is_gimple_call (call
)
4471 && (decl
= gimple_call_fndecl (call
))
4472 && DECL_EXTERNAL (decl
)
4473 && TREE_PUBLIC (decl
)
4474 && DECL_INITIAL (decl
) == NULL
)
4478 if (DECL_NAME (decl
) == thr_num_id
)
4480 /* In #pragma omp task untied omp_get_thread_num () can change
4481 during the execution of the task region. */
4484 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4486 else if (DECL_NAME (decl
) == num_thr_id
)
4487 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
4491 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
4492 || gimple_call_num_args (call
) != 0)
4495 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
4498 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
4499 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
4500 TREE_TYPE (TREE_TYPE (built_in
))))
4503 gimple_call_set_fndecl (call
, built_in
);
4508 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
4512 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
4516 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
4517 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
4520 if (TREE_CODE (t
) == ADDR_EXPR
)
4521 recompute_tree_invariant_for_addr_expr (t
);
4523 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
4527 /* Prepend TO = FROM assignment before *GSI_P. */
4530 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
)
4532 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
4533 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
4534 true, GSI_SAME_STMT
);
4535 gimple stmt
= gimple_build_assign (to
, from
);
4536 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
4537 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
4538 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
4540 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
4541 gimple_regimplify_operands (stmt
, &gsi
);
4545 /* Expand the OpenMP parallel or task directive starting at REGION. */
4548 expand_omp_taskreg (struct omp_region
*region
)
4550 basic_block entry_bb
, exit_bb
, new_bb
;
4551 struct function
*child_cfun
;
4552 tree child_fn
, block
, t
;
4553 gimple_stmt_iterator gsi
;
4554 gimple entry_stmt
, stmt
;
4556 vec
<tree
, va_gc
> *ws_args
;
4558 entry_stmt
= last_stmt (region
->entry
);
4559 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
4560 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
4562 entry_bb
= region
->entry
;
4563 exit_bb
= region
->exit
;
4565 if (is_combined_parallel (region
))
4566 ws_args
= region
->ws_args
;
4570 if (child_cfun
->cfg
)
4572 /* Due to inlining, it may happen that we have already outlined
4573 the region, in which case all we need to do is make the
4574 sub-graph unreachable and emit the parallel call. */
4575 edge entry_succ_e
, exit_succ_e
;
4576 gimple_stmt_iterator gsi
;
4578 entry_succ_e
= single_succ_edge (entry_bb
);
4580 gsi
= gsi_last_bb (entry_bb
);
4581 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
4582 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
4583 gsi_remove (&gsi
, true);
4588 exit_succ_e
= single_succ_edge (exit_bb
);
4589 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
4591 remove_edge_and_dominated_blocks (entry_succ_e
);
4595 unsigned srcidx
, dstidx
, num
;
4597 /* If the parallel region needs data sent from the parent
4598 function, then the very first statement (except possible
4599 tree profile counter updates) of the parallel body
4600 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
4601 &.OMP_DATA_O is passed as an argument to the child function,
4602 we need to replace it with the argument as seen by the child
4605 In most cases, this will end up being the identity assignment
4606 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
4607 a function call that has been inlined, the original PARM_DECL
4608 .OMP_DATA_I may have been converted into a different local
4609 variable. In which case, we need to keep the assignment. */
4610 if (gimple_omp_taskreg_data_arg (entry_stmt
))
4612 basic_block entry_succ_bb
= single_succ (entry_bb
);
4613 gimple_stmt_iterator gsi
;
4615 gimple parcopy_stmt
= NULL
;
4617 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
4621 gcc_assert (!gsi_end_p (gsi
));
4622 stmt
= gsi_stmt (gsi
);
4623 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
4626 if (gimple_num_ops (stmt
) == 2)
4628 tree arg
= gimple_assign_rhs1 (stmt
);
4630 /* We're ignore the subcode because we're
4631 effectively doing a STRIP_NOPS. */
4633 if (TREE_CODE (arg
) == ADDR_EXPR
4634 && TREE_OPERAND (arg
, 0)
4635 == gimple_omp_taskreg_data_arg (entry_stmt
))
4637 parcopy_stmt
= stmt
;
4643 gcc_assert (parcopy_stmt
!= NULL
);
4644 arg
= DECL_ARGUMENTS (child_fn
);
4646 if (!gimple_in_ssa_p (cfun
))
4648 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
4649 gsi_remove (&gsi
, true);
4652 /* ?? Is setting the subcode really necessary ?? */
4653 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
4654 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
4659 /* If we are in ssa form, we must load the value from the default
4660 definition of the argument. That should not be defined now,
4661 since the argument is not used uninitialized. */
4662 gcc_assert (ssa_default_def (cfun
, arg
) == NULL
);
4663 narg
= make_ssa_name (arg
, gimple_build_nop ());
4664 set_ssa_default_def (cfun
, arg
, narg
);
4665 /* ?? Is setting the subcode really necessary ?? */
4666 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (narg
));
4667 gimple_assign_set_rhs1 (parcopy_stmt
, narg
);
4668 update_stmt (parcopy_stmt
);
4672 /* Declare local variables needed in CHILD_CFUN. */
4673 block
= DECL_INITIAL (child_fn
);
4674 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
4675 /* The gimplifier could record temporaries in parallel/task block
4676 rather than in containing function's local_decls chain,
4677 which would mean cgraph missed finalizing them. Do it now. */
4678 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
4679 if (TREE_CODE (t
) == VAR_DECL
4681 && !DECL_EXTERNAL (t
))
4682 varpool_finalize_decl (t
);
4683 DECL_SAVED_TREE (child_fn
) = NULL
;
4684 /* We'll create a CFG for child_fn, so no gimple body is needed. */
4685 gimple_set_body (child_fn
, NULL
);
4686 TREE_USED (block
) = 1;
4688 /* Reset DECL_CONTEXT on function arguments. */
4689 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
4690 DECL_CONTEXT (t
) = child_fn
;
4692 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
4693 so that it can be moved to the child function. */
4694 gsi
= gsi_last_bb (entry_bb
);
4695 stmt
= gsi_stmt (gsi
);
4696 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
4697 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
4698 gsi_remove (&gsi
, true);
4699 e
= split_block (entry_bb
, stmt
);
4701 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
4703 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
4706 gsi
= gsi_last_bb (exit_bb
);
4707 gcc_assert (!gsi_end_p (gsi
)
4708 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
4709 stmt
= gimple_build_return (NULL
);
4710 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
4711 gsi_remove (&gsi
, true);
4714 /* Move the parallel region into CHILD_CFUN. */
4716 if (gimple_in_ssa_p (cfun
))
4718 init_tree_ssa (child_cfun
);
4719 init_ssa_operands (child_cfun
);
4720 child_cfun
->gimple_df
->in_ssa_p
= true;
4724 block
= gimple_block (entry_stmt
);
4726 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
4728 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
4729 /* When the OMP expansion process cannot guarantee an up-to-date
4730 loop tree arrange for the child function to fixup loops. */
4731 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
4732 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
4734 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
4735 num
= vec_safe_length (child_cfun
->local_decls
);
4736 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
4738 t
= (*child_cfun
->local_decls
)[srcidx
];
4739 if (DECL_CONTEXT (t
) == cfun
->decl
)
4741 if (srcidx
!= dstidx
)
4742 (*child_cfun
->local_decls
)[dstidx
] = t
;
4746 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
4748 /* Inform the callgraph about the new function. */
4749 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
4750 cgraph_add_new_function (child_fn
, true);
4752 /* Fix the callgraph edges for child_cfun. Those for cfun will be
4753 fixed in a following pass. */
4754 push_cfun (child_cfun
);
4756 optimize_omp_library_calls (entry_stmt
);
4757 rebuild_cgraph_edges ();
4759 /* Some EH regions might become dead, see PR34608. If
4760 pass_cleanup_cfg isn't the first pass to happen with the
4761 new child, these dead EH edges might cause problems.
4762 Clean them up now. */
4763 if (flag_exceptions
)
4766 bool changed
= false;
4769 changed
|= gimple_purge_dead_eh_edges (bb
);
4771 cleanup_tree_cfg ();
4773 if (gimple_in_ssa_p (cfun
))
4774 update_ssa (TODO_update_ssa
);
4778 /* Emit a library call to launch the children threads. */
4779 if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
4780 expand_parallel_call (region
, new_bb
, entry_stmt
, ws_args
);
4782 expand_task_call (new_bb
, entry_stmt
);
4783 if (gimple_in_ssa_p (cfun
))
4784 update_ssa (TODO_update_ssa_only_virtuals
);
4788 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
4789 of the combined collapse > 1 loop constructs, generate code like:
4790 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
4795 count3 = (adj + N32 - N31) / STEP3;
4796 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
4801 count2 = (adj + N22 - N21) / STEP2;
4802 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
4807 count1 = (adj + N12 - N11) / STEP1;
4808 count = count1 * count2 * count3;
4809 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
4811 and set ZERO_ITER_BB to that bb. If this isn't the outermost
4812 of the combined loop constructs, just initialize COUNTS array
4813 from the _looptemp_ clauses. */
4815 /* NOTE: It *could* be better to moosh all of the BBs together,
4816 creating one larger BB with all the computation and the unexpected
4817 jump at the end. I.e.
4819 bool zero3, zero2, zero1, zero;
4822 count3 = (N32 - N31) /[cl] STEP3;
4824 count2 = (N22 - N21) /[cl] STEP2;
4826 count1 = (N12 - N11) /[cl] STEP1;
4827 zero = zero3 || zero2 || zero1;
4828 count = count1 * count2 * count3;
4829 if (__builtin_expect(zero, false)) goto zero_iter_bb;
4831 After all, we expect the zero=false, and thus we expect to have to
4832 evaluate all of the comparison expressions, so short-circuiting
4833 oughtn't be a win. Since the condition isn't protecting a
4834 denominator, we're not concerned about divide-by-zero, so we can
4835 fully evaluate count even if a numerator turned out to be wrong.
4837 It seems like putting this all together would create much better
4838 scheduling opportunities, and less pressure on the chip's branch
4842 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
4843 basic_block
&entry_bb
, tree
*counts
,
4844 basic_block
&zero_iter_bb
, int &first_zero_iter
,
4845 basic_block
&l2_dom_bb
)
4847 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
4852 /* Collapsed loops need work for expansion into SSA form. */
4853 gcc_assert (!gimple_in_ssa_p (cfun
));
4855 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
4856 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
4858 /* First two _looptemp_ clauses are for istart/iend, counts[0]
4859 isn't supposed to be handled, as the inner loop doesn't
4861 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
4862 OMP_CLAUSE__LOOPTEMP_
);
4863 gcc_assert (innerc
);
4864 for (i
= 0; i
< fd
->collapse
; i
++)
4866 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
4867 OMP_CLAUSE__LOOPTEMP_
);
4868 gcc_assert (innerc
);
4870 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
4872 counts
[0] = NULL_TREE
;
4877 for (i
= 0; i
< fd
->collapse
; i
++)
4879 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
4881 if (SSA_VAR_P (fd
->loop
.n2
)
4882 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
4883 fold_convert (itype
, fd
->loops
[i
].n1
),
4884 fold_convert (itype
, fd
->loops
[i
].n2
)))
4885 == NULL_TREE
|| !integer_onep (t
)))
4888 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
4889 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
4890 true, GSI_SAME_STMT
);
4891 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
4892 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
4893 true, GSI_SAME_STMT
);
4894 stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
4895 NULL_TREE
, NULL_TREE
);
4896 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
4897 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
4898 expand_omp_regimplify_p
, NULL
, NULL
)
4899 || walk_tree (gimple_cond_rhs_ptr (stmt
),
4900 expand_omp_regimplify_p
, NULL
, NULL
))
4902 *gsi
= gsi_for_stmt (stmt
);
4903 gimple_regimplify_operands (stmt
, gsi
);
4905 e
= split_block (entry_bb
, stmt
);
4906 if (zero_iter_bb
== NULL
)
4908 first_zero_iter
= i
;
4909 zero_iter_bb
= create_empty_bb (entry_bb
);
4911 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
4912 *gsi
= gsi_after_labels (zero_iter_bb
);
4913 stmt
= gimple_build_assign (fd
->loop
.n2
,
4914 build_zero_cst (type
));
4915 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
4916 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
4919 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
4920 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
4921 e
->flags
= EDGE_TRUE_VALUE
;
4922 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
4923 if (l2_dom_bb
== NULL
)
4924 l2_dom_bb
= entry_bb
;
4926 *gsi
= gsi_last_bb (entry_bb
);
4929 if (POINTER_TYPE_P (itype
))
4930 itype
= signed_type_for (itype
);
4931 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
4933 t
= fold_build2 (PLUS_EXPR
, itype
,
4934 fold_convert (itype
, fd
->loops
[i
].step
), t
);
4935 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
4936 fold_convert (itype
, fd
->loops
[i
].n2
));
4937 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
4938 fold_convert (itype
, fd
->loops
[i
].n1
));
4939 /* ?? We could probably use CEIL_DIV_EXPR instead of
4940 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
4941 generate the same code in the end because generically we
4942 don't know that the values involved must be negative for
4944 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
4945 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
4946 fold_build1 (NEGATE_EXPR
, itype
, t
),
4947 fold_build1 (NEGATE_EXPR
, itype
,
4948 fold_convert (itype
,
4949 fd
->loops
[i
].step
)));
4951 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
4952 fold_convert (itype
, fd
->loops
[i
].step
));
4953 t
= fold_convert (type
, t
);
4954 if (TREE_CODE (t
) == INTEGER_CST
)
4958 counts
[i
] = create_tmp_reg (type
, ".count");
4959 expand_omp_build_assign (gsi
, counts
[i
], t
);
4961 if (SSA_VAR_P (fd
->loop
.n2
))
4966 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
4967 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
4973 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
4975 V3 = N31 + (T % count3) * STEP3;
4977 V2 = N21 + (T % count2) * STEP2;
4979 V1 = N11 + T * STEP1;
4980 if this loop doesn't have an inner loop construct combined with it.
4981 If it does have an inner loop construct combined with it and the
4982 iteration count isn't known constant, store values from counts array
4983 into its _looptemp_ temporaries instead. */
4986 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
4987 tree
*counts
, gimple inner_stmt
, tree startvar
)
4990 if (gimple_omp_for_combined_p (fd
->for_stmt
))
4992 /* If fd->loop.n2 is constant, then no propagation of the counts
4993 is needed, they are constant. */
4994 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
4997 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
4998 ? gimple_omp_parallel_clauses (inner_stmt
)
4999 : gimple_omp_for_clauses (inner_stmt
);
5000 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5001 isn't supposed to be handled, as the inner loop doesn't
5003 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
5004 gcc_assert (innerc
);
5005 for (i
= 0; i
< fd
->collapse
; i
++)
5007 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5008 OMP_CLAUSE__LOOPTEMP_
);
5009 gcc_assert (innerc
);
5012 tree tem
= OMP_CLAUSE_DECL (innerc
);
5013 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
5014 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5015 false, GSI_CONTINUE_LINKING
);
5016 gimple stmt
= gimple_build_assign (tem
, t
);
5017 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5023 tree type
= TREE_TYPE (fd
->loop
.v
);
5024 tree tem
= create_tmp_reg (type
, ".tem");
5025 gimple stmt
= gimple_build_assign (tem
, startvar
);
5026 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5028 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5030 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
5032 if (POINTER_TYPE_P (vtype
))
5033 itype
= signed_type_for (vtype
);
5035 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
5038 t
= fold_convert (itype
, t
);
5039 t
= fold_build2 (MULT_EXPR
, itype
, t
,
5040 fold_convert (itype
, fd
->loops
[i
].step
));
5041 if (POINTER_TYPE_P (vtype
))
5042 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
5044 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
5045 t
= force_gimple_operand_gsi (gsi
, t
,
5046 DECL_P (fd
->loops
[i
].v
)
5047 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5049 GSI_CONTINUE_LINKING
);
5050 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5051 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5054 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
5055 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5056 false, GSI_CONTINUE_LINKING
);
5057 stmt
= gimple_build_assign (tem
, t
);
5058 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5064 /* Helper function for expand_omp_for_*. Generate code like:
5067 if (V3 cond3 N32) goto BODY_BB; else goto L11;
5071 if (V2 cond2 N22) goto BODY_BB; else goto L12;
5078 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
5079 basic_block body_bb
)
5081 basic_block last_bb
, bb
, collapse_bb
= NULL
;
5083 gimple_stmt_iterator gsi
;
5089 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5091 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
5093 bb
= create_empty_bb (last_bb
);
5095 add_bb_to_loop (bb
, last_bb
->loop_father
);
5096 gsi
= gsi_start_bb (bb
);
5098 if (i
< fd
->collapse
- 1)
5100 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
5101 e
->probability
= REG_BR_PROB_BASE
/ 8;
5103 t
= fd
->loops
[i
+ 1].n1
;
5104 t
= force_gimple_operand_gsi (&gsi
, t
,
5105 DECL_P (fd
->loops
[i
+ 1].v
)
5106 && TREE_ADDRESSABLE (fd
->loops
[i
5109 GSI_CONTINUE_LINKING
);
5110 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
5111 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5116 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
5118 if (POINTER_TYPE_P (vtype
))
5119 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5121 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5122 t
= force_gimple_operand_gsi (&gsi
, t
,
5123 DECL_P (fd
->loops
[i
].v
)
5124 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5125 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5126 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5127 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5131 t
= fd
->loops
[i
].n2
;
5132 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5133 false, GSI_CONTINUE_LINKING
);
5134 tree v
= fd
->loops
[i
].v
;
5135 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
5136 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
5137 false, GSI_CONTINUE_LINKING
);
5138 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
5139 stmt
= gimple_build_cond_empty (t
);
5140 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5141 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
5142 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5145 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
5153 /* A subroutine of expand_omp_for. Generate code for a parallel
5154 loop with any schedule. Given parameters:
5156 for (V = N1; V cond N2; V += STEP) BODY;
5158 where COND is "<" or ">", we generate pseudocode
5160 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
5161 if (more) goto L0; else goto L3;
5168 if (V cond iend) goto L1; else goto L2;
5170 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5173 If this is a combined omp parallel loop, instead of the call to
5174 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
5175 If this is gimple_omp_for_combined_p loop, then instead of assigning
5176 V and iend in L0 we assign the first two _looptemp_ clause decls of the
5177 inner GIMPLE_OMP_FOR and V += STEP; and
5178 if (V cond iend) goto L1; else goto L2; are removed.
5180 For collapsed loops, given parameters:
5182 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
5183 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
5184 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
5187 we generate pseudocode
5189 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
5194 count3 = (adj + N32 - N31) / STEP3;
5195 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
5200 count2 = (adj + N22 - N21) / STEP2;
5201 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
5206 count1 = (adj + N12 - N11) / STEP1;
5207 count = count1 * count2 * count3;
5212 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
5213 if (more) goto L0; else goto L3;
5217 V3 = N31 + (T % count3) * STEP3;
5219 V2 = N21 + (T % count2) * STEP2;
5221 V1 = N11 + T * STEP1;
5226 if (V < iend) goto L10; else goto L2;
5229 if (V3 cond3 N32) goto L1; else goto L11;
5233 if (V2 cond2 N22) goto L1; else goto L12;
5239 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5245 expand_omp_for_generic (struct omp_region
*region
,
5246 struct omp_for_data
*fd
,
5247 enum built_in_function start_fn
,
5248 enum built_in_function next_fn
,
5251 tree type
, istart0
, iend0
, iend
;
5252 tree t
, vmain
, vback
, bias
= NULL_TREE
;
5253 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
5254 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
5255 gimple_stmt_iterator gsi
;
5257 bool in_combined_parallel
= is_combined_parallel (region
);
5258 bool broken_loop
= region
->cont
== NULL
;
5260 tree
*counts
= NULL
;
5263 gcc_assert (!broken_loop
|| !in_combined_parallel
);
5264 gcc_assert (fd
->iter_type
== long_integer_type_node
5265 || !in_combined_parallel
);
5267 type
= TREE_TYPE (fd
->loop
.v
);
5268 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
5269 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
5270 TREE_ADDRESSABLE (istart0
) = 1;
5271 TREE_ADDRESSABLE (iend0
) = 1;
5273 /* See if we need to bias by LLONG_MIN. */
5274 if (fd
->iter_type
== long_long_unsigned_type_node
5275 && TREE_CODE (type
) == INTEGER_TYPE
5276 && !TYPE_UNSIGNED (type
))
5280 if (fd
->loop
.cond_code
== LT_EXPR
)
5283 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5287 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5290 if (TREE_CODE (n1
) != INTEGER_CST
5291 || TREE_CODE (n2
) != INTEGER_CST
5292 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
5293 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
5296 entry_bb
= region
->entry
;
5297 cont_bb
= region
->cont
;
5299 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
5300 gcc_assert (broken_loop
5301 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
5302 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
5303 l1_bb
= single_succ (l0_bb
);
5306 l2_bb
= create_empty_bb (cont_bb
);
5307 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
);
5308 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
5312 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
5313 exit_bb
= region
->exit
;
5315 gsi
= gsi_last_bb (entry_bb
);
5317 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5318 if (fd
->collapse
> 1)
5320 int first_zero_iter
= -1;
5321 basic_block zero_iter_bb
= NULL
, l2_dom_bb
= NULL
;
5323 counts
= XALLOCAVEC (tree
, fd
->collapse
);
5324 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
5325 zero_iter_bb
, first_zero_iter
,
5330 /* Some counts[i] vars might be uninitialized if
5331 some loop has zero iterations. But the body shouldn't
5332 be executed in that case, so just avoid uninit warnings. */
5333 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
5334 if (SSA_VAR_P (counts
[i
]))
5335 TREE_NO_WARNING (counts
[i
]) = 1;
5337 e
= split_block (entry_bb
, gsi_stmt (gsi
));
5339 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
5340 gsi
= gsi_last_bb (entry_bb
);
5341 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
5342 get_immediate_dominator (CDI_DOMINATORS
,
5346 if (in_combined_parallel
)
5348 /* In a combined parallel loop, emit a call to
5349 GOMP_loop_foo_next. */
5350 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
5351 build_fold_addr_expr (istart0
),
5352 build_fold_addr_expr (iend0
));
5356 tree t0
, t1
, t2
, t3
, t4
;
5357 /* If this is not a combined parallel loop, emit a call to
5358 GOMP_loop_foo_start in ENTRY_BB. */
5359 t4
= build_fold_addr_expr (iend0
);
5360 t3
= build_fold_addr_expr (istart0
);
5361 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
5364 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5366 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5367 OMP_CLAUSE__LOOPTEMP_
);
5368 gcc_assert (innerc
);
5369 t0
= OMP_CLAUSE_DECL (innerc
);
5370 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5371 OMP_CLAUSE__LOOPTEMP_
);
5372 gcc_assert (innerc
);
5373 t1
= OMP_CLAUSE_DECL (innerc
);
5375 if (POINTER_TYPE_P (TREE_TYPE (t0
))
5376 && TYPE_PRECISION (TREE_TYPE (t0
))
5377 != TYPE_PRECISION (fd
->iter_type
))
5379 /* Avoid casting pointers to integer of a different size. */
5380 tree itype
= signed_type_for (type
);
5381 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
5382 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
5386 t1
= fold_convert (fd
->iter_type
, t1
);
5387 t0
= fold_convert (fd
->iter_type
, t0
);
5391 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
5392 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
5394 if (fd
->iter_type
== long_integer_type_node
)
5398 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
5399 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5400 6, t0
, t1
, t2
, t
, t3
, t4
);
5403 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5404 5, t0
, t1
, t2
, t3
, t4
);
5412 /* The GOMP_loop_ull_*start functions have additional boolean
5413 argument, true for < loops and false for > loops.
5414 In Fortran, the C bool type can be different from
5415 boolean_type_node. */
5416 bfn_decl
= builtin_decl_explicit (start_fn
);
5417 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
5418 t5
= build_int_cst (c_bool_type
,
5419 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
5422 tree bfn_decl
= builtin_decl_explicit (start_fn
);
5423 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
5424 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
5427 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5428 6, t5
, t0
, t1
, t2
, t3
, t4
);
5431 if (TREE_TYPE (t
) != boolean_type_node
)
5432 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
5433 t
, build_int_cst (TREE_TYPE (t
), 0));
5434 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5435 true, GSI_SAME_STMT
);
5436 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
5438 /* Remove the GIMPLE_OMP_FOR statement. */
5439 gsi_remove (&gsi
, true);
5441 /* Iteration setup for sequential loop goes in L0_BB. */
5442 tree startvar
= fd
->loop
.v
;
5443 tree endvar
= NULL_TREE
;
5445 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5447 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
5448 && gimple_omp_for_kind (inner_stmt
)
5449 == GF_OMP_FOR_KIND_SIMD
);
5450 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
5451 OMP_CLAUSE__LOOPTEMP_
);
5452 gcc_assert (innerc
);
5453 startvar
= OMP_CLAUSE_DECL (innerc
);
5454 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5455 OMP_CLAUSE__LOOPTEMP_
);
5456 gcc_assert (innerc
);
5457 endvar
= OMP_CLAUSE_DECL (innerc
);
5460 gsi
= gsi_start_bb (l0_bb
);
5463 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
5464 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
5465 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
5466 t
= fold_convert (TREE_TYPE (startvar
), t
);
5467 t
= force_gimple_operand_gsi (&gsi
, t
,
5469 && TREE_ADDRESSABLE (startvar
),
5470 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5471 stmt
= gimple_build_assign (startvar
, t
);
5472 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5476 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
5477 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
5478 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
5479 t
= fold_convert (TREE_TYPE (startvar
), t
);
5480 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5481 false, GSI_CONTINUE_LINKING
);
5484 stmt
= gimple_build_assign (endvar
, iend
);
5485 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5487 if (fd
->collapse
> 1)
5488 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
5492 /* Code to control the increment and predicate for the sequential
5493 loop goes in the CONT_BB. */
5494 gsi
= gsi_last_bb (cont_bb
);
5495 stmt
= gsi_stmt (gsi
);
5496 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
5497 vmain
= gimple_omp_continue_control_use (stmt
);
5498 vback
= gimple_omp_continue_control_def (stmt
);
5500 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5502 if (POINTER_TYPE_P (type
))
5503 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
5505 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
5506 t
= force_gimple_operand_gsi (&gsi
, t
,
5508 && TREE_ADDRESSABLE (vback
),
5509 NULL_TREE
, true, GSI_SAME_STMT
);
5510 stmt
= gimple_build_assign (vback
, t
);
5511 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5513 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
5514 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
5516 stmt
= gimple_build_cond_empty (t
);
5517 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5520 /* Remove GIMPLE_OMP_CONTINUE. */
5521 gsi_remove (&gsi
, true);
5523 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
5524 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
5526 /* Emit code to get the next parallel iteration in L2_BB. */
5527 gsi
= gsi_start_bb (l2_bb
);
5529 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
5530 build_fold_addr_expr (istart0
),
5531 build_fold_addr_expr (iend0
));
5532 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5533 false, GSI_CONTINUE_LINKING
);
5534 if (TREE_TYPE (t
) != boolean_type_node
)
5535 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
5536 t
, build_int_cst (TREE_TYPE (t
), 0));
5537 stmt
= gimple_build_cond_empty (t
);
5538 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5541 /* Add the loop cleanup function. */
5542 gsi
= gsi_last_bb (exit_bb
);
5543 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
5544 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
5545 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
5546 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
5548 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
5549 stmt
= gimple_build_call (t
, 0);
5550 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
5551 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
5552 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
5553 gsi_remove (&gsi
, true);
5555 /* Connect the new blocks. */
5556 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
5557 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
5563 e
= find_edge (cont_bb
, l3_bb
);
5564 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
5566 phis
= phi_nodes (l3_bb
);
5567 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5569 gimple phi
= gsi_stmt (gsi
);
5570 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
5571 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
5575 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
5577 add_bb_to_loop (l2_bb
, cont_bb
->loop_father
);
5578 e
= find_edge (cont_bb
, l1_bb
);
5579 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5584 else if (fd
->collapse
> 1)
5587 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
5590 e
->flags
= EDGE_TRUE_VALUE
;
5593 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5594 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
5598 e
= find_edge (cont_bb
, l2_bb
);
5599 e
->flags
= EDGE_FALLTHRU
;
5601 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
5603 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
5604 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
5605 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
5606 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
5607 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
5608 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
5609 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
5610 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
5612 struct loop
*outer_loop
= alloc_loop ();
5613 outer_loop
->header
= l0_bb
;
5614 outer_loop
->latch
= l2_bb
;
5615 add_loop (outer_loop
, l0_bb
->loop_father
);
5617 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5619 struct loop
*loop
= alloc_loop ();
5620 loop
->header
= l1_bb
;
5621 /* The loop may have multiple latches. */
5622 add_loop (loop
, outer_loop
);
5628 /* A subroutine of expand_omp_for. Generate code for a parallel
5629 loop with static schedule and no specified chunk size. Given
5632 for (V = N1; V cond N2; V += STEP) BODY;
5634 where COND is "<" or ">", we generate pseudocode
5636 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
5641 if ((__typeof (V)) -1 > 0 && cond is >)
5642 n = -(adj + N2 - N1) / -STEP;
5644 n = (adj + N2 - N1) / STEP;
5647 if (threadid < tt) goto L3; else goto L4;
5652 s0 = q * threadid + tt;
5655 if (s0 >= e0) goto L2; else goto L0;
5661 if (V cond e) goto L1;
5666 expand_omp_for_static_nochunk (struct omp_region
*region
,
5667 struct omp_for_data
*fd
,
5670 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
5671 tree type
, itype
, vmain
, vback
;
5672 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
5673 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
5675 gimple_stmt_iterator gsi
;
5678 enum built_in_function get_num_threads
= BUILT_IN_OMP_GET_NUM_THREADS
;
5679 enum built_in_function get_thread_num
= BUILT_IN_OMP_GET_THREAD_NUM
;
5680 bool broken_loop
= region
->cont
== NULL
;
5681 tree
*counts
= NULL
;
5684 itype
= type
= TREE_TYPE (fd
->loop
.v
);
5685 if (POINTER_TYPE_P (type
))
5686 itype
= signed_type_for (type
);
5688 entry_bb
= region
->entry
;
5689 cont_bb
= region
->cont
;
5690 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
5691 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
5692 gcc_assert (broken_loop
5693 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
5694 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
5695 body_bb
= single_succ (seq_start_bb
);
5698 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
5699 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
5701 exit_bb
= region
->exit
;
5703 /* Iteration space partitioning goes in ENTRY_BB. */
5704 gsi
= gsi_last_bb (entry_bb
);
5705 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5707 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
5709 get_num_threads
= BUILT_IN_OMP_GET_NUM_TEAMS
;
5710 get_thread_num
= BUILT_IN_OMP_GET_TEAM_NUM
;
5713 if (fd
->collapse
> 1)
5715 int first_zero_iter
= -1;
5716 basic_block l2_dom_bb
= NULL
;
5718 counts
= XALLOCAVEC (tree
, fd
->collapse
);
5719 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
5720 fin_bb
, first_zero_iter
,
5724 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5725 t
= integer_one_node
;
5727 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
5728 fold_convert (type
, fd
->loop
.n1
),
5729 fold_convert (type
, fd
->loop
.n2
));
5730 if (fd
->collapse
== 1
5731 && TYPE_UNSIGNED (type
)
5732 && (t
== NULL_TREE
|| !integer_onep (t
)))
5734 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
5735 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
5736 true, GSI_SAME_STMT
);
5737 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
5738 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
5739 true, GSI_SAME_STMT
);
5740 stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
5741 NULL_TREE
, NULL_TREE
);
5742 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5743 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
5744 expand_omp_regimplify_p
, NULL
, NULL
)
5745 || walk_tree (gimple_cond_rhs_ptr (stmt
),
5746 expand_omp_regimplify_p
, NULL
, NULL
))
5748 gsi
= gsi_for_stmt (stmt
);
5749 gimple_regimplify_operands (stmt
, &gsi
);
5751 ep
= split_block (entry_bb
, stmt
);
5752 ep
->flags
= EDGE_TRUE_VALUE
;
5753 entry_bb
= ep
->dest
;
5754 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
5755 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
5756 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
5757 if (gimple_in_ssa_p (cfun
))
5759 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
5760 for (gsi
= gsi_start_phis (fin_bb
);
5761 !gsi_end_p (gsi
); gsi_next (&gsi
))
5763 gimple phi
= gsi_stmt (gsi
);
5764 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
5765 ep
, UNKNOWN_LOCATION
);
5768 gsi
= gsi_last_bb (entry_bb
);
5771 t
= build_call_expr (builtin_decl_explicit (get_num_threads
), 0);
5772 t
= fold_convert (itype
, t
);
5773 nthreads
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5774 true, GSI_SAME_STMT
);
5776 t
= build_call_expr (builtin_decl_explicit (get_thread_num
), 0);
5777 t
= fold_convert (itype
, t
);
5778 threadid
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5779 true, GSI_SAME_STMT
);
5783 step
= fd
->loop
.step
;
5784 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5786 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5787 OMP_CLAUSE__LOOPTEMP_
);
5788 gcc_assert (innerc
);
5789 n1
= OMP_CLAUSE_DECL (innerc
);
5790 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5791 OMP_CLAUSE__LOOPTEMP_
);
5792 gcc_assert (innerc
);
5793 n2
= OMP_CLAUSE_DECL (innerc
);
5795 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
5796 true, NULL_TREE
, true, GSI_SAME_STMT
);
5797 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
5798 true, NULL_TREE
, true, GSI_SAME_STMT
);
5799 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
5800 true, NULL_TREE
, true, GSI_SAME_STMT
);
5802 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
5803 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
5804 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
5805 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
5806 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
5807 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5808 fold_build1 (NEGATE_EXPR
, itype
, t
),
5809 fold_build1 (NEGATE_EXPR
, itype
, step
));
5811 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
5812 t
= fold_convert (itype
, t
);
5813 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
5815 q
= create_tmp_reg (itype
, "q");
5816 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
5817 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
5818 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
5820 tt
= create_tmp_reg (itype
, "tt");
5821 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
5822 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
5823 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
5825 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
5826 stmt
= gimple_build_cond_empty (t
);
5827 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5829 second_bb
= split_block (entry_bb
, stmt
)->dest
;
5830 gsi
= gsi_last_bb (second_bb
);
5831 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5833 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
5835 stmt
= gimple_build_assign_with_ops (PLUS_EXPR
, q
, q
,
5836 build_int_cst (itype
, 1));
5837 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5839 third_bb
= split_block (second_bb
, stmt
)->dest
;
5840 gsi
= gsi_last_bb (third_bb
);
5841 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5843 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
5844 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
5845 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
5847 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
5848 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
5850 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
5851 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
5853 /* Remove the GIMPLE_OMP_FOR statement. */
5854 gsi_remove (&gsi
, true);
5856 /* Setup code for sequential iteration goes in SEQ_START_BB. */
5857 gsi
= gsi_start_bb (seq_start_bb
);
5859 tree startvar
= fd
->loop
.v
;
5860 tree endvar
= NULL_TREE
;
5862 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5864 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
5865 ? gimple_omp_parallel_clauses (inner_stmt
)
5866 : gimple_omp_for_clauses (inner_stmt
);
5867 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
5868 gcc_assert (innerc
);
5869 startvar
= OMP_CLAUSE_DECL (innerc
);
5870 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5871 OMP_CLAUSE__LOOPTEMP_
);
5872 gcc_assert (innerc
);
5873 endvar
= OMP_CLAUSE_DECL (innerc
);
5875 t
= fold_convert (itype
, s0
);
5876 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
5877 if (POINTER_TYPE_P (type
))
5878 t
= fold_build_pointer_plus (n1
, t
);
5880 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
5881 t
= fold_convert (TREE_TYPE (startvar
), t
);
5882 t
= force_gimple_operand_gsi (&gsi
, t
,
5884 && TREE_ADDRESSABLE (startvar
),
5885 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5886 stmt
= gimple_build_assign (startvar
, t
);
5887 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5889 t
= fold_convert (itype
, e0
);
5890 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
5891 if (POINTER_TYPE_P (type
))
5892 t
= fold_build_pointer_plus (n1
, t
);
5894 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
5895 t
= fold_convert (TREE_TYPE (startvar
), t
);
5896 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5897 false, GSI_CONTINUE_LINKING
);
5900 stmt
= gimple_build_assign (endvar
, e
);
5901 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5903 if (fd
->collapse
> 1)
5904 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
5908 /* The code controlling the sequential loop replaces the
5909 GIMPLE_OMP_CONTINUE. */
5910 gsi
= gsi_last_bb (cont_bb
);
5911 stmt
= gsi_stmt (gsi
);
5912 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
5913 vmain
= gimple_omp_continue_control_use (stmt
);
5914 vback
= gimple_omp_continue_control_def (stmt
);
5916 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5918 if (POINTER_TYPE_P (type
))
5919 t
= fold_build_pointer_plus (vmain
, step
);
5921 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
5922 t
= force_gimple_operand_gsi (&gsi
, t
,
5924 && TREE_ADDRESSABLE (vback
),
5925 NULL_TREE
, true, GSI_SAME_STMT
);
5926 stmt
= gimple_build_assign (vback
, t
);
5927 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5929 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
5930 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
5932 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
5935 /* Remove the GIMPLE_OMP_CONTINUE statement. */
5936 gsi_remove (&gsi
, true);
5938 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
5939 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
5942 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
5943 gsi
= gsi_last_bb (exit_bb
);
5944 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
5946 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
5947 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
5949 gsi_remove (&gsi
, true);
5951 /* Connect all the blocks. */
5952 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
5953 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
5954 ep
= find_edge (entry_bb
, second_bb
);
5955 ep
->flags
= EDGE_TRUE_VALUE
;
5956 ep
->probability
= REG_BR_PROB_BASE
/ 4;
5957 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
5958 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
5962 ep
= find_edge (cont_bb
, body_bb
);
5963 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5968 else if (fd
->collapse
> 1)
5971 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
5974 ep
->flags
= EDGE_TRUE_VALUE
;
5975 find_edge (cont_bb
, fin_bb
)->flags
5976 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
5979 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
5980 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
5981 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
5983 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
5984 recompute_dominator (CDI_DOMINATORS
, body_bb
));
5985 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
5986 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
5988 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
5990 struct loop
*loop
= alloc_loop ();
5991 loop
->header
= body_bb
;
5992 if (collapse_bb
== NULL
)
5993 loop
->latch
= cont_bb
;
5994 add_loop (loop
, body_bb
->loop_father
);
5999 /* A subroutine of expand_omp_for. Generate code for a parallel
6000 loop with static schedule and a specified chunk size. Given
6003 for (V = N1; V cond N2; V += STEP) BODY;
6005 where COND is "<" or ">", we generate pseudocode
6007 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6012 if ((__typeof (V)) -1 > 0 && cond is >)
6013 n = -(adj + N2 - N1) / -STEP;
6015 n = (adj + N2 - N1) / STEP;
6017 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
6018 here so that V is defined
6019 if the loop is not entered
6021 s0 = (trip * nthreads + threadid) * CHUNK;
6022 e0 = min(s0 + CHUNK, n);
6023 if (s0 < n) goto L1; else goto L4;
6030 if (V cond e) goto L2; else goto L3;
6038 expand_omp_for_static_chunk (struct omp_region
*region
,
6039 struct omp_for_data
*fd
, gimple inner_stmt
)
6041 tree n
, s0
, e0
, e
, t
;
6042 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
6043 tree type
, itype
, v_main
, v_back
, v_extra
;
6044 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
6045 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
6046 gimple_stmt_iterator si
;
6049 enum built_in_function get_num_threads
= BUILT_IN_OMP_GET_NUM_THREADS
;
6050 enum built_in_function get_thread_num
= BUILT_IN_OMP_GET_THREAD_NUM
;
6051 bool broken_loop
= region
->cont
== NULL
;
6052 tree
*counts
= NULL
;
6055 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6056 if (POINTER_TYPE_P (type
))
6057 itype
= signed_type_for (type
);
6059 entry_bb
= region
->entry
;
6060 se
= split_block (entry_bb
, last_stmt (entry_bb
));
6062 iter_part_bb
= se
->dest
;
6063 cont_bb
= region
->cont
;
6064 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
6065 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
6066 gcc_assert (broken_loop
6067 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
6068 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
6069 body_bb
= single_succ (seq_start_bb
);
6072 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6073 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6074 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
6076 exit_bb
= region
->exit
;
6078 /* Trip and adjustment setup goes in ENTRY_BB. */
6079 si
= gsi_last_bb (entry_bb
);
6080 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_FOR
);
6082 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
6084 get_num_threads
= BUILT_IN_OMP_GET_NUM_TEAMS
;
6085 get_thread_num
= BUILT_IN_OMP_GET_TEAM_NUM
;
6088 if (fd
->collapse
> 1)
6090 int first_zero_iter
= -1;
6091 basic_block l2_dom_bb
= NULL
;
6093 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6094 expand_omp_for_init_counts (fd
, &si
, entry_bb
, counts
,
6095 fin_bb
, first_zero_iter
,
6099 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6100 t
= integer_one_node
;
6102 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6103 fold_convert (type
, fd
->loop
.n1
),
6104 fold_convert (type
, fd
->loop
.n2
));
6105 if (fd
->collapse
== 1
6106 && TYPE_UNSIGNED (type
)
6107 && (t
== NULL_TREE
|| !integer_onep (t
)))
6109 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6110 n1
= force_gimple_operand_gsi (&si
, n1
, true, NULL_TREE
,
6111 true, GSI_SAME_STMT
);
6112 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6113 n2
= force_gimple_operand_gsi (&si
, n2
, true, NULL_TREE
,
6114 true, GSI_SAME_STMT
);
6115 stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6116 NULL_TREE
, NULL_TREE
);
6117 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
6118 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
6119 expand_omp_regimplify_p
, NULL
, NULL
)
6120 || walk_tree (gimple_cond_rhs_ptr (stmt
),
6121 expand_omp_regimplify_p
, NULL
, NULL
))
6123 si
= gsi_for_stmt (stmt
);
6124 gimple_regimplify_operands (stmt
, &si
);
6126 se
= split_block (entry_bb
, stmt
);
6127 se
->flags
= EDGE_TRUE_VALUE
;
6128 entry_bb
= se
->dest
;
6129 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6130 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6131 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6132 if (gimple_in_ssa_p (cfun
))
6134 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6135 for (si
= gsi_start_phis (fin_bb
);
6136 !gsi_end_p (si
); gsi_next (&si
))
6138 gimple phi
= gsi_stmt (si
);
6139 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6140 se
, UNKNOWN_LOCATION
);
6143 si
= gsi_last_bb (entry_bb
);
6146 t
= build_call_expr (builtin_decl_explicit (get_num_threads
), 0);
6147 t
= fold_convert (itype
, t
);
6148 nthreads
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6149 true, GSI_SAME_STMT
);
6151 t
= build_call_expr (builtin_decl_explicit (get_thread_num
), 0);
6152 t
= fold_convert (itype
, t
);
6153 threadid
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6154 true, GSI_SAME_STMT
);
6158 step
= fd
->loop
.step
;
6159 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6161 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6162 OMP_CLAUSE__LOOPTEMP_
);
6163 gcc_assert (innerc
);
6164 n1
= OMP_CLAUSE_DECL (innerc
);
6165 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6166 OMP_CLAUSE__LOOPTEMP_
);
6167 gcc_assert (innerc
);
6168 n2
= OMP_CLAUSE_DECL (innerc
);
6170 n1
= force_gimple_operand_gsi (&si
, fold_convert (type
, n1
),
6171 true, NULL_TREE
, true, GSI_SAME_STMT
);
6172 n2
= force_gimple_operand_gsi (&si
, fold_convert (itype
, n2
),
6173 true, NULL_TREE
, true, GSI_SAME_STMT
);
6174 step
= force_gimple_operand_gsi (&si
, fold_convert (itype
, step
),
6175 true, NULL_TREE
, true, GSI_SAME_STMT
);
6177 = force_gimple_operand_gsi (&si
, fold_convert (itype
, fd
->chunk_size
),
6178 true, NULL_TREE
, true, GSI_SAME_STMT
);
6180 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
6181 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
6182 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
6183 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
6184 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
6185 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6186 fold_build1 (NEGATE_EXPR
, itype
, t
),
6187 fold_build1 (NEGATE_EXPR
, itype
, step
));
6189 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
6190 t
= fold_convert (itype
, t
);
6191 n
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6192 true, GSI_SAME_STMT
);
6194 trip_var
= create_tmp_reg (itype
, ".trip");
6195 if (gimple_in_ssa_p (cfun
))
6197 trip_init
= make_ssa_name (trip_var
, NULL
);
6198 trip_main
= make_ssa_name (trip_var
, NULL
);
6199 trip_back
= make_ssa_name (trip_var
, NULL
);
6203 trip_init
= trip_var
;
6204 trip_main
= trip_var
;
6205 trip_back
= trip_var
;
6208 stmt
= gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
6209 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
6211 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, fd
->chunk_size
);
6212 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6213 if (POINTER_TYPE_P (type
))
6214 t
= fold_build_pointer_plus (n1
, t
);
6216 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6217 v_extra
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6218 true, GSI_SAME_STMT
);
6220 /* Remove the GIMPLE_OMP_FOR. */
6221 gsi_remove (&si
, true);
6223 /* Iteration space partitioning goes in ITER_PART_BB. */
6224 si
= gsi_last_bb (iter_part_bb
);
6226 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
6227 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
6228 t
= fold_build2 (MULT_EXPR
, itype
, t
, fd
->chunk_size
);
6229 s0
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6230 false, GSI_CONTINUE_LINKING
);
6232 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, fd
->chunk_size
);
6233 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
6234 e0
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6235 false, GSI_CONTINUE_LINKING
);
6237 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
6238 gsi_insert_after (&si
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
6240 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6241 si
= gsi_start_bb (seq_start_bb
);
6243 tree startvar
= fd
->loop
.v
;
6244 tree endvar
= NULL_TREE
;
6246 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6248 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6249 ? gimple_omp_parallel_clauses (inner_stmt
)
6250 : gimple_omp_for_clauses (inner_stmt
);
6251 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6252 gcc_assert (innerc
);
6253 startvar
= OMP_CLAUSE_DECL (innerc
);
6254 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6255 OMP_CLAUSE__LOOPTEMP_
);
6256 gcc_assert (innerc
);
6257 endvar
= OMP_CLAUSE_DECL (innerc
);
6260 t
= fold_convert (itype
, s0
);
6261 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6262 if (POINTER_TYPE_P (type
))
6263 t
= fold_build_pointer_plus (n1
, t
);
6265 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6266 t
= fold_convert (TREE_TYPE (startvar
), t
);
6267 t
= force_gimple_operand_gsi (&si
, t
,
6269 && TREE_ADDRESSABLE (startvar
),
6270 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6271 stmt
= gimple_build_assign (startvar
, t
);
6272 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
6274 t
= fold_convert (itype
, e0
);
6275 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6276 if (POINTER_TYPE_P (type
))
6277 t
= fold_build_pointer_plus (n1
, t
);
6279 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6280 t
= fold_convert (TREE_TYPE (startvar
), t
);
6281 e
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6282 false, GSI_CONTINUE_LINKING
);
6285 stmt
= gimple_build_assign (endvar
, e
);
6286 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
6288 if (fd
->collapse
> 1)
6289 expand_omp_for_init_vars (fd
, &si
, counts
, inner_stmt
, startvar
);
6293 /* The code controlling the sequential loop goes in CONT_BB,
6294 replacing the GIMPLE_OMP_CONTINUE. */
6295 si
= gsi_last_bb (cont_bb
);
6296 stmt
= gsi_stmt (si
);
6297 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6298 v_main
= gimple_omp_continue_control_use (stmt
);
6299 v_back
= gimple_omp_continue_control_def (stmt
);
6301 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6303 if (POINTER_TYPE_P (type
))
6304 t
= fold_build_pointer_plus (v_main
, step
);
6306 t
= fold_build2 (PLUS_EXPR
, type
, v_main
, step
);
6307 if (DECL_P (v_back
) && TREE_ADDRESSABLE (v_back
))
6308 t
= force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
,
6309 true, GSI_SAME_STMT
);
6310 stmt
= gimple_build_assign (v_back
, t
);
6311 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
6313 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6314 DECL_P (v_back
) && TREE_ADDRESSABLE (v_back
)
6316 gsi_insert_before (&si
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6319 /* Remove GIMPLE_OMP_CONTINUE. */
6320 gsi_remove (&si
, true);
6322 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6323 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6325 /* Trip update code goes into TRIP_UPDATE_BB. */
6326 si
= gsi_start_bb (trip_update_bb
);
6328 t
= build_int_cst (itype
, 1);
6329 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
6330 stmt
= gimple_build_assign (trip_back
, t
);
6331 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
6334 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6335 si
= gsi_last_bb (exit_bb
);
6336 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
6338 t
= gimple_omp_return_lhs (gsi_stmt (si
));
6339 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
6341 gsi_remove (&si
, true);
6343 /* Connect the new blocks. */
6344 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
6345 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
6349 se
= find_edge (cont_bb
, body_bb
);
6350 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6355 else if (fd
->collapse
> 1)
6358 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6361 se
->flags
= EDGE_TRUE_VALUE
;
6362 find_edge (cont_bb
, trip_update_bb
)->flags
6363 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6365 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
6368 if (gimple_in_ssa_p (cfun
))
6370 gimple_stmt_iterator psi
;
6373 edge_var_map_vector
*head
;
6377 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
6379 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
6380 remove arguments of the phi nodes in fin_bb. We need to create
6381 appropriate phi nodes in iter_part_bb instead. */
6382 se
= single_pred_edge (fin_bb
);
6383 re
= single_succ_edge (trip_update_bb
);
6384 head
= redirect_edge_var_map_vector (re
);
6385 ene
= single_succ_edge (entry_bb
);
6387 psi
= gsi_start_phis (fin_bb
);
6388 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
6389 gsi_next (&psi
), ++i
)
6392 source_location locus
;
6394 phi
= gsi_stmt (psi
);
6395 t
= gimple_phi_result (phi
);
6396 gcc_assert (t
== redirect_edge_var_map_result (vm
));
6397 nphi
= create_phi_node (t
, iter_part_bb
);
6399 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
6400 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
6402 /* A special case -- fd->loop.v is not yet computed in
6403 iter_part_bb, we need to use v_extra instead. */
6404 if (t
== fd
->loop
.v
)
6406 add_phi_arg (nphi
, t
, ene
, locus
);
6407 locus
= redirect_edge_var_map_location (vm
);
6408 add_phi_arg (nphi
, redirect_edge_var_map_def (vm
), re
, locus
);
6410 gcc_assert (!gsi_end_p (psi
) && i
== head
->length ());
6411 redirect_edge_var_map_clear (re
);
6414 psi
= gsi_start_phis (fin_bb
);
6415 if (gsi_end_p (psi
))
6417 remove_phi_node (&psi
, false);
6420 /* Make phi node for trip. */
6421 phi
= create_phi_node (trip_main
, iter_part_bb
);
6422 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
6424 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
6429 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
6430 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
6431 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
6432 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6433 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6434 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
6435 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
6436 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6437 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6441 struct loop
*trip_loop
= alloc_loop ();
6442 trip_loop
->header
= iter_part_bb
;
6443 trip_loop
->latch
= trip_update_bb
;
6444 add_loop (trip_loop
, iter_part_bb
->loop_father
);
6446 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6448 struct loop
*loop
= alloc_loop ();
6449 loop
->header
= body_bb
;
6450 loop
->latch
= cont_bb
;
6451 add_loop (loop
, trip_loop
);
6457 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
6458 loop. Given parameters:
6460 for (V = N1; V cond N2; V += STEP) BODY;
6462 where COND is "<" or ">", we generate pseudocode
6470 if (V cond N2) goto L0; else goto L2;
6473 For collapsed loops, given parameters:
6475 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
6476 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
6477 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
6480 we generate pseudocode
6486 count3 = (adj + N32 - N31) / STEP3;
6491 count2 = (adj + N22 - N21) / STEP2;
6496 count1 = (adj + N12 - N11) / STEP1;
6497 count = count1 * count2 * count3;
6507 V2 += (V3 cond3 N32) ? 0 : STEP2;
6508 V3 = (V3 cond3 N32) ? V3 : N31;
6509 V1 += (V2 cond2 N22) ? 0 : STEP1;
6510 V2 = (V2 cond2 N22) ? V2 : N21;
6512 if (V < count) goto L0; else goto L2;
6518 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
6521 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
6522 gimple_stmt_iterator gsi
;
6524 bool broken_loop
= region
->cont
== NULL
;
6526 tree
*counts
= NULL
;
6528 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6529 OMP_CLAUSE_SAFELEN
);
6530 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6531 OMP_CLAUSE__SIMDUID_
);
6534 type
= TREE_TYPE (fd
->loop
.v
);
6535 entry_bb
= region
->entry
;
6536 cont_bb
= region
->cont
;
6537 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6538 gcc_assert (broken_loop
6539 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
6540 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
6543 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
6544 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6545 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
6546 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6550 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
6551 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
6552 l2_bb
= single_succ (l1_bb
);
6554 exit_bb
= region
->exit
;
6557 gsi
= gsi_last_bb (entry_bb
);
6559 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6560 /* Not needed in SSA form right now. */
6561 gcc_assert (!gimple_in_ssa_p (cfun
));
6562 if (fd
->collapse
> 1)
6564 int first_zero_iter
= -1;
6565 basic_block zero_iter_bb
= l2_bb
;
6567 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6568 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6569 zero_iter_bb
, first_zero_iter
,
6572 if (l2_dom_bb
== NULL
)
6577 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6579 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6580 OMP_CLAUSE__LOOPTEMP_
);
6581 gcc_assert (innerc
);
6582 n1
= OMP_CLAUSE_DECL (innerc
);
6583 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6584 OMP_CLAUSE__LOOPTEMP_
);
6585 gcc_assert (innerc
);
6586 n2
= OMP_CLAUSE_DECL (innerc
);
6587 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
6588 fold_convert (type
, n1
));
6589 if (fd
->collapse
> 1)
6592 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
6598 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
6599 fold_convert (type
, fd
->loop
.n1
));
6600 if (fd
->collapse
> 1)
6601 for (i
= 0; i
< fd
->collapse
; i
++)
6603 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
6604 if (POINTER_TYPE_P (itype
))
6605 itype
= signed_type_for (itype
);
6606 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
6607 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6611 /* Remove the GIMPLE_OMP_FOR statement. */
6612 gsi_remove (&gsi
, true);
6616 /* Code to control the increment goes in the CONT_BB. */
6617 gsi
= gsi_last_bb (cont_bb
);
6618 stmt
= gsi_stmt (gsi
);
6619 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6621 if (POINTER_TYPE_P (type
))
6622 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
6624 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
6625 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
6627 if (fd
->collapse
> 1)
6629 i
= fd
->collapse
- 1;
6630 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
6632 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
6633 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
6637 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
6639 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
6642 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6644 for (i
= fd
->collapse
- 1; i
> 0; i
--)
6646 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
6647 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
6648 if (POINTER_TYPE_P (itype2
))
6649 itype2
= signed_type_for (itype2
);
6650 t
= build3 (COND_EXPR
, itype2
,
6651 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
6653 fold_convert (itype
, fd
->loops
[i
].n2
)),
6654 build_int_cst (itype2
, 0),
6655 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
6656 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
6657 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
6659 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
6660 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
6662 t
= build3 (COND_EXPR
, itype
,
6663 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
6665 fold_convert (itype
, fd
->loops
[i
].n2
)),
6667 fold_convert (itype
, fd
->loops
[i
].n1
));
6668 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6672 /* Remove GIMPLE_OMP_CONTINUE. */
6673 gsi_remove (&gsi
, true);
6676 /* Emit the condition in L1_BB. */
6677 gsi
= gsi_start_bb (l1_bb
);
6679 t
= fold_convert (type
, n2
);
6680 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6681 false, GSI_CONTINUE_LINKING
);
6682 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, fd
->loop
.v
, t
);
6683 stmt
= gimple_build_cond_empty (t
);
6684 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6685 if (walk_tree (gimple_cond_lhs_ptr (stmt
), expand_omp_regimplify_p
,
6687 || walk_tree (gimple_cond_rhs_ptr (stmt
), expand_omp_regimplify_p
,
6690 gsi
= gsi_for_stmt (stmt
);
6691 gimple_regimplify_operands (stmt
, &gsi
);
6694 /* Remove GIMPLE_OMP_RETURN. */
6695 gsi
= gsi_last_bb (exit_bb
);
6696 gsi_remove (&gsi
, true);
6698 /* Connect the new blocks. */
6699 remove_edge (FALLTHRU_EDGE (entry_bb
));
6703 remove_edge (BRANCH_EDGE (entry_bb
));
6704 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
6706 e
= BRANCH_EDGE (l1_bb
);
6707 ne
= FALLTHRU_EDGE (l1_bb
);
6708 e
->flags
= EDGE_TRUE_VALUE
;
6712 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
6714 ne
= single_succ_edge (l1_bb
);
6715 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
6718 ne
->flags
= EDGE_FALSE_VALUE
;
6719 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
6720 ne
->probability
= REG_BR_PROB_BASE
/ 8;
6722 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
6723 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
6724 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
6728 struct loop
*loop
= alloc_loop ();
6729 loop
->header
= l1_bb
;
6730 loop
->latch
= e
->dest
;
6731 add_loop (loop
, l1_bb
->loop_father
);
6732 if (safelen
== NULL_TREE
)
6733 loop
->safelen
= INT_MAX
;
6736 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
6737 if (!host_integerp (safelen
, 1)
6738 || (unsigned HOST_WIDE_INT
) tree_low_cst (safelen
, 1)
6740 loop
->safelen
= INT_MAX
;
6742 loop
->safelen
= tree_low_cst (safelen
, 1);
6743 if (loop
->safelen
== 1)
6748 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
6749 cfun
->has_simduid_loops
= true;
6751 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
6753 if ((flag_tree_loop_vectorize
6754 || (!global_options_set
.x_flag_tree_loop_vectorize
6755 && !global_options_set
.x_flag_tree_vectorize
))
6756 && loop
->safelen
> 1)
6758 loop
->force_vect
= true;
6759 cfun
->has_force_vect_loops
= true;
6765 /* Expand the OpenMP loop defined by REGION. */
6768 expand_omp_for (struct omp_region
*region
, gimple inner_stmt
)
6770 struct omp_for_data fd
;
6771 struct omp_for_data_loop
*loops
;
6774 = (struct omp_for_data_loop
*)
6775 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
6776 * sizeof (struct omp_for_data_loop
));
6777 extract_omp_for_data (last_stmt (region
->entry
), &fd
, loops
);
6778 region
->sched_kind
= fd
.sched_kind
;
6780 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
6781 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
6782 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
6785 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
6786 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
6787 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
6790 /* If there isn't a continue then this is a degerate case where
6791 the introduction of abnormal edges during lowering will prevent
6792 original loops from being detected. Fix that up. */
6793 loops_state_set (LOOPS_NEED_FIXUP
);
6795 if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_SIMD
)
6796 expand_omp_simd (region
, &fd
);
6797 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
6798 && !fd
.have_ordered
)
6800 if (fd
.chunk_size
== NULL
)
6801 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
6803 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
6807 int fn_index
, start_ix
, next_ix
;
6809 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
6810 == GF_OMP_FOR_KIND_FOR
);
6811 if (fd
.chunk_size
== NULL
6812 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
6813 fd
.chunk_size
= integer_zero_node
;
6814 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
6815 fn_index
= (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
6816 ? 3 : fd
.sched_kind
;
6817 fn_index
+= fd
.have_ordered
* 4;
6818 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
6819 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
6820 if (fd
.iter_type
== long_long_unsigned_type_node
)
6822 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
6823 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
6824 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
6825 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
6827 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
6828 (enum built_in_function
) next_ix
, inner_stmt
);
6831 if (gimple_in_ssa_p (cfun
))
6832 update_ssa (TODO_update_ssa_only_virtuals
);
6836 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
6838 v = GOMP_sections_start (n);
6855 v = GOMP_sections_next ();
6860 If this is a combined parallel sections, replace the call to
6861 GOMP_sections_start with call to GOMP_sections_next. */
6864 expand_omp_sections (struct omp_region
*region
)
6866 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
6867 vec
<tree
> label_vec
;
6869 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
6870 gimple_stmt_iterator si
, switch_si
;
6871 gimple sections_stmt
, stmt
, cont
;
6874 struct omp_region
*inner
;
6876 bool exit_reachable
= region
->cont
!= NULL
;
6878 gcc_assert (region
->exit
!= NULL
);
6879 entry_bb
= region
->entry
;
6880 l0_bb
= single_succ (entry_bb
);
6881 l1_bb
= region
->cont
;
6882 l2_bb
= region
->exit
;
6883 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
6884 l2
= gimple_block_label (l2_bb
);
6887 /* This can happen if there are reductions. */
6888 len
= EDGE_COUNT (l0_bb
->succs
);
6889 gcc_assert (len
> 0);
6890 e
= EDGE_SUCC (l0_bb
, len
- 1);
6891 si
= gsi_last_bb (e
->dest
);
6894 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
6895 l2
= gimple_block_label (e
->dest
);
6897 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
6899 si
= gsi_last_bb (e
->dest
);
6901 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
6903 l2
= gimple_block_label (e
->dest
);
6909 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
6911 default_bb
= create_empty_bb (l0_bb
);
6913 /* We will build a switch() with enough cases for all the
6914 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
6915 and a default case to abort if something goes wrong. */
6916 len
= EDGE_COUNT (l0_bb
->succs
);
6918 /* Use vec::quick_push on label_vec throughout, since we know the size
6920 label_vec
.create (len
);
6922 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
6923 GIMPLE_OMP_SECTIONS statement. */
6924 si
= gsi_last_bb (entry_bb
);
6925 sections_stmt
= gsi_stmt (si
);
6926 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
6927 vin
= gimple_omp_sections_control (sections_stmt
);
6928 if (!is_combined_parallel (region
))
6930 /* If we are not inside a combined parallel+sections region,
6931 call GOMP_sections_start. */
6932 t
= build_int_cst (unsigned_type_node
, len
- 1);
6933 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
6934 stmt
= gimple_build_call (u
, 1, t
);
6938 /* Otherwise, call GOMP_sections_next. */
6939 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
6940 stmt
= gimple_build_call (u
, 0);
6942 gimple_call_set_lhs (stmt
, vin
);
6943 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
6944 gsi_remove (&si
, true);
6946 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
6948 switch_si
= gsi_last_bb (l0_bb
);
6949 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
6952 cont
= last_stmt (l1_bb
);
6953 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
6954 vmain
= gimple_omp_continue_control_use (cont
);
6955 vnext
= gimple_omp_continue_control_def (cont
);
6963 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
6964 label_vec
.quick_push (t
);
6967 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
6968 for (inner
= region
->inner
, casei
= 1;
6970 inner
= inner
->next
, i
++, casei
++)
6972 basic_block s_entry_bb
, s_exit_bb
;
6974 /* Skip optional reduction region. */
6975 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
6982 s_entry_bb
= inner
->entry
;
6983 s_exit_bb
= inner
->exit
;
6985 t
= gimple_block_label (s_entry_bb
);
6986 u
= build_int_cst (unsigned_type_node
, casei
);
6987 u
= build_case_label (u
, NULL
, t
);
6988 label_vec
.quick_push (u
);
6990 si
= gsi_last_bb (s_entry_bb
);
6991 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
6992 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
6993 gsi_remove (&si
, true);
6994 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
6996 if (s_exit_bb
== NULL
)
6999 si
= gsi_last_bb (s_exit_bb
);
7000 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
7001 gsi_remove (&si
, true);
7003 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
7006 /* Error handling code goes in DEFAULT_BB. */
7007 t
= gimple_block_label (default_bb
);
7008 u
= build_case_label (NULL
, NULL
, t
);
7009 make_edge (l0_bb
, default_bb
, 0);
7011 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
7013 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
7014 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
7015 gsi_remove (&switch_si
, true);
7016 label_vec
.release ();
7018 si
= gsi_start_bb (default_bb
);
7019 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
7020 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
7026 /* Code to get the next section goes in L1_BB. */
7027 si
= gsi_last_bb (l1_bb
);
7028 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
7030 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
7031 stmt
= gimple_build_call (bfn_decl
, 0);
7032 gimple_call_set_lhs (stmt
, vnext
);
7033 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7034 gsi_remove (&si
, true);
7036 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
7039 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
7040 si
= gsi_last_bb (l2_bb
);
7041 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
7042 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
7043 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
7044 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
7046 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
7047 stmt
= gimple_build_call (t
, 0);
7048 if (gimple_omp_return_lhs (gsi_stmt (si
)))
7049 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
7050 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7051 gsi_remove (&si
, true);
7053 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
7057 /* Expand code for an OpenMP single directive. We've already expanded
7058 much of the code, here we simply place the GOMP_barrier call. */
7061 expand_omp_single (struct omp_region
*region
)
7063 basic_block entry_bb
, exit_bb
;
7064 gimple_stmt_iterator si
;
7066 entry_bb
= region
->entry
;
7067 exit_bb
= region
->exit
;
7069 si
= gsi_last_bb (entry_bb
);
7070 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
7071 gsi_remove (&si
, true);
7072 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7074 si
= gsi_last_bb (exit_bb
);
7075 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
7077 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
7078 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
7080 gsi_remove (&si
, true);
7081 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
7085 /* Generic expansion for OpenMP synchronization directives: master,
7086 ordered and critical. All we need to do here is remove the entry
7087 and exit markers for REGION. */
7090 expand_omp_synch (struct omp_region
*region
)
7092 basic_block entry_bb
, exit_bb
;
7093 gimple_stmt_iterator si
;
7095 entry_bb
= region
->entry
;
7096 exit_bb
= region
->exit
;
7098 si
= gsi_last_bb (entry_bb
);
7099 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
7100 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
7101 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
7102 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
7103 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
7104 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
7105 gsi_remove (&si
, true);
7106 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7110 si
= gsi_last_bb (exit_bb
);
7111 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
7112 gsi_remove (&si
, true);
7113 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
7117 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7118 operation as a normal volatile load. */
7121 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
7122 tree loaded_val
, int index
)
7124 enum built_in_function tmpbase
;
7125 gimple_stmt_iterator gsi
;
7126 basic_block store_bb
;
7129 tree decl
, call
, type
, itype
;
7131 gsi
= gsi_last_bb (load_bb
);
7132 stmt
= gsi_stmt (gsi
);
7133 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7134 loc
= gimple_location (stmt
);
7136 /* ??? If the target does not implement atomic_load_optab[mode], and mode
7137 is smaller than word size, then expand_atomic_load assumes that the load
7138 is atomic. We could avoid the builtin entirely in this case. */
7140 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
7141 decl
= builtin_decl_explicit (tmpbase
);
7142 if (decl
== NULL_TREE
)
7145 type
= TREE_TYPE (loaded_val
);
7146 itype
= TREE_TYPE (TREE_TYPE (decl
));
7148 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
7149 build_int_cst (NULL
,
7150 gimple_omp_atomic_seq_cst_p (stmt
)
7152 : MEMMODEL_RELAXED
));
7153 if (!useless_type_conversion_p (type
, itype
))
7154 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7155 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7157 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7158 gsi_remove (&gsi
, true);
7160 store_bb
= single_succ (load_bb
);
7161 gsi
= gsi_last_bb (store_bb
);
7162 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7163 gsi_remove (&gsi
, true);
7165 if (gimple_in_ssa_p (cfun
))
7166 update_ssa (TODO_update_ssa_no_phi
);
7171 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7172 operation as a normal volatile store. */
7175 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
7176 tree loaded_val
, tree stored_val
, int index
)
7178 enum built_in_function tmpbase
;
7179 gimple_stmt_iterator gsi
;
7180 basic_block store_bb
= single_succ (load_bb
);
7183 tree decl
, call
, type
, itype
;
7184 enum machine_mode imode
;
7187 gsi
= gsi_last_bb (load_bb
);
7188 stmt
= gsi_stmt (gsi
);
7189 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7191 /* If the load value is needed, then this isn't a store but an exchange. */
7192 exchange
= gimple_omp_atomic_need_value_p (stmt
);
7194 gsi
= gsi_last_bb (store_bb
);
7195 stmt
= gsi_stmt (gsi
);
7196 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
7197 loc
= gimple_location (stmt
);
7199 /* ??? If the target does not implement atomic_store_optab[mode], and mode
7200 is smaller than word size, then expand_atomic_store assumes that the store
7201 is atomic. We could avoid the builtin entirely in this case. */
7203 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
7204 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
7205 decl
= builtin_decl_explicit (tmpbase
);
7206 if (decl
== NULL_TREE
)
7209 type
= TREE_TYPE (stored_val
);
7211 /* Dig out the type of the function's second argument. */
7212 itype
= TREE_TYPE (decl
);
7213 itype
= TYPE_ARG_TYPES (itype
);
7214 itype
= TREE_CHAIN (itype
);
7215 itype
= TREE_VALUE (itype
);
7216 imode
= TYPE_MODE (itype
);
7218 if (exchange
&& !can_atomic_exchange_p (imode
, true))
7221 if (!useless_type_conversion_p (itype
, type
))
7222 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
7223 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
7224 build_int_cst (NULL
,
7225 gimple_omp_atomic_seq_cst_p (stmt
)
7227 : MEMMODEL_RELAXED
));
7230 if (!useless_type_conversion_p (type
, itype
))
7231 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7232 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7235 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7236 gsi_remove (&gsi
, true);
7238 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
7239 gsi
= gsi_last_bb (load_bb
);
7240 gsi_remove (&gsi
, true);
7242 if (gimple_in_ssa_p (cfun
))
7243 update_ssa (TODO_update_ssa_no_phi
);
7248 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7249 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
7250 size of the data type, and thus usable to find the index of the builtin
7251 decl. Returns false if the expression is not of the proper form. */
7254 expand_omp_atomic_fetch_op (basic_block load_bb
,
7255 tree addr
, tree loaded_val
,
7256 tree stored_val
, int index
)
7258 enum built_in_function oldbase
, newbase
, tmpbase
;
7259 tree decl
, itype
, call
;
7261 basic_block store_bb
= single_succ (load_bb
);
7262 gimple_stmt_iterator gsi
;
7265 enum tree_code code
;
7266 bool need_old
, need_new
;
7267 enum machine_mode imode
;
7270 /* We expect to find the following sequences:
7273 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
7276 val = tmp OP something; (or: something OP tmp)
7277 GIMPLE_OMP_STORE (val)
7279 ???FIXME: Allow a more flexible sequence.
7280 Perhaps use data flow to pick the statements.
7284 gsi
= gsi_after_labels (store_bb
);
7285 stmt
= gsi_stmt (gsi
);
7286 loc
= gimple_location (stmt
);
7287 if (!is_gimple_assign (stmt
))
7290 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
7292 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
7293 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
7294 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
7295 gcc_checking_assert (!need_old
|| !need_new
);
7297 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
7300 /* Check for one of the supported fetch-op operations. */
7301 code
= gimple_assign_rhs_code (stmt
);
7305 case POINTER_PLUS_EXPR
:
7306 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
7307 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
7310 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
7311 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
7314 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
7315 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
7318 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
7319 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
7322 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
7323 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
7329 /* Make sure the expression is of the proper form. */
7330 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
7331 rhs
= gimple_assign_rhs2 (stmt
);
7332 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
7333 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
7334 rhs
= gimple_assign_rhs1 (stmt
);
7338 tmpbase
= ((enum built_in_function
)
7339 ((need_new
? newbase
: oldbase
) + index
+ 1));
7340 decl
= builtin_decl_explicit (tmpbase
);
7341 if (decl
== NULL_TREE
)
7343 itype
= TREE_TYPE (TREE_TYPE (decl
));
7344 imode
= TYPE_MODE (itype
);
7346 /* We could test all of the various optabs involved, but the fact of the
7347 matter is that (with the exception of i486 vs i586 and xadd) all targets
7348 that support any atomic operaton optab also implements compare-and-swap.
7349 Let optabs.c take care of expanding any compare-and-swap loop. */
7350 if (!can_compare_and_swap_p (imode
, true))
7353 gsi
= gsi_last_bb (load_bb
);
7354 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7356 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
7357 It only requires that the operation happen atomically. Thus we can
7358 use the RELAXED memory model. */
7359 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
7360 fold_convert_loc (loc
, itype
, rhs
),
7361 build_int_cst (NULL
,
7362 seq_cst
? MEMMODEL_SEQ_CST
7363 : MEMMODEL_RELAXED
));
7365 if (need_old
|| need_new
)
7367 lhs
= need_old
? loaded_val
: stored_val
;
7368 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
7369 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
7372 call
= fold_convert_loc (loc
, void_type_node
, call
);
7373 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7374 gsi_remove (&gsi
, true);
7376 gsi
= gsi_last_bb (store_bb
);
7377 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7378 gsi_remove (&gsi
, true);
7379 gsi
= gsi_last_bb (store_bb
);
7380 gsi_remove (&gsi
, true);
7382 if (gimple_in_ssa_p (cfun
))
7383 update_ssa (TODO_update_ssa_no_phi
);
7388 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7392 newval = rhs; // with oldval replacing *addr in rhs
7393 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
7394 if (oldval != newval)
7397 INDEX is log2 of the size of the data type, and thus usable to find the
7398 index of the builtin decl. */
7401 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
7402 tree addr
, tree loaded_val
, tree stored_val
,
7405 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
7406 tree type
, itype
, cmpxchg
, iaddr
;
7407 gimple_stmt_iterator si
;
7408 basic_block loop_header
= single_succ (load_bb
);
7411 enum built_in_function fncode
;
7413 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
7414 order to use the RELAXED memory model effectively. */
7415 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
7417 cmpxchg
= builtin_decl_explicit (fncode
);
7418 if (cmpxchg
== NULL_TREE
)
7420 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
7421 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
7423 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
7426 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
7427 si
= gsi_last_bb (load_bb
);
7428 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7430 /* For floating-point values, we'll need to view-convert them to integers
7431 so that we can perform the atomic compare and swap. Simplify the
7432 following code by always setting up the "i"ntegral variables. */
7433 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
7437 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
7440 = force_gimple_operand_gsi (&si
,
7441 fold_convert (TREE_TYPE (iaddr
), addr
),
7442 false, NULL_TREE
, true, GSI_SAME_STMT
);
7443 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
7444 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7445 loadedi
= create_tmp_var (itype
, NULL
);
7446 if (gimple_in_ssa_p (cfun
))
7447 loadedi
= make_ssa_name (loadedi
, NULL
);
7452 loadedi
= loaded_val
;
7456 = force_gimple_operand_gsi (&si
,
7457 build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)),
7459 build_int_cst (TREE_TYPE (iaddr
), 0)),
7460 true, NULL_TREE
, true, GSI_SAME_STMT
);
7462 /* Move the value to the LOADEDI temporary. */
7463 if (gimple_in_ssa_p (cfun
))
7465 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
7466 phi
= create_phi_node (loadedi
, loop_header
);
7467 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
7471 gsi_insert_before (&si
,
7472 gimple_build_assign (loadedi
, initial
),
7474 if (loadedi
!= loaded_val
)
7476 gimple_stmt_iterator gsi2
;
7479 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
7480 gsi2
= gsi_start_bb (loop_header
);
7481 if (gimple_in_ssa_p (cfun
))
7484 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
7485 true, GSI_SAME_STMT
);
7486 stmt
= gimple_build_assign (loaded_val
, x
);
7487 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
7491 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
7492 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
7493 true, GSI_SAME_STMT
);
7496 gsi_remove (&si
, true);
7498 si
= gsi_last_bb (store_bb
);
7499 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
7502 storedi
= stored_val
;
7505 force_gimple_operand_gsi (&si
,
7506 build1 (VIEW_CONVERT_EXPR
, itype
,
7507 stored_val
), true, NULL_TREE
, true,
7510 /* Build the compare&swap statement. */
7511 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
7512 new_storedi
= force_gimple_operand_gsi (&si
,
7513 fold_convert (TREE_TYPE (loadedi
),
7516 true, GSI_SAME_STMT
);
7518 if (gimple_in_ssa_p (cfun
))
7522 old_vali
= create_tmp_var (TREE_TYPE (loadedi
), NULL
);
7523 stmt
= gimple_build_assign (old_vali
, loadedi
);
7524 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7526 stmt
= gimple_build_assign (loadedi
, new_storedi
);
7527 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7530 /* Note that we always perform the comparison as an integer, even for
7531 floating point. This allows the atomic operation to properly
7532 succeed even with NaNs and -0.0. */
7533 stmt
= gimple_build_cond_empty
7534 (build2 (NE_EXPR
, boolean_type_node
,
7535 new_storedi
, old_vali
));
7536 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7539 e
= single_succ_edge (store_bb
);
7540 e
->flags
&= ~EDGE_FALLTHRU
;
7541 e
->flags
|= EDGE_FALSE_VALUE
;
7543 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
7545 /* Copy the new value to loadedi (we already did that before the condition
7546 if we are not in SSA). */
7547 if (gimple_in_ssa_p (cfun
))
7549 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
7550 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
7553 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
7554 gsi_remove (&si
, true);
7556 struct loop
*loop
= alloc_loop ();
7557 loop
->header
= loop_header
;
7558 loop
->latch
= store_bb
;
7559 add_loop (loop
, loop_header
->loop_father
);
7561 if (gimple_in_ssa_p (cfun
))
7562 update_ssa (TODO_update_ssa_no_phi
);
7567 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7569 GOMP_atomic_start ();
7573 The result is not globally atomic, but works so long as all parallel
7574 references are within #pragma omp atomic directives. According to
7575 responses received from omp@openmp.org, appears to be within spec.
7576 Which makes sense, since that's how several other compilers handle
7577 this situation as well.
7578 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
7579 expanding. STORED_VAL is the operand of the matching
7580 GIMPLE_OMP_ATOMIC_STORE.
7583 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
7587 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
7592 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
7593 tree addr
, tree loaded_val
, tree stored_val
)
7595 gimple_stmt_iterator si
;
7599 si
= gsi_last_bb (load_bb
);
7600 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7602 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
7603 t
= build_call_expr (t
, 0);
7604 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7606 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
7607 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7608 gsi_remove (&si
, true);
7610 si
= gsi_last_bb (store_bb
);
7611 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
7613 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
7615 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7617 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
7618 t
= build_call_expr (t
, 0);
7619 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7620 gsi_remove (&si
, true);
7622 if (gimple_in_ssa_p (cfun
))
7623 update_ssa (TODO_update_ssa_no_phi
);
7627 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
7628 using expand_omp_atomic_fetch_op. If it failed, we try to
7629 call expand_omp_atomic_pipeline, and if it fails too, the
7630 ultimate fallback is wrapping the operation in a mutex
7631 (expand_omp_atomic_mutex). REGION is the atomic region built
7632 by build_omp_regions_1(). */
7635 expand_omp_atomic (struct omp_region
*region
)
7637 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
7638 gimple load
= last_stmt (load_bb
), store
= last_stmt (store_bb
);
7639 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
7640 tree addr
= gimple_omp_atomic_load_rhs (load
);
7641 tree stored_val
= gimple_omp_atomic_store_val (store
);
7642 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
7643 HOST_WIDE_INT index
;
7645 /* Make sure the type is one of the supported sizes. */
7646 index
= tree_low_cst (TYPE_SIZE_UNIT (type
), 1);
7647 index
= exact_log2 (index
);
7648 if (index
>= 0 && index
<= 4)
7650 unsigned int align
= TYPE_ALIGN_UNIT (type
);
7652 /* __sync builtins require strict data alignment. */
7653 if (exact_log2 (align
) >= index
)
7656 if (loaded_val
== stored_val
7657 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
7658 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
7659 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
7660 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
7664 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
7665 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
7666 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
7667 && store_bb
== single_succ (load_bb
)
7668 && first_stmt (store_bb
) == store
7669 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
7673 /* When possible, use specialized atomic update functions. */
7674 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
7675 && store_bb
== single_succ (load_bb
)
7676 && expand_omp_atomic_fetch_op (load_bb
, addr
,
7677 loaded_val
, stored_val
, index
))
7680 /* If we don't have specialized __sync builtins, try and implement
7681 as a compare and swap loop. */
7682 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
7683 loaded_val
, stored_val
, index
))
7688 /* The ultimate fallback is wrapping the operation in a mutex. */
7689 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
7693 /* Expand the OpenMP target{, data, update} directive starting at REGION. */
7696 expand_omp_target (struct omp_region
*region
)
7698 basic_block entry_bb
, exit_bb
, new_bb
;
7699 struct function
*child_cfun
= NULL
;
7700 tree child_fn
= NULL_TREE
, block
, t
;
7701 gimple_stmt_iterator gsi
;
7702 gimple entry_stmt
, stmt
;
7705 entry_stmt
= last_stmt (region
->entry
);
7706 new_bb
= region
->entry
;
7707 int kind
= gimple_omp_target_kind (entry_stmt
);
7708 if (kind
== GF_OMP_TARGET_KIND_REGION
)
7710 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
7711 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
7714 entry_bb
= region
->entry
;
7715 exit_bb
= region
->exit
;
7717 if (kind
== GF_OMP_TARGET_KIND_REGION
)
7719 unsigned srcidx
, dstidx
, num
;
7721 /* If the target region needs data sent from the parent
7722 function, then the very first statement (except possible
7723 tree profile counter updates) of the parallel body
7724 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
7725 &.OMP_DATA_O is passed as an argument to the child function,
7726 we need to replace it with the argument as seen by the child
7729 In most cases, this will end up being the identity assignment
7730 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
7731 a function call that has been inlined, the original PARM_DECL
7732 .OMP_DATA_I may have been converted into a different local
7733 variable. In which case, we need to keep the assignment. */
7734 if (gimple_omp_target_data_arg (entry_stmt
))
7736 basic_block entry_succ_bb
= single_succ (entry_bb
);
7737 gimple_stmt_iterator gsi
;
7739 gimple tgtcopy_stmt
= NULL
;
7741 = TREE_VEC_ELT (gimple_omp_target_data_arg (entry_stmt
), 0);
7743 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
7745 gcc_assert (!gsi_end_p (gsi
));
7746 stmt
= gsi_stmt (gsi
);
7747 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
7750 if (gimple_num_ops (stmt
) == 2)
7752 tree arg
= gimple_assign_rhs1 (stmt
);
7754 /* We're ignoring the subcode because we're
7755 effectively doing a STRIP_NOPS. */
7757 if (TREE_CODE (arg
) == ADDR_EXPR
7758 && TREE_OPERAND (arg
, 0) == sender
)
7760 tgtcopy_stmt
= stmt
;
7766 gcc_assert (tgtcopy_stmt
!= NULL
);
7767 arg
= DECL_ARGUMENTS (child_fn
);
7769 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
7770 gsi_remove (&gsi
, true);
7773 /* Declare local variables needed in CHILD_CFUN. */
7774 block
= DECL_INITIAL (child_fn
);
7775 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
7776 /* The gimplifier could record temporaries in target block
7777 rather than in containing function's local_decls chain,
7778 which would mean cgraph missed finalizing them. Do it now. */
7779 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
7780 if (TREE_CODE (t
) == VAR_DECL
7782 && !DECL_EXTERNAL (t
))
7783 varpool_finalize_decl (t
);
7784 DECL_SAVED_TREE (child_fn
) = NULL
;
7785 /* We'll create a CFG for child_fn, so no gimple body is needed. */
7786 gimple_set_body (child_fn
, NULL
);
7787 TREE_USED (block
) = 1;
7789 /* Reset DECL_CONTEXT on function arguments. */
7790 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
7791 DECL_CONTEXT (t
) = child_fn
;
7793 /* Split ENTRY_BB at GIMPLE_OMP_TARGET,
7794 so that it can be moved to the child function. */
7795 gsi
= gsi_last_bb (entry_bb
);
7796 stmt
= gsi_stmt (gsi
);
7797 gcc_assert (stmt
&& gimple_code (stmt
) == GIMPLE_OMP_TARGET
7798 && gimple_omp_target_kind (stmt
)
7799 == GF_OMP_TARGET_KIND_REGION
);
7800 gsi_remove (&gsi
, true);
7801 e
= split_block (entry_bb
, stmt
);
7803 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7805 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
7808 gsi
= gsi_last_bb (exit_bb
);
7809 gcc_assert (!gsi_end_p (gsi
)
7810 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
7811 stmt
= gimple_build_return (NULL
);
7812 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
7813 gsi_remove (&gsi
, true);
7816 /* Move the target region into CHILD_CFUN. */
7818 block
= gimple_block (entry_stmt
);
7820 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
7822 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
7823 /* When the OMP expansion process cannot guarantee an up-to-date
7824 loop tree arrange for the child function to fixup loops. */
7825 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
7826 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
7828 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
7829 num
= vec_safe_length (child_cfun
->local_decls
);
7830 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
7832 t
= (*child_cfun
->local_decls
)[srcidx
];
7833 if (DECL_CONTEXT (t
) == cfun
->decl
)
7835 if (srcidx
!= dstidx
)
7836 (*child_cfun
->local_decls
)[dstidx
] = t
;
7840 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
7842 /* Inform the callgraph about the new function. */
7843 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
7844 cgraph_add_new_function (child_fn
, true);
7846 /* Fix the callgraph edges for child_cfun. Those for cfun will be
7847 fixed in a following pass. */
7848 push_cfun (child_cfun
);
7849 rebuild_cgraph_edges ();
7851 /* Some EH regions might become dead, see PR34608. If
7852 pass_cleanup_cfg isn't the first pass to happen with the
7853 new child, these dead EH edges might cause problems.
7854 Clean them up now. */
7855 if (flag_exceptions
)
7858 bool changed
= false;
7861 changed
|= gimple_purge_dead_eh_edges (bb
);
7863 cleanup_tree_cfg ();
7868 /* Emit a library call to launch the target region, or do data
7870 tree t1
, t2
, t3
, t4
, device
, cond
, c
, clauses
;
7871 enum built_in_function start_ix
;
7872 location_t clause_loc
;
7874 clauses
= gimple_omp_target_clauses (entry_stmt
);
7876 if (kind
== GF_OMP_TARGET_KIND_REGION
)
7877 start_ix
= BUILT_IN_GOMP_TARGET
;
7878 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
7879 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
7881 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
7883 /* By default, the value of DEVICE is -1 (let runtime library choose)
7884 and there is no conditional. */
7886 device
= build_int_cst (integer_type_node
, -1);
7888 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
7890 cond
= OMP_CLAUSE_IF_EXPR (c
);
7892 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
7895 device
= OMP_CLAUSE_DEVICE_ID (c
);
7896 clause_loc
= OMP_CLAUSE_LOCATION (c
);
7899 clause_loc
= gimple_location (entry_stmt
);
7901 /* Ensure 'device' is of the correct type. */
7902 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
7904 /* If we found the clause 'if (cond)', build
7905 (cond ? device : -2). */
7908 cond
= gimple_boolify (cond
);
7910 basic_block cond_bb
, then_bb
, else_bb
;
7914 tmp_var
= create_tmp_var (TREE_TYPE (device
), NULL
);
7915 if (kind
!= GF_OMP_TARGET_KIND_REGION
)
7917 gsi
= gsi_last_bb (new_bb
);
7919 e
= split_block (new_bb
, gsi_stmt (gsi
));
7922 e
= split_block (new_bb
, NULL
);
7927 then_bb
= create_empty_bb (cond_bb
);
7928 else_bb
= create_empty_bb (then_bb
);
7929 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
7930 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
7932 stmt
= gimple_build_cond_empty (cond
);
7933 gsi
= gsi_last_bb (cond_bb
);
7934 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7936 gsi
= gsi_start_bb (then_bb
);
7937 stmt
= gimple_build_assign (tmp_var
, device
);
7938 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7940 gsi
= gsi_start_bb (else_bb
);
7941 stmt
= gimple_build_assign (tmp_var
,
7942 build_int_cst (integer_type_node
, -2));
7943 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7945 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
7946 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
7949 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
7950 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
7952 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
7953 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
7958 gsi
= gsi_last_bb (new_bb
);
7959 t
= gimple_omp_target_data_arg (entry_stmt
);
7962 t1
= size_zero_node
;
7963 t2
= build_zero_cst (ptr_type_node
);
7969 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
7970 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
7971 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
7972 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
7973 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
7977 /* FIXME: This will be address of
7978 extern char __OPENMP_TARGET__[] __attribute__((visibility ("hidden")))
7979 symbol, as soon as the linker plugin is able to create it for us. */
7980 tree openmp_target
= build_zero_cst (ptr_type_node
);
7981 if (kind
== GF_OMP_TARGET_KIND_REGION
)
7983 tree fnaddr
= build_fold_addr_expr (child_fn
);
7984 g
= gimple_build_call (builtin_decl_explicit (start_ix
), 7,
7985 device
, fnaddr
, openmp_target
, t1
, t2
, t3
, t4
);
7988 g
= gimple_build_call (builtin_decl_explicit (start_ix
), 6,
7989 device
, openmp_target
, t1
, t2
, t3
, t4
);
7990 gimple_set_location (g
, gimple_location (entry_stmt
));
7991 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
7992 if (kind
!= GF_OMP_TARGET_KIND_REGION
)
7995 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
7996 gsi_remove (&gsi
, true);
7998 if (kind
== GF_OMP_TARGET_KIND_DATA
&& region
->exit
)
8000 gsi
= gsi_last_bb (region
->exit
);
8002 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
8003 gsi_remove (&gsi
, true);
8008 /* Expand the parallel region tree rooted at REGION. Expansion
8009 proceeds in depth-first order. Innermost regions are expanded
8010 first. This way, parallel regions that require a new function to
8011 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
8012 internal dependencies in their body. */
8015 expand_omp (struct omp_region
*region
)
8019 location_t saved_location
;
8020 gimple inner_stmt
= NULL
;
8022 /* First, determine whether this is a combined parallel+workshare
8024 if (region
->type
== GIMPLE_OMP_PARALLEL
)
8025 determine_parallel_type (region
);
8027 if (region
->type
== GIMPLE_OMP_FOR
8028 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
8029 inner_stmt
= last_stmt (region
->inner
->entry
);
8032 expand_omp (region
->inner
);
8034 saved_location
= input_location
;
8035 if (gimple_has_location (last_stmt (region
->entry
)))
8036 input_location
= gimple_location (last_stmt (region
->entry
));
8038 switch (region
->type
)
8040 case GIMPLE_OMP_PARALLEL
:
8041 case GIMPLE_OMP_TASK
:
8042 expand_omp_taskreg (region
);
8045 case GIMPLE_OMP_FOR
:
8046 expand_omp_for (region
, inner_stmt
);
8049 case GIMPLE_OMP_SECTIONS
:
8050 expand_omp_sections (region
);
8053 case GIMPLE_OMP_SECTION
:
8054 /* Individual omp sections are handled together with their
8055 parent GIMPLE_OMP_SECTIONS region. */
8058 case GIMPLE_OMP_SINGLE
:
8059 expand_omp_single (region
);
8062 case GIMPLE_OMP_MASTER
:
8063 case GIMPLE_OMP_TASKGROUP
:
8064 case GIMPLE_OMP_ORDERED
:
8065 case GIMPLE_OMP_CRITICAL
:
8066 case GIMPLE_OMP_TEAMS
:
8067 expand_omp_synch (region
);
8070 case GIMPLE_OMP_ATOMIC_LOAD
:
8071 expand_omp_atomic (region
);
8074 case GIMPLE_OMP_TARGET
:
8075 expand_omp_target (region
);
8082 input_location
= saved_location
;
8083 region
= region
->next
;
8088 /* Helper for build_omp_regions. Scan the dominator tree starting at
8089 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
8090 true, the function ends once a single tree is built (otherwise, whole
8091 forest of OMP constructs may be built). */
8094 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
8097 gimple_stmt_iterator gsi
;
8101 gsi
= gsi_last_bb (bb
);
8102 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
8104 struct omp_region
*region
;
8105 enum gimple_code code
;
8107 stmt
= gsi_stmt (gsi
);
8108 code
= gimple_code (stmt
);
8109 if (code
== GIMPLE_OMP_RETURN
)
8111 /* STMT is the return point out of region PARENT. Mark it
8112 as the exit point and make PARENT the immediately
8113 enclosing region. */
8114 gcc_assert (parent
);
8117 parent
= parent
->outer
;
8119 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
8121 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
8122 GIMPLE_OMP_RETURN, but matches with
8123 GIMPLE_OMP_ATOMIC_LOAD. */
8124 gcc_assert (parent
);
8125 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
8128 parent
= parent
->outer
;
8131 else if (code
== GIMPLE_OMP_CONTINUE
)
8133 gcc_assert (parent
);
8136 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
8138 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
8139 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
8142 else if (code
== GIMPLE_OMP_TARGET
8143 && gimple_omp_target_kind (stmt
) == GF_OMP_TARGET_KIND_UPDATE
)
8144 new_omp_region (bb
, code
, parent
);
8147 /* Otherwise, this directive becomes the parent for a new
8149 region
= new_omp_region (bb
, code
, parent
);
8154 if (single_tree
&& !parent
)
8157 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
8159 son
= next_dom_son (CDI_DOMINATORS
, son
))
8160 build_omp_regions_1 (son
, parent
, single_tree
);
8163 /* Builds the tree of OMP regions rooted at ROOT, storing it to
8167 build_omp_regions_root (basic_block root
)
8169 gcc_assert (root_omp_region
== NULL
);
8170 build_omp_regions_1 (root
, NULL
, true);
8171 gcc_assert (root_omp_region
!= NULL
);
8174 /* Expands omp construct (and its subconstructs) starting in HEAD. */
8177 omp_expand_local (basic_block head
)
8179 build_omp_regions_root (head
);
8180 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
8182 fprintf (dump_file
, "\nOMP region tree\n\n");
8183 dump_omp_region (dump_file
, root_omp_region
, 0);
8184 fprintf (dump_file
, "\n");
8187 remove_exit_barriers (root_omp_region
);
8188 expand_omp (root_omp_region
);
8190 free_omp_regions ();
8193 /* Scan the CFG and build a tree of OMP regions. Return the root of
8194 the OMP region tree. */
8197 build_omp_regions (void)
8199 gcc_assert (root_omp_region
== NULL
);
8200 calculate_dominance_info (CDI_DOMINATORS
);
8201 build_omp_regions_1 (ENTRY_BLOCK_PTR
, NULL
, false);
8204 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
8207 execute_expand_omp (void)
8209 build_omp_regions ();
8211 if (!root_omp_region
)
8216 fprintf (dump_file
, "\nOMP region tree\n\n");
8217 dump_omp_region (dump_file
, root_omp_region
, 0);
8218 fprintf (dump_file
, "\n");
8221 remove_exit_barriers (root_omp_region
);
8223 expand_omp (root_omp_region
);
8225 cleanup_tree_cfg ();
8227 free_omp_regions ();
8232 /* OMP expansion -- the default pass, run before creation of SSA form. */
8235 gate_expand_omp (void)
8237 return ((flag_openmp
!= 0 || flag_openmp_simd
!= 0) && !seen_error ());
8242 const pass_data pass_data_expand_omp
=
8244 GIMPLE_PASS
, /* type */
8245 "ompexp", /* name */
8246 OPTGROUP_NONE
, /* optinfo_flags */
8247 true, /* has_gate */
8248 true, /* has_execute */
8249 TV_NONE
, /* tv_id */
8250 PROP_gimple_any
, /* properties_required */
8251 0, /* properties_provided */
8252 0, /* properties_destroyed */
8253 0, /* todo_flags_start */
8254 0, /* todo_flags_finish */
8257 class pass_expand_omp
: public gimple_opt_pass
8260 pass_expand_omp (gcc::context
*ctxt
)
8261 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
8264 /* opt_pass methods: */
8265 bool gate () { return gate_expand_omp (); }
8266 unsigned int execute () { return execute_expand_omp (); }
8268 }; // class pass_expand_omp
8273 make_pass_expand_omp (gcc::context
*ctxt
)
8275 return new pass_expand_omp (ctxt
);
8278 /* Routines to lower OpenMP directives into OMP-GIMPLE. */
8280 /* If ctx is a worksharing context inside of a cancellable parallel
8281 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
8282 and conditional branch to parallel's cancel_label to handle
8283 cancellation in the implicit barrier. */
8286 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
8288 gimple omp_return
= gimple_seq_last_stmt (*body
);
8289 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
8290 if (gimple_omp_return_nowait_p (omp_return
))
8293 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
8294 && ctx
->outer
->cancellable
)
8296 tree lhs
= create_tmp_var (boolean_type_node
, NULL
);
8297 gimple_omp_return_set_lhs (omp_return
, lhs
);
8298 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
8299 gimple g
= gimple_build_cond (NE_EXPR
, lhs
, boolean_false_node
,
8300 ctx
->outer
->cancel_label
, fallthru_label
);
8301 gimple_seq_add_stmt (body
, g
);
8302 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
8306 /* Lower the OpenMP sections directive in the current statement in GSI_P.
8307 CTX is the enclosing OMP context for the current statement. */
8310 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8312 tree block
, control
;
8313 gimple_stmt_iterator tgsi
;
8314 gimple stmt
, new_stmt
, bind
, t
;
8315 gimple_seq ilist
, dlist
, olist
, new_body
;
8316 struct gimplify_ctx gctx
;
8318 stmt
= gsi_stmt (*gsi_p
);
8320 push_gimplify_context (&gctx
);
8324 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
8325 &ilist
, &dlist
, ctx
, NULL
);
8327 new_body
= gimple_omp_body (stmt
);
8328 gimple_omp_set_body (stmt
, NULL
);
8329 tgsi
= gsi_start (new_body
);
8330 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
8335 sec_start
= gsi_stmt (tgsi
);
8336 sctx
= maybe_lookup_ctx (sec_start
);
8339 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
8340 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
8341 GSI_CONTINUE_LINKING
);
8342 gimple_omp_set_body (sec_start
, NULL
);
8344 if (gsi_one_before_end_p (tgsi
))
8346 gimple_seq l
= NULL
;
8347 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
8349 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
8350 gimple_omp_section_set_last (sec_start
);
8353 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
8354 GSI_CONTINUE_LINKING
);
8357 block
= make_node (BLOCK
);
8358 bind
= gimple_build_bind (NULL
, new_body
, block
);
8361 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
8363 block
= make_node (BLOCK
);
8364 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
8365 gsi_replace (gsi_p
, new_stmt
, true);
8367 pop_gimplify_context (new_stmt
);
8368 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
8369 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8370 if (BLOCK_VARS (block
))
8371 TREE_USED (block
) = 1;
8374 gimple_seq_add_seq (&new_body
, ilist
);
8375 gimple_seq_add_stmt (&new_body
, stmt
);
8376 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
8377 gimple_seq_add_stmt (&new_body
, bind
);
8379 control
= create_tmp_var (unsigned_type_node
, ".section");
8380 t
= gimple_build_omp_continue (control
, control
);
8381 gimple_omp_sections_set_control (stmt
, control
);
8382 gimple_seq_add_stmt (&new_body
, t
);
8384 gimple_seq_add_seq (&new_body
, olist
);
8385 if (ctx
->cancellable
)
8386 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
8387 gimple_seq_add_seq (&new_body
, dlist
);
8389 new_body
= maybe_catch_exception (new_body
);
8391 t
= gimple_build_omp_return
8392 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
8393 OMP_CLAUSE_NOWAIT
));
8394 gimple_seq_add_stmt (&new_body
, t
);
8395 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
8397 gimple_bind_set_body (new_stmt
, new_body
);
8401 /* A subroutine of lower_omp_single. Expand the simple form of
8402 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
8404 if (GOMP_single_start ())
8406 [ GOMP_barrier (); ] -> unless 'nowait' is present.
8408 FIXME. It may be better to delay expanding the logic of this until
8409 pass_expand_omp. The expanded logic may make the job more difficult
8410 to a synchronization analysis pass. */
8413 lower_omp_single_simple (gimple single_stmt
, gimple_seq
*pre_p
)
8415 location_t loc
= gimple_location (single_stmt
);
8416 tree tlabel
= create_artificial_label (loc
);
8417 tree flabel
= create_artificial_label (loc
);
8421 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
8422 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)), NULL
);
8423 call
= gimple_build_call (decl
, 0);
8424 gimple_call_set_lhs (call
, lhs
);
8425 gimple_seq_add_stmt (pre_p
, call
);
8427 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
8428 fold_convert_loc (loc
, TREE_TYPE (lhs
),
8431 gimple_seq_add_stmt (pre_p
, cond
);
8432 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
8433 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
8434 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
8438 /* A subroutine of lower_omp_single. Expand the simple form of
8439 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
8441 #pragma omp single copyprivate (a, b, c)
8443 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
8446 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
8452 GOMP_single_copy_end (©out);
8463 FIXME. It may be better to delay expanding the logic of this until
8464 pass_expand_omp. The expanded logic may make the job more difficult
8465 to a synchronization analysis pass. */
8468 lower_omp_single_copy (gimple single_stmt
, gimple_seq
*pre_p
, omp_context
*ctx
)
8470 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
8471 gimple_seq copyin_seq
;
8472 location_t loc
= gimple_location (single_stmt
);
8474 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
8476 ptr_type
= build_pointer_type (ctx
->record_type
);
8477 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
8479 l0
= create_artificial_label (loc
);
8480 l1
= create_artificial_label (loc
);
8481 l2
= create_artificial_label (loc
);
8483 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
8484 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
8485 t
= fold_convert_loc (loc
, ptr_type
, t
);
8486 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
8488 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
8489 build_int_cst (ptr_type
, 0));
8490 t
= build3 (COND_EXPR
, void_type_node
, t
,
8491 build_and_jump (&l0
), build_and_jump (&l1
));
8492 gimplify_and_add (t
, pre_p
);
8494 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
8496 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
8499 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
8502 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
8503 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
8504 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
8505 gimplify_and_add (t
, pre_p
);
8507 t
= build_and_jump (&l2
);
8508 gimplify_and_add (t
, pre_p
);
8510 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
8512 gimple_seq_add_seq (pre_p
, copyin_seq
);
8514 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
8518 /* Expand code for an OpenMP single directive. */
8521 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8524 gimple t
, bind
, single_stmt
= gsi_stmt (*gsi_p
);
8525 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
8526 struct gimplify_ctx gctx
;
8528 push_gimplify_context (&gctx
);
8530 block
= make_node (BLOCK
);
8531 bind
= gimple_build_bind (NULL
, NULL
, block
);
8532 gsi_replace (gsi_p
, bind
, true);
8535 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
8536 &bind_body
, &dlist
, ctx
, NULL
);
8537 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
8539 gimple_seq_add_stmt (&bind_body
, single_stmt
);
8541 if (ctx
->record_type
)
8542 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
8544 lower_omp_single_simple (single_stmt
, &bind_body
);
8546 gimple_omp_set_body (single_stmt
, NULL
);
8548 gimple_seq_add_seq (&bind_body
, dlist
);
8550 bind_body
= maybe_catch_exception (bind_body
);
8552 t
= gimple_build_omp_return
8553 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
8554 OMP_CLAUSE_NOWAIT
));
8555 gimple_seq_add_stmt (&bind_body_tail
, t
);
8556 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
8557 if (ctx
->record_type
)
8559 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
8560 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
8561 TREE_THIS_VOLATILE (clobber
) = 1;
8562 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
8563 clobber
), GSI_SAME_STMT
);
8565 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
8566 gimple_bind_set_body (bind
, bind_body
);
8568 pop_gimplify_context (bind
);
8570 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8571 BLOCK_VARS (block
) = ctx
->block_vars
;
8572 if (BLOCK_VARS (block
))
8573 TREE_USED (block
) = 1;
8577 /* Expand code for an OpenMP master directive. */
8580 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8582 tree block
, lab
= NULL
, x
, bfn_decl
;
8583 gimple stmt
= gsi_stmt (*gsi_p
), bind
;
8584 location_t loc
= gimple_location (stmt
);
8586 struct gimplify_ctx gctx
;
8588 push_gimplify_context (&gctx
);
8590 block
= make_node (BLOCK
);
8591 bind
= gimple_build_bind (NULL
, NULL
, block
);
8592 gsi_replace (gsi_p
, bind
, true);
8593 gimple_bind_add_stmt (bind
, stmt
);
8595 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
8596 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
8597 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
8598 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
8600 gimplify_and_add (x
, &tseq
);
8601 gimple_bind_add_seq (bind
, tseq
);
8603 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8604 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
8605 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8606 gimple_omp_set_body (stmt
, NULL
);
8608 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
8610 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8612 pop_gimplify_context (bind
);
8614 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8615 BLOCK_VARS (block
) = ctx
->block_vars
;
8619 /* Expand code for an OpenMP taskgroup directive. */
8622 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8624 gimple stmt
= gsi_stmt (*gsi_p
), bind
, x
;
8625 tree block
= make_node (BLOCK
);
8627 bind
= gimple_build_bind (NULL
, NULL
, block
);
8628 gsi_replace (gsi_p
, bind
, true);
8629 gimple_bind_add_stmt (bind
, stmt
);
8631 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
8633 gimple_bind_add_stmt (bind
, x
);
8635 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8636 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8637 gimple_omp_set_body (stmt
, NULL
);
8639 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8641 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8642 BLOCK_VARS (block
) = ctx
->block_vars
;
8646 /* Expand code for an OpenMP ordered directive. */
8649 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8652 gimple stmt
= gsi_stmt (*gsi_p
), bind
, x
;
8653 struct gimplify_ctx gctx
;
8655 push_gimplify_context (&gctx
);
8657 block
= make_node (BLOCK
);
8658 bind
= gimple_build_bind (NULL
, NULL
, block
);
8659 gsi_replace (gsi_p
, bind
, true);
8660 gimple_bind_add_stmt (bind
, stmt
);
8662 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
8664 gimple_bind_add_stmt (bind
, x
);
8666 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8667 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
8668 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8669 gimple_omp_set_body (stmt
, NULL
);
8671 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
), 0);
8672 gimple_bind_add_stmt (bind
, x
);
8674 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8676 pop_gimplify_context (bind
);
8678 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8679 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8683 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
8684 substitution of a couple of function calls. But in the NAMED case,
8685 requires that languages coordinate a symbol name. It is therefore
8686 best put here in common code. */
8688 static GTY((param1_is (tree
), param2_is (tree
)))
8689 splay_tree critical_name_mutexes
;
8692 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8695 tree name
, lock
, unlock
;
8696 gimple stmt
= gsi_stmt (*gsi_p
), bind
;
8697 location_t loc
= gimple_location (stmt
);
8699 struct gimplify_ctx gctx
;
8701 name
= gimple_omp_critical_name (stmt
);
8707 if (!critical_name_mutexes
)
8708 critical_name_mutexes
8709 = splay_tree_new_ggc (splay_tree_compare_pointers
,
8710 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_s
,
8711 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_node_s
);
8713 n
= splay_tree_lookup (critical_name_mutexes
, (splay_tree_key
) name
);
8718 decl
= create_tmp_var_raw (ptr_type_node
, NULL
);
8720 new_str
= ACONCAT ((".gomp_critical_user_",
8721 IDENTIFIER_POINTER (name
), NULL
));
8722 DECL_NAME (decl
) = get_identifier (new_str
);
8723 TREE_PUBLIC (decl
) = 1;
8724 TREE_STATIC (decl
) = 1;
8725 DECL_COMMON (decl
) = 1;
8726 DECL_ARTIFICIAL (decl
) = 1;
8727 DECL_IGNORED_P (decl
) = 1;
8728 varpool_finalize_decl (decl
);
8730 splay_tree_insert (critical_name_mutexes
, (splay_tree_key
) name
,
8731 (splay_tree_value
) decl
);
8734 decl
= (tree
) n
->value
;
8736 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
8737 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
8739 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
8740 unlock
= build_call_expr_loc (loc
, unlock
, 1,
8741 build_fold_addr_expr_loc (loc
, decl
));
8745 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
8746 lock
= build_call_expr_loc (loc
, lock
, 0);
8748 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
8749 unlock
= build_call_expr_loc (loc
, unlock
, 0);
8752 push_gimplify_context (&gctx
);
8754 block
= make_node (BLOCK
);
8755 bind
= gimple_build_bind (NULL
, NULL
, block
);
8756 gsi_replace (gsi_p
, bind
, true);
8757 gimple_bind_add_stmt (bind
, stmt
);
8759 tbody
= gimple_bind_body (bind
);
8760 gimplify_and_add (lock
, &tbody
);
8761 gimple_bind_set_body (bind
, tbody
);
8763 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8764 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
8765 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8766 gimple_omp_set_body (stmt
, NULL
);
8768 tbody
= gimple_bind_body (bind
);
8769 gimplify_and_add (unlock
, &tbody
);
8770 gimple_bind_set_body (bind
, tbody
);
8772 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8774 pop_gimplify_context (bind
);
8775 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8776 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8780 /* A subroutine of lower_omp_for. Generate code to emit the predicate
8781 for a lastprivate clause. Given a loop control predicate of (V
8782 cond N2), we gate the clause on (!(V cond N2)). The lowered form
8783 is appended to *DLIST, iterator initialization is appended to
8787 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
8788 gimple_seq
*dlist
, struct omp_context
*ctx
)
8790 tree clauses
, cond
, vinit
;
8791 enum tree_code cond_code
;
8794 cond_code
= fd
->loop
.cond_code
;
8795 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
8797 /* When possible, use a strict equality expression. This can let VRP
8798 type optimizations deduce the value and remove a copy. */
8799 if (host_integerp (fd
->loop
.step
, 0))
8801 HOST_WIDE_INT step
= TREE_INT_CST_LOW (fd
->loop
.step
);
8802 if (step
== 1 || step
== -1)
8803 cond_code
= EQ_EXPR
;
8806 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, fd
->loop
.n2
);
8808 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
8810 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
8811 if (!gimple_seq_empty_p (stmts
))
8813 gimple_seq_add_seq (&stmts
, *dlist
);
8816 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
8817 vinit
= fd
->loop
.n1
;
8818 if (cond_code
== EQ_EXPR
8819 && host_integerp (fd
->loop
.n2
, 0)
8820 && ! integer_zerop (fd
->loop
.n2
))
8821 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
8823 vinit
= unshare_expr (vinit
);
8825 /* Initialize the iterator variable, so that threads that don't execute
8826 any iterations don't execute the lastprivate clauses by accident. */
8827 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
8832 /* Lower code for an OpenMP loop directive. */
8835 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8838 struct omp_for_data fd
, *fdp
= NULL
;
8839 gimple stmt
= gsi_stmt (*gsi_p
), new_stmt
;
8840 gimple_seq omp_for_body
, body
, dlist
;
8842 struct gimplify_ctx gctx
;
8844 push_gimplify_context (&gctx
);
8846 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
8848 block
= make_node (BLOCK
);
8849 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
8850 /* Replace at gsi right away, so that 'stmt' is no member
8851 of a sequence anymore as we're going to add to to a different
8853 gsi_replace (gsi_p
, new_stmt
, true);
8855 /* Move declaration of temporaries in the loop body before we make
8857 omp_for_body
= gimple_omp_body (stmt
);
8858 if (!gimple_seq_empty_p (omp_for_body
)
8859 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
8861 tree vars
= gimple_bind_vars (gimple_seq_first_stmt (omp_for_body
));
8862 gimple_bind_append_vars (new_stmt
, vars
);
8865 if (gimple_omp_for_combined_into_p (stmt
))
8867 extract_omp_for_data (stmt
, &fd
, NULL
);
8870 /* We need two temporaries with fd.loop.v type (istart/iend)
8871 and then (fd.collapse - 1) temporaries with the same
8872 type for count2 ... countN-1 vars if not constant. */
8874 tree type
= fd
.iter_type
;
8876 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
8877 count
+= fd
.collapse
- 1;
8878 bool parallel_for
= gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
;
8879 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
8883 = find_omp_clause (gimple_omp_parallel_clauses (ctx
->outer
->stmt
),
8884 OMP_CLAUSE__LOOPTEMP_
);
8885 for (i
= 0; i
< count
; i
++)
8890 gcc_assert (outerc
);
8891 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
8892 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
8893 OMP_CLAUSE__LOOPTEMP_
);
8896 temp
= create_tmp_var (type
, NULL
);
8897 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
8898 OMP_CLAUSE_DECL (*pc
) = temp
;
8899 pc
= &OMP_CLAUSE_CHAIN (*pc
);
8904 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
8907 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
8909 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
8911 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8913 /* Lower the header expressions. At this point, we can assume that
8914 the header is of the form:
8916 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
8918 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
8919 using the .omp_data_s mapping, if needed. */
8920 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
8922 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
8923 if (!is_gimple_min_invariant (*rhs_p
))
8924 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
8926 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
8927 if (!is_gimple_min_invariant (*rhs_p
))
8928 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
8930 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
8931 if (!is_gimple_min_invariant (*rhs_p
))
8932 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
8935 /* Once lowered, extract the bounds and clauses. */
8936 extract_omp_for_data (stmt
, &fd
, NULL
);
8938 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
8940 gimple_seq_add_stmt (&body
, stmt
);
8941 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
8943 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
8946 /* After the loop, add exit clauses. */
8947 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
8949 if (ctx
->cancellable
)
8950 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
8952 gimple_seq_add_seq (&body
, dlist
);
8954 body
= maybe_catch_exception (body
);
8956 /* Region exit marker goes at the end of the loop body. */
8957 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
8958 maybe_add_implicit_barrier_cancel (ctx
, &body
);
8959 pop_gimplify_context (new_stmt
);
8961 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
8962 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
8963 if (BLOCK_VARS (block
))
8964 TREE_USED (block
) = 1;
8966 gimple_bind_set_body (new_stmt
, body
);
8967 gimple_omp_set_body (stmt
, NULL
);
8968 gimple_omp_for_set_pre_body (stmt
, NULL
);
8971 /* Callback for walk_stmts. Check if the current statement only contains
8972 GIMPLE_OMP_FOR or GIMPLE_OMP_PARALLEL. */
8975 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
8976 bool *handled_ops_p
,
8977 struct walk_stmt_info
*wi
)
8979 int *info
= (int *) wi
->info
;
8980 gimple stmt
= gsi_stmt (*gsi_p
);
8982 *handled_ops_p
= true;
8983 switch (gimple_code (stmt
))
8987 case GIMPLE_OMP_FOR
:
8988 case GIMPLE_OMP_SECTIONS
:
8989 *info
= *info
== 0 ? 1 : -1;
8998 struct omp_taskcopy_context
9000 /* This field must be at the beginning, as we do "inheritance": Some
9001 callback functions for tree-inline.c (e.g., omp_copy_decl)
9002 receive a copy_body_data pointer that is up-casted to an
9003 omp_context pointer. */
9009 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
9011 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
9013 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
9014 return create_tmp_var (TREE_TYPE (var
), NULL
);
9020 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
9022 tree name
, new_fields
= NULL
, type
, f
;
9024 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
9025 name
= DECL_NAME (TYPE_NAME (orig_type
));
9026 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
9027 TYPE_DECL
, name
, type
);
9028 TYPE_NAME (type
) = name
;
9030 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
9032 tree new_f
= copy_node (f
);
9033 DECL_CONTEXT (new_f
) = type
;
9034 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
9035 TREE_CHAIN (new_f
) = new_fields
;
9036 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
9037 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
9038 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
9041 *pointer_map_insert (tcctx
->cb
.decl_map
, f
) = new_f
;
9043 TYPE_FIELDS (type
) = nreverse (new_fields
);
9048 /* Create task copyfn. */
9051 create_task_copyfn (gimple task_stmt
, omp_context
*ctx
)
9053 struct function
*child_cfun
;
9054 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
9055 tree record_type
, srecord_type
, bind
, list
;
9056 bool record_needs_remap
= false, srecord_needs_remap
= false;
9058 struct omp_taskcopy_context tcctx
;
9059 struct gimplify_ctx gctx
;
9060 location_t loc
= gimple_location (task_stmt
);
9062 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
9063 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
9064 gcc_assert (child_cfun
->cfg
== NULL
);
9065 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
9067 /* Reset DECL_CONTEXT on function arguments. */
9068 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
9069 DECL_CONTEXT (t
) = child_fn
;
9071 /* Populate the function. */
9072 push_gimplify_context (&gctx
);
9073 push_cfun (child_cfun
);
9075 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
9076 TREE_SIDE_EFFECTS (bind
) = 1;
9078 DECL_SAVED_TREE (child_fn
) = bind
;
9079 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
9081 /* Remap src and dst argument types if needed. */
9082 record_type
= ctx
->record_type
;
9083 srecord_type
= ctx
->srecord_type
;
9084 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
9085 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
9087 record_needs_remap
= true;
9090 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
9091 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
9093 srecord_needs_remap
= true;
9097 if (record_needs_remap
|| srecord_needs_remap
)
9099 memset (&tcctx
, '\0', sizeof (tcctx
));
9100 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
9101 tcctx
.cb
.dst_fn
= child_fn
;
9102 tcctx
.cb
.src_node
= cgraph_get_node (tcctx
.cb
.src_fn
);
9103 gcc_checking_assert (tcctx
.cb
.src_node
);
9104 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
9105 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
9106 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
9107 tcctx
.cb
.eh_lp_nr
= 0;
9108 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
9109 tcctx
.cb
.decl_map
= pointer_map_create ();
9112 if (record_needs_remap
)
9113 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
9114 if (srecord_needs_remap
)
9115 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
9118 tcctx
.cb
.decl_map
= NULL
;
9120 arg
= DECL_ARGUMENTS (child_fn
);
9121 TREE_TYPE (arg
) = build_pointer_type (record_type
);
9122 sarg
= DECL_CHAIN (arg
);
9123 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
9125 /* First pass: initialize temporaries used in record_type and srecord_type
9126 sizes and field offsets. */
9127 if (tcctx
.cb
.decl_map
)
9128 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9129 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
9133 decl
= OMP_CLAUSE_DECL (c
);
9134 p
= (tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, decl
);
9137 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9138 sf
= (tree
) n
->value
;
9139 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9140 src
= build_simple_mem_ref_loc (loc
, sarg
);
9141 src
= omp_build_component_ref (src
, sf
);
9142 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
9143 append_to_statement_list (t
, &list
);
9146 /* Second pass: copy shared var pointers and copy construct non-VLA
9147 firstprivate vars. */
9148 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9149 switch (OMP_CLAUSE_CODE (c
))
9151 case OMP_CLAUSE_SHARED
:
9152 decl
= OMP_CLAUSE_DECL (c
);
9153 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9156 f
= (tree
) n
->value
;
9157 if (tcctx
.cb
.decl_map
)
9158 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9159 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9160 sf
= (tree
) n
->value
;
9161 if (tcctx
.cb
.decl_map
)
9162 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9163 src
= build_simple_mem_ref_loc (loc
, sarg
);
9164 src
= omp_build_component_ref (src
, sf
);
9165 dst
= build_simple_mem_ref_loc (loc
, arg
);
9166 dst
= omp_build_component_ref (dst
, f
);
9167 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
9168 append_to_statement_list (t
, &list
);
9170 case OMP_CLAUSE_FIRSTPRIVATE
:
9171 decl
= OMP_CLAUSE_DECL (c
);
9172 if (is_variable_sized (decl
))
9174 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9177 f
= (tree
) n
->value
;
9178 if (tcctx
.cb
.decl_map
)
9179 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9180 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9183 sf
= (tree
) n
->value
;
9184 if (tcctx
.cb
.decl_map
)
9185 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9186 src
= build_simple_mem_ref_loc (loc
, sarg
);
9187 src
= omp_build_component_ref (src
, sf
);
9188 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
9189 src
= build_simple_mem_ref_loc (loc
, src
);
9193 dst
= build_simple_mem_ref_loc (loc
, arg
);
9194 dst
= omp_build_component_ref (dst
, f
);
9195 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
9196 append_to_statement_list (t
, &list
);
9198 case OMP_CLAUSE_PRIVATE
:
9199 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
9201 decl
= OMP_CLAUSE_DECL (c
);
9202 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9203 f
= (tree
) n
->value
;
9204 if (tcctx
.cb
.decl_map
)
9205 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9206 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9209 sf
= (tree
) n
->value
;
9210 if (tcctx
.cb
.decl_map
)
9211 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9212 src
= build_simple_mem_ref_loc (loc
, sarg
);
9213 src
= omp_build_component_ref (src
, sf
);
9214 if (use_pointer_for_field (decl
, NULL
))
9215 src
= build_simple_mem_ref_loc (loc
, src
);
9219 dst
= build_simple_mem_ref_loc (loc
, arg
);
9220 dst
= omp_build_component_ref (dst
, f
);
9221 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
9222 append_to_statement_list (t
, &list
);
9228 /* Last pass: handle VLA firstprivates. */
9229 if (tcctx
.cb
.decl_map
)
9230 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9231 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
9235 decl
= OMP_CLAUSE_DECL (c
);
9236 if (!is_variable_sized (decl
))
9238 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9241 f
= (tree
) n
->value
;
9242 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9243 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
9244 ind
= DECL_VALUE_EXPR (decl
);
9245 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
9246 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
9247 n
= splay_tree_lookup (ctx
->sfield_map
,
9248 (splay_tree_key
) TREE_OPERAND (ind
, 0));
9249 sf
= (tree
) n
->value
;
9250 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9251 src
= build_simple_mem_ref_loc (loc
, sarg
);
9252 src
= omp_build_component_ref (src
, sf
);
9253 src
= build_simple_mem_ref_loc (loc
, src
);
9254 dst
= build_simple_mem_ref_loc (loc
, arg
);
9255 dst
= omp_build_component_ref (dst
, f
);
9256 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
9257 append_to_statement_list (t
, &list
);
9258 n
= splay_tree_lookup (ctx
->field_map
,
9259 (splay_tree_key
) TREE_OPERAND (ind
, 0));
9260 df
= (tree
) n
->value
;
9261 df
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, df
);
9262 ptr
= build_simple_mem_ref_loc (loc
, arg
);
9263 ptr
= omp_build_component_ref (ptr
, df
);
9264 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
9265 build_fold_addr_expr_loc (loc
, dst
));
9266 append_to_statement_list (t
, &list
);
9269 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
9270 append_to_statement_list (t
, &list
);
9272 if (tcctx
.cb
.decl_map
)
9273 pointer_map_destroy (tcctx
.cb
.decl_map
);
9274 pop_gimplify_context (NULL
);
9275 BIND_EXPR_BODY (bind
) = list
;
9280 lower_depend_clauses (gimple stmt
, gimple_seq
*iseq
, gimple_seq
*oseq
)
9284 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
9286 clauses
= find_omp_clause (gimple_omp_task_clauses (stmt
),
9288 gcc_assert (clauses
);
9289 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9290 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
9291 switch (OMP_CLAUSE_DEPEND_KIND (c
))
9293 case OMP_CLAUSE_DEPEND_IN
:
9296 case OMP_CLAUSE_DEPEND_OUT
:
9297 case OMP_CLAUSE_DEPEND_INOUT
:
9303 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
9304 tree array
= create_tmp_var (type
, NULL
);
9305 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
9307 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
9308 gimple_seq_add_stmt (iseq
, g
);
9309 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
9311 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
9312 gimple_seq_add_stmt (iseq
, g
);
9313 for (i
= 0; i
< 2; i
++)
9315 if ((i
? n_in
: n_out
) == 0)
9317 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9318 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
9319 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
9321 tree t
= OMP_CLAUSE_DECL (c
);
9322 t
= fold_convert (ptr_type_node
, t
);
9323 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
9324 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
9325 NULL_TREE
, NULL_TREE
);
9326 g
= gimple_build_assign (r
, t
);
9327 gimple_seq_add_stmt (iseq
, g
);
9330 tree
*p
= gimple_omp_task_clauses_ptr (stmt
);
9331 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
9332 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
9333 OMP_CLAUSE_CHAIN (c
) = *p
;
9335 tree clobber
= build_constructor (type
, NULL
);
9336 TREE_THIS_VOLATILE (clobber
) = 1;
9337 g
= gimple_build_assign (array
, clobber
);
9338 gimple_seq_add_stmt (oseq
, g
);
9341 /* Lower the OpenMP parallel or task directive in the current statement
9342 in GSI_P. CTX holds context information for the directive. */
9345 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9349 gimple stmt
= gsi_stmt (*gsi_p
);
9350 gimple par_bind
, bind
, dep_bind
= NULL
;
9351 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
9352 struct gimplify_ctx gctx
, dep_gctx
;
9353 location_t loc
= gimple_location (stmt
);
9355 clauses
= gimple_omp_taskreg_clauses (stmt
);
9356 par_bind
= gimple_seq_first_stmt (gimple_omp_body (stmt
));
9357 par_body
= gimple_bind_body (par_bind
);
9358 child_fn
= ctx
->cb
.dst_fn
;
9359 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
9360 && !gimple_omp_parallel_combined_p (stmt
))
9362 struct walk_stmt_info wi
;
9365 memset (&wi
, 0, sizeof (wi
));
9368 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
9370 gimple_omp_parallel_set_combined_p (stmt
, true);
9372 gimple_seq dep_ilist
= NULL
;
9373 gimple_seq dep_olist
= NULL
;
9374 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
9375 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
9377 push_gimplify_context (&dep_gctx
);
9378 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
9379 lower_depend_clauses (stmt
, &dep_ilist
, &dep_olist
);
9382 if (ctx
->srecord_type
)
9383 create_task_copyfn (stmt
, ctx
);
9385 push_gimplify_context (&gctx
);
9390 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
9391 lower_omp (&par_body
, ctx
);
9392 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
9393 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
9395 /* Declare all the variables created by mapping and the variables
9396 declared in the scope of the parallel body. */
9397 record_vars_into (ctx
->block_vars
, child_fn
);
9398 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
9400 if (ctx
->record_type
)
9403 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
9404 : ctx
->record_type
, ".omp_data_o");
9405 DECL_NAMELESS (ctx
->sender_decl
) = 1;
9406 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
9407 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
9412 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
9413 lower_send_shared_vars (&ilist
, &olist
, ctx
);
9415 if (ctx
->record_type
)
9417 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
9418 TREE_THIS_VOLATILE (clobber
) = 1;
9419 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
9423 /* Once all the expansions are done, sequence all the different
9424 fragments inside gimple_omp_body. */
9428 if (ctx
->record_type
)
9430 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
9431 /* fixup_child_record_type might have changed receiver_decl's type. */
9432 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
9433 gimple_seq_add_stmt (&new_body
,
9434 gimple_build_assign (ctx
->receiver_decl
, t
));
9437 gimple_seq_add_seq (&new_body
, par_ilist
);
9438 gimple_seq_add_seq (&new_body
, par_body
);
9439 gimple_seq_add_seq (&new_body
, par_rlist
);
9440 if (ctx
->cancellable
)
9441 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
9442 gimple_seq_add_seq (&new_body
, par_olist
);
9443 new_body
= maybe_catch_exception (new_body
);
9444 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
9445 gimple_omp_set_body (stmt
, new_body
);
9447 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
9448 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
9449 gimple_bind_add_seq (bind
, ilist
);
9450 gimple_bind_add_stmt (bind
, stmt
);
9451 gimple_bind_add_seq (bind
, olist
);
9453 pop_gimplify_context (NULL
);
9457 gimple_bind_add_seq (dep_bind
, dep_ilist
);
9458 gimple_bind_add_stmt (dep_bind
, bind
);
9459 gimple_bind_add_seq (dep_bind
, dep_olist
);
9460 pop_gimplify_context (dep_bind
);
9464 /* Lower the OpenMP target directive in the current statement
9465 in GSI_P. CTX holds context information for the directive. */
9468 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9471 tree child_fn
, t
, c
;
9472 gimple stmt
= gsi_stmt (*gsi_p
);
9473 gimple tgt_bind
= NULL
, bind
;
9474 gimple_seq tgt_body
= NULL
, olist
, ilist
, new_body
;
9475 struct gimplify_ctx gctx
;
9476 location_t loc
= gimple_location (stmt
);
9477 int kind
= gimple_omp_target_kind (stmt
);
9478 unsigned int map_cnt
= 0;
9480 clauses
= gimple_omp_target_clauses (stmt
);
9481 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9483 tgt_bind
= gimple_seq_first_stmt (gimple_omp_body (stmt
));
9484 tgt_body
= gimple_bind_body (tgt_bind
);
9486 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
9487 tgt_body
= gimple_omp_body (stmt
);
9488 child_fn
= ctx
->cb
.dst_fn
;
9490 push_gimplify_context (&gctx
);
9492 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9493 switch (OMP_CLAUSE_CODE (c
))
9499 case OMP_CLAUSE_MAP
:
9501 case OMP_CLAUSE_FROM
:
9502 var
= OMP_CLAUSE_DECL (c
);
9505 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
9506 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
9512 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
9514 tree var2
= DECL_VALUE_EXPR (var
);
9515 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
9516 var2
= TREE_OPERAND (var2
, 0);
9517 gcc_assert (DECL_P (var2
));
9521 if (!maybe_lookup_field (var
, ctx
))
9524 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9526 x
= build_receiver_ref (var
, true, ctx
);
9527 tree new_var
= lookup_decl (var
, ctx
);
9528 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9529 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
9530 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
9531 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
9532 x
= build_simple_mem_ref (x
);
9533 SET_DECL_VALUE_EXPR (new_var
, x
);
9534 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
9539 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9541 target_nesting_level
++;
9542 lower_omp (&tgt_body
, ctx
);
9543 target_nesting_level
--;
9545 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
9546 lower_omp (&tgt_body
, ctx
);
9548 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9550 /* Declare all the variables created by mapping and the variables
9551 declared in the scope of the target body. */
9552 record_vars_into (ctx
->block_vars
, child_fn
);
9553 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
9558 if (ctx
->record_type
)
9561 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
9562 DECL_NAMELESS (ctx
->sender_decl
) = 1;
9563 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
9564 t
= make_tree_vec (3);
9565 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
9567 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
9569 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
9570 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
9571 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
9573 = create_tmp_var (build_array_type_nelts (unsigned_char_type_node
,
9576 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
9577 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
9578 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
9579 gimple_omp_target_set_data_arg (stmt
, t
);
9581 vec
<constructor_elt
, va_gc
> *vsize
;
9582 vec
<constructor_elt
, va_gc
> *vkind
;
9583 vec_alloc (vsize
, map_cnt
);
9584 vec_alloc (vkind
, map_cnt
);
9585 unsigned int map_idx
= 0;
9587 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9588 switch (OMP_CLAUSE_CODE (c
))
9594 case OMP_CLAUSE_MAP
:
9596 case OMP_CLAUSE_FROM
:
9598 ovar
= OMP_CLAUSE_DECL (c
);
9601 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9602 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
9604 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
9605 == get_base_address (ovar
));
9606 nc
= OMP_CLAUSE_CHAIN (c
);
9607 ovar
= OMP_CLAUSE_DECL (nc
);
9611 tree x
= build_sender_ref (ovar
, ctx
);
9613 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
9614 gimplify_assign (x
, v
, &ilist
);
9620 if (DECL_SIZE (ovar
)
9621 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
9623 tree ovar2
= DECL_VALUE_EXPR (ovar
);
9624 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
9625 ovar2
= TREE_OPERAND (ovar2
, 0);
9626 gcc_assert (DECL_P (ovar2
));
9629 if (!maybe_lookup_field (ovar
, ctx
))
9635 tree var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
9636 tree x
= build_sender_ref (ovar
, ctx
);
9637 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9638 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
9639 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
9640 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
9642 gcc_assert (kind
== GF_OMP_TARGET_KIND_REGION
);
9644 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)), NULL
);
9645 mark_addressable (avar
);
9646 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
9647 avar
= build_fold_addr_expr (avar
);
9648 gimplify_assign (x
, avar
, &ilist
);
9650 else if (is_gimple_reg (var
))
9652 gcc_assert (kind
== GF_OMP_TARGET_KIND_REGION
);
9653 tree avar
= create_tmp_var (TREE_TYPE (var
), NULL
);
9654 mark_addressable (avar
);
9655 if (OMP_CLAUSE_MAP_KIND (c
) != OMP_CLAUSE_MAP_ALLOC
9656 && OMP_CLAUSE_MAP_KIND (c
) != OMP_CLAUSE_MAP_FROM
)
9657 gimplify_assign (avar
, var
, &ilist
);
9658 avar
= build_fold_addr_expr (avar
);
9659 gimplify_assign (x
, avar
, &ilist
);
9660 if ((OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_FROM
9661 || OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_TOFROM
)
9662 && !TYPE_READONLY (TREE_TYPE (var
)))
9664 x
= build_sender_ref (ovar
, ctx
);
9665 x
= build_simple_mem_ref (x
);
9666 gimplify_assign (var
, x
, &olist
);
9671 var
= build_fold_addr_expr (var
);
9672 gimplify_assign (x
, var
, &ilist
);
9675 tree s
= OMP_CLAUSE_SIZE (c
);
9677 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
9678 s
= fold_convert (size_type_node
, s
);
9679 tree purpose
= size_int (map_idx
++);
9680 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
9681 if (TREE_CODE (s
) != INTEGER_CST
)
9682 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
9684 unsigned char tkind
= 0;
9685 switch (OMP_CLAUSE_CODE (c
))
9687 case OMP_CLAUSE_MAP
:
9688 tkind
= OMP_CLAUSE_MAP_KIND (c
);
9691 tkind
= OMP_CLAUSE_MAP_TO
;
9693 case OMP_CLAUSE_FROM
:
9694 tkind
= OMP_CLAUSE_MAP_FROM
;
9699 unsigned int talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
9700 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
9701 talign
= DECL_ALIGN_UNIT (ovar
);
9702 talign
= ceil_log2 (talign
);
9703 tkind
|= talign
<< 3;
9704 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
9705 build_int_cst (unsigned_char_type_node
,
9711 gcc_assert (map_idx
== map_cnt
);
9713 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
9714 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
9715 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
9716 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
9717 if (!TREE_STATIC (TREE_VEC_ELT (t
, 1)))
9719 gimple_seq initlist
= NULL
;
9720 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
9721 TREE_VEC_ELT (t
, 1)),
9722 &initlist
, true, NULL_TREE
);
9723 gimple_seq_add_seq (&ilist
, initlist
);
9726 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
9727 TREE_THIS_VOLATILE (clobber
) = 1;
9728 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
9732 /* Once all the expansions are done, sequence all the different
9733 fragments inside gimple_omp_body. */
9737 if (ctx
->record_type
&& kind
== GF_OMP_TARGET_KIND_REGION
)
9739 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
9740 /* fixup_child_record_type might have changed receiver_decl's type. */
9741 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
9742 gimple_seq_add_stmt (&new_body
,
9743 gimple_build_assign (ctx
->receiver_decl
, t
));
9746 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9748 gimple_seq_add_seq (&new_body
, tgt_body
);
9749 new_body
= maybe_catch_exception (new_body
);
9751 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
9752 new_body
= tgt_body
;
9753 if (kind
!= GF_OMP_TARGET_KIND_UPDATE
)
9755 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
9756 gimple_omp_set_body (stmt
, new_body
);
9759 bind
= gimple_build_bind (NULL
, NULL
,
9760 tgt_bind
? gimple_bind_block (tgt_bind
)
9762 gsi_replace (gsi_p
, bind
, true);
9763 gimple_bind_add_seq (bind
, ilist
);
9764 gimple_bind_add_stmt (bind
, stmt
);
9765 gimple_bind_add_seq (bind
, olist
);
9767 pop_gimplify_context (NULL
);
9770 /* Expand code for an OpenMP teams directive. */
9773 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9775 gimple teams_stmt
= gsi_stmt (*gsi_p
);
9776 struct gimplify_ctx gctx
;
9777 push_gimplify_context (&gctx
);
9779 tree block
= make_node (BLOCK
);
9780 gimple bind
= gimple_build_bind (NULL
, NULL
, block
);
9781 gsi_replace (gsi_p
, bind
, true);
9782 gimple_seq bind_body
= NULL
;
9783 gimple_seq dlist
= NULL
;
9784 gimple_seq olist
= NULL
;
9786 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
9787 OMP_CLAUSE_NUM_TEAMS
);
9788 if (num_teams
== NULL_TREE
)
9789 num_teams
= build_int_cst (unsigned_type_node
, 0);
9792 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
9793 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
9794 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
9796 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
9797 OMP_CLAUSE_THREAD_LIMIT
);
9798 if (thread_limit
== NULL_TREE
)
9799 thread_limit
= build_int_cst (unsigned_type_node
, 0);
9802 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
9803 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
9804 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
9808 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
9809 &bind_body
, &dlist
, ctx
, NULL
);
9810 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
9811 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
9812 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
9814 location_t loc
= gimple_location (teams_stmt
);
9815 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
9816 gimple call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
9817 gimple_set_location (call
, loc
);
9818 gimple_seq_add_stmt (&bind_body
, call
);
9820 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
9821 gimple_omp_set_body (teams_stmt
, NULL
);
9822 gimple_seq_add_seq (&bind_body
, olist
);
9823 gimple_seq_add_seq (&bind_body
, dlist
);
9824 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
9825 gimple_bind_set_body (bind
, bind_body
);
9827 pop_gimplify_context (bind
);
9829 gimple_bind_append_vars (bind
, ctx
->block_vars
);
9830 BLOCK_VARS (block
) = ctx
->block_vars
;
9831 if (BLOCK_VARS (block
))
9832 TREE_USED (block
) = 1;
9836 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
9837 regimplified. If DATA is non-NULL, lower_omp_1 is outside
9838 of OpenMP context, but with task_shared_vars set. */
9841 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
9846 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
9847 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
9850 if (task_shared_vars
9852 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
9855 /* If a global variable has been privatized, TREE_CONSTANT on
9856 ADDR_EXPR might be wrong. */
9857 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
9858 recompute_tree_invariant_for_addr_expr (t
);
9860 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
9865 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9867 gimple stmt
= gsi_stmt (*gsi_p
);
9868 struct walk_stmt_info wi
;
9870 if (gimple_has_location (stmt
))
9871 input_location
= gimple_location (stmt
);
9873 if (task_shared_vars
)
9874 memset (&wi
, '\0', sizeof (wi
));
9876 /* If we have issued syntax errors, avoid doing any heavy lifting.
9877 Just replace the OpenMP directives with a NOP to avoid
9878 confusing RTL expansion. */
9879 if (seen_error () && is_gimple_omp (stmt
))
9881 gsi_replace (gsi_p
, gimple_build_nop (), true);
9885 switch (gimple_code (stmt
))
9888 if ((ctx
|| task_shared_vars
)
9889 && (walk_tree (gimple_cond_lhs_ptr (stmt
), lower_omp_regimplify_p
,
9890 ctx
? NULL
: &wi
, NULL
)
9891 || walk_tree (gimple_cond_rhs_ptr (stmt
), lower_omp_regimplify_p
,
9892 ctx
? NULL
: &wi
, NULL
)))
9893 gimple_regimplify_operands (stmt
, gsi_p
);
9896 lower_omp (gimple_catch_handler_ptr (stmt
), ctx
);
9898 case GIMPLE_EH_FILTER
:
9899 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
9902 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
9903 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
9905 case GIMPLE_TRANSACTION
:
9906 lower_omp (gimple_transaction_body_ptr (stmt
), ctx
);
9909 lower_omp (gimple_bind_body_ptr (stmt
), ctx
);
9911 case GIMPLE_OMP_PARALLEL
:
9912 case GIMPLE_OMP_TASK
:
9913 ctx
= maybe_lookup_ctx (stmt
);
9915 if (ctx
->cancellable
)
9916 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
9917 lower_omp_taskreg (gsi_p
, ctx
);
9919 case GIMPLE_OMP_FOR
:
9920 ctx
= maybe_lookup_ctx (stmt
);
9922 if (ctx
->cancellable
)
9923 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
9924 lower_omp_for (gsi_p
, ctx
);
9926 case GIMPLE_OMP_SECTIONS
:
9927 ctx
= maybe_lookup_ctx (stmt
);
9929 if (ctx
->cancellable
)
9930 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
9931 lower_omp_sections (gsi_p
, ctx
);
9933 case GIMPLE_OMP_SINGLE
:
9934 ctx
= maybe_lookup_ctx (stmt
);
9936 lower_omp_single (gsi_p
, ctx
);
9938 case GIMPLE_OMP_MASTER
:
9939 ctx
= maybe_lookup_ctx (stmt
);
9941 lower_omp_master (gsi_p
, ctx
);
9943 case GIMPLE_OMP_TASKGROUP
:
9944 ctx
= maybe_lookup_ctx (stmt
);
9946 lower_omp_taskgroup (gsi_p
, ctx
);
9948 case GIMPLE_OMP_ORDERED
:
9949 ctx
= maybe_lookup_ctx (stmt
);
9951 lower_omp_ordered (gsi_p
, ctx
);
9953 case GIMPLE_OMP_CRITICAL
:
9954 ctx
= maybe_lookup_ctx (stmt
);
9956 lower_omp_critical (gsi_p
, ctx
);
9958 case GIMPLE_OMP_ATOMIC_LOAD
:
9959 if ((ctx
|| task_shared_vars
)
9960 && walk_tree (gimple_omp_atomic_load_rhs_ptr (stmt
),
9961 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
9962 gimple_regimplify_operands (stmt
, gsi_p
);
9964 case GIMPLE_OMP_TARGET
:
9965 ctx
= maybe_lookup_ctx (stmt
);
9967 lower_omp_target (gsi_p
, ctx
);
9969 case GIMPLE_OMP_TEAMS
:
9970 ctx
= maybe_lookup_ctx (stmt
);
9972 lower_omp_teams (gsi_p
, ctx
);
9976 fndecl
= gimple_call_fndecl (stmt
);
9978 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
9979 switch (DECL_FUNCTION_CODE (fndecl
))
9981 case BUILT_IN_GOMP_BARRIER
:
9985 case BUILT_IN_GOMP_CANCEL
:
9986 case BUILT_IN_GOMP_CANCELLATION_POINT
:
9989 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
9991 gcc_assert (gimple_call_lhs (stmt
) == NULL_TREE
);
9992 if (!cctx
->cancellable
)
9994 if (DECL_FUNCTION_CODE (fndecl
)
9995 == BUILT_IN_GOMP_CANCELLATION_POINT
)
9997 stmt
= gimple_build_nop ();
9998 gsi_replace (gsi_p
, stmt
, false);
10003 lhs
= create_tmp_var (boolean_type_node
, NULL
);
10004 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
10006 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
10007 gimple_call_set_fndecl (stmt
, fndecl
);
10008 gimple_call_set_fntype (stmt
, TREE_TYPE (fndecl
));
10010 gimple_call_set_lhs (stmt
, lhs
);
10011 tree fallthru_label
;
10012 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
10014 g
= gimple_build_label (fallthru_label
);
10015 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
10016 g
= gimple_build_cond (NE_EXPR
, lhs
, boolean_false_node
,
10017 cctx
->cancel_label
, fallthru_label
);
10018 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
10025 if ((ctx
|| task_shared_vars
)
10026 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
10028 gimple_regimplify_operands (stmt
, gsi_p
);
10034 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
10036 location_t saved_location
= input_location
;
10037 gimple_stmt_iterator gsi
;
10038 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
10039 lower_omp_1 (&gsi
, ctx
);
10040 /* Inside target region we haven't called fold_stmt during gimplification,
10041 because it can break code by adding decl references that weren't in the
10042 source. Call fold_stmt now. */
10043 if (target_nesting_level
)
10044 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
10046 input_location
= saved_location
;
10049 /* Main entry point. */
10051 static unsigned int
10052 execute_lower_omp (void)
10056 /* This pass always runs, to provide PROP_gimple_lomp.
10057 But there is nothing to do unless -fopenmp is given. */
10058 if (flag_openmp
== 0 && flag_openmp_simd
== 0)
10061 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
10062 delete_omp_context
);
10064 body
= gimple_body (current_function_decl
);
10065 scan_omp (&body
, NULL
);
10066 gcc_assert (taskreg_nesting_level
== 0);
10068 if (all_contexts
->root
)
10070 struct gimplify_ctx gctx
;
10072 if (task_shared_vars
)
10073 push_gimplify_context (&gctx
);
10074 lower_omp (&body
, NULL
);
10075 if (task_shared_vars
)
10076 pop_gimplify_context (NULL
);
10081 splay_tree_delete (all_contexts
);
10082 all_contexts
= NULL
;
10084 BITMAP_FREE (task_shared_vars
);
10090 const pass_data pass_data_lower_omp
=
10092 GIMPLE_PASS
, /* type */
10093 "omplower", /* name */
10094 OPTGROUP_NONE
, /* optinfo_flags */
10095 false, /* has_gate */
10096 true, /* has_execute */
10097 TV_NONE
, /* tv_id */
10098 PROP_gimple_any
, /* properties_required */
10099 PROP_gimple_lomp
, /* properties_provided */
10100 0, /* properties_destroyed */
10101 0, /* todo_flags_start */
10102 0, /* todo_flags_finish */
10105 class pass_lower_omp
: public gimple_opt_pass
10108 pass_lower_omp (gcc::context
*ctxt
)
10109 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
10112 /* opt_pass methods: */
10113 unsigned int execute () { return execute_lower_omp (); }
10115 }; // class pass_lower_omp
10117 } // anon namespace
10120 make_pass_lower_omp (gcc::context
*ctxt
)
10122 return new pass_lower_omp (ctxt
);
10125 /* The following is a utility to diagnose OpenMP structured block violations.
10126 It is not part of the "omplower" pass, as that's invoked too late. It
10127 should be invoked by the respective front ends after gimplification. */
10129 static splay_tree all_labels
;
10131 /* Check for mismatched contexts and generate an error if needed. Return
10132 true if an error is detected. */
10135 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
10136 gimple branch_ctx
, gimple label_ctx
)
10138 if (label_ctx
== branch_ctx
)
10143 Previously we kept track of the label's entire context in diagnose_sb_[12]
10144 so we could traverse it and issue a correct "exit" or "enter" error
10145 message upon a structured block violation.
10147 We built the context by building a list with tree_cons'ing, but there is
10148 no easy counterpart in gimple tuples. It seems like far too much work
10149 for issuing exit/enter error messages. If someone really misses the
10150 distinct error message... patches welcome.
10154 /* Try to avoid confusing the user by producing and error message
10155 with correct "exit" or "enter" verbiage. We prefer "exit"
10156 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
10157 if (branch_ctx
== NULL
)
10163 if (TREE_VALUE (label_ctx
) == branch_ctx
)
10168 label_ctx
= TREE_CHAIN (label_ctx
);
10173 error ("invalid exit from OpenMP structured block");
10175 error ("invalid entry to OpenMP structured block");
10178 /* If it's obvious we have an invalid entry, be specific about the error. */
10179 if (branch_ctx
== NULL
)
10180 error ("invalid entry to OpenMP structured block");
10182 /* Otherwise, be vague and lazy, but efficient. */
10183 error ("invalid branch to/from an OpenMP structured block");
10185 gsi_replace (gsi_p
, gimple_build_nop (), false);
10189 /* Pass 1: Create a minimal tree of OpenMP structured blocks, and record
10190 where each label is found. */
10193 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
10194 struct walk_stmt_info
*wi
)
10196 gimple context
= (gimple
) wi
->info
;
10197 gimple inner_context
;
10198 gimple stmt
= gsi_stmt (*gsi_p
);
10200 *handled_ops_p
= true;
10202 switch (gimple_code (stmt
))
10206 case GIMPLE_OMP_PARALLEL
:
10207 case GIMPLE_OMP_TASK
:
10208 case GIMPLE_OMP_SECTIONS
:
10209 case GIMPLE_OMP_SINGLE
:
10210 case GIMPLE_OMP_SECTION
:
10211 case GIMPLE_OMP_MASTER
:
10212 case GIMPLE_OMP_ORDERED
:
10213 case GIMPLE_OMP_CRITICAL
:
10214 case GIMPLE_OMP_TARGET
:
10215 case GIMPLE_OMP_TEAMS
:
10216 case GIMPLE_OMP_TASKGROUP
:
10217 /* The minimal context here is just the current OMP construct. */
10218 inner_context
= stmt
;
10219 wi
->info
= inner_context
;
10220 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
10221 wi
->info
= context
;
10224 case GIMPLE_OMP_FOR
:
10225 inner_context
= stmt
;
10226 wi
->info
= inner_context
;
10227 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10229 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
10230 diagnose_sb_1
, NULL
, wi
);
10231 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
10232 wi
->info
= context
;
10236 splay_tree_insert (all_labels
, (splay_tree_key
) gimple_label_label (stmt
),
10237 (splay_tree_value
) context
);
10247 /* Pass 2: Check each branch and see if its context differs from that of
10248 the destination label's context. */
10251 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
10252 struct walk_stmt_info
*wi
)
10254 gimple context
= (gimple
) wi
->info
;
10256 gimple stmt
= gsi_stmt (*gsi_p
);
10258 *handled_ops_p
= true;
10260 switch (gimple_code (stmt
))
10264 case GIMPLE_OMP_PARALLEL
:
10265 case GIMPLE_OMP_TASK
:
10266 case GIMPLE_OMP_SECTIONS
:
10267 case GIMPLE_OMP_SINGLE
:
10268 case GIMPLE_OMP_SECTION
:
10269 case GIMPLE_OMP_MASTER
:
10270 case GIMPLE_OMP_ORDERED
:
10271 case GIMPLE_OMP_CRITICAL
:
10272 case GIMPLE_OMP_TARGET
:
10273 case GIMPLE_OMP_TEAMS
:
10274 case GIMPLE_OMP_TASKGROUP
:
10276 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
10277 wi
->info
= context
;
10280 case GIMPLE_OMP_FOR
:
10282 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10284 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
10285 diagnose_sb_2
, NULL
, wi
);
10286 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
10287 wi
->info
= context
;
10292 tree lab
= gimple_cond_true_label (stmt
);
10295 n
= splay_tree_lookup (all_labels
,
10296 (splay_tree_key
) lab
);
10297 diagnose_sb_0 (gsi_p
, context
,
10298 n
? (gimple
) n
->value
: NULL
);
10300 lab
= gimple_cond_false_label (stmt
);
10303 n
= splay_tree_lookup (all_labels
,
10304 (splay_tree_key
) lab
);
10305 diagnose_sb_0 (gsi_p
, context
,
10306 n
? (gimple
) n
->value
: NULL
);
10313 tree lab
= gimple_goto_dest (stmt
);
10314 if (TREE_CODE (lab
) != LABEL_DECL
)
10317 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
10318 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
) n
->value
: NULL
);
10322 case GIMPLE_SWITCH
:
10325 for (i
= 0; i
< gimple_switch_num_labels (stmt
); ++i
)
10327 tree lab
= CASE_LABEL (gimple_switch_label (stmt
, i
));
10328 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
10329 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
) n
->value
))
10335 case GIMPLE_RETURN
:
10336 diagnose_sb_0 (gsi_p
, context
, NULL
);
10346 /* Called from tree-cfg.c::make_edges to create cfg edges for all GIMPLE_OMP
10349 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
)
10351 gimple last
= last_stmt (bb
);
10352 enum gimple_code code
= gimple_code (last
);
10353 struct omp_region
*cur_region
= *region
;
10354 bool fallthru
= false;
10358 case GIMPLE_OMP_PARALLEL
:
10359 case GIMPLE_OMP_TASK
:
10360 case GIMPLE_OMP_FOR
:
10361 case GIMPLE_OMP_SINGLE
:
10362 case GIMPLE_OMP_TEAMS
:
10363 case GIMPLE_OMP_MASTER
:
10364 case GIMPLE_OMP_TASKGROUP
:
10365 case GIMPLE_OMP_ORDERED
:
10366 case GIMPLE_OMP_CRITICAL
:
10367 case GIMPLE_OMP_SECTION
:
10368 cur_region
= new_omp_region (bb
, code
, cur_region
);
10372 case GIMPLE_OMP_TARGET
:
10373 cur_region
= new_omp_region (bb
, code
, cur_region
);
10375 if (gimple_omp_target_kind (last
) == GF_OMP_TARGET_KIND_UPDATE
)
10376 cur_region
= cur_region
->outer
;
10379 case GIMPLE_OMP_SECTIONS
:
10380 cur_region
= new_omp_region (bb
, code
, cur_region
);
10384 case GIMPLE_OMP_SECTIONS_SWITCH
:
10388 case GIMPLE_OMP_ATOMIC_LOAD
:
10389 case GIMPLE_OMP_ATOMIC_STORE
:
10393 case GIMPLE_OMP_RETURN
:
10394 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
10395 somewhere other than the next block. This will be
10397 cur_region
->exit
= bb
;
10398 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
10399 cur_region
= cur_region
->outer
;
10402 case GIMPLE_OMP_CONTINUE
:
10403 cur_region
->cont
= bb
;
10404 switch (cur_region
->type
)
10406 case GIMPLE_OMP_FOR
:
10407 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
10408 succs edges as abnormal to prevent splitting
10410 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
10411 /* Make the loopback edge. */
10412 make_edge (bb
, single_succ (cur_region
->entry
),
10415 /* Create an edge from GIMPLE_OMP_FOR to exit, which
10416 corresponds to the case that the body of the loop
10417 is not executed at all. */
10418 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
10419 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
10423 case GIMPLE_OMP_SECTIONS
:
10424 /* Wire up the edges into and out of the nested sections. */
10426 basic_block switch_bb
= single_succ (cur_region
->entry
);
10428 struct omp_region
*i
;
10429 for (i
= cur_region
->inner
; i
; i
= i
->next
)
10431 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
10432 make_edge (switch_bb
, i
->entry
, 0);
10433 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
10436 /* Make the loopback edge to the block with
10437 GIMPLE_OMP_SECTIONS_SWITCH. */
10438 make_edge (bb
, switch_bb
, 0);
10440 /* Make the edge from the switch to exit. */
10441 make_edge (switch_bb
, bb
->next_bb
, 0);
10447 gcc_unreachable ();
10452 gcc_unreachable ();
10455 if (*region
!= cur_region
)
10456 *region
= cur_region
;
10461 static unsigned int
10462 diagnose_omp_structured_block_errors (void)
10464 struct walk_stmt_info wi
;
10465 gimple_seq body
= gimple_body (current_function_decl
);
10467 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
10469 memset (&wi
, 0, sizeof (wi
));
10470 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
10472 memset (&wi
, 0, sizeof (wi
));
10473 wi
.want_locations
= true;
10474 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
10476 gimple_set_body (current_function_decl
, body
);
10478 splay_tree_delete (all_labels
);
10485 gate_diagnose_omp_blocks (void)
10487 return flag_openmp
!= 0;
10492 const pass_data pass_data_diagnose_omp_blocks
=
10494 GIMPLE_PASS
, /* type */
10495 "*diagnose_omp_blocks", /* name */
10496 OPTGROUP_NONE
, /* optinfo_flags */
10497 true, /* has_gate */
10498 true, /* has_execute */
10499 TV_NONE
, /* tv_id */
10500 PROP_gimple_any
, /* properties_required */
10501 0, /* properties_provided */
10502 0, /* properties_destroyed */
10503 0, /* todo_flags_start */
10504 0, /* todo_flags_finish */
10507 class pass_diagnose_omp_blocks
: public gimple_opt_pass
10510 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
10511 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
10514 /* opt_pass methods: */
10515 bool gate () { return gate_diagnose_omp_blocks (); }
10516 unsigned int execute () {
10517 return diagnose_omp_structured_block_errors ();
10520 }; // class pass_diagnose_omp_blocks
10522 } // anon namespace
10525 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
10527 return new pass_diagnose_omp_blocks (ctxt
);
10530 #include "gt-omp-low.h"