1 /* Expansion pass for OMP directives. Outlines regions of certain OMP
2 directives to separate functions, converts others into explicit calls to the
3 runtime library (libgomp) and so forth
5 Copyright (C) 2005-2019 Free Software Foundation, Inc.
7 This file is part of GCC.
9 GCC is free software; you can redistribute it and/or modify it under
10 the terms of the GNU General Public License as published by the Free
11 Software Foundation; either version 3, or (at your option) any later
14 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
15 WARRANTY; without even the implied warranty of MERCHANTABILITY or
16 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19 You should have received a copy of the GNU General Public License
20 along with GCC; see the file COPYING3. If not see
21 <http://www.gnu.org/licenses/>. */
25 #include "coretypes.h"
33 #include "tree-pass.h"
37 #include "pretty-print.h"
38 #include "diagnostic-core.h"
39 #include "fold-const.h"
40 #include "stor-layout.h"
42 #include "internal-fn.h"
44 #include "gimple-iterator.h"
45 #include "gimplify-me.h"
46 #include "gimple-walk.h"
48 #include "tree-into-ssa.h"
50 #include "splay-tree.h"
52 #include "omp-general.h"
53 #include "omp-offload.h"
54 #include "tree-cfgcleanup.h"
55 #include "symbol-summary.h"
56 #include "gomp-constants.h"
57 #include "gimple-pretty-print.h"
58 #include "hsa-common.h"
59 #include "stringpool.h"
62 /* OMP region information. Every parallel and workshare
63 directive is enclosed between two markers, the OMP_* directive
64 and a corresponding GIMPLE_OMP_RETURN statement. */
68 /* The enclosing region. */
69 struct omp_region
*outer
;
71 /* First child region. */
72 struct omp_region
*inner
;
74 /* Next peer region. */
75 struct omp_region
*next
;
77 /* Block containing the omp directive as its last stmt. */
80 /* Block containing the GIMPLE_OMP_RETURN as its last stmt. */
83 /* Block containing the GIMPLE_OMP_CONTINUE as its last stmt. */
86 /* If this is a combined parallel+workshare region, this is a list
87 of additional arguments needed by the combined parallel+workshare
89 vec
<tree
, va_gc
> *ws_args
;
91 /* The code for the omp directive of this region. */
92 enum gimple_code type
;
94 /* Schedule kind, only used for GIMPLE_OMP_FOR type regions. */
95 enum omp_clause_schedule_kind sched_kind
;
97 /* Schedule modifiers. */
98 unsigned char sched_modifiers
;
100 /* True if this is a combined parallel+workshare region. */
101 bool is_combined_parallel
;
103 /* Copy of fd.lastprivate_conditional != 0. */
104 bool has_lastprivate_conditional
;
106 /* The ordered stmt if type is GIMPLE_OMP_ORDERED and it has
108 gomp_ordered
*ord_stmt
;
111 static struct omp_region
*root_omp_region
;
112 static bool omp_any_child_fn_dumped
;
114 static void expand_omp_build_assign (gimple_stmt_iterator
*, tree
, tree
,
116 static gphi
*find_phi_with_arg_on_edge (tree
, edge
);
117 static void expand_omp (struct omp_region
*region
);
119 /* Return true if REGION is a combined parallel+workshare region. */
122 is_combined_parallel (struct omp_region
*region
)
124 return region
->is_combined_parallel
;
127 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
128 is the immediate dominator of PAR_ENTRY_BB, return true if there
129 are no data dependencies that would prevent expanding the parallel
130 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
132 When expanding a combined parallel+workshare region, the call to
133 the child function may need additional arguments in the case of
134 GIMPLE_OMP_FOR regions. In some cases, these arguments are
135 computed out of variables passed in from the parent to the child
136 via 'struct .omp_data_s'. For instance:
138 #pragma omp parallel for schedule (guided, i * 4)
143 # BLOCK 2 (PAR_ENTRY_BB)
145 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
147 # BLOCK 3 (WS_ENTRY_BB)
148 .omp_data_i = &.omp_data_o;
149 D.1667 = .omp_data_i->i;
151 #pragma omp for schedule (guided, D.1598)
153 When we outline the parallel region, the call to the child function
154 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
155 that value is computed *after* the call site. So, in principle we
156 cannot do the transformation.
158 To see whether the code in WS_ENTRY_BB blocks the combined
159 parallel+workshare call, we collect all the variables used in the
160 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
161 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
164 FIXME. If we had the SSA form built at this point, we could merely
165 hoist the code in block 3 into block 2 and be done with it. But at
166 this point we don't have dataflow information and though we could
167 hack something up here, it is really not worth the aggravation. */
170 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
172 struct omp_for_data fd
;
173 gimple
*ws_stmt
= last_stmt (ws_entry_bb
);
175 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
178 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
179 if (gimple_omp_for_kind (ws_stmt
) != GF_OMP_FOR_KIND_FOR
)
182 omp_extract_for_data (as_a
<gomp_for
*> (ws_stmt
), &fd
, NULL
);
184 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
186 if (fd
.iter_type
!= long_integer_type_node
)
189 /* FIXME. We give up too easily here. If any of these arguments
190 are not constants, they will likely involve variables that have
191 been mapped into fields of .omp_data_s for sharing with the child
192 function. With appropriate data flow, it would be possible to
194 if (!is_gimple_min_invariant (fd
.loop
.n1
)
195 || !is_gimple_min_invariant (fd
.loop
.n2
)
196 || !is_gimple_min_invariant (fd
.loop
.step
)
197 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
203 /* Adjust CHUNK_SIZE from SCHEDULE clause, depending on simd modifier
204 presence (SIMD_SCHEDULE). */
207 omp_adjust_chunk_size (tree chunk_size
, bool simd_schedule
)
209 if (!simd_schedule
|| integer_zerop (chunk_size
))
212 poly_uint64 vf
= omp_max_vf ();
213 if (known_eq (vf
, 1U))
216 tree type
= TREE_TYPE (chunk_size
);
217 chunk_size
= fold_build2 (PLUS_EXPR
, type
, chunk_size
,
218 build_int_cst (type
, vf
- 1));
219 return fold_build2 (BIT_AND_EXPR
, type
, chunk_size
,
220 build_int_cst (type
, -vf
));
223 /* Collect additional arguments needed to emit a combined
224 parallel+workshare call. WS_STMT is the workshare directive being
227 static vec
<tree
, va_gc
> *
228 get_ws_args_for (gimple
*par_stmt
, gimple
*ws_stmt
)
231 location_t loc
= gimple_location (ws_stmt
);
232 vec
<tree
, va_gc
> *ws_args
;
234 if (gomp_for
*for_stmt
= dyn_cast
<gomp_for
*> (ws_stmt
))
236 struct omp_for_data fd
;
239 omp_extract_for_data (for_stmt
, &fd
, NULL
);
243 if (gimple_omp_for_combined_into_p (for_stmt
))
246 = omp_find_clause (gimple_omp_parallel_clauses (par_stmt
),
247 OMP_CLAUSE__LOOPTEMP_
);
249 n1
= OMP_CLAUSE_DECL (innerc
);
250 innerc
= omp_find_clause (OMP_CLAUSE_CHAIN (innerc
),
251 OMP_CLAUSE__LOOPTEMP_
);
253 n2
= OMP_CLAUSE_DECL (innerc
);
256 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
258 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
259 ws_args
->quick_push (t
);
261 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
262 ws_args
->quick_push (t
);
264 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
265 ws_args
->quick_push (t
);
269 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
270 t
= omp_adjust_chunk_size (t
, fd
.simd_schedule
);
271 ws_args
->quick_push (t
);
276 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
278 /* Number of sections is equal to the number of edges from the
279 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
280 the exit of the sections region. */
281 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
282 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
283 vec_alloc (ws_args
, 1);
284 ws_args
->quick_push (t
);
291 /* Discover whether REGION is a combined parallel+workshare region. */
294 determine_parallel_type (struct omp_region
*region
)
296 basic_block par_entry_bb
, par_exit_bb
;
297 basic_block ws_entry_bb
, ws_exit_bb
;
299 if (region
== NULL
|| region
->inner
== NULL
300 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
301 || region
->inner
->cont
== NULL
)
304 /* We only support parallel+for and parallel+sections. */
305 if (region
->type
!= GIMPLE_OMP_PARALLEL
306 || (region
->inner
->type
!= GIMPLE_OMP_FOR
307 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
310 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
311 WS_EXIT_BB -> PAR_EXIT_BB. */
312 par_entry_bb
= region
->entry
;
313 par_exit_bb
= region
->exit
;
314 ws_entry_bb
= region
->inner
->entry
;
315 ws_exit_bb
= region
->inner
->exit
;
317 /* Give up for task reductions on the parallel, while it is implementable,
318 adding another big set of APIs or slowing down the normal paths is
320 tree pclauses
= gimple_omp_parallel_clauses (last_stmt (par_entry_bb
));
321 if (omp_find_clause (pclauses
, OMP_CLAUSE__REDUCTEMP_
))
324 if (single_succ (par_entry_bb
) == ws_entry_bb
325 && single_succ (ws_exit_bb
) == par_exit_bb
326 && workshare_safe_to_combine_p (ws_entry_bb
)
327 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
328 || (last_and_only_stmt (ws_entry_bb
)
329 && last_and_only_stmt (par_exit_bb
))))
331 gimple
*par_stmt
= last_stmt (par_entry_bb
);
332 gimple
*ws_stmt
= last_stmt (ws_entry_bb
);
334 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
336 /* If this is a combined parallel loop, we need to determine
337 whether or not to use the combined library calls. There
338 are two cases where we do not apply the transformation:
339 static loops and any kind of ordered loop. In the first
340 case, we already open code the loop so there is no need
341 to do anything else. In the latter case, the combined
342 parallel loop call would still need extra synchronization
343 to implement ordered semantics, so there would not be any
344 gain in using the combined call. */
345 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
346 tree c
= omp_find_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
348 || ((OMP_CLAUSE_SCHEDULE_KIND (c
) & OMP_CLAUSE_SCHEDULE_MASK
)
349 == OMP_CLAUSE_SCHEDULE_STATIC
)
350 || omp_find_clause (clauses
, OMP_CLAUSE_ORDERED
)
351 || omp_find_clause (clauses
, OMP_CLAUSE__REDUCTEMP_
)
352 || ((c
= omp_find_clause (clauses
, OMP_CLAUSE__CONDTEMP_
))
353 && POINTER_TYPE_P (TREE_TYPE (OMP_CLAUSE_DECL (c
)))))
356 else if (region
->inner
->type
== GIMPLE_OMP_SECTIONS
357 && (omp_find_clause (gimple_omp_sections_clauses (ws_stmt
),
358 OMP_CLAUSE__REDUCTEMP_
)
359 || omp_find_clause (gimple_omp_sections_clauses (ws_stmt
),
360 OMP_CLAUSE__CONDTEMP_
)))
363 region
->is_combined_parallel
= true;
364 region
->inner
->is_combined_parallel
= true;
365 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
369 /* Debugging dumps for parallel regions. */
370 void dump_omp_region (FILE *, struct omp_region
*, int);
371 void debug_omp_region (struct omp_region
*);
372 void debug_all_omp_regions (void);
374 /* Dump the parallel region tree rooted at REGION. */
377 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
379 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
380 gimple_code_name
[region
->type
]);
383 dump_omp_region (file
, region
->inner
, indent
+ 4);
387 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
388 region
->cont
->index
);
392 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
393 region
->exit
->index
);
395 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
398 dump_omp_region (file
, region
->next
, indent
);
402 debug_omp_region (struct omp_region
*region
)
404 dump_omp_region (stderr
, region
, 0);
408 debug_all_omp_regions (void)
410 dump_omp_region (stderr
, root_omp_region
, 0);
413 /* Create a new parallel region starting at STMT inside region PARENT. */
415 static struct omp_region
*
416 new_omp_region (basic_block bb
, enum gimple_code type
,
417 struct omp_region
*parent
)
419 struct omp_region
*region
= XCNEW (struct omp_region
);
421 region
->outer
= parent
;
427 /* This is a nested region. Add it to the list of inner
428 regions in PARENT. */
429 region
->next
= parent
->inner
;
430 parent
->inner
= region
;
434 /* This is a toplevel region. Add it to the list of toplevel
435 regions in ROOT_OMP_REGION. */
436 region
->next
= root_omp_region
;
437 root_omp_region
= region
;
443 /* Release the memory associated with the region tree rooted at REGION. */
446 free_omp_region_1 (struct omp_region
*region
)
448 struct omp_region
*i
, *n
;
450 for (i
= region
->inner
; i
; i
= n
)
453 free_omp_region_1 (i
);
459 /* Release the memory for the entire omp region tree. */
462 omp_free_regions (void)
464 struct omp_region
*r
, *n
;
465 for (r
= root_omp_region
; r
; r
= n
)
468 free_omp_region_1 (r
);
470 root_omp_region
= NULL
;
473 /* A convenience function to build an empty GIMPLE_COND with just the
477 gimple_build_cond_empty (tree cond
)
479 enum tree_code pred_code
;
482 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
483 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
486 /* Return true if a parallel REGION is within a declare target function or
487 within a target region and is not a part of a gridified target. */
490 parallel_needs_hsa_kernel_p (struct omp_region
*region
)
492 bool indirect
= false;
493 for (region
= region
->outer
; region
; region
= region
->outer
)
495 if (region
->type
== GIMPLE_OMP_PARALLEL
)
497 else if (region
->type
== GIMPLE_OMP_TARGET
)
499 gomp_target
*tgt_stmt
500 = as_a
<gomp_target
*> (last_stmt (region
->entry
));
502 if (omp_find_clause (gimple_omp_target_clauses (tgt_stmt
),
503 OMP_CLAUSE__GRIDDIM_
))
510 if (lookup_attribute ("omp declare target",
511 DECL_ATTRIBUTES (current_function_decl
)))
517 /* Change DECL_CONTEXT of CHILD_FNDECL to that of the parent function.
518 Add CHILD_FNDECL to decl chain of the supercontext of the block
519 ENTRY_BLOCK - this is the block which originally contained the
520 code from which CHILD_FNDECL was created.
522 Together, these actions ensure that the debug info for the outlined
523 function will be emitted with the correct lexical scope. */
526 adjust_context_and_scope (struct omp_region
*region
, tree entry_block
,
529 tree parent_fndecl
= NULL_TREE
;
531 /* OMP expansion expands inner regions before outer ones, so if
532 we e.g. have explicit task region nested in parallel region, when
533 expanding the task region current_function_decl will be the original
534 source function, but we actually want to use as context the child
535 function of the parallel. */
536 for (region
= region
->outer
;
537 region
&& parent_fndecl
== NULL_TREE
; region
= region
->outer
)
538 switch (region
->type
)
540 case GIMPLE_OMP_PARALLEL
:
541 case GIMPLE_OMP_TASK
:
542 case GIMPLE_OMP_TEAMS
:
543 entry_stmt
= last_stmt (region
->entry
);
544 parent_fndecl
= gimple_omp_taskreg_child_fn (entry_stmt
);
546 case GIMPLE_OMP_TARGET
:
547 entry_stmt
= last_stmt (region
->entry
);
549 = gimple_omp_target_child_fn (as_a
<gomp_target
*> (entry_stmt
));
555 if (parent_fndecl
== NULL_TREE
)
556 parent_fndecl
= current_function_decl
;
557 DECL_CONTEXT (child_fndecl
) = parent_fndecl
;
559 if (entry_block
!= NULL_TREE
&& TREE_CODE (entry_block
) == BLOCK
)
561 tree b
= BLOCK_SUPERCONTEXT (entry_block
);
562 if (TREE_CODE (b
) == BLOCK
)
564 DECL_CHAIN (child_fndecl
) = BLOCK_VARS (b
);
565 BLOCK_VARS (b
) = child_fndecl
;
570 /* Build the function calls to GOMP_parallel etc to actually
571 generate the parallel operation. REGION is the parallel region
572 being expanded. BB is the block where to insert the code. WS_ARGS
573 will be set if this is a call to a combined parallel+workshare
574 construct, it contains the list of additional arguments needed by
575 the workshare construct. */
578 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
579 gomp_parallel
*entry_stmt
,
580 vec
<tree
, va_gc
> *ws_args
)
582 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
583 gimple_stmt_iterator gsi
;
585 enum built_in_function start_ix
;
587 location_t clause_loc
;
588 vec
<tree
, va_gc
> *args
;
590 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
592 /* Determine what flavor of GOMP_parallel we will be
594 start_ix
= BUILT_IN_GOMP_PARALLEL
;
595 tree rtmp
= omp_find_clause (clauses
, OMP_CLAUSE__REDUCTEMP_
);
597 start_ix
= BUILT_IN_GOMP_PARALLEL_REDUCTIONS
;
598 else if (is_combined_parallel (region
))
600 switch (region
->inner
->type
)
603 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
604 switch (region
->inner
->sched_kind
)
606 case OMP_CLAUSE_SCHEDULE_RUNTIME
:
607 /* For lastprivate(conditional:), our implementation
608 requires monotonic behavior. */
609 if (region
->inner
->has_lastprivate_conditional
!= 0)
611 else if ((region
->inner
->sched_modifiers
612 & OMP_CLAUSE_SCHEDULE_NONMONOTONIC
) != 0)
614 else if ((region
->inner
->sched_modifiers
615 & OMP_CLAUSE_SCHEDULE_MONOTONIC
) == 0)
620 case OMP_CLAUSE_SCHEDULE_DYNAMIC
:
621 case OMP_CLAUSE_SCHEDULE_GUIDED
:
622 if ((region
->inner
->sched_modifiers
623 & OMP_CLAUSE_SCHEDULE_MONOTONIC
) == 0
624 && !region
->inner
->has_lastprivate_conditional
)
626 start_ix2
= 3 + region
->inner
->sched_kind
;
631 start_ix2
= region
->inner
->sched_kind
;
634 start_ix2
+= (int) BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
;
635 start_ix
= (enum built_in_function
) start_ix2
;
637 case GIMPLE_OMP_SECTIONS
:
638 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
645 /* By default, the value of NUM_THREADS is zero (selected at run time)
646 and there is no conditional. */
648 val
= build_int_cst (unsigned_type_node
, 0);
649 flags
= build_int_cst (unsigned_type_node
, 0);
651 c
= omp_find_clause (clauses
, OMP_CLAUSE_IF
);
653 cond
= OMP_CLAUSE_IF_EXPR (c
);
655 c
= omp_find_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
658 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
659 clause_loc
= OMP_CLAUSE_LOCATION (c
);
662 clause_loc
= gimple_location (entry_stmt
);
664 c
= omp_find_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
666 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
668 /* Ensure 'val' is of the correct type. */
669 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
671 /* If we found the clause 'if (cond)', build either
672 (cond != 0) or (cond ? val : 1u). */
675 cond
= gimple_boolify (cond
);
677 if (integer_zerop (val
))
678 val
= fold_build2_loc (clause_loc
,
679 EQ_EXPR
, unsigned_type_node
, cond
,
680 build_int_cst (TREE_TYPE (cond
), 0));
683 basic_block cond_bb
, then_bb
, else_bb
;
684 edge e
, e_then
, e_else
;
685 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
687 tmp_var
= create_tmp_var (TREE_TYPE (val
));
688 if (gimple_in_ssa_p (cfun
))
690 tmp_then
= make_ssa_name (tmp_var
);
691 tmp_else
= make_ssa_name (tmp_var
);
692 tmp_join
= make_ssa_name (tmp_var
);
701 e
= split_block_after_labels (bb
);
706 then_bb
= create_empty_bb (cond_bb
);
707 else_bb
= create_empty_bb (then_bb
);
708 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
709 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
711 stmt
= gimple_build_cond_empty (cond
);
712 gsi
= gsi_start_bb (cond_bb
);
713 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
715 gsi
= gsi_start_bb (then_bb
);
716 expand_omp_build_assign (&gsi
, tmp_then
, val
, true);
718 gsi
= gsi_start_bb (else_bb
);
719 expand_omp_build_assign (&gsi
, tmp_else
,
720 build_int_cst (unsigned_type_node
, 1),
723 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
724 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
725 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
726 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
727 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
728 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
730 if (gimple_in_ssa_p (cfun
))
732 gphi
*phi
= create_phi_node (tmp_join
, bb
);
733 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
734 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
740 gsi
= gsi_start_bb (bb
);
741 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
742 false, GSI_CONTINUE_LINKING
);
745 gsi
= gsi_last_nondebug_bb (bb
);
746 t
= gimple_omp_parallel_data_arg (entry_stmt
);
748 t1
= null_pointer_node
;
750 t1
= build_fold_addr_expr (t
);
751 tree child_fndecl
= gimple_omp_parallel_child_fn (entry_stmt
);
752 t2
= build_fold_addr_expr (child_fndecl
);
754 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
755 args
->quick_push (t2
);
756 args
->quick_push (t1
);
757 args
->quick_push (val
);
759 args
->splice (*ws_args
);
760 args
->quick_push (flags
);
762 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
763 builtin_decl_explicit (start_ix
), args
);
767 tree type
= TREE_TYPE (OMP_CLAUSE_DECL (rtmp
));
768 t
= build2 (MODIFY_EXPR
, type
, OMP_CLAUSE_DECL (rtmp
),
770 fold_convert (pointer_sized_int_node
, t
)));
772 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
773 false, GSI_CONTINUE_LINKING
);
775 if (hsa_gen_requested_p ()
776 && parallel_needs_hsa_kernel_p (region
))
778 cgraph_node
*child_cnode
= cgraph_node::get (child_fndecl
);
779 hsa_register_kernel (child_cnode
);
783 /* Build the function call to GOMP_task to actually
784 generate the task operation. BB is the block where to insert the code. */
787 expand_task_call (struct omp_region
*region
, basic_block bb
,
788 gomp_task
*entry_stmt
)
791 gimple_stmt_iterator gsi
;
792 location_t loc
= gimple_location (entry_stmt
);
794 tree clauses
= gimple_omp_task_clauses (entry_stmt
);
796 tree ifc
= omp_find_clause (clauses
, OMP_CLAUSE_IF
);
797 tree untied
= omp_find_clause (clauses
, OMP_CLAUSE_UNTIED
);
798 tree mergeable
= omp_find_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
799 tree depend
= omp_find_clause (clauses
, OMP_CLAUSE_DEPEND
);
800 tree finalc
= omp_find_clause (clauses
, OMP_CLAUSE_FINAL
);
801 tree priority
= omp_find_clause (clauses
, OMP_CLAUSE_PRIORITY
);
804 = (untied
? GOMP_TASK_FLAG_UNTIED
: 0)
805 | (mergeable
? GOMP_TASK_FLAG_MERGEABLE
: 0)
806 | (depend
? GOMP_TASK_FLAG_DEPEND
: 0);
808 bool taskloop_p
= gimple_omp_task_taskloop_p (entry_stmt
);
809 tree startvar
= NULL_TREE
, endvar
= NULL_TREE
, step
= NULL_TREE
;
810 tree num_tasks
= NULL_TREE
;
814 gimple
*g
= last_stmt (region
->outer
->entry
);
815 gcc_assert (gimple_code (g
) == GIMPLE_OMP_FOR
816 && gimple_omp_for_kind (g
) == GF_OMP_FOR_KIND_TASKLOOP
);
817 struct omp_for_data fd
;
818 omp_extract_for_data (as_a
<gomp_for
*> (g
), &fd
, NULL
);
819 startvar
= omp_find_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
820 endvar
= omp_find_clause (OMP_CLAUSE_CHAIN (startvar
),
821 OMP_CLAUSE__LOOPTEMP_
);
822 startvar
= OMP_CLAUSE_DECL (startvar
);
823 endvar
= OMP_CLAUSE_DECL (endvar
);
824 step
= fold_convert_loc (loc
, fd
.iter_type
, fd
.loop
.step
);
825 if (fd
.loop
.cond_code
== LT_EXPR
)
826 iflags
|= GOMP_TASK_FLAG_UP
;
827 tree tclauses
= gimple_omp_for_clauses (g
);
828 num_tasks
= omp_find_clause (tclauses
, OMP_CLAUSE_NUM_TASKS
);
830 num_tasks
= OMP_CLAUSE_NUM_TASKS_EXPR (num_tasks
);
833 num_tasks
= omp_find_clause (tclauses
, OMP_CLAUSE_GRAINSIZE
);
836 iflags
|= GOMP_TASK_FLAG_GRAINSIZE
;
837 num_tasks
= OMP_CLAUSE_GRAINSIZE_EXPR (num_tasks
);
840 num_tasks
= integer_zero_node
;
842 num_tasks
= fold_convert_loc (loc
, long_integer_type_node
, num_tasks
);
843 if (ifc
== NULL_TREE
)
844 iflags
|= GOMP_TASK_FLAG_IF
;
845 if (omp_find_clause (tclauses
, OMP_CLAUSE_NOGROUP
))
846 iflags
|= GOMP_TASK_FLAG_NOGROUP
;
847 ull
= fd
.iter_type
== long_long_unsigned_type_node
;
848 if (omp_find_clause (clauses
, OMP_CLAUSE_REDUCTION
))
849 iflags
|= GOMP_TASK_FLAG_REDUCTION
;
852 iflags
|= GOMP_TASK_FLAG_PRIORITY
;
854 tree flags
= build_int_cst (unsigned_type_node
, iflags
);
856 tree cond
= boolean_true_node
;
861 tree t
= gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc
));
862 t
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, t
,
863 build_int_cst (unsigned_type_node
,
865 build_int_cst (unsigned_type_node
, 0));
866 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
,
870 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc
));
875 tree t
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (finalc
));
876 t
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, t
,
877 build_int_cst (unsigned_type_node
,
878 GOMP_TASK_FLAG_FINAL
),
879 build_int_cst (unsigned_type_node
, 0));
880 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, t
);
883 depend
= OMP_CLAUSE_DECL (depend
);
885 depend
= build_int_cst (ptr_type_node
, 0);
887 priority
= fold_convert (integer_type_node
,
888 OMP_CLAUSE_PRIORITY_EXPR (priority
));
890 priority
= integer_zero_node
;
892 gsi
= gsi_last_nondebug_bb (bb
);
893 tree t
= gimple_omp_task_data_arg (entry_stmt
);
895 t2
= null_pointer_node
;
897 t2
= build_fold_addr_expr_loc (loc
, t
);
898 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
899 t
= gimple_omp_task_copy_fn (entry_stmt
);
901 t3
= null_pointer_node
;
903 t3
= build_fold_addr_expr_loc (loc
, t
);
906 t
= build_call_expr (ull
907 ? builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP_ULL
)
908 : builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP
),
910 gimple_omp_task_arg_size (entry_stmt
),
911 gimple_omp_task_arg_align (entry_stmt
), flags
,
912 num_tasks
, priority
, startvar
, endvar
, step
);
914 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
916 gimple_omp_task_arg_size (entry_stmt
),
917 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
920 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
921 false, GSI_CONTINUE_LINKING
);
924 /* Build the function call to GOMP_taskwait_depend to actually
925 generate the taskwait operation. BB is the block where to insert the
929 expand_taskwait_call (basic_block bb
, gomp_task
*entry_stmt
)
931 tree clauses
= gimple_omp_task_clauses (entry_stmt
);
932 tree depend
= omp_find_clause (clauses
, OMP_CLAUSE_DEPEND
);
933 if (depend
== NULL_TREE
)
936 depend
= OMP_CLAUSE_DECL (depend
);
938 gimple_stmt_iterator gsi
= gsi_last_nondebug_bb (bb
);
940 = build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASKWAIT_DEPEND
),
943 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
944 false, GSI_CONTINUE_LINKING
);
947 /* Build the function call to GOMP_teams_reg to actually
948 generate the host teams operation. REGION is the teams region
949 being expanded. BB is the block where to insert the code. */
952 expand_teams_call (basic_block bb
, gomp_teams
*entry_stmt
)
954 tree clauses
= gimple_omp_teams_clauses (entry_stmt
);
955 tree num_teams
= omp_find_clause (clauses
, OMP_CLAUSE_NUM_TEAMS
);
956 if (num_teams
== NULL_TREE
)
957 num_teams
= build_int_cst (unsigned_type_node
, 0);
960 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
961 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
963 tree thread_limit
= omp_find_clause (clauses
, OMP_CLAUSE_THREAD_LIMIT
);
964 if (thread_limit
== NULL_TREE
)
965 thread_limit
= build_int_cst (unsigned_type_node
, 0);
968 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
969 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
972 gimple_stmt_iterator gsi
= gsi_last_nondebug_bb (bb
);
973 tree t
= gimple_omp_teams_data_arg (entry_stmt
), t1
;
975 t1
= null_pointer_node
;
977 t1
= build_fold_addr_expr (t
);
978 tree child_fndecl
= gimple_omp_teams_child_fn (entry_stmt
);
979 tree t2
= build_fold_addr_expr (child_fndecl
);
981 vec
<tree
, va_gc
> *args
;
983 args
->quick_push (t2
);
984 args
->quick_push (t1
);
985 args
->quick_push (num_teams
);
986 args
->quick_push (thread_limit
);
987 /* For future extensibility. */
988 args
->quick_push (build_zero_cst (unsigned_type_node
));
990 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
991 builtin_decl_explicit (BUILT_IN_GOMP_TEAMS_REG
),
994 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
995 false, GSI_CONTINUE_LINKING
);
998 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
1001 vec2chain (vec
<tree
, va_gc
> *v
)
1003 tree chain
= NULL_TREE
, t
;
1006 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
1008 DECL_CHAIN (t
) = chain
;
1015 /* Remove barriers in REGION->EXIT's block. Note that this is only
1016 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
1017 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
1018 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
1022 remove_exit_barrier (struct omp_region
*region
)
1024 gimple_stmt_iterator gsi
;
1025 basic_block exit_bb
;
1029 int any_addressable_vars
= -1;
1031 exit_bb
= region
->exit
;
1033 /* If the parallel region doesn't return, we don't have REGION->EXIT
1038 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
1039 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
1040 statements that can appear in between are extremely limited -- no
1041 memory operations at all. Here, we allow nothing at all, so the
1042 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
1043 gsi
= gsi_last_nondebug_bb (exit_bb
);
1044 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
1045 gsi_prev_nondebug (&gsi
);
1046 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
1049 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
1051 gsi
= gsi_last_nondebug_bb (e
->src
);
1052 if (gsi_end_p (gsi
))
1054 stmt
= gsi_stmt (gsi
);
1055 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
1056 && !gimple_omp_return_nowait_p (stmt
))
1058 /* OpenMP 3.0 tasks unfortunately prevent this optimization
1059 in many cases. If there could be tasks queued, the barrier
1060 might be needed to let the tasks run before some local
1061 variable of the parallel that the task uses as shared
1062 runs out of scope. The task can be spawned either
1063 from within current function (this would be easy to check)
1064 or from some function it calls and gets passed an address
1065 of such a variable. */
1066 if (any_addressable_vars
< 0)
1068 gomp_parallel
*parallel_stmt
1069 = as_a
<gomp_parallel
*> (last_stmt (region
->entry
));
1070 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
1071 tree local_decls
, block
, decl
;
1074 any_addressable_vars
= 0;
1075 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
1076 if (TREE_ADDRESSABLE (decl
))
1078 any_addressable_vars
= 1;
1081 for (block
= gimple_block (stmt
);
1082 !any_addressable_vars
1084 && TREE_CODE (block
) == BLOCK
;
1085 block
= BLOCK_SUPERCONTEXT (block
))
1087 for (local_decls
= BLOCK_VARS (block
);
1089 local_decls
= DECL_CHAIN (local_decls
))
1090 if (TREE_ADDRESSABLE (local_decls
))
1092 any_addressable_vars
= 1;
1095 if (block
== gimple_block (parallel_stmt
))
1099 if (!any_addressable_vars
)
1100 gimple_omp_return_set_nowait (stmt
);
1106 remove_exit_barriers (struct omp_region
*region
)
1108 if (region
->type
== GIMPLE_OMP_PARALLEL
)
1109 remove_exit_barrier (region
);
1113 region
= region
->inner
;
1114 remove_exit_barriers (region
);
1115 while (region
->next
)
1117 region
= region
->next
;
1118 remove_exit_barriers (region
);
1123 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
1124 calls. These can't be declared as const functions, but
1125 within one parallel body they are constant, so they can be
1126 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
1127 which are declared const. Similarly for task body, except
1128 that in untied task omp_get_thread_num () can change at any task
1129 scheduling point. */
1132 optimize_omp_library_calls (gimple
*entry_stmt
)
1135 gimple_stmt_iterator gsi
;
1136 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
1137 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
1138 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
1139 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
1140 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
1141 && omp_find_clause (gimple_omp_task_clauses (entry_stmt
),
1142 OMP_CLAUSE_UNTIED
) != NULL
);
1144 FOR_EACH_BB_FN (bb
, cfun
)
1145 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1147 gimple
*call
= gsi_stmt (gsi
);
1150 if (is_gimple_call (call
)
1151 && (decl
= gimple_call_fndecl (call
))
1152 && DECL_EXTERNAL (decl
)
1153 && TREE_PUBLIC (decl
)
1154 && DECL_INITIAL (decl
) == NULL
)
1158 if (DECL_NAME (decl
) == thr_num_id
)
1160 /* In #pragma omp task untied omp_get_thread_num () can change
1161 during the execution of the task region. */
1164 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
1166 else if (DECL_NAME (decl
) == num_thr_id
)
1167 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
1171 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
1172 || gimple_call_num_args (call
) != 0)
1175 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
1178 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
1179 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
1180 TREE_TYPE (TREE_TYPE (built_in
))))
1183 gimple_call_set_fndecl (call
, built_in
);
1188 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
1192 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
1196 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
1197 if (VAR_P (t
) && DECL_HAS_VALUE_EXPR_P (t
))
1200 if (TREE_CODE (t
) == ADDR_EXPR
)
1201 recompute_tree_invariant_for_addr_expr (t
);
1203 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
1207 /* Prepend or append TO = FROM assignment before or after *GSI_P. */
1210 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
,
1213 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
1214 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
1215 !after
, after
? GSI_CONTINUE_LINKING
1217 gimple
*stmt
= gimple_build_assign (to
, from
);
1219 gsi_insert_after (gsi_p
, stmt
, GSI_CONTINUE_LINKING
);
1221 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
1222 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
1223 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
1225 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
1226 gimple_regimplify_operands (stmt
, &gsi
);
1230 /* Expand the OpenMP parallel or task directive starting at REGION. */
1233 expand_omp_taskreg (struct omp_region
*region
)
1235 basic_block entry_bb
, exit_bb
, new_bb
;
1236 struct function
*child_cfun
;
1237 tree child_fn
, block
, t
;
1238 gimple_stmt_iterator gsi
;
1239 gimple
*entry_stmt
, *stmt
;
1241 vec
<tree
, va_gc
> *ws_args
;
1243 entry_stmt
= last_stmt (region
->entry
);
1244 if (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
1245 && gimple_omp_task_taskwait_p (entry_stmt
))
1247 new_bb
= region
->entry
;
1248 gsi
= gsi_last_nondebug_bb (region
->entry
);
1249 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
1250 gsi_remove (&gsi
, true);
1251 expand_taskwait_call (new_bb
, as_a
<gomp_task
*> (entry_stmt
));
1255 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
1256 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1258 entry_bb
= region
->entry
;
1259 if (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
)
1260 exit_bb
= region
->cont
;
1262 exit_bb
= region
->exit
;
1264 if (is_combined_parallel (region
))
1265 ws_args
= region
->ws_args
;
1269 if (child_cfun
->cfg
)
1271 /* Due to inlining, it may happen that we have already outlined
1272 the region, in which case all we need to do is make the
1273 sub-graph unreachable and emit the parallel call. */
1274 edge entry_succ_e
, exit_succ_e
;
1276 entry_succ_e
= single_succ_edge (entry_bb
);
1278 gsi
= gsi_last_nondebug_bb (entry_bb
);
1279 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
1280 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
1281 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TEAMS
);
1282 gsi_remove (&gsi
, true);
1287 exit_succ_e
= single_succ_edge (exit_bb
);
1288 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
1290 remove_edge_and_dominated_blocks (entry_succ_e
);
1294 unsigned srcidx
, dstidx
, num
;
1296 /* If the parallel region needs data sent from the parent
1297 function, then the very first statement (except possible
1298 tree profile counter updates) of the parallel body
1299 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
1300 &.OMP_DATA_O is passed as an argument to the child function,
1301 we need to replace it with the argument as seen by the child
1304 In most cases, this will end up being the identity assignment
1305 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
1306 a function call that has been inlined, the original PARM_DECL
1307 .OMP_DATA_I may have been converted into a different local
1308 variable. In which case, we need to keep the assignment. */
1309 if (gimple_omp_taskreg_data_arg (entry_stmt
))
1311 basic_block entry_succ_bb
1312 = single_succ_p (entry_bb
) ? single_succ (entry_bb
)
1313 : FALLTHRU_EDGE (entry_bb
)->dest
;
1315 gimple
*parcopy_stmt
= NULL
;
1317 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
1321 gcc_assert (!gsi_end_p (gsi
));
1322 stmt
= gsi_stmt (gsi
);
1323 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
1326 if (gimple_num_ops (stmt
) == 2)
1328 tree arg
= gimple_assign_rhs1 (stmt
);
1330 /* We're ignore the subcode because we're
1331 effectively doing a STRIP_NOPS. */
1333 if (TREE_CODE (arg
) == ADDR_EXPR
1334 && (TREE_OPERAND (arg
, 0)
1335 == gimple_omp_taskreg_data_arg (entry_stmt
)))
1337 parcopy_stmt
= stmt
;
1343 gcc_assert (parcopy_stmt
!= NULL
);
1344 arg
= DECL_ARGUMENTS (child_fn
);
1346 if (!gimple_in_ssa_p (cfun
))
1348 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
1349 gsi_remove (&gsi
, true);
1352 /* ?? Is setting the subcode really necessary ?? */
1353 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
1354 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
1359 tree lhs
= gimple_assign_lhs (parcopy_stmt
);
1360 gcc_assert (SSA_NAME_VAR (lhs
) == arg
);
1361 /* We'd like to set the rhs to the default def in the child_fn,
1362 but it's too early to create ssa names in the child_fn.
1363 Instead, we set the rhs to the parm. In
1364 move_sese_region_to_fn, we introduce a default def for the
1365 parm, map the parm to it's default def, and once we encounter
1366 this stmt, replace the parm with the default def. */
1367 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
1368 update_stmt (parcopy_stmt
);
1372 /* Declare local variables needed in CHILD_CFUN. */
1373 block
= DECL_INITIAL (child_fn
);
1374 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
1375 /* The gimplifier could record temporaries in parallel/task block
1376 rather than in containing function's local_decls chain,
1377 which would mean cgraph missed finalizing them. Do it now. */
1378 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
1379 if (VAR_P (t
) && TREE_STATIC (t
) && !DECL_EXTERNAL (t
))
1380 varpool_node::finalize_decl (t
);
1381 DECL_SAVED_TREE (child_fn
) = NULL
;
1382 /* We'll create a CFG for child_fn, so no gimple body is needed. */
1383 gimple_set_body (child_fn
, NULL
);
1384 TREE_USED (block
) = 1;
1386 /* Reset DECL_CONTEXT on function arguments. */
1387 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
1388 DECL_CONTEXT (t
) = child_fn
;
1390 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
1391 so that it can be moved to the child function. */
1392 gsi
= gsi_last_nondebug_bb (entry_bb
);
1393 stmt
= gsi_stmt (gsi
);
1394 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
1395 || gimple_code (stmt
) == GIMPLE_OMP_TASK
1396 || gimple_code (stmt
) == GIMPLE_OMP_TEAMS
));
1397 e
= split_block (entry_bb
, stmt
);
1398 gsi_remove (&gsi
, true);
1401 if (gimple_code (entry_stmt
) != GIMPLE_OMP_TASK
)
1402 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
1405 e2
= make_edge (e
->src
, BRANCH_EDGE (entry_bb
)->dest
, EDGE_ABNORMAL
);
1406 gcc_assert (e2
->dest
== region
->exit
);
1407 remove_edge (BRANCH_EDGE (entry_bb
));
1408 set_immediate_dominator (CDI_DOMINATORS
, e2
->dest
, e
->src
);
1409 gsi
= gsi_last_nondebug_bb (region
->exit
);
1410 gcc_assert (!gsi_end_p (gsi
)
1411 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
1412 gsi_remove (&gsi
, true);
1415 /* Convert GIMPLE_OMP_{RETURN,CONTINUE} into a RETURN_EXPR. */
1418 gsi
= gsi_last_nondebug_bb (exit_bb
);
1419 gcc_assert (!gsi_end_p (gsi
)
1420 && (gimple_code (gsi_stmt (gsi
))
1421 == (e2
? GIMPLE_OMP_CONTINUE
: GIMPLE_OMP_RETURN
)));
1422 stmt
= gimple_build_return (NULL
);
1423 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
1424 gsi_remove (&gsi
, true);
1427 /* Move the parallel region into CHILD_CFUN. */
1429 if (gimple_in_ssa_p (cfun
))
1431 init_tree_ssa (child_cfun
);
1432 init_ssa_operands (child_cfun
);
1433 child_cfun
->gimple_df
->in_ssa_p
= true;
1437 block
= gimple_block (entry_stmt
);
1439 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
1441 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
1444 basic_block dest_bb
= e2
->dest
;
1446 make_edge (new_bb
, dest_bb
, EDGE_FALLTHRU
);
1448 set_immediate_dominator (CDI_DOMINATORS
, dest_bb
, new_bb
);
1450 /* When the OMP expansion process cannot guarantee an up-to-date
1451 loop tree arrange for the child function to fixup loops. */
1452 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
1453 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
1455 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
1456 num
= vec_safe_length (child_cfun
->local_decls
);
1457 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
1459 t
= (*child_cfun
->local_decls
)[srcidx
];
1460 if (DECL_CONTEXT (t
) == cfun
->decl
)
1462 if (srcidx
!= dstidx
)
1463 (*child_cfun
->local_decls
)[dstidx
] = t
;
1467 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
1469 /* Inform the callgraph about the new function. */
1470 child_cfun
->curr_properties
= cfun
->curr_properties
;
1471 child_cfun
->has_simduid_loops
|= cfun
->has_simduid_loops
;
1472 child_cfun
->has_force_vectorize_loops
|= cfun
->has_force_vectorize_loops
;
1473 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
1474 node
->parallelized_function
= 1;
1475 cgraph_node::add_new_function (child_fn
, true);
1477 bool need_asm
= DECL_ASSEMBLER_NAME_SET_P (current_function_decl
)
1478 && !DECL_ASSEMBLER_NAME_SET_P (child_fn
);
1480 /* Fix the callgraph edges for child_cfun. Those for cfun will be
1481 fixed in a following pass. */
1482 push_cfun (child_cfun
);
1484 assign_assembler_name_if_needed (child_fn
);
1487 optimize_omp_library_calls (entry_stmt
);
1488 update_max_bb_count ();
1489 cgraph_edge::rebuild_edges ();
1491 /* Some EH regions might become dead, see PR34608. If
1492 pass_cleanup_cfg isn't the first pass to happen with the
1493 new child, these dead EH edges might cause problems.
1494 Clean them up now. */
1495 if (flag_exceptions
)
1498 bool changed
= false;
1500 FOR_EACH_BB_FN (bb
, cfun
)
1501 changed
|= gimple_purge_dead_eh_edges (bb
);
1503 cleanup_tree_cfg ();
1505 if (gimple_in_ssa_p (cfun
))
1506 update_ssa (TODO_update_ssa
);
1507 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
1508 verify_loop_structure ();
1511 if (dump_file
&& !gimple_in_ssa_p (cfun
))
1513 omp_any_child_fn_dumped
= true;
1514 dump_function_header (dump_file
, child_fn
, dump_flags
);
1515 dump_function_to_file (child_fn
, dump_file
, dump_flags
);
1519 adjust_context_and_scope (region
, gimple_block (entry_stmt
), child_fn
);
1521 if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
1522 expand_parallel_call (region
, new_bb
,
1523 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
1524 else if (gimple_code (entry_stmt
) == GIMPLE_OMP_TEAMS
)
1525 expand_teams_call (new_bb
, as_a
<gomp_teams
*> (entry_stmt
));
1527 expand_task_call (region
, new_bb
, as_a
<gomp_task
*> (entry_stmt
));
1528 if (gimple_in_ssa_p (cfun
))
1529 update_ssa (TODO_update_ssa_only_virtuals
);
1532 /* Information about members of an OpenACC collapsed loop nest. */
1534 struct oacc_collapse
1536 tree base
; /* Base value. */
1537 tree iters
; /* Number of steps. */
1538 tree step
; /* Step size. */
1539 tree tile
; /* Tile increment (if tiled). */
1540 tree outer
; /* Tile iterator var. */
1543 /* Helper for expand_oacc_for. Determine collapsed loop information.
1544 Fill in COUNTS array. Emit any initialization code before GSI.
1545 Return the calculated outer loop bound of BOUND_TYPE. */
1548 expand_oacc_collapse_init (const struct omp_for_data
*fd
,
1549 gimple_stmt_iterator
*gsi
,
1550 oacc_collapse
*counts
, tree bound_type
,
1553 tree tiling
= fd
->tiling
;
1554 tree total
= build_int_cst (bound_type
, 1);
1557 gcc_assert (integer_onep (fd
->loop
.step
));
1558 gcc_assert (integer_zerop (fd
->loop
.n1
));
1560 /* When tiling, the first operand of the tile clause applies to the
1561 innermost loop, and we work outwards from there. Seems
1562 backwards, but whatever. */
1563 for (ix
= fd
->collapse
; ix
--;)
1565 const omp_for_data_loop
*loop
= &fd
->loops
[ix
];
1567 tree iter_type
= TREE_TYPE (loop
->v
);
1568 tree diff_type
= iter_type
;
1569 tree plus_type
= iter_type
;
1571 gcc_assert (loop
->cond_code
== fd
->loop
.cond_code
);
1573 if (POINTER_TYPE_P (iter_type
))
1574 plus_type
= sizetype
;
1575 if (POINTER_TYPE_P (diff_type
) || TYPE_UNSIGNED (diff_type
))
1576 diff_type
= signed_type_for (diff_type
);
1577 if (TYPE_PRECISION (diff_type
) < TYPE_PRECISION (integer_type_node
))
1578 diff_type
= integer_type_node
;
1582 tree num
= build_int_cst (integer_type_node
, fd
->collapse
);
1583 tree loop_no
= build_int_cst (integer_type_node
, ix
);
1584 tree tile
= TREE_VALUE (tiling
);
1586 = gimple_build_call_internal (IFN_GOACC_TILE
, 5, num
, loop_no
, tile
,
1587 /* gwv-outer=*/integer_zero_node
,
1588 /* gwv-inner=*/integer_zero_node
);
1590 counts
[ix
].outer
= create_tmp_var (iter_type
, ".outer");
1591 counts
[ix
].tile
= create_tmp_var (diff_type
, ".tile");
1592 gimple_call_set_lhs (call
, counts
[ix
].tile
);
1593 gimple_set_location (call
, loc
);
1594 gsi_insert_before (gsi
, call
, GSI_SAME_STMT
);
1596 tiling
= TREE_CHAIN (tiling
);
1600 counts
[ix
].tile
= NULL
;
1601 counts
[ix
].outer
= loop
->v
;
1606 tree s
= loop
->step
;
1607 bool up
= loop
->cond_code
== LT_EXPR
;
1608 tree dir
= build_int_cst (diff_type
, up
? +1 : -1);
1612 b
= force_gimple_operand_gsi (gsi
, b
, true, NULL_TREE
,
1613 true, GSI_SAME_STMT
);
1614 e
= force_gimple_operand_gsi (gsi
, e
, true, NULL_TREE
,
1615 true, GSI_SAME_STMT
);
1617 /* Convert the step, avoiding possible unsigned->signed overflow. */
1618 negating
= !up
&& TYPE_UNSIGNED (TREE_TYPE (s
));
1620 s
= fold_build1 (NEGATE_EXPR
, TREE_TYPE (s
), s
);
1621 s
= fold_convert (diff_type
, s
);
1623 s
= fold_build1 (NEGATE_EXPR
, diff_type
, s
);
1624 s
= force_gimple_operand_gsi (gsi
, s
, true, NULL_TREE
,
1625 true, GSI_SAME_STMT
);
1627 /* Determine the range, avoiding possible unsigned->signed overflow. */
1628 negating
= !up
&& TYPE_UNSIGNED (iter_type
);
1629 expr
= fold_build2 (MINUS_EXPR
, plus_type
,
1630 fold_convert (plus_type
, negating
? b
: e
),
1631 fold_convert (plus_type
, negating
? e
: b
));
1632 expr
= fold_convert (diff_type
, expr
);
1634 expr
= fold_build1 (NEGATE_EXPR
, diff_type
, expr
);
1635 tree range
= force_gimple_operand_gsi
1636 (gsi
, expr
, true, NULL_TREE
, true, GSI_SAME_STMT
);
1638 /* Determine number of iterations. */
1639 expr
= fold_build2 (MINUS_EXPR
, diff_type
, range
, dir
);
1640 expr
= fold_build2 (PLUS_EXPR
, diff_type
, expr
, s
);
1641 expr
= fold_build2 (TRUNC_DIV_EXPR
, diff_type
, expr
, s
);
1643 tree iters
= force_gimple_operand_gsi (gsi
, expr
, true, NULL_TREE
,
1644 true, GSI_SAME_STMT
);
1646 counts
[ix
].base
= b
;
1647 counts
[ix
].iters
= iters
;
1648 counts
[ix
].step
= s
;
1650 total
= fold_build2 (MULT_EXPR
, bound_type
, total
,
1651 fold_convert (bound_type
, iters
));
1657 /* Emit initializers for collapsed loop members. INNER is true if
1658 this is for the element loop of a TILE. IVAR is the outer
1659 loop iteration variable, from which collapsed loop iteration values
1660 are calculated. COUNTS array has been initialized by
1661 expand_oacc_collapse_inits. */
1664 expand_oacc_collapse_vars (const struct omp_for_data
*fd
, bool inner
,
1665 gimple_stmt_iterator
*gsi
,
1666 const oacc_collapse
*counts
, tree ivar
)
1668 tree ivar_type
= TREE_TYPE (ivar
);
1670 /* The most rapidly changing iteration variable is the innermost
1672 for (int ix
= fd
->collapse
; ix
--;)
1674 const omp_for_data_loop
*loop
= &fd
->loops
[ix
];
1675 const oacc_collapse
*collapse
= &counts
[ix
];
1676 tree v
= inner
? loop
->v
: collapse
->outer
;
1677 tree iter_type
= TREE_TYPE (v
);
1678 tree diff_type
= TREE_TYPE (collapse
->step
);
1679 tree plus_type
= iter_type
;
1680 enum tree_code plus_code
= PLUS_EXPR
;
1683 if (POINTER_TYPE_P (iter_type
))
1685 plus_code
= POINTER_PLUS_EXPR
;
1686 plus_type
= sizetype
;
1692 tree mod
= fold_convert (ivar_type
, collapse
->iters
);
1693 ivar
= fold_build2 (TRUNC_DIV_EXPR
, ivar_type
, expr
, mod
);
1694 expr
= fold_build2 (TRUNC_MOD_EXPR
, ivar_type
, expr
, mod
);
1695 ivar
= force_gimple_operand_gsi (gsi
, ivar
, true, NULL_TREE
,
1696 true, GSI_SAME_STMT
);
1699 expr
= fold_build2 (MULT_EXPR
, diff_type
, fold_convert (diff_type
, expr
),
1701 expr
= fold_build2 (plus_code
, iter_type
,
1702 inner
? collapse
->outer
: collapse
->base
,
1703 fold_convert (plus_type
, expr
));
1704 expr
= force_gimple_operand_gsi (gsi
, expr
, false, NULL_TREE
,
1705 true, GSI_SAME_STMT
);
1706 gassign
*ass
= gimple_build_assign (v
, expr
);
1707 gsi_insert_before (gsi
, ass
, GSI_SAME_STMT
);
1711 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
1712 of the combined collapse > 1 loop constructs, generate code like:
1713 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
1718 count3 = (adj + N32 - N31) / STEP3;
1719 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
1724 count2 = (adj + N22 - N21) / STEP2;
1725 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
1730 count1 = (adj + N12 - N11) / STEP1;
1731 count = count1 * count2 * count3;
1732 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
1734 and set ZERO_ITER_BB to that bb. If this isn't the outermost
1735 of the combined loop constructs, just initialize COUNTS array
1736 from the _looptemp_ clauses. */
1738 /* NOTE: It *could* be better to moosh all of the BBs together,
1739 creating one larger BB with all the computation and the unexpected
1740 jump at the end. I.e.
1742 bool zero3, zero2, zero1, zero;
1745 count3 = (N32 - N31) /[cl] STEP3;
1747 count2 = (N22 - N21) /[cl] STEP2;
1749 count1 = (N12 - N11) /[cl] STEP1;
1750 zero = zero3 || zero2 || zero1;
1751 count = count1 * count2 * count3;
1752 if (__builtin_expect(zero, false)) goto zero_iter_bb;
1754 After all, we expect the zero=false, and thus we expect to have to
1755 evaluate all of the comparison expressions, so short-circuiting
1756 oughtn't be a win. Since the condition isn't protecting a
1757 denominator, we're not concerned about divide-by-zero, so we can
1758 fully evaluate count even if a numerator turned out to be wrong.
1760 It seems like putting this all together would create much better
1761 scheduling opportunities, and less pressure on the chip's branch
1765 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
1766 basic_block
&entry_bb
, tree
*counts
,
1767 basic_block
&zero_iter1_bb
, int &first_zero_iter1
,
1768 basic_block
&zero_iter2_bb
, int &first_zero_iter2
,
1769 basic_block
&l2_dom_bb
)
1771 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
1775 /* Collapsed loops need work for expansion into SSA form. */
1776 gcc_assert (!gimple_in_ssa_p (cfun
));
1778 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
1779 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
1781 gcc_assert (fd
->ordered
== 0);
1782 /* First two _looptemp_ clauses are for istart/iend, counts[0]
1783 isn't supposed to be handled, as the inner loop doesn't
1785 tree innerc
= omp_find_clause (gimple_omp_for_clauses (fd
->for_stmt
),
1786 OMP_CLAUSE__LOOPTEMP_
);
1787 gcc_assert (innerc
);
1788 for (i
= 0; i
< fd
->collapse
; i
++)
1790 innerc
= omp_find_clause (OMP_CLAUSE_CHAIN (innerc
),
1791 OMP_CLAUSE__LOOPTEMP_
);
1792 gcc_assert (innerc
);
1794 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
1796 counts
[0] = NULL_TREE
;
1801 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
1803 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
1804 counts
[i
] = NULL_TREE
;
1805 t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
1806 fold_convert (itype
, fd
->loops
[i
].n1
),
1807 fold_convert (itype
, fd
->loops
[i
].n2
));
1808 if (t
&& integer_zerop (t
))
1810 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
1811 counts
[i
] = build_int_cst (type
, 0);
1815 for (i
= 0; i
< (fd
->ordered
? fd
->ordered
: fd
->collapse
); i
++)
1817 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
1819 if (i
>= fd
->collapse
&& counts
[i
])
1821 if ((SSA_VAR_P (fd
->loop
.n2
) || i
>= fd
->collapse
)
1822 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
1823 fold_convert (itype
, fd
->loops
[i
].n1
),
1824 fold_convert (itype
, fd
->loops
[i
].n2
)))
1825 == NULL_TREE
|| !integer_onep (t
)))
1829 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
1830 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
1831 true, GSI_SAME_STMT
);
1832 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
1833 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
1834 true, GSI_SAME_STMT
);
1835 cond_stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
1836 NULL_TREE
, NULL_TREE
);
1837 gsi_insert_before (gsi
, cond_stmt
, GSI_SAME_STMT
);
1838 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
1839 expand_omp_regimplify_p
, NULL
, NULL
)
1840 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
1841 expand_omp_regimplify_p
, NULL
, NULL
))
1843 *gsi
= gsi_for_stmt (cond_stmt
);
1844 gimple_regimplify_operands (cond_stmt
, gsi
);
1846 e
= split_block (entry_bb
, cond_stmt
);
1847 basic_block
&zero_iter_bb
1848 = i
< fd
->collapse
? zero_iter1_bb
: zero_iter2_bb
;
1849 int &first_zero_iter
1850 = i
< fd
->collapse
? first_zero_iter1
: first_zero_iter2
;
1851 if (zero_iter_bb
== NULL
)
1853 gassign
*assign_stmt
;
1854 first_zero_iter
= i
;
1855 zero_iter_bb
= create_empty_bb (entry_bb
);
1856 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
1857 *gsi
= gsi_after_labels (zero_iter_bb
);
1858 if (i
< fd
->collapse
)
1859 assign_stmt
= gimple_build_assign (fd
->loop
.n2
,
1860 build_zero_cst (type
));
1863 counts
[i
] = create_tmp_reg (type
, ".count");
1865 = gimple_build_assign (counts
[i
], build_zero_cst (type
));
1867 gsi_insert_before (gsi
, assign_stmt
, GSI_SAME_STMT
);
1868 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
1871 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
1872 ne
->probability
= profile_probability::very_unlikely ();
1873 e
->flags
= EDGE_TRUE_VALUE
;
1874 e
->probability
= ne
->probability
.invert ();
1875 if (l2_dom_bb
== NULL
)
1876 l2_dom_bb
= entry_bb
;
1878 *gsi
= gsi_last_nondebug_bb (entry_bb
);
1881 if (POINTER_TYPE_P (itype
))
1882 itype
= signed_type_for (itype
);
1883 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
1885 t
= fold_build2 (PLUS_EXPR
, itype
,
1886 fold_convert (itype
, fd
->loops
[i
].step
), t
);
1887 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
1888 fold_convert (itype
, fd
->loops
[i
].n2
));
1889 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
1890 fold_convert (itype
, fd
->loops
[i
].n1
));
1891 /* ?? We could probably use CEIL_DIV_EXPR instead of
1892 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
1893 generate the same code in the end because generically we
1894 don't know that the values involved must be negative for
1896 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
1897 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
1898 fold_build1 (NEGATE_EXPR
, itype
, t
),
1899 fold_build1 (NEGATE_EXPR
, itype
,
1900 fold_convert (itype
,
1901 fd
->loops
[i
].step
)));
1903 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
1904 fold_convert (itype
, fd
->loops
[i
].step
));
1905 t
= fold_convert (type
, t
);
1906 if (TREE_CODE (t
) == INTEGER_CST
)
1910 if (i
< fd
->collapse
|| i
!= first_zero_iter2
)
1911 counts
[i
] = create_tmp_reg (type
, ".count");
1912 expand_omp_build_assign (gsi
, counts
[i
], t
);
1914 if (SSA_VAR_P (fd
->loop
.n2
) && i
< fd
->collapse
)
1919 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
1920 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
1925 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
1927 V3 = N31 + (T % count3) * STEP3;
1929 V2 = N21 + (T % count2) * STEP2;
1931 V1 = N11 + T * STEP1;
1932 if this loop doesn't have an inner loop construct combined with it.
1933 If it does have an inner loop construct combined with it and the
1934 iteration count isn't known constant, store values from counts array
1935 into its _looptemp_ temporaries instead. */
1938 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
1939 tree
*counts
, gimple
*inner_stmt
, tree startvar
)
1942 if (gimple_omp_for_combined_p (fd
->for_stmt
))
1944 /* If fd->loop.n2 is constant, then no propagation of the counts
1945 is needed, they are constant. */
1946 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
1949 tree clauses
= gimple_code (inner_stmt
) != GIMPLE_OMP_FOR
1950 ? gimple_omp_taskreg_clauses (inner_stmt
)
1951 : gimple_omp_for_clauses (inner_stmt
);
1952 /* First two _looptemp_ clauses are for istart/iend, counts[0]
1953 isn't supposed to be handled, as the inner loop doesn't
1955 tree innerc
= omp_find_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
1956 gcc_assert (innerc
);
1957 for (i
= 0; i
< fd
->collapse
; i
++)
1959 innerc
= omp_find_clause (OMP_CLAUSE_CHAIN (innerc
),
1960 OMP_CLAUSE__LOOPTEMP_
);
1961 gcc_assert (innerc
);
1964 tree tem
= OMP_CLAUSE_DECL (innerc
);
1965 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
1966 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
1967 false, GSI_CONTINUE_LINKING
);
1968 gassign
*stmt
= gimple_build_assign (tem
, t
);
1969 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
1975 tree type
= TREE_TYPE (fd
->loop
.v
);
1976 tree tem
= create_tmp_reg (type
, ".tem");
1977 gassign
*stmt
= gimple_build_assign (tem
, startvar
);
1978 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
1980 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
1982 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
1984 if (POINTER_TYPE_P (vtype
))
1985 itype
= signed_type_for (vtype
);
1987 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
1990 t
= fold_convert (itype
, t
);
1991 t
= fold_build2 (MULT_EXPR
, itype
, t
,
1992 fold_convert (itype
, fd
->loops
[i
].step
));
1993 if (POINTER_TYPE_P (vtype
))
1994 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
1996 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
1997 t
= force_gimple_operand_gsi (gsi
, t
,
1998 DECL_P (fd
->loops
[i
].v
)
1999 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
2001 GSI_CONTINUE_LINKING
);
2002 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
2003 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
2006 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
2007 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
2008 false, GSI_CONTINUE_LINKING
);
2009 stmt
= gimple_build_assign (tem
, t
);
2010 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
2015 /* Helper function for expand_omp_for_*. Generate code like:
2018 if (V3 cond3 N32) goto BODY_BB; else goto L11;
2022 if (V2 cond2 N22) goto BODY_BB; else goto L12;
2029 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
2030 basic_block body_bb
)
2032 basic_block last_bb
, bb
, collapse_bb
= NULL
;
2034 gimple_stmt_iterator gsi
;
2040 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
2042 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
2044 bb
= create_empty_bb (last_bb
);
2045 add_bb_to_loop (bb
, last_bb
->loop_father
);
2046 gsi
= gsi_start_bb (bb
);
2048 if (i
< fd
->collapse
- 1)
2050 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
2051 e
->probability
= profile_probability::guessed_always ().apply_scale (1, 8);
2053 t
= fd
->loops
[i
+ 1].n1
;
2054 t
= force_gimple_operand_gsi (&gsi
, t
,
2055 DECL_P (fd
->loops
[i
+ 1].v
)
2056 && TREE_ADDRESSABLE (fd
->loops
[i
2059 GSI_CONTINUE_LINKING
);
2060 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
2061 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2066 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
2068 if (POINTER_TYPE_P (vtype
))
2069 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
2071 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
2072 t
= force_gimple_operand_gsi (&gsi
, t
,
2073 DECL_P (fd
->loops
[i
].v
)
2074 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
2075 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
2076 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
2077 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2081 t
= fd
->loops
[i
].n2
;
2082 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
2083 false, GSI_CONTINUE_LINKING
);
2084 tree v
= fd
->loops
[i
].v
;
2085 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
2086 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
2087 false, GSI_CONTINUE_LINKING
);
2088 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
2089 stmt
= gimple_build_cond_empty (t
);
2090 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2091 if (walk_tree (gimple_cond_lhs_ptr (as_a
<gcond
*> (stmt
)),
2092 expand_omp_regimplify_p
, NULL
, NULL
)
2093 || walk_tree (gimple_cond_rhs_ptr (as_a
<gcond
*> (stmt
)),
2094 expand_omp_regimplify_p
, NULL
, NULL
))
2095 gimple_regimplify_operands (stmt
, &gsi
);
2096 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
2097 e
->probability
= profile_probability::guessed_always ().apply_scale (7, 8);
2100 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
2107 /* Expand #pragma omp ordered depend(source). */
2110 expand_omp_ordered_source (gimple_stmt_iterator
*gsi
, struct omp_for_data
*fd
,
2111 tree
*counts
, location_t loc
)
2113 enum built_in_function source_ix
2114 = fd
->iter_type
== long_integer_type_node
2115 ? BUILT_IN_GOMP_DOACROSS_POST
: BUILT_IN_GOMP_DOACROSS_ULL_POST
;
2117 = gimple_build_call (builtin_decl_explicit (source_ix
), 1,
2118 build_fold_addr_expr (counts
[fd
->ordered
]));
2119 gimple_set_location (g
, loc
);
2120 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
2123 /* Expand a single depend from #pragma omp ordered depend(sink:...). */
2126 expand_omp_ordered_sink (gimple_stmt_iterator
*gsi
, struct omp_for_data
*fd
,
2127 tree
*counts
, tree c
, location_t loc
)
2129 auto_vec
<tree
, 10> args
;
2130 enum built_in_function sink_ix
2131 = fd
->iter_type
== long_integer_type_node
2132 ? BUILT_IN_GOMP_DOACROSS_WAIT
: BUILT_IN_GOMP_DOACROSS_ULL_WAIT
;
2133 tree t
, off
, coff
= NULL_TREE
, deps
= OMP_CLAUSE_DECL (c
), cond
= NULL_TREE
;
2135 gimple_stmt_iterator gsi2
= *gsi
;
2136 bool warned_step
= false;
2138 for (i
= 0; i
< fd
->ordered
; i
++)
2140 tree step
= NULL_TREE
;
2141 off
= TREE_PURPOSE (deps
);
2142 if (TREE_CODE (off
) == TRUNC_DIV_EXPR
)
2144 step
= TREE_OPERAND (off
, 1);
2145 off
= TREE_OPERAND (off
, 0);
2147 if (!integer_zerop (off
))
2149 gcc_assert (fd
->loops
[i
].cond_code
== LT_EXPR
2150 || fd
->loops
[i
].cond_code
== GT_EXPR
);
2151 bool forward
= fd
->loops
[i
].cond_code
== LT_EXPR
;
2154 /* Non-simple Fortran DO loops. If step is variable,
2155 we don't know at compile even the direction, so can't
2157 if (TREE_CODE (step
) != INTEGER_CST
)
2159 forward
= tree_int_cst_sgn (step
) != -1;
2161 if (forward
^ OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
2162 warning_at (loc
, 0, "%<depend%> clause with %<sink%> modifier "
2163 "waiting for lexically later iteration");
2166 deps
= TREE_CHAIN (deps
);
2168 /* If all offsets corresponding to the collapsed loops are zero,
2169 this depend clause can be ignored. FIXME: but there is still a
2170 flush needed. We need to emit one __sync_synchronize () for it
2171 though (perhaps conditionally)? Solve this together with the
2172 conservative dependence folding optimization.
2173 if (i >= fd->collapse)
2176 deps
= OMP_CLAUSE_DECL (c
);
2178 edge e1
= split_block (gsi_bb (gsi2
), gsi_stmt (gsi2
));
2179 edge e2
= split_block_after_labels (e1
->dest
);
2181 gsi2
= gsi_after_labels (e1
->dest
);
2182 *gsi
= gsi_last_bb (e1
->src
);
2183 for (i
= 0; i
< fd
->ordered
; i
++)
2185 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
2186 tree step
= NULL_TREE
;
2187 tree orig_off
= NULL_TREE
;
2188 if (POINTER_TYPE_P (itype
))
2191 deps
= TREE_CHAIN (deps
);
2192 off
= TREE_PURPOSE (deps
);
2193 if (TREE_CODE (off
) == TRUNC_DIV_EXPR
)
2195 step
= TREE_OPERAND (off
, 1);
2196 off
= TREE_OPERAND (off
, 0);
2197 gcc_assert (fd
->loops
[i
].cond_code
== LT_EXPR
2198 && integer_onep (fd
->loops
[i
].step
)
2199 && !POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)));
2201 tree s
= fold_convert_loc (loc
, itype
, step
? step
: fd
->loops
[i
].step
);
2204 off
= fold_convert_loc (loc
, itype
, off
);
2206 off
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, off
, s
);
2209 if (integer_zerop (off
))
2210 t
= boolean_true_node
;
2214 tree co
= fold_convert_loc (loc
, itype
, off
);
2215 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
2217 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
2218 co
= fold_build1_loc (loc
, NEGATE_EXPR
, itype
, co
);
2219 a
= fold_build2_loc (loc
, POINTER_PLUS_EXPR
,
2220 TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].v
,
2223 else if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
2224 a
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
2225 fd
->loops
[i
].v
, co
);
2227 a
= fold_build2_loc (loc
, PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
2228 fd
->loops
[i
].v
, co
);
2232 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
2233 t1
= fold_build2_loc (loc
, GE_EXPR
, boolean_type_node
, a
,
2236 t1
= fold_build2_loc (loc
, LT_EXPR
, boolean_type_node
, a
,
2238 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
2239 t2
= fold_build2_loc (loc
, LT_EXPR
, boolean_type_node
, a
,
2242 t2
= fold_build2_loc (loc
, GE_EXPR
, boolean_type_node
, a
,
2244 t
= fold_build2_loc (loc
, LT_EXPR
, boolean_type_node
,
2245 step
, build_int_cst (TREE_TYPE (step
), 0));
2246 if (TREE_CODE (step
) != INTEGER_CST
)
2248 t1
= unshare_expr (t1
);
2249 t1
= force_gimple_operand_gsi (gsi
, t1
, true, NULL_TREE
,
2250 false, GSI_CONTINUE_LINKING
);
2251 t2
= unshare_expr (t2
);
2252 t2
= force_gimple_operand_gsi (gsi
, t2
, true, NULL_TREE
,
2253 false, GSI_CONTINUE_LINKING
);
2255 t
= fold_build3_loc (loc
, COND_EXPR
, boolean_type_node
,
2258 else if (fd
->loops
[i
].cond_code
== LT_EXPR
)
2260 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
2261 t
= fold_build2_loc (loc
, GE_EXPR
, boolean_type_node
, a
,
2264 t
= fold_build2_loc (loc
, LT_EXPR
, boolean_type_node
, a
,
2267 else if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
2268 t
= fold_build2_loc (loc
, GT_EXPR
, boolean_type_node
, a
,
2271 t
= fold_build2_loc (loc
, LE_EXPR
, boolean_type_node
, a
,
2275 cond
= fold_build2_loc (loc
, BIT_AND_EXPR
, boolean_type_node
, cond
, t
);
2279 off
= fold_convert_loc (loc
, itype
, off
);
2282 || (fd
->loops
[i
].cond_code
== LT_EXPR
2283 ? !integer_onep (fd
->loops
[i
].step
)
2284 : !integer_minus_onep (fd
->loops
[i
].step
)))
2286 if (step
== NULL_TREE
2287 && TYPE_UNSIGNED (itype
)
2288 && fd
->loops
[i
].cond_code
== GT_EXPR
)
2289 t
= fold_build2_loc (loc
, TRUNC_MOD_EXPR
, itype
, off
,
2290 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
2293 t
= fold_build2_loc (loc
, TRUNC_MOD_EXPR
, itype
,
2294 orig_off
? orig_off
: off
, s
);
2295 t
= fold_build2_loc (loc
, EQ_EXPR
, boolean_type_node
, t
,
2296 build_int_cst (itype
, 0));
2297 if (integer_zerop (t
) && !warned_step
)
2299 warning_at (loc
, 0, "%<depend%> clause with %<sink%> modifier "
2300 "refers to iteration never in the iteration "
2304 cond
= fold_build2_loc (loc
, BIT_AND_EXPR
, boolean_type_node
,
2308 if (i
<= fd
->collapse
- 1 && fd
->collapse
> 1)
2314 t
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
2315 fd
->loops
[i
].v
, fd
->loops
[i
].n1
);
2316 t
= fold_convert_loc (loc
, fd
->iter_type
, t
);
2319 /* We have divided off by step already earlier. */;
2320 else if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
2321 off
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, off
,
2322 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
2325 off
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, off
, s
);
2326 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
2327 off
= fold_build1_loc (loc
, NEGATE_EXPR
, itype
, off
);
2328 off
= fold_convert_loc (loc
, fd
->iter_type
, off
);
2329 if (i
<= fd
->collapse
- 1 && fd
->collapse
> 1)
2332 off
= fold_build2_loc (loc
, PLUS_EXPR
, fd
->iter_type
, coff
,
2334 if (i
< fd
->collapse
- 1)
2336 coff
= fold_build2_loc (loc
, MULT_EXPR
, fd
->iter_type
, off
,
2341 off
= unshare_expr (off
);
2342 t
= fold_build2_loc (loc
, PLUS_EXPR
, fd
->iter_type
, t
, off
);
2343 t
= force_gimple_operand_gsi (&gsi2
, t
, true, NULL_TREE
,
2344 true, GSI_SAME_STMT
);
2347 gimple
*g
= gimple_build_call_vec (builtin_decl_explicit (sink_ix
), args
);
2348 gimple_set_location (g
, loc
);
2349 gsi_insert_before (&gsi2
, g
, GSI_SAME_STMT
);
2351 cond
= unshare_expr (cond
);
2352 cond
= force_gimple_operand_gsi (gsi
, cond
, true, NULL_TREE
, false,
2353 GSI_CONTINUE_LINKING
);
2354 gsi_insert_after (gsi
, gimple_build_cond_empty (cond
), GSI_NEW_STMT
);
2355 edge e3
= make_edge (e1
->src
, e2
->dest
, EDGE_FALSE_VALUE
);
2356 e3
->probability
= profile_probability::guessed_always ().apply_scale (1, 8);
2357 e1
->probability
= e3
->probability
.invert ();
2358 e1
->flags
= EDGE_TRUE_VALUE
;
2359 set_immediate_dominator (CDI_DOMINATORS
, e2
->dest
, e1
->src
);
2361 *gsi
= gsi_after_labels (e2
->dest
);
2364 /* Expand all #pragma omp ordered depend(source) and
2365 #pragma omp ordered depend(sink:...) constructs in the current
2366 #pragma omp for ordered(n) region. */
2369 expand_omp_ordered_source_sink (struct omp_region
*region
,
2370 struct omp_for_data
*fd
, tree
*counts
,
2371 basic_block cont_bb
)
2373 struct omp_region
*inner
;
2375 for (i
= fd
->collapse
- 1; i
< fd
->ordered
; i
++)
2376 if (i
== fd
->collapse
- 1 && fd
->collapse
> 1)
2377 counts
[i
] = NULL_TREE
;
2378 else if (i
>= fd
->collapse
&& !cont_bb
)
2379 counts
[i
] = build_zero_cst (fd
->iter_type
);
2380 else if (!POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
))
2381 && integer_onep (fd
->loops
[i
].step
))
2382 counts
[i
] = NULL_TREE
;
2384 counts
[i
] = create_tmp_var (fd
->iter_type
, ".orditer");
2386 = build_array_type_nelts (fd
->iter_type
, fd
->ordered
- fd
->collapse
+ 1);
2387 counts
[fd
->ordered
] = create_tmp_var (atype
, ".orditera");
2388 TREE_ADDRESSABLE (counts
[fd
->ordered
]) = 1;
2390 for (inner
= region
->inner
; inner
; inner
= inner
->next
)
2391 if (inner
->type
== GIMPLE_OMP_ORDERED
)
2393 gomp_ordered
*ord_stmt
= inner
->ord_stmt
;
2394 gimple_stmt_iterator gsi
= gsi_for_stmt (ord_stmt
);
2395 location_t loc
= gimple_location (ord_stmt
);
2397 for (c
= gimple_omp_ordered_clauses (ord_stmt
);
2398 c
; c
= OMP_CLAUSE_CHAIN (c
))
2399 if (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
)
2402 expand_omp_ordered_source (&gsi
, fd
, counts
, loc
);
2403 for (c
= gimple_omp_ordered_clauses (ord_stmt
);
2404 c
; c
= OMP_CLAUSE_CHAIN (c
))
2405 if (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
)
2406 expand_omp_ordered_sink (&gsi
, fd
, counts
, c
, loc
);
2407 gsi_remove (&gsi
, true);
2411 /* Wrap the body into fd->ordered - fd->collapse loops that aren't
2415 expand_omp_for_ordered_loops (struct omp_for_data
*fd
, tree
*counts
,
2416 basic_block cont_bb
, basic_block body_bb
,
2417 bool ordered_lastprivate
)
2419 if (fd
->ordered
== fd
->collapse
)
2424 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
2425 for (int i
= fd
->collapse
; i
< fd
->ordered
; i
++)
2427 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
2428 tree n1
= fold_convert (type
, fd
->loops
[i
].n1
);
2429 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, n1
);
2430 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
2431 size_int (i
- fd
->collapse
+ 1),
2432 NULL_TREE
, NULL_TREE
);
2433 expand_omp_build_assign (&gsi
, aref
, build_zero_cst (fd
->iter_type
));
2438 for (int i
= fd
->ordered
- 1; i
>= fd
->collapse
; i
--)
2440 tree t
, type
= TREE_TYPE (fd
->loops
[i
].v
);
2441 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
2442 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
,
2443 fold_convert (type
, fd
->loops
[i
].n1
));
2445 expand_omp_build_assign (&gsi
, counts
[i
],
2446 build_zero_cst (fd
->iter_type
));
2447 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
2448 size_int (i
- fd
->collapse
+ 1),
2449 NULL_TREE
, NULL_TREE
);
2450 expand_omp_build_assign (&gsi
, aref
, build_zero_cst (fd
->iter_type
));
2451 if (!gsi_end_p (gsi
))
2454 gsi
= gsi_last_bb (body_bb
);
2455 edge e1
= split_block (body_bb
, gsi_stmt (gsi
));
2456 basic_block new_body
= e1
->dest
;
2457 if (body_bb
== cont_bb
)
2460 basic_block new_header
;
2461 if (EDGE_COUNT (cont_bb
->preds
) > 0)
2463 gsi
= gsi_last_bb (cont_bb
);
2464 if (POINTER_TYPE_P (type
))
2465 t
= fold_build_pointer_plus (fd
->loops
[i
].v
,
2466 fold_convert (sizetype
,
2467 fd
->loops
[i
].step
));
2469 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loops
[i
].v
,
2470 fold_convert (type
, fd
->loops
[i
].step
));
2471 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
2474 t
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, counts
[i
],
2475 build_int_cst (fd
->iter_type
, 1));
2476 expand_omp_build_assign (&gsi
, counts
[i
], t
);
2481 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
2482 fd
->loops
[i
].v
, fd
->loops
[i
].n1
);
2483 t
= fold_convert (fd
->iter_type
, t
);
2484 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
2485 true, GSI_SAME_STMT
);
2487 aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
2488 size_int (i
- fd
->collapse
+ 1),
2489 NULL_TREE
, NULL_TREE
);
2490 expand_omp_build_assign (&gsi
, aref
, t
);
2492 e2
= split_block (cont_bb
, gsi_stmt (gsi
));
2493 new_header
= e2
->dest
;
2496 new_header
= cont_bb
;
2497 gsi
= gsi_after_labels (new_header
);
2498 tree v
= force_gimple_operand_gsi (&gsi
, fd
->loops
[i
].v
, true, NULL_TREE
,
2499 true, GSI_SAME_STMT
);
2501 = force_gimple_operand_gsi (&gsi
, fold_convert (type
, fd
->loops
[i
].n2
),
2502 true, NULL_TREE
, true, GSI_SAME_STMT
);
2503 t
= build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, n2
);
2504 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_NEW_STMT
);
2505 edge e3
= split_block (new_header
, gsi_stmt (gsi
));
2508 make_edge (body_bb
, new_header
, EDGE_FALLTHRU
);
2509 e3
->flags
= EDGE_FALSE_VALUE
;
2510 e3
->probability
= profile_probability::guessed_always ().apply_scale (1, 8);
2511 e1
= make_edge (new_header
, new_body
, EDGE_TRUE_VALUE
);
2512 e1
->probability
= e3
->probability
.invert ();
2514 set_immediate_dominator (CDI_DOMINATORS
, new_header
, body_bb
);
2515 set_immediate_dominator (CDI_DOMINATORS
, new_body
, new_header
);
2519 class loop
*loop
= alloc_loop ();
2520 loop
->header
= new_header
;
2521 loop
->latch
= e2
->src
;
2522 add_loop (loop
, body_bb
->loop_father
);
2526 /* If there are any lastprivate clauses and it is possible some loops
2527 might have zero iterations, ensure all the decls are initialized,
2528 otherwise we could crash evaluating C++ class iterators with lastprivate
2530 bool need_inits
= false;
2531 for (int i
= fd
->collapse
; ordered_lastprivate
&& i
< fd
->ordered
; i
++)
2534 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
2535 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
2536 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
,
2537 fold_convert (type
, fd
->loops
[i
].n1
));
2541 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
2542 tree this_cond
= fold_build2 (fd
->loops
[i
].cond_code
,
2544 fold_convert (type
, fd
->loops
[i
].n1
),
2545 fold_convert (type
, fd
->loops
[i
].n2
));
2546 if (!integer_onep (this_cond
))
2553 /* A subroutine of expand_omp_for. Generate code for a parallel
2554 loop with any schedule. Given parameters:
2556 for (V = N1; V cond N2; V += STEP) BODY;
2558 where COND is "<" or ">", we generate pseudocode
2560 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
2561 if (more) goto L0; else goto L3;
2568 if (V cond iend) goto L1; else goto L2;
2570 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
2573 If this is a combined omp parallel loop, instead of the call to
2574 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
2575 If this is gimple_omp_for_combined_p loop, then instead of assigning
2576 V and iend in L0 we assign the first two _looptemp_ clause decls of the
2577 inner GIMPLE_OMP_FOR and V += STEP; and
2578 if (V cond iend) goto L1; else goto L2; are removed.
2580 For collapsed loops, given parameters:
2582 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
2583 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
2584 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
2587 we generate pseudocode
2589 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
2594 count3 = (adj + N32 - N31) / STEP3;
2595 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
2600 count2 = (adj + N22 - N21) / STEP2;
2601 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
2606 count1 = (adj + N12 - N11) / STEP1;
2607 count = count1 * count2 * count3;
2612 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
2613 if (more) goto L0; else goto L3;
2617 V3 = N31 + (T % count3) * STEP3;
2619 V2 = N21 + (T % count2) * STEP2;
2621 V1 = N11 + T * STEP1;
2626 if (V < iend) goto L10; else goto L2;
2629 if (V3 cond3 N32) goto L1; else goto L11;
2633 if (V2 cond2 N22) goto L1; else goto L12;
2639 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
2645 expand_omp_for_generic (struct omp_region
*region
,
2646 struct omp_for_data
*fd
,
2647 enum built_in_function start_fn
,
2648 enum built_in_function next_fn
,
2652 tree type
, istart0
, iend0
, iend
;
2653 tree t
, vmain
, vback
, bias
= NULL_TREE
;
2654 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
2655 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
2656 gimple_stmt_iterator gsi
;
2657 gassign
*assign_stmt
;
2658 bool in_combined_parallel
= is_combined_parallel (region
);
2659 bool broken_loop
= region
->cont
== NULL
;
2661 tree
*counts
= NULL
;
2663 bool ordered_lastprivate
= false;
2665 gcc_assert (!broken_loop
|| !in_combined_parallel
);
2666 gcc_assert (fd
->iter_type
== long_integer_type_node
2667 || !in_combined_parallel
);
2669 entry_bb
= region
->entry
;
2670 cont_bb
= region
->cont
;
2672 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
2673 gcc_assert (broken_loop
2674 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
2675 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
2676 l1_bb
= single_succ (l0_bb
);
2679 l2_bb
= create_empty_bb (cont_bb
);
2680 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
2681 || (single_succ_edge (BRANCH_EDGE (cont_bb
)->dest
)->dest
2683 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
2687 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
2688 exit_bb
= region
->exit
;
2690 gsi
= gsi_last_nondebug_bb (entry_bb
);
2692 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
2694 && omp_find_clause (gimple_omp_for_clauses (fd
->for_stmt
),
2695 OMP_CLAUSE_LASTPRIVATE
))
2696 ordered_lastprivate
= false;
2697 tree reductions
= NULL_TREE
;
2698 tree mem
= NULL_TREE
, cond_var
= NULL_TREE
, condtemp
= NULL_TREE
;
2699 tree memv
= NULL_TREE
;
2700 if (fd
->lastprivate_conditional
)
2702 tree c
= omp_find_clause (gimple_omp_for_clauses (fd
->for_stmt
),
2703 OMP_CLAUSE__CONDTEMP_
);
2704 if (fd
->have_pointer_condtemp
)
2705 condtemp
= OMP_CLAUSE_DECL (c
);
2706 c
= omp_find_clause (OMP_CLAUSE_CHAIN (c
), OMP_CLAUSE__CONDTEMP_
);
2707 cond_var
= OMP_CLAUSE_DECL (c
);
2711 if (fd
->have_reductemp
)
2713 tree c
= omp_find_clause (gimple_omp_for_clauses (fd
->for_stmt
),
2714 OMP_CLAUSE__REDUCTEMP_
);
2715 reductions
= OMP_CLAUSE_DECL (c
);
2716 gcc_assert (TREE_CODE (reductions
) == SSA_NAME
);
2717 gimple
*g
= SSA_NAME_DEF_STMT (reductions
);
2718 reductions
= gimple_assign_rhs1 (g
);
2719 OMP_CLAUSE_DECL (c
) = reductions
;
2720 entry_bb
= gimple_bb (g
);
2721 edge e
= split_block (entry_bb
, g
);
2722 if (region
->entry
== entry_bb
)
2723 region
->entry
= e
->dest
;
2724 gsi
= gsi_last_bb (entry_bb
);
2727 reductions
= null_pointer_node
;
2728 if (fd
->have_pointer_condtemp
)
2730 tree type
= TREE_TYPE (condtemp
);
2731 memv
= create_tmp_var (type
);
2732 TREE_ADDRESSABLE (memv
) = 1;
2733 unsigned HOST_WIDE_INT sz
2734 = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (type
)));
2735 sz
*= fd
->lastprivate_conditional
;
2736 expand_omp_build_assign (&gsi
, memv
, build_int_cst (type
, sz
),
2738 mem
= build_fold_addr_expr (memv
);
2741 mem
= null_pointer_node
;
2743 if (fd
->collapse
> 1 || fd
->ordered
)
2745 int first_zero_iter1
= -1, first_zero_iter2
= -1;
2746 basic_block zero_iter1_bb
= NULL
, zero_iter2_bb
= NULL
, l2_dom_bb
= NULL
;
2748 counts
= XALLOCAVEC (tree
, fd
->ordered
? fd
->ordered
+ 1 : fd
->collapse
);
2749 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
2750 zero_iter1_bb
, first_zero_iter1
,
2751 zero_iter2_bb
, first_zero_iter2
, l2_dom_bb
);
2755 /* Some counts[i] vars might be uninitialized if
2756 some loop has zero iterations. But the body shouldn't
2757 be executed in that case, so just avoid uninit warnings. */
2758 for (i
= first_zero_iter1
;
2759 i
< (fd
->ordered
? fd
->ordered
: fd
->collapse
); i
++)
2760 if (SSA_VAR_P (counts
[i
]))
2761 TREE_NO_WARNING (counts
[i
]) = 1;
2763 e
= split_block (entry_bb
, gsi_stmt (gsi
));
2765 make_edge (zero_iter1_bb
, entry_bb
, EDGE_FALLTHRU
);
2766 gsi
= gsi_last_nondebug_bb (entry_bb
);
2767 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
2768 get_immediate_dominator (CDI_DOMINATORS
,
2773 /* Some counts[i] vars might be uninitialized if
2774 some loop has zero iterations. But the body shouldn't
2775 be executed in that case, so just avoid uninit warnings. */
2776 for (i
= first_zero_iter2
; i
< fd
->ordered
; i
++)
2777 if (SSA_VAR_P (counts
[i
]))
2778 TREE_NO_WARNING (counts
[i
]) = 1;
2780 make_edge (zero_iter2_bb
, entry_bb
, EDGE_FALLTHRU
);
2784 e
= split_block (entry_bb
, gsi_stmt (gsi
));
2786 make_edge (zero_iter2_bb
, entry_bb
, EDGE_FALLTHRU
);
2787 gsi
= gsi_last_nondebug_bb (entry_bb
);
2788 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
2789 get_immediate_dominator
2790 (CDI_DOMINATORS
, zero_iter2_bb
));
2793 if (fd
->collapse
== 1)
2795 counts
[0] = fd
->loop
.n2
;
2796 fd
->loop
= fd
->loops
[0];
2800 type
= TREE_TYPE (fd
->loop
.v
);
2801 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
2802 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
2803 TREE_ADDRESSABLE (istart0
) = 1;
2804 TREE_ADDRESSABLE (iend0
) = 1;
2806 /* See if we need to bias by LLONG_MIN. */
2807 if (fd
->iter_type
== long_long_unsigned_type_node
2808 && TREE_CODE (type
) == INTEGER_TYPE
2809 && !TYPE_UNSIGNED (type
)
2810 && fd
->ordered
== 0)
2814 if (fd
->loop
.cond_code
== LT_EXPR
)
2817 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
2821 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
2824 if (TREE_CODE (n1
) != INTEGER_CST
2825 || TREE_CODE (n2
) != INTEGER_CST
2826 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
2827 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
2830 gimple_stmt_iterator gsif
= gsi
;
2833 tree arr
= NULL_TREE
;
2834 if (in_combined_parallel
)
2836 gcc_assert (fd
->ordered
== 0);
2837 /* In a combined parallel loop, emit a call to
2838 GOMP_loop_foo_next. */
2839 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
2840 build_fold_addr_expr (istart0
),
2841 build_fold_addr_expr (iend0
));
2845 tree t0
, t1
, t2
, t3
, t4
;
2846 /* If this is not a combined parallel loop, emit a call to
2847 GOMP_loop_foo_start in ENTRY_BB. */
2848 t4
= build_fold_addr_expr (iend0
);
2849 t3
= build_fold_addr_expr (istart0
);
2852 t0
= build_int_cst (unsigned_type_node
,
2853 fd
->ordered
- fd
->collapse
+ 1);
2854 arr
= create_tmp_var (build_array_type_nelts (fd
->iter_type
,
2856 - fd
->collapse
+ 1),
2858 DECL_NAMELESS (arr
) = 1;
2859 TREE_ADDRESSABLE (arr
) = 1;
2860 TREE_STATIC (arr
) = 1;
2861 vec
<constructor_elt
, va_gc
> *v
;
2862 vec_alloc (v
, fd
->ordered
- fd
->collapse
+ 1);
2865 for (idx
= 0; idx
< fd
->ordered
- fd
->collapse
+ 1; idx
++)
2868 if (idx
== 0 && fd
->collapse
> 1)
2871 c
= counts
[idx
+ fd
->collapse
- 1];
2872 tree purpose
= size_int (idx
);
2873 CONSTRUCTOR_APPEND_ELT (v
, purpose
, c
);
2874 if (TREE_CODE (c
) != INTEGER_CST
)
2875 TREE_STATIC (arr
) = 0;
2878 DECL_INITIAL (arr
) = build_constructor (TREE_TYPE (arr
), v
);
2879 if (!TREE_STATIC (arr
))
2880 force_gimple_operand_gsi (&gsi
, build1 (DECL_EXPR
,
2881 void_type_node
, arr
),
2882 true, NULL_TREE
, true, GSI_SAME_STMT
);
2883 t1
= build_fold_addr_expr (arr
);
2888 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
2891 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
2894 = omp_find_clause (gimple_omp_for_clauses (fd
->for_stmt
),
2895 OMP_CLAUSE__LOOPTEMP_
);
2896 gcc_assert (innerc
);
2897 t0
= OMP_CLAUSE_DECL (innerc
);
2898 innerc
= omp_find_clause (OMP_CLAUSE_CHAIN (innerc
),
2899 OMP_CLAUSE__LOOPTEMP_
);
2900 gcc_assert (innerc
);
2901 t1
= OMP_CLAUSE_DECL (innerc
);
2903 if (POINTER_TYPE_P (TREE_TYPE (t0
))
2904 && TYPE_PRECISION (TREE_TYPE (t0
))
2905 != TYPE_PRECISION (fd
->iter_type
))
2907 /* Avoid casting pointers to integer of a different size. */
2908 tree itype
= signed_type_for (type
);
2909 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
2910 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
2914 t1
= fold_convert (fd
->iter_type
, t1
);
2915 t0
= fold_convert (fd
->iter_type
, t0
);
2919 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
2920 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
2923 if (fd
->iter_type
== long_integer_type_node
|| fd
->ordered
)
2927 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
2928 t
= omp_adjust_chunk_size (t
, fd
->simd_schedule
);
2932 t
= build_call_expr (builtin_decl_explicit (start_fn
),
2933 8, t0
, t1
, sched_arg
, t
, t3
, t4
,
2936 t
= build_call_expr (builtin_decl_explicit (start_fn
),
2937 9, t0
, t1
, t2
, sched_arg
, t
, t3
, t4
,
2940 else if (fd
->ordered
)
2941 t
= build_call_expr (builtin_decl_explicit (start_fn
),
2942 5, t0
, t1
, t
, t3
, t4
);
2944 t
= build_call_expr (builtin_decl_explicit (start_fn
),
2945 6, t0
, t1
, t2
, t
, t3
, t4
);
2947 else if (fd
->ordered
)
2948 t
= build_call_expr (builtin_decl_explicit (start_fn
),
2951 t
= build_call_expr (builtin_decl_explicit (start_fn
),
2952 5, t0
, t1
, t2
, t3
, t4
);
2960 /* The GOMP_loop_ull_*start functions have additional boolean
2961 argument, true for < loops and false for > loops.
2962 In Fortran, the C bool type can be different from
2963 boolean_type_node. */
2964 bfn_decl
= builtin_decl_explicit (start_fn
);
2965 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
2966 t5
= build_int_cst (c_bool_type
,
2967 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
2970 tree bfn_decl
= builtin_decl_explicit (start_fn
);
2971 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
2972 t
= omp_adjust_chunk_size (t
, fd
->simd_schedule
);
2974 t
= build_call_expr (bfn_decl
, 10, t5
, t0
, t1
, t2
, sched_arg
,
2975 t
, t3
, t4
, reductions
, mem
);
2977 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
2980 t
= build_call_expr (builtin_decl_explicit (start_fn
),
2981 6, t5
, t0
, t1
, t2
, t3
, t4
);
2984 if (TREE_TYPE (t
) != boolean_type_node
)
2985 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
2986 t
, build_int_cst (TREE_TYPE (t
), 0));
2987 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
2988 true, GSI_SAME_STMT
);
2989 if (arr
&& !TREE_STATIC (arr
))
2991 tree clobber
= build_clobber (TREE_TYPE (arr
));
2992 gsi_insert_before (&gsi
, gimple_build_assign (arr
, clobber
),
2995 if (fd
->have_pointer_condtemp
)
2996 expand_omp_build_assign (&gsi
, condtemp
, memv
, false);
2997 if (fd
->have_reductemp
)
2999 gimple
*g
= gsi_stmt (gsi
);
3000 gsi_remove (&gsi
, true);
3001 release_ssa_name (gimple_assign_lhs (g
));
3003 entry_bb
= region
->entry
;
3004 gsi
= gsi_last_nondebug_bb (entry_bb
);
3006 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
3008 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
3010 /* Remove the GIMPLE_OMP_FOR statement. */
3011 gsi_remove (&gsi
, true);
3013 if (gsi_end_p (gsif
))
3014 gsif
= gsi_after_labels (gsi_bb (gsif
));
3017 /* Iteration setup for sequential loop goes in L0_BB. */
3018 tree startvar
= fd
->loop
.v
;
3019 tree endvar
= NULL_TREE
;
3021 if (gimple_omp_for_combined_p (fd
->for_stmt
))
3023 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
3024 && gimple_omp_for_kind (inner_stmt
)
3025 == GF_OMP_FOR_KIND_SIMD
);
3026 tree innerc
= omp_find_clause (gimple_omp_for_clauses (inner_stmt
),
3027 OMP_CLAUSE__LOOPTEMP_
);
3028 gcc_assert (innerc
);
3029 startvar
= OMP_CLAUSE_DECL (innerc
);
3030 innerc
= omp_find_clause (OMP_CLAUSE_CHAIN (innerc
),
3031 OMP_CLAUSE__LOOPTEMP_
);
3032 gcc_assert (innerc
);
3033 endvar
= OMP_CLAUSE_DECL (innerc
);
3036 gsi
= gsi_start_bb (l0_bb
);
3038 if (fd
->ordered
&& fd
->collapse
== 1)
3039 t
= fold_build2 (MULT_EXPR
, fd
->iter_type
, t
,
3040 fold_convert (fd
->iter_type
, fd
->loop
.step
));
3042 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
3043 if (fd
->ordered
&& fd
->collapse
== 1)
3045 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
3046 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (startvar
),
3047 fd
->loop
.n1
, fold_convert (sizetype
, t
));
3050 t
= fold_convert (TREE_TYPE (startvar
), t
);
3051 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (startvar
),
3057 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
3058 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
3059 t
= fold_convert (TREE_TYPE (startvar
), t
);
3061 t
= force_gimple_operand_gsi (&gsi
, t
,
3063 && TREE_ADDRESSABLE (startvar
),
3064 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
3065 assign_stmt
= gimple_build_assign (startvar
, t
);
3066 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
3069 tree itype
= TREE_TYPE (cond_var
);
3070 /* For lastprivate(conditional:) itervar, we need some iteration
3071 counter that starts at unsigned non-zero and increases.
3072 Prefer as few IVs as possible, so if we can use startvar
3073 itself, use that, or startvar + constant (those would be
3074 incremented with step), and as last resort use the s0 + 1
3075 incremented by 1. */
3076 if ((fd
->ordered
&& fd
->collapse
== 1)
3078 || POINTER_TYPE_P (type
)
3079 || TREE_CODE (fd
->loop
.n1
) != INTEGER_CST
3080 || fd
->loop
.cond_code
!= LT_EXPR
)
3081 t
= fold_build2 (PLUS_EXPR
, itype
, fold_convert (itype
, istart0
),
3082 build_int_cst (itype
, 1));
3083 else if (tree_int_cst_sgn (fd
->loop
.n1
) == 1)
3084 t
= fold_convert (itype
, t
);
3087 tree c
= fold_convert (itype
, fd
->loop
.n1
);
3088 c
= fold_build2 (MINUS_EXPR
, itype
, build_int_cst (itype
, 1), c
);
3089 t
= fold_build2 (PLUS_EXPR
, itype
, fold_convert (itype
, t
), c
);
3091 t
= force_gimple_operand_gsi (&gsi
, t
, false,
3092 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
3093 assign_stmt
= gimple_build_assign (cond_var
, t
);
3094 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
3098 if (fd
->ordered
&& fd
->collapse
== 1)
3099 t
= fold_build2 (MULT_EXPR
, fd
->iter_type
, t
,
3100 fold_convert (fd
->iter_type
, fd
->loop
.step
));
3102 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
3103 if (fd
->ordered
&& fd
->collapse
== 1)
3105 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
3106 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (startvar
),
3107 fd
->loop
.n1
, fold_convert (sizetype
, t
));
3110 t
= fold_convert (TREE_TYPE (startvar
), t
);
3111 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (startvar
),
3117 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
3118 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
3119 t
= fold_convert (TREE_TYPE (startvar
), t
);
3121 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
3122 false, GSI_CONTINUE_LINKING
);
3125 assign_stmt
= gimple_build_assign (endvar
, iend
);
3126 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
3127 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
3128 assign_stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
3130 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, iend
);
3131 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
3133 /* Handle linear clause adjustments. */
3134 tree itercnt
= NULL_TREE
;
3135 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
3136 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
3137 c
; c
= OMP_CLAUSE_CHAIN (c
))
3138 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3139 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
3141 tree d
= OMP_CLAUSE_DECL (c
);
3142 bool is_ref
= omp_is_reference (d
);
3143 tree t
= d
, a
, dest
;
3145 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
3146 tree type
= TREE_TYPE (t
);
3147 if (POINTER_TYPE_P (type
))
3149 dest
= unshare_expr (t
);
3150 tree v
= create_tmp_var (TREE_TYPE (t
), NULL
);
3151 expand_omp_build_assign (&gsif
, v
, t
);
3152 if (itercnt
== NULL_TREE
)
3155 tree n1
= fd
->loop
.n1
;
3156 if (POINTER_TYPE_P (TREE_TYPE (itercnt
)))
3159 = fold_convert (signed_type_for (TREE_TYPE (itercnt
)),
3161 n1
= fold_convert (TREE_TYPE (itercnt
), n1
);
3163 itercnt
= fold_build2 (MINUS_EXPR
, TREE_TYPE (itercnt
),
3165 itercnt
= fold_build2 (EXACT_DIV_EXPR
, TREE_TYPE (itercnt
),
3166 itercnt
, fd
->loop
.step
);
3167 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
3169 GSI_CONTINUE_LINKING
);
3171 a
= fold_build2 (MULT_EXPR
, type
,
3172 fold_convert (type
, itercnt
),
3173 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
3174 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
3175 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), v
, a
);
3176 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
3177 false, GSI_CONTINUE_LINKING
);
3178 assign_stmt
= gimple_build_assign (dest
, t
);
3179 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
3181 if (fd
->collapse
> 1)
3182 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
3186 /* Until now, counts array contained number of iterations or
3187 variable containing it for ith loop. From now on, we need
3188 those counts only for collapsed loops, and only for the 2nd
3189 till the last collapsed one. Move those one element earlier,
3190 we'll use counts[fd->collapse - 1] for the first source/sink
3191 iteration counter and so on and counts[fd->ordered]
3192 as the array holding the current counter values for
3194 if (fd
->collapse
> 1)
3195 memmove (counts
, counts
+ 1, (fd
->collapse
- 1) * sizeof (counts
[0]));
3199 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
3201 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
3203 = fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
3204 fold_convert (type
, fd
->loops
[i
].n1
),
3205 fold_convert (type
, fd
->loops
[i
].n2
));
3206 if (!integer_onep (this_cond
))
3209 if (i
< fd
->ordered
)
3212 = create_empty_bb (EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
);
3213 add_bb_to_loop (cont_bb
, l1_bb
->loop_father
);
3214 gimple_stmt_iterator gsi
= gsi_after_labels (cont_bb
);
3215 gimple
*g
= gimple_build_omp_continue (fd
->loop
.v
, fd
->loop
.v
);
3216 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
3217 make_edge (cont_bb
, l3_bb
, EDGE_FALLTHRU
);
3218 make_edge (cont_bb
, l1_bb
, 0);
3219 l2_bb
= create_empty_bb (cont_bb
);
3220 broken_loop
= false;
3223 expand_omp_ordered_source_sink (region
, fd
, counts
, cont_bb
);
3224 cont_bb
= expand_omp_for_ordered_loops (fd
, counts
, cont_bb
, l1_bb
,
3225 ordered_lastprivate
);
3226 if (counts
[fd
->collapse
- 1])
3228 gcc_assert (fd
->collapse
== 1);
3229 gsi
= gsi_last_bb (l0_bb
);
3230 expand_omp_build_assign (&gsi
, counts
[fd
->collapse
- 1],
3232 gsi
= gsi_last_bb (cont_bb
);
3233 t
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, counts
[fd
->collapse
- 1],
3234 build_int_cst (fd
->iter_type
, 1));
3235 expand_omp_build_assign (&gsi
, counts
[fd
->collapse
- 1], t
);
3236 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
3237 size_zero_node
, NULL_TREE
, NULL_TREE
);
3238 expand_omp_build_assign (&gsi
, aref
, counts
[fd
->collapse
- 1]);
3239 t
= counts
[fd
->collapse
- 1];
3241 else if (fd
->collapse
> 1)
3245 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[0].v
),
3246 fd
->loops
[0].v
, fd
->loops
[0].n1
);
3247 t
= fold_convert (fd
->iter_type
, t
);
3249 gsi
= gsi_last_bb (l0_bb
);
3250 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
3251 size_zero_node
, NULL_TREE
, NULL_TREE
);
3252 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
3253 false, GSI_CONTINUE_LINKING
);
3254 expand_omp_build_assign (&gsi
, aref
, t
, true);
3259 /* Code to control the increment and predicate for the sequential
3260 loop goes in the CONT_BB. */
3261 gsi
= gsi_last_nondebug_bb (cont_bb
);
3262 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
3263 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
3264 vmain
= gimple_omp_continue_control_use (cont_stmt
);
3265 vback
= gimple_omp_continue_control_def (cont_stmt
);
3269 tree itype
= TREE_TYPE (cond_var
);
3271 if ((fd
->ordered
&& fd
->collapse
== 1)
3273 || POINTER_TYPE_P (type
)
3274 || TREE_CODE (fd
->loop
.n1
) != INTEGER_CST
3275 || fd
->loop
.cond_code
!= LT_EXPR
)
3276 t2
= build_int_cst (itype
, 1);
3278 t2
= fold_convert (itype
, fd
->loop
.step
);
3279 t2
= fold_build2 (PLUS_EXPR
, itype
, cond_var
, t2
);
3280 t2
= force_gimple_operand_gsi (&gsi
, t2
, false,
3281 NULL_TREE
, true, GSI_SAME_STMT
);
3282 assign_stmt
= gimple_build_assign (cond_var
, t2
);
3283 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
3286 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
3288 if (POINTER_TYPE_P (type
))
3289 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
3291 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
3292 t
= force_gimple_operand_gsi (&gsi
, t
,
3294 && TREE_ADDRESSABLE (vback
),
3295 NULL_TREE
, true, GSI_SAME_STMT
);
3296 assign_stmt
= gimple_build_assign (vback
, t
);
3297 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
3299 if (fd
->ordered
&& counts
[fd
->collapse
- 1] == NULL_TREE
)
3302 if (fd
->collapse
> 1)
3306 tem
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[0].v
),
3307 fd
->loops
[0].v
, fd
->loops
[0].n1
);
3308 tem
= fold_convert (fd
->iter_type
, tem
);
3310 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
,
3311 counts
[fd
->ordered
], size_zero_node
,
3312 NULL_TREE
, NULL_TREE
);
3313 tem
= force_gimple_operand_gsi (&gsi
, tem
, true, NULL_TREE
,
3314 true, GSI_SAME_STMT
);
3315 expand_omp_build_assign (&gsi
, aref
, tem
);
3318 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
3319 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
3321 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
3322 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
3325 /* Remove GIMPLE_OMP_CONTINUE. */
3326 gsi_remove (&gsi
, true);
3328 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
3329 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
3331 /* Emit code to get the next parallel iteration in L2_BB. */
3332 gsi
= gsi_start_bb (l2_bb
);
3334 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
3335 build_fold_addr_expr (istart0
),
3336 build_fold_addr_expr (iend0
));
3337 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
3338 false, GSI_CONTINUE_LINKING
);
3339 if (TREE_TYPE (t
) != boolean_type_node
)
3340 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
3341 t
, build_int_cst (TREE_TYPE (t
), 0));
3342 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
3343 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
3346 /* Add the loop cleanup function. */
3347 gsi
= gsi_last_nondebug_bb (exit_bb
);
3348 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
3349 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
3350 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
3351 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
3353 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
3354 gcall
*call_stmt
= gimple_build_call (t
, 0);
3357 tree arr
= counts
[fd
->ordered
];
3358 tree clobber
= build_clobber (TREE_TYPE (arr
));
3359 gsi_insert_after (&gsi
, gimple_build_assign (arr
, clobber
),
3362 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
3364 gimple_call_set_lhs (call_stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
3365 if (fd
->have_reductemp
)
3367 gimple
*g
= gimple_build_assign (reductions
, NOP_EXPR
,
3368 gimple_call_lhs (call_stmt
));
3369 gsi_insert_after (&gsi
, g
, GSI_SAME_STMT
);
3372 gsi_insert_after (&gsi
, call_stmt
, GSI_SAME_STMT
);
3373 gsi_remove (&gsi
, true);
3375 /* Connect the new blocks. */
3376 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
3377 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
3383 e
= find_edge (cont_bb
, l3_bb
);
3384 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
3386 phis
= phi_nodes (l3_bb
);
3387 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3389 gimple
*phi
= gsi_stmt (gsi
);
3390 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
3391 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
3395 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
3396 e
= find_edge (cont_bb
, l1_bb
);
3399 e
= BRANCH_EDGE (cont_bb
);
3400 gcc_assert (single_succ (e
->dest
) == l1_bb
);
3402 if (gimple_omp_for_combined_p (fd
->for_stmt
))
3407 else if (fd
->collapse
> 1)
3410 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
3413 e
->flags
= EDGE_TRUE_VALUE
;
3416 e
->probability
= profile_probability::guessed_always ().apply_scale (7, 8);
3417 find_edge (cont_bb
, l2_bb
)->probability
= e
->probability
.invert ();
3421 e
= find_edge (cont_bb
, l2_bb
);
3422 e
->flags
= EDGE_FALLTHRU
;
3424 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
3426 if (gimple_in_ssa_p (cfun
))
3428 /* Add phis to the outer loop that connect to the phis in the inner,
3429 original loop, and move the loop entry value of the inner phi to
3430 the loop entry value of the outer phi. */
3432 for (psi
= gsi_start_phis (l3_bb
); !gsi_end_p (psi
); gsi_next (&psi
))
3436 gphi
*exit_phi
= psi
.phi ();
3438 if (virtual_operand_p (gimple_phi_result (exit_phi
)))
3441 edge l2_to_l3
= find_edge (l2_bb
, l3_bb
);
3442 tree exit_res
= PHI_ARG_DEF_FROM_EDGE (exit_phi
, l2_to_l3
);
3444 basic_block latch
= BRANCH_EDGE (cont_bb
)->dest
;
3445 edge latch_to_l1
= find_edge (latch
, l1_bb
);
3447 = find_phi_with_arg_on_edge (exit_res
, latch_to_l1
);
3449 tree t
= gimple_phi_result (exit_phi
);
3450 tree new_res
= copy_ssa_name (t
, NULL
);
3451 nphi
= create_phi_node (new_res
, l0_bb
);
3453 edge l0_to_l1
= find_edge (l0_bb
, l1_bb
);
3454 t
= PHI_ARG_DEF_FROM_EDGE (inner_phi
, l0_to_l1
);
3455 locus
= gimple_phi_arg_location_from_edge (inner_phi
, l0_to_l1
);
3456 edge entry_to_l0
= find_edge (entry_bb
, l0_bb
);
3457 add_phi_arg (nphi
, t
, entry_to_l0
, locus
);
3459 edge l2_to_l0
= find_edge (l2_bb
, l0_bb
);
3460 add_phi_arg (nphi
, exit_res
, l2_to_l0
, UNKNOWN_LOCATION
);
3462 add_phi_arg (inner_phi
, new_res
, l0_to_l1
, UNKNOWN_LOCATION
);
3466 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
3467 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
3468 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
3469 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
3470 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
3471 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
3472 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
3473 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
3475 /* We enter expand_omp_for_generic with a loop. This original loop may
3476 have its own loop struct, or it may be part of an outer loop struct
3477 (which may be the fake loop). */
3478 class loop
*outer_loop
= entry_bb
->loop_father
;
3479 bool orig_loop_has_loop_struct
= l1_bb
->loop_father
!= outer_loop
;
3481 add_bb_to_loop (l2_bb
, outer_loop
);
3483 /* We've added a new loop around the original loop. Allocate the
3484 corresponding loop struct. */
3485 class loop
*new_loop
= alloc_loop ();
3486 new_loop
->header
= l0_bb
;
3487 new_loop
->latch
= l2_bb
;
3488 add_loop (new_loop
, outer_loop
);
3490 /* Allocate a loop structure for the original loop unless we already
3492 if (!orig_loop_has_loop_struct
3493 && !gimple_omp_for_combined_p (fd
->for_stmt
))
3495 class loop
*orig_loop
= alloc_loop ();
3496 orig_loop
->header
= l1_bb
;
3497 /* The loop may have multiple latches. */
3498 add_loop (orig_loop
, new_loop
);
3503 /* Helper function for expand_omp_for_static_nochunk. If PTR is NULL,
3504 compute needed allocation size. If !ALLOC of team allocations,
3505 if ALLOC of thread allocation. SZ is the initial needed size for
3506 other purposes, ALLOC_ALIGN guaranteed alignment of allocation in bytes,
3507 CNT number of elements of each array, for !ALLOC this is
3508 omp_get_num_threads (), for ALLOC number of iterations handled by the
3509 current thread. If PTR is non-NULL, it is the start of the allocation
3510 and this routine shall assign to OMP_CLAUSE_DECL (c) of those _scantemp_
3511 clauses pointers to the corresponding arrays. */
3514 expand_omp_scantemp_alloc (tree clauses
, tree ptr
, unsigned HOST_WIDE_INT sz
,
3515 unsigned HOST_WIDE_INT alloc_align
, tree cnt
,
3516 gimple_stmt_iterator
*gsi
, bool alloc
)
3518 tree eltsz
= NULL_TREE
;
3519 unsigned HOST_WIDE_INT preval
= 0;
3521 ptr
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (ptr
),
3522 ptr
, size_int (sz
));
3523 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3524 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE__SCANTEMP_
3525 && !OMP_CLAUSE__SCANTEMP__CONTROL (c
)
3526 && (!OMP_CLAUSE__SCANTEMP__ALLOC (c
)) != alloc
)
3528 tree pointee_type
= TREE_TYPE (TREE_TYPE (OMP_CLAUSE_DECL (c
)));
3529 unsigned HOST_WIDE_INT al
= TYPE_ALIGN_UNIT (pointee_type
);
3530 if (tree_fits_uhwi_p (TYPE_SIZE_UNIT (pointee_type
)))
3532 unsigned HOST_WIDE_INT szl
3533 = tree_to_uhwi (TYPE_SIZE_UNIT (pointee_type
));
3534 szl
= least_bit_hwi (szl
);
3538 if (ptr
== NULL_TREE
)
3540 if (eltsz
== NULL_TREE
)
3541 eltsz
= TYPE_SIZE_UNIT (pointee_type
);
3543 eltsz
= size_binop (PLUS_EXPR
, eltsz
,
3544 TYPE_SIZE_UNIT (pointee_type
));
3546 if (preval
== 0 && al
<= alloc_align
)
3548 unsigned HOST_WIDE_INT diff
= ROUND_UP (sz
, al
) - sz
;
3551 ptr
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (ptr
),
3552 ptr
, size_int (diff
));
3554 else if (al
> preval
)
3558 ptr
= fold_convert (pointer_sized_int_node
, ptr
);
3559 ptr
= fold_build2 (PLUS_EXPR
, pointer_sized_int_node
, ptr
,
3560 build_int_cst (pointer_sized_int_node
,
3562 ptr
= fold_build2 (BIT_AND_EXPR
, pointer_sized_int_node
, ptr
,
3563 build_int_cst (pointer_sized_int_node
,
3564 -(HOST_WIDE_INT
) al
));
3565 ptr
= fold_convert (ptr_type_node
, ptr
);
3570 if (tree_fits_uhwi_p (TYPE_SIZE_UNIT (pointee_type
)))
3576 expand_omp_build_assign (gsi
, OMP_CLAUSE_DECL (c
), ptr
, false);
3577 ptr
= OMP_CLAUSE_DECL (c
);
3578 ptr
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (ptr
), ptr
,
3579 size_binop (MULT_EXPR
, cnt
,
3580 TYPE_SIZE_UNIT (pointee_type
)));
3584 if (ptr
== NULL_TREE
)
3586 eltsz
= size_binop (MULT_EXPR
, eltsz
, cnt
);
3588 eltsz
= size_binop (PLUS_EXPR
, eltsz
, size_int (sz
));
3595 /* A subroutine of expand_omp_for. Generate code for a parallel
3596 loop with static schedule and no specified chunk size. Given
3599 for (V = N1; V cond N2; V += STEP) BODY;
3601 where COND is "<" or ">", we generate pseudocode
3603 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
3608 if ((__typeof (V)) -1 > 0 && cond is >)
3609 n = -(adj + N2 - N1) / -STEP;
3611 n = (adj + N2 - N1) / STEP;
3614 if (threadid < tt) goto L3; else goto L4;
3619 s0 = q * threadid + tt;
3622 if (s0 >= e0) goto L2; else goto L0;
3628 if (V cond e) goto L1;
3633 expand_omp_for_static_nochunk (struct omp_region
*region
,
3634 struct omp_for_data
*fd
,
3637 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
= NULL_TREE
, threadid
;
3638 tree type
, itype
, vmain
, vback
;
3639 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
3640 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
3641 basic_block fin_bb
, fourth_bb
= NULL
, fifth_bb
= NULL
, sixth_bb
= NULL
;
3642 basic_block exit1_bb
= NULL
, exit2_bb
= NULL
, exit3_bb
= NULL
;
3643 gimple_stmt_iterator gsi
, gsip
;
3645 bool broken_loop
= region
->cont
== NULL
;
3646 tree
*counts
= NULL
;
3648 tree reductions
= NULL_TREE
;
3649 tree cond_var
= NULL_TREE
, condtemp
= NULL_TREE
;
3651 itype
= type
= TREE_TYPE (fd
->loop
.v
);
3652 if (POINTER_TYPE_P (type
))
3653 itype
= signed_type_for (type
);
3655 entry_bb
= region
->entry
;
3656 cont_bb
= region
->cont
;
3657 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
3658 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
3659 gcc_assert (broken_loop
3660 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
3661 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
3662 body_bb
= single_succ (seq_start_bb
);
3665 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
3666 || single_succ (BRANCH_EDGE (cont_bb
)->dest
) == body_bb
);
3667 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
3669 exit_bb
= region
->exit
;
3671 /* Iteration space partitioning goes in ENTRY_BB. */
3672 gsi
= gsi_last_nondebug_bb (entry_bb
);
3673 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
3677 if (fd
->collapse
> 1)
3679 int first_zero_iter
= -1, dummy
= -1;
3680 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
3682 counts
= XALLOCAVEC (tree
, fd
->collapse
);
3683 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
3684 fin_bb
, first_zero_iter
,
3685 dummy_bb
, dummy
, l2_dom_bb
);
3688 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
3689 t
= integer_one_node
;
3691 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
3692 fold_convert (type
, fd
->loop
.n1
),
3693 fold_convert (type
, fd
->loop
.n2
));
3694 if (fd
->collapse
== 1
3695 && TYPE_UNSIGNED (type
)
3696 && (t
== NULL_TREE
|| !integer_onep (t
)))
3698 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
3699 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
3700 true, GSI_SAME_STMT
);
3701 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
3702 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
3703 true, GSI_SAME_STMT
);
3704 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
3705 NULL_TREE
, NULL_TREE
);
3706 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
3707 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
3708 expand_omp_regimplify_p
, NULL
, NULL
)
3709 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
3710 expand_omp_regimplify_p
, NULL
, NULL
))
3712 gsi
= gsi_for_stmt (cond_stmt
);
3713 gimple_regimplify_operands (cond_stmt
, &gsi
);
3715 ep
= split_block (entry_bb
, cond_stmt
);
3716 ep
->flags
= EDGE_TRUE_VALUE
;
3717 entry_bb
= ep
->dest
;
3718 ep
->probability
= profile_probability::very_likely ();
3719 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
3720 ep
->probability
= profile_probability::very_unlikely ();
3721 if (gimple_in_ssa_p (cfun
))
3723 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
3724 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
3725 !gsi_end_p (gpi
); gsi_next (&gpi
))
3727 gphi
*phi
= gpi
.phi ();
3728 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
3729 ep
, UNKNOWN_LOCATION
);
3732 gsi
= gsi_last_bb (entry_bb
);
3735 if (fd
->lastprivate_conditional
)
3737 tree clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
3738 tree c
= omp_find_clause (clauses
, OMP_CLAUSE__CONDTEMP_
);
3739 if (fd
->have_pointer_condtemp
)
3740 condtemp
= OMP_CLAUSE_DECL (c
);
3741 c
= omp_find_clause (OMP_CLAUSE_CHAIN (c
), OMP_CLAUSE__CONDTEMP_
);
3742 cond_var
= OMP_CLAUSE_DECL (c
);
3744 if (fd
->have_reductemp
3745 /* For scan, we don't want to reinitialize condtemp before the
3747 || (fd
->have_pointer_condtemp
&& !fd
->have_scantemp
)
3748 || fd
->have_nonctrl_scantemp
)
3750 tree t1
= build_int_cst (long_integer_type_node
, 0);
3751 tree t2
= build_int_cst (long_integer_type_node
, 1);
3752 tree t3
= build_int_cstu (long_integer_type_node
,
3753 (HOST_WIDE_INT_1U
<< 31) + 1);
3754 tree clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
3755 gimple_stmt_iterator gsi2
= gsi_none ();
3757 tree mem
= null_pointer_node
, memv
= NULL_TREE
;
3758 unsigned HOST_WIDE_INT condtemp_sz
= 0;
3759 unsigned HOST_WIDE_INT alloc_align
= 0;
3760 if (fd
->have_reductemp
)
3762 gcc_assert (!fd
->have_nonctrl_scantemp
);
3763 tree c
= omp_find_clause (clauses
, OMP_CLAUSE__REDUCTEMP_
);
3764 reductions
= OMP_CLAUSE_DECL (c
);
3765 gcc_assert (TREE_CODE (reductions
) == SSA_NAME
);
3766 g
= SSA_NAME_DEF_STMT (reductions
);
3767 reductions
= gimple_assign_rhs1 (g
);
3768 OMP_CLAUSE_DECL (c
) = reductions
;
3769 gsi2
= gsi_for_stmt (g
);
3773 if (gsi_end_p (gsip
))
3774 gsi2
= gsi_after_labels (region
->entry
);
3777 reductions
= null_pointer_node
;
3779 if (fd
->have_pointer_condtemp
|| fd
->have_nonctrl_scantemp
)
3782 if (fd
->have_pointer_condtemp
)
3783 type
= TREE_TYPE (condtemp
);
3785 type
= ptr_type_node
;
3786 memv
= create_tmp_var (type
);
3787 TREE_ADDRESSABLE (memv
) = 1;
3788 unsigned HOST_WIDE_INT sz
= 0;
3789 tree size
= NULL_TREE
;
3790 if (fd
->have_pointer_condtemp
)
3792 sz
= tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (type
)));
3793 sz
*= fd
->lastprivate_conditional
;
3796 if (fd
->have_nonctrl_scantemp
)
3798 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
3799 gimple
*g
= gimple_build_call (nthreads
, 0);
3800 nthreads
= create_tmp_var (integer_type_node
);
3801 gimple_call_set_lhs (g
, nthreads
);
3802 gsi_insert_before (&gsi2
, g
, GSI_SAME_STMT
);
3803 nthreads
= fold_convert (sizetype
, nthreads
);
3804 alloc_align
= TYPE_ALIGN_UNIT (long_long_integer_type_node
);
3805 size
= expand_omp_scantemp_alloc (clauses
, NULL_TREE
, sz
,
3806 alloc_align
, nthreads
, NULL
,
3808 size
= fold_convert (type
, size
);
3811 size
= build_int_cst (type
, sz
);
3812 expand_omp_build_assign (&gsi2
, memv
, size
, false);
3813 mem
= build_fold_addr_expr (memv
);
3816 = build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_LOOP_START
),
3817 9, t1
, t2
, t2
, t3
, t1
, null_pointer_node
,
3818 null_pointer_node
, reductions
, mem
);
3819 force_gimple_operand_gsi (&gsi2
, t
, true, NULL_TREE
,
3820 true, GSI_SAME_STMT
);
3821 if (fd
->have_pointer_condtemp
)
3822 expand_omp_build_assign (&gsi2
, condtemp
, memv
, false);
3823 if (fd
->have_nonctrl_scantemp
)
3825 tree ptr
= fd
->have_pointer_condtemp
? condtemp
: memv
;
3826 expand_omp_scantemp_alloc (clauses
, ptr
, condtemp_sz
,
3827 alloc_align
, nthreads
, &gsi2
, false);
3829 if (fd
->have_reductemp
)
3831 gsi_remove (&gsi2
, true);
3832 release_ssa_name (gimple_assign_lhs (g
));
3835 switch (gimple_omp_for_kind (fd
->for_stmt
))
3837 case GF_OMP_FOR_KIND_FOR
:
3838 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
3839 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
3841 case GF_OMP_FOR_KIND_DISTRIBUTE
:
3842 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
3843 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
3848 nthreads
= build_call_expr (nthreads
, 0);
3849 nthreads
= fold_convert (itype
, nthreads
);
3850 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
3851 true, GSI_SAME_STMT
);
3852 threadid
= build_call_expr (threadid
, 0);
3853 threadid
= fold_convert (itype
, threadid
);
3854 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
3855 true, GSI_SAME_STMT
);
3859 step
= fd
->loop
.step
;
3860 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
3862 tree innerc
= omp_find_clause (gimple_omp_for_clauses (fd
->for_stmt
),
3863 OMP_CLAUSE__LOOPTEMP_
);
3864 gcc_assert (innerc
);
3865 n1
= OMP_CLAUSE_DECL (innerc
);
3866 innerc
= omp_find_clause (OMP_CLAUSE_CHAIN (innerc
),
3867 OMP_CLAUSE__LOOPTEMP_
);
3868 gcc_assert (innerc
);
3869 n2
= OMP_CLAUSE_DECL (innerc
);
3871 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
3872 true, NULL_TREE
, true, GSI_SAME_STMT
);
3873 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
3874 true, NULL_TREE
, true, GSI_SAME_STMT
);
3875 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
3876 true, NULL_TREE
, true, GSI_SAME_STMT
);
3878 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
3879 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
3880 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
3881 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
3882 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
3883 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
3884 fold_build1 (NEGATE_EXPR
, itype
, t
),
3885 fold_build1 (NEGATE_EXPR
, itype
, step
));
3887 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
3888 t
= fold_convert (itype
, t
);
3889 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
3891 q
= create_tmp_reg (itype
, "q");
3892 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
3893 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
3894 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
3896 tt
= create_tmp_reg (itype
, "tt");
3897 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
3898 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
3899 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
3901 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
3902 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
3903 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
3905 second_bb
= split_block (entry_bb
, cond_stmt
)->dest
;
3906 gsi
= gsi_last_nondebug_bb (second_bb
);
3907 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
3909 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
3911 gassign
*assign_stmt
3912 = gimple_build_assign (q
, PLUS_EXPR
, q
, build_int_cst (itype
, 1));
3913 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
3915 third_bb
= split_block (second_bb
, assign_stmt
)->dest
;
3916 gsi
= gsi_last_nondebug_bb (third_bb
);
3917 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
3919 if (fd
->have_nonctrl_scantemp
)
3921 tree clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
3922 tree controlp
= NULL_TREE
, controlb
= NULL_TREE
;
3923 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3924 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE__SCANTEMP_
3925 && OMP_CLAUSE__SCANTEMP__CONTROL (c
))
3927 if (TREE_TYPE (OMP_CLAUSE_DECL (c
)) == boolean_type_node
)
3928 controlb
= OMP_CLAUSE_DECL (c
);
3930 controlp
= OMP_CLAUSE_DECL (c
);
3931 if (controlb
&& controlp
)
3934 gcc_assert (controlp
&& controlb
);
3935 tree cnt
= create_tmp_var (sizetype
);
3936 gimple
*g
= gimple_build_assign (cnt
, NOP_EXPR
, q
);
3937 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
3938 unsigned HOST_WIDE_INT alloc_align
= TYPE_ALIGN_UNIT (ptr_type_node
);
3939 tree sz
= expand_omp_scantemp_alloc (clauses
, NULL_TREE
, 0,
3940 alloc_align
, cnt
, NULL
, true);
3941 tree size
= create_tmp_var (sizetype
);
3942 expand_omp_build_assign (&gsi
, size
, sz
, false);
3943 tree cmp
= fold_build2 (GT_EXPR
, boolean_type_node
,
3944 size
, size_int (16384));
3945 expand_omp_build_assign (&gsi
, controlb
, cmp
);
3946 g
= gimple_build_cond (NE_EXPR
, controlb
, boolean_false_node
,
3947 NULL_TREE
, NULL_TREE
);
3948 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
3949 fourth_bb
= split_block (third_bb
, g
)->dest
;
3950 gsi
= gsi_last_nondebug_bb (fourth_bb
);
3951 /* FIXME: Once we have allocators, this should use allocator. */
3952 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_MALLOC
), 1, size
);
3953 gimple_call_set_lhs (g
, controlp
);
3954 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
3955 expand_omp_scantemp_alloc (clauses
, controlp
, 0, alloc_align
, cnt
,
3959 fifth_bb
= split_block (fourth_bb
, g
)->dest
;
3960 gsi
= gsi_last_nondebug_bb (fifth_bb
);
3962 g
= gimple_build_call (builtin_decl_implicit (BUILT_IN_STACK_SAVE
), 0);
3963 gimple_call_set_lhs (g
, controlp
);
3964 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
3965 tree alloca_decl
= builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
3966 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3967 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE__SCANTEMP_
3968 && OMP_CLAUSE__SCANTEMP__ALLOC (c
))
3970 tree tmp
= create_tmp_var (sizetype
);
3971 tree pointee_type
= TREE_TYPE (TREE_TYPE (OMP_CLAUSE_DECL (c
)));
3972 g
= gimple_build_assign (tmp
, MULT_EXPR
, cnt
,
3973 TYPE_SIZE_UNIT (pointee_type
));
3974 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
3975 g
= gimple_build_call (alloca_decl
, 2, tmp
,
3976 size_int (TYPE_ALIGN (pointee_type
)));
3977 gimple_call_set_lhs (g
, OMP_CLAUSE_DECL (c
));
3978 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
3981 sixth_bb
= split_block (fifth_bb
, g
)->dest
;
3982 gsi
= gsi_last_nondebug_bb (sixth_bb
);
3985 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
3986 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
3987 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
3989 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
3990 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
3992 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
3993 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
3995 /* Remove the GIMPLE_OMP_FOR statement. */
3996 gsi_remove (&gsi
, true);
3998 /* Setup code for sequential iteration goes in SEQ_START_BB. */
3999 gsi
= gsi_start_bb (seq_start_bb
);
4001 tree startvar
= fd
->loop
.v
;
4002 tree endvar
= NULL_TREE
;
4004 if (gimple_omp_for_combined_p (fd
->for_stmt
))
4006 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
4007 ? gimple_omp_parallel_clauses (inner_stmt
)
4008 : gimple_omp_for_clauses (inner_stmt
);
4009 tree innerc
= omp_find_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
4010 gcc_assert (innerc
);
4011 startvar
= OMP_CLAUSE_DECL (innerc
);
4012 innerc
= omp_find_clause (OMP_CLAUSE_CHAIN (innerc
),
4013 OMP_CLAUSE__LOOPTEMP_
);
4014 gcc_assert (innerc
);
4015 endvar
= OMP_CLAUSE_DECL (innerc
);
4016 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
4017 && gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
4020 for (i
= 1; i
< fd
->collapse
; i
++)
4022 innerc
= omp_find_clause (OMP_CLAUSE_CHAIN (innerc
),
4023 OMP_CLAUSE__LOOPTEMP_
);
4024 gcc_assert (innerc
);
4026 innerc
= omp_find_clause (OMP_CLAUSE_CHAIN (innerc
),
4027 OMP_CLAUSE__LOOPTEMP_
);
4030 /* If needed (distribute parallel for with lastprivate),
4031 propagate down the total number of iterations. */
4032 tree t
= fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (innerc
)),
4034 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, false,
4035 GSI_CONTINUE_LINKING
);
4036 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
4037 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
4041 t
= fold_convert (itype
, s0
);
4042 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
4043 if (POINTER_TYPE_P (type
))
4045 t
= fold_build_pointer_plus (n1
, t
);
4046 if (!POINTER_TYPE_P (TREE_TYPE (startvar
))
4047 && TYPE_PRECISION (TREE_TYPE (startvar
)) > TYPE_PRECISION (type
))
4048 t
= fold_convert (signed_type_for (type
), t
);
4051 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
4052 t
= fold_convert (TREE_TYPE (startvar
), t
);
4053 t
= force_gimple_operand_gsi (&gsi
, t
,
4055 && TREE_ADDRESSABLE (startvar
),
4056 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
4057 assign_stmt
= gimple_build_assign (startvar
, t
);
4058 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
4061 tree itype
= TREE_TYPE (cond_var
);
4062 /* For lastprivate(conditional:) itervar, we need some iteration
4063 counter that starts at unsigned non-zero and increases.
4064 Prefer as few IVs as possible, so if we can use startvar
4065 itself, use that, or startvar + constant (those would be
4066 incremented with step), and as last resort use the s0 + 1
4067 incremented by 1. */
4068 if (POINTER_TYPE_P (type
)
4069 || TREE_CODE (n1
) != INTEGER_CST
4070 || fd
->loop
.cond_code
!= LT_EXPR
)
4071 t
= fold_build2 (PLUS_EXPR
, itype
, fold_convert (itype
, s0
),
4072 build_int_cst (itype
, 1));
4073 else if (tree_int_cst_sgn (n1
) == 1)
4074 t
= fold_convert (itype
, t
);
4077 tree c
= fold_convert (itype
, n1
);
4078 c
= fold_build2 (MINUS_EXPR
, itype
, build_int_cst (itype
, 1), c
);
4079 t
= fold_build2 (PLUS_EXPR
, itype
, fold_convert (itype
, t
), c
);
4081 t
= force_gimple_operand_gsi (&gsi
, t
, false,
4082 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
4083 assign_stmt
= gimple_build_assign (cond_var
, t
);
4084 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
4087 t
= fold_convert (itype
, e0
);
4088 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
4089 if (POINTER_TYPE_P (type
))
4091 t
= fold_build_pointer_plus (n1
, t
);
4092 if (!POINTER_TYPE_P (TREE_TYPE (startvar
))
4093 && TYPE_PRECISION (TREE_TYPE (startvar
)) > TYPE_PRECISION (type
))
4094 t
= fold_convert (signed_type_for (type
), t
);
4097 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
4098 t
= fold_convert (TREE_TYPE (startvar
), t
);
4099 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4100 false, GSI_CONTINUE_LINKING
);
4103 assign_stmt
= gimple_build_assign (endvar
, e
);
4104 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
4105 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
4106 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
4108 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
4109 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
4111 /* Handle linear clause adjustments. */
4112 tree itercnt
= NULL_TREE
;
4113 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
4114 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
4115 c
; c
= OMP_CLAUSE_CHAIN (c
))
4116 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
4117 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
4119 tree d
= OMP_CLAUSE_DECL (c
);
4120 bool is_ref
= omp_is_reference (d
);
4121 tree t
= d
, a
, dest
;
4123 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
4124 if (itercnt
== NULL_TREE
)
4126 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
4128 itercnt
= fold_build2 (MINUS_EXPR
, itype
,
4129 fold_convert (itype
, n1
),
4130 fold_convert (itype
, fd
->loop
.n1
));
4131 itercnt
= fold_build2 (EXACT_DIV_EXPR
, itype
, itercnt
, step
);
4132 itercnt
= fold_build2 (PLUS_EXPR
, itype
, itercnt
, s0
);
4133 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
4135 GSI_CONTINUE_LINKING
);
4140 tree type
= TREE_TYPE (t
);
4141 if (POINTER_TYPE_P (type
))
4143 a
= fold_build2 (MULT_EXPR
, type
,
4144 fold_convert (type
, itercnt
),
4145 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
4146 dest
= unshare_expr (t
);
4147 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
4148 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), t
, a
);
4149 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4150 false, GSI_CONTINUE_LINKING
);
4151 assign_stmt
= gimple_build_assign (dest
, t
);
4152 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
4154 if (fd
->collapse
> 1)
4155 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
4159 /* The code controlling the sequential loop replaces the
4160 GIMPLE_OMP_CONTINUE. */
4161 gsi
= gsi_last_nondebug_bb (cont_bb
);
4162 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
4163 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
4164 vmain
= gimple_omp_continue_control_use (cont_stmt
);
4165 vback
= gimple_omp_continue_control_def (cont_stmt
);
4169 tree itype
= TREE_TYPE (cond_var
);
4171 if (POINTER_TYPE_P (type
)
4172 || TREE_CODE (n1
) != INTEGER_CST
4173 || fd
->loop
.cond_code
!= LT_EXPR
)
4174 t2
= build_int_cst (itype
, 1);
4176 t2
= fold_convert (itype
, step
);
4177 t2
= fold_build2 (PLUS_EXPR
, itype
, cond_var
, t2
);
4178 t2
= force_gimple_operand_gsi (&gsi
, t2
, false,
4179 NULL_TREE
, true, GSI_SAME_STMT
);
4180 assign_stmt
= gimple_build_assign (cond_var
, t2
);
4181 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
4184 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
4186 if (POINTER_TYPE_P (type
))
4187 t
= fold_build_pointer_plus (vmain
, step
);
4189 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
4190 t
= force_gimple_operand_gsi (&gsi
, t
,
4192 && TREE_ADDRESSABLE (vback
),
4193 NULL_TREE
, true, GSI_SAME_STMT
);
4194 assign_stmt
= gimple_build_assign (vback
, t
);
4195 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
4197 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
4198 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
4200 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
4203 /* Remove the GIMPLE_OMP_CONTINUE statement. */
4204 gsi_remove (&gsi
, true);
4206 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
4207 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
4210 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
4211 gsi
= gsi_last_nondebug_bb (exit_bb
);
4212 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
4214 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
4215 if (fd
->have_reductemp
4216 || ((fd
->have_pointer_condtemp
|| fd
->have_scantemp
)
4217 && !fd
->have_nonctrl_scantemp
))
4221 fn
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
4223 fn
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
4224 gcall
*g
= gimple_build_call (fn
, 0);
4227 gimple_call_set_lhs (g
, t
);
4228 if (fd
->have_reductemp
)
4229 gsi_insert_after (&gsi
, gimple_build_assign (reductions
,
4233 gsi_insert_after (&gsi
, g
, GSI_SAME_STMT
);
4236 gsi_insert_after (&gsi
, omp_build_barrier (t
), GSI_SAME_STMT
);
4238 else if ((fd
->have_pointer_condtemp
|| fd
->have_scantemp
)
4239 && !fd
->have_nonctrl_scantemp
)
4241 tree fn
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
4242 gcall
*g
= gimple_build_call (fn
, 0);
4243 gsi_insert_after (&gsi
, g
, GSI_SAME_STMT
);
4245 if (fd
->have_scantemp
&& !fd
->have_nonctrl_scantemp
)
4247 tree clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
4248 tree controlp
= NULL_TREE
, controlb
= NULL_TREE
;
4249 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4250 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE__SCANTEMP_
4251 && OMP_CLAUSE__SCANTEMP__CONTROL (c
))
4253 if (TREE_TYPE (OMP_CLAUSE_DECL (c
)) == boolean_type_node
)
4254 controlb
= OMP_CLAUSE_DECL (c
);
4256 controlp
= OMP_CLAUSE_DECL (c
);
4257 if (controlb
&& controlp
)
4260 gcc_assert (controlp
&& controlb
);
4261 gimple
*g
= gimple_build_cond (NE_EXPR
, controlb
, boolean_false_node
,
4262 NULL_TREE
, NULL_TREE
);
4263 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
4264 exit1_bb
= split_block (exit_bb
, g
)->dest
;
4265 gsi
= gsi_after_labels (exit1_bb
);
4266 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_FREE
), 1,
4268 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
4269 exit2_bb
= split_block (exit1_bb
, g
)->dest
;
4270 gsi
= gsi_after_labels (exit2_bb
);
4271 g
= gimple_build_call (builtin_decl_implicit (BUILT_IN_STACK_RESTORE
), 1,
4273 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
4274 exit3_bb
= split_block (exit2_bb
, g
)->dest
;
4275 gsi
= gsi_after_labels (exit3_bb
);
4277 gsi_remove (&gsi
, true);
4279 /* Connect all the blocks. */
4280 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
4281 ep
->probability
= profile_probability::guessed_always ().apply_scale (3, 4);
4282 ep
= find_edge (entry_bb
, second_bb
);
4283 ep
->flags
= EDGE_TRUE_VALUE
;
4284 ep
->probability
= profile_probability::guessed_always ().apply_scale (1, 4);
4287 ep
= make_edge (third_bb
, fifth_bb
, EDGE_FALSE_VALUE
);
4289 = profile_probability::guessed_always ().apply_scale (1, 2);
4290 ep
= find_edge (third_bb
, fourth_bb
);
4291 ep
->flags
= EDGE_TRUE_VALUE
;
4293 = profile_probability::guessed_always ().apply_scale (1, 2);
4294 ep
= find_edge (fourth_bb
, fifth_bb
);
4295 redirect_edge_and_branch (ep
, sixth_bb
);
4298 sixth_bb
= third_bb
;
4299 find_edge (sixth_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
4300 find_edge (sixth_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
4303 ep
= make_edge (exit_bb
, exit2_bb
, EDGE_FALSE_VALUE
);
4305 = profile_probability::guessed_always ().apply_scale (1, 2);
4306 ep
= find_edge (exit_bb
, exit1_bb
);
4307 ep
->flags
= EDGE_TRUE_VALUE
;
4309 = profile_probability::guessed_always ().apply_scale (1, 2);
4310 ep
= find_edge (exit1_bb
, exit2_bb
);
4311 redirect_edge_and_branch (ep
, exit3_bb
);
4316 ep
= find_edge (cont_bb
, body_bb
);
4319 ep
= BRANCH_EDGE (cont_bb
);
4320 gcc_assert (single_succ (ep
->dest
) == body_bb
);
4322 if (gimple_omp_for_combined_p (fd
->for_stmt
))
4327 else if (fd
->collapse
> 1)
4330 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
4333 ep
->flags
= EDGE_TRUE_VALUE
;
4334 find_edge (cont_bb
, fin_bb
)->flags
4335 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
4338 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
4339 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
4342 set_immediate_dominator (CDI_DOMINATORS
, fifth_bb
, third_bb
);
4343 set_immediate_dominator (CDI_DOMINATORS
, sixth_bb
, third_bb
);
4345 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, sixth_bb
);
4347 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
4348 recompute_dominator (CDI_DOMINATORS
, body_bb
));
4349 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
4350 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
4353 set_immediate_dominator (CDI_DOMINATORS
, exit2_bb
, exit_bb
);
4354 set_immediate_dominator (CDI_DOMINATORS
, exit3_bb
, exit_bb
);
4357 class loop
*loop
= body_bb
->loop_father
;
4358 if (loop
!= entry_bb
->loop_father
)
4360 gcc_assert (broken_loop
|| loop
->header
== body_bb
);
4361 gcc_assert (broken_loop
4362 || loop
->latch
== region
->cont
4363 || single_pred (loop
->latch
) == region
->cont
);
4367 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
4369 loop
= alloc_loop ();
4370 loop
->header
= body_bb
;
4371 if (collapse_bb
== NULL
)
4372 loop
->latch
= cont_bb
;
4373 add_loop (loop
, body_bb
->loop_father
);
4377 /* Return phi in E->DEST with ARG on edge E. */
4380 find_phi_with_arg_on_edge (tree arg
, edge e
)
4382 basic_block bb
= e
->dest
;
4384 for (gphi_iterator gpi
= gsi_start_phis (bb
);
4388 gphi
*phi
= gpi
.phi ();
4389 if (PHI_ARG_DEF_FROM_EDGE (phi
, e
) == arg
)
4396 /* A subroutine of expand_omp_for. Generate code for a parallel
4397 loop with static schedule and a specified chunk size. Given
4400 for (V = N1; V cond N2; V += STEP) BODY;
4402 where COND is "<" or ">", we generate pseudocode
4404 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
4409 if ((__typeof (V)) -1 > 0 && cond is >)
4410 n = -(adj + N2 - N1) / -STEP;
4412 n = (adj + N2 - N1) / STEP;
4414 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
4415 here so that V is defined
4416 if the loop is not entered
4418 s0 = (trip * nthreads + threadid) * CHUNK;
4419 e0 = min (s0 + CHUNK, n);
4420 if (s0 < n) goto L1; else goto L4;
4427 if (V cond e) goto L2; else goto L3;
4435 expand_omp_for_static_chunk (struct omp_region
*region
,
4436 struct omp_for_data
*fd
, gimple
*inner_stmt
)
4438 tree n
, s0
, e0
, e
, t
;
4439 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
4440 tree type
, itype
, vmain
, vback
, vextra
;
4441 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
4442 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
4443 gimple_stmt_iterator gsi
, gsip
;
4445 bool broken_loop
= region
->cont
== NULL
;
4446 tree
*counts
= NULL
;
4448 tree reductions
= NULL_TREE
;
4449 tree cond_var
= NULL_TREE
, condtemp
= NULL_TREE
;
4451 itype
= type
= TREE_TYPE (fd
->loop
.v
);
4452 if (POINTER_TYPE_P (type
))
4453 itype
= signed_type_for (type
);
4455 entry_bb
= region
->entry
;
4456 se
= split_block (entry_bb
, last_stmt (entry_bb
));
4458 iter_part_bb
= se
->dest
;
4459 cont_bb
= region
->cont
;
4460 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
4461 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
4462 gcc_assert (broken_loop
4463 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
4464 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
4465 body_bb
= single_succ (seq_start_bb
);
4468 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
4469 || single_succ (BRANCH_EDGE (cont_bb
)->dest
) == body_bb
);
4470 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
4471 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
4473 exit_bb
= region
->exit
;
4475 /* Trip and adjustment setup goes in ENTRY_BB. */
4476 gsi
= gsi_last_nondebug_bb (entry_bb
);
4477 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
4481 if (fd
->collapse
> 1)
4483 int first_zero_iter
= -1, dummy
= -1;
4484 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
4486 counts
= XALLOCAVEC (tree
, fd
->collapse
);
4487 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
4488 fin_bb
, first_zero_iter
,
4489 dummy_bb
, dummy
, l2_dom_bb
);
4492 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
4493 t
= integer_one_node
;
4495 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
4496 fold_convert (type
, fd
->loop
.n1
),
4497 fold_convert (type
, fd
->loop
.n2
));
4498 if (fd
->collapse
== 1
4499 && TYPE_UNSIGNED (type
)
4500 && (t
== NULL_TREE
|| !integer_onep (t
)))
4502 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
4503 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
4504 true, GSI_SAME_STMT
);
4505 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
4506 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
4507 true, GSI_SAME_STMT
);
4508 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
4509 NULL_TREE
, NULL_TREE
);
4510 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
4511 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
4512 expand_omp_regimplify_p
, NULL
, NULL
)
4513 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
4514 expand_omp_regimplify_p
, NULL
, NULL
))
4516 gsi
= gsi_for_stmt (cond_stmt
);
4517 gimple_regimplify_operands (cond_stmt
, &gsi
);
4519 se
= split_block (entry_bb
, cond_stmt
);
4520 se
->flags
= EDGE_TRUE_VALUE
;
4521 entry_bb
= se
->dest
;
4522 se
->probability
= profile_probability::very_likely ();
4523 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
4524 se
->probability
= profile_probability::very_unlikely ();
4525 if (gimple_in_ssa_p (cfun
))
4527 int dest_idx
= find_edge (iter_part_bb
, fin_bb
)->dest_idx
;
4528 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
4529 !gsi_end_p (gpi
); gsi_next (&gpi
))
4531 gphi
*phi
= gpi
.phi ();
4532 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
4533 se
, UNKNOWN_LOCATION
);
4536 gsi
= gsi_last_bb (entry_bb
);
4539 if (fd
->lastprivate_conditional
)
4541 tree clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
4542 tree c
= omp_find_clause (clauses
, OMP_CLAUSE__CONDTEMP_
);
4543 if (fd
->have_pointer_condtemp
)
4544 condtemp
= OMP_CLAUSE_DECL (c
);
4545 c
= omp_find_clause (OMP_CLAUSE_CHAIN (c
), OMP_CLAUSE__CONDTEMP_
);
4546 cond_var
= OMP_CLAUSE_DECL (c
);
4548 if (fd
->have_reductemp
|| fd
->have_pointer_condtemp
)
4550 tree t1
= build_int_cst (long_integer_type_node
, 0);
4551 tree t2
= build_int_cst (long_integer_type_node
, 1);
4552 tree t3
= build_int_cstu (long_integer_type_node
,
4553 (HOST_WIDE_INT_1U
<< 31) + 1);
4554 tree clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
4555 gimple_stmt_iterator gsi2
= gsi_none ();
4557 tree mem
= null_pointer_node
, memv
= NULL_TREE
;
4558 if (fd
->have_reductemp
)
4560 tree c
= omp_find_clause (clauses
, OMP_CLAUSE__REDUCTEMP_
);
4561 reductions
= OMP_CLAUSE_DECL (c
);
4562 gcc_assert (TREE_CODE (reductions
) == SSA_NAME
);
4563 g
= SSA_NAME_DEF_STMT (reductions
);
4564 reductions
= gimple_assign_rhs1 (g
);
4565 OMP_CLAUSE_DECL (c
) = reductions
;
4566 gsi2
= gsi_for_stmt (g
);
4570 if (gsi_end_p (gsip
))
4571 gsi2
= gsi_after_labels (region
->entry
);
4574 reductions
= null_pointer_node
;
4576 if (fd
->have_pointer_condtemp
)
4578 tree type
= TREE_TYPE (condtemp
);
4579 memv
= create_tmp_var (type
);
4580 TREE_ADDRESSABLE (memv
) = 1;
4581 unsigned HOST_WIDE_INT sz
4582 = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4583 sz
*= fd
->lastprivate_conditional
;
4584 expand_omp_build_assign (&gsi2
, memv
, build_int_cst (type
, sz
),
4586 mem
= build_fold_addr_expr (memv
);
4589 = build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_LOOP_START
),
4590 9, t1
, t2
, t2
, t3
, t1
, null_pointer_node
,
4591 null_pointer_node
, reductions
, mem
);
4592 force_gimple_operand_gsi (&gsi2
, t
, true, NULL_TREE
,
4593 true, GSI_SAME_STMT
);
4594 if (fd
->have_pointer_condtemp
)
4595 expand_omp_build_assign (&gsi2
, condtemp
, memv
, false);
4596 if (fd
->have_reductemp
)
4598 gsi_remove (&gsi2
, true);
4599 release_ssa_name (gimple_assign_lhs (g
));
4602 switch (gimple_omp_for_kind (fd
->for_stmt
))
4604 case GF_OMP_FOR_KIND_FOR
:
4605 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
4606 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4608 case GF_OMP_FOR_KIND_DISTRIBUTE
:
4609 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
4610 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
4615 nthreads
= build_call_expr (nthreads
, 0);
4616 nthreads
= fold_convert (itype
, nthreads
);
4617 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
4618 true, GSI_SAME_STMT
);
4619 threadid
= build_call_expr (threadid
, 0);
4620 threadid
= fold_convert (itype
, threadid
);
4621 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
4622 true, GSI_SAME_STMT
);
4626 step
= fd
->loop
.step
;
4627 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
4629 tree innerc
= omp_find_clause (gimple_omp_for_clauses (fd
->for_stmt
),
4630 OMP_CLAUSE__LOOPTEMP_
);
4631 gcc_assert (innerc
);
4632 n1
= OMP_CLAUSE_DECL (innerc
);
4633 innerc
= omp_find_clause (OMP_CLAUSE_CHAIN (innerc
),
4634 OMP_CLAUSE__LOOPTEMP_
);
4635 gcc_assert (innerc
);
4636 n2
= OMP_CLAUSE_DECL (innerc
);
4638 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
4639 true, NULL_TREE
, true, GSI_SAME_STMT
);
4640 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
4641 true, NULL_TREE
, true, GSI_SAME_STMT
);
4642 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
4643 true, NULL_TREE
, true, GSI_SAME_STMT
);
4644 tree chunk_size
= fold_convert (itype
, fd
->chunk_size
);
4645 chunk_size
= omp_adjust_chunk_size (chunk_size
, fd
->simd_schedule
);
4647 = force_gimple_operand_gsi (&gsi
, chunk_size
, true, NULL_TREE
, true,
4650 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
4651 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
4652 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
4653 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
4654 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
4655 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
4656 fold_build1 (NEGATE_EXPR
, itype
, t
),
4657 fold_build1 (NEGATE_EXPR
, itype
, step
));
4659 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
4660 t
= fold_convert (itype
, t
);
4661 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4662 true, GSI_SAME_STMT
);
4664 trip_var
= create_tmp_reg (itype
, ".trip");
4665 if (gimple_in_ssa_p (cfun
))
4667 trip_init
= make_ssa_name (trip_var
);
4668 trip_main
= make_ssa_name (trip_var
);
4669 trip_back
= make_ssa_name (trip_var
);
4673 trip_init
= trip_var
;
4674 trip_main
= trip_var
;
4675 trip_back
= trip_var
;
4678 gassign
*assign_stmt
4679 = gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
4680 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
4682 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, chunk_size
);
4683 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
4684 if (POINTER_TYPE_P (type
))
4685 t
= fold_build_pointer_plus (n1
, t
);
4687 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
4688 vextra
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4689 true, GSI_SAME_STMT
);
4691 /* Remove the GIMPLE_OMP_FOR. */
4692 gsi_remove (&gsi
, true);
4694 gimple_stmt_iterator gsif
= gsi
;
4696 /* Iteration space partitioning goes in ITER_PART_BB. */
4697 gsi
= gsi_last_bb (iter_part_bb
);
4699 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
4700 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
4701 t
= fold_build2 (MULT_EXPR
, itype
, t
, chunk_size
);
4702 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4703 false, GSI_CONTINUE_LINKING
);
4705 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, chunk_size
);
4706 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
4707 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4708 false, GSI_CONTINUE_LINKING
);
4710 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
4711 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
4713 /* Setup code for sequential iteration goes in SEQ_START_BB. */
4714 gsi
= gsi_start_bb (seq_start_bb
);
4716 tree startvar
= fd
->loop
.v
;
4717 tree endvar
= NULL_TREE
;
4719 if (gimple_omp_for_combined_p (fd
->for_stmt
))
4721 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
4722 ? gimple_omp_parallel_clauses (inner_stmt
)
4723 : gimple_omp_for_clauses (inner_stmt
);
4724 tree innerc
= omp_find_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
4725 gcc_assert (innerc
);
4726 startvar
= OMP_CLAUSE_DECL (innerc
);
4727 innerc
= omp_find_clause (OMP_CLAUSE_CHAIN (innerc
),
4728 OMP_CLAUSE__LOOPTEMP_
);
4729 gcc_assert (innerc
);
4730 endvar
= OMP_CLAUSE_DECL (innerc
);
4731 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
4732 && gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
4735 for (i
= 1; i
< fd
->collapse
; i
++)
4737 innerc
= omp_find_clause (OMP_CLAUSE_CHAIN (innerc
),
4738 OMP_CLAUSE__LOOPTEMP_
);
4739 gcc_assert (innerc
);
4741 innerc
= omp_find_clause (OMP_CLAUSE_CHAIN (innerc
),
4742 OMP_CLAUSE__LOOPTEMP_
);
4745 /* If needed (distribute parallel for with lastprivate),
4746 propagate down the total number of iterations. */
4747 tree t
= fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (innerc
)),
4749 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, false,
4750 GSI_CONTINUE_LINKING
);
4751 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
4752 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
4757 t
= fold_convert (itype
, s0
);
4758 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
4759 if (POINTER_TYPE_P (type
))
4761 t
= fold_build_pointer_plus (n1
, t
);
4762 if (!POINTER_TYPE_P (TREE_TYPE (startvar
))
4763 && TYPE_PRECISION (TREE_TYPE (startvar
)) > TYPE_PRECISION (type
))
4764 t
= fold_convert (signed_type_for (type
), t
);
4767 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
4768 t
= fold_convert (TREE_TYPE (startvar
), t
);
4769 t
= force_gimple_operand_gsi (&gsi
, t
,
4771 && TREE_ADDRESSABLE (startvar
),
4772 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
4773 assign_stmt
= gimple_build_assign (startvar
, t
);
4774 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
4777 tree itype
= TREE_TYPE (cond_var
);
4778 /* For lastprivate(conditional:) itervar, we need some iteration
4779 counter that starts at unsigned non-zero and increases.
4780 Prefer as few IVs as possible, so if we can use startvar
4781 itself, use that, or startvar + constant (those would be
4782 incremented with step), and as last resort use the s0 + 1
4783 incremented by 1. */
4784 if (POINTER_TYPE_P (type
)
4785 || TREE_CODE (n1
) != INTEGER_CST
4786 || fd
->loop
.cond_code
!= LT_EXPR
)
4787 t
= fold_build2 (PLUS_EXPR
, itype
, fold_convert (itype
, s0
),
4788 build_int_cst (itype
, 1));
4789 else if (tree_int_cst_sgn (n1
) == 1)
4790 t
= fold_convert (itype
, t
);
4793 tree c
= fold_convert (itype
, n1
);
4794 c
= fold_build2 (MINUS_EXPR
, itype
, build_int_cst (itype
, 1), c
);
4795 t
= fold_build2 (PLUS_EXPR
, itype
, fold_convert (itype
, t
), c
);
4797 t
= force_gimple_operand_gsi (&gsi
, t
, false,
4798 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
4799 assign_stmt
= gimple_build_assign (cond_var
, t
);
4800 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
4803 t
= fold_convert (itype
, e0
);
4804 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
4805 if (POINTER_TYPE_P (type
))
4807 t
= fold_build_pointer_plus (n1
, t
);
4808 if (!POINTER_TYPE_P (TREE_TYPE (startvar
))
4809 && TYPE_PRECISION (TREE_TYPE (startvar
)) > TYPE_PRECISION (type
))
4810 t
= fold_convert (signed_type_for (type
), t
);
4813 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
4814 t
= fold_convert (TREE_TYPE (startvar
), t
);
4815 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4816 false, GSI_CONTINUE_LINKING
);
4819 assign_stmt
= gimple_build_assign (endvar
, e
);
4820 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
4821 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
4822 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
4824 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
4825 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
4827 /* Handle linear clause adjustments. */
4828 tree itercnt
= NULL_TREE
, itercntbias
= NULL_TREE
;
4829 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
4830 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
4831 c
; c
= OMP_CLAUSE_CHAIN (c
))
4832 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
4833 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
4835 tree d
= OMP_CLAUSE_DECL (c
);
4836 bool is_ref
= omp_is_reference (d
);
4837 tree t
= d
, a
, dest
;
4839 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
4840 tree type
= TREE_TYPE (t
);
4841 if (POINTER_TYPE_P (type
))
4843 dest
= unshare_expr (t
);
4844 tree v
= create_tmp_var (TREE_TYPE (t
), NULL
);
4845 expand_omp_build_assign (&gsif
, v
, t
);
4846 if (itercnt
== NULL_TREE
)
4848 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
4851 = fold_build2 (MINUS_EXPR
, itype
, fold_convert (itype
, n1
),
4852 fold_convert (itype
, fd
->loop
.n1
));
4853 itercntbias
= fold_build2 (EXACT_DIV_EXPR
, itype
,
4856 = force_gimple_operand_gsi (&gsif
, itercntbias
, true,
4859 itercnt
= fold_build2 (PLUS_EXPR
, itype
, itercntbias
, s0
);
4860 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
4862 GSI_CONTINUE_LINKING
);
4867 a
= fold_build2 (MULT_EXPR
, type
,
4868 fold_convert (type
, itercnt
),
4869 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
4870 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
4871 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), v
, a
);
4872 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4873 false, GSI_CONTINUE_LINKING
);
4874 assign_stmt
= gimple_build_assign (dest
, t
);
4875 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
4877 if (fd
->collapse
> 1)
4878 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
4882 /* The code controlling the sequential loop goes in CONT_BB,
4883 replacing the GIMPLE_OMP_CONTINUE. */
4884 gsi
= gsi_last_nondebug_bb (cont_bb
);
4885 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
4886 vmain
= gimple_omp_continue_control_use (cont_stmt
);
4887 vback
= gimple_omp_continue_control_def (cont_stmt
);
4891 tree itype
= TREE_TYPE (cond_var
);
4893 if (POINTER_TYPE_P (type
)
4894 || TREE_CODE (n1
) != INTEGER_CST
4895 || fd
->loop
.cond_code
!= LT_EXPR
)
4896 t2
= build_int_cst (itype
, 1);
4898 t2
= fold_convert (itype
, step
);
4899 t2
= fold_build2 (PLUS_EXPR
, itype
, cond_var
, t2
);
4900 t2
= force_gimple_operand_gsi (&gsi
, t2
, false,
4901 NULL_TREE
, true, GSI_SAME_STMT
);
4902 assign_stmt
= gimple_build_assign (cond_var
, t2
);
4903 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
4906 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
4908 if (POINTER_TYPE_P (type
))
4909 t
= fold_build_pointer_plus (vmain
, step
);
4911 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
4912 if (DECL_P (vback
) && TREE_ADDRESSABLE (vback
))
4913 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4914 true, GSI_SAME_STMT
);
4915 assign_stmt
= gimple_build_assign (vback
, t
);
4916 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
4918 if (tree_int_cst_equal (fd
->chunk_size
, integer_one_node
))
4919 t
= build2 (EQ_EXPR
, boolean_type_node
,
4920 build_int_cst (itype
, 0),
4921 build_int_cst (itype
, 1));
4923 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
4924 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
4926 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
4929 /* Remove GIMPLE_OMP_CONTINUE. */
4930 gsi_remove (&gsi
, true);
4932 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
4933 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
4935 /* Trip update code goes into TRIP_UPDATE_BB. */
4936 gsi
= gsi_start_bb (trip_update_bb
);
4938 t
= build_int_cst (itype
, 1);
4939 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
4940 assign_stmt
= gimple_build_assign (trip_back
, t
);
4941 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
4944 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
4945 gsi
= gsi_last_nondebug_bb (exit_bb
);
4946 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
4948 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
4949 if (fd
->have_reductemp
|| fd
->have_pointer_condtemp
)
4953 fn
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
4955 fn
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
4956 gcall
*g
= gimple_build_call (fn
, 0);
4959 gimple_call_set_lhs (g
, t
);
4960 if (fd
->have_reductemp
)
4961 gsi_insert_after (&gsi
, gimple_build_assign (reductions
,
4965 gsi_insert_after (&gsi
, g
, GSI_SAME_STMT
);
4968 gsi_insert_after (&gsi
, omp_build_barrier (t
), GSI_SAME_STMT
);
4970 else if (fd
->have_pointer_condtemp
)
4972 tree fn
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
4973 gcall
*g
= gimple_build_call (fn
, 0);
4974 gsi_insert_after (&gsi
, g
, GSI_SAME_STMT
);
4976 gsi_remove (&gsi
, true);
4978 /* Connect the new blocks. */
4979 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
4980 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
4984 se
= find_edge (cont_bb
, body_bb
);
4987 se
= BRANCH_EDGE (cont_bb
);
4988 gcc_assert (single_succ (se
->dest
) == body_bb
);
4990 if (gimple_omp_for_combined_p (fd
->for_stmt
))
4995 else if (fd
->collapse
> 1)
4998 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
5001 se
->flags
= EDGE_TRUE_VALUE
;
5002 find_edge (cont_bb
, trip_update_bb
)->flags
5003 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
5005 redirect_edge_and_branch (single_succ_edge (trip_update_bb
),
5009 if (gimple_in_ssa_p (cfun
))
5017 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
5019 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
5020 remove arguments of the phi nodes in fin_bb. We need to create
5021 appropriate phi nodes in iter_part_bb instead. */
5022 se
= find_edge (iter_part_bb
, fin_bb
);
5023 re
= single_succ_edge (trip_update_bb
);
5024 vec
<edge_var_map
> *head
= redirect_edge_var_map_vector (re
);
5025 ene
= single_succ_edge (entry_bb
);
5027 psi
= gsi_start_phis (fin_bb
);
5028 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
5029 gsi_next (&psi
), ++i
)
5035 if (operand_equal_p (gimple_phi_arg_def (phi
, 0),
5036 redirect_edge_var_map_def (vm
), 0))
5039 t
= gimple_phi_result (phi
);
5040 gcc_assert (t
== redirect_edge_var_map_result (vm
));
5042 if (!single_pred_p (fin_bb
))
5043 t
= copy_ssa_name (t
, phi
);
5045 nphi
= create_phi_node (t
, iter_part_bb
);
5047 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
5048 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
5050 /* A special case -- fd->loop.v is not yet computed in
5051 iter_part_bb, we need to use vextra instead. */
5052 if (t
== fd
->loop
.v
)
5054 add_phi_arg (nphi
, t
, ene
, locus
);
5055 locus
= redirect_edge_var_map_location (vm
);
5056 tree back_arg
= redirect_edge_var_map_def (vm
);
5057 add_phi_arg (nphi
, back_arg
, re
, locus
);
5058 edge ce
= find_edge (cont_bb
, body_bb
);
5061 ce
= BRANCH_EDGE (cont_bb
);
5062 gcc_assert (single_succ (ce
->dest
) == body_bb
);
5063 ce
= single_succ_edge (ce
->dest
);
5065 gphi
*inner_loop_phi
= find_phi_with_arg_on_edge (back_arg
, ce
);
5066 gcc_assert (inner_loop_phi
!= NULL
);
5067 add_phi_arg (inner_loop_phi
, gimple_phi_result (nphi
),
5068 find_edge (seq_start_bb
, body_bb
), locus
);
5070 if (!single_pred_p (fin_bb
))
5071 add_phi_arg (phi
, gimple_phi_result (nphi
), se
, locus
);
5073 gcc_assert (gsi_end_p (psi
) && (head
== NULL
|| i
== head
->length ()));
5074 redirect_edge_var_map_clear (re
);
5075 if (single_pred_p (fin_bb
))
5078 psi
= gsi_start_phis (fin_bb
);
5079 if (gsi_end_p (psi
))
5081 remove_phi_node (&psi
, false);
5084 /* Make phi node for trip. */
5085 phi
= create_phi_node (trip_main
, iter_part_bb
);
5086 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
5088 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
5093 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
5094 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
5095 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
5096 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
5097 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
5098 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
5099 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
5100 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
5101 recompute_dominator (CDI_DOMINATORS
, body_bb
));
5105 class loop
*loop
= body_bb
->loop_father
;
5106 class loop
*trip_loop
= alloc_loop ();
5107 trip_loop
->header
= iter_part_bb
;
5108 trip_loop
->latch
= trip_update_bb
;
5109 add_loop (trip_loop
, iter_part_bb
->loop_father
);
5111 if (loop
!= entry_bb
->loop_father
)
5113 gcc_assert (loop
->header
== body_bb
);
5114 gcc_assert (loop
->latch
== region
->cont
5115 || single_pred (loop
->latch
) == region
->cont
);
5116 trip_loop
->inner
= loop
;
5120 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5122 loop
= alloc_loop ();
5123 loop
->header
= body_bb
;
5124 if (collapse_bb
== NULL
)
5125 loop
->latch
= cont_bb
;
5126 add_loop (loop
, trip_loop
);
5131 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
5132 loop. Given parameters:
5134 for (V = N1; V cond N2; V += STEP) BODY;
5136 where COND is "<" or ">", we generate pseudocode
5144 if (V cond N2) goto L0; else goto L2;
5147 For collapsed loops, given parameters:
5149 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
5150 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
5151 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
5154 we generate pseudocode
5160 count3 = (adj + N32 - N31) / STEP3;
5165 count2 = (adj + N22 - N21) / STEP2;
5170 count1 = (adj + N12 - N11) / STEP1;
5171 count = count1 * count2 * count3;
5181 V2 += (V3 cond3 N32) ? 0 : STEP2;
5182 V3 = (V3 cond3 N32) ? V3 : N31;
5183 V1 += (V2 cond2 N22) ? 0 : STEP1;
5184 V2 = (V2 cond2 N22) ? V2 : N21;
5186 if (V < count) goto L0; else goto L2;
5192 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
5195 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
5196 gimple_stmt_iterator gsi
;
5199 bool broken_loop
= region
->cont
== NULL
;
5201 tree
*counts
= NULL
;
5203 int safelen_int
= INT_MAX
;
5204 bool dont_vectorize
= false;
5205 tree safelen
= omp_find_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5206 OMP_CLAUSE_SAFELEN
);
5207 tree simduid
= omp_find_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5208 OMP_CLAUSE__SIMDUID_
);
5209 tree ifc
= omp_find_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5211 tree simdlen
= omp_find_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5212 OMP_CLAUSE_SIMDLEN
);
5213 tree condtemp
= omp_find_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5214 OMP_CLAUSE__CONDTEMP_
);
5216 tree cond_var
= condtemp
? OMP_CLAUSE_DECL (condtemp
) : NULL_TREE
;
5221 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
5222 if (!poly_int_tree_p (safelen
, &val
))
5225 safelen_int
= MIN (constant_lower_bound (val
), INT_MAX
);
5226 if (safelen_int
== 1)
5229 if ((ifc
&& integer_zerop (OMP_CLAUSE_IF_EXPR (ifc
)))
5230 || (simdlen
&& integer_onep (OMP_CLAUSE_SIMDLEN_EXPR (simdlen
))))
5233 dont_vectorize
= true;
5235 type
= TREE_TYPE (fd
->loop
.v
);
5236 entry_bb
= region
->entry
;
5237 cont_bb
= region
->cont
;
5238 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
5239 gcc_assert (broken_loop
5240 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
5241 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
5244 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
5245 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
5246 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
5247 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
5251 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
5252 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
5253 l2_bb
= single_succ (l1_bb
);
5255 exit_bb
= region
->exit
;
5258 gsi
= gsi_last_nondebug_bb (entry_bb
);
5260 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5261 /* Not needed in SSA form right now. */
5262 gcc_assert (!gimple_in_ssa_p (cfun
));
5263 if (fd
->collapse
> 1)
5265 int first_zero_iter
= -1, dummy
= -1;
5266 basic_block zero_iter_bb
= l2_bb
, dummy_bb
= NULL
;
5268 counts
= XALLOCAVEC (tree
, fd
->collapse
);
5269 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
5270 zero_iter_bb
, first_zero_iter
,
5271 dummy_bb
, dummy
, l2_dom_bb
);
5273 if (l2_dom_bb
== NULL
)
5278 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5280 tree innerc
= omp_find_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5281 OMP_CLAUSE__LOOPTEMP_
);
5282 gcc_assert (innerc
);
5283 n1
= OMP_CLAUSE_DECL (innerc
);
5284 innerc
= omp_find_clause (OMP_CLAUSE_CHAIN (innerc
),
5285 OMP_CLAUSE__LOOPTEMP_
);
5286 gcc_assert (innerc
);
5287 n2
= OMP_CLAUSE_DECL (innerc
);
5289 tree step
= fd
->loop
.step
;
5291 bool is_simt
= omp_find_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5295 cfun
->curr_properties
&= ~PROP_gimple_lomp_dev
;
5296 is_simt
= safelen_int
> 1;
5298 tree simt_lane
= NULL_TREE
, simt_maxlane
= NULL_TREE
;
5301 simt_lane
= create_tmp_var (unsigned_type_node
);
5302 gimple
*g
= gimple_build_call_internal (IFN_GOMP_SIMT_LANE
, 0);
5303 gimple_call_set_lhs (g
, simt_lane
);
5304 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
5305 tree offset
= fold_build2 (MULT_EXPR
, TREE_TYPE (step
), step
,
5306 fold_convert (TREE_TYPE (step
), simt_lane
));
5307 n1
= fold_convert (type
, n1
);
5308 if (POINTER_TYPE_P (type
))
5309 n1
= fold_build_pointer_plus (n1
, offset
);
5311 n1
= fold_build2 (PLUS_EXPR
, type
, n1
, fold_convert (type
, offset
));
5313 /* Collapsed loops not handled for SIMT yet: limit to one lane only. */
5314 if (fd
->collapse
> 1)
5315 simt_maxlane
= build_one_cst (unsigned_type_node
);
5316 else if (safelen_int
< omp_max_simt_vf ())
5317 simt_maxlane
= build_int_cst (unsigned_type_node
, safelen_int
);
5319 = build_call_expr_internal_loc (UNKNOWN_LOCATION
, IFN_GOMP_SIMT_VF
,
5320 unsigned_type_node
, 0);
5322 vf
= fold_build2 (MIN_EXPR
, unsigned_type_node
, vf
, simt_maxlane
);
5323 vf
= fold_convert (TREE_TYPE (step
), vf
);
5324 step
= fold_build2 (MULT_EXPR
, TREE_TYPE (step
), step
, vf
);
5327 expand_omp_build_assign (&gsi
, fd
->loop
.v
, fold_convert (type
, n1
));
5328 if (fd
->collapse
> 1)
5330 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5333 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
5337 for (i
= 0; i
< fd
->collapse
; i
++)
5339 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
5340 if (POINTER_TYPE_P (itype
))
5341 itype
= signed_type_for (itype
);
5342 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
5343 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
5348 if (POINTER_TYPE_P (type
)
5349 || TREE_CODE (n1
) != INTEGER_CST
5350 || fd
->loop
.cond_code
!= LT_EXPR
5351 || tree_int_cst_sgn (n1
) != 1)
5352 expand_omp_build_assign (&gsi
, cond_var
,
5353 build_one_cst (TREE_TYPE (cond_var
)));
5355 expand_omp_build_assign (&gsi
, cond_var
,
5356 fold_convert (TREE_TYPE (cond_var
), n1
));
5359 /* Remove the GIMPLE_OMP_FOR statement. */
5360 gsi_remove (&gsi
, true);
5364 /* Code to control the increment goes in the CONT_BB. */
5365 gsi
= gsi_last_nondebug_bb (cont_bb
);
5366 stmt
= gsi_stmt (gsi
);
5367 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
5369 if (POINTER_TYPE_P (type
))
5370 t
= fold_build_pointer_plus (fd
->loop
.v
, step
);
5372 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, step
);
5373 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
5375 if (fd
->collapse
> 1)
5377 i
= fd
->collapse
- 1;
5378 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
5380 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
5381 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
5385 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
5387 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
5390 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
5392 for (i
= fd
->collapse
- 1; i
> 0; i
--)
5394 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
5395 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
5396 if (POINTER_TYPE_P (itype2
))
5397 itype2
= signed_type_for (itype2
);
5398 t
= fold_convert (itype2
, fd
->loops
[i
- 1].step
);
5399 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true,
5401 t
= build3 (COND_EXPR
, itype2
,
5402 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
5404 fold_convert (itype
, fd
->loops
[i
].n2
)),
5405 build_int_cst (itype2
, 0), t
);
5406 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
5407 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
5409 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
5410 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
5412 t
= fold_convert (itype
, fd
->loops
[i
].n1
);
5413 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true,
5415 t
= build3 (COND_EXPR
, itype
,
5416 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
5418 fold_convert (itype
, fd
->loops
[i
].n2
)),
5420 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
5425 if (POINTER_TYPE_P (type
)
5426 || TREE_CODE (n1
) != INTEGER_CST
5427 || fd
->loop
.cond_code
!= LT_EXPR
5428 || tree_int_cst_sgn (n1
) != 1)
5429 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (cond_var
), cond_var
,
5430 build_one_cst (TREE_TYPE (cond_var
)));
5432 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (cond_var
), cond_var
,
5433 fold_convert (TREE_TYPE (cond_var
), step
));
5434 expand_omp_build_assign (&gsi
, cond_var
, t
);
5437 /* Remove GIMPLE_OMP_CONTINUE. */
5438 gsi_remove (&gsi
, true);
5441 /* Emit the condition in L1_BB. */
5442 gsi
= gsi_start_bb (l1_bb
);
5444 t
= fold_convert (type
, n2
);
5445 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5446 false, GSI_CONTINUE_LINKING
);
5447 tree v
= fd
->loop
.v
;
5448 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
5449 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
5450 false, GSI_CONTINUE_LINKING
);
5451 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, v
, t
);
5452 cond_stmt
= gimple_build_cond_empty (t
);
5453 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
5454 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
5456 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
5459 gsi
= gsi_for_stmt (cond_stmt
);
5460 gimple_regimplify_operands (cond_stmt
, &gsi
);
5463 /* Add 'V -= STEP * (SIMT_VF - 1)' after the loop. */
5466 gsi
= gsi_start_bb (l2_bb
);
5467 step
= fold_build2 (MINUS_EXPR
, TREE_TYPE (step
), fd
->loop
.step
, step
);
5468 if (POINTER_TYPE_P (type
))
5469 t
= fold_build_pointer_plus (fd
->loop
.v
, step
);
5471 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, step
);
5472 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
5475 /* Remove GIMPLE_OMP_RETURN. */
5476 gsi
= gsi_last_nondebug_bb (exit_bb
);
5477 gsi_remove (&gsi
, true);
5479 /* Connect the new blocks. */
5480 remove_edge (FALLTHRU_EDGE (entry_bb
));
5484 remove_edge (BRANCH_EDGE (entry_bb
));
5485 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
5487 e
= BRANCH_EDGE (l1_bb
);
5488 ne
= FALLTHRU_EDGE (l1_bb
);
5489 e
->flags
= EDGE_TRUE_VALUE
;
5493 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
5495 ne
= single_succ_edge (l1_bb
);
5496 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
5499 ne
->flags
= EDGE_FALSE_VALUE
;
5500 e
->probability
= profile_probability::guessed_always ().apply_scale (7, 8);
5501 ne
->probability
= e
->probability
.invert ();
5503 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
5504 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
5508 cond_stmt
= gimple_build_cond (LT_EXPR
, simt_lane
, simt_maxlane
,
5509 NULL_TREE
, NULL_TREE
);
5510 gsi
= gsi_last_bb (entry_bb
);
5511 gsi_insert_after (&gsi
, cond_stmt
, GSI_NEW_STMT
);
5512 make_edge (entry_bb
, l2_bb
, EDGE_FALSE_VALUE
);
5513 FALLTHRU_EDGE (entry_bb
)->flags
= EDGE_TRUE_VALUE
;
5514 FALLTHRU_EDGE (entry_bb
)->probability
5515 = profile_probability::guessed_always ().apply_scale (7, 8);
5516 BRANCH_EDGE (entry_bb
)->probability
5517 = FALLTHRU_EDGE (entry_bb
)->probability
.invert ();
5518 l2_dom_bb
= entry_bb
;
5520 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
5524 class loop
*loop
= alloc_loop ();
5525 loop
->header
= l1_bb
;
5526 loop
->latch
= cont_bb
;
5527 add_loop (loop
, l1_bb
->loop_father
);
5528 loop
->safelen
= safelen_int
;
5531 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
5532 cfun
->has_simduid_loops
= true;
5534 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
5536 if ((flag_tree_loop_vectorize
5537 || !global_options_set
.x_flag_tree_loop_vectorize
)
5538 && flag_tree_loop_optimize
5539 && loop
->safelen
> 1)
5541 loop
->force_vectorize
= true;
5542 if (simdlen
&& tree_fits_uhwi_p (OMP_CLAUSE_SIMDLEN_EXPR (simdlen
)))
5544 unsigned HOST_WIDE_INT v
5545 = tree_to_uhwi (OMP_CLAUSE_SIMDLEN_EXPR (simdlen
));
5546 if (v
< INT_MAX
&& v
<= (unsigned HOST_WIDE_INT
) loop
->safelen
)
5549 cfun
->has_force_vectorize_loops
= true;
5551 else if (dont_vectorize
)
5552 loop
->dont_vectorize
= true;
5555 cfun
->has_simduid_loops
= true;
5558 /* Taskloop construct is represented after gimplification with
5559 two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
5560 in between them. This routine expands the outer GIMPLE_OMP_FOR,
5561 which should just compute all the needed loop temporaries
5562 for GIMPLE_OMP_TASK. */
5565 expand_omp_taskloop_for_outer (struct omp_region
*region
,
5566 struct omp_for_data
*fd
,
5569 tree type
, bias
= NULL_TREE
;
5570 basic_block entry_bb
, cont_bb
, exit_bb
;
5571 gimple_stmt_iterator gsi
;
5572 gassign
*assign_stmt
;
5573 tree
*counts
= NULL
;
5576 gcc_assert (inner_stmt
);
5577 gcc_assert (region
->cont
);
5578 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_TASK
5579 && gimple_omp_task_taskloop_p (inner_stmt
));
5580 type
= TREE_TYPE (fd
->loop
.v
);
5582 /* See if we need to bias by LLONG_MIN. */
5583 if (fd
->iter_type
== long_long_unsigned_type_node
5584 && TREE_CODE (type
) == INTEGER_TYPE
5585 && !TYPE_UNSIGNED (type
))
5589 if (fd
->loop
.cond_code
== LT_EXPR
)
5592 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5596 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5599 if (TREE_CODE (n1
) != INTEGER_CST
5600 || TREE_CODE (n2
) != INTEGER_CST
5601 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
5602 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
5605 entry_bb
= region
->entry
;
5606 cont_bb
= region
->cont
;
5607 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
5608 gcc_assert (BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
5609 exit_bb
= region
->exit
;
5611 gsi
= gsi_last_nondebug_bb (entry_bb
);
5612 gimple
*for_stmt
= gsi_stmt (gsi
);
5613 gcc_assert (gimple_code (for_stmt
) == GIMPLE_OMP_FOR
);
5614 if (fd
->collapse
> 1)
5616 int first_zero_iter
= -1, dummy
= -1;
5617 basic_block zero_iter_bb
= NULL
, dummy_bb
= NULL
, l2_dom_bb
= NULL
;
5619 counts
= XALLOCAVEC (tree
, fd
->collapse
);
5620 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
5621 zero_iter_bb
, first_zero_iter
,
5622 dummy_bb
, dummy
, l2_dom_bb
);
5626 /* Some counts[i] vars might be uninitialized if
5627 some loop has zero iterations. But the body shouldn't
5628 be executed in that case, so just avoid uninit warnings. */
5629 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
5630 if (SSA_VAR_P (counts
[i
]))
5631 TREE_NO_WARNING (counts
[i
]) = 1;
5633 edge e
= split_block (entry_bb
, gsi_stmt (gsi
));
5635 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
5636 gsi
= gsi_last_bb (entry_bb
);
5637 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
5638 get_immediate_dominator (CDI_DOMINATORS
,
5646 if (POINTER_TYPE_P (TREE_TYPE (t0
))
5647 && TYPE_PRECISION (TREE_TYPE (t0
))
5648 != TYPE_PRECISION (fd
->iter_type
))
5650 /* Avoid casting pointers to integer of a different size. */
5651 tree itype
= signed_type_for (type
);
5652 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
5653 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
5657 t1
= fold_convert (fd
->iter_type
, t1
);
5658 t0
= fold_convert (fd
->iter_type
, t0
);
5662 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
5663 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
5666 tree innerc
= omp_find_clause (gimple_omp_task_clauses (inner_stmt
),
5667 OMP_CLAUSE__LOOPTEMP_
);
5668 gcc_assert (innerc
);
5669 tree startvar
= OMP_CLAUSE_DECL (innerc
);
5670 innerc
= omp_find_clause (OMP_CLAUSE_CHAIN (innerc
), OMP_CLAUSE__LOOPTEMP_
);
5671 gcc_assert (innerc
);
5672 tree endvar
= OMP_CLAUSE_DECL (innerc
);
5673 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
5675 gcc_assert (innerc
);
5676 for (i
= 1; i
< fd
->collapse
; i
++)
5678 innerc
= omp_find_clause (OMP_CLAUSE_CHAIN (innerc
),
5679 OMP_CLAUSE__LOOPTEMP_
);
5680 gcc_assert (innerc
);
5682 innerc
= omp_find_clause (OMP_CLAUSE_CHAIN (innerc
),
5683 OMP_CLAUSE__LOOPTEMP_
);
5686 /* If needed (inner taskloop has lastprivate clause), propagate
5687 down the total number of iterations. */
5688 tree t
= force_gimple_operand_gsi (&gsi
, fd
->loop
.n2
, false,
5690 GSI_CONTINUE_LINKING
);
5691 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
5692 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
5696 t0
= force_gimple_operand_gsi (&gsi
, t0
, false, NULL_TREE
, false,
5697 GSI_CONTINUE_LINKING
);
5698 assign_stmt
= gimple_build_assign (startvar
, t0
);
5699 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
5701 t1
= force_gimple_operand_gsi (&gsi
, t1
, false, NULL_TREE
, false,
5702 GSI_CONTINUE_LINKING
);
5703 assign_stmt
= gimple_build_assign (endvar
, t1
);
5704 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
5705 if (fd
->collapse
> 1)
5706 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
5708 /* Remove the GIMPLE_OMP_FOR statement. */
5709 gsi
= gsi_for_stmt (for_stmt
);
5710 gsi_remove (&gsi
, true);
5712 gsi
= gsi_last_nondebug_bb (cont_bb
);
5713 gsi_remove (&gsi
, true);
5715 gsi
= gsi_last_nondebug_bb (exit_bb
);
5716 gsi_remove (&gsi
, true);
5718 FALLTHRU_EDGE (entry_bb
)->probability
= profile_probability::always ();
5719 remove_edge (BRANCH_EDGE (entry_bb
));
5720 FALLTHRU_EDGE (cont_bb
)->probability
= profile_probability::always ();
5721 remove_edge (BRANCH_EDGE (cont_bb
));
5722 set_immediate_dominator (CDI_DOMINATORS
, exit_bb
, cont_bb
);
5723 set_immediate_dominator (CDI_DOMINATORS
, region
->entry
,
5724 recompute_dominator (CDI_DOMINATORS
, region
->entry
));
5727 /* Taskloop construct is represented after gimplification with
5728 two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
5729 in between them. This routine expands the inner GIMPLE_OMP_FOR.
5730 GOMP_taskloop{,_ull} function arranges for each task to be given just
5731 a single range of iterations. */
5734 expand_omp_taskloop_for_inner (struct omp_region
*region
,
5735 struct omp_for_data
*fd
,
5738 tree e
, t
, type
, itype
, vmain
, vback
, bias
= NULL_TREE
;
5739 basic_block entry_bb
, exit_bb
, body_bb
, cont_bb
, collapse_bb
= NULL
;
5741 gimple_stmt_iterator gsi
;
5743 bool broken_loop
= region
->cont
== NULL
;
5744 tree
*counts
= NULL
;
5747 itype
= type
= TREE_TYPE (fd
->loop
.v
);
5748 if (POINTER_TYPE_P (type
))
5749 itype
= signed_type_for (type
);
5751 /* See if we need to bias by LLONG_MIN. */
5752 if (fd
->iter_type
== long_long_unsigned_type_node
5753 && TREE_CODE (type
) == INTEGER_TYPE
5754 && !TYPE_UNSIGNED (type
))
5758 if (fd
->loop
.cond_code
== LT_EXPR
)
5761 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5765 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5768 if (TREE_CODE (n1
) != INTEGER_CST
5769 || TREE_CODE (n2
) != INTEGER_CST
5770 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
5771 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
5774 entry_bb
= region
->entry
;
5775 cont_bb
= region
->cont
;
5776 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
5777 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
5778 gcc_assert (broken_loop
5779 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
5780 body_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
5783 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
5784 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
5786 exit_bb
= region
->exit
;
5788 /* Iteration space partitioning goes in ENTRY_BB. */
5789 gsi
= gsi_last_nondebug_bb (entry_bb
);
5790 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5792 if (fd
->collapse
> 1)
5794 int first_zero_iter
= -1, dummy
= -1;
5795 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
5797 counts
= XALLOCAVEC (tree
, fd
->collapse
);
5798 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
5799 fin_bb
, first_zero_iter
,
5800 dummy_bb
, dummy
, l2_dom_bb
);
5804 t
= integer_one_node
;
5806 step
= fd
->loop
.step
;
5807 tree innerc
= omp_find_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5808 OMP_CLAUSE__LOOPTEMP_
);
5809 gcc_assert (innerc
);
5810 n1
= OMP_CLAUSE_DECL (innerc
);
5811 innerc
= omp_find_clause (OMP_CLAUSE_CHAIN (innerc
), OMP_CLAUSE__LOOPTEMP_
);
5812 gcc_assert (innerc
);
5813 n2
= OMP_CLAUSE_DECL (innerc
);
5816 n1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, n1
, bias
);
5817 n2
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, n2
, bias
);
5819 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
5820 true, NULL_TREE
, true, GSI_SAME_STMT
);
5821 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
5822 true, NULL_TREE
, true, GSI_SAME_STMT
);
5823 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
5824 true, NULL_TREE
, true, GSI_SAME_STMT
);
5826 tree startvar
= fd
->loop
.v
;
5827 tree endvar
= NULL_TREE
;
5829 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5831 tree clauses
= gimple_omp_for_clauses (inner_stmt
);
5832 tree innerc
= omp_find_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
5833 gcc_assert (innerc
);
5834 startvar
= OMP_CLAUSE_DECL (innerc
);
5835 innerc
= omp_find_clause (OMP_CLAUSE_CHAIN (innerc
),
5836 OMP_CLAUSE__LOOPTEMP_
);
5837 gcc_assert (innerc
);
5838 endvar
= OMP_CLAUSE_DECL (innerc
);
5840 t
= fold_convert (TREE_TYPE (startvar
), n1
);
5841 t
= force_gimple_operand_gsi (&gsi
, t
,
5843 && TREE_ADDRESSABLE (startvar
),
5844 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5845 gimple
*assign_stmt
= gimple_build_assign (startvar
, t
);
5846 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
5848 t
= fold_convert (TREE_TYPE (startvar
), n2
);
5849 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5850 false, GSI_CONTINUE_LINKING
);
5853 assign_stmt
= gimple_build_assign (endvar
, e
);
5854 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
5855 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
5856 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
5858 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
5859 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
5861 if (fd
->collapse
> 1)
5862 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
5866 /* The code controlling the sequential loop replaces the
5867 GIMPLE_OMP_CONTINUE. */
5868 gsi
= gsi_last_nondebug_bb (cont_bb
);
5869 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
5870 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
5871 vmain
= gimple_omp_continue_control_use (cont_stmt
);
5872 vback
= gimple_omp_continue_control_def (cont_stmt
);
5874 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5876 if (POINTER_TYPE_P (type
))
5877 t
= fold_build_pointer_plus (vmain
, step
);
5879 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
5880 t
= force_gimple_operand_gsi (&gsi
, t
,
5882 && TREE_ADDRESSABLE (vback
),
5883 NULL_TREE
, true, GSI_SAME_STMT
);
5884 assign_stmt
= gimple_build_assign (vback
, t
);
5885 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
5887 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
5888 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
5890 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
5893 /* Remove the GIMPLE_OMP_CONTINUE statement. */
5894 gsi_remove (&gsi
, true);
5896 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
5897 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
5900 /* Remove the GIMPLE_OMP_FOR statement. */
5901 gsi
= gsi_for_stmt (fd
->for_stmt
);
5902 gsi_remove (&gsi
, true);
5904 /* Remove the GIMPLE_OMP_RETURN statement. */
5905 gsi
= gsi_last_nondebug_bb (exit_bb
);
5906 gsi_remove (&gsi
, true);
5908 FALLTHRU_EDGE (entry_bb
)->probability
= profile_probability::always ();
5910 remove_edge (BRANCH_EDGE (entry_bb
));
5913 remove_edge_and_dominated_blocks (BRANCH_EDGE (entry_bb
));
5914 region
->outer
->cont
= NULL
;
5917 /* Connect all the blocks. */
5920 ep
= find_edge (cont_bb
, body_bb
);
5921 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5926 else if (fd
->collapse
> 1)
5929 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
5932 ep
->flags
= EDGE_TRUE_VALUE
;
5933 find_edge (cont_bb
, fin_bb
)->flags
5934 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
5937 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
5938 recompute_dominator (CDI_DOMINATORS
, body_bb
));
5940 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
5941 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
5943 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
5945 class loop
*loop
= alloc_loop ();
5946 loop
->header
= body_bb
;
5947 if (collapse_bb
== NULL
)
5948 loop
->latch
= cont_bb
;
5949 add_loop (loop
, body_bb
->loop_father
);
5953 /* A subroutine of expand_omp_for. Generate code for an OpenACC
5954 partitioned loop. The lowering here is abstracted, in that the
5955 loop parameters are passed through internal functions, which are
5956 further lowered by oacc_device_lower, once we get to the target
5957 compiler. The loop is of the form:
5959 for (V = B; V LTGT E; V += S) {BODY}
5961 where LTGT is < or >. We may have a specified chunking size, CHUNKING
5962 (constant 0 for no chunking) and we will have a GWV partitioning
5963 mask, specifying dimensions over which the loop is to be
5964 partitioned (see note below). We generate code that looks like
5965 (this ignores tiling):
5967 <entry_bb> [incoming FALL->body, BRANCH->exit]
5968 typedef signedintify (typeof (V)) T; // underlying signed integral type
5971 T DIR = LTGT == '<' ? +1 : -1;
5972 T chunk_max = GOACC_LOOP_CHUNK (dir, range, S, CHUNK_SIZE, GWV);
5973 T step = GOACC_LOOP_STEP (dir, range, S, CHUNK_SIZE, GWV);
5975 <head_bb> [created by splitting end of entry_bb]
5976 T offset = GOACC_LOOP_OFFSET (dir, range, S, CHUNK_SIZE, GWV, chunk_no);
5977 T bound = GOACC_LOOP_BOUND (dir, range, S, CHUNK_SIZE, GWV, offset);
5978 if (!(offset LTGT bound)) goto bottom_bb;
5980 <body_bb> [incoming]
5984 <cont_bb> [incoming, may == body_bb FALL->exit_bb, BRANCH->body_bb]
5986 if (offset LTGT bound) goto body_bb; [*]
5988 <bottom_bb> [created by splitting start of exit_bb] insert BRANCH->head_bb
5990 if (chunk < chunk_max) goto head_bb;
5992 <exit_bb> [incoming]
5993 V = B + ((range -/+ 1) / S +/- 1) * S [*]
5995 [*] Needed if V live at end of loop. */
5998 expand_oacc_for (struct omp_region
*region
, struct omp_for_data
*fd
)
6000 tree v
= fd
->loop
.v
;
6001 enum tree_code cond_code
= fd
->loop
.cond_code
;
6002 enum tree_code plus_code
= PLUS_EXPR
;
6004 tree chunk_size
= integer_minus_one_node
;
6005 tree gwv
= integer_zero_node
;
6006 tree iter_type
= TREE_TYPE (v
);
6007 tree diff_type
= iter_type
;
6008 tree plus_type
= iter_type
;
6009 struct oacc_collapse
*counts
= NULL
;
6011 gcc_checking_assert (gimple_omp_for_kind (fd
->for_stmt
)
6012 == GF_OMP_FOR_KIND_OACC_LOOP
);
6013 gcc_assert (!gimple_omp_for_combined_into_p (fd
->for_stmt
));
6014 gcc_assert (cond_code
== LT_EXPR
|| cond_code
== GT_EXPR
);
6016 if (POINTER_TYPE_P (iter_type
))
6018 plus_code
= POINTER_PLUS_EXPR
;
6019 plus_type
= sizetype
;
6021 if (POINTER_TYPE_P (diff_type
) || TYPE_UNSIGNED (diff_type
))
6022 diff_type
= signed_type_for (diff_type
);
6023 if (TYPE_PRECISION (diff_type
) < TYPE_PRECISION (integer_type_node
))
6024 diff_type
= integer_type_node
;
6026 basic_block entry_bb
= region
->entry
; /* BB ending in OMP_FOR */
6027 basic_block exit_bb
= region
->exit
; /* BB ending in OMP_RETURN */
6028 basic_block cont_bb
= region
->cont
; /* BB ending in OMP_CONTINUE */
6029 basic_block bottom_bb
= NULL
;
6031 /* entry_bb has two sucessors; the branch edge is to the exit
6032 block, fallthrough edge to body. */
6033 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2
6034 && BRANCH_EDGE (entry_bb
)->dest
== exit_bb
);
6036 /* If cont_bb non-NULL, it has 2 successors. The branch successor is
6037 body_bb, or to a block whose only successor is the body_bb. Its
6038 fallthrough successor is the final block (same as the branch
6039 successor of the entry_bb). */
6042 basic_block body_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
6043 basic_block bed
= BRANCH_EDGE (cont_bb
)->dest
;
6045 gcc_assert (FALLTHRU_EDGE (cont_bb
)->dest
== exit_bb
);
6046 gcc_assert (bed
== body_bb
|| single_succ_edge (bed
)->dest
== body_bb
);
6049 gcc_assert (!gimple_in_ssa_p (cfun
));
6051 /* The exit block only has entry_bb and cont_bb as predecessors. */
6052 gcc_assert (EDGE_COUNT (exit_bb
->preds
) == 1 + (cont_bb
!= NULL
));
6055 tree chunk_max
= NULL_TREE
;
6057 tree step
= create_tmp_var (diff_type
, ".step");
6058 bool up
= cond_code
== LT_EXPR
;
6059 tree dir
= build_int_cst (diff_type
, up
? +1 : -1);
6060 bool chunking
= !gimple_in_ssa_p (cfun
);
6064 tree tile_size
= NULL_TREE
;
6065 tree element_s
= NULL_TREE
;
6066 tree e_bound
= NULL_TREE
, e_offset
= NULL_TREE
, e_step
= NULL_TREE
;
6067 basic_block elem_body_bb
= NULL
;
6068 basic_block elem_cont_bb
= NULL
;
6070 /* SSA instances. */
6071 tree offset_incr
= NULL_TREE
;
6072 tree offset_init
= NULL_TREE
;
6074 gimple_stmt_iterator gsi
;
6080 edge split
, be
, fte
;
6082 /* Split the end of entry_bb to create head_bb. */
6083 split
= split_block (entry_bb
, last_stmt (entry_bb
));
6084 basic_block head_bb
= split
->dest
;
6085 entry_bb
= split
->src
;
6087 /* Chunk setup goes at end of entry_bb, replacing the omp_for. */
6088 gsi
= gsi_last_nondebug_bb (entry_bb
);
6089 gomp_for
*for_stmt
= as_a
<gomp_for
*> (gsi_stmt (gsi
));
6090 loc
= gimple_location (for_stmt
);
6092 if (gimple_in_ssa_p (cfun
))
6094 offset_init
= gimple_omp_for_index (for_stmt
, 0);
6095 gcc_assert (integer_zerop (fd
->loop
.n1
));
6096 /* The SSA parallelizer does gang parallelism. */
6097 gwv
= build_int_cst (integer_type_node
, GOMP_DIM_MASK (GOMP_DIM_GANG
));
6100 if (fd
->collapse
> 1 || fd
->tiling
)
6102 gcc_assert (!gimple_in_ssa_p (cfun
) && up
);
6103 counts
= XALLOCAVEC (struct oacc_collapse
, fd
->collapse
);
6104 tree total
= expand_oacc_collapse_init (fd
, &gsi
, counts
,
6105 TREE_TYPE (fd
->loop
.n2
), loc
);
6107 if (SSA_VAR_P (fd
->loop
.n2
))
6109 total
= force_gimple_operand_gsi (&gsi
, total
, false, NULL_TREE
,
6110 true, GSI_SAME_STMT
);
6111 ass
= gimple_build_assign (fd
->loop
.n2
, total
);
6112 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
6116 tree b
= fd
->loop
.n1
;
6117 tree e
= fd
->loop
.n2
;
6118 tree s
= fd
->loop
.step
;
6120 b
= force_gimple_operand_gsi (&gsi
, b
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6121 e
= force_gimple_operand_gsi (&gsi
, e
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6123 /* Convert the step, avoiding possible unsigned->signed overflow. */
6124 negating
= !up
&& TYPE_UNSIGNED (TREE_TYPE (s
));
6126 s
= fold_build1 (NEGATE_EXPR
, TREE_TYPE (s
), s
);
6127 s
= fold_convert (diff_type
, s
);
6129 s
= fold_build1 (NEGATE_EXPR
, diff_type
, s
);
6130 s
= force_gimple_operand_gsi (&gsi
, s
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6133 chunk_size
= integer_zero_node
;
6134 expr
= fold_convert (diff_type
, chunk_size
);
6135 chunk_size
= force_gimple_operand_gsi (&gsi
, expr
, true,
6136 NULL_TREE
, true, GSI_SAME_STMT
);
6140 /* Determine the tile size and element step,
6141 modify the outer loop step size. */
6142 tile_size
= create_tmp_var (diff_type
, ".tile_size");
6143 expr
= build_int_cst (diff_type
, 1);
6144 for (int ix
= 0; ix
< fd
->collapse
; ix
++)
6145 expr
= fold_build2 (MULT_EXPR
, diff_type
, counts
[ix
].tile
, expr
);
6146 expr
= force_gimple_operand_gsi (&gsi
, expr
, true,
6147 NULL_TREE
, true, GSI_SAME_STMT
);
6148 ass
= gimple_build_assign (tile_size
, expr
);
6149 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
6151 element_s
= create_tmp_var (diff_type
, ".element_s");
6152 ass
= gimple_build_assign (element_s
, s
);
6153 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
6155 expr
= fold_build2 (MULT_EXPR
, diff_type
, s
, tile_size
);
6156 s
= force_gimple_operand_gsi (&gsi
, expr
, true,
6157 NULL_TREE
, true, GSI_SAME_STMT
);
6160 /* Determine the range, avoiding possible unsigned->signed overflow. */
6161 negating
= !up
&& TYPE_UNSIGNED (iter_type
);
6162 expr
= fold_build2 (MINUS_EXPR
, plus_type
,
6163 fold_convert (plus_type
, negating
? b
: e
),
6164 fold_convert (plus_type
, negating
? e
: b
));
6165 expr
= fold_convert (diff_type
, expr
);
6167 expr
= fold_build1 (NEGATE_EXPR
, diff_type
, expr
);
6168 tree range
= force_gimple_operand_gsi (&gsi
, expr
, true,
6169 NULL_TREE
, true, GSI_SAME_STMT
);
6171 chunk_no
= build_int_cst (diff_type
, 0);
6174 gcc_assert (!gimple_in_ssa_p (cfun
));
6177 chunk_max
= create_tmp_var (diff_type
, ".chunk_max");
6178 chunk_no
= create_tmp_var (diff_type
, ".chunk_no");
6180 ass
= gimple_build_assign (chunk_no
, expr
);
6181 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
6183 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 6,
6184 build_int_cst (integer_type_node
,
6185 IFN_GOACC_LOOP_CHUNKS
),
6186 dir
, range
, s
, chunk_size
, gwv
);
6187 gimple_call_set_lhs (call
, chunk_max
);
6188 gimple_set_location (call
, loc
);
6189 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
6192 chunk_size
= chunk_no
;
6194 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 6,
6195 build_int_cst (integer_type_node
,
6196 IFN_GOACC_LOOP_STEP
),
6197 dir
, range
, s
, chunk_size
, gwv
);
6198 gimple_call_set_lhs (call
, step
);
6199 gimple_set_location (call
, loc
);
6200 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
6202 /* Remove the GIMPLE_OMP_FOR. */
6203 gsi_remove (&gsi
, true);
6205 /* Fixup edges from head_bb. */
6206 be
= BRANCH_EDGE (head_bb
);
6207 fte
= FALLTHRU_EDGE (head_bb
);
6208 be
->flags
|= EDGE_FALSE_VALUE
;
6209 fte
->flags
^= EDGE_FALLTHRU
| EDGE_TRUE_VALUE
;
6211 basic_block body_bb
= fte
->dest
;
6213 if (gimple_in_ssa_p (cfun
))
6215 gsi
= gsi_last_nondebug_bb (cont_bb
);
6216 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
6218 offset
= gimple_omp_continue_control_use (cont_stmt
);
6219 offset_incr
= gimple_omp_continue_control_def (cont_stmt
);
6223 offset
= create_tmp_var (diff_type
, ".offset");
6224 offset_init
= offset_incr
= offset
;
6226 bound
= create_tmp_var (TREE_TYPE (offset
), ".bound");
6228 /* Loop offset & bound go into head_bb. */
6229 gsi
= gsi_start_bb (head_bb
);
6231 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 7,
6232 build_int_cst (integer_type_node
,
6233 IFN_GOACC_LOOP_OFFSET
),
6235 chunk_size
, gwv
, chunk_no
);
6236 gimple_call_set_lhs (call
, offset_init
);
6237 gimple_set_location (call
, loc
);
6238 gsi_insert_after (&gsi
, call
, GSI_CONTINUE_LINKING
);
6240 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 7,
6241 build_int_cst (integer_type_node
,
6242 IFN_GOACC_LOOP_BOUND
),
6244 chunk_size
, gwv
, offset_init
);
6245 gimple_call_set_lhs (call
, bound
);
6246 gimple_set_location (call
, loc
);
6247 gsi_insert_after (&gsi
, call
, GSI_CONTINUE_LINKING
);
6249 expr
= build2 (cond_code
, boolean_type_node
, offset_init
, bound
);
6250 gsi_insert_after (&gsi
, gimple_build_cond_empty (expr
),
6251 GSI_CONTINUE_LINKING
);
6253 /* V assignment goes into body_bb. */
6254 if (!gimple_in_ssa_p (cfun
))
6256 gsi
= gsi_start_bb (body_bb
);
6258 expr
= build2 (plus_code
, iter_type
, b
,
6259 fold_convert (plus_type
, offset
));
6260 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
6261 true, GSI_SAME_STMT
);
6262 ass
= gimple_build_assign (v
, expr
);
6263 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
6265 if (fd
->collapse
> 1 || fd
->tiling
)
6266 expand_oacc_collapse_vars (fd
, false, &gsi
, counts
, v
);
6270 /* Determine the range of the element loop -- usually simply
6271 the tile_size, but could be smaller if the final
6272 iteration of the outer loop is a partial tile. */
6273 tree e_range
= create_tmp_var (diff_type
, ".e_range");
6275 expr
= build2 (MIN_EXPR
, diff_type
,
6276 build2 (MINUS_EXPR
, diff_type
, bound
, offset
),
6277 build2 (MULT_EXPR
, diff_type
, tile_size
,
6279 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
6280 true, GSI_SAME_STMT
);
6281 ass
= gimple_build_assign (e_range
, expr
);
6282 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
6284 /* Determine bound, offset & step of inner loop. */
6285 e_bound
= create_tmp_var (diff_type
, ".e_bound");
6286 e_offset
= create_tmp_var (diff_type
, ".e_offset");
6287 e_step
= create_tmp_var (diff_type
, ".e_step");
6289 /* Mark these as element loops. */
6290 tree t
, e_gwv
= integer_minus_one_node
;
6291 tree chunk
= build_int_cst (diff_type
, 0); /* Never chunked. */
6293 t
= build_int_cst (integer_type_node
, IFN_GOACC_LOOP_OFFSET
);
6294 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 7, t
, dir
, e_range
,
6295 element_s
, chunk
, e_gwv
, chunk
);
6296 gimple_call_set_lhs (call
, e_offset
);
6297 gimple_set_location (call
, loc
);
6298 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
6300 t
= build_int_cst (integer_type_node
, IFN_GOACC_LOOP_BOUND
);
6301 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 7, t
, dir
, e_range
,
6302 element_s
, chunk
, e_gwv
, e_offset
);
6303 gimple_call_set_lhs (call
, e_bound
);
6304 gimple_set_location (call
, loc
);
6305 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
6307 t
= build_int_cst (integer_type_node
, IFN_GOACC_LOOP_STEP
);
6308 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 6, t
, dir
, e_range
,
6309 element_s
, chunk
, e_gwv
);
6310 gimple_call_set_lhs (call
, e_step
);
6311 gimple_set_location (call
, loc
);
6312 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
6314 /* Add test and split block. */
6315 expr
= build2 (cond_code
, boolean_type_node
, e_offset
, e_bound
);
6316 stmt
= gimple_build_cond_empty (expr
);
6317 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6318 split
= split_block (body_bb
, stmt
);
6319 elem_body_bb
= split
->dest
;
6320 if (cont_bb
== body_bb
)
6321 cont_bb
= elem_body_bb
;
6322 body_bb
= split
->src
;
6324 split
->flags
^= EDGE_FALLTHRU
| EDGE_TRUE_VALUE
;
6326 /* Add a dummy exit for the tiled block when cont_bb is missing. */
6327 if (cont_bb
== NULL
)
6329 edge e
= make_edge (body_bb
, exit_bb
, EDGE_FALSE_VALUE
);
6330 e
->probability
= profile_probability::even ();
6331 split
->probability
= profile_probability::even ();
6334 /* Initialize the user's loop vars. */
6335 gsi
= gsi_start_bb (elem_body_bb
);
6336 expand_oacc_collapse_vars (fd
, true, &gsi
, counts
, e_offset
);
6340 /* Loop increment goes into cont_bb. If this is not a loop, we
6341 will have spawned threads as if it was, and each one will
6342 execute one iteration. The specification is not explicit about
6343 whether such constructs are ill-formed or not, and they can
6344 occur, especially when noreturn routines are involved. */
6347 gsi
= gsi_last_nondebug_bb (cont_bb
);
6348 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
6349 loc
= gimple_location (cont_stmt
);
6353 /* Insert element loop increment and test. */
6354 expr
= build2 (PLUS_EXPR
, diff_type
, e_offset
, e_step
);
6355 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
6356 true, GSI_SAME_STMT
);
6357 ass
= gimple_build_assign (e_offset
, expr
);
6358 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
6359 expr
= build2 (cond_code
, boolean_type_node
, e_offset
, e_bound
);
6361 stmt
= gimple_build_cond_empty (expr
);
6362 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6363 split
= split_block (cont_bb
, stmt
);
6364 elem_cont_bb
= split
->src
;
6365 cont_bb
= split
->dest
;
6367 split
->flags
^= EDGE_FALLTHRU
| EDGE_FALSE_VALUE
;
6368 split
->probability
= profile_probability::unlikely ().guessed ();
6370 = make_edge (elem_cont_bb
, elem_body_bb
, EDGE_TRUE_VALUE
);
6371 latch_edge
->probability
= profile_probability::likely ().guessed ();
6373 edge skip_edge
= make_edge (body_bb
, cont_bb
, EDGE_FALSE_VALUE
);
6374 skip_edge
->probability
= profile_probability::unlikely ().guessed ();
6375 edge loop_entry_edge
= EDGE_SUCC (body_bb
, 1 - skip_edge
->dest_idx
);
6376 loop_entry_edge
->probability
6377 = profile_probability::likely ().guessed ();
6379 gsi
= gsi_for_stmt (cont_stmt
);
6382 /* Increment offset. */
6383 if (gimple_in_ssa_p (cfun
))
6384 expr
= build2 (plus_code
, iter_type
, offset
,
6385 fold_convert (plus_type
, step
));
6387 expr
= build2 (PLUS_EXPR
, diff_type
, offset
, step
);
6388 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
6389 true, GSI_SAME_STMT
);
6390 ass
= gimple_build_assign (offset_incr
, expr
);
6391 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
6392 expr
= build2 (cond_code
, boolean_type_node
, offset_incr
, bound
);
6393 gsi_insert_before (&gsi
, gimple_build_cond_empty (expr
), GSI_SAME_STMT
);
6395 /* Remove the GIMPLE_OMP_CONTINUE. */
6396 gsi_remove (&gsi
, true);
6398 /* Fixup edges from cont_bb. */
6399 be
= BRANCH_EDGE (cont_bb
);
6400 fte
= FALLTHRU_EDGE (cont_bb
);
6401 be
->flags
|= EDGE_TRUE_VALUE
;
6402 fte
->flags
^= EDGE_FALLTHRU
| EDGE_FALSE_VALUE
;
6406 /* Split the beginning of exit_bb to make bottom_bb. We
6407 need to insert a nop at the start, because splitting is
6408 after a stmt, not before. */
6409 gsi
= gsi_start_bb (exit_bb
);
6410 stmt
= gimple_build_nop ();
6411 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6412 split
= split_block (exit_bb
, stmt
);
6413 bottom_bb
= split
->src
;
6414 exit_bb
= split
->dest
;
6415 gsi
= gsi_last_bb (bottom_bb
);
6417 /* Chunk increment and test goes into bottom_bb. */
6418 expr
= build2 (PLUS_EXPR
, diff_type
, chunk_no
,
6419 build_int_cst (diff_type
, 1));
6420 ass
= gimple_build_assign (chunk_no
, expr
);
6421 gsi_insert_after (&gsi
, ass
, GSI_CONTINUE_LINKING
);
6423 /* Chunk test at end of bottom_bb. */
6424 expr
= build2 (LT_EXPR
, boolean_type_node
, chunk_no
, chunk_max
);
6425 gsi_insert_after (&gsi
, gimple_build_cond_empty (expr
),
6426 GSI_CONTINUE_LINKING
);
6428 /* Fixup edges from bottom_bb. */
6429 split
->flags
^= EDGE_FALLTHRU
| EDGE_FALSE_VALUE
;
6430 split
->probability
= profile_probability::unlikely ().guessed ();
6431 edge latch_edge
= make_edge (bottom_bb
, head_bb
, EDGE_TRUE_VALUE
);
6432 latch_edge
->probability
= profile_probability::likely ().guessed ();
6436 gsi
= gsi_last_nondebug_bb (exit_bb
);
6437 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
6438 loc
= gimple_location (gsi_stmt (gsi
));
6440 if (!gimple_in_ssa_p (cfun
))
6442 /* Insert the final value of V, in case it is live. This is the
6443 value for the only thread that survives past the join. */
6444 expr
= fold_build2 (MINUS_EXPR
, diff_type
, range
, dir
);
6445 expr
= fold_build2 (PLUS_EXPR
, diff_type
, expr
, s
);
6446 expr
= fold_build2 (TRUNC_DIV_EXPR
, diff_type
, expr
, s
);
6447 expr
= fold_build2 (MULT_EXPR
, diff_type
, expr
, s
);
6448 expr
= build2 (plus_code
, iter_type
, b
, fold_convert (plus_type
, expr
));
6449 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
6450 true, GSI_SAME_STMT
);
6451 ass
= gimple_build_assign (v
, expr
);
6452 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
6455 /* Remove the OMP_RETURN. */
6456 gsi_remove (&gsi
, true);
6460 /* We now have one, two or three nested loops. Update the loop
6462 class loop
*parent
= entry_bb
->loop_father
;
6463 class loop
*body
= body_bb
->loop_father
;
6467 class loop
*chunk_loop
= alloc_loop ();
6468 chunk_loop
->header
= head_bb
;
6469 chunk_loop
->latch
= bottom_bb
;
6470 add_loop (chunk_loop
, parent
);
6471 parent
= chunk_loop
;
6473 else if (parent
!= body
)
6475 gcc_assert (body
->header
== body_bb
);
6476 gcc_assert (body
->latch
== cont_bb
6477 || single_pred (body
->latch
) == cont_bb
);
6483 class loop
*body_loop
= alloc_loop ();
6484 body_loop
->header
= body_bb
;
6485 body_loop
->latch
= cont_bb
;
6486 add_loop (body_loop
, parent
);
6490 /* Insert tiling's element loop. */
6491 class loop
*inner_loop
= alloc_loop ();
6492 inner_loop
->header
= elem_body_bb
;
6493 inner_loop
->latch
= elem_cont_bb
;
6494 add_loop (inner_loop
, body_loop
);
6500 /* Expand the OMP loop defined by REGION. */
6503 expand_omp_for (struct omp_region
*region
, gimple
*inner_stmt
)
6505 struct omp_for_data fd
;
6506 struct omp_for_data_loop
*loops
;
6509 = (struct omp_for_data_loop
*)
6510 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
6511 * sizeof (struct omp_for_data_loop
));
6512 omp_extract_for_data (as_a
<gomp_for
*> (last_stmt (region
->entry
)),
6514 region
->sched_kind
= fd
.sched_kind
;
6515 region
->sched_modifiers
= fd
.sched_modifiers
;
6516 region
->has_lastprivate_conditional
= fd
.lastprivate_conditional
!= 0;
6518 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
6519 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
6520 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
6523 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
6524 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
6525 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
6528 /* If there isn't a continue then this is a degerate case where
6529 the introduction of abnormal edges during lowering will prevent
6530 original loops from being detected. Fix that up. */
6531 loops_state_set (LOOPS_NEED_FIXUP
);
6533 if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_SIMD
)
6534 expand_omp_simd (region
, &fd
);
6535 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
6537 gcc_assert (!inner_stmt
);
6538 expand_oacc_for (region
, &fd
);
6540 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
6542 if (gimple_omp_for_combined_into_p (fd
.for_stmt
))
6543 expand_omp_taskloop_for_inner (region
, &fd
, inner_stmt
);
6545 expand_omp_taskloop_for_outer (region
, &fd
, inner_stmt
);
6547 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
6548 && !fd
.have_ordered
)
6550 if (fd
.chunk_size
== NULL
)
6551 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
6553 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
6557 int fn_index
, start_ix
, next_ix
;
6558 unsigned HOST_WIDE_INT sched
= 0;
6559 tree sched_arg
= NULL_TREE
;
6561 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
6562 == GF_OMP_FOR_KIND_FOR
);
6563 if (fd
.chunk_size
== NULL
6564 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
6565 fd
.chunk_size
= integer_zero_node
;
6566 switch (fd
.sched_kind
)
6568 case OMP_CLAUSE_SCHEDULE_RUNTIME
:
6569 if ((fd
.sched_modifiers
& OMP_CLAUSE_SCHEDULE_NONMONOTONIC
) != 0
6570 && fd
.lastprivate_conditional
== 0)
6572 gcc_assert (!fd
.have_ordered
);
6576 else if ((fd
.sched_modifiers
& OMP_CLAUSE_SCHEDULE_MONOTONIC
) == 0
6578 && fd
.lastprivate_conditional
== 0)
6583 sched
= (HOST_WIDE_INT_1U
<< 31);
6586 case OMP_CLAUSE_SCHEDULE_DYNAMIC
:
6587 case OMP_CLAUSE_SCHEDULE_GUIDED
:
6588 if ((fd
.sched_modifiers
& OMP_CLAUSE_SCHEDULE_MONOTONIC
) == 0
6590 && fd
.lastprivate_conditional
== 0)
6592 fn_index
= 3 + fd
.sched_kind
;
6593 sched
= (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_GUIDED
) + 2;
6596 fn_index
= fd
.sched_kind
;
6597 sched
= (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_GUIDED
) + 2;
6598 sched
+= (HOST_WIDE_INT_1U
<< 31);
6600 case OMP_CLAUSE_SCHEDULE_STATIC
:
6601 gcc_assert (fd
.have_ordered
);
6603 sched
= (HOST_WIDE_INT_1U
<< 31) + 1;
6609 fn_index
+= fd
.have_ordered
* 8;
6611 start_ix
= ((int)BUILT_IN_GOMP_LOOP_DOACROSS_STATIC_START
) + fn_index
;
6613 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
6614 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
6615 if (fd
.have_reductemp
|| fd
.have_pointer_condtemp
)
6618 start_ix
= (int)BUILT_IN_GOMP_LOOP_DOACROSS_START
;
6619 else if (fd
.have_ordered
)
6620 start_ix
= (int)BUILT_IN_GOMP_LOOP_ORDERED_START
;
6622 start_ix
= (int)BUILT_IN_GOMP_LOOP_START
;
6623 sched_arg
= build_int_cstu (long_integer_type_node
, sched
);
6625 fd
.chunk_size
= integer_zero_node
;
6627 if (fd
.iter_type
== long_long_unsigned_type_node
)
6629 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
6630 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
6631 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
6632 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
6634 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
6635 (enum built_in_function
) next_ix
, sched_arg
,
6639 if (gimple_in_ssa_p (cfun
))
6640 update_ssa (TODO_update_ssa_only_virtuals
);
6643 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
6645 v = GOMP_sections_start (n);
6662 v = GOMP_sections_next ();
6667 If this is a combined parallel sections, replace the call to
6668 GOMP_sections_start with call to GOMP_sections_next. */
6671 expand_omp_sections (struct omp_region
*region
)
6673 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
6675 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
6676 gimple_stmt_iterator si
, switch_si
;
6677 gomp_sections
*sections_stmt
;
6679 gomp_continue
*cont
;
6682 struct omp_region
*inner
;
6684 bool exit_reachable
= region
->cont
!= NULL
;
6686 gcc_assert (region
->exit
!= NULL
);
6687 entry_bb
= region
->entry
;
6688 l0_bb
= single_succ (entry_bb
);
6689 l1_bb
= region
->cont
;
6690 l2_bb
= region
->exit
;
6691 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
6692 l2
= gimple_block_label (l2_bb
);
6695 /* This can happen if there are reductions. */
6696 len
= EDGE_COUNT (l0_bb
->succs
);
6697 gcc_assert (len
> 0);
6698 e
= EDGE_SUCC (l0_bb
, len
- 1);
6699 si
= gsi_last_nondebug_bb (e
->dest
);
6702 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
6703 l2
= gimple_block_label (e
->dest
);
6705 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
6707 si
= gsi_last_nondebug_bb (e
->dest
);
6709 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
6711 l2
= gimple_block_label (e
->dest
);
6717 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
6719 default_bb
= create_empty_bb (l0_bb
);
6721 /* We will build a switch() with enough cases for all the
6722 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
6723 and a default case to abort if something goes wrong. */
6724 len
= EDGE_COUNT (l0_bb
->succs
);
6726 /* Use vec::quick_push on label_vec throughout, since we know the size
6728 auto_vec
<tree
> label_vec (len
);
6730 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
6731 GIMPLE_OMP_SECTIONS statement. */
6732 si
= gsi_last_nondebug_bb (entry_bb
);
6733 sections_stmt
= as_a
<gomp_sections
*> (gsi_stmt (si
));
6734 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
6735 vin
= gimple_omp_sections_control (sections_stmt
);
6736 tree clauses
= gimple_omp_sections_clauses (sections_stmt
);
6737 tree reductmp
= omp_find_clause (clauses
, OMP_CLAUSE__REDUCTEMP_
);
6738 tree condtmp
= omp_find_clause (clauses
, OMP_CLAUSE__CONDTEMP_
);
6739 tree cond_var
= NULL_TREE
;
6740 if (reductmp
|| condtmp
)
6742 tree reductions
= null_pointer_node
, mem
= null_pointer_node
;
6743 tree memv
= NULL_TREE
, condtemp
= NULL_TREE
;
6744 gimple_stmt_iterator gsi
= gsi_none ();
6748 reductions
= OMP_CLAUSE_DECL (reductmp
);
6749 gcc_assert (TREE_CODE (reductions
) == SSA_NAME
);
6750 g
= SSA_NAME_DEF_STMT (reductions
);
6751 reductions
= gimple_assign_rhs1 (g
);
6752 OMP_CLAUSE_DECL (reductmp
) = reductions
;
6753 gsi
= gsi_for_stmt (g
);
6759 condtemp
= OMP_CLAUSE_DECL (condtmp
);
6760 tree c
= omp_find_clause (OMP_CLAUSE_CHAIN (condtmp
),
6761 OMP_CLAUSE__CONDTEMP_
);
6762 cond_var
= OMP_CLAUSE_DECL (c
);
6763 tree type
= TREE_TYPE (condtemp
);
6764 memv
= create_tmp_var (type
);
6765 TREE_ADDRESSABLE (memv
) = 1;
6767 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
6768 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
6769 && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c
))
6771 unsigned HOST_WIDE_INT sz
6772 = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (type
))) * cnt
;
6773 expand_omp_build_assign (&gsi
, memv
, build_int_cst (type
, sz
),
6775 mem
= build_fold_addr_expr (memv
);
6777 t
= build_int_cst (unsigned_type_node
, len
- 1);
6778 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS2_START
);
6779 stmt
= gimple_build_call (u
, 3, t
, reductions
, mem
);
6780 gimple_call_set_lhs (stmt
, vin
);
6781 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6784 expand_omp_build_assign (&gsi
, condtemp
, memv
, false);
6785 tree t
= build2 (PLUS_EXPR
, TREE_TYPE (cond_var
),
6786 vin
, build_one_cst (TREE_TYPE (cond_var
)));
6787 expand_omp_build_assign (&gsi
, cond_var
, t
, false);
6791 gsi_remove (&gsi
, true);
6792 release_ssa_name (gimple_assign_lhs (g
));
6795 else if (!is_combined_parallel (region
))
6797 /* If we are not inside a combined parallel+sections region,
6798 call GOMP_sections_start. */
6799 t
= build_int_cst (unsigned_type_node
, len
- 1);
6800 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
6801 stmt
= gimple_build_call (u
, 1, t
);
6805 /* Otherwise, call GOMP_sections_next. */
6806 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
6807 stmt
= gimple_build_call (u
, 0);
6809 if (!reductmp
&& !condtmp
)
6811 gimple_call_set_lhs (stmt
, vin
);
6812 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
6814 gsi_remove (&si
, true);
6816 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
6818 switch_si
= gsi_last_nondebug_bb (l0_bb
);
6819 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
6822 cont
= as_a
<gomp_continue
*> (last_stmt (l1_bb
));
6823 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
6824 vmain
= gimple_omp_continue_control_use (cont
);
6825 vnext
= gimple_omp_continue_control_def (cont
);
6833 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
6834 label_vec
.quick_push (t
);
6837 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
6838 for (inner
= region
->inner
, casei
= 1;
6840 inner
= inner
->next
, i
++, casei
++)
6842 basic_block s_entry_bb
, s_exit_bb
;
6844 /* Skip optional reduction region. */
6845 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
6852 s_entry_bb
= inner
->entry
;
6853 s_exit_bb
= inner
->exit
;
6855 t
= gimple_block_label (s_entry_bb
);
6856 u
= build_int_cst (unsigned_type_node
, casei
);
6857 u
= build_case_label (u
, NULL
, t
);
6858 label_vec
.quick_push (u
);
6860 si
= gsi_last_nondebug_bb (s_entry_bb
);
6861 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
6862 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
6863 gsi_remove (&si
, true);
6864 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
6866 if (s_exit_bb
== NULL
)
6869 si
= gsi_last_nondebug_bb (s_exit_bb
);
6870 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
6871 gsi_remove (&si
, true);
6873 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
6876 /* Error handling code goes in DEFAULT_BB. */
6877 t
= gimple_block_label (default_bb
);
6878 u
= build_case_label (NULL
, NULL
, t
);
6879 make_edge (l0_bb
, default_bb
, 0);
6880 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
6882 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
6883 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
6884 gsi_remove (&switch_si
, true);
6886 si
= gsi_start_bb (default_bb
);
6887 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
6888 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
6894 /* Code to get the next section goes in L1_BB. */
6895 si
= gsi_last_nondebug_bb (l1_bb
);
6896 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
6898 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
6899 stmt
= gimple_build_call (bfn_decl
, 0);
6900 gimple_call_set_lhs (stmt
, vnext
);
6901 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
6904 tree t
= build2 (PLUS_EXPR
, TREE_TYPE (cond_var
),
6905 vnext
, build_one_cst (TREE_TYPE (cond_var
)));
6906 expand_omp_build_assign (&si
, cond_var
, t
, false);
6908 gsi_remove (&si
, true);
6910 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
6913 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
6914 si
= gsi_last_nondebug_bb (l2_bb
);
6915 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
6916 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
6917 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
6918 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
6920 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
6921 stmt
= gimple_build_call (t
, 0);
6922 if (gimple_omp_return_lhs (gsi_stmt (si
)))
6923 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
6924 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
6925 gsi_remove (&si
, true);
6927 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
6930 /* Expand code for an OpenMP single directive. We've already expanded
6931 much of the code, here we simply place the GOMP_barrier call. */
6934 expand_omp_single (struct omp_region
*region
)
6936 basic_block entry_bb
, exit_bb
;
6937 gimple_stmt_iterator si
;
6939 entry_bb
= region
->entry
;
6940 exit_bb
= region
->exit
;
6942 si
= gsi_last_nondebug_bb (entry_bb
);
6943 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
6944 gsi_remove (&si
, true);
6945 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
6947 si
= gsi_last_nondebug_bb (exit_bb
);
6948 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
6950 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
6951 gsi_insert_after (&si
, omp_build_barrier (t
), GSI_SAME_STMT
);
6953 gsi_remove (&si
, true);
6954 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
6957 /* Generic expansion for OpenMP synchronization directives: master,
6958 ordered and critical. All we need to do here is remove the entry
6959 and exit markers for REGION. */
6962 expand_omp_synch (struct omp_region
*region
)
6964 basic_block entry_bb
, exit_bb
;
6965 gimple_stmt_iterator si
;
6967 entry_bb
= region
->entry
;
6968 exit_bb
= region
->exit
;
6970 si
= gsi_last_nondebug_bb (entry_bb
);
6971 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
6972 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
6973 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
6974 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
6975 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
6976 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
6977 if (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
6978 && gimple_omp_teams_host (as_a
<gomp_teams
*> (gsi_stmt (si
))))
6980 expand_omp_taskreg (region
);
6983 gsi_remove (&si
, true);
6984 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
6988 si
= gsi_last_nondebug_bb (exit_bb
);
6989 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
6990 gsi_remove (&si
, true);
6991 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
6995 /* Translate enum omp_memory_order to enum memmodel. The two enums
6996 are using different numbers so that OMP_MEMORY_ORDER_UNSPECIFIED
6999 static enum memmodel
7000 omp_memory_order_to_memmodel (enum omp_memory_order mo
)
7004 case OMP_MEMORY_ORDER_RELAXED
: return MEMMODEL_RELAXED
;
7005 case OMP_MEMORY_ORDER_ACQUIRE
: return MEMMODEL_ACQUIRE
;
7006 case OMP_MEMORY_ORDER_RELEASE
: return MEMMODEL_RELEASE
;
7007 case OMP_MEMORY_ORDER_ACQ_REL
: return MEMMODEL_ACQ_REL
;
7008 case OMP_MEMORY_ORDER_SEQ_CST
: return MEMMODEL_SEQ_CST
;
7009 default: gcc_unreachable ();
7013 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7014 operation as a normal volatile load. */
7017 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
7018 tree loaded_val
, int index
)
7020 enum built_in_function tmpbase
;
7021 gimple_stmt_iterator gsi
;
7022 basic_block store_bb
;
7025 tree decl
, call
, type
, itype
;
7027 gsi
= gsi_last_nondebug_bb (load_bb
);
7028 stmt
= gsi_stmt (gsi
);
7029 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7030 loc
= gimple_location (stmt
);
7032 /* ??? If the target does not implement atomic_load_optab[mode], and mode
7033 is smaller than word size, then expand_atomic_load assumes that the load
7034 is atomic. We could avoid the builtin entirely in this case. */
7036 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
7037 decl
= builtin_decl_explicit (tmpbase
);
7038 if (decl
== NULL_TREE
)
7041 type
= TREE_TYPE (loaded_val
);
7042 itype
= TREE_TYPE (TREE_TYPE (decl
));
7044 enum omp_memory_order omo
= gimple_omp_atomic_memory_order (stmt
);
7045 tree mo
= build_int_cst (NULL
, omp_memory_order_to_memmodel (omo
));
7046 call
= build_call_expr_loc (loc
, decl
, 2, addr
, mo
);
7047 if (!useless_type_conversion_p (type
, itype
))
7048 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7049 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7051 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7052 gsi_remove (&gsi
, true);
7054 store_bb
= single_succ (load_bb
);
7055 gsi
= gsi_last_nondebug_bb (store_bb
);
7056 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7057 gsi_remove (&gsi
, true);
7059 if (gimple_in_ssa_p (cfun
))
7060 update_ssa (TODO_update_ssa_no_phi
);
7065 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7066 operation as a normal volatile store. */
7069 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
7070 tree loaded_val
, tree stored_val
, int index
)
7072 enum built_in_function tmpbase
;
7073 gimple_stmt_iterator gsi
;
7074 basic_block store_bb
= single_succ (load_bb
);
7077 tree decl
, call
, type
, itype
;
7081 gsi
= gsi_last_nondebug_bb (load_bb
);
7082 stmt
= gsi_stmt (gsi
);
7083 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7085 /* If the load value is needed, then this isn't a store but an exchange. */
7086 exchange
= gimple_omp_atomic_need_value_p (stmt
);
7088 gsi
= gsi_last_nondebug_bb (store_bb
);
7089 stmt
= gsi_stmt (gsi
);
7090 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
7091 loc
= gimple_location (stmt
);
7093 /* ??? If the target does not implement atomic_store_optab[mode], and mode
7094 is smaller than word size, then expand_atomic_store assumes that the store
7095 is atomic. We could avoid the builtin entirely in this case. */
7097 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
7098 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
7099 decl
= builtin_decl_explicit (tmpbase
);
7100 if (decl
== NULL_TREE
)
7103 type
= TREE_TYPE (stored_val
);
7105 /* Dig out the type of the function's second argument. */
7106 itype
= TREE_TYPE (decl
);
7107 itype
= TYPE_ARG_TYPES (itype
);
7108 itype
= TREE_CHAIN (itype
);
7109 itype
= TREE_VALUE (itype
);
7110 imode
= TYPE_MODE (itype
);
7112 if (exchange
&& !can_atomic_exchange_p (imode
, true))
7115 if (!useless_type_conversion_p (itype
, type
))
7116 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
7117 enum omp_memory_order omo
= gimple_omp_atomic_memory_order (stmt
);
7118 tree mo
= build_int_cst (NULL
, omp_memory_order_to_memmodel (omo
));
7119 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
, mo
);
7122 if (!useless_type_conversion_p (type
, itype
))
7123 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7124 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7127 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7128 gsi_remove (&gsi
, true);
7130 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
7131 gsi
= gsi_last_nondebug_bb (load_bb
);
7132 gsi_remove (&gsi
, true);
7134 if (gimple_in_ssa_p (cfun
))
7135 update_ssa (TODO_update_ssa_no_phi
);
7140 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7141 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
7142 size of the data type, and thus usable to find the index of the builtin
7143 decl. Returns false if the expression is not of the proper form. */
7146 expand_omp_atomic_fetch_op (basic_block load_bb
,
7147 tree addr
, tree loaded_val
,
7148 tree stored_val
, int index
)
7150 enum built_in_function oldbase
, newbase
, tmpbase
;
7151 tree decl
, itype
, call
;
7153 basic_block store_bb
= single_succ (load_bb
);
7154 gimple_stmt_iterator gsi
;
7157 enum tree_code code
;
7158 bool need_old
, need_new
;
7161 /* We expect to find the following sequences:
7164 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
7167 val = tmp OP something; (or: something OP tmp)
7168 GIMPLE_OMP_STORE (val)
7170 ???FIXME: Allow a more flexible sequence.
7171 Perhaps use data flow to pick the statements.
7175 gsi
= gsi_after_labels (store_bb
);
7176 stmt
= gsi_stmt (gsi
);
7177 if (is_gimple_debug (stmt
))
7179 gsi_next_nondebug (&gsi
);
7180 if (gsi_end_p (gsi
))
7182 stmt
= gsi_stmt (gsi
);
7184 loc
= gimple_location (stmt
);
7185 if (!is_gimple_assign (stmt
))
7187 gsi_next_nondebug (&gsi
);
7188 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
7190 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
7191 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
7192 enum omp_memory_order omo
7193 = gimple_omp_atomic_memory_order (last_stmt (load_bb
));
7194 enum memmodel mo
= omp_memory_order_to_memmodel (omo
);
7195 gcc_checking_assert (!need_old
|| !need_new
);
7197 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
7200 /* Check for one of the supported fetch-op operations. */
7201 code
= gimple_assign_rhs_code (stmt
);
7205 case POINTER_PLUS_EXPR
:
7206 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
7207 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
7210 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
7211 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
7214 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
7215 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
7218 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
7219 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
7222 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
7223 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
7229 /* Make sure the expression is of the proper form. */
7230 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
7231 rhs
= gimple_assign_rhs2 (stmt
);
7232 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
7233 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
7234 rhs
= gimple_assign_rhs1 (stmt
);
7238 tmpbase
= ((enum built_in_function
)
7239 ((need_new
? newbase
: oldbase
) + index
+ 1));
7240 decl
= builtin_decl_explicit (tmpbase
);
7241 if (decl
== NULL_TREE
)
7243 itype
= TREE_TYPE (TREE_TYPE (decl
));
7244 imode
= TYPE_MODE (itype
);
7246 /* We could test all of the various optabs involved, but the fact of the
7247 matter is that (with the exception of i486 vs i586 and xadd) all targets
7248 that support any atomic operaton optab also implements compare-and-swap.
7249 Let optabs.c take care of expanding any compare-and-swap loop. */
7250 if (!can_compare_and_swap_p (imode
, true) || !can_atomic_load_p (imode
))
7253 gsi
= gsi_last_nondebug_bb (load_bb
);
7254 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7256 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
7257 It only requires that the operation happen atomically. Thus we can
7258 use the RELAXED memory model. */
7259 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
7260 fold_convert_loc (loc
, itype
, rhs
),
7261 build_int_cst (NULL
, mo
));
7263 if (need_old
|| need_new
)
7265 lhs
= need_old
? loaded_val
: stored_val
;
7266 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
7267 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
7270 call
= fold_convert_loc (loc
, void_type_node
, call
);
7271 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7272 gsi_remove (&gsi
, true);
7274 gsi
= gsi_last_nondebug_bb (store_bb
);
7275 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7276 gsi_remove (&gsi
, true);
7277 gsi
= gsi_last_nondebug_bb (store_bb
);
7278 stmt
= gsi_stmt (gsi
);
7279 gsi_remove (&gsi
, true);
7281 if (gimple_in_ssa_p (cfun
))
7283 release_defs (stmt
);
7284 update_ssa (TODO_update_ssa_no_phi
);
7290 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7294 newval = rhs; // with oldval replacing *addr in rhs
7295 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
7296 if (oldval != newval)
7299 INDEX is log2 of the size of the data type, and thus usable to find the
7300 index of the builtin decl. */
7303 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
7304 tree addr
, tree loaded_val
, tree stored_val
,
7307 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
7308 tree type
, itype
, cmpxchg
, iaddr
, atype
;
7309 gimple_stmt_iterator si
;
7310 basic_block loop_header
= single_succ (load_bb
);
7313 enum built_in_function fncode
;
7315 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
7316 order to use the RELAXED memory model effectively. */
7317 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
7319 cmpxchg
= builtin_decl_explicit (fncode
);
7320 if (cmpxchg
== NULL_TREE
)
7322 type
= TYPE_MAIN_VARIANT (TREE_TYPE (loaded_val
));
7324 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
7326 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true)
7327 || !can_atomic_load_p (TYPE_MODE (itype
)))
7330 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
7331 si
= gsi_last_nondebug_bb (load_bb
);
7332 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7334 /* For floating-point values, we'll need to view-convert them to integers
7335 so that we can perform the atomic compare and swap. Simplify the
7336 following code by always setting up the "i"ntegral variables. */
7337 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
7341 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
7345 = force_gimple_operand_gsi (&si
,
7346 fold_convert (TREE_TYPE (iaddr
), addr
),
7347 false, NULL_TREE
, true, GSI_SAME_STMT
);
7348 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
7349 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7350 loadedi
= create_tmp_var (itype
);
7351 if (gimple_in_ssa_p (cfun
))
7352 loadedi
= make_ssa_name (loadedi
);
7357 loadedi
= loaded_val
;
7360 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
7361 tree loaddecl
= builtin_decl_explicit (fncode
);
7364 = fold_convert (atype
,
7365 build_call_expr (loaddecl
, 2, iaddr
,
7366 build_int_cst (NULL_TREE
,
7367 MEMMODEL_RELAXED
)));
7371 = build_int_cst (build_pointer_type_for_mode (atype
, ptr_mode
,
7373 initial
= build2 (MEM_REF
, atype
, iaddr
, off
);
7377 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
7380 /* Move the value to the LOADEDI temporary. */
7381 if (gimple_in_ssa_p (cfun
))
7383 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
7384 phi
= create_phi_node (loadedi
, loop_header
);
7385 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
7389 gsi_insert_before (&si
,
7390 gimple_build_assign (loadedi
, initial
),
7392 if (loadedi
!= loaded_val
)
7394 gimple_stmt_iterator gsi2
;
7397 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
7398 gsi2
= gsi_start_bb (loop_header
);
7399 if (gimple_in_ssa_p (cfun
))
7402 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
7403 true, GSI_SAME_STMT
);
7404 stmt
= gimple_build_assign (loaded_val
, x
);
7405 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
7409 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
7410 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
7411 true, GSI_SAME_STMT
);
7414 gsi_remove (&si
, true);
7416 si
= gsi_last_nondebug_bb (store_bb
);
7417 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
7420 storedi
= stored_val
;
7423 = force_gimple_operand_gsi (&si
,
7424 build1 (VIEW_CONVERT_EXPR
, itype
,
7425 stored_val
), true, NULL_TREE
, true,
7428 /* Build the compare&swap statement. */
7429 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
7430 new_storedi
= force_gimple_operand_gsi (&si
,
7431 fold_convert (TREE_TYPE (loadedi
),
7434 true, GSI_SAME_STMT
);
7436 if (gimple_in_ssa_p (cfun
))
7440 old_vali
= create_tmp_var (TREE_TYPE (loadedi
));
7441 stmt
= gimple_build_assign (old_vali
, loadedi
);
7442 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7444 stmt
= gimple_build_assign (loadedi
, new_storedi
);
7445 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7448 /* Note that we always perform the comparison as an integer, even for
7449 floating point. This allows the atomic operation to properly
7450 succeed even with NaNs and -0.0. */
7451 tree ne
= build2 (NE_EXPR
, boolean_type_node
, new_storedi
, old_vali
);
7452 stmt
= gimple_build_cond_empty (ne
);
7453 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7456 e
= single_succ_edge (store_bb
);
7457 e
->flags
&= ~EDGE_FALLTHRU
;
7458 e
->flags
|= EDGE_FALSE_VALUE
;
7459 /* Expect no looping. */
7460 e
->probability
= profile_probability::guessed_always ();
7462 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
7463 e
->probability
= profile_probability::guessed_never ();
7465 /* Copy the new value to loadedi (we already did that before the condition
7466 if we are not in SSA). */
7467 if (gimple_in_ssa_p (cfun
))
7469 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
7470 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
7473 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
7474 gsi_remove (&si
, true);
7476 class loop
*loop
= alloc_loop ();
7477 loop
->header
= loop_header
;
7478 loop
->latch
= store_bb
;
7479 add_loop (loop
, loop_header
->loop_father
);
7481 if (gimple_in_ssa_p (cfun
))
7482 update_ssa (TODO_update_ssa_no_phi
);
7487 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7489 GOMP_atomic_start ();
7493 The result is not globally atomic, but works so long as all parallel
7494 references are within #pragma omp atomic directives. According to
7495 responses received from omp@openmp.org, appears to be within spec.
7496 Which makes sense, since that's how several other compilers handle
7497 this situation as well.
7498 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
7499 expanding. STORED_VAL is the operand of the matching
7500 GIMPLE_OMP_ATOMIC_STORE.
7503 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
7507 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
7512 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
7513 tree addr
, tree loaded_val
, tree stored_val
)
7515 gimple_stmt_iterator si
;
7519 si
= gsi_last_nondebug_bb (load_bb
);
7520 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7522 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
7523 t
= build_call_expr (t
, 0);
7524 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7526 tree mem
= build_simple_mem_ref (addr
);
7527 TREE_TYPE (mem
) = TREE_TYPE (loaded_val
);
7528 TREE_OPERAND (mem
, 1)
7529 = fold_convert (build_pointer_type_for_mode (TREE_TYPE (mem
), ptr_mode
,
7531 TREE_OPERAND (mem
, 1));
7532 stmt
= gimple_build_assign (loaded_val
, mem
);
7533 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7534 gsi_remove (&si
, true);
7536 si
= gsi_last_nondebug_bb (store_bb
);
7537 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
7539 stmt
= gimple_build_assign (unshare_expr (mem
), stored_val
);
7540 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7542 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
7543 t
= build_call_expr (t
, 0);
7544 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7545 gsi_remove (&si
, true);
7547 if (gimple_in_ssa_p (cfun
))
7548 update_ssa (TODO_update_ssa_no_phi
);
7552 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
7553 using expand_omp_atomic_fetch_op. If it failed, we try to
7554 call expand_omp_atomic_pipeline, and if it fails too, the
7555 ultimate fallback is wrapping the operation in a mutex
7556 (expand_omp_atomic_mutex). REGION is the atomic region built
7557 by build_omp_regions_1(). */
7560 expand_omp_atomic (struct omp_region
*region
)
7562 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
7563 gomp_atomic_load
*load
= as_a
<gomp_atomic_load
*> (last_stmt (load_bb
));
7564 gomp_atomic_store
*store
= as_a
<gomp_atomic_store
*> (last_stmt (store_bb
));
7565 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
7566 tree addr
= gimple_omp_atomic_load_rhs (load
);
7567 tree stored_val
= gimple_omp_atomic_store_val (store
);
7568 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (loaded_val
));
7569 HOST_WIDE_INT index
;
7571 /* Make sure the type is one of the supported sizes. */
7572 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
7573 index
= exact_log2 (index
);
7574 if (index
>= 0 && index
<= 4)
7576 unsigned int align
= TYPE_ALIGN_UNIT (type
);
7578 /* __sync builtins require strict data alignment. */
7579 if (exact_log2 (align
) >= index
)
7583 if (loaded_val
== stored_val
7584 && (is_int_mode (TYPE_MODE (type
), &smode
)
7585 || is_float_mode (TYPE_MODE (type
), &smode
))
7586 && GET_MODE_BITSIZE (smode
) <= BITS_PER_WORD
7587 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
7591 if ((is_int_mode (TYPE_MODE (type
), &smode
)
7592 || is_float_mode (TYPE_MODE (type
), &smode
))
7593 && GET_MODE_BITSIZE (smode
) <= BITS_PER_WORD
7594 && store_bb
== single_succ (load_bb
)
7595 && first_stmt (store_bb
) == store
7596 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
7600 /* When possible, use specialized atomic update functions. */
7601 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
7602 && store_bb
== single_succ (load_bb
)
7603 && expand_omp_atomic_fetch_op (load_bb
, addr
,
7604 loaded_val
, stored_val
, index
))
7607 /* If we don't have specialized __sync builtins, try and implement
7608 as a compare and swap loop. */
7609 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
7610 loaded_val
, stored_val
, index
))
7615 /* The ultimate fallback is wrapping the operation in a mutex. */
7616 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
7619 /* Mark the loops inside the kernels region starting at REGION_ENTRY and ending
7623 mark_loops_in_oacc_kernels_region (basic_block region_entry
,
7624 basic_block region_exit
)
7626 class loop
*outer
= region_entry
->loop_father
;
7627 gcc_assert (region_exit
== NULL
|| outer
== region_exit
->loop_father
);
7629 /* Don't parallelize the kernels region if it contains more than one outer
7631 unsigned int nr_outer_loops
= 0;
7632 class loop
*single_outer
= NULL
;
7633 for (class loop
*loop
= outer
->inner
; loop
!= NULL
; loop
= loop
->next
)
7635 gcc_assert (loop_outer (loop
) == outer
);
7637 if (!dominated_by_p (CDI_DOMINATORS
, loop
->header
, region_entry
))
7640 if (region_exit
!= NULL
7641 && dominated_by_p (CDI_DOMINATORS
, loop
->header
, region_exit
))
7645 single_outer
= loop
;
7647 if (nr_outer_loops
!= 1)
7650 for (class loop
*loop
= single_outer
->inner
;
7656 /* Mark the loops in the region. */
7657 for (class loop
*loop
= single_outer
; loop
!= NULL
; loop
= loop
->inner
)
7658 loop
->in_oacc_kernels_region
= true;
7661 /* Types used to pass grid and wortkgroup sizes to kernel invocation. */
7663 struct GTY(()) grid_launch_attributes_trees
7665 tree kernel_dim_array_type
;
7666 tree kernel_lattrs_dimnum_decl
;
7667 tree kernel_lattrs_grid_decl
;
7668 tree kernel_lattrs_group_decl
;
7669 tree kernel_launch_attributes_type
;
7672 static GTY(()) struct grid_launch_attributes_trees
*grid_attr_trees
;
7674 /* Create types used to pass kernel launch attributes to target. */
7677 grid_create_kernel_launch_attr_types (void)
7679 if (grid_attr_trees
)
7681 grid_attr_trees
= ggc_alloc
<grid_launch_attributes_trees
> ();
7683 tree dim_arr_index_type
7684 = build_index_type (build_int_cst (integer_type_node
, 2));
7685 grid_attr_trees
->kernel_dim_array_type
7686 = build_array_type (uint32_type_node
, dim_arr_index_type
);
7688 grid_attr_trees
->kernel_launch_attributes_type
= make_node (RECORD_TYPE
);
7689 grid_attr_trees
->kernel_lattrs_dimnum_decl
7690 = build_decl (BUILTINS_LOCATION
, FIELD_DECL
, get_identifier ("ndim"),
7692 DECL_CHAIN (grid_attr_trees
->kernel_lattrs_dimnum_decl
) = NULL_TREE
;
7694 grid_attr_trees
->kernel_lattrs_grid_decl
7695 = build_decl (BUILTINS_LOCATION
, FIELD_DECL
, get_identifier ("grid_size"),
7696 grid_attr_trees
->kernel_dim_array_type
);
7697 DECL_CHAIN (grid_attr_trees
->kernel_lattrs_grid_decl
)
7698 = grid_attr_trees
->kernel_lattrs_dimnum_decl
;
7699 grid_attr_trees
->kernel_lattrs_group_decl
7700 = build_decl (BUILTINS_LOCATION
, FIELD_DECL
, get_identifier ("group_size"),
7701 grid_attr_trees
->kernel_dim_array_type
);
7702 DECL_CHAIN (grid_attr_trees
->kernel_lattrs_group_decl
)
7703 = grid_attr_trees
->kernel_lattrs_grid_decl
;
7704 finish_builtin_struct (grid_attr_trees
->kernel_launch_attributes_type
,
7705 "__gomp_kernel_launch_attributes",
7706 grid_attr_trees
->kernel_lattrs_group_decl
, NULL_TREE
);
7709 /* Insert before the current statement in GSI a store of VALUE to INDEX of
7710 array (of type kernel_dim_array_type) FLD_DECL of RANGE_VAR. VALUE must be
7711 of type uint32_type_node. */
7714 grid_insert_store_range_dim (gimple_stmt_iterator
*gsi
, tree range_var
,
7715 tree fld_decl
, int index
, tree value
)
7717 tree ref
= build4 (ARRAY_REF
, uint32_type_node
,
7718 build3 (COMPONENT_REF
,
7719 grid_attr_trees
->kernel_dim_array_type
,
7720 range_var
, fld_decl
, NULL_TREE
),
7721 build_int_cst (integer_type_node
, index
),
7722 NULL_TREE
, NULL_TREE
);
7723 gsi_insert_before (gsi
, gimple_build_assign (ref
, value
), GSI_SAME_STMT
);
7726 /* Return a tree representation of a pointer to a structure with grid and
7727 work-group size information. Statements filling that information will be
7728 inserted before GSI, TGT_STMT is the target statement which has the
7729 necessary information in it. */
7732 grid_get_kernel_launch_attributes (gimple_stmt_iterator
*gsi
,
7733 gomp_target
*tgt_stmt
)
7735 grid_create_kernel_launch_attr_types ();
7736 tree lattrs
= create_tmp_var (grid_attr_trees
->kernel_launch_attributes_type
,
7737 "__kernel_launch_attrs");
7739 unsigned max_dim
= 0;
7740 for (tree clause
= gimple_omp_target_clauses (tgt_stmt
);
7742 clause
= OMP_CLAUSE_CHAIN (clause
))
7744 if (OMP_CLAUSE_CODE (clause
) != OMP_CLAUSE__GRIDDIM_
)
7747 unsigned dim
= OMP_CLAUSE__GRIDDIM__DIMENSION (clause
);
7748 max_dim
= MAX (dim
, max_dim
);
7750 grid_insert_store_range_dim (gsi
, lattrs
,
7751 grid_attr_trees
->kernel_lattrs_grid_decl
,
7752 dim
, OMP_CLAUSE__GRIDDIM__SIZE (clause
));
7753 grid_insert_store_range_dim (gsi
, lattrs
,
7754 grid_attr_trees
->kernel_lattrs_group_decl
,
7755 dim
, OMP_CLAUSE__GRIDDIM__GROUP (clause
));
7758 tree dimref
= build3 (COMPONENT_REF
, uint32_type_node
, lattrs
,
7759 grid_attr_trees
->kernel_lattrs_dimnum_decl
, NULL_TREE
);
7760 gcc_checking_assert (max_dim
<= 2);
7761 tree dimensions
= build_int_cstu (uint32_type_node
, max_dim
+ 1);
7762 gsi_insert_before (gsi
, gimple_build_assign (dimref
, dimensions
),
7764 TREE_ADDRESSABLE (lattrs
) = 1;
7765 return build_fold_addr_expr (lattrs
);
7768 /* Build target argument identifier from the DEVICE identifier, value
7769 identifier ID and whether the element also has a SUBSEQUENT_PARAM. */
7772 get_target_argument_identifier_1 (int device
, bool subseqent_param
, int id
)
7774 tree t
= build_int_cst (integer_type_node
, device
);
7775 if (subseqent_param
)
7776 t
= fold_build2 (BIT_IOR_EXPR
, integer_type_node
, t
,
7777 build_int_cst (integer_type_node
,
7778 GOMP_TARGET_ARG_SUBSEQUENT_PARAM
));
7779 t
= fold_build2 (BIT_IOR_EXPR
, integer_type_node
, t
,
7780 build_int_cst (integer_type_node
, id
));
7784 /* Like above but return it in type that can be directly stored as an element
7785 of the argument array. */
7788 get_target_argument_identifier (int device
, bool subseqent_param
, int id
)
7790 tree t
= get_target_argument_identifier_1 (device
, subseqent_param
, id
);
7791 return fold_convert (ptr_type_node
, t
);
7794 /* Return a target argument consisting of DEVICE identifier, value identifier
7795 ID, and the actual VALUE. */
7798 get_target_argument_value (gimple_stmt_iterator
*gsi
, int device
, int id
,
7801 tree t
= fold_build2 (LSHIFT_EXPR
, integer_type_node
,
7802 fold_convert (integer_type_node
, value
),
7803 build_int_cst (unsigned_type_node
,
7804 GOMP_TARGET_ARG_VALUE_SHIFT
));
7805 t
= fold_build2 (BIT_IOR_EXPR
, integer_type_node
, t
,
7806 get_target_argument_identifier_1 (device
, false, id
));
7807 t
= fold_convert (ptr_type_node
, t
);
7808 return force_gimple_operand_gsi (gsi
, t
, true, NULL
, true, GSI_SAME_STMT
);
7811 /* If VALUE is an integer constant greater than -2^15 and smaller than 2^15,
7812 push one argument to ARGS with both the DEVICE, ID and VALUE embedded in it,
7813 otherwise push an identifier (with DEVICE and ID) and the VALUE in two
7817 push_target_argument_according_to_value (gimple_stmt_iterator
*gsi
, int device
,
7818 int id
, tree value
, vec
<tree
> *args
)
7820 if (tree_fits_shwi_p (value
)
7821 && tree_to_shwi (value
) > -(1 << 15)
7822 && tree_to_shwi (value
) < (1 << 15))
7823 args
->quick_push (get_target_argument_value (gsi
, device
, id
, value
));
7826 args
->quick_push (get_target_argument_identifier (device
, true, id
));
7827 value
= fold_convert (ptr_type_node
, value
);
7828 value
= force_gimple_operand_gsi (gsi
, value
, true, NULL
, true,
7830 args
->quick_push (value
);
7834 /* Create an array of arguments that is then passed to GOMP_target. */
7837 get_target_arguments (gimple_stmt_iterator
*gsi
, gomp_target
*tgt_stmt
)
7839 auto_vec
<tree
, 6> args
;
7840 tree clauses
= gimple_omp_target_clauses (tgt_stmt
);
7841 tree t
, c
= omp_find_clause (clauses
, OMP_CLAUSE_NUM_TEAMS
);
7843 t
= OMP_CLAUSE_NUM_TEAMS_EXPR (c
);
7845 t
= integer_minus_one_node
;
7846 push_target_argument_according_to_value (gsi
, GOMP_TARGET_ARG_DEVICE_ALL
,
7847 GOMP_TARGET_ARG_NUM_TEAMS
, t
, &args
);
7849 c
= omp_find_clause (clauses
, OMP_CLAUSE_THREAD_LIMIT
);
7851 t
= OMP_CLAUSE_THREAD_LIMIT_EXPR (c
);
7853 t
= integer_minus_one_node
;
7854 push_target_argument_according_to_value (gsi
, GOMP_TARGET_ARG_DEVICE_ALL
,
7855 GOMP_TARGET_ARG_THREAD_LIMIT
, t
,
7858 /* Add HSA-specific grid sizes, if available. */
7859 if (omp_find_clause (gimple_omp_target_clauses (tgt_stmt
),
7860 OMP_CLAUSE__GRIDDIM_
))
7862 int id
= GOMP_TARGET_ARG_HSA_KERNEL_ATTRIBUTES
;
7863 t
= get_target_argument_identifier (GOMP_DEVICE_HSA
, true, id
);
7864 args
.quick_push (t
);
7865 args
.quick_push (grid_get_kernel_launch_attributes (gsi
, tgt_stmt
));
7868 /* Produce more, perhaps device specific, arguments here. */
7870 tree argarray
= create_tmp_var (build_array_type_nelts (ptr_type_node
,
7871 args
.length () + 1),
7872 ".omp_target_args");
7873 for (unsigned i
= 0; i
< args
.length (); i
++)
7875 tree ref
= build4 (ARRAY_REF
, ptr_type_node
, argarray
,
7876 build_int_cst (integer_type_node
, i
),
7877 NULL_TREE
, NULL_TREE
);
7878 gsi_insert_before (gsi
, gimple_build_assign (ref
, args
[i
]),
7881 tree ref
= build4 (ARRAY_REF
, ptr_type_node
, argarray
,
7882 build_int_cst (integer_type_node
, args
.length ()),
7883 NULL_TREE
, NULL_TREE
);
7884 gsi_insert_before (gsi
, gimple_build_assign (ref
, null_pointer_node
),
7886 TREE_ADDRESSABLE (argarray
) = 1;
7887 return build_fold_addr_expr (argarray
);
7890 /* Expand the GIMPLE_OMP_TARGET starting at REGION. */
7893 expand_omp_target (struct omp_region
*region
)
7895 basic_block entry_bb
, exit_bb
, new_bb
;
7896 struct function
*child_cfun
;
7897 tree child_fn
, block
, t
;
7898 gimple_stmt_iterator gsi
;
7899 gomp_target
*entry_stmt
;
7902 bool offloaded
, data_region
;
7904 entry_stmt
= as_a
<gomp_target
*> (last_stmt (region
->entry
));
7905 new_bb
= region
->entry
;
7907 offloaded
= is_gimple_omp_offloaded (entry_stmt
);
7908 switch (gimple_omp_target_kind (entry_stmt
))
7910 case GF_OMP_TARGET_KIND_REGION
:
7911 case GF_OMP_TARGET_KIND_UPDATE
:
7912 case GF_OMP_TARGET_KIND_ENTER_DATA
:
7913 case GF_OMP_TARGET_KIND_EXIT_DATA
:
7914 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
7915 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
7916 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
7917 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
7918 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
7919 data_region
= false;
7921 case GF_OMP_TARGET_KIND_DATA
:
7922 case GF_OMP_TARGET_KIND_OACC_DATA
:
7923 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
7930 child_fn
= NULL_TREE
;
7934 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
7935 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
7938 /* Supported by expand_omp_taskreg, but not here. */
7939 if (child_cfun
!= NULL
)
7940 gcc_checking_assert (!child_cfun
->cfg
);
7941 gcc_checking_assert (!gimple_in_ssa_p (cfun
));
7943 entry_bb
= region
->entry
;
7944 exit_bb
= region
->exit
;
7946 if (gimple_omp_target_kind (entry_stmt
) == GF_OMP_TARGET_KIND_OACC_KERNELS
)
7948 mark_loops_in_oacc_kernels_region (region
->entry
, region
->exit
);
7950 /* Further down, both OpenACC kernels and OpenACC parallel constructs
7951 will be mappted to BUILT_IN_GOACC_PARALLEL, and to distinguish the
7952 two, there is an "oacc kernels" attribute set for OpenACC kernels. */
7953 DECL_ATTRIBUTES (child_fn
)
7954 = tree_cons (get_identifier ("oacc kernels"),
7955 NULL_TREE
, DECL_ATTRIBUTES (child_fn
));
7960 unsigned srcidx
, dstidx
, num
;
7962 /* If the offloading region needs data sent from the parent
7963 function, then the very first statement (except possible
7964 tree profile counter updates) of the offloading body
7965 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
7966 &.OMP_DATA_O is passed as an argument to the child function,
7967 we need to replace it with the argument as seen by the child
7970 In most cases, this will end up being the identity assignment
7971 .OMP_DATA_I = .OMP_DATA_I. However, if the offloading body had
7972 a function call that has been inlined, the original PARM_DECL
7973 .OMP_DATA_I may have been converted into a different local
7974 variable. In which case, we need to keep the assignment. */
7975 tree data_arg
= gimple_omp_target_data_arg (entry_stmt
);
7978 basic_block entry_succ_bb
= single_succ (entry_bb
);
7979 gimple_stmt_iterator gsi
;
7981 gimple
*tgtcopy_stmt
= NULL
;
7982 tree sender
= TREE_VEC_ELT (data_arg
, 0);
7984 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
7986 gcc_assert (!gsi_end_p (gsi
));
7987 stmt
= gsi_stmt (gsi
);
7988 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
7991 if (gimple_num_ops (stmt
) == 2)
7993 tree arg
= gimple_assign_rhs1 (stmt
);
7995 /* We're ignoring the subcode because we're
7996 effectively doing a STRIP_NOPS. */
7998 if (TREE_CODE (arg
) == ADDR_EXPR
7999 && TREE_OPERAND (arg
, 0) == sender
)
8001 tgtcopy_stmt
= stmt
;
8007 gcc_assert (tgtcopy_stmt
!= NULL
);
8008 arg
= DECL_ARGUMENTS (child_fn
);
8010 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
8011 gsi_remove (&gsi
, true);
8014 /* Declare local variables needed in CHILD_CFUN. */
8015 block
= DECL_INITIAL (child_fn
);
8016 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
8017 /* The gimplifier could record temporaries in the offloading block
8018 rather than in containing function's local_decls chain,
8019 which would mean cgraph missed finalizing them. Do it now. */
8020 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
8021 if (VAR_P (t
) && TREE_STATIC (t
) && !DECL_EXTERNAL (t
))
8022 varpool_node::finalize_decl (t
);
8023 DECL_SAVED_TREE (child_fn
) = NULL
;
8024 /* We'll create a CFG for child_fn, so no gimple body is needed. */
8025 gimple_set_body (child_fn
, NULL
);
8026 TREE_USED (block
) = 1;
8028 /* Reset DECL_CONTEXT on function arguments. */
8029 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
8030 DECL_CONTEXT (t
) = child_fn
;
8032 /* Split ENTRY_BB at GIMPLE_*,
8033 so that it can be moved to the child function. */
8034 gsi
= gsi_last_nondebug_bb (entry_bb
);
8035 stmt
= gsi_stmt (gsi
);
8037 && gimple_code (stmt
) == gimple_code (entry_stmt
));
8038 e
= split_block (entry_bb
, stmt
);
8039 gsi_remove (&gsi
, true);
8041 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8043 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
8046 gsi
= gsi_last_nondebug_bb (exit_bb
);
8047 gcc_assert (!gsi_end_p (gsi
)
8048 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
8049 stmt
= gimple_build_return (NULL
);
8050 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
8051 gsi_remove (&gsi
, true);
8054 /* Move the offloading region into CHILD_CFUN. */
8056 block
= gimple_block (entry_stmt
);
8058 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
8060 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
8061 /* When the OMP expansion process cannot guarantee an up-to-date
8062 loop tree arrange for the child function to fixup loops. */
8063 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
8064 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
8066 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
8067 num
= vec_safe_length (child_cfun
->local_decls
);
8068 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
8070 t
= (*child_cfun
->local_decls
)[srcidx
];
8071 if (DECL_CONTEXT (t
) == cfun
->decl
)
8073 if (srcidx
!= dstidx
)
8074 (*child_cfun
->local_decls
)[dstidx
] = t
;
8078 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
8080 /* Inform the callgraph about the new function. */
8081 child_cfun
->curr_properties
= cfun
->curr_properties
;
8082 child_cfun
->has_simduid_loops
|= cfun
->has_simduid_loops
;
8083 child_cfun
->has_force_vectorize_loops
|= cfun
->has_force_vectorize_loops
;
8084 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
8085 node
->parallelized_function
= 1;
8086 cgraph_node::add_new_function (child_fn
, true);
8088 /* Add the new function to the offload table. */
8089 if (ENABLE_OFFLOADING
)
8092 DECL_PRESERVE_P (child_fn
) = 1;
8093 vec_safe_push (offload_funcs
, child_fn
);
8096 bool need_asm
= DECL_ASSEMBLER_NAME_SET_P (current_function_decl
)
8097 && !DECL_ASSEMBLER_NAME_SET_P (child_fn
);
8099 /* Fix the callgraph edges for child_cfun. Those for cfun will be
8100 fixed in a following pass. */
8101 push_cfun (child_cfun
);
8103 assign_assembler_name_if_needed (child_fn
);
8104 cgraph_edge::rebuild_edges ();
8106 /* Some EH regions might become dead, see PR34608. If
8107 pass_cleanup_cfg isn't the first pass to happen with the
8108 new child, these dead EH edges might cause problems.
8109 Clean them up now. */
8110 if (flag_exceptions
)
8113 bool changed
= false;
8115 FOR_EACH_BB_FN (bb
, cfun
)
8116 changed
|= gimple_purge_dead_eh_edges (bb
);
8118 cleanup_tree_cfg ();
8120 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
8121 verify_loop_structure ();
8124 if (dump_file
&& !gimple_in_ssa_p (cfun
))
8126 omp_any_child_fn_dumped
= true;
8127 dump_function_header (dump_file
, child_fn
, dump_flags
);
8128 dump_function_to_file (child_fn
, dump_file
, dump_flags
);
8131 adjust_context_and_scope (region
, gimple_block (entry_stmt
), child_fn
);
8134 /* Emit a library call to launch the offloading region, or do data
8136 tree t1
, t2
, t3
, t4
, depend
, c
, clauses
;
8137 enum built_in_function start_ix
;
8138 unsigned int flags_i
= 0;
8140 switch (gimple_omp_target_kind (entry_stmt
))
8142 case GF_OMP_TARGET_KIND_REGION
:
8143 start_ix
= BUILT_IN_GOMP_TARGET
;
8145 case GF_OMP_TARGET_KIND_DATA
:
8146 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
8148 case GF_OMP_TARGET_KIND_UPDATE
:
8149 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
8151 case GF_OMP_TARGET_KIND_ENTER_DATA
:
8152 start_ix
= BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
;
8154 case GF_OMP_TARGET_KIND_EXIT_DATA
:
8155 start_ix
= BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
;
8156 flags_i
|= GOMP_TARGET_FLAG_EXIT_DATA
;
8158 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
8159 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
8160 start_ix
= BUILT_IN_GOACC_PARALLEL
;
8162 case GF_OMP_TARGET_KIND_OACC_DATA
:
8163 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
8164 start_ix
= BUILT_IN_GOACC_DATA_START
;
8166 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
8167 start_ix
= BUILT_IN_GOACC_UPDATE
;
8169 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
8170 start_ix
= BUILT_IN_GOACC_ENTER_EXIT_DATA
;
8172 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
8173 start_ix
= BUILT_IN_GOACC_DECLARE
;
8179 clauses
= gimple_omp_target_clauses (entry_stmt
);
8181 tree device
= NULL_TREE
;
8182 location_t device_loc
= UNKNOWN_LOCATION
;
8183 tree goacc_flags
= NULL_TREE
;
8184 if (is_gimple_omp_oacc (entry_stmt
))
8186 /* By default, no GOACC_FLAGs are set. */
8187 goacc_flags
= integer_zero_node
;
8191 c
= omp_find_clause (clauses
, OMP_CLAUSE_DEVICE
);
8194 device
= OMP_CLAUSE_DEVICE_ID (c
);
8195 device_loc
= OMP_CLAUSE_LOCATION (c
);
8199 /* By default, the value of DEVICE is GOMP_DEVICE_ICV (let runtime
8201 device
= build_int_cst (integer_type_node
, GOMP_DEVICE_ICV
);
8202 device_loc
= gimple_location (entry_stmt
);
8205 c
= omp_find_clause (clauses
, OMP_CLAUSE_NOWAIT
);
8207 flags_i
|= GOMP_TARGET_FLAG_NOWAIT
;
8210 /* By default, there is no conditional. */
8211 tree cond
= NULL_TREE
;
8212 c
= omp_find_clause (clauses
, OMP_CLAUSE_IF
);
8214 cond
= OMP_CLAUSE_IF_EXPR (c
);
8215 /* If we found the clause 'if (cond)', build:
8216 OpenACC: goacc_flags = (cond ? goacc_flags : flags | GOACC_FLAG_HOST_FALLBACK)
8217 OpenMP: device = (cond ? device : GOMP_DEVICE_HOST_FALLBACK) */
8221 if (is_gimple_omp_oacc (entry_stmt
))
8225 /* Ensure 'device' is of the correct type. */
8226 device
= fold_convert_loc (device_loc
, integer_type_node
, device
);
8231 cond
= gimple_boolify (cond
);
8233 basic_block cond_bb
, then_bb
, else_bb
;
8237 tmp_var
= create_tmp_var (TREE_TYPE (*tp
));
8239 e
= split_block_after_labels (new_bb
);
8242 gsi
= gsi_last_nondebug_bb (new_bb
);
8244 e
= split_block (new_bb
, gsi_stmt (gsi
));
8250 then_bb
= create_empty_bb (cond_bb
);
8251 else_bb
= create_empty_bb (then_bb
);
8252 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
8253 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
8255 stmt
= gimple_build_cond_empty (cond
);
8256 gsi
= gsi_last_bb (cond_bb
);
8257 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8259 gsi
= gsi_start_bb (then_bb
);
8260 stmt
= gimple_build_assign (tmp_var
, *tp
);
8261 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8263 gsi
= gsi_start_bb (else_bb
);
8264 if (is_gimple_omp_oacc (entry_stmt
))
8265 stmt
= gimple_build_assign (tmp_var
,
8268 build_int_cst (integer_type_node
,
8269 GOACC_FLAG_HOST_FALLBACK
));
8271 stmt
= gimple_build_assign (tmp_var
,
8272 build_int_cst (integer_type_node
,
8273 GOMP_DEVICE_HOST_FALLBACK
));
8274 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8276 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
8277 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
8278 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
8279 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
8280 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
8281 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
8285 gsi
= gsi_last_nondebug_bb (new_bb
);
8289 gsi
= gsi_last_nondebug_bb (new_bb
);
8291 if (device
!= NULL_TREE
)
8292 device
= force_gimple_operand_gsi (&gsi
, device
, true, NULL_TREE
,
8293 true, GSI_SAME_STMT
);
8296 t
= gimple_omp_target_data_arg (entry_stmt
);
8299 t1
= size_zero_node
;
8300 t2
= build_zero_cst (ptr_type_node
);
8306 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
8307 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
8308 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
8309 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
8310 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
8314 bool tagging
= false;
8315 /* The maximum number used by any start_ix, without varargs. */
8316 auto_vec
<tree
, 11> args
;
8317 if (is_gimple_omp_oacc (entry_stmt
))
8319 tree goacc_flags_m
= fold_build1 (GOACC_FLAGS_MARSHAL_OP
,
8320 TREE_TYPE (goacc_flags
), goacc_flags
);
8321 goacc_flags_m
= force_gimple_operand_gsi (&gsi
, goacc_flags_m
, true,
8324 args
.quick_push (goacc_flags_m
);
8327 args
.quick_push (device
);
8329 args
.quick_push (build_fold_addr_expr (child_fn
));
8330 args
.quick_push (t1
);
8331 args
.quick_push (t2
);
8332 args
.quick_push (t3
);
8333 args
.quick_push (t4
);
8336 case BUILT_IN_GOACC_DATA_START
:
8337 case BUILT_IN_GOACC_DECLARE
:
8338 case BUILT_IN_GOMP_TARGET_DATA
:
8340 case BUILT_IN_GOMP_TARGET
:
8341 case BUILT_IN_GOMP_TARGET_UPDATE
:
8342 case BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
:
8343 args
.quick_push (build_int_cst (unsigned_type_node
, flags_i
));
8344 c
= omp_find_clause (clauses
, OMP_CLAUSE_DEPEND
);
8346 depend
= OMP_CLAUSE_DECL (c
);
8348 depend
= build_int_cst (ptr_type_node
, 0);
8349 args
.quick_push (depend
);
8350 if (start_ix
== BUILT_IN_GOMP_TARGET
)
8351 args
.quick_push (get_target_arguments (&gsi
, entry_stmt
));
8353 case BUILT_IN_GOACC_PARALLEL
:
8354 oacc_set_fn_attrib (child_fn
, clauses
, &args
);
8357 case BUILT_IN_GOACC_ENTER_EXIT_DATA
:
8358 case BUILT_IN_GOACC_UPDATE
:
8360 tree t_async
= NULL_TREE
;
8362 /* If present, use the value specified by the respective
8363 clause, making sure that is of the correct type. */
8364 c
= omp_find_clause (clauses
, OMP_CLAUSE_ASYNC
);
8366 t_async
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
8368 OMP_CLAUSE_ASYNC_EXPR (c
));
8370 /* Default values for t_async. */
8371 t_async
= fold_convert_loc (gimple_location (entry_stmt
),
8373 build_int_cst (integer_type_node
,
8375 if (tagging
&& t_async
)
8377 unsigned HOST_WIDE_INT i_async
= GOMP_LAUNCH_OP_MAX
;
8379 if (TREE_CODE (t_async
) == INTEGER_CST
)
8381 /* See if we can pack the async arg in to the tag's
8383 i_async
= TREE_INT_CST_LOW (t_async
);
8384 if (i_async
< GOMP_LAUNCH_OP_MAX
)
8385 t_async
= NULL_TREE
;
8387 i_async
= GOMP_LAUNCH_OP_MAX
;
8389 args
.safe_push (oacc_launch_pack (GOMP_LAUNCH_ASYNC
, NULL_TREE
,
8393 args
.safe_push (t_async
);
8395 /* Save the argument index, and ... */
8396 unsigned t_wait_idx
= args
.length ();
8397 unsigned num_waits
= 0;
8398 c
= omp_find_clause (clauses
, OMP_CLAUSE_WAIT
);
8400 /* ... push a placeholder. */
8401 args
.safe_push (integer_zero_node
);
8403 for (; c
; c
= OMP_CLAUSE_CHAIN (c
))
8404 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_WAIT
)
8406 args
.safe_push (fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
8408 OMP_CLAUSE_WAIT_EXPR (c
)));
8412 if (!tagging
|| num_waits
)
8416 /* Now that we know the number, update the placeholder. */
8418 len
= oacc_launch_pack (GOMP_LAUNCH_WAIT
, NULL_TREE
, num_waits
);
8420 len
= build_int_cst (integer_type_node
, num_waits
);
8421 len
= fold_convert_loc (gimple_location (entry_stmt
),
8422 unsigned_type_node
, len
);
8423 args
[t_wait_idx
] = len
;
8431 /* Push terminal marker - zero. */
8432 args
.safe_push (oacc_launch_pack (0, NULL_TREE
, 0));
8434 g
= gimple_build_call_vec (builtin_decl_explicit (start_ix
), args
);
8435 gimple_set_location (g
, gimple_location (entry_stmt
));
8436 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
8440 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
8441 gsi_remove (&gsi
, true);
8443 if (data_region
&& region
->exit
)
8445 gsi
= gsi_last_nondebug_bb (region
->exit
);
8447 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
8448 gsi_remove (&gsi
, true);
8452 /* Expand KFOR loop as a HSA grifidied kernel, i.e. as a body only with
8453 iteration variable derived from the thread number. INTRA_GROUP means this
8454 is an expansion of a loop iterating over work-items within a separate
8455 iteration over groups. */
8458 grid_expand_omp_for_loop (struct omp_region
*kfor
, bool intra_group
)
8460 gimple_stmt_iterator gsi
;
8461 gomp_for
*for_stmt
= as_a
<gomp_for
*> (last_stmt (kfor
->entry
));
8462 gcc_checking_assert (gimple_omp_for_kind (for_stmt
)
8463 == GF_OMP_FOR_KIND_GRID_LOOP
);
8464 size_t collapse
= gimple_omp_for_collapse (for_stmt
);
8465 struct omp_for_data_loop
*loops
8466 = XALLOCAVEC (struct omp_for_data_loop
,
8467 gimple_omp_for_collapse (for_stmt
));
8468 struct omp_for_data fd
;
8470 remove_edge (BRANCH_EDGE (kfor
->entry
));
8471 basic_block body_bb
= FALLTHRU_EDGE (kfor
->entry
)->dest
;
8473 gcc_assert (kfor
->cont
);
8474 omp_extract_for_data (for_stmt
, &fd
, loops
);
8476 gsi
= gsi_start_bb (body_bb
);
8478 for (size_t dim
= 0; dim
< collapse
; dim
++)
8481 itype
= type
= TREE_TYPE (fd
.loops
[dim
].v
);
8482 if (POINTER_TYPE_P (type
))
8483 itype
= signed_type_for (type
);
8485 tree n1
= fd
.loops
[dim
].n1
;
8486 tree step
= fd
.loops
[dim
].step
;
8487 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
8488 true, NULL_TREE
, true, GSI_SAME_STMT
);
8489 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
8490 true, NULL_TREE
, true, GSI_SAME_STMT
);
8492 if (gimple_omp_for_grid_group_iter (for_stmt
))
8494 gcc_checking_assert (!intra_group
);
8495 threadid
= build_call_expr (builtin_decl_explicit
8496 (BUILT_IN_HSA_WORKGROUPID
), 1,
8497 build_int_cstu (unsigned_type_node
, dim
));
8499 else if (intra_group
)
8500 threadid
= build_call_expr (builtin_decl_explicit
8501 (BUILT_IN_HSA_WORKITEMID
), 1,
8502 build_int_cstu (unsigned_type_node
, dim
));
8504 threadid
= build_call_expr (builtin_decl_explicit
8505 (BUILT_IN_HSA_WORKITEMABSID
), 1,
8506 build_int_cstu (unsigned_type_node
, dim
));
8507 threadid
= fold_convert (itype
, threadid
);
8508 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
8509 true, GSI_SAME_STMT
);
8511 tree startvar
= fd
.loops
[dim
].v
;
8512 tree t
= fold_build2 (MULT_EXPR
, itype
, threadid
, step
);
8513 if (POINTER_TYPE_P (type
))
8514 t
= fold_build_pointer_plus (n1
, t
);
8516 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
8517 t
= fold_convert (type
, t
);
8518 t
= force_gimple_operand_gsi (&gsi
, t
,
8520 && TREE_ADDRESSABLE (startvar
),
8521 NULL_TREE
, true, GSI_SAME_STMT
);
8522 gassign
*assign_stmt
= gimple_build_assign (startvar
, t
);
8523 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
8525 /* Remove the omp for statement. */
8526 gsi
= gsi_last_nondebug_bb (kfor
->entry
);
8527 gsi_remove (&gsi
, true);
8529 /* Remove the GIMPLE_OMP_CONTINUE statement. */
8530 gsi
= gsi_last_nondebug_bb (kfor
->cont
);
8531 gcc_assert (!gsi_end_p (gsi
)
8532 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_CONTINUE
);
8533 gsi_remove (&gsi
, true);
8535 /* Replace the GIMPLE_OMP_RETURN with a barrier, if necessary. */
8536 gsi
= gsi_last_nondebug_bb (kfor
->exit
);
8537 gcc_assert (!gsi_end_p (gsi
)
8538 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
8540 gsi_insert_before (&gsi
, omp_build_barrier (NULL_TREE
), GSI_SAME_STMT
);
8541 gsi_remove (&gsi
, true);
8543 /* Fixup the much simpler CFG. */
8544 remove_edge (find_edge (kfor
->cont
, body_bb
));
8546 if (kfor
->cont
!= body_bb
)
8547 set_immediate_dominator (CDI_DOMINATORS
, kfor
->cont
, body_bb
);
8548 set_immediate_dominator (CDI_DOMINATORS
, kfor
->exit
, kfor
->cont
);
8551 /* Structure passed to grid_remap_kernel_arg_accesses so that it can remap
8554 struct grid_arg_decl_map
8560 /* Invoked through walk_gimple_op, will remap all PARM_DECLs to the ones
8561 pertaining to kernel function. */
8564 grid_remap_kernel_arg_accesses (tree
*tp
, int *walk_subtrees
, void *data
)
8566 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
8567 struct grid_arg_decl_map
*adm
= (struct grid_arg_decl_map
*) wi
->info
;
8570 if (t
== adm
->old_arg
)
8572 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
8576 /* If TARGET region contains a kernel body for loop, remove its region from the
8577 TARGET and expand it in HSA gridified kernel fashion. */
8580 grid_expand_target_grid_body (struct omp_region
*target
)
8582 if (!hsa_gen_requested_p ())
8585 gomp_target
*tgt_stmt
= as_a
<gomp_target
*> (last_stmt (target
->entry
));
8586 struct omp_region
**pp
;
8588 for (pp
= &target
->inner
; *pp
; pp
= &(*pp
)->next
)
8589 if ((*pp
)->type
== GIMPLE_OMP_GRID_BODY
)
8592 struct omp_region
*gpukernel
= *pp
;
8594 tree orig_child_fndecl
= gimple_omp_target_child_fn (tgt_stmt
);
8597 /* HSA cannot handle OACC stuff. */
8598 if (gimple_omp_target_kind (tgt_stmt
) != GF_OMP_TARGET_KIND_REGION
)
8600 gcc_checking_assert (orig_child_fndecl
);
8601 gcc_assert (!omp_find_clause (gimple_omp_target_clauses (tgt_stmt
),
8602 OMP_CLAUSE__GRIDDIM_
));
8603 cgraph_node
*n
= cgraph_node::get (orig_child_fndecl
);
8605 hsa_register_kernel (n
);
8609 gcc_assert (omp_find_clause (gimple_omp_target_clauses (tgt_stmt
),
8610 OMP_CLAUSE__GRIDDIM_
));
8612 = gimple_block (first_stmt (single_succ (gpukernel
->entry
)));
8613 *pp
= gpukernel
->next
;
8614 for (pp
= &gpukernel
->inner
; *pp
; pp
= &(*pp
)->next
)
8615 if ((*pp
)->type
== GIMPLE_OMP_FOR
)
8618 struct omp_region
*kfor
= *pp
;
8620 gomp_for
*for_stmt
= as_a
<gomp_for
*> (last_stmt (kfor
->entry
));
8621 gcc_assert (gimple_omp_for_kind (for_stmt
) == GF_OMP_FOR_KIND_GRID_LOOP
);
8625 if (gimple_omp_for_grid_group_iter (for_stmt
))
8627 struct omp_region
**next_pp
;
8628 for (pp
= &kfor
->inner
; *pp
; pp
= next_pp
)
8630 next_pp
= &(*pp
)->next
;
8631 if ((*pp
)->type
!= GIMPLE_OMP_FOR
)
8633 gomp_for
*inner
= as_a
<gomp_for
*> (last_stmt ((*pp
)->entry
));
8634 gcc_assert (gimple_omp_for_kind (inner
)
8635 == GF_OMP_FOR_KIND_GRID_LOOP
);
8636 grid_expand_omp_for_loop (*pp
, true);
8641 expand_omp (kfor
->inner
);
8643 if (gpukernel
->inner
)
8644 expand_omp (gpukernel
->inner
);
8646 tree kern_fndecl
= copy_node (orig_child_fndecl
);
8647 DECL_NAME (kern_fndecl
) = clone_function_name_numbered (kern_fndecl
,
8649 SET_DECL_ASSEMBLER_NAME (kern_fndecl
, DECL_NAME (kern_fndecl
));
8650 tree tgtblock
= gimple_block (tgt_stmt
);
8651 tree fniniblock
= make_node (BLOCK
);
8652 BLOCK_ABSTRACT_ORIGIN (fniniblock
) = BLOCK_ORIGIN (tgtblock
);
8653 BLOCK_SOURCE_LOCATION (fniniblock
) = BLOCK_SOURCE_LOCATION (tgtblock
);
8654 BLOCK_SOURCE_END_LOCATION (fniniblock
) = BLOCK_SOURCE_END_LOCATION (tgtblock
);
8655 BLOCK_SUPERCONTEXT (fniniblock
) = kern_fndecl
;
8656 DECL_INITIAL (kern_fndecl
) = fniniblock
;
8657 push_struct_function (kern_fndecl
);
8658 cfun
->function_end_locus
= gimple_location (tgt_stmt
);
8659 init_tree_ssa (cfun
);
8662 tree old_parm_decl
= DECL_ARGUMENTS (kern_fndecl
);
8663 gcc_assert (!DECL_CHAIN (old_parm_decl
));
8664 tree new_parm_decl
= copy_node (DECL_ARGUMENTS (kern_fndecl
));
8665 DECL_CONTEXT (new_parm_decl
) = kern_fndecl
;
8666 DECL_ARGUMENTS (kern_fndecl
) = new_parm_decl
;
8667 gcc_assert (VOID_TYPE_P (TREE_TYPE (DECL_RESULT (kern_fndecl
))));
8668 DECL_RESULT (kern_fndecl
) = copy_node (DECL_RESULT (kern_fndecl
));
8669 DECL_CONTEXT (DECL_RESULT (kern_fndecl
)) = kern_fndecl
;
8670 struct function
*kern_cfun
= DECL_STRUCT_FUNCTION (kern_fndecl
);
8671 kern_cfun
->curr_properties
= cfun
->curr_properties
;
8673 grid_expand_omp_for_loop (kfor
, false);
8675 /* Remove the omp for statement. */
8676 gimple_stmt_iterator gsi
= gsi_last_nondebug_bb (gpukernel
->entry
);
8677 gsi_remove (&gsi
, true);
8678 /* Replace the GIMPLE_OMP_RETURN at the end of the kernel region with a real
8680 gsi
= gsi_last_nondebug_bb (gpukernel
->exit
);
8681 gcc_assert (!gsi_end_p (gsi
)
8682 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
8683 gimple
*ret_stmt
= gimple_build_return (NULL
);
8684 gsi_insert_after (&gsi
, ret_stmt
, GSI_SAME_STMT
);
8685 gsi_remove (&gsi
, true);
8687 /* Statements in the first BB in the target construct have been produced by
8688 target lowering and must be copied inside the GPUKERNEL, with the two
8689 exceptions of the first OMP statement and the OMP_DATA assignment
8691 gsi
= gsi_start_bb (single_succ (gpukernel
->entry
));
8692 tree data_arg
= gimple_omp_target_data_arg (tgt_stmt
);
8693 tree sender
= data_arg
? TREE_VEC_ELT (data_arg
, 0) : NULL
;
8694 for (gimple_stmt_iterator tsi
= gsi_start_bb (single_succ (target
->entry
));
8695 !gsi_end_p (tsi
); gsi_next (&tsi
))
8697 gimple
*stmt
= gsi_stmt (tsi
);
8698 if (is_gimple_omp (stmt
))
8701 && is_gimple_assign (stmt
)
8702 && TREE_CODE (gimple_assign_rhs1 (stmt
)) == ADDR_EXPR
8703 && TREE_OPERAND (gimple_assign_rhs1 (stmt
), 0) == sender
)
8705 gimple
*copy
= gimple_copy (stmt
);
8706 gsi_insert_before (&gsi
, copy
, GSI_SAME_STMT
);
8707 gimple_set_block (copy
, fniniblock
);
8710 move_sese_region_to_fn (kern_cfun
, single_succ (gpukernel
->entry
),
8711 gpukernel
->exit
, inside_block
);
8713 cgraph_node
*kcn
= cgraph_node::get_create (kern_fndecl
);
8714 kcn
->mark_force_output ();
8715 cgraph_node
*orig_child
= cgraph_node::get (orig_child_fndecl
);
8717 hsa_register_kernel (kcn
, orig_child
);
8719 cgraph_node::add_new_function (kern_fndecl
, true);
8720 push_cfun (kern_cfun
);
8721 cgraph_edge::rebuild_edges ();
8723 /* Re-map any mention of the PARM_DECL of the original function to the
8724 PARM_DECL of the new one.
8726 TODO: It would be great if lowering produced references into the GPU
8727 kernel decl straight away and we did not have to do this. */
8728 struct grid_arg_decl_map adm
;
8729 adm
.old_arg
= old_parm_decl
;
8730 adm
.new_arg
= new_parm_decl
;
8732 FOR_EACH_BB_FN (bb
, kern_cfun
)
8734 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
8736 gimple
*stmt
= gsi_stmt (gsi
);
8737 struct walk_stmt_info wi
;
8738 memset (&wi
, 0, sizeof (wi
));
8740 walk_gimple_op (stmt
, grid_remap_kernel_arg_accesses
, &wi
);
8748 /* Expand the parallel region tree rooted at REGION. Expansion
8749 proceeds in depth-first order. Innermost regions are expanded
8750 first. This way, parallel regions that require a new function to
8751 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
8752 internal dependencies in their body. */
8755 expand_omp (struct omp_region
*region
)
8757 omp_any_child_fn_dumped
= false;
8760 location_t saved_location
;
8761 gimple
*inner_stmt
= NULL
;
8763 /* First, determine whether this is a combined parallel+workshare
8765 if (region
->type
== GIMPLE_OMP_PARALLEL
)
8766 determine_parallel_type (region
);
8767 else if (region
->type
== GIMPLE_OMP_TARGET
)
8768 grid_expand_target_grid_body (region
);
8770 if (region
->type
== GIMPLE_OMP_FOR
8771 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
8772 inner_stmt
= last_stmt (region
->inner
->entry
);
8775 expand_omp (region
->inner
);
8777 saved_location
= input_location
;
8778 if (gimple_has_location (last_stmt (region
->entry
)))
8779 input_location
= gimple_location (last_stmt (region
->entry
));
8781 switch (region
->type
)
8783 case GIMPLE_OMP_PARALLEL
:
8784 case GIMPLE_OMP_TASK
:
8785 expand_omp_taskreg (region
);
8788 case GIMPLE_OMP_FOR
:
8789 expand_omp_for (region
, inner_stmt
);
8792 case GIMPLE_OMP_SECTIONS
:
8793 expand_omp_sections (region
);
8796 case GIMPLE_OMP_SECTION
:
8797 /* Individual omp sections are handled together with their
8798 parent GIMPLE_OMP_SECTIONS region. */
8801 case GIMPLE_OMP_SINGLE
:
8802 expand_omp_single (region
);
8805 case GIMPLE_OMP_ORDERED
:
8807 gomp_ordered
*ord_stmt
8808 = as_a
<gomp_ordered
*> (last_stmt (region
->entry
));
8809 if (omp_find_clause (gimple_omp_ordered_clauses (ord_stmt
),
8812 /* We'll expand these when expanding corresponding
8813 worksharing region with ordered(n) clause. */
8814 gcc_assert (region
->outer
8815 && region
->outer
->type
== GIMPLE_OMP_FOR
);
8816 region
->ord_stmt
= ord_stmt
;
8821 case GIMPLE_OMP_MASTER
:
8822 case GIMPLE_OMP_TASKGROUP
:
8823 case GIMPLE_OMP_CRITICAL
:
8824 case GIMPLE_OMP_TEAMS
:
8825 expand_omp_synch (region
);
8828 case GIMPLE_OMP_ATOMIC_LOAD
:
8829 expand_omp_atomic (region
);
8832 case GIMPLE_OMP_TARGET
:
8833 expand_omp_target (region
);
8840 input_location
= saved_location
;
8841 region
= region
->next
;
8843 if (omp_any_child_fn_dumped
)
8846 dump_function_header (dump_file
, current_function_decl
, dump_flags
);
8847 omp_any_child_fn_dumped
= false;
8851 /* Helper for build_omp_regions. Scan the dominator tree starting at
8852 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
8853 true, the function ends once a single tree is built (otherwise, whole
8854 forest of OMP constructs may be built). */
8857 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
8860 gimple_stmt_iterator gsi
;
8864 gsi
= gsi_last_nondebug_bb (bb
);
8865 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
8867 struct omp_region
*region
;
8868 enum gimple_code code
;
8870 stmt
= gsi_stmt (gsi
);
8871 code
= gimple_code (stmt
);
8872 if (code
== GIMPLE_OMP_RETURN
)
8874 /* STMT is the return point out of region PARENT. Mark it
8875 as the exit point and make PARENT the immediately
8876 enclosing region. */
8877 gcc_assert (parent
);
8880 parent
= parent
->outer
;
8882 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
8884 /* GIMPLE_OMP_ATOMIC_STORE is analogous to
8885 GIMPLE_OMP_RETURN, but matches with
8886 GIMPLE_OMP_ATOMIC_LOAD. */
8887 gcc_assert (parent
);
8888 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
8891 parent
= parent
->outer
;
8893 else if (code
== GIMPLE_OMP_CONTINUE
)
8895 gcc_assert (parent
);
8898 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
8900 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
8901 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
8905 region
= new_omp_region (bb
, code
, parent
);
8907 if (code
== GIMPLE_OMP_TARGET
)
8909 switch (gimple_omp_target_kind (stmt
))
8911 case GF_OMP_TARGET_KIND_REGION
:
8912 case GF_OMP_TARGET_KIND_DATA
:
8913 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
8914 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
8915 case GF_OMP_TARGET_KIND_OACC_DATA
:
8916 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
8918 case GF_OMP_TARGET_KIND_UPDATE
:
8919 case GF_OMP_TARGET_KIND_ENTER_DATA
:
8920 case GF_OMP_TARGET_KIND_EXIT_DATA
:
8921 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
8922 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
8923 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
8924 /* ..., other than for those stand-alone directives... */
8931 else if (code
== GIMPLE_OMP_ORDERED
8932 && omp_find_clause (gimple_omp_ordered_clauses
8933 (as_a
<gomp_ordered
*> (stmt
)),
8935 /* #pragma omp ordered depend is also just a stand-alone
8938 else if (code
== GIMPLE_OMP_TASK
8939 && gimple_omp_task_taskwait_p (stmt
))
8940 /* #pragma omp taskwait depend(...) is a stand-alone directive. */
8942 /* ..., this directive becomes the parent for a new region. */
8948 if (single_tree
&& !parent
)
8951 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
8953 son
= next_dom_son (CDI_DOMINATORS
, son
))
8954 build_omp_regions_1 (son
, parent
, single_tree
);
8957 /* Builds the tree of OMP regions rooted at ROOT, storing it to
8961 build_omp_regions_root (basic_block root
)
8963 gcc_assert (root_omp_region
== NULL
);
8964 build_omp_regions_1 (root
, NULL
, true);
8965 gcc_assert (root_omp_region
!= NULL
);
8968 /* Expands omp construct (and its subconstructs) starting in HEAD. */
8971 omp_expand_local (basic_block head
)
8973 build_omp_regions_root (head
);
8974 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
8976 fprintf (dump_file
, "\nOMP region tree\n\n");
8977 dump_omp_region (dump_file
, root_omp_region
, 0);
8978 fprintf (dump_file
, "\n");
8981 remove_exit_barriers (root_omp_region
);
8982 expand_omp (root_omp_region
);
8984 omp_free_regions ();
8987 /* Scan the CFG and build a tree of OMP regions. Return the root of
8988 the OMP region tree. */
8991 build_omp_regions (void)
8993 gcc_assert (root_omp_region
== NULL
);
8994 calculate_dominance_info (CDI_DOMINATORS
);
8995 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
8998 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
9001 execute_expand_omp (void)
9003 build_omp_regions ();
9005 if (!root_omp_region
)
9010 fprintf (dump_file
, "\nOMP region tree\n\n");
9011 dump_omp_region (dump_file
, root_omp_region
, 0);
9012 fprintf (dump_file
, "\n");
9015 remove_exit_barriers (root_omp_region
);
9017 expand_omp (root_omp_region
);
9019 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
9020 verify_loop_structure ();
9021 cleanup_tree_cfg ();
9023 omp_free_regions ();
9028 /* OMP expansion -- the default pass, run before creation of SSA form. */
9032 const pass_data pass_data_expand_omp
=
9034 GIMPLE_PASS
, /* type */
9035 "ompexp", /* name */
9036 OPTGROUP_OMP
, /* optinfo_flags */
9037 TV_NONE
, /* tv_id */
9038 PROP_gimple_any
, /* properties_required */
9039 PROP_gimple_eomp
, /* properties_provided */
9040 0, /* properties_destroyed */
9041 0, /* todo_flags_start */
9042 0, /* todo_flags_finish */
9045 class pass_expand_omp
: public gimple_opt_pass
9048 pass_expand_omp (gcc::context
*ctxt
)
9049 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
9052 /* opt_pass methods: */
9053 virtual unsigned int execute (function
*)
9055 bool gate
= ((flag_openacc
!= 0 || flag_openmp
!= 0
9056 || flag_openmp_simd
!= 0)
9059 /* This pass always runs, to provide PROP_gimple_eomp.
9060 But often, there is nothing to do. */
9064 return execute_expand_omp ();
9067 }; // class pass_expand_omp
9072 make_pass_expand_omp (gcc::context
*ctxt
)
9074 return new pass_expand_omp (ctxt
);
9079 const pass_data pass_data_expand_omp_ssa
=
9081 GIMPLE_PASS
, /* type */
9082 "ompexpssa", /* name */
9083 OPTGROUP_OMP
, /* optinfo_flags */
9084 TV_NONE
, /* tv_id */
9085 PROP_cfg
| PROP_ssa
, /* properties_required */
9086 PROP_gimple_eomp
, /* properties_provided */
9087 0, /* properties_destroyed */
9088 0, /* todo_flags_start */
9089 TODO_cleanup_cfg
| TODO_rebuild_alias
, /* todo_flags_finish */
9092 class pass_expand_omp_ssa
: public gimple_opt_pass
9095 pass_expand_omp_ssa (gcc::context
*ctxt
)
9096 : gimple_opt_pass (pass_data_expand_omp_ssa
, ctxt
)
9099 /* opt_pass methods: */
9100 virtual bool gate (function
*fun
)
9102 return !(fun
->curr_properties
& PROP_gimple_eomp
);
9104 virtual unsigned int execute (function
*) { return execute_expand_omp (); }
9105 opt_pass
* clone () { return new pass_expand_omp_ssa (m_ctxt
); }
9107 }; // class pass_expand_omp_ssa
9112 make_pass_expand_omp_ssa (gcc::context
*ctxt
)
9114 return new pass_expand_omp_ssa (ctxt
);
9117 /* Called from tree-cfg.c::make_edges to create cfg edges for all relevant
9121 omp_make_gimple_edges (basic_block bb
, struct omp_region
**region
,
9124 gimple
*last
= last_stmt (bb
);
9125 enum gimple_code code
= gimple_code (last
);
9126 struct omp_region
*cur_region
= *region
;
9127 bool fallthru
= false;
9131 case GIMPLE_OMP_PARALLEL
:
9132 case GIMPLE_OMP_FOR
:
9133 case GIMPLE_OMP_SINGLE
:
9134 case GIMPLE_OMP_TEAMS
:
9135 case GIMPLE_OMP_MASTER
:
9136 case GIMPLE_OMP_TASKGROUP
:
9137 case GIMPLE_OMP_CRITICAL
:
9138 case GIMPLE_OMP_SECTION
:
9139 case GIMPLE_OMP_GRID_BODY
:
9140 cur_region
= new_omp_region (bb
, code
, cur_region
);
9144 case GIMPLE_OMP_TASK
:
9145 cur_region
= new_omp_region (bb
, code
, cur_region
);
9147 if (gimple_omp_task_taskwait_p (last
))
9148 cur_region
= cur_region
->outer
;
9151 case GIMPLE_OMP_ORDERED
:
9152 cur_region
= new_omp_region (bb
, code
, cur_region
);
9154 if (omp_find_clause (gimple_omp_ordered_clauses
9155 (as_a
<gomp_ordered
*> (last
)),
9157 cur_region
= cur_region
->outer
;
9160 case GIMPLE_OMP_TARGET
:
9161 cur_region
= new_omp_region (bb
, code
, cur_region
);
9163 switch (gimple_omp_target_kind (last
))
9165 case GF_OMP_TARGET_KIND_REGION
:
9166 case GF_OMP_TARGET_KIND_DATA
:
9167 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
9168 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
9169 case GF_OMP_TARGET_KIND_OACC_DATA
:
9170 case GF_OMP_TARGET_KIND_OACC_HOST_DATA
:
9172 case GF_OMP_TARGET_KIND_UPDATE
:
9173 case GF_OMP_TARGET_KIND_ENTER_DATA
:
9174 case GF_OMP_TARGET_KIND_EXIT_DATA
:
9175 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
9176 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
9177 case GF_OMP_TARGET_KIND_OACC_DECLARE
:
9178 cur_region
= cur_region
->outer
;
9185 case GIMPLE_OMP_SECTIONS
:
9186 cur_region
= new_omp_region (bb
, code
, cur_region
);
9190 case GIMPLE_OMP_SECTIONS_SWITCH
:
9194 case GIMPLE_OMP_ATOMIC_LOAD
:
9195 case GIMPLE_OMP_ATOMIC_STORE
:
9199 case GIMPLE_OMP_RETURN
:
9200 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
9201 somewhere other than the next block. This will be
9203 cur_region
->exit
= bb
;
9204 if (cur_region
->type
== GIMPLE_OMP_TASK
)
9205 /* Add an edge corresponding to not scheduling the task
9207 make_edge (cur_region
->entry
, bb
, EDGE_ABNORMAL
);
9208 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
9209 cur_region
= cur_region
->outer
;
9212 case GIMPLE_OMP_CONTINUE
:
9213 cur_region
->cont
= bb
;
9214 switch (cur_region
->type
)
9216 case GIMPLE_OMP_FOR
:
9217 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
9218 succs edges as abnormal to prevent splitting
9220 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
9221 /* Make the loopback edge. */
9222 make_edge (bb
, single_succ (cur_region
->entry
),
9225 /* Create an edge from GIMPLE_OMP_FOR to exit, which
9226 corresponds to the case that the body of the loop
9227 is not executed at all. */
9228 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
9229 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
9233 case GIMPLE_OMP_SECTIONS
:
9234 /* Wire up the edges into and out of the nested sections. */
9236 basic_block switch_bb
= single_succ (cur_region
->entry
);
9238 struct omp_region
*i
;
9239 for (i
= cur_region
->inner
; i
; i
= i
->next
)
9241 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
9242 make_edge (switch_bb
, i
->entry
, 0);
9243 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
9246 /* Make the loopback edge to the block with
9247 GIMPLE_OMP_SECTIONS_SWITCH. */
9248 make_edge (bb
, switch_bb
, 0);
9250 /* Make the edge from the switch to exit. */
9251 make_edge (switch_bb
, bb
->next_bb
, 0);
9256 case GIMPLE_OMP_TASK
:
9269 if (*region
!= cur_region
)
9271 *region
= cur_region
;
9273 *region_idx
= cur_region
->entry
->index
;
9281 #include "gt-omp-expand.h"