directives to separate functions, converts others into explicit calls to the
runtime library (libgomp) and so forth
-Copyright (C) 2005-2020 Free Software Foundation, Inc.
+Copyright (C) 2005-2021 Free Software Foundation, Inc.
This file is part of GCC.
tree depend = omp_find_clause (clauses, OMP_CLAUSE_DEPEND);
tree finalc = omp_find_clause (clauses, OMP_CLAUSE_FINAL);
tree priority = omp_find_clause (clauses, OMP_CLAUSE_PRIORITY);
+ tree detach = omp_find_clause (clauses, OMP_CLAUSE_DETACH);
unsigned int iflags
= (untied ? GOMP_TASK_FLAG_UNTIED : 0)
tree tclauses = gimple_omp_for_clauses (g);
num_tasks = omp_find_clause (tclauses, OMP_CLAUSE_NUM_TASKS);
if (num_tasks)
- num_tasks = OMP_CLAUSE_NUM_TASKS_EXPR (num_tasks);
+ {
+ if (OMP_CLAUSE_NUM_TASKS_STRICT (num_tasks))
+ iflags |= GOMP_TASK_FLAG_STRICT;
+ num_tasks = OMP_CLAUSE_NUM_TASKS_EXPR (num_tasks);
+ }
else
{
num_tasks = omp_find_clause (tclauses, OMP_CLAUSE_GRAINSIZE);
if (num_tasks)
{
iflags |= GOMP_TASK_FLAG_GRAINSIZE;
+ if (OMP_CLAUSE_GRAINSIZE_STRICT (num_tasks))
+ iflags |= GOMP_TASK_FLAG_STRICT;
num_tasks = OMP_CLAUSE_GRAINSIZE_EXPR (num_tasks);
}
else
if (omp_find_clause (clauses, OMP_CLAUSE_REDUCTION))
iflags |= GOMP_TASK_FLAG_REDUCTION;
}
- else if (priority)
- iflags |= GOMP_TASK_FLAG_PRIORITY;
+ else
+ {
+ if (priority)
+ iflags |= GOMP_TASK_FLAG_PRIORITY;
+ if (detach)
+ iflags |= GOMP_TASK_FLAG_DETACH;
+ }
tree flags = build_int_cst (unsigned_type_node, iflags);
priority = integer_zero_node;
gsi = gsi_last_nondebug_bb (bb);
+
+ detach = (detach
+ ? build_fold_addr_expr (OMP_CLAUSE_DECL (detach))
+ : null_pointer_node);
+
tree t = gimple_omp_task_data_arg (entry_stmt);
if (t == NULL)
t2 = null_pointer_node;
num_tasks, priority, startvar, endvar, step);
else
t = build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK),
- 9, t1, t2, t3,
+ 10, t1, t2, t3,
gimple_omp_task_arg_size (entry_stmt),
gimple_omp_task_arg_align (entry_stmt), cond, flags,
- depend, priority);
+ depend, priority, detach);
force_gimple_operand_gsi (&gsi, t, true, NULL_TREE,
false, GSI_CONTINUE_LINKING);
static tree
expand_oacc_collapse_init (const struct omp_for_data *fd,
gimple_stmt_iterator *gsi,
- oacc_collapse *counts, tree bound_type,
- location_t loc)
+ oacc_collapse *counts, tree diff_type,
+ tree bound_type, location_t loc)
{
tree tiling = fd->tiling;
tree total = build_int_cst (bound_type, 1);
const omp_for_data_loop *loop = &fd->loops[ix];
tree iter_type = TREE_TYPE (loop->v);
- tree diff_type = iter_type;
tree plus_type = iter_type;
- gcc_assert (loop->cond_code == fd->loop.cond_code);
+ gcc_assert (loop->cond_code == LT_EXPR || loop->cond_code == GT_EXPR);
if (POINTER_TYPE_P (iter_type))
plus_type = sizetype;
- if (POINTER_TYPE_P (diff_type) || TYPE_UNSIGNED (diff_type))
- diff_type = signed_type_for (diff_type);
- if (TYPE_PRECISION (diff_type) < TYPE_PRECISION (integer_type_node))
- diff_type = integer_type_node;
if (tiling)
{
static void
expand_oacc_collapse_vars (const struct omp_for_data *fd, bool inner,
gimple_stmt_iterator *gsi,
- const oacc_collapse *counts, tree ivar)
+ const oacc_collapse *counts, tree ivar,
+ tree diff_type)
{
tree ivar_type = TREE_TYPE (ivar);
const oacc_collapse *collapse = &counts[ix];
tree v = inner ? loop->v : collapse->outer;
tree iter_type = TREE_TYPE (v);
- tree diff_type = TREE_TYPE (collapse->step);
tree plus_type = iter_type;
enum tree_code plus_code = PLUS_EXPR;
tree expr;
}
expr = fold_build2 (MULT_EXPR, diff_type, fold_convert (diff_type, expr),
- collapse->step);
+ fold_convert (diff_type, collapse->step));
expr = fold_build2 (plus_code, iter_type,
inner ? collapse->outer : collapse->base,
fold_convert (plus_type, expr));
&& (TREE_CODE (fd->loop.n2) == INTEGER_CST
|| fd->first_inner_iterations)
&& (optab_handler (sqrt_optab, TYPE_MODE (double_type_node))
- != CODE_FOR_nothing))
+ != CODE_FOR_nothing)
+ && !integer_zerop (fd->loop.n2))
{
tree outer_n1 = fd->adjn1 ? fd->adjn1 : fd->loops[i - 1].n1;
tree itype = TREE_TYPE (fd->loops[i].v);
for (i = first_zero_iter1;
i < (fd->ordered ? fd->ordered : fd->collapse); i++)
if (SSA_VAR_P (counts[i]))
- TREE_NO_WARNING (counts[i]) = 1;
+ suppress_warning (counts[i], OPT_Wuninitialized);
gsi_prev (&gsi);
e = split_block (entry_bb, gsi_stmt (gsi));
entry_bb = e->dest;
be executed in that case, so just avoid uninit warnings. */
for (i = first_zero_iter2; i < fd->ordered; i++)
if (SSA_VAR_P (counts[i]))
- TREE_NO_WARNING (counts[i]) = 1;
+ suppress_warning (counts[i], OPT_Wuninitialized);
if (zero_iter1_bb)
make_edge (zero_iter2_bb, entry_bb, EDGE_FALLTHRU);
else
&& !OMP_CLAUSE_LINEAR_NO_COPYIN (c))
{
tree d = OMP_CLAUSE_DECL (c);
- bool is_ref = omp_is_reference (d);
tree t = d, a, dest;
- if (is_ref)
+ if (omp_privatize_by_reference (t))
t = build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c), t);
tree type = TREE_TYPE (t);
if (POINTER_TYPE_P (type))
gsi = gsi_last_bb (l0_bb);
expand_omp_build_assign (&gsi, counts[fd->collapse - 1],
istart0, true);
- gsi = gsi_last_bb (cont_bb);
- t = fold_build2 (PLUS_EXPR, fd->iter_type, counts[fd->collapse - 1],
- build_int_cst (fd->iter_type, 1));
- expand_omp_build_assign (&gsi, counts[fd->collapse - 1], t);
- tree aref = build4 (ARRAY_REF, fd->iter_type, counts[fd->ordered],
- size_zero_node, NULL_TREE, NULL_TREE);
- expand_omp_build_assign (&gsi, aref, counts[fd->collapse - 1]);
+ if (cont_bb)
+ {
+ gsi = gsi_last_bb (cont_bb);
+ t = fold_build2 (PLUS_EXPR, fd->iter_type,
+ counts[fd->collapse - 1],
+ build_int_cst (fd->iter_type, 1));
+ expand_omp_build_assign (&gsi, counts[fd->collapse - 1], t);
+ tree aref = build4 (ARRAY_REF, fd->iter_type,
+ counts[fd->ordered], size_zero_node,
+ NULL_TREE, NULL_TREE);
+ expand_omp_build_assign (&gsi, aref, counts[fd->collapse - 1]);
+ }
t = counts[fd->collapse - 1];
}
else if (fd->collapse > 1)
&& !OMP_CLAUSE_LINEAR_NO_COPYIN (c))
{
tree d = OMP_CLAUSE_DECL (c);
- bool is_ref = omp_is_reference (d);
tree t = d, a, dest;
- if (is_ref)
+ if (omp_privatize_by_reference (t))
t = build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c), t);
if (itercnt == NULL_TREE)
{
&& !OMP_CLAUSE_LINEAR_NO_COPYIN (c))
{
tree d = OMP_CLAUSE_DECL (c);
- bool is_ref = omp_is_reference (d);
tree t = d, a, dest;
- if (is_ref)
+ if (omp_privatize_by_reference (t))
t = build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c), t);
tree type = TREE_TYPE (t);
if (POINTER_TYPE_P (type))
n2 = OMP_CLAUSE_DECL (innerc);
}
tree step = fd->loop.step;
+ tree orig_step = step; /* May be different from step if is_simt. */
bool is_simt = omp_find_clause (gimple_omp_for_clauses (fd->for_stmt),
OMP_CLAUSE__SIMT_);
tree altv = NULL_TREE, altn2 = NULL_TREE;
if (fd->collapse == 1
&& !broken_loop
- && TREE_CODE (fd->loops[0].step) != INTEGER_CST)
+ && TREE_CODE (orig_step) != INTEGER_CST)
{
/* The vectorizer currently punts on loops with non-constant steps
for the main IV (can't compute number of iterations and gives up
itype = signed_type_for (itype);
t = build_int_cst (itype, (fd->loop.cond_code == LT_EXPR ? -1 : 1));
t = fold_build2 (PLUS_EXPR, itype,
- fold_convert (itype, fd->loop.step), t);
+ fold_convert (itype, step), t);
t = fold_build2 (PLUS_EXPR, itype, t, fold_convert (itype, n2));
t = fold_build2 (MINUS_EXPR, itype, t,
fold_convert (itype, fd->loop.v));
t = fold_build2 (TRUNC_DIV_EXPR, itype,
fold_build1 (NEGATE_EXPR, itype, t),
fold_build1 (NEGATE_EXPR, itype,
- fold_convert (itype, fd->loop.step)));
+ fold_convert (itype, step)));
else
t = fold_build2 (TRUNC_DIV_EXPR, itype, t,
- fold_convert (itype, fd->loop.step));
+ fold_convert (itype, step));
t = fold_convert (TREE_TYPE (altv), t);
altn2 = create_tmp_var (TREE_TYPE (altv));
expand_omp_build_assign (&gsi, altn2, t);
if (is_simt)
{
gsi = gsi_start_bb (l2_bb);
- step = fold_build2 (MINUS_EXPR, TREE_TYPE (step), fd->loop.step, step);
+ step = fold_build2 (MINUS_EXPR, TREE_TYPE (step), orig_step, step);
if (POINTER_TYPE_P (type))
t = fold_build_pointer_plus (fd->loop.v, step);
else
be executed in that case, so just avoid uninit warnings. */
for (i = first_zero_iter; i < fd->collapse; i++)
if (SSA_VAR_P (counts[i]))
- TREE_NO_WARNING (counts[i]) = 1;
+ suppress_warning (counts[i], OPT_Wuninitialized);
gsi_prev (&gsi);
edge e = split_block (entry_bb, gsi_stmt (gsi));
entry_bb = e->dest;
static void
expand_oacc_for (struct omp_region *region, struct omp_for_data *fd)
{
+ bool is_oacc_kernels_parallelized
+ = (lookup_attribute ("oacc kernels parallelized",
+ DECL_ATTRIBUTES (current_function_decl)) != NULL);
+ {
+ bool is_oacc_kernels
+ = (lookup_attribute ("oacc kernels",
+ DECL_ATTRIBUTES (current_function_decl)) != NULL);
+ if (is_oacc_kernels_parallelized)
+ gcc_checking_assert (is_oacc_kernels);
+ }
+ gcc_assert (gimple_in_ssa_p (cfun) == is_oacc_kernels_parallelized);
+ /* In the following, some of the 'gimple_in_ssa_p (cfun)' conditionals are
+ for SSA specifics, and some are for 'parloops' OpenACC
+ 'kernels'-parallelized specifics. */
+
tree v = fd->loop.v;
enum tree_code cond_code = fd->loop.cond_code;
enum tree_code plus_code = PLUS_EXPR;
plus_code = POINTER_PLUS_EXPR;
plus_type = sizetype;
}
+ for (int ix = fd->collapse; ix--;)
+ {
+ tree diff_type2 = TREE_TYPE (fd->loops[ix].step);
+ if (TYPE_PRECISION (diff_type) < TYPE_PRECISION (diff_type2))
+ diff_type = diff_type2;
+ }
if (POINTER_TYPE_P (diff_type) || TYPE_UNSIGNED (diff_type))
diff_type = signed_type_for (diff_type);
if (TYPE_PRECISION (diff_type) < TYPE_PRECISION (integer_type_node))
{
gcc_assert (!gimple_in_ssa_p (cfun) && up);
counts = XALLOCAVEC (struct oacc_collapse, fd->collapse);
- tree total = expand_oacc_collapse_init (fd, &gsi, counts,
+ tree total = expand_oacc_collapse_init (fd, &gsi, counts, diff_type,
TREE_TYPE (fd->loop.n2), loc);
if (SSA_VAR_P (fd->loop.n2))
gsi_insert_before (&gsi, ass, GSI_SAME_STMT);
if (fd->collapse > 1 || fd->tiling)
- expand_oacc_collapse_vars (fd, false, &gsi, counts, v);
+ expand_oacc_collapse_vars (fd, false, &gsi, counts, v, diff_type);
if (fd->tiling)
{
/* Initialize the user's loop vars. */
gsi = gsi_start_bb (elem_body_bb);
- expand_oacc_collapse_vars (fd, true, &gsi, counts, e_offset);
+ expand_oacc_collapse_vars (fd, true, &gsi, counts, e_offset,
+ diff_type);
}
}
set_immediate_dominator (CDI_DOMINATORS, default_bb, l0_bb);
}
-/* Expand code for an OpenMP single directive. We've already expanded
+/* Expand code for an OpenMP single or scope directive. We've already expanded
much of the code, here we simply place the GOMP_barrier call. */
static void
exit_bb = region->exit;
si = gsi_last_nondebug_bb (entry_bb);
- gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SINGLE);
+ gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SINGLE
+ || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SCOPE);
gsi_remove (&si, true);
single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
si = gsi_last_nondebug_bb (entry_bb);
gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SINGLE
|| gimple_code (gsi_stmt (si)) == GIMPLE_OMP_MASTER
+ || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_MASKED
|| gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TASKGROUP
|| gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ORDERED
|| gimple_code (gsi_stmt (si)) == GIMPLE_OMP_CRITICAL
}
}
+/* Translate enum omp_memory_order to enum memmodel for the embedded
+ fail clause in there. */
+
+static enum memmodel
+omp_memory_order_to_fail_memmodel (enum omp_memory_order mo)
+{
+ switch (mo & OMP_FAIL_MEMORY_ORDER_MASK)
+ {
+ case OMP_FAIL_MEMORY_ORDER_UNSPECIFIED:
+ switch (mo & OMP_MEMORY_ORDER_MASK)
+ {
+ case OMP_MEMORY_ORDER_RELAXED: return MEMMODEL_RELAXED;
+ case OMP_MEMORY_ORDER_ACQUIRE: return MEMMODEL_ACQUIRE;
+ case OMP_MEMORY_ORDER_RELEASE: return MEMMODEL_RELAXED;
+ case OMP_MEMORY_ORDER_ACQ_REL: return MEMMODEL_ACQUIRE;
+ case OMP_MEMORY_ORDER_SEQ_CST: return MEMMODEL_SEQ_CST;
+ default: break;
+ }
+ gcc_unreachable ();
+ case OMP_FAIL_MEMORY_ORDER_RELAXED: return MEMMODEL_RELAXED;
+ case OMP_FAIL_MEMORY_ORDER_ACQUIRE: return MEMMODEL_ACQUIRE;
+ case OMP_FAIL_MEMORY_ORDER_SEQ_CST: return MEMMODEL_SEQ_CST;
+ default: gcc_unreachable ();
+ }
+}
+
/* Translate enum omp_memory_order to enum memmodel. The two enums
are using different numbers so that OMP_MEMORY_ORDER_UNSPECIFIED
- is 0. */
+ is 0 and omp_memory_order has the fail mode encoded in it too. */
static enum memmodel
omp_memory_order_to_memmodel (enum omp_memory_order mo)
{
- switch (mo)
- {
- case OMP_MEMORY_ORDER_RELAXED: return MEMMODEL_RELAXED;
- case OMP_MEMORY_ORDER_ACQUIRE: return MEMMODEL_ACQUIRE;
- case OMP_MEMORY_ORDER_RELEASE: return MEMMODEL_RELEASE;
- case OMP_MEMORY_ORDER_ACQ_REL: return MEMMODEL_ACQ_REL;
- case OMP_MEMORY_ORDER_SEQ_CST: return MEMMODEL_SEQ_CST;
+ enum memmodel ret, fail_ret;
+ switch (mo & OMP_MEMORY_ORDER_MASK)
+ {
+ case OMP_MEMORY_ORDER_RELAXED: ret = MEMMODEL_RELAXED; break;
+ case OMP_MEMORY_ORDER_ACQUIRE: ret = MEMMODEL_ACQUIRE; break;
+ case OMP_MEMORY_ORDER_RELEASE: ret = MEMMODEL_RELEASE; break;
+ case OMP_MEMORY_ORDER_ACQ_REL: ret = MEMMODEL_ACQ_REL; break;
+ case OMP_MEMORY_ORDER_SEQ_CST: ret = MEMMODEL_SEQ_CST; break;
default: gcc_unreachable ();
}
+ /* If we drop the -Winvalid-memory-model warning for C++17 P0418R2,
+ we can just return ret here unconditionally. Otherwise, work around
+ it here and make sure fail memmodel is not stronger. */
+ if ((mo & OMP_FAIL_MEMORY_ORDER_MASK) == OMP_FAIL_MEMORY_ORDER_UNSPECIFIED)
+ return ret;
+ fail_ret = omp_memory_order_to_fail_memmodel (mo);
+ if (fail_ret > ret)
+ return fail_ret;
+ return ret;
}
/* A subroutine of expand_omp_atomic. Attempt to implement the atomic
return true;
}
+/* A subroutine of expand_omp_atomic. Attempt to implement the atomic
+ compare and exchange as an ATOMIC_COMPARE_EXCHANGE internal function.
+ Returns false if the expression is not of the proper form. */
+
+static bool
+expand_omp_atomic_cas (basic_block load_bb, tree addr,
+ tree loaded_val, tree stored_val, int index)
+{
+ /* We expect to find the following sequences:
+
+ load_bb:
+ GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
+
+ store_bb:
+ val = tmp == e ? d : tmp;
+ GIMPLE_OMP_ATOMIC_STORE (val)
+
+ or in store_bb instead:
+ tmp2 = tmp == e;
+ val = tmp2 ? d : tmp;
+ GIMPLE_OMP_ATOMIC_STORE (val)
+
+ or:
+ tmp3 = VIEW_CONVERT_EXPR<integral_type>(tmp);
+ val = e == tmp3 ? d : tmp;
+ GIMPLE_OMP_ATOMIC_STORE (val)
+
+ etc. */
+
+
+ basic_block store_bb = single_succ (load_bb);
+ gimple_stmt_iterator gsi = gsi_last_nondebug_bb (store_bb);
+ gimple *store_stmt = gsi_stmt (gsi);
+ if (!store_stmt || gimple_code (store_stmt) != GIMPLE_OMP_ATOMIC_STORE)
+ return false;
+ gsi_prev_nondebug (&gsi);
+ if (gsi_end_p (gsi))
+ return false;
+ gimple *condexpr_stmt = gsi_stmt (gsi);
+ if (!is_gimple_assign (condexpr_stmt)
+ || gimple_assign_rhs_code (condexpr_stmt) != COND_EXPR)
+ return false;
+ if (!operand_equal_p (gimple_assign_lhs (condexpr_stmt), stored_val, 0))
+ return false;
+ gimple *cond_stmt = NULL;
+ gimple *vce_stmt = NULL;
+ gsi_prev_nondebug (&gsi);
+ if (!gsi_end_p (gsi))
+ {
+ cond_stmt = gsi_stmt (gsi);
+ if (!is_gimple_assign (cond_stmt))
+ return false;
+ if (gimple_assign_rhs_code (cond_stmt) == EQ_EXPR)
+ {
+ gsi_prev_nondebug (&gsi);
+ if (!gsi_end_p (gsi))
+ {
+ vce_stmt = gsi_stmt (gsi);
+ if (!is_gimple_assign (vce_stmt)
+ || gimple_assign_rhs_code (vce_stmt) != VIEW_CONVERT_EXPR)
+ return false;
+ }
+ }
+ else if (gimple_assign_rhs_code (cond_stmt) == VIEW_CONVERT_EXPR)
+ std::swap (vce_stmt, cond_stmt);
+ else
+ return false;
+ if (vce_stmt)
+ {
+ tree vce_rhs = gimple_assign_rhs1 (vce_stmt);
+ if (TREE_CODE (vce_rhs) != VIEW_CONVERT_EXPR
+ || !operand_equal_p (TREE_OPERAND (vce_rhs, 0), loaded_val))
+ return false;
+ if (!INTEGRAL_TYPE_P (TREE_TYPE (vce_rhs))
+ || !SCALAR_FLOAT_TYPE_P (TREE_TYPE (loaded_val))
+ || !tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vce_rhs)),
+ TYPE_SIZE (TREE_TYPE (loaded_val))))
+ return false;
+ gsi_prev_nondebug (&gsi);
+ if (!gsi_end_p (gsi))
+ return false;
+ }
+ }
+ tree cond = gimple_assign_rhs1 (condexpr_stmt);
+ tree cond_op1, cond_op2;
+ if (cond_stmt)
+ {
+ if (!operand_equal_p (cond, gimple_assign_lhs (cond_stmt)))
+ return false;
+ cond_op1 = gimple_assign_rhs1 (cond_stmt);
+ cond_op2 = gimple_assign_rhs2 (cond_stmt);
+ }
+ else if (TREE_CODE (cond) != EQ_EXPR && TREE_CODE (cond) != NE_EXPR)
+ return false;
+ else
+ {
+ cond_op1 = TREE_OPERAND (cond, 0);
+ cond_op2 = TREE_OPERAND (cond, 1);
+ }
+ tree d;
+ if (TREE_CODE (cond) == NE_EXPR)
+ {
+ if (!operand_equal_p (gimple_assign_rhs2 (condexpr_stmt), loaded_val))
+ return false;
+ d = gimple_assign_rhs3 (condexpr_stmt);
+ }
+ else if (!operand_equal_p (gimple_assign_rhs3 (condexpr_stmt), loaded_val))
+ return false;
+ else
+ d = gimple_assign_rhs2 (condexpr_stmt);
+ tree e = vce_stmt ? gimple_assign_lhs (vce_stmt) : loaded_val;
+ if (operand_equal_p (e, cond_op1))
+ e = cond_op2;
+ else if (operand_equal_p (e, cond_op2))
+ e = cond_op1;
+ else
+ return false;
+
+ location_t loc = gimple_location (store_stmt);
+ gimple *load_stmt = last_stmt (load_bb);
+ bool need_new = gimple_omp_atomic_need_value_p (store_stmt);
+ bool need_old = gimple_omp_atomic_need_value_p (load_stmt);
+ bool weak = gimple_omp_atomic_weak_p (load_stmt);
+ enum omp_memory_order omo = gimple_omp_atomic_memory_order (load_stmt);
+ tree mo = build_int_cst (NULL, omp_memory_order_to_memmodel (omo));
+ tree fmo = build_int_cst (NULL, omp_memory_order_to_fail_memmodel (omo));
+ gcc_checking_assert (!need_old || !need_new);
+
+ enum built_in_function fncode
+ = (enum built_in_function) ((int) BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
+ + index + 1);
+ tree cmpxchg = builtin_decl_explicit (fncode);
+ if (cmpxchg == NULL_TREE)
+ return false;
+ tree itype = TREE_TYPE (TREE_TYPE (cmpxchg));
+
+ if (!can_compare_and_swap_p (TYPE_MODE (itype), true)
+ || !can_atomic_load_p (TYPE_MODE (itype)))
+ return false;
+
+ tree type = TYPE_MAIN_VARIANT (TREE_TYPE (loaded_val));
+ if (SCALAR_FLOAT_TYPE_P (type) && !vce_stmt)
+ return false;
+
+ gsi = gsi_for_stmt (store_stmt);
+ if (!useless_type_conversion_p (itype, TREE_TYPE (e)))
+ {
+ tree ne = create_tmp_reg (itype);
+ gimple *g = gimple_build_assign (ne, NOP_EXPR, e);
+ gimple_set_location (g, loc);
+ gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+ e = ne;
+ }
+ if (!useless_type_conversion_p (itype, TREE_TYPE (d)))
+ {
+ tree nd = create_tmp_reg (itype);
+ enum tree_code code;
+ if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (d)))
+ {
+ code = VIEW_CONVERT_EXPR;
+ d = build1 (VIEW_CONVERT_EXPR, itype, d);
+ }
+ else
+ code = NOP_EXPR;
+ gimple *g = gimple_build_assign (nd, code, d);
+ gimple_set_location (g, loc);
+ gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+ d = nd;
+ }
+
+ tree ctype = build_complex_type (itype);
+ int flag = int_size_in_bytes (itype) + (weak ? 256 : 0);
+ gimple *g
+ = gimple_build_call_internal (IFN_ATOMIC_COMPARE_EXCHANGE, 6, addr, e, d,
+ build_int_cst (integer_type_node, flag),
+ mo, fmo);
+ tree cres = create_tmp_reg (ctype);
+ gimple_call_set_lhs (g, cres);
+ gimple_set_location (g, loc);
+ gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+
+ if (cond_stmt || need_old || need_new)
+ {
+ tree im = create_tmp_reg (itype);
+ g = gimple_build_assign (im, IMAGPART_EXPR,
+ build1 (IMAGPART_EXPR, itype, cres));
+ gimple_set_location (g, loc);
+ gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+
+ tree re = NULL_TREE;
+ if (need_old || need_new)
+ {
+ re = create_tmp_reg (itype);
+ g = gimple_build_assign (re, REALPART_EXPR,
+ build1 (REALPART_EXPR, itype, cres));
+ gimple_set_location (g, loc);
+ gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+ }
+
+ if (cond_stmt)
+ {
+ g = gimple_build_assign (gimple_assign_lhs (cond_stmt),
+ NOP_EXPR, im);
+ gimple_set_location (g, loc);
+ gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+ }
+ else if (need_new)
+ {
+ g = gimple_build_assign (create_tmp_reg (itype), COND_EXPR,
+ build2 (NE_EXPR, boolean_type_node,
+ im, build_zero_cst (itype)),
+ d, re);
+ gimple_set_location (g, loc);
+ gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+ re = gimple_assign_lhs (g);
+ }
+
+ if (need_old || need_new)
+ {
+ tree v = need_old ? loaded_val : stored_val;
+ enum tree_code code;
+ if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (v)))
+ {
+ code = VIEW_CONVERT_EXPR;
+ re = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (v), re);
+ }
+ else if (!useless_type_conversion_p (TREE_TYPE (v), itype))
+ code = NOP_EXPR;
+ else
+ code = TREE_CODE (re);
+ g = gimple_build_assign (v, code, re);
+ gimple_set_location (g, loc);
+ gsi_insert_before (&gsi, g, GSI_SAME_STMT);
+ }
+ }
+
+ gsi_remove (&gsi, true);
+ gsi = gsi_for_stmt (load_stmt);
+ gsi_remove (&gsi, true);
+ gsi = gsi_for_stmt (condexpr_stmt);
+ gsi_remove (&gsi, true);
+ if (cond_stmt)
+ {
+ gsi = gsi_for_stmt (cond_stmt);
+ gsi_remove (&gsi, true);
+ }
+ if (vce_stmt)
+ {
+ gsi = gsi_for_stmt (vce_stmt);
+ gsi_remove (&gsi, true);
+ }
+
+ return true;
+}
+
/* A subroutine of expand_omp_atomic. Implement the atomic operation as:
oldval = *addr;
edge e;
enum built_in_function fncode;
- /* ??? We need a non-pointer interface to __atomic_compare_exchange in
- order to use the RELAXED memory model effectively. */
fncode = (enum built_in_function)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
+ index + 1);
cmpxchg = builtin_decl_explicit (fncode);
/* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
si = gsi_last_nondebug_bb (load_bb);
gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD);
+ location_t loc = gimple_location (gsi_stmt (si));
+ enum omp_memory_order omo = gimple_omp_atomic_memory_order (gsi_stmt (si));
+ tree mo = build_int_cst (NULL, omp_memory_order_to_memmodel (omo));
+ tree fmo = build_int_cst (NULL, omp_memory_order_to_fail_memmodel (omo));
/* For floating-point values, we'll need to view-convert them to integers
so that we can perform the atomic compare and swap. Simplify the
GSI_SAME_STMT);
/* Build the compare&swap statement. */
- new_storedi = build_call_expr (cmpxchg, 3, iaddr, loadedi, storedi);
+ tree ctype = build_complex_type (itype);
+ int flag = int_size_in_bytes (itype);
+ new_storedi = build_call_expr_internal_loc (loc, IFN_ATOMIC_COMPARE_EXCHANGE,
+ ctype, 6, iaddr, loadedi,
+ storedi,
+ build_int_cst (integer_type_node,
+ flag),
+ mo, fmo);
+ new_storedi = build1 (REALPART_EXPR, itype, new_storedi);
new_storedi = force_gimple_operand_gsi (&si,
fold_convert (TREE_TYPE (loadedi),
new_storedi),
loaded_val, stored_val, index))
return;
+ /* When possible, use ATOMIC_COMPARE_EXCHANGE ifn without a loop. */
+ if (store_bb == single_succ (load_bb)
+ && !gimple_in_ssa_p (cfun)
+ && expand_omp_atomic_cas (load_bb, addr, loaded_val, stored_val,
+ index))
+ return;
+
/* If we don't have specialized __sync builtins, try and implement
as a compare and swap loop. */
if (expand_omp_atomic_pipeline (load_bb, store_bb, addr,
gomp_target *entry_stmt;
gimple *stmt;
edge e;
- bool offloaded, data_region;
+ bool offloaded;
int target_kind;
entry_stmt = as_a <gomp_target *> (last_stmt (region->entry));
case GF_OMP_TARGET_KIND_OACC_KERNELS:
case GF_OMP_TARGET_KIND_OACC_SERIAL:
case GF_OMP_TARGET_KIND_OACC_UPDATE:
- case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA:
+ case GF_OMP_TARGET_KIND_OACC_ENTER_DATA:
+ case GF_OMP_TARGET_KIND_OACC_EXIT_DATA:
case GF_OMP_TARGET_KIND_OACC_DECLARE:
case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
- data_region = false;
- break;
case GF_OMP_TARGET_KIND_DATA:
case GF_OMP_TARGET_KIND_OACC_DATA:
case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
- data_region = true;
break;
default:
gcc_unreachable ();
case GF_OMP_TARGET_KIND_OACC_UPDATE:
start_ix = BUILT_IN_GOACC_UPDATE;
break;
- case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA:
- start_ix = BUILT_IN_GOACC_ENTER_EXIT_DATA;
+ case GF_OMP_TARGET_KIND_OACC_ENTER_DATA:
+ start_ix = BUILT_IN_GOACC_ENTER_DATA;
+ break;
+ case GF_OMP_TARGET_KIND_OACC_EXIT_DATA:
+ start_ix = BUILT_IN_GOACC_EXIT_DATA;
break;
case GF_OMP_TARGET_KIND_OACC_DECLARE:
start_ix = BUILT_IN_GOACC_DECLARE;
{
device = OMP_CLAUSE_DEVICE_ID (c);
device_loc = OMP_CLAUSE_LOCATION (c);
+ if (OMP_CLAUSE_DEVICE_ANCESTOR (c))
+ sorry_at (device_loc, "%<ancestor%> not yet supported");
}
else
{
}
c = omp_find_clause (clauses, OMP_CLAUSE_NOWAIT);
+ /* FIXME: in_reduction(...) nowait is unimplemented yet, pretend
+ nowait doesn't appear. */
+ if (c && omp_find_clause (clauses, OMP_CLAUSE_IN_REDUCTION))
+ c = NULL;
if (c)
flags_i |= GOMP_TARGET_FLAG_NOWAIT;
}
oacc_set_fn_attrib (child_fn, clauses, &args);
tagging = true;
/* FALLTHRU */
- case BUILT_IN_GOACC_ENTER_EXIT_DATA:
+ case BUILT_IN_GOACC_ENTER_DATA:
+ case BUILT_IN_GOACC_EXIT_DATA:
case BUILT_IN_GOACC_UPDATE:
{
tree t_async = NULL_TREE;
gcc_assert (g && gimple_code (g) == GIMPLE_OMP_TARGET);
gsi_remove (&gsi, true);
}
- if (data_region && region->exit)
- {
- gsi = gsi_last_nondebug_bb (region->exit);
- g = gsi_stmt (gsi);
- gcc_assert (g && gimple_code (g) == GIMPLE_OMP_RETURN);
- gsi_remove (&gsi, true);
- }
}
/* Expand the parallel region tree rooted at REGION. Expansion
break;
case GIMPLE_OMP_SINGLE:
+ case GIMPLE_OMP_SCOPE:
expand_omp_single (region);
break;
}
/* FALLTHRU */
case GIMPLE_OMP_MASTER:
+ case GIMPLE_OMP_MASKED:
case GIMPLE_OMP_TASKGROUP:
case GIMPLE_OMP_CRITICAL:
case GIMPLE_OMP_TEAMS:
switch (gimple_omp_target_kind (stmt))
{
case GF_OMP_TARGET_KIND_REGION:
- case GF_OMP_TARGET_KIND_DATA:
case GF_OMP_TARGET_KIND_OACC_PARALLEL:
case GF_OMP_TARGET_KIND_OACC_KERNELS:
case GF_OMP_TARGET_KIND_OACC_SERIAL:
- case GF_OMP_TARGET_KIND_OACC_DATA:
- case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
- case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
break;
case GF_OMP_TARGET_KIND_UPDATE:
case GF_OMP_TARGET_KIND_ENTER_DATA:
case GF_OMP_TARGET_KIND_EXIT_DATA:
+ case GF_OMP_TARGET_KIND_DATA:
+ case GF_OMP_TARGET_KIND_OACC_DATA:
+ case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
+ case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
case GF_OMP_TARGET_KIND_OACC_UPDATE:
- case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA:
+ case GF_OMP_TARGET_KIND_OACC_ENTER_DATA:
+ case GF_OMP_TARGET_KIND_OACC_EXIT_DATA:
case GF_OMP_TARGET_KIND_OACC_DECLARE:
/* ..., other than for those stand-alone directives... */
region = NULL;
case GIMPLE_OMP_SINGLE:
case GIMPLE_OMP_TEAMS:
case GIMPLE_OMP_MASTER:
+ case GIMPLE_OMP_MASKED:
+ case GIMPLE_OMP_SCOPE:
case GIMPLE_OMP_TASKGROUP:
case GIMPLE_OMP_CRITICAL:
case GIMPLE_OMP_SECTION:
switch (gimple_omp_target_kind (last))
{
case GF_OMP_TARGET_KIND_REGION:
- case GF_OMP_TARGET_KIND_DATA:
case GF_OMP_TARGET_KIND_OACC_PARALLEL:
case GF_OMP_TARGET_KIND_OACC_KERNELS:
case GF_OMP_TARGET_KIND_OACC_SERIAL:
- case GF_OMP_TARGET_KIND_OACC_DATA:
- case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
- case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
break;
case GF_OMP_TARGET_KIND_UPDATE:
case GF_OMP_TARGET_KIND_ENTER_DATA:
case GF_OMP_TARGET_KIND_EXIT_DATA:
+ case GF_OMP_TARGET_KIND_DATA:
+ case GF_OMP_TARGET_KIND_OACC_DATA:
+ case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
+ case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
case GF_OMP_TARGET_KIND_OACC_UPDATE:
- case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA:
+ case GF_OMP_TARGET_KIND_OACC_ENTER_DATA:
+ case GF_OMP_TARGET_KIND_OACC_EXIT_DATA:
case GF_OMP_TARGET_KIND_OACC_DECLARE:
cur_region = cur_region->outer;
break;
projects / thirdparty / gcc.git / blobdiff