/* Function splitting pass
- Copyright (C) 2010-2014 Free Software Foundation, Inc.
+ Copyright (C) 2010-2021 Free Software Foundation, Inc.
Contributed by Jan Hubicka <jh@suse.cz>
This file is part of GCC.
#include "config.h"
#include "system.h"
#include "coretypes.h"
+#include "backend.h"
+#include "rtl.h"
#include "tree.h"
-#include "predict.h"
-#include "vec.h"
-#include "hashtab.h"
-#include "hash-set.h"
-#include "machmode.h"
-#include "tm.h"
-#include "hard-reg-set.h"
-#include "input.h"
-#include "function.h"
-#include "dominance.h"
-#include "cfg.h"
-#include "cfganal.h"
-#include "basic-block.h"
-#include "tree-ssa-alias.h"
-#include "internal-fn.h"
-#include "gimple-expr.h"
-#include "is-a.h"
#include "gimple.h"
-#include "stringpool.h"
-#include "expr.h"
+#include "cfghooks.h"
+#include "alloc-pool.h"
+#include "tree-pass.h"
+#include "ssa.h"
+#include "cgraph.h"
+#include "diagnostic.h"
+#include "fold-const.h"
+#include "cfganal.h"
#include "calls.h"
#include "gimplify.h"
#include "gimple-iterator.h"
#include "gimplify-me.h"
#include "gimple-walk.h"
-#include "target.h"
-#include "hash-map.h"
-#include "plugin-api.h"
-#include "ipa-ref.h"
-#include "cgraph.h"
-#include "alloc-pool.h"
#include "symbol-summary.h"
#include "ipa-prop.h"
-#include "gimple-ssa.h"
#include "tree-cfg.h"
-#include "tree-phinodes.h"
-#include "ssa-iterators.h"
-#include "stringpool.h"
-#include "tree-ssanames.h"
#include "tree-into-ssa.h"
#include "tree-dfa.h"
-#include "tree-pass.h"
-#include "flags.h"
-#include "diagnostic.h"
-#include "tree-dump.h"
#include "tree-inline.h"
-#include "params.h"
#include "gimple-pretty-print.h"
-#include "ipa-inline.h"
+#include "ipa-fnsummary.h"
#include "cfgloop.h"
-#include "tree-chkp.h"
+#include "attribs.h"
/* Per basic block info. */
-typedef struct
+class split_bb_info
{
+public:
unsigned int size;
- unsigned int time;
-} split_bb_info;
+ sreal time;
+};
static vec<split_bb_info> bb_info_vec;
/* Description of split point. */
-struct split_point
+class split_point
{
+public:
/* Size of the partitions. */
- unsigned int header_time, header_size, split_time, split_size;
+ sreal header_time, split_time;
+ unsigned int header_size, split_size;
/* SSA names that need to be passed into spit function. */
bitmap ssa_names_to_pass;
/* Basic block where we split (that will become entry point of new function. */
basic_block entry_bb;
+ /* Count for entering the split part.
+ This is not count of the entry_bb because it may be in loop. */
+ profile_count count;
+
/* Basic blocks we are splitting away. */
bitmap split_bbs;
/* Best split point found. */
-struct split_point best_split_point;
+class split_point best_split_point;
/* Set of basic blocks that are not allowed to dominate a split point. */
static bitmap forbidden_dominators;
static tree find_retval (basic_block return_bb);
-static tree find_retbnd (basic_block return_bb);
/* Callback for walk_stmt_load_store_addr_ops. If T is non-SSA automatic
variable, check it if it is present in bitmap passed via DATA. */
static bool
-test_nonssa_use (gimple, tree t, tree, void *data)
+test_nonssa_use (gimple *, tree t, tree, void *data)
{
t = get_base_address (t);
return false;
if (TREE_CODE (t) == PARM_DECL
- || (TREE_CODE (t) == VAR_DECL
+ || (VAR_P (t)
&& auto_var_in_fn_p (t, current_function_decl))
|| TREE_CODE (t) == RESULT_DECL
- /* Normal labels are part of CFG and will be handled gratefuly.
+ /* Normal labels are part of CFG and will be handled gratefully.
Forced labels however can be used directly by statements and
need to stay in one partition along with their uses. */
|| (TREE_CODE (t) == LABEL_DECL
/* Dump split point CURRENT. */
static void
-dump_split_point (FILE * file, struct split_point *current)
+dump_split_point (FILE * file, class split_point *current)
{
fprintf (file,
"Split point at BB %i\n"
- " header time: %i header size: %i\n"
- " split time: %i split size: %i\n bbs: ",
- current->entry_bb->index, current->header_time,
- current->header_size, current->split_time, current->split_size);
+ " header time: %f header size: %i\n"
+ " split time: %f split size: %i\n bbs: ",
+ current->entry_bb->index, current->header_time.to_double (),
+ current->header_size, current->split_time.to_double (),
+ current->split_size);
dump_bitmap (file, current->split_bbs);
fprintf (file, " SSA names to pass: ");
dump_bitmap (file, current->ssa_names_to_pass);
Parameters are the same as for consider_split. */
static bool
-verify_non_ssa_vars (struct split_point *current, bitmap non_ssa_vars,
+verify_non_ssa_vars (class split_point *current, bitmap non_ssa_vars,
basic_block return_bb)
{
bitmap seen = BITMAP_ALLOC (NULL);
for (gimple_stmt_iterator bsi = gsi_start_bb (bb); !gsi_end_p (bsi);
gsi_next (&bsi))
{
- gimple stmt = gsi_stmt (bsi);
+ gimple *stmt = gsi_stmt (bsi);
if (is_gimple_debug (stmt))
continue;
if (walk_stmt_load_store_addr_ops
to optimize away an unused function call. */
static void
-check_forbidden_calls (gimple stmt)
+check_forbidden_calls (gimple *stmt)
{
imm_use_iterator use_iter;
use_operand_p use_p;
/* For give split point CURRENT and return block RETURN_BB return 1
if ssa name VAL is set by split part and 0 otherwise. */
static bool
-split_part_set_ssa_name_p (tree val, struct split_point *current,
+split_part_set_ssa_name_p (tree val, class split_point *current,
basic_block return_bb)
{
if (TREE_CODE (val) != SSA_NAME)
See if we can split function here. */
static void
-consider_split (struct split_point *current, bitmap non_ssa_vars,
+consider_split (class split_point *current, bitmap non_ssa_vars,
basic_block return_bb)
{
tree parm;
edge_iterator ei;
gphi_iterator bsi;
unsigned int i;
- int incoming_freq = 0;
tree retval;
- tree retbnd;
bool back_edge = false;
if (dump_file && (dump_flags & TDF_DETAILS))
dump_split_point (dump_file, current);
+ current->count = profile_count::zero ();
FOR_EACH_EDGE (e, ei, current->entry_bb->preds)
{
if (e->flags & EDGE_DFS_BACK)
back_edge = true;
if (!bitmap_bit_p (current->split_bbs, e->src->index))
- incoming_freq += EDGE_FREQUENCY (e);
+ current->count += e->count ();
}
- /* Do not split when we would end up calling function anyway. */
- if (incoming_freq
- >= (ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency
- * PARAM_VALUE (PARAM_PARTIAL_INLINING_ENTRY_PROBABILITY) / 100))
+ /* Do not split when we would end up calling function anyway.
+ Compares are three state, use !(...<...) to also give up when outcome
+ is unknown. */
+ if (!(current->count
+ < (ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.apply_scale
+ (param_partial_inlining_entry_probability, 100))))
{
- /* When profile is guessed, we can not expect it to give us
- realistic estimate on likelyness of function taking the
+ /* When profile is guessed, we cannot expect it to give us
+ realistic estimate on likeliness of function taking the
complex path. As a special case, when tail of the function is
a loop, enable splitting since inlining code skipping the loop
is likely noticeable win. */
if (back_edge
&& profile_status_for_fn (cfun) != PROFILE_READ
- && incoming_freq < ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency)
+ && current->count
+ < ENTRY_BLOCK_PTR_FOR_FN (cfun)->count)
{
if (dump_file && (dump_flags & TDF_DETAILS))
- fprintf (dump_file,
- " Split before loop, accepting despite low frequencies %i %i.\n",
- incoming_freq,
- ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency);
+ {
+ fprintf (dump_file,
+ " Split before loop, accepting despite low counts");
+ current->count.dump (dump_file);
+ fprintf (dump_file, " ");
+ ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.dump (dump_file);
+ }
}
else
{
}
}
}
- if (!VOID_TYPE_P (TREE_TYPE (current_function_decl)))
- call_overhead += estimate_move_cost (TREE_TYPE (current_function_decl),
+ if (!VOID_TYPE_P (TREE_TYPE (TREE_TYPE (current_function_decl))))
+ call_overhead += estimate_move_cost (TREE_TYPE (TREE_TYPE
+ (current_function_decl)),
false);
if (current->split_size <= call_overhead)
" Refused: split size is smaller than call overhead\n");
return;
}
+ /* FIXME: The logic here is not very precise, because inliner does use
+ inline predicates to reduce function body size. We add 10 to anticipate
+ that. Next stage1 we should try to be more meaningful here. */
if (current->header_size + call_overhead
>= (unsigned int)(DECL_DECLARED_INLINE_P (current_function_decl)
- ? MAX_INLINE_INSNS_SINGLE
- : MAX_INLINE_INSNS_AUTO))
+ ? param_max_inline_insns_single
+ : param_max_inline_insns_auto) + 10)
{
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file,
return;
}
+ /* Splitting functions brings the target out of comdat group; this will
+ lead to code duplication if the function is reused by other unit.
+ Limit this duplication. This is consistent with limit in tree-sra.c
+ FIXME: with LTO we ought to be able to do better! */
+ if (DECL_ONE_ONLY (current_function_decl)
+ && current->split_size >= (unsigned int) param_max_inline_insns_auto + 10)
+ {
+ if (dump_file && (dump_flags & TDF_DETAILS))
+ fprintf (dump_file,
+ " Refused: function is COMDAT and tail is too large\n");
+ return;
+ }
+ /* For comdat functions also reject very small tails; those will likely get
+ inlined back and we do not want to risk the duplication overhead.
+ FIXME: with LTO we ought to be able to do better! */
+ if (DECL_ONE_ONLY (current_function_decl)
+ && current->split_size
+ <= (unsigned int) param_early_inlining_insns / 2)
+ {
+ if (dump_file && (dump_flags & TDF_DETAILS))
+ fprintf (dump_file,
+ " Refused: function is COMDAT and tail is too small\n");
+ return;
+ }
+
/* FIXME: we currently can pass only SSA function parameters to the split
arguments. Once parm_adjustment infrastructure is supported by cloning,
we can pass more than that. */
4) For non-SSA we need to look where the var is computed. */
retval = find_retval (return_bb);
if (!retval)
- current->split_part_set_retval = true;
+ {
+ /* If there is a return_bb with no return value in function returning
+ value by reference, also make the split part return void, otherwise
+ we expansion would try to create a non-POD temporary, which is
+ invalid. */
+ if (return_bb != EXIT_BLOCK_PTR_FOR_FN (cfun)
+ && DECL_RESULT (current_function_decl)
+ && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
+ current->split_part_set_retval = false;
+ else
+ current->split_part_set_retval = true;
+ }
else if (is_gimple_min_invariant (retval))
current->split_part_set_retval = false;
/* Special case is value returned by reference we record as if it was non-ssa
= split_part_set_ssa_name_p (retval, current, return_bb);
else if (TREE_CODE (retval) == PARM_DECL)
current->split_part_set_retval = false;
- else if (TREE_CODE (retval) == VAR_DECL
+ else if (VAR_P (retval)
|| TREE_CODE (retval) == RESULT_DECL)
current->split_part_set_retval
= bitmap_bit_p (non_ssa_vars, DECL_UID (retval));
else
current->split_part_set_retval = true;
- /* See if retbnd used by return bb is computed by header or split part. */
- retbnd = find_retbnd (return_bb);
- if (retbnd)
- {
- bool split_part_set_retbnd
- = split_part_set_ssa_name_p (retbnd, current, return_bb);
-
- /* If we have both return value and bounds then keep their definitions
- in a single function. We use SSA names to link returned bounds and
- value and therefore do not handle cases when result is passed by
- reference (which should not be our case anyway since bounds are
- returned for pointers only). */
- if ((DECL_BY_REFERENCE (DECL_RESULT (current_function_decl))
- && current->split_part_set_retval)
- || split_part_set_retbnd != current->split_part_set_retval)
- {
- if (dump_file && (dump_flags & TDF_DETAILS))
- fprintf (dump_file,
- " Refused: split point splits return value and bounds\n");
- return;
- }
- }
-
/* split_function fixes up at most one PHI non-virtual PHI node in return_bb,
for the return value. If there are other PHIs, give up. */
if (return_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file, " Accepted!\n");
- /* At the moment chose split point with lowest frequency and that leaves
+ /* At the moment chose split point with lowest count and that leaves
out smallest size of header.
In future we might re-consider this heuristics. */
if (!best_split_point.split_bbs
- || best_split_point.entry_bb->frequency > current->entry_bb->frequency
- || (best_split_point.entry_bb->frequency == current->entry_bb->frequency
+ || best_split_point.count
+ > current->count
+ || (best_split_point.count == current->count
&& best_split_point.split_size < current->split_size))
{
of the form:
<retval> = tmp_var;
return <retval>
- but return_bb can not be more complex than this.
+ but return_bb cannot be more complex than this (except for
+ -fsanitize=thread we allow TSAN_FUNC_EXIT () internal call in there).
If nothing is found, return the exit block.
When there are multiple RETURN statement, chose one with return value,
e = single_pred_edge (EXIT_BLOCK_PTR_FOR_FN (cfun));
for (bsi = gsi_last_bb (e->src); !gsi_end_p (bsi); gsi_prev (&bsi))
{
- gimple stmt = gsi_stmt (bsi);
+ gimple *stmt = gsi_stmt (bsi);
if (gimple_code (stmt) == GIMPLE_LABEL
|| is_gimple_debug (stmt)
|| gimple_clobber_p (stmt))
found_return = true;
retval = gimple_return_retval (return_stmt);
}
+ /* For -fsanitize=thread, allow also TSAN_FUNC_EXIT () in the return
+ bb. */
+ else if ((flag_sanitize & SANITIZE_THREAD)
+ && gimple_call_internal_p (stmt, IFN_TSAN_FUNC_EXIT))
+ ;
else
break;
}
return NULL;
}
-/* Given return basic block RETURN_BB, see where return bounds are really
- stored. */
-static tree
-find_retbnd (basic_block return_bb)
-{
- gimple_stmt_iterator bsi;
- for (bsi = gsi_last_bb (return_bb); !gsi_end_p (bsi); gsi_prev (&bsi))
- if (gimple_code (gsi_stmt (bsi)) == GIMPLE_RETURN)
- return gimple_return_retbnd (gsi_stmt (bsi));
- return NULL;
-}
-
/* Callback for walk_stmt_load_store_addr_ops. If T is non-SSA automatic
variable, mark it as used in bitmap passed via DATA.
Return true when access to T prevents splitting the function. */
static bool
-mark_nonssa_use (gimple, tree t, tree, void *data)
+mark_nonssa_use (gimple *, tree t, tree, void *data)
{
t = get_base_address (t);
return true;
}
- if ((TREE_CODE (t) == VAR_DECL
- && auto_var_in_fn_p (t, current_function_decl))
+ if ((VAR_P (t) && auto_var_in_fn_p (t, current_function_decl))
|| TREE_CODE (t) == RESULT_DECL
- || (TREE_CODE (t) == LABEL_DECL
- && FORCED_LABEL (t)))
+ || (TREE_CODE (t) == LABEL_DECL && FORCED_LABEL (t)))
bitmap_set_bit ((bitmap)data, DECL_UID (t));
/* For DECL_BY_REFERENCE, the return value is actually a pointer. We want
for (gimple_stmt_iterator bsi = gsi_start_bb (bb); !gsi_end_p (bsi);
gsi_next (&bsi))
{
- gimple stmt = gsi_stmt (bsi);
+ gimple *stmt = gsi_stmt (bsi);
tree op;
ssa_op_iter iter;
tree decl;
if (gimple_clobber_p (stmt))
continue;
- /* FIXME: We can split regions containing EH. We can not however
+ /* FIXME: We can split regions containing EH. We cannot however
split RESX, EH_DISPATCH and EH_POINTER referring to same region
into different partitions. This would require tracking of
EH regions and checking in consider_split_point if they
/* Check builtins that prevent splitting. */
if (gimple_code (stmt) == GIMPLE_CALL
&& (decl = gimple_call_fndecl (stmt)) != NULL_TREE
- && DECL_BUILT_IN (decl)
- && DECL_BUILT_IN_CLASS (decl) == BUILT_IN_NORMAL)
+ && fndecl_built_in_p (decl, BUILT_IN_NORMAL))
switch (DECL_FUNCTION_CODE (decl))
{
/* FIXME: once we will allow passing non-parm values to split part,
/* Stack entry for recursive DFS walk in find_split_point. */
-typedef struct
+class stack_entry
{
+public:
/* Basic block we are examining. */
basic_block bb;
int earliest;
/* Overall time and size of all BBs reached from this BB in DFS walk. */
- int overall_time, overall_size;
+ sreal overall_time;
+ int overall_size;
- /* When false we can not split on this BB. */
+ /* When false we cannot split on this BB. */
bool can_split;
-} stack_entry;
+};
/* Find all articulations and call consider_split on them.
the component used by consider_split. */
static void
-find_split_points (int overall_time, int overall_size)
+find_split_points (basic_block return_bb, sreal overall_time, int overall_size)
{
stack_entry first;
vec<stack_entry> stack = vNULL;
basic_block bb;
- basic_block return_bb = find_return_bb ();
- struct split_point current;
+ class split_point current;
current.header_time = overall_time;
current.header_size = overall_size;
if (pos <= entry->earliest && !entry->can_split
&& dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file,
- "found articulation at bb %i but can not split\n",
+ "found articulation at bb %i but cannot split\n",
entry->bb->index);
if (pos <= entry->earliest && entry->can_split)
{
BITMAP_FREE (current.ssa_names_to_pass);
}
-/* Build and insert initialization of returned bounds RETBND
- for returned value RETVAL. Statements are inserted after
- a statement pointed by GSI and GSI is modified to point to
- the last inserted statement. */
-
-static void
-insert_bndret_call_after (tree retbnd, tree retval, gimple_stmt_iterator *gsi)
-{
- tree fndecl = targetm.builtin_chkp_function (BUILT_IN_CHKP_BNDRET);
- gimple bndret = gimple_build_call (fndecl, 1, retval);
- gimple_call_set_lhs (bndret, retbnd);
- gsi_insert_after (gsi, bndret, GSI_CONTINUE_LINKING);
-}
/* Split function at SPLIT_POINT. */
static void
-split_function (struct split_point *split_point)
+split_function (basic_block return_bb, class split_point *split_point,
+ bool add_tsan_func_exit)
{
vec<tree> args_to_pass = vNULL;
bitmap args_to_skip;
tree parm;
int num = 0;
cgraph_node *node, *cur_node = cgraph_node::get (current_function_decl);
- basic_block return_bb = find_return_bb ();
basic_block call_bb;
- gcall *call;
+ gcall *call, *tsan_func_exit_call = NULL;
edge e;
edge_iterator ei;
- tree retval = NULL, real_retval = NULL, retbnd = NULL;
- bool split_part_return_p = false;
- bool with_bounds = chkp_function_instrumented_p (current_function_decl);
- gimple last_stmt = NULL;
+ tree retval = NULL, real_retval = NULL;
+ gimple *last_stmt = NULL;
unsigned int i;
tree arg, ddef;
- vec<tree, va_gc> **debug_args = NULL;
if (dump_file)
{
dump_split_point (dump_file, split_point);
}
- if (cur_node->local.can_change_signature)
+ if (cur_node->can_change_signature)
args_to_skip = BITMAP_ALLOC (NULL);
else
args_to_skip = NULL;
}
/* See if the split function will return. */
+ bool split_part_return_p = false;
FOR_EACH_EDGE (e, ei, return_bb->preds)
- if (bitmap_bit_p (split_point->split_bbs, e->src->index))
- break;
- if (e)
- split_part_return_p = true;
+ {
+ if (bitmap_bit_p (split_point->split_bbs, e->src->index))
+ split_part_return_p = true;
+ }
/* Add return block to what will become the split function.
We do not return; no return block is needed. */
FIXME: Once we are able to change return type, we should change function
to return void instead of just outputting function with undefined return
value. For structures this affects quality of codegen. */
- else if (!split_point->split_part_set_retval
- && find_retval (return_bb))
+ else if ((retval = find_retval (return_bb))
+ && !split_point->split_part_set_retval)
{
bool redirected = true;
basic_block new_return_bb = create_basic_block (NULL, 0, return_bb);
gimple_stmt_iterator gsi = gsi_start_bb (new_return_bb);
gsi_insert_after (&gsi, gimple_build_return (NULL), GSI_NEW_STMT);
+ new_return_bb->count = profile_count::zero ();
while (redirected)
{
redirected = false;
FOR_EACH_EDGE (e, ei, return_bb->preds)
if (bitmap_bit_p (split_point->split_bbs, e->src->index))
{
- new_return_bb->count += e->count;
- new_return_bb->frequency += EDGE_FREQUENCY (e);
+ new_return_bb->count += e->count ();
redirect_edge_and_branch (e, new_return_bb);
redirected = true;
break;
}
}
- e = make_edge (new_return_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
- e->probability = REG_BR_PROB_BASE;
- e->count = new_return_bb->count;
+ e = make_single_succ_edge (new_return_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
add_bb_to_loop (new_return_bb, current_loops->tree_root);
bitmap_set_bit (split_point->split_bbs, new_return_bb->index);
}
!gsi_end_p (gsi);
gsi_next (&gsi))
{
- gimple stmt = gsi_stmt (gsi);
+ gimple *stmt = gsi_stmt (gsi);
if (gimple_vuse (stmt))
{
gimple_set_vuse (stmt, NULL_TREE);
}
}
+ ipa_param_adjustments *adjustments;
+ bool skip_return = (!split_part_return_p
+ || !split_point->split_part_set_retval);
+ /* TODO: Perhaps get rid of args_to_skip entirely, after we make sure the
+ debug info generation and discrepancy avoiding works well too. */
+ if ((args_to_skip && !bitmap_empty_p (args_to_skip))
+ || skip_return)
+ {
+ vec<ipa_adjusted_param, va_gc> *new_params = NULL;
+ unsigned j;
+ for (parm = DECL_ARGUMENTS (current_function_decl), j = 0;
+ parm; parm = DECL_CHAIN (parm), j++)
+ if (!args_to_skip || !bitmap_bit_p (args_to_skip, j))
+ {
+ ipa_adjusted_param adj;
+ memset (&adj, 0, sizeof (adj));
+ adj.op = IPA_PARAM_OP_COPY;
+ adj.base_index = j;
+ adj.prev_clone_index = j;
+ vec_safe_push (new_params, adj);
+ }
+ adjustments = new ipa_param_adjustments (new_params, j, skip_return);
+ }
+ else
+ adjustments = NULL;
+
/* Now create the actual clone. */
cgraph_edge::rebuild_edges ();
node = cur_node->create_version_clone_with_body
- (vNULL, NULL, args_to_skip, !split_part_return_p, split_point->split_bbs,
- split_point->entry_bb, "part");
+ (vNULL, NULL, adjustments,
+ split_point->split_bbs, split_point->entry_bb, "part");
+ delete adjustments;
+ node->split_part = true;
+
+ if (cur_node->same_comdat_group)
+ {
+ /* TODO: call is versionable if we make sure that all
+ callers are inside of a comdat group. */
+ cur_node->calls_comdat_local = true;
+ node->add_to_same_comdat_group (cur_node);
+ }
+
/* Let's take a time profile for splitted function. */
- node->tp_first_run = cur_node->tp_first_run + 1;
+ if (cur_node->tp_first_run)
+ node->tp_first_run = cur_node->tp_first_run + 1;
/* For usual cloning it is enough to clear builtin only when signature
- changes. For partial inlining we however can not expect the part
+ changes. For partial inlining we however cannot expect the part
of builtin implementation to have same semantic as the whole. */
- if (DECL_BUILT_IN (node->decl))
+ if (fndecl_built_in_p (node->decl))
+ set_decl_built_in_function (node->decl, NOT_BUILT_IN, 0);
+
+ /* If return_bb contains any clobbers that refer to SSA_NAMEs
+ set in the split part, remove them. Also reset debug stmts that
+ refer to SSA_NAMEs set in the split part. */
+ if (return_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
{
- DECL_BUILT_IN_CLASS (node->decl) = NOT_BUILT_IN;
- DECL_FUNCTION_CODE (node->decl) = (enum built_in_function) 0;
+ gimple_stmt_iterator gsi = gsi_start_bb (return_bb);
+ while (!gsi_end_p (gsi))
+ {
+ tree op;
+ ssa_op_iter iter;
+ gimple *stmt = gsi_stmt (gsi);
+ bool remove = false;
+ if (gimple_clobber_p (stmt) || is_gimple_debug (stmt))
+ FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
+ {
+ basic_block bb = gimple_bb (SSA_NAME_DEF_STMT (op));
+ if (op != retval
+ && bb
+ && bb != return_bb
+ && bitmap_bit_p (split_point->split_bbs, bb->index))
+ {
+ if (is_gimple_debug (stmt))
+ {
+ gimple_debug_bind_reset_value (stmt);
+ update_stmt (stmt);
+ }
+ else
+ remove = true;
+ break;
+ }
+ }
+ if (remove)
+ gsi_remove (&gsi, true);
+ else
+ gsi_next (&gsi);
+ }
}
- /* If the original function is instrumented then it's
- part is also instrumented. */
- if (with_bounds)
- chkp_function_mark_instrumented (node->decl);
-
/* If the original function is declared inline, there is no point in issuing
a warning for the non-inlinable part. */
DECL_NO_INLINE_WARNING_P (node->decl) = 1;
}
else
break;
+ call_bb->count = split_point->count;
e = split_block (split_point->entry_bb, last_stmt);
remove_edge (e);
args_to_pass[i] = arg;
}
call = gimple_build_call_vec (node->decl, args_to_pass);
- gimple_call_set_with_bounds (call, with_bounds);
gimple_set_block (call, DECL_INITIAL (current_function_decl));
args_to_pass.release ();
vector to say for debug info that if parameter parm had been passed,
it would have value parm_Y(D). */
if (args_to_skip)
- for (parm = DECL_ARGUMENTS (current_function_decl), num = 0;
- parm; parm = DECL_CHAIN (parm), num++)
- if (bitmap_bit_p (args_to_skip, num)
- && is_gimple_reg (parm))
- {
- tree ddecl;
- gimple def_temp;
-
- /* This needs to be done even without MAY_HAVE_DEBUG_STMTS,
- otherwise if it didn't exist before, we'd end up with
- different SSA_NAME_VERSIONs between -g and -g0. */
- arg = get_or_create_ssa_default_def (cfun, parm);
- if (!MAY_HAVE_DEBUG_STMTS)
- continue;
-
- if (debug_args == NULL)
- debug_args = decl_debug_args_insert (node->decl);
- ddecl = make_node (DEBUG_EXPR_DECL);
- DECL_ARTIFICIAL (ddecl) = 1;
- TREE_TYPE (ddecl) = TREE_TYPE (parm);
- DECL_MODE (ddecl) = DECL_MODE (parm);
- vec_safe_push (*debug_args, DECL_ORIGIN (parm));
- vec_safe_push (*debug_args, ddecl);
- def_temp = gimple_build_debug_bind (ddecl, unshare_expr (arg),
- call);
- gsi_insert_after (&gsi, def_temp, GSI_NEW_STMT);
- }
- /* And on the callee side, add
- DEBUG D#Y s=> parm
- DEBUG var => D#Y
- stmts to the first bb where var is a VAR_DECL created for the
- optimized away parameter in DECL_INITIAL block. This hints
- in the debug info that var (whole DECL_ORIGIN is the parm PARM_DECL)
- is optimized away, but could be looked up at the call site
- as value of D#X there. */
- if (debug_args != NULL)
{
- unsigned int i;
- tree var, vexpr;
- gimple_stmt_iterator cgsi;
- gimple def_temp;
-
- push_cfun (DECL_STRUCT_FUNCTION (node->decl));
- var = BLOCK_VARS (DECL_INITIAL (node->decl));
- i = vec_safe_length (*debug_args);
- cgsi = gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
- do
+ vec<tree, va_gc> **debug_args = NULL;
+ unsigned i = 0, len = 0;
+ if (MAY_HAVE_DEBUG_BIND_STMTS)
{
- i -= 2;
- while (var != NULL_TREE
- && DECL_ABSTRACT_ORIGIN (var) != (**debug_args)[i])
- var = TREE_CHAIN (var);
- if (var == NULL_TREE)
- break;
- vexpr = make_node (DEBUG_EXPR_DECL);
- parm = (**debug_args)[i];
- DECL_ARTIFICIAL (vexpr) = 1;
- TREE_TYPE (vexpr) = TREE_TYPE (parm);
- DECL_MODE (vexpr) = DECL_MODE (parm);
- def_temp = gimple_build_debug_source_bind (vexpr, parm,
- NULL);
- gsi_insert_before (&cgsi, def_temp, GSI_SAME_STMT);
- def_temp = gimple_build_debug_bind (var, vexpr, NULL);
- gsi_insert_before (&cgsi, def_temp, GSI_SAME_STMT);
+ debug_args = decl_debug_args_lookup (node->decl);
+ if (debug_args)
+ len = vec_safe_length (*debug_args);
}
- while (i);
- pop_cfun ();
+ for (parm = DECL_ARGUMENTS (current_function_decl), num = 0;
+ parm; parm = DECL_CHAIN (parm), num++)
+ if (bitmap_bit_p (args_to_skip, num) && is_gimple_reg (parm))
+ {
+ tree ddecl;
+ gimple *def_temp;
+
+ /* This needs to be done even without
+ MAY_HAVE_DEBUG_BIND_STMTS, otherwise if it didn't exist
+ before, we'd end up with different SSA_NAME_VERSIONs
+ between -g and -g0. */
+ arg = get_or_create_ssa_default_def (cfun, parm);
+ if (!MAY_HAVE_DEBUG_BIND_STMTS || debug_args == NULL)
+ continue;
+
+ while (i < len && (**debug_args)[i] != DECL_ORIGIN (parm))
+ i += 2;
+ if (i >= len)
+ continue;
+ ddecl = (**debug_args)[i + 1];
+ def_temp
+ = gimple_build_debug_bind (ddecl, unshare_expr (arg), call);
+ gsi_insert_after (&gsi, def_temp, GSI_NEW_STMT);
+ }
+ BITMAP_FREE (args_to_skip);
}
/* We avoid address being taken on any variable used by split part,
|| DECL_BY_REFERENCE (DECL_RESULT (current_function_decl))))
gimple_call_set_return_slot_opt (call, true);
+ if (add_tsan_func_exit)
+ tsan_func_exit_call = gimple_build_call_internal (IFN_TSAN_FUNC_EXIT, 0);
+
/* Update return value. This is bit tricky. When we do not return,
do nothing. When we return we might need to update return_bb
or produce a new return statement. */
if (!split_part_return_p)
- gsi_insert_after (&gsi, call, GSI_NEW_STMT);
+ {
+ gsi_insert_after (&gsi, call, GSI_NEW_STMT);
+ if (tsan_func_exit_call)
+ gsi_insert_after (&gsi, tsan_func_exit_call, GSI_NEW_STMT);
+ }
else
{
- e = make_edge (call_bb, return_bb,
- return_bb == EXIT_BLOCK_PTR_FOR_FN (cfun)
- ? 0 : EDGE_FALLTHRU);
- e->count = call_bb->count;
- e->probability = REG_BR_PROB_BASE;
+ e = make_single_succ_edge (call_bb, return_bb,
+ return_bb == EXIT_BLOCK_PTR_FOR_FN (cfun)
+ ? 0 : EDGE_FALLTHRU);
/* If there is return basic block, see what value we need to store
return value into and put call just before it. */
if (return_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
{
- real_retval = retval = find_retval (return_bb);
- retbnd = find_retbnd (return_bb);
-
+ real_retval = retval;
if (real_retval && split_point->split_part_set_retval)
{
gphi_iterator psi;
break;
}
update_stmt (gsi_stmt (bsi));
- }
-
- /* Replace retbnd with new one. */
- if (retbnd)
- {
- gimple_stmt_iterator bsi;
- for (bsi = gsi_last_bb (return_bb); !gsi_end_p (bsi);
- gsi_prev (&bsi))
- if (gimple_code (gsi_stmt (bsi)) == GIMPLE_RETURN)
- {
- retbnd = copy_ssa_name (retbnd, call);
- gimple_return_set_retbnd (gsi_stmt (bsi), retbnd);
- update_stmt (gsi_stmt (bsi));
- break;
- }
+ /* Also adjust clobbers and debug stmts in return_bb. */
+ for (bsi = gsi_start_bb (return_bb); !gsi_end_p (bsi);
+ gsi_next (&bsi))
+ {
+ gimple *stmt = gsi_stmt (bsi);
+ if (gimple_clobber_p (stmt)
+ || is_gimple_debug (stmt))
+ {
+ ssa_op_iter iter;
+ use_operand_p use_p;
+ bool update = false;
+ FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter,
+ SSA_OP_USE)
+ if (USE_FROM_PTR (use_p) == real_retval)
+ {
+ SET_USE (use_p, retval);
+ update = true;
+ }
+ if (update)
+ update_stmt (stmt);
+ }
+ }
}
}
if (DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
gsi_insert_after (&gsi, call, GSI_NEW_STMT);
if (!useless_type_conversion_p (TREE_TYPE (retval), restype))
{
- gimple cpy;
+ gimple *cpy;
tree tem = create_tmp_reg (restype);
tem = make_ssa_name (tem, call);
cpy = gimple_build_assign (retval, NOP_EXPR, tem);
gsi_insert_after (&gsi, cpy, GSI_NEW_STMT);
retval = tem;
}
- /* Build bndret call to obtain returned bounds. */
- if (retbnd)
- insert_bndret_call_after (retbnd, retval, &gsi);
gimple_call_set_lhs (call, retval);
update_stmt (call);
}
}
else
gsi_insert_after (&gsi, call, GSI_NEW_STMT);
+ if (tsan_func_exit_call)
+ gsi_insert_after (&gsi, tsan_func_exit_call, GSI_NEW_STMT);
}
/* We don't use return block (there is either no return in function or
multiple of them). So create new basic block with return statement.
{
retval = DECL_RESULT (current_function_decl);
- if (chkp_function_instrumented_p (current_function_decl)
- && BOUNDED_P (retval))
- retbnd = create_tmp_reg (pointer_bounds_type_node);
-
/* We use temporary register to hold value when aggregate_value_p
is false. Similarly for DECL_BY_REFERENCE we must avoid extra
copy. */
gimple_call_set_lhs (call, build_simple_mem_ref (retval));
else
gimple_call_set_lhs (call, retval);
+ gsi_insert_after (&gsi, call, GSI_NEW_STMT);
}
- gsi_insert_after (&gsi, call, GSI_NEW_STMT);
- /* Build bndret call to obtain returned bounds. */
- if (retbnd)
- insert_bndret_call_after (retbnd, retval, &gsi);
+ else
+ {
+ gsi_insert_after (&gsi, call, GSI_NEW_STMT);
+ if (retval
+ && is_gimple_reg_type (TREE_TYPE (retval))
+ && !is_gimple_val (retval))
+ {
+ gassign *g
+ = gimple_build_assign (make_ssa_name (TREE_TYPE (retval)),
+ retval);
+ retval = gimple_assign_lhs (g);
+ gsi_insert_after (&gsi, g, GSI_NEW_STMT);
+ }
+ }
+ if (tsan_func_exit_call)
+ gsi_insert_after (&gsi, tsan_func_exit_call, GSI_NEW_STMT);
ret = gimple_build_return (retval);
gsi_insert_after (&gsi, ret, GSI_NEW_STMT);
}
}
free_dominance_info (CDI_DOMINATORS);
free_dominance_info (CDI_POST_DOMINATORS);
- compute_inline_parameters (node, true);
+ compute_fn_summary (node, true);
}
/* Execute function splitting pass. */
{
gimple_stmt_iterator bsi;
basic_block bb;
- int overall_time = 0, overall_size = 0;
+ sreal overall_time = 0;
+ int overall_size = 0;
int todo = 0;
struct cgraph_node *node = cgraph_node::get (current_function_decl);
fprintf (dump_file, "Not splitting: main function.\n");
return 0;
}
+ if (node->frequency == NODE_FREQUENCY_UNLIKELY_EXECUTED)
+ {
+ if (dump_file)
+ fprintf (dump_file, "Not splitting: function is unlikely executed.\n");
+ return 0;
+ }
/* This can be relaxed; function might become inlinable after splitting
away the uninlinable part. */
- if (inline_edge_summary_vec.exists ()
- && !inline_summary (node)->inlinable)
+ if (ipa_fn_summaries
+ && ipa_fn_summaries->get (node)
+ && !ipa_fn_summaries->get (node)->inlinable)
{
if (dump_file)
fprintf (dump_file, "Not splitting: not inlinable.\n");
then inlining would still benefit. */
if ((!node->callers
/* Local functions called once will be completely inlined most of time. */
- || (!node->callers->next_caller && node->local.local))
+ || (!node->callers->next_caller && node->local))
&& !node->address_taken
+ && !node->has_aliases_p ()
&& (!flag_lto || !node->externally_visible))
{
if (dump_file)
return 0;
}
+ if (lookup_attribute ("noinline", DECL_ATTRIBUTES (current_function_decl)))
+ {
+ if (dump_file)
+ fprintf (dump_file, "Not splitting: function is noinline.\n");
+ return 0;
+ }
+ if (lookup_attribute ("section", DECL_ATTRIBUTES (current_function_decl)))
+ {
+ if (dump_file)
+ fprintf (dump_file, "Not splitting: function is in user defined "
+ "section.\n");
+ return 0;
+ }
+
/* We enforce splitting after loop headers when profile info is not
available. */
if (profile_status_for_fn (cfun) != PROFILE_READ)
calculate_dominance_info (CDI_DOMINATORS);
/* Compute local info about basic blocks and determine function size/time. */
- bb_info_vec.safe_grow_cleared (last_basic_block_for_fn (cfun) + 1);
- memset (&best_split_point, 0, sizeof (best_split_point));
+ bb_info_vec.safe_grow_cleared (last_basic_block_for_fn (cfun) + 1, true);
+ best_split_point.split_bbs = NULL;
+ basic_block return_bb = find_return_bb ();
+ int tsan_exit_found = -1;
FOR_EACH_BB_FN (bb, cfun)
{
- int time = 0;
+ sreal time = 0;
int size = 0;
- int freq = compute_call_stmt_bb_frequency (current_function_decl, bb);
+ sreal freq = bb->count.to_sreal_scale
+ (ENTRY_BLOCK_PTR_FOR_FN (cfun)->count);
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file, "Basic block %i\n", bb->index);
for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
{
- int this_time, this_size;
- gimple stmt = gsi_stmt (bsi);
+ sreal this_time;
+ int this_size;
+ gimple *stmt = gsi_stmt (bsi);
this_size = estimate_num_insns (stmt, &eni_size_weights);
- this_time = estimate_num_insns (stmt, &eni_time_weights) * freq;
+ this_time = (sreal)estimate_num_insns (stmt, &eni_time_weights)
+ * freq;
size += this_size;
time += this_time;
check_forbidden_calls (stmt);
if (dump_file && (dump_flags & TDF_DETAILS))
{
- fprintf (dump_file, " freq:%6i size:%3i time:%3i ",
- freq, this_size, this_time);
- print_gimple_stmt (dump_file, stmt, 0, 0);
+ fprintf (dump_file, " freq:%4.2f size:%3i time:%4.2f ",
+ freq.to_double (), this_size, this_time.to_double ());
+ print_gimple_stmt (dump_file, stmt, 0);
+ }
+
+ if ((flag_sanitize & SANITIZE_THREAD)
+ && gimple_call_internal_p (stmt, IFN_TSAN_FUNC_EXIT))
+ {
+ /* We handle TSAN_FUNC_EXIT for splitting either in the
+ return_bb, or in its immediate predecessors. */
+ if ((bb != return_bb && !find_edge (bb, return_bb))
+ || (tsan_exit_found != -1
+ && tsan_exit_found != (bb != return_bb)))
+ {
+ if (dump_file)
+ fprintf (dump_file, "Not splitting: TSAN_FUNC_EXIT"
+ " in unexpected basic block.\n");
+ BITMAP_FREE (forbidden_dominators);
+ bb_info_vec.release ();
+ return 0;
+ }
+ tsan_exit_found = bb != return_bb;
}
}
overall_time += time;
bb_info_vec[bb->index].time = time;
bb_info_vec[bb->index].size = size;
}
- find_split_points (overall_time, overall_size);
+ find_split_points (return_bb, overall_time, overall_size);
if (best_split_point.split_bbs)
{
- split_function (&best_split_point);
+ split_function (return_bb, &best_split_point, tsan_exit_found == 1);
BITMAP_FREE (best_split_point.ssa_names_to_pass);
BITMAP_FREE (best_split_point.split_bbs);
todo = TODO_update_ssa | TODO_cleanup_cfg;
projects / thirdparty / gcc.git / blobdiff