/* Interprocedural constant propagation
- Copyright (C) 2005-2019 Free Software Foundation, Inc.
+ Copyright (C) 2005-2020 Free Software Foundation, Inc.
Contributed by Razya Ladelsky <RAZYA@il.ibm.com> and Martin Jambor
<mjambor@suse.cz>
inline bool set_contains_variable ();
bool add_value (valtype newval, cgraph_edge *cs,
ipcp_value<valtype> *src_val = NULL,
- int src_idx = 0, HOST_WIDE_INT offset = -1);
+ int src_idx = 0, HOST_WIDE_INT offset = -1,
+ ipcp_value<valtype> **val_p = NULL,
+ bool unlimited = false);
void print (FILE * f, bool dump_sources, bool dump_benefits);
};
/* Original overall size of the program. */
-static long overall_size, max_new_size;
+static long overall_size, orig_overall_size;
/* Node name to unique clone suffix number map. */
static hash_map<const char *, unsigned> *clone_num_suffixes;
class ipa_node_params *info;
info = IPA_NODE_REF (node);
- /* Skip constprop clones since we don't make lattices for them. */
- if (info->ipcp_orig_node)
+ /* Skip unoptimized functions and constprop clones since we don't make
+ lattices for them. */
+ if (!info || info->ipcp_orig_node)
continue;
fprintf (f, " Node: %s:\n", node->dump_name ());
count = ipa_get_param_count (info);
if (dump_file)
fprintf (dump_file, "Not considering %s for cloning; "
"-fipa-cp-clone disabled.\n",
- node->name ());
+ node->dump_name ());
return false;
}
if (dump_file)
fprintf (dump_file, "Not considering %s for cloning; "
"optimizing it for size.\n",
- node->name ());
+ node->dump_name ());
return false;
}
{
if (dump_file)
fprintf (dump_file, "Considering %s for cloning; code might shrink.\n",
- node->name ());
+ node->dump_name ());
return true;
}
if (dump_file)
fprintf (dump_file, "Considering %s for cloning; "
"usually called directly.\n",
- node->name ());
+ node->dump_name ());
return true;
}
}
{
if (dump_file)
fprintf (dump_file, "Not considering %s for cloning; no hot calls.\n",
- node->name ());
+ node->dump_name ());
return false;
}
if (dump_file)
fprintf (dump_file, "Considering %s for cloning.\n",
- node->name ());
+ node->dump_name ());
return true;
}
= e->callee->function_or_virtual_thunk_symbol (&avail, e->caller);
return (avail <= AVAIL_INTERPOSABLE
- || !opt_for_fn (e->caller->decl, optimize)
|| !opt_for_fn (ultimate_target->decl, optimize)
- || !opt_for_fn (e->caller->decl, flag_ipa_cp)
|| !opt_for_fn (ultimate_target->decl, flag_ipa_cp));
}
/* Local thunks can be handled transparently, skip them. */
while (cs && cs->caller->thunk.thunk_p && cs->caller->local)
cs = cs->next_caller;
- if (cs)
+ if (cs && IPA_NODE_REF (cs->caller))
{
IPA_NODE_REF (cs->caller)->node_calling_single_call = true;
return true;
}
}
-/* Return the result of a (possibly arithmetic) pass through jump function
- JFUNC on the constant value INPUT. RES_TYPE is the type of the parameter
- to which the result is passed. Return NULL_TREE if that cannot be
- determined or be considered an interprocedural invariant. */
+/* Return the result of a (possibly arithmetic) operation on the constant
+ value INPUT. OPERAND is 2nd operand for binary operation. RES_TYPE is
+ the type of the parameter to which the result is passed. Return
+ NULL_TREE if that cannot be determined or be considered an
+ interprocedural invariant. */
static tree
-ipa_get_jf_pass_through_result (struct ipa_jump_func *jfunc, tree input,
- tree res_type)
+ipa_get_jf_arith_result (enum tree_code opcode, tree input, tree operand,
+ tree res_type)
{
tree res;
- if (ipa_get_jf_pass_through_operation (jfunc) == NOP_EXPR)
+ if (opcode == NOP_EXPR)
return input;
if (!is_gimple_ip_invariant (input))
return NULL_TREE;
- tree_code opcode = ipa_get_jf_pass_through_operation (jfunc);
if (!res_type)
{
if (TREE_CODE_CLASS (opcode) == tcc_comparison)
if (TREE_CODE_CLASS (opcode) == tcc_unary)
res = fold_unary (opcode, res_type, input);
else
- res = fold_binary (opcode, res_type, input,
- ipa_get_jf_pass_through_operand (jfunc));
+ res = fold_binary (opcode, res_type, input, operand);
if (res && !is_gimple_ip_invariant (res))
return NULL_TREE;
return res;
}
+/* Return the result of a (possibly arithmetic) pass through jump function
+ JFUNC on the constant value INPUT. RES_TYPE is the type of the parameter
+ to which the result is passed. Return NULL_TREE if that cannot be
+ determined or be considered an interprocedural invariant. */
+
+static tree
+ipa_get_jf_pass_through_result (struct ipa_jump_func *jfunc, tree input,
+ tree res_type)
+{
+ return ipa_get_jf_arith_result (ipa_get_jf_pass_through_operation (jfunc),
+ input,
+ ipa_get_jf_pass_through_operand (jfunc),
+ res_type);
+}
+
/* Return the result of an ancestor jump function JFUNC on the constant value
INPUT. Return NULL_TREE if that cannot be determined. */
gcc_checking_assert (TREE_CODE (input) != TREE_BINFO);
if (TREE_CODE (input) == ADDR_EXPR)
{
- tree t = TREE_OPERAND (input, 0);
- t = build_ref_for_offset (EXPR_LOCATION (t), t,
- ipa_get_jf_ancestor_offset (jfunc), false,
- ptr_type_node, NULL, false);
- return build_fold_addr_expr (t);
+ gcc_checking_assert (is_gimple_ip_invariant_address (input));
+ poly_int64 off = ipa_get_jf_ancestor_offset (jfunc);
+ if (known_eq (off, 0))
+ return input;
+ poly_int64 byte_offset = exact_div (off, BITS_PER_UNIT);
+ return build1 (ADDR_EXPR, TREE_TYPE (input),
+ fold_build2 (MEM_REF, TREE_TYPE (TREE_TYPE (input)), input,
+ build_int_cst (ptr_type_node, byte_offset)));
}
else
return NULL_TREE;
return ctx;
}
+/* Emulate effects of unary OPERATION and/or conversion from SRC_TYPE to
+ DST_TYPE on value range in SRC_VR and store it to DST_VR. Return true if
+ the result is a range or an anti-range. */
+
+static bool
+ipa_vr_operation_and_type_effects (value_range *dst_vr,
+ value_range *src_vr,
+ enum tree_code operation,
+ tree dst_type, tree src_type)
+{
+ range_fold_unary_expr (dst_vr, operation, dst_type, src_vr, src_type);
+ if (dst_vr->varying_p () || dst_vr->undefined_p ())
+ return false;
+ return true;
+}
+
+/* Determine value_range of JFUNC given that INFO describes the caller node or
+ the one it is inlined to, CS is the call graph edge corresponding to JFUNC
+ and PARM_TYPE of the parameter. */
+
+value_range
+ipa_value_range_from_jfunc (ipa_node_params *info, cgraph_edge *cs,
+ ipa_jump_func *jfunc, tree parm_type)
+{
+ value_range vr;
+ return vr;
+ if (jfunc->m_vr)
+ ipa_vr_operation_and_type_effects (&vr,
+ jfunc->m_vr,
+ NOP_EXPR, parm_type,
+ jfunc->m_vr->type ());
+ if (vr.singleton_p ())
+ return vr;
+ if (jfunc->type == IPA_JF_PASS_THROUGH)
+ {
+ int idx;
+ ipcp_transformation *sum
+ = ipcp_get_transformation_summary (cs->caller->inlined_to
+ ? cs->caller->inlined_to
+ : cs->caller);
+ if (!sum || !sum->m_vr)
+ return vr;
+
+ idx = ipa_get_jf_pass_through_formal_id (jfunc);
+
+ if (!(*sum->m_vr)[idx].known)
+ return vr;
+ tree vr_type = ipa_get_type (info, idx);
+ value_range srcvr (wide_int_to_tree (vr_type, (*sum->m_vr)[idx].min),
+ wide_int_to_tree (vr_type, (*sum->m_vr)[idx].max),
+ (*sum->m_vr)[idx].type);
+
+ enum tree_code operation = ipa_get_jf_pass_through_operation (jfunc);
+
+ if (TREE_CODE_CLASS (operation) == tcc_unary)
+ {
+ value_range res;
+
+ if (ipa_vr_operation_and_type_effects (&res,
+ &srcvr,
+ operation, parm_type,
+ vr_type))
+ vr.intersect (res);
+ }
+ else
+ {
+ value_range op_res, res;
+ tree op = ipa_get_jf_pass_through_operand (jfunc);
+ value_range op_vr (op, op);
+
+ range_fold_binary_expr (&op_res, operation, vr_type, &srcvr, &op_vr);
+ if (ipa_vr_operation_and_type_effects (&res,
+ &op_res,
+ NOP_EXPR, parm_type,
+ vr_type))
+ vr.intersect (res);
+ }
+ }
+ return vr;
+}
+
+/* See if NODE is a clone with a known aggregate value at a given OFFSET of a
+ parameter with the given INDEX. */
+
+static tree
+get_clone_agg_value (struct cgraph_node *node, HOST_WIDE_INT offset,
+ int index)
+{
+ struct ipa_agg_replacement_value *aggval;
+
+ aggval = ipa_get_agg_replacements_for_node (node);
+ while (aggval)
+ {
+ if (aggval->offset == offset
+ && aggval->index == index)
+ return aggval->value;
+ aggval = aggval->next;
+ }
+ return NULL_TREE;
+}
+
+/* Determine whether ITEM, jump function for an aggregate part, evaluates to a
+ single known constant value and if so, return it. Otherwise return NULL.
+ NODE and INFO describes the caller node or the one it is inlined to, and
+ its related info. */
+
+static tree
+ipa_agg_value_from_node (class ipa_node_params *info,
+ struct cgraph_node *node,
+ struct ipa_agg_jf_item *item)
+{
+ tree value = NULL_TREE;
+ int src_idx;
+
+ if (item->offset < 0 || item->jftype == IPA_JF_UNKNOWN)
+ return NULL_TREE;
+
+ if (item->jftype == IPA_JF_CONST)
+ return item->value.constant;
+
+ gcc_checking_assert (item->jftype == IPA_JF_PASS_THROUGH
+ || item->jftype == IPA_JF_LOAD_AGG);
+
+ src_idx = item->value.pass_through.formal_id;
+
+ if (info->ipcp_orig_node)
+ {
+ if (item->jftype == IPA_JF_PASS_THROUGH)
+ value = info->known_csts[src_idx];
+ else
+ value = get_clone_agg_value (node, item->value.load_agg.offset,
+ src_idx);
+ }
+ else if (info->lattices)
+ {
+ class ipcp_param_lattices *src_plats
+ = ipa_get_parm_lattices (info, src_idx);
+
+ if (item->jftype == IPA_JF_PASS_THROUGH)
+ {
+ struct ipcp_lattice<tree> *lat = &src_plats->itself;
+
+ if (!lat->is_single_const ())
+ return NULL_TREE;
+
+ value = lat->values->value;
+ }
+ else if (src_plats->aggs
+ && !src_plats->aggs_bottom
+ && !src_plats->aggs_contain_variable
+ && src_plats->aggs_by_ref == item->value.load_agg.by_ref)
+ {
+ struct ipcp_agg_lattice *aglat;
+
+ for (aglat = src_plats->aggs; aglat; aglat = aglat->next)
+ {
+ if (aglat->offset > item->value.load_agg.offset)
+ break;
+
+ if (aglat->offset == item->value.load_agg.offset)
+ {
+ if (aglat->is_single_const ())
+ value = aglat->values->value;
+ break;
+ }
+ }
+ }
+ }
+
+ if (!value)
+ return NULL_TREE;
+
+ if (item->jftype == IPA_JF_LOAD_AGG)
+ {
+ tree load_type = item->value.load_agg.type;
+ tree value_type = TREE_TYPE (value);
+
+ /* Ensure value type is compatible with load type. */
+ if (!useless_type_conversion_p (load_type, value_type))
+ return NULL_TREE;
+ }
+
+ return ipa_get_jf_arith_result (item->value.pass_through.operation,
+ value,
+ item->value.pass_through.operand,
+ item->type);
+}
+
+/* Determine whether AGG_JFUNC evaluates to a set of known constant value for
+ an aggregate and if so, return it. Otherwise return an empty set. NODE
+ and INFO describes the caller node or the one it is inlined to, and its
+ related info. */
+
+struct ipa_agg_value_set
+ipa_agg_value_set_from_jfunc (class ipa_node_params *info, cgraph_node *node,
+ struct ipa_agg_jump_function *agg_jfunc)
+{
+ struct ipa_agg_value_set agg;
+ struct ipa_agg_jf_item *item;
+ int i;
+
+ agg.items = vNULL;
+ agg.by_ref = agg_jfunc->by_ref;
+
+ FOR_EACH_VEC_SAFE_ELT (agg_jfunc->items, i, item)
+ {
+ tree value = ipa_agg_value_from_node (info, node, item);
+
+ if (value)
+ {
+ struct ipa_agg_value value_item;
+
+ value_item.offset = item->offset;
+ value_item.value = value;
+
+ agg.items.safe_push (value_item);
+ }
+ }
+ return agg;
+}
+
/* If checking is enabled, verify that no lattice is in the TOP state, i.e. not
bottom, not containing a variable component and without any known value at
the same time. */
/* Try to add NEWVAL to LAT, potentially creating a new ipcp_value for it. CS,
SRC_VAL SRC_INDEX and OFFSET are meant for add_source and have the same
meaning. OFFSET -1 means the source is scalar and not a part of an
- aggregate. */
+ aggregate. If non-NULL, VAL_P records address of existing or newly added
+ ipcp_value. UNLIMITED means whether value count should not exceed the limit
+ given by PARAM_IPA_CP_VALUE_LIST_SIZE. */
template <typename valtype>
bool
ipcp_lattice<valtype>::add_value (valtype newval, cgraph_edge *cs,
ipcp_value<valtype> *src_val,
- int src_idx, HOST_WIDE_INT offset)
+ int src_idx, HOST_WIDE_INT offset,
+ ipcp_value<valtype> **val_p,
+ bool unlimited)
{
- ipcp_value<valtype> *val;
+ ipcp_value<valtype> *val, *last_val = NULL;
+
+ if (val_p)
+ *val_p = NULL;
if (bottom)
return false;
- for (val = values; val; val = val->next)
+ for (val = values; val; last_val = val, val = val->next)
if (values_equal_for_ipcp_p (val->value, newval))
{
+ if (val_p)
+ *val_p = val;
+
if (ipa_edge_within_scc (cs))
{
ipcp_value_source<valtype> *s;
for (s = val->sources; s; s = s->next)
- if (s->cs == cs)
+ if (s->cs == cs && s->val == src_val)
break;
if (s)
return false;
return false;
}
- if (values_count == param_ipa_cp_value_list_size)
+ if (!unlimited && values_count == opt_for_fn (cs->caller->decl,
+ param_ipa_cp_value_list_size))
{
/* We can only free sources, not the values themselves, because sources
of other values in this SCC might point to them. */
ipcp_sources_pool.remove ((ipcp_value_source<tree>*)src);
}
}
-
values = NULL;
return set_to_bottom ();
}
values_count++;
val = allocate_and_init_ipcp_value (newval);
val->add_source (cs, src_val, src_idx, offset);
- val->next = values;
- values = val;
+ val->next = NULL;
+
+ /* Add the new value to end of value list, which can reduce iterations
+ of propagation stage for recursive function. */
+ if (last_val)
+ last_val->next = val;
+ else
+ values = val;
+
+ if (val_p)
+ *val_p = val;
+
return true;
}
-/* Propagate values through a pass-through jump function JFUNC associated with
- edge CS, taking values from SRC_LAT and putting them into DEST_LAT. SRC_IDX
- is the index of the source parameter. PARM_TYPE is the type of the
- parameter to which the result is passed. */
+/* Return true, if a ipcp_value VAL is orginated from parameter value of
+ self-feeding recursive function via some kind of pass-through jump
+ function. */
static bool
-propagate_vals_across_pass_through (cgraph_edge *cs, ipa_jump_func *jfunc,
- ipcp_lattice<tree> *src_lat,
- ipcp_lattice<tree> *dest_lat, int src_idx,
- tree parm_type)
+self_recursively_generated_p (ipcp_value<tree> *val)
+{
+ class ipa_node_params *info = NULL;
+
+ for (ipcp_value_source<tree> *src = val->sources; src; src = src->next)
+ {
+ cgraph_edge *cs = src->cs;
+
+ if (!src->val || cs->caller != cs->callee->function_symbol ())
+ return false;
+
+ if (src->val == val)
+ continue;
+
+ if (!info)
+ info = IPA_NODE_REF (cs->caller);
+
+ class ipcp_param_lattices *plats = ipa_get_parm_lattices (info,
+ src->index);
+ ipcp_lattice<tree> *src_lat;
+ ipcp_value<tree> *src_val;
+
+ if (src->offset == -1)
+ src_lat = &plats->itself;
+ else
+ {
+ struct ipcp_agg_lattice *src_aglat;
+
+ for (src_aglat = plats->aggs; src_aglat; src_aglat = src_aglat->next)
+ if (src_aglat->offset == src->offset)
+ break;
+
+ if (!src_aglat)
+ return false;
+
+ src_lat = src_aglat;
+ }
+
+ for (src_val = src_lat->values; src_val; src_val = src_val->next)
+ if (src_val == val)
+ break;
+
+ if (!src_val)
+ return false;
+ }
+
+ return true;
+}
+
+/* A helper function that returns result of operation specified by OPCODE on
+ the value of SRC_VAL. If non-NULL, OPND1_TYPE is expected type for the
+ value of SRC_VAL. If the operation is binary, OPND2 is a constant value
+ acting as its second operand. If non-NULL, RES_TYPE is expected type of
+ the result. */
+
+static tree
+get_val_across_arith_op (enum tree_code opcode,
+ tree opnd1_type,
+ tree opnd2,
+ ipcp_value<tree> *src_val,
+ tree res_type)
+{
+ tree opnd1 = src_val->value;
+
+ /* Skip source values that is incompatible with specified type. */
+ if (opnd1_type
+ && !useless_type_conversion_p (opnd1_type, TREE_TYPE (opnd1)))
+ return NULL_TREE;
+
+ return ipa_get_jf_arith_result (opcode, opnd1, opnd2, res_type);
+}
+
+/* Propagate values through an arithmetic transformation described by a jump
+ function associated with edge CS, taking values from SRC_LAT and putting
+ them into DEST_LAT. OPND1_TYPE is expected type for the values in SRC_LAT.
+ OPND2 is a constant value if transformation is a binary operation.
+ SRC_OFFSET specifies offset in an aggregate if SRC_LAT describes lattice of
+ a part of the aggregate. SRC_IDX is the index of the source parameter.
+ RES_TYPE is the value type of result being propagated into. Return true if
+ DEST_LAT changed. */
+
+static bool
+propagate_vals_across_arith_jfunc (cgraph_edge *cs,
+ enum tree_code opcode,
+ tree opnd1_type,
+ tree opnd2,
+ ipcp_lattice<tree> *src_lat,
+ ipcp_lattice<tree> *dest_lat,
+ HOST_WIDE_INT src_offset,
+ int src_idx,
+ tree res_type)
{
ipcp_value<tree> *src_val;
bool ret = false;
- /* Do not create new values when propagating within an SCC because if there
- are arithmetic functions with circular dependencies, there is infinite
- number of them and we would just make lattices bottom. If this condition
- is ever relaxed we have to detect self-feeding recursive calls in
- cgraph_edge_brings_value_p in a smarter way. */
- if ((ipa_get_jf_pass_through_operation (jfunc) != NOP_EXPR)
- && ipa_edge_within_scc (cs))
- ret = dest_lat->set_contains_variable ();
+ /* Due to circular dependencies, propagating within an SCC through arithmetic
+ transformation would create infinite number of values. But for
+ self-feeding recursive function, we could allow propagation in a limited
+ count, and this can enable a simple kind of recursive function versioning.
+ For other scenario, we would just make lattices bottom. */
+ if (opcode != NOP_EXPR && ipa_edge_within_scc (cs))
+ {
+ int i;
+
+ int max_recursive_depth = opt_for_fn(cs->caller->decl,
+ param_ipa_cp_max_recursive_depth);
+ if (src_lat != dest_lat || max_recursive_depth < 1)
+ return dest_lat->set_contains_variable ();
+
+ /* No benefit if recursive execution is in low probability. */
+ if (cs->sreal_frequency () * 100
+ <= ((sreal) 1) * opt_for_fn (cs->caller->decl,
+ param_ipa_cp_min_recursive_probability))
+ return dest_lat->set_contains_variable ();
+
+ auto_vec<ipcp_value<tree> *, 8> val_seeds;
+
+ for (src_val = src_lat->values; src_val; src_val = src_val->next)
+ {
+ /* Now we do not use self-recursively generated value as propagation
+ source, this is absolutely conservative, but could avoid explosion
+ of lattice's value space, especially when one recursive function
+ calls another recursive. */
+ if (self_recursively_generated_p (src_val))
+ {
+ ipcp_value_source<tree> *s;
+
+ /* If the lattice has already been propagated for the call site,
+ no need to do that again. */
+ for (s = src_val->sources; s; s = s->next)
+ if (s->cs == cs)
+ return dest_lat->set_contains_variable ();
+ }
+ else
+ val_seeds.safe_push (src_val);
+ }
+
+ gcc_assert ((int) val_seeds.length () <= param_ipa_cp_value_list_size);
+
+ /* Recursively generate lattice values with a limited count. */
+ FOR_EACH_VEC_ELT (val_seeds, i, src_val)
+ {
+ for (int j = 1; j < max_recursive_depth; j++)
+ {
+ tree cstval = get_val_across_arith_op (opcode, opnd1_type, opnd2,
+ src_val, res_type);
+ if (!cstval)
+ break;
+
+ ret |= dest_lat->add_value (cstval, cs, src_val, src_idx,
+ src_offset, &src_val, true);
+ gcc_checking_assert (src_val);
+ }
+ }
+ ret |= dest_lat->set_contains_variable ();
+ }
else
for (src_val = src_lat->values; src_val; src_val = src_val->next)
{
- tree cstval = ipa_get_jf_pass_through_result (jfunc, src_val->value,
- parm_type);
+ /* Now we do not use self-recursively generated value as propagation
+ source, otherwise it is easy to make value space of normal lattice
+ overflow. */
+ if (self_recursively_generated_p (src_val))
+ {
+ ret |= dest_lat->set_contains_variable ();
+ continue;
+ }
+ tree cstval = get_val_across_arith_op (opcode, opnd1_type, opnd2,
+ src_val, res_type);
if (cstval)
- ret |= dest_lat->add_value (cstval, cs, src_val, src_idx);
+ ret |= dest_lat->add_value (cstval, cs, src_val, src_idx,
+ src_offset);
else
ret |= dest_lat->set_contains_variable ();
}
return ret;
}
+/* Propagate values through a pass-through jump function JFUNC associated with
+ edge CS, taking values from SRC_LAT and putting them into DEST_LAT. SRC_IDX
+ is the index of the source parameter. PARM_TYPE is the type of the
+ parameter to which the result is passed. */
+
+static bool
+propagate_vals_across_pass_through (cgraph_edge *cs, ipa_jump_func *jfunc,
+ ipcp_lattice<tree> *src_lat,
+ ipcp_lattice<tree> *dest_lat, int src_idx,
+ tree parm_type)
+{
+ return propagate_vals_across_arith_jfunc (cs,
+ ipa_get_jf_pass_through_operation (jfunc),
+ NULL_TREE,
+ ipa_get_jf_pass_through_operand (jfunc),
+ src_lat, dest_lat, -1, src_idx, parm_type);
+}
+
/* Propagate values through an ancestor jump function JFUNC associated with
edge CS, taking values from SRC_LAT and putting them into DEST_LAT. SRC_IDX
is the index of the source parameter. */
added_sth = true;
}
}
-
}
prop_fail:
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file, "Setting dest_lattice to bottom, because type of "
"param %i of %s is NULL or unsuitable for bits propagation\n",
- idx, cs->callee->name ());
+ idx, cs->callee->dump_name ());
return dest_lattice->set_to_bottom ();
}
return dest_lattice->set_to_bottom ();
}
-/* Emulate effects of unary OPERATION and/or conversion from SRC_TYPE to
- DST_TYPE on value range in SRC_VR and store it to DST_VR. Return true if
- the result is a range or an anti-range. */
-
-static bool
-ipa_vr_operation_and_type_effects (value_range *dst_vr,
- value_range *src_vr,
- enum tree_code operation,
- tree dst_type, tree src_type)
-{
- range_fold_unary_expr (dst_vr, operation, dst_type, src_vr, src_type);
- if (dst_vr->varying_p () || dst_vr->undefined_p ())
- return false;
- return true;
-}
-
/* Propagate value range across jump function JFUNC that is associated with
edge CS with param of callee of PARAM_TYPE and update DEST_PLATS
accordingly. */
if (jfunc->type == IPA_JF_PASS_THROUGH)
{
enum tree_code operation = ipa_get_jf_pass_through_operation (jfunc);
+ class ipa_node_params *caller_info = IPA_NODE_REF (cs->caller);
+ int src_idx = ipa_get_jf_pass_through_formal_id (jfunc);
+ class ipcp_param_lattices *src_lats
+ = ipa_get_parm_lattices (caller_info, src_idx);
+ tree operand_type = ipa_get_type (caller_info, src_idx);
+ if (src_lats->m_value_range.bottom_p ())
+ return dest_lat->set_to_bottom ();
+
+ value_range vr;
if (TREE_CODE_CLASS (operation) == tcc_unary)
+ ipa_vr_operation_and_type_effects (&vr,
+ &src_lats->m_value_range.m_vr,
+ operation, param_type,
+ operand_type);
+ /* A crude way to prevent unbounded number of value range updates
+ in SCC components. We should allow limited number of updates within
+ SCC, too. */
+ else if (!ipa_edge_within_scc (cs))
{
- class ipa_node_params *caller_info = IPA_NODE_REF (cs->caller);
- int src_idx = ipa_get_jf_pass_through_formal_id (jfunc);
- tree operand_type = ipa_get_type (caller_info, src_idx);
- class ipcp_param_lattices *src_lats
- = ipa_get_parm_lattices (caller_info, src_idx);
-
- if (src_lats->m_value_range.bottom_p ())
- return dest_lat->set_to_bottom ();
- value_range vr;
- if (ipa_vr_operation_and_type_effects (&vr,
- &src_lats->m_value_range.m_vr,
- operation, param_type,
- operand_type))
- return dest_lat->meet_with (&vr);
+ tree op = ipa_get_jf_pass_through_operand (jfunc);
+ value_range op_vr (op, op);
+ value_range op_res,res;
+
+ range_fold_binary_expr (&op_res, operation, operand_type,
+ &src_lats->m_value_range.m_vr, &op_vr);
+ ipa_vr_operation_and_type_effects (&vr,
+ &op_res,
+ NOP_EXPR, param_type,
+ operand_type);
+ }
+ if (!vr.undefined_p () && !vr.varying_p ())
+ {
+ if (jfunc->m_vr)
+ {
+ value_range jvr;
+ if (ipa_vr_operation_and_type_effects (&jvr, jfunc->m_vr,
+ NOP_EXPR,
+ param_type,
+ jfunc->m_vr->type ()))
+ vr.intersect (jvr);
+ }
+ return dest_lat->meet_with (&vr);
}
}
else if (jfunc->type == IPA_JF_CONST)
unless there are too many already. If there are two many, return false. If
there are overlaps turn whole DEST_PLATS to bottom and return false. If any
skipped lattices were newly marked as containing variable, set *CHANGE to
- true. */
+ true. MAX_AGG_ITEMS is the maximum number of lattices. */
static bool
merge_agg_lats_step (class ipcp_param_lattices *dest_plats,
HOST_WIDE_INT offset, HOST_WIDE_INT val_size,
struct ipcp_agg_lattice ***aglat,
- bool pre_existing, bool *change)
+ bool pre_existing, bool *change, int max_agg_items)
{
gcc_checking_assert (offset >= 0);
set_agg_lats_to_bottom (dest_plats);
return false;
}
- if (dest_plats->aggs_count == param_ipa_max_agg_items)
+ if (dest_plats->aggs_count == max_agg_items)
return false;
dest_plats->aggs_count++;
new_al = ipcp_agg_lattice_pool.allocate ();
ret |= set_agg_lats_contain_variable (dest_plats);
dst_aglat = &dest_plats->aggs;
+ int max_agg_items = opt_for_fn (cs->callee->function_symbol ()->decl,
+ param_ipa_max_agg_items);
for (struct ipcp_agg_lattice *src_aglat = src_plats->aggs;
src_aglat;
src_aglat = src_aglat->next)
if (new_offset < 0)
continue;
if (merge_agg_lats_step (dest_plats, new_offset, src_aglat->size,
- &dst_aglat, pre_existing, &ret))
+ &dst_aglat, pre_existing, &ret, max_agg_items))
{
struct ipcp_agg_lattice *new_al = *dst_aglat;
|| ipa_get_jf_pass_through_agg_preserved (jfunc));
}
+/* Propagate values through ITEM, jump function for a part of an aggregate,
+ into corresponding aggregate lattice AGLAT. CS is the call graph edge
+ associated with the jump function. Return true if AGLAT changed in any
+ way. */
+
+static bool
+propagate_aggregate_lattice (struct cgraph_edge *cs,
+ struct ipa_agg_jf_item *item,
+ struct ipcp_agg_lattice *aglat)
+{
+ class ipa_node_params *caller_info;
+ class ipcp_param_lattices *src_plats;
+ struct ipcp_lattice<tree> *src_lat;
+ HOST_WIDE_INT src_offset;
+ int src_idx;
+ tree load_type;
+ bool ret;
+
+ if (item->jftype == IPA_JF_CONST)
+ {
+ tree value = item->value.constant;
+
+ gcc_checking_assert (is_gimple_ip_invariant (value));
+ return aglat->add_value (value, cs, NULL, 0);
+ }
+
+ gcc_checking_assert (item->jftype == IPA_JF_PASS_THROUGH
+ || item->jftype == IPA_JF_LOAD_AGG);
+
+ caller_info = IPA_NODE_REF (cs->caller);
+ src_idx = item->value.pass_through.formal_id;
+ src_plats = ipa_get_parm_lattices (caller_info, src_idx);
+
+ if (item->jftype == IPA_JF_PASS_THROUGH)
+ {
+ load_type = NULL_TREE;
+ src_lat = &src_plats->itself;
+ src_offset = -1;
+ }
+ else
+ {
+ HOST_WIDE_INT load_offset = item->value.load_agg.offset;
+ struct ipcp_agg_lattice *src_aglat;
+
+ for (src_aglat = src_plats->aggs; src_aglat; src_aglat = src_aglat->next)
+ if (src_aglat->offset >= load_offset)
+ break;
+
+ load_type = item->value.load_agg.type;
+ if (!src_aglat
+ || src_aglat->offset > load_offset
+ || src_aglat->size != tree_to_shwi (TYPE_SIZE (load_type))
+ || src_plats->aggs_by_ref != item->value.load_agg.by_ref)
+ return aglat->set_contains_variable ();
+
+ src_lat = src_aglat;
+ src_offset = load_offset;
+ }
+
+ if (src_lat->bottom
+ || (!ipcp_versionable_function_p (cs->caller)
+ && !src_lat->is_single_const ()))
+ return aglat->set_contains_variable ();
+
+ ret = propagate_vals_across_arith_jfunc (cs,
+ item->value.pass_through.operation,
+ load_type,
+ item->value.pass_through.operand,
+ src_lat, aglat,
+ src_offset,
+ src_idx,
+ item->type);
+
+ if (src_lat->contains_variable)
+ ret |= aglat->set_contains_variable ();
+
+ return ret;
+}
+
/* Propagate scalar values across jump function JFUNC that is associated with
edge CS and put the values into DEST_LAT. */
if (set_check_aggs_by_ref (dest_plats, jfunc->agg.by_ref))
return true;
+ int max_agg_items = opt_for_fn (cs->callee->function_symbol ()->decl,
+ param_ipa_max_agg_items);
FOR_EACH_VEC_ELT (*jfunc->agg.items, i, item)
{
HOST_WIDE_INT val_size;
- if (item->offset < 0)
+ if (item->offset < 0 || item->jftype == IPA_JF_UNKNOWN)
continue;
- gcc_checking_assert (is_gimple_ip_invariant (item->value));
- val_size = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (item->value)));
+ val_size = tree_to_shwi (TYPE_SIZE (item->type));
if (merge_agg_lats_step (dest_plats, item->offset, val_size,
- &aglat, pre_existing, &ret))
+ &aglat, pre_existing, &ret, max_agg_items))
{
- ret |= (*aglat)->add_value (item->value, cs, NULL, 0, 0);
+ ret |= propagate_aggregate_lattice (cs, item, *aglat);
aglat = &(*aglat)->next;
}
else if (dest_plats->aggs_bottom)
ipa_get_indirect_edge_target_1 (struct cgraph_edge *ie,
vec<tree> known_csts,
vec<ipa_polymorphic_call_context> known_contexts,
- vec<ipa_agg_jump_function_p> known_aggs,
+ vec<ipa_agg_value_set> known_aggs,
struct ipa_agg_replacement_value *agg_reps,
bool *speculative)
{
int param_index = ie->indirect_info->param_index;
HOST_WIDE_INT anc_offset;
- tree t;
+ tree t = NULL;
tree target = NULL;
*speculative = false;
- if (param_index == -1
- || known_csts.length () <= (unsigned int) param_index)
+ if (param_index == -1)
return NULL_TREE;
if (!ie->indirect_info->polymorphic)
{
- tree t;
+ tree t = NULL;
if (ie->indirect_info->agg_contents)
{
}
if (!t)
{
- struct ipa_agg_jump_function *agg;
+ struct ipa_agg_value_set *agg;
if (known_aggs.length () > (unsigned int) param_index)
- agg = known_aggs[param_index];
+ agg = &known_aggs[param_index];
else
agg = NULL;
bool from_global_constant;
- t = ipa_find_agg_cst_for_param (agg, known_csts[param_index],
+ t = ipa_find_agg_cst_for_param (agg,
+ (unsigned) param_index
+ < known_csts.length ()
+ ? known_csts[param_index]
+ : NULL,
ie->indirect_info->offset,
ie->indirect_info->by_ref,
&from_global_constant);
t = NULL_TREE;
}
}
- else
+ else if ((unsigned) param_index < known_csts.length ())
t = known_csts[param_index];
if (t
if (!t && known_aggs.length () > (unsigned int) param_index
&& !ie->indirect_info->by_ref)
{
- struct ipa_agg_jump_function *agg;
- agg = known_aggs[param_index];
- t = ipa_find_agg_cst_for_param (agg, known_csts[param_index],
+ struct ipa_agg_value_set *agg = &known_aggs[param_index];
+ t = ipa_find_agg_cst_for_param (agg,
+ (unsigned) param_index
+ < known_csts.length ()
+ ? known_csts[param_index] : NULL,
ie->indirect_info->offset, true);
}
}
/* Do we know the constant value of pointer? */
- if (!t)
+ if (!t && (unsigned) param_index < known_csts.length ())
t = known_csts[param_index];
gcc_checking_assert (!t || TREE_CODE (t) != TREE_BINFO);
ipa_get_indirect_edge_target (struct cgraph_edge *ie,
vec<tree> known_csts,
vec<ipa_polymorphic_call_context> known_contexts,
- vec<ipa_agg_jump_function_p> known_aggs,
+ vec<ipa_agg_value_set> known_aggs,
bool *speculative)
{
return ipa_get_indirect_edge_target_1 (ie, known_csts, known_contexts,
devirtualization_time_bonus (struct cgraph_node *node,
vec<tree> known_csts,
vec<ipa_polymorphic_call_context> known_contexts,
- vec<ipa_agg_jump_function_p> known_aggs)
+ vec<ipa_agg_value_set> known_aggs)
{
struct cgraph_edge *ie;
int res = 0;
if (avail < AVAIL_AVAILABLE)
continue;
isummary = ipa_fn_summaries->get (callee);
- if (!isummary->inlinable)
+ if (!isummary || !isummary->inlinable)
continue;
int size = ipa_size_summaries->get (callee)->size;
/* FIXME: The values below need re-considering and perhaps also
integrating into the cost metrics, at lest in some very basic way. */
- if (size <= param_max_inline_insns_auto / 4)
+ int max_inline_insns_auto
+ = opt_for_fn (callee->decl, param_max_inline_insns_auto);
+ if (size <= max_inline_insns_auto / 4)
res += 31 / ((int)speculative + 1);
- else if (size <= param_max_inline_insns_auto / 2)
+ else if (size <= max_inline_insns_auto / 2)
res += 15 / ((int)speculative + 1);
- else if (size <= param_max_inline_insns_auto
+ else if (size <= max_inline_insns_auto
|| DECL_DECLARED_INLINE_P (callee->decl))
res += 7 / ((int)speculative + 1);
}
/* Return time bonus incurred because of HINTS. */
static int
-hint_time_bonus (ipa_hints hints)
+hint_time_bonus (cgraph_node *node, ipa_hints hints)
{
int result = 0;
if (hints & (INLINE_HINT_loop_iterations | INLINE_HINT_loop_stride))
- result += param_ipa_cp_loop_hint_bonus;
+ result += opt_for_fn (node->decl, param_ipa_cp_loop_hint_bonus);
return result;
}
cloning goodness evaluation, do so. */
static inline int64_t
-incorporate_penalties (ipa_node_params *info, int64_t evaluation)
+incorporate_penalties (cgraph_node *node, ipa_node_params *info,
+ int64_t evaluation)
{
- if (info->node_within_scc)
+ if (info->node_within_scc && !info->node_is_self_scc)
evaluation = (evaluation
- * (100 - param_ipa_cp_recursion_penalty)) / 100;
+ * (100 - opt_for_fn (node->decl,
+ param_ipa_cp_recursion_penalty))) / 100;
if (info->node_calling_single_call)
evaluation = (evaluation
- * (100 - param_ipa_cp_single_call_penalty))
+ * (100 - opt_for_fn (node->decl,
+ param_ipa_cp_single_call_penalty)))
/ 100;
return evaluation;
gcc_assert (size_cost > 0);
class ipa_node_params *info = IPA_NODE_REF (node);
+ int eval_threshold = opt_for_fn (node->decl, param_ipa_cp_eval_threshold);
if (max_count > profile_count::zero ())
{
int factor = RDIV (count_sum.probability_in
* 1000, REG_BR_PROB_BASE);
int64_t evaluation = (((int64_t) time_benefit * factor)
/ size_cost);
- evaluation = incorporate_penalties (info, evaluation);
+ evaluation = incorporate_penalties (node, info, evaluation);
if (dump_file && (dump_flags & TDF_DETAILS))
{
count_sum.dump (dump_file);
fprintf (dump_file, "%s%s) -> evaluation: " "%" PRId64
", threshold: %i\n",
- info->node_within_scc ? ", scc" : "",
+ info->node_within_scc
+ ? (info->node_is_self_scc ? ", self_scc" : ", scc") : "",
info->node_calling_single_call ? ", single_call" : "",
- evaluation, param_ipa_cp_eval_threshold);
+ evaluation, eval_threshold);
}
- return evaluation >= param_ipa_cp_eval_threshold;
+ return evaluation >= eval_threshold;
}
else
{
int64_t evaluation = (((int64_t) time_benefit * freq_sum)
/ size_cost);
- evaluation = incorporate_penalties (info, evaluation);
+ evaluation = incorporate_penalties (node, info, evaluation);
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file, " good_cloning_opportunity_p (time: %i, "
"size: %i, freq_sum: %i%s%s) -> evaluation: "
"%" PRId64 ", threshold: %i\n",
time_benefit, size_cost, freq_sum,
- info->node_within_scc ? ", scc" : "",
+ info->node_within_scc
+ ? (info->node_is_self_scc ? ", self_scc" : ", scc") : "",
info->node_calling_single_call ? ", single_call" : "",
- evaluation, param_ipa_cp_eval_threshold);
+ evaluation, eval_threshold);
- return evaluation >= param_ipa_cp_eval_threshold;
+ return evaluation >= eval_threshold;
}
}
/* Return all context independent values from aggregate lattices in PLATS in a
vector. Return NULL if there are none. */
-static vec<ipa_agg_jf_item, va_gc> *
+static vec<ipa_agg_value>
context_independent_aggregate_values (class ipcp_param_lattices *plats)
{
- vec<ipa_agg_jf_item, va_gc> *res = NULL;
+ vec<ipa_agg_value> res = vNULL;
if (plats->aggs_bottom
|| plats->aggs_contain_variable
|| plats->aggs_count == 0)
- return NULL;
+ return vNULL;
for (struct ipcp_agg_lattice *aglat = plats->aggs;
aglat;
aglat = aglat->next)
if (aglat->is_single_const ())
{
- struct ipa_agg_jf_item item;
+ struct ipa_agg_value item;
item.offset = aglat->offset;
item.value = aglat->values->value;
- vec_safe_push (res, item);
+ res.safe_push (item);
}
return res;
}
vec<tree> *known_csts,
vec<ipa_polymorphic_call_context>
*known_contexts,
- vec<ipa_agg_jump_function> *known_aggs,
+ vec<ipa_agg_value_set> *known_aggs,
int *removable_params_cost)
{
int i, count = ipa_get_param_count (info);
if (known_aggs)
{
- vec<ipa_agg_jf_item, va_gc> *agg_items;
- struct ipa_agg_jump_function *ajf;
+ vec<ipa_agg_value> agg_items;
+ struct ipa_agg_value_set *agg;
agg_items = context_independent_aggregate_values (plats);
- ajf = &(*known_aggs)[i];
- ajf->items = agg_items;
- ajf->by_ref = plats->aggs_by_ref;
- ret |= agg_items != NULL;
+ agg = &(*known_aggs)[i];
+ agg->items = agg_items;
+ agg->by_ref = plats->aggs_by_ref;
+ ret |= !agg_items.is_empty ();
}
}
return ret;
}
-/* The current interface in ipa-inline-analysis requires a pointer vector.
- Create it.
-
- FIXME: That interface should be re-worked, this is slightly silly. Still,
- I'd like to discuss how to change it first and this demonstrates the
- issue. */
-
-static vec<ipa_agg_jump_function_p>
-agg_jmp_p_vec_for_t_vec (vec<ipa_agg_jump_function> known_aggs)
-{
- vec<ipa_agg_jump_function_p> ret;
- struct ipa_agg_jump_function *ajf;
- int i;
-
- ret.create (known_aggs.length ());
- FOR_EACH_VEC_ELT (known_aggs, i, ajf)
- ret.quick_push (ajf);
- return ret;
-}
-
/* Perform time and size measurement of NODE with the context given in
KNOWN_CSTS, KNOWN_CONTEXTS and KNOWN_AGGS, calculate the benefit and cost
given BASE_TIME of the node without specialization, REMOVABLE_PARAMS_COST of
static void
perform_estimation_of_a_value (cgraph_node *node, vec<tree> known_csts,
vec<ipa_polymorphic_call_context> known_contexts,
- vec<ipa_agg_jump_function_p> known_aggs_ptrs,
+ vec<ipa_agg_value_set> known_aggs,
int removable_params_cost,
int est_move_cost, ipcp_value_base *val)
{
ipa_hints hints;
estimate_ipcp_clone_size_and_time (node, known_csts, known_contexts,
- known_aggs_ptrs, &size, &time,
+ known_aggs, &size, &time,
&base_time, &hints);
base_time -= time;
if (base_time > 65535)
else
time_benefit = base_time.to_int ()
+ devirtualization_time_bonus (node, known_csts, known_contexts,
- known_aggs_ptrs)
- + hint_time_bonus (hints)
+ known_aggs)
+ + hint_time_bonus (node, hints)
+ removable_params_cost + est_move_cost;
gcc_checking_assert (size >=0);
val->local_size_cost = size;
}
+/* Get the overall limit oof growth based on parameters extracted from growth.
+ it does not really make sense to mix functions with different overall growth
+ limits but it is possible and if it happens, we do not want to select one
+ limit at random. */
+
+static long
+get_max_overall_size (cgraph_node *node)
+{
+ long max_new_size = orig_overall_size;
+ long large_unit = opt_for_fn (node->decl, param_large_unit_insns);
+ if (max_new_size < large_unit)
+ max_new_size = large_unit;
+ int unit_growth = opt_for_fn (node->decl, param_ipa_cp_unit_growth);
+ max_new_size += max_new_size * unit_growth / 100 + 1;
+ return max_new_size;
+}
+
/* Iterate over known values of parameters of NODE and estimate the local
effects in terms of time and size they have. */
int i, count = ipa_get_param_count (info);
vec<tree> known_csts;
vec<ipa_polymorphic_call_context> known_contexts;
- vec<ipa_agg_jump_function> known_aggs;
- vec<ipa_agg_jump_function_p> known_aggs_ptrs;
+ vec<ipa_agg_value_set> known_aggs;
bool always_const;
int removable_params_cost;
always_const = gather_context_independent_values (info, &known_csts,
&known_contexts, &known_aggs,
&removable_params_cost);
- known_aggs_ptrs = agg_jmp_p_vec_for_t_vec (known_aggs);
int devirt_bonus = devirtualization_time_bonus (node, known_csts,
- known_contexts, known_aggs_ptrs);
+ known_contexts, known_aggs);
if (always_const || devirt_bonus
|| (removable_params_cost && node->can_change_signature))
{
node->call_for_symbol_thunks_and_aliases (gather_caller_stats, &stats,
false);
estimate_ipcp_clone_size_and_time (node, known_csts, known_contexts,
- known_aggs_ptrs, &size, &time,
+ known_aggs, &size, &time,
&base_time, &hints);
time -= devirt_bonus;
- time -= hint_time_bonus (hints);
+ time -= hint_time_bonus (node, hints);
time -= removable_params_cost;
size -= stats.n_calls * removable_params_cost;
stats.freq_sum, stats.count_sum,
size))
{
- if (size + overall_size <= max_new_size)
+ if (size + overall_size <= get_max_overall_size (node))
{
info->do_clone_for_all_contexts = true;
overall_size += size;
"known contexts, growth deemed beneficial.\n");
}
else if (dump_file && (dump_flags & TDF_DETAILS))
- fprintf (dump_file, " Not cloning for all contexts because "
- "max_new_size would be reached with %li.\n",
+ fprintf (dump_file, " Not cloning for all contexts because "
+ "maximum unit size would be reached with %li.\n",
size + overall_size);
}
else if (dump_file && (dump_flags & TDF_DETAILS))
int emc = estimate_move_cost (TREE_TYPE (val->value), true);
perform_estimation_of_a_value (node, known_csts, known_contexts,
- known_aggs_ptrs,
+ known_aggs,
removable_params_cost, emc, val);
if (dump_file && (dump_flags & TDF_DETAILS))
{
known_contexts[i] = val->value;
perform_estimation_of_a_value (node, known_csts, known_contexts,
- known_aggs_ptrs,
+ known_aggs,
removable_params_cost, 0, val);
if (dump_file && (dump_flags & TDF_DETAILS))
for (i = 0; i < count; i++)
{
class ipcp_param_lattices *plats = ipa_get_parm_lattices (info, i);
- struct ipa_agg_jump_function *ajf;
+ struct ipa_agg_value_set *agg;
struct ipcp_agg_lattice *aglat;
if (plats->aggs_bottom || !plats->aggs)
continue;
- ajf = &known_aggs[i];
+ agg = &known_aggs[i];
for (aglat = plats->aggs; aglat; aglat = aglat->next)
{
ipcp_value<tree> *val;
for (val = aglat->values; val; val = val->next)
{
- struct ipa_agg_jf_item item;
+ struct ipa_agg_value item;
item.offset = aglat->offset;
item.value = val->value;
- vec_safe_push (ajf->items, item);
+ agg->items.safe_push (item);
perform_estimation_of_a_value (node, known_csts, known_contexts,
- known_aggs_ptrs,
+ known_aggs,
removable_params_cost, 0, val);
if (dump_file && (dump_flags & TDF_DETAILS))
val->local_time_benefit, val->local_size_cost);
}
- ajf->items->pop ();
+ agg->items.pop ();
}
}
}
- for (i = 0; i < count; i++)
- vec_free (known_aggs[i].items);
-
known_csts.release ();
known_contexts.release ();
- known_aggs.release ();
- known_aggs_ptrs.release ();
+ ipa_release_agg_values (known_aggs);
}
if (opt_for_fn (v->decl, flag_ipa_cp)
&& opt_for_fn (v->decl, optimize))
push_node_to_stack (topo, v);
- /* When V is not optimized, we can not push it to stac, but
+ /* When V is not optimized, we can not push it to stack, but
still we need to set all its callees lattices to bottom. */
else
{
while (v)
{
struct cgraph_edge *cs;
+ class ipa_node_params *info = NULL;
+ bool self_scc = true;
for (cs = v->callees; cs; cs = cs->next_callee)
if (ipa_edge_within_scc (cs))
{
- IPA_NODE_REF (v)->node_within_scc = true;
+ cgraph_node *callee = cs->callee->function_symbol ();
+
+ if (v != callee)
+ self_scc = false;
+
+ if (!info)
+ {
+ info = IPA_NODE_REF (v);
+ info->node_within_scc = true;
+ }
+
if (propagate_constants_across_call (cs))
- push_node_to_stack (topo, cs->callee->function_symbol ());
+ push_node_to_stack (topo, callee);
}
+
+ if (info)
+ info->node_is_self_scc = self_scc;
+
v = pop_node_from_stack (topo);
}
max_count = max_count.max (node->count.ipa ());
}
- max_new_size = overall_size;
- if (max_new_size < param_large_unit_insns)
- max_new_size = param_large_unit_insns;
- max_new_size += max_new_size * param_ipcp_unit_growth / 100 + 1;
+ orig_overall_size = overall_size;
if (dump_file)
- fprintf (dump_file, "\noverall_size: %li, max_new_size: %li\n",
- overall_size, max_new_size);
+ fprintf (dump_file, "\noverall_size: %li\n", overall_size);
propagate_constants_topo (topo);
if (flag_checking)
src_data->next_clone = dst_edge;
}
-/* See if NODE is a clone with a known aggregate value at a given OFFSET of a
- parameter with the given INDEX. */
-
-static tree
-get_clone_agg_value (struct cgraph_node *node, HOST_WIDE_INT offset,
- int index)
-{
- struct ipa_agg_replacement_value *aggval;
-
- aggval = ipa_get_agg_replacements_for_node (node);
- while (aggval)
- {
- if (aggval->offset == offset
- && aggval->index == index)
- return aggval->value;
- aggval = aggval->next;
- }
- return NULL_TREE;
-}
-
-/* Return true is NODE is DEST or its clone for all contexts. */
+/* Return true is CS calls DEST or its clone for all contexts. When
+ ALLOW_RECURSION_TO_CLONE is false, also return false for self-recursive
+ edges from/to an all-context clone. */
static bool
-same_node_or_its_all_contexts_clone_p (cgraph_node *node, cgraph_node *dest)
+calls_same_node_or_its_all_contexts_clone_p (cgraph_edge *cs, cgraph_node *dest,
+ bool allow_recursion_to_clone)
{
- if (node == dest)
+ enum availability availability;
+ cgraph_node *callee = cs->callee->function_symbol (&availability);
+
+ if (availability <= AVAIL_INTERPOSABLE)
+ return false;
+ if (callee == dest)
return true;
+ if (!allow_recursion_to_clone && cs->caller == callee)
+ return false;
- class ipa_node_params *info = IPA_NODE_REF (node);
+ class ipa_node_params *info = IPA_NODE_REF (callee);
return info->is_all_contexts_clone && info->ipcp_orig_node == dest;
}
cgraph_node *dest, ipcp_value<tree> *dest_val)
{
class ipa_node_params *caller_info = IPA_NODE_REF (cs->caller);
- enum availability availability;
- cgraph_node *real_dest = cs->callee->function_symbol (&availability);
- if (availability <= AVAIL_INTERPOSABLE
- || !same_node_or_its_all_contexts_clone_p (real_dest, dest)
+ if (!calls_same_node_or_its_all_contexts_clone_p (cs, dest, !src->val)
|| caller_info->node_dead)
return false;
}
else
{
- /* At the moment we do not propagate over arithmetic jump functions in
- SCCs, so it is safe to detect self-feeding recursive calls in this
- way. */
if (src->val == dest_val)
return true;
ipcp_value<ipa_polymorphic_call_context> *)
{
class ipa_node_params *caller_info = IPA_NODE_REF (cs->caller);
- enum availability avail;
- cgraph_node *real_dest = cs->callee->function_symbol (&avail);
- if (avail <= AVAIL_INTERPOSABLE
- || !same_node_or_its_all_contexts_clone_p (real_dest, dest)
+ if (!calls_same_node_or_its_all_contexts_clone_p (cs, dest, true)
|| caller_info->node_dead)
return false;
if (!src->val)
*freq_sum = freq;
*count_sum = cnt;
*caller_count = count;
+
+ if (!hot && IPA_NODE_REF (dest)->node_within_scc)
+ {
+ struct cgraph_edge *cs;
+
+ /* Cold non-SCC source edge could trigger hot recursive execution of
+ function. Consider the case as hot and rely on following cost model
+ computation to further select right one. */
+ for (cs = dest->callers; cs; cs = cs->next_caller)
+ if (cs->caller == dest && cs->maybe_hot_p ())
+ return true;
+ }
+
return hot;
}
+/* Given a NODE, and a set of its CALLERS, try to adjust order of the callers
+ to let a non-self-recursive caller be the first element. Thus, we can
+ simplify intersecting operations on values that arrive from all of these
+ callers, especially when there exists self-recursive call. Return true if
+ this kind of adjustment is possible. */
+
+static bool
+adjust_callers_for_value_intersection (vec<cgraph_edge *> callers,
+ cgraph_node *node)
+{
+ for (unsigned i = 0; i < callers.length (); i++)
+ {
+ cgraph_edge *cs = callers[i];
+
+ if (cs->caller != node)
+ {
+ if (i > 0)
+ {
+ callers[i] = callers[0];
+ callers[0] = cs;
+ }
+ return true;
+ }
+ }
+ return false;
+}
+
/* Return a vector of incoming edges that do bring value VAL to node DEST. It
is assumed their number is known and equal to CALLER_COUNT. */
}
}
+ if (caller_count > 1)
+ adjust_callers_for_value_intersection (ret, dest);
+
return ret;
}
{
struct ipa_replace_map *replace_map;
-
replace_map = ggc_alloc<ipa_replace_map> ();
if (dump_file)
{
for (cs = new_node->callees; cs; cs = cs->next_callee)
{
fprintf (dump_file, " edge to %s has count ",
- cs->callee->name ());
+ cs->callee->dump_name ());
cs->count.dump (dump_file);
fprintf (dump_file, "\n");
}
for (cs = orig_node->callees; cs; cs = cs->next_callee)
{
fprintf (dump_file, " edge to %s is left with ",
- cs->callee->name ());
+ cs->callee->dump_name ());
cs->count.dump (dump_file);
fprintf (dump_file, "\n");
}
struct caller_statistics stats;
profile_count new_sum, orig_sum;
profile_count remainder, orig_node_count = orig_node->count;
+ profile_count orig_new_node_count = new_node->count;
if (!(orig_node_count.ipa () > profile_count::zero ()))
return;
remainder = orig_node_count.combine_with_ipa_count (orig_node_count.ipa ()
- new_sum.ipa ());
+
+ /* With partial train run we do not want to assume that original's
+ count is zero whenever we redurect all executed edges to clone.
+ Simply drop profile to local one in this case. */
+ if (remainder.ipa_p () && !remainder.ipa ().nonzero_p ()
+ && orig_node->count.ipa_p () && orig_node->count.ipa ().nonzero_p ()
+ && flag_profile_partial_training)
+ remainder = remainder.guessed_local ();
+
new_sum = orig_node_count.combine_with_ipa_count (new_sum);
+ new_node->count = new_sum;
orig_node->count = remainder;
- profile_count::adjust_for_ipa_scaling (&new_sum, &orig_node_count);
+ profile_count::adjust_for_ipa_scaling (&new_sum, &orig_new_node_count);
for (cs = new_node->callees; cs; cs = cs->next_callee)
- cs->count = cs->count.apply_scale (new_sum, orig_node_count);
+ cs->count = cs->count.apply_scale (new_sum, orig_new_node_count);
+ for (cs = new_node->indirect_calls; cs; cs = cs->next_callee)
+ cs->count = cs->count.apply_scale (new_sum, orig_new_node_count);
profile_count::adjust_for_ipa_scaling (&remainder, &orig_node_count);
for (cs = orig_node->callees; cs; cs = cs->next_callee)
cs->count = cs->count.apply_scale (remainder, orig_node_count);
+ for (cs = orig_node->indirect_calls; cs; cs = cs->next_callee)
+ cs->count = cs->count.apply_scale (remainder, orig_node_count);
if (dump_file)
dump_profile_updates (orig_node, new_node);
return new_node;
}
-/* Return true, if JFUNC, which describes a i-th parameter of call CS, is a
- simple no-operation pass-through function to itself. */
+/* Return true if JFUNC, which describes a i-th parameter of call CS, is a
+ pass-through function to itself when the cgraph_node involved is not an
+ IPA-CP clone. When SIMPLE is true, further check if JFUNC is a simple
+ no-operation pass-through. */
static bool
-self_recursive_pass_through_p (cgraph_edge *cs, ipa_jump_func *jfunc, int i)
+self_recursive_pass_through_p (cgraph_edge *cs, ipa_jump_func *jfunc, int i,
+ bool simple = true)
{
enum availability availability;
if (cs->caller == cs->callee->function_symbol (&availability)
&& availability > AVAIL_INTERPOSABLE
&& jfunc->type == IPA_JF_PASS_THROUGH
- && ipa_get_jf_pass_through_operation (jfunc) == NOP_EXPR
- && ipa_get_jf_pass_through_formal_id (jfunc) == i)
+ && (!simple || ipa_get_jf_pass_through_operation (jfunc) == NOP_EXPR)
+ && ipa_get_jf_pass_through_formal_id (jfunc) == i
+ && IPA_NODE_REF (cs->caller)
+ && !IPA_NODE_REF (cs->caller)->ipcp_orig_node)
+ return true;
+ return false;
+}
+
+/* Return true if JFUNC, which describes a part of an aggregate represented or
+ pointed to by the i-th parameter of call CS, is a pass-through function to
+ itself when the cgraph_node involved is not an IPA-CP clone.. When
+ SIMPLE is true, further check if JFUNC is a simple no-operation
+ pass-through. */
+
+static bool
+self_recursive_agg_pass_through_p (cgraph_edge *cs, ipa_agg_jf_item *jfunc,
+ int i, bool simple = true)
+{
+ enum availability availability;
+ if (cs->caller == cs->callee->function_symbol (&availability)
+ && availability > AVAIL_INTERPOSABLE
+ && jfunc->jftype == IPA_JF_LOAD_AGG
+ && jfunc->offset == jfunc->value.load_agg.offset
+ && (!simple || jfunc->value.pass_through.operation == NOP_EXPR)
+ && jfunc->value.pass_through.formal_id == i
+ && useless_type_conversion_p (jfunc->value.load_agg.type, jfunc->type)
+ && IPA_NODE_REF (cs->caller)
+ && !IPA_NODE_REF (cs->caller)->ipcp_orig_node)
return true;
return false;
}
struct ipa_jump_func *jump_func;
tree t;
- if (IPA_NODE_REF (cs->caller)->node_dead)
- continue;
-
if (!IPA_EDGE_REF (cs)
|| i >= ipa_get_cs_argument_count (IPA_EDGE_REF (cs))
|| (i == 0
break;
}
jump_func = ipa_get_ith_jump_func (IPA_EDGE_REF (cs), i);
- if (self_recursive_pass_through_p (cs, jump_func, i))
- continue;
- t = ipa_value_from_jfunc (IPA_NODE_REF (cs->caller), jump_func, type);
+ /* Besides simple pass-through jump function, arithmetic jump
+ function could also introduce argument-direct-pass-through for
+ self-feeding recursive call. For example,
+
+ fn (int i)
+ {
+ fn (i & 1);
+ }
+
+ Given that i is 0, recursive propagation via (i & 1) also gets
+ 0. */
+ if (self_recursive_pass_through_p (cs, jump_func, i, false))
+ {
+ gcc_assert (newval);
+ t = ipa_get_jf_arith_result (
+ ipa_get_jf_pass_through_operation (jump_func),
+ newval,
+ ipa_get_jf_pass_through_operand (jump_func),
+ type);
+ }
+ else
+ t = ipa_value_from_jfunc (IPA_NODE_REF (cs->caller), jump_func,
+ type);
if (!t
|| (newval
&& !values_equal_for_ipcp_p (t, newval))
/* Go through PLATS and create a vector of values consisting of values and
offsets (minus OFFSET) of lattices that contain only a single value. */
-static vec<ipa_agg_jf_item>
+static vec<ipa_agg_value>
copy_plats_to_inter (class ipcp_param_lattices *plats, HOST_WIDE_INT offset)
{
- vec<ipa_agg_jf_item> res = vNULL;
+ vec<ipa_agg_value> res = vNULL;
if (!plats->aggs || plats->aggs_contain_variable || plats->aggs_bottom)
return vNULL;
for (struct ipcp_agg_lattice *aglat = plats->aggs; aglat; aglat = aglat->next)
if (aglat->is_single_const ())
{
- struct ipa_agg_jf_item ti;
+ struct ipa_agg_value ti;
ti.offset = aglat->offset - offset;
ti.value = aglat->values->value;
res.safe_push (ti);
static void
intersect_with_plats (class ipcp_param_lattices *plats,
- vec<ipa_agg_jf_item> *inter,
+ vec<ipa_agg_value> *inter,
HOST_WIDE_INT offset)
{
struct ipcp_agg_lattice *aglat;
- struct ipa_agg_jf_item *item;
+ struct ipa_agg_value *item;
int k;
if (!plats->aggs || plats->aggs_contain_variable || plats->aggs_bottom)
break;
if (aglat->offset - offset == item->offset)
{
- gcc_checking_assert (item->value);
- if (aglat->is_single_const ()
- && values_equal_for_ipcp_p (item->value,
- aglat->values->value))
- found = true;
+ if (aglat->is_single_const ())
+ {
+ tree value = aglat->values->value;
+
+ if (values_equal_for_ipcp_p (item->value, value))
+ found = true;
+ }
break;
}
aglat = aglat->next;
/* Copy aggregate replacement values of NODE (which is an IPA-CP clone) to the
vector result while subtracting OFFSET from the individual value offsets. */
-static vec<ipa_agg_jf_item>
+static vec<ipa_agg_value>
agg_replacements_to_vector (struct cgraph_node *node, int index,
HOST_WIDE_INT offset)
{
struct ipa_agg_replacement_value *av;
- vec<ipa_agg_jf_item> res = vNULL;
+ vec<ipa_agg_value> res = vNULL;
for (av = ipa_get_agg_replacements_for_node (node); av; av = av->next)
if (av->index == index
&& (av->offset - offset) >= 0)
{
- struct ipa_agg_jf_item item;
+ struct ipa_agg_value item;
gcc_checking_assert (av->value);
item.offset = av->offset - offset;
item.value = av->value;
static void
intersect_with_agg_replacements (struct cgraph_node *node, int index,
- vec<ipa_agg_jf_item> *inter,
+ vec<ipa_agg_value> *inter,
HOST_WIDE_INT offset)
{
struct ipa_agg_replacement_value *srcvals;
- struct ipa_agg_jf_item *item;
+ struct ipa_agg_value *item;
int i;
srcvals = ipa_get_agg_replacements_for_node (node);
copy all incoming values to it. If we determine we ended up with no values
whatsoever, return a released vector. */
-static vec<ipa_agg_jf_item>
+static vec<ipa_agg_value>
intersect_aggregates_with_edge (struct cgraph_edge *cs, int index,
- vec<ipa_agg_jf_item> inter)
+ vec<ipa_agg_value> inter)
{
struct ipa_jump_func *jfunc;
jfunc = ipa_get_ith_jump_func (IPA_EDGE_REF (cs), index);
}
else if (jfunc->agg.items)
{
- struct ipa_agg_jf_item *item;
+ class ipa_node_params *caller_info = IPA_NODE_REF (cs->caller);
+ struct ipa_agg_value *item;
int k;
if (!inter.exists ())
for (unsigned i = 0; i < jfunc->agg.items->length (); i++)
- inter.safe_push ((*jfunc->agg.items)[i]);
+ {
+ struct ipa_agg_jf_item *agg_item = &(*jfunc->agg.items)[i];
+ tree value = ipa_agg_value_from_node (caller_info, cs->caller,
+ agg_item);
+ if (value)
+ {
+ struct ipa_agg_value agg_value;
+
+ agg_value.value = value;
+ agg_value.offset = agg_item->offset;
+ inter.safe_push (agg_value);
+ }
+ }
else
FOR_EACH_VEC_ELT (inter, k, item)
{
break;
if (ti->offset == item->offset)
{
- gcc_checking_assert (ti->value);
- if (values_equal_for_ipcp_p (item->value,
- ti->value))
+ tree value;
+
+ /* Besides simple pass-through aggregate jump function,
+ arithmetic aggregate jump function could also bring
+ same aggregate value as parameter passed-in for
+ self-feeding recursive call. For example,
+
+ fn (int *i)
+ {
+ int j = *i & 1;
+ fn (&j);
+ }
+
+ Given that *i is 0, recursive propagation via (*i & 1)
+ also gets 0. */
+ if (self_recursive_agg_pass_through_p (cs, ti, index,
+ false))
+ value = ipa_get_jf_arith_result (
+ ti->value.pass_through.operation,
+ item->value,
+ ti->value.pass_through.operand,
+ ti->type);
+ else
+ value = ipa_agg_value_from_node (caller_info,
+ cs->caller, ti);
+
+ if (value && values_equal_for_ipcp_p (item->value, value))
found = true;
break;
}
else
{
inter.release ();
- return vec<ipa_agg_jf_item>();
+ return vNULL;
}
return inter;
}
for (i = 0; i < count; i++)
{
struct cgraph_edge *cs;
- vec<ipa_agg_jf_item> inter = vNULL;
- struct ipa_agg_jf_item *item;
+ vec<ipa_agg_value> inter = vNULL;
+ struct ipa_agg_value *item;
class ipcp_param_lattices *plats = ipa_get_parm_lattices (dest_info, i);
int j;
for (i = 0; i < count; i++)
{
- static vec<ipa_agg_jf_item> values = vec<ipa_agg_jf_item>();
class ipcp_param_lattices *plats;
bool interesting = false;
for (struct ipa_agg_replacement_value *av = aggval; av; av = av->next)
if (plats->aggs_bottom)
return false;
- values = intersect_aggregates_with_edge (cs, i, values);
+ vec<ipa_agg_value> values = intersect_aggregates_with_edge (cs, i, vNULL);
if (!values.exists ())
return false;
for (struct ipa_agg_replacement_value *av = aggval; av; av = av->next)
if (aggval->index == i)
{
- struct ipa_agg_jf_item *item;
+ struct ipa_agg_value *item;
int j;
bool found = false;
FOR_EACH_VEC_ELT (values, j, item)
return false;
}
}
+ values.release ();
}
return true;
}
perhaps_add_new_callers (node, val);
return false;
}
- else if (val->local_size_cost + overall_size > max_new_size)
+ else if (val->local_size_cost + overall_size > get_max_overall_size (node))
{
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file, " Ignoring candidate value because "
- "max_new_size would be reached with %li.\n",
+ "maximum unit size would be reached with %li.\n",
val->local_size_cost + overall_size);
return false;
}
int i, count = ipa_get_param_count (info);
vec<tree> known_csts;
vec<ipa_polymorphic_call_context> known_contexts;
- vec<ipa_agg_jump_function> known_aggs = vNULL;
bool ret = false;
if (count == 0)
node->dump_name ());
gather_context_independent_values (info, &known_csts, &known_contexts,
- info->do_clone_for_all_contexts ? &known_aggs
- : NULL, NULL);
+ NULL, NULL);
for (i = 0; i < count;i++)
{
if (info->do_clone_for_all_contexts)
{
struct cgraph_node *clone;
- vec<cgraph_edge *> callers;
+ vec<cgraph_edge *> callers = node->collect_callers ();
+
+ for (int i = callers.length () - 1; i >= 0; i--)
+ {
+ cgraph_edge *cs = callers[i];
+ class ipa_node_params *caller_info = IPA_NODE_REF (cs->caller);
+
+ if (caller_info && caller_info->node_dead)
+ callers.unordered_remove (i);
+ }
+
+ if (!adjust_callers_for_value_intersection (callers, node))
+ {
+ /* If node is not called by anyone, or all its caller edges are
+ self-recursive, the node is not really be in use, no need to
+ do cloning. */
+ callers.release ();
+ known_csts.release ();
+ known_contexts.release ();
+ info->do_clone_for_all_contexts = false;
+ return ret;
+ }
if (dump_file)
fprintf (dump_file, " - Creating a specialized node of %s "
"for all known contexts.\n", node->dump_name ());
- callers = node->collect_callers ();
find_more_scalar_values_for_callers_subset (node, known_csts, callers);
find_more_contexts_for_caller_subset (node, &known_contexts, callers);
ipa_agg_replacement_value *aggvals
info = IPA_NODE_REF (node);
info->do_clone_for_all_contexts = false;
IPA_NODE_REF (clone)->is_all_contexts_clone = true;
- for (i = 0; i < count; i++)
- vec_free (known_aggs[i].items);
- known_aggs.release ();
ret = true;
}
else
if (dump_file)
fprintf (dump_file, "Not considering %s for ipa bitwise propagation "
"; -fipa-bit-cp: disabled.\n",
- node->name ());
+ node->dump_name ());
continue;
}
if (info->ipcp_orig_node)
info = IPA_NODE_REF (info->ipcp_orig_node);
+ if (!info->lattices)
+ /* Newly expanded artificial thunks do not have lattices. */
+ continue;
unsigned count = ipa_get_param_count (info);
for (unsigned i = 0; i < count; i++)
ipa_node_params *info = IPA_NODE_REF (node);
bool found_useful_result = false;
- if (!opt_for_fn (node->decl, flag_ipa_vrp))
+ if (!info || !opt_for_fn (node->decl, flag_ipa_vrp))
{
if (dump_file)
fprintf (dump_file, "Not considering %s for VR discovery "
"and propagate; -fipa-ipa-vrp: disabled.\n",
- node->name ());
+ node->dump_name ());
continue;
}
if (info->ipcp_orig_node)
info = IPA_NODE_REF (info->ipcp_orig_node);
+ if (!info->lattices)
+ /* Newly expanded artificial thunks do not have lattices. */
+ continue;
unsigned count = ipa_get_param_count (info);
for (unsigned i = 0; i < count; i++)
{
max_count = profile_count::uninitialized ();
overall_size = 0;
- max_new_size = 0;
+ orig_overall_size = 0;
ipcp_free_transformation_sum ();
}
projects / thirdparty / gcc.git / blobdiff