/* LTO partitioning logic routines.
- Copyright (C) 2009-2013 Free Software Foundation, Inc.
+ Copyright (C) 2009-2020 Free Software Foundation, Inc.
This file is part of GCC.
#include "config.h"
#include "system.h"
#include "coretypes.h"
-#include "toplev.h"
-#include "tree.h"
-#include "gcc-symtab.h"
+#include "target.h"
+#include "function.h"
#include "basic-block.h"
-#include "tree-ssa-alias.h"
-#include "internal-fn.h"
-#include "gimple-expr.h"
-#include "is-a.h"
+#include "tree.h"
#include "gimple.h"
-#include "tm.h"
+#include "alloc-pool.h"
+#include "stringpool.h"
#include "cgraph.h"
#include "lto-streamer.h"
-#include "timevar.h"
-#include "params.h"
-#include "ipa-inline.h"
-#include "ipa-utils.h"
+#include "symbol-summary.h"
+#include "tree-vrp.h"
+#include "ipa-prop.h"
+#include "ipa-fnsummary.h"
#include "lto-partition.h"
-
-/* Classifcation of symbols into classes WRT partitioning. */
-enum symbol_class
-{
- /* External declarations are ignored by partitioning algorithms and they are
- added into the boundary later via compute_ltrans_boundary. */
- SYMBOL_EXTERNAL,
- /* Partitioned symbols are pur into one of partitions. */
- SYMBOL_PARTITION,
- /* Duplicated symbols (such as comdat or constant pool references) are
- copied into every node needing them via add_symbol_to_partition. */
- SYMBOL_DUPLICATE
-};
+#include "sreal.h"
vec<ltrans_partition> ltrans_partitions;
static void add_symbol_to_partition (ltrans_partition part, symtab_node *node);
-/* Classify symbol NODE. */
-
-enum symbol_class
-get_symbol_class (symtab_node *node)
-{
- /* Inline clones are always duplicated.
- This include external delcarations. */
- cgraph_node *cnode = dyn_cast <cgraph_node> (node);
-
- if (DECL_ABSTRACT (node->decl))
- return SYMBOL_EXTERNAL;
-
- if (cnode && cnode->global.inlined_to)
- return SYMBOL_DUPLICATE;
-
- /* Weakref aliases are always duplicated. */
- if (node->weakref)
- return SYMBOL_DUPLICATE;
- /* External declarations are external. */
- if (DECL_EXTERNAL (node->decl))
- return SYMBOL_EXTERNAL;
+/* Helper for qsort; compare partitions and return one with smaller order. */
- if (varpool_node *vnode = dyn_cast <varpool_node> (node))
- {
- /* Constant pool references use local symbol names that can not
- be promoted global. We should never put into a constant pool
- objects that can not be duplicated across partitions. */
- if (DECL_IN_CONSTANT_POOL (node->decl))
- return SYMBOL_DUPLICATE;
- gcc_checking_assert (vnode->definition);
- }
- /* Functions that are cloned may stay in callgraph even if they are unused.
- Handle them as external; compute_ltrans_boundary take care to make
- proper things to happen (i.e. to make them appear in the boundary but
- with body streamed, so clone can me materialized). */
- else if (!cgraph (node)->definition)
- return SYMBOL_EXTERNAL;
-
- /* Comdats are duplicated to every use unless they are keyed.
- Those do not need duplication. */
- if (DECL_COMDAT (node->decl)
- && !node->force_output
- && !symtab_used_from_object_file_p (node))
- return SYMBOL_DUPLICATE;
-
- return SYMBOL_PARTITION;
+static int
+cmp_partitions_order (const void *a, const void *b)
+{
+ const struct ltrans_partition_def *pa
+ = *(struct ltrans_partition_def *const *)a;
+ const struct ltrans_partition_def *pb
+ = *(struct ltrans_partition_def *const *)b;
+ int ordera = -1, orderb = -1;
+
+ if (lto_symtab_encoder_size (pa->encoder))
+ ordera = lto_symtab_encoder_deref (pa->encoder, 0)->order;
+ if (lto_symtab_encoder_size (pb->encoder))
+ orderb = lto_symtab_encoder_deref (pb->encoder, 0)->order;
+ return orderb - ordera;
}
/* Create new partition with name NAME. */
part->encoder = lto_symtab_encoder_new (false);
part->name = name;
part->insns = 0;
+ part->symbols = 0;
ltrans_partitions.safe_push (part);
return part;
}
for (idx = 0; ltrans_partitions.iterate (idx, &part); idx++)
{
if (part->initializers_visited)
- pointer_set_destroy (part->initializers_visited);
+ delete part->initializers_visited;
/* Symtab encoder is freed after streaming. */
free (part);
}
add_references_to_partition (ltrans_partition part, symtab_node *node)
{
int i;
- struct ipa_ref *ref;
+ struct ipa_ref *ref = NULL;
/* Add all duplicated references to the partition. */
- for (i = 0; ipa_ref_list_reference_iterate (&node->ref_list, i, ref); i++)
- if (get_symbol_class (ref->referred) == SYMBOL_DUPLICATE)
+ for (i = 0; node->iterate_reference (i, ref); i++)
+ if (ref->referred->get_partitioning_class () == SYMBOL_DUPLICATE)
add_symbol_to_partition (part, ref->referred);
/* References to a readonly variable may be constant foled into its value.
Recursively look into the initializers of the constant variable and add
references, too. */
- else if (is_a <varpool_node> (ref->referred)
- && ctor_for_folding (ref->referred->decl) != error_mark_node
+ else if (is_a <varpool_node *> (ref->referred)
+ && (dyn_cast <varpool_node *> (ref->referred)
+ ->ctor_useable_for_folding_p ())
&& !lto_symtab_encoder_in_partition_p (part->encoder, ref->referred))
{
if (!part->initializers_visited)
- part->initializers_visited = pointer_set_create ();
- if (!pointer_set_insert (part->initializers_visited, ref->referred))
+ part->initializers_visited = new hash_set<symtab_node *>;
+ if (!part->initializers_visited->add (ref->referred))
add_references_to_partition (part, ref->referred);
}
}
static bool
add_symbol_to_partition_1 (ltrans_partition part, symtab_node *node)
{
- enum symbol_class c = get_symbol_class (node);
- int i;
+ enum symbol_partitioning_class c = node->get_partitioning_class ();
struct ipa_ref *ref;
symtab_node *node1;
gcc_assert (c != SYMBOL_EXTERNAL
&& (c == SYMBOL_DUPLICATE || !symbol_partitioned_p (node)));
+ part->symbols++;
+
lto_set_symtab_encoder_in_partition (part->encoder, node);
if (symbol_partitioned_p (node))
{
node->in_other_partition = 1;
- if (cgraph_dump_file)
- fprintf (cgraph_dump_file, "Symbol node %s now used in multiple partitions\n",
+ if (dump_file)
+ fprintf (dump_file,
+ "Symbol node %s now used in multiple partitions\n",
node->name ());
}
node->aux = (void *)((size_t)node->aux + 1);
- if (cgraph_node *cnode = dyn_cast <cgraph_node> (node))
+ if (cgraph_node *cnode = dyn_cast <cgraph_node *> (node))
{
struct cgraph_edge *e;
- part->insns += inline_summary (cnode)->self_size;
+ if (!node->alias && c == SYMBOL_PARTITION)
+ part->insns += ipa_size_summaries->get (cnode)->size;
/* Add all inline clones and callees that are duplicated. */
for (e = cnode->callees; e; e = e->next_callee)
if (!e->inline_failed)
add_symbol_to_partition_1 (part, e->callee);
- else if (get_symbol_class (e->callee) == SYMBOL_DUPLICATE)
+ else if (e->callee->get_partitioning_class () == SYMBOL_DUPLICATE)
add_symbol_to_partition (part, e->callee);
/* Add all thunks associated with the function. */
for (e = cnode->callers; e; e = e->next_caller)
- if (e->caller->thunk.thunk_p)
+ if (e->caller->thunk.thunk_p && !e->caller->inlined_to)
add_symbol_to_partition_1 (part, e->caller);
}
add_references_to_partition (part, node);
/* Add all aliases associated with the symbol. */
- for (i = 0; ipa_ref_list_referring_iterate (&node->ref_list, i, ref); i++)
- if (ref->use == IPA_REF_ALIAS && !node->weakref)
+
+ FOR_EACH_ALIAS (node, ref)
+ if (!ref->referring->transparent_alias)
add_symbol_to_partition_1 (part, ref->referring);
+ else
+ {
+ struct ipa_ref *ref2;
+ /* We do not need to add transparent aliases if they are not used.
+ However we must add aliases of transparent aliases if they exist. */
+ FOR_EACH_ALIAS (ref->referring, ref2)
+ {
+ /* Nested transparent aliases are not permitted. */
+ gcc_checking_assert (!ref2->referring->transparent_alias);
+ add_symbol_to_partition_1 (part, ref2->referring);
+ }
+ }
/* Ensure that SAME_COMDAT_GROUP lists all allways added in a group. */
if (node->same_comdat_group)
for (node1 = node->same_comdat_group;
node1 != node; node1 = node1->same_comdat_group)
- {
- bool added = add_symbol_to_partition_1 (part, node1);
- gcc_assert (added);
- }
+ if (!node->alias)
+ {
+ bool added = add_symbol_to_partition_1 (part, node1);
+ gcc_assert (added);
+ }
return true;
}
static symtab_node *
contained_in_symbol (symtab_node *node)
{
- /* Weakrefs are never contained in anything. */
- if (node->weakref)
+ /* There is no need to consider transparent aliases to be part of the
+ definition: they are only useful insite the partition they are output
+ and thus we will always see an explicit reference to it. */
+ if (node->transparent_alias)
return node;
- if (cgraph_node *cnode = dyn_cast <cgraph_node> (node))
+ if (cgraph_node *cnode = dyn_cast <cgraph_node *> (node))
{
- cnode = cgraph_function_node (cnode, NULL);
- if (cnode->global.inlined_to)
- cnode = cnode->global.inlined_to;
+ cnode = cnode->function_symbol ();
+ if (cnode->inlined_to)
+ cnode = cnode->inlined_to;
return cnode;
}
- else if (varpool_node *vnode = dyn_cast <varpool_node> (node))
- return varpool_variable_node (vnode, NULL);
+ else if (varpool_node *vnode = dyn_cast <varpool_node *> (node))
+ return vnode->ultimate_alias_target ();
return node;
}
{
symtab_node *node1;
- /* Verify that we do not try to duplicate something that can not be. */
- gcc_checking_assert (get_symbol_class (node) == SYMBOL_DUPLICATE
+ /* Verify that we do not try to duplicate something that cannot be. */
+ gcc_checking_assert (node->get_partitioning_class () == SYMBOL_DUPLICATE
|| !symbol_partitioned_p (node));
while ((node1 = contained_in_symbol (node)) != node)
node = node1;
- /* If we have duplicated symbol contained in something we can not duplicate,
+ /* If we have duplicated symbol contained in something we cannot duplicate,
we are very badly screwed. The other way is possible, so we do not
assert this in add_symbol_to_partition_1.
Be lax about comdats; they may or may not be duplicated and we may
end up in need to duplicate keyed comdat because it has unkeyed alias. */
- gcc_assert (get_symbol_class (node) == SYMBOL_DUPLICATE
+ gcc_assert (node->get_partitioning_class () == SYMBOL_DUPLICATE
|| DECL_COMDAT (node->decl)
|| !symbol_partitioned_p (node));
{
symtab_node *node = lto_symtab_encoder_deref (partition->encoder,
n_nodes);
+ partition->symbols--;
+ cgraph_node *cnode;
/* After UNDO we no longer know what was visited. */
if (partition->initializers_visited)
- pointer_set_destroy (partition->initializers_visited);
+ delete partition->initializers_visited;
partition->initializers_visited = NULL;
- if (cgraph_node *cnode = dyn_cast <cgraph_node> (node))
- partition->insns -= inline_summary (cnode)->self_size;
+ if (!node->alias && (cnode = dyn_cast <cgraph_node *> (node))
+ && node->get_partitioning_class () == SYMBOL_PARTITION)
+ partition->insns -= ipa_size_summaries->get (cnode)->size;
lto_symtab_encoder_delete_node (partition->encoder, node);
node->aux = (void *)((size_t)node->aux - 1);
}
{
symtab_node *node;
struct lto_file_decl_data *file_data;
- struct pointer_map_t *pmap;
+ hash_map<lto_file_decl_data *, ltrans_partition> pmap;
ltrans_partition partition;
- void **slot;
int npartitions = 0;
- pmap = pointer_map_create ();
-
FOR_EACH_SYMBOL (node)
{
- if (get_symbol_class (node) != SYMBOL_PARTITION
+ if (node->get_partitioning_class () != SYMBOL_PARTITION
|| symbol_partitioned_p (node))
continue;
if (file_data)
{
- slot = pointer_map_contains (pmap, file_data);
- if (slot)
- partition = (ltrans_partition) *slot;
+ ltrans_partition *slot = &pmap.get_or_insert (file_data);
+ if (*slot)
+ partition = *slot;
else
{
partition = new_partition (file_data->file_name);
- slot = pointer_map_insert (pmap, file_data);
*slot = partition;
npartitions++;
}
else
{
partition = new_partition ("");
- slot = pointer_map_insert (pmap, NULL);
- *slot = partition;
+ pmap.put (NULL, partition);
npartitions++;
}
if (!npartitions)
new_partition ("empty");
- pointer_map_destroy (pmap);
-
+ /* Order partitions by order of symbols because they are linked into binary
+ that way. */
+ ltrans_partitions.qsort (cmp_partitions_order);
}
/* Maximal partitioning. Put every new symbol into new partition if possible. */
FOR_EACH_SYMBOL (node)
{
- if (get_symbol_class (node) != SYMBOL_PARTITION
+ if (node->get_partitioning_class () != SYMBOL_PARTITION
|| symbol_partitioned_p (node))
continue;
partition = new_partition (node->asm_name ());
static int
node_cmp (const void *pa, const void *pb)
{
- const struct cgraph_node *a = *(const struct cgraph_node * const *) pa;
- const struct cgraph_node *b = *(const struct cgraph_node * const *) pb;
-
- /* Profile reorder flag enables function reordering based on first execution
- of a function. All functions with profile are placed in ascending
- order at the beginning. */
-
- if (flag_profile_reorder_functions)
- {
- /* Functions with time profile are sorted in ascending order. */
- if (a->tp_first_run && b->tp_first_run)
- return a->tp_first_run != b->tp_first_run
- ? a->tp_first_run - b->tp_first_run
- : a->order - b->order;
-
- /* Functions with time profile are sorted before the functions
- that do not have the profile. */
- if (a->tp_first_run || b->tp_first_run)
- return b->tp_first_run - a->tp_first_run;
- }
-
+ const symtab_node *a = *static_cast<const symtab_node * const *> (pa);
+ const symtab_node *b = *static_cast<const symtab_node * const *> (pb);
return b->order - a->order;
}
-/* Helper function for qsort; sort nodes by order. */
-static int
-varpool_node_cmp (const void *pa, const void *pb)
+/* Add all symtab nodes from NEXT_NODE to PARTITION in order. */
+
+static void
+add_sorted_nodes (vec<symtab_node *> &next_nodes, ltrans_partition partition)
{
- const varpool_node *a = *(const varpool_node * const *) pa;
- const varpool_node *b = *(const varpool_node * const *) pb;
- return b->order - a->order;
+ unsigned i;
+ symtab_node *node;
+
+ next_nodes.qsort (node_cmp);
+ FOR_EACH_VEC_ELT (next_nodes, i, node)
+ if (!symbol_partitioned_p (node))
+ add_symbol_to_partition (partition, node);
}
+/* Return true if we should account reference from N1 to N2 in cost
+ of partition boundary. */
+
+bool
+account_reference_p (symtab_node *n1, symtab_node *n2)
+{
+ if (cgraph_node *cnode = dyn_cast <cgraph_node *> (n1))
+ n1 = cnode;
+ /* Do not account references from aliases - they are never split across
+ partitions. */
+ if (n1->alias)
+ return false;
+ /* Do not account recursion - the code below will handle it incorrectly
+ otherwise. Do not account references to external symbols: they will
+ never become local. Finally do not account references to duplicated
+ symbols: they will be always local. */
+ if (n1 == n2
+ || !n2->definition
+ || n2->get_partitioning_class () != SYMBOL_PARTITION)
+ return false;
+ /* If referring node is external symbol do not account it to boundary
+ cost. Those are added into units only to enable possible constant
+ folding and devirtulization.
+
+ Here we do not know if it will ever be added to some partition
+ (this is decided by compute_ltrans_boundary) and second it is not
+ that likely that constant folding will actually use the reference. */
+ if (contained_in_symbol (n1)
+ ->get_partitioning_class () == SYMBOL_EXTERNAL)
+ return false;
+ return true;
+}
+
+
/* Group cgraph nodes into equally-sized partitions.
The partitioning algorithm is simple: nodes are taken in predefined order.
and in-partition calls was reached. */
void
-lto_balanced_map (void)
+lto_balanced_map (int n_lto_partitions, int max_partition_size)
{
- int n_nodes = 0;
int n_varpool_nodes = 0, varpool_pos = 0, best_varpool_pos = 0;
- struct cgraph_node **order = XNEWVEC (struct cgraph_node *, cgraph_max_uid);
- varpool_node **varpool_order = NULL;
- int i;
+ auto_vec <cgraph_node *> order (symtab->cgraph_count);
+ auto_vec<cgraph_node *> noreorder;
+ auto_vec<varpool_node *> varpool_order;
struct cgraph_node *node;
- int total_size = 0, best_total_size = 0;
- int partition_size;
+ int64_t original_total_size, total_size = 0;
+ int64_t partition_size;
ltrans_partition partition;
int last_visited_node = 0;
varpool_node *vnode;
- int cost = 0, internal = 0;
- int best_n_nodes = 0, best_i = 0, best_cost =
- INT_MAX, best_internal = 0;
+ int64_t cost = 0, internal = 0;
+ unsigned int best_n_nodes = 0, best_i = 0;
+ int64_t best_cost = -1, best_internal = 0, best_size = 0;
int npartitions;
int current_order = -1;
+ int noreorder_pos = 0;
FOR_EACH_VARIABLE (vnode)
gcc_assert (!vnode->aux);
-
+
FOR_EACH_DEFINED_FUNCTION (node)
- if (get_symbol_class (node) == SYMBOL_PARTITION)
+ if (node->get_partitioning_class () == SYMBOL_PARTITION)
{
- order[n_nodes++] = node;
- total_size += inline_summary (node)->size;
+ if (node->no_reorder)
+ noreorder.safe_push (node);
+ else
+ order.safe_push (node);
+ if (!node->alias)
+ total_size += ipa_size_summaries->get (node)->size;
}
+ original_total_size = total_size;
+
/* Streaming works best when the source units do not cross partition
boundaries much. This is because importing function from a source
unit tends to import a lot of global trees defined there. We should
get better about minimizing the function bounday, but until that
things works smoother if we order in source order. */
- qsort (order, n_nodes, sizeof (struct cgraph_node *), node_cmp);
-
- if (cgraph_dump_file)
- for(i = 0; i < n_nodes; i++)
- fprintf (cgraph_dump_file, "Balanced map symbol order:%s:%u\n", order[i]->name (), order[i]->tp_first_run);
+ order.qsort (tp_first_run_node_cmp);
+ noreorder.qsort (node_cmp);
- if (!flag_toplevel_reorder)
+ if (dump_file)
{
- FOR_EACH_VARIABLE (vnode)
- if (get_symbol_class (vnode) == SYMBOL_PARTITION)
- n_varpool_nodes++;
- varpool_order = XNEWVEC (varpool_node *, n_varpool_nodes);
-
- n_varpool_nodes = 0;
- FOR_EACH_VARIABLE (vnode)
- if (get_symbol_class (vnode) == SYMBOL_PARTITION)
- varpool_order[n_varpool_nodes++] = vnode;
- qsort (varpool_order, n_varpool_nodes, sizeof (varpool_node *),
- varpool_node_cmp);
+ for (unsigned i = 0; i < order.length (); i++)
+ fprintf (dump_file, "Balanced map symbol order:%s:%u\n",
+ order[i]->name (), order[i]->tp_first_run);
+ for (unsigned i = 0; i < noreorder.length (); i++)
+ fprintf (dump_file, "Balanced map symbol no_reorder:%s:%u\n",
+ noreorder[i]->name (), noreorder[i]->tp_first_run);
}
+ /* Collect all variables that should not be reordered. */
+ FOR_EACH_VARIABLE (vnode)
+ if (vnode->get_partitioning_class () == SYMBOL_PARTITION
+ && vnode->no_reorder)
+ varpool_order.safe_push (vnode);
+ n_varpool_nodes = varpool_order.length ();
+ varpool_order.qsort (node_cmp);
+
/* Compute partition size and create the first partition. */
- partition_size = total_size / PARAM_VALUE (PARAM_LTO_PARTITIONS);
- if (partition_size < PARAM_VALUE (MIN_PARTITION_SIZE))
- partition_size = PARAM_VALUE (MIN_PARTITION_SIZE);
+ if (param_min_partition_size > max_partition_size)
+ fatal_error (input_location, "min partition size cannot be greater "
+ "than max partition size");
+
+ partition_size = total_size / n_lto_partitions;
+ if (partition_size < param_min_partition_size)
+ partition_size = param_min_partition_size;
npartitions = 1;
partition = new_partition ("");
- if (cgraph_dump_file)
- fprintf (cgraph_dump_file, "Total unit size: %i, partition size: %i\n",
+ if (dump_file)
+ fprintf (dump_file, "Total unit size: %" PRId64 ", partition size: %" PRId64 "\n",
total_size, partition_size);
- for (i = 0; i < n_nodes; i++)
+ auto_vec<symtab_node *> next_nodes;
+
+ for (unsigned i = 0; i < order.length (); i++)
{
if (symbol_partitioned_p (order[i]))
continue;
current_order = order[i]->order;
- if (!flag_toplevel_reorder)
- while (varpool_pos < n_varpool_nodes
- && varpool_order[varpool_pos]->order < current_order)
- {
- if (!symbol_partitioned_p (varpool_order[varpool_pos]))
- add_symbol_to_partition (partition, varpool_order[varpool_pos]);
- varpool_pos++;
- }
-
- add_symbol_to_partition (partition, order[i]);
- total_size -= inline_summary (order[i])->size;
+ /* Output noreorder and varpool in program order first. */
+ next_nodes.truncate (0);
+ while (varpool_pos < n_varpool_nodes
+ && varpool_order[varpool_pos]->order < current_order)
+ next_nodes.safe_push (varpool_order[varpool_pos++]);
+ while (noreorder_pos < (int)noreorder.length ()
+ && noreorder[noreorder_pos]->order < current_order)
+ next_nodes.safe_push (noreorder[noreorder_pos++]);
+ add_sorted_nodes (next_nodes, partition);
+
+ if (!symbol_partitioned_p (order[i]))
+ add_symbol_to_partition (partition, order[i]);
/* Once we added a new node to the partition, we also want to add
it and thus we need to subtract it from COST. */
while (last_visited_node < lto_symtab_encoder_size (partition->encoder))
{
- struct ipa_ref_list *refs;
int j;
- struct ipa_ref *ref;
+ struct ipa_ref *ref = NULL;
symtab_node *snode = lto_symtab_encoder_deref (partition->encoder,
last_visited_node);
- if (cgraph_node *node = dyn_cast <cgraph_node> (snode))
+ if (cgraph_node *node = dyn_cast <cgraph_node *> (snode))
{
struct cgraph_edge *edge;
- refs = &node->ref_list;
last_visited_node++;
/* Compute boundary cost of callgraph edges. */
for (edge = node->callees; edge; edge = edge->next_callee)
- if (edge->callee->definition)
+ /* Inline edges will always end up local. */
+ if (edge->inline_failed
+ && account_reference_p (node, edge->callee))
{
- int edge_cost = edge->frequency;
+ int edge_cost = edge->frequency ();
int index;
if (!edge_cost)
cost += edge_cost;
}
for (edge = node->callers; edge; edge = edge->next_caller)
+ if (edge->inline_failed
+ && account_reference_p (edge->caller, node))
{
- int edge_cost = edge->frequency;
+ int edge_cost = edge->frequency ();
int index;
gcc_assert (edge->caller->definition);
edge->caller);
if (index != LCC_NOT_FOUND
&& index < last_visited_node - 1)
- cost -= edge_cost;
+ cost -= edge_cost, internal += edge_cost;
else
cost += edge_cost;
}
}
else
- {
- refs = &snode->ref_list;
- last_visited_node++;
- }
+ last_visited_node++;
/* Compute boundary cost of IPA REF edges and at the same time look into
variables referenced from current partition and try to add them. */
- for (j = 0; ipa_ref_list_reference_iterate (refs, j, ref); j++)
- if (is_a <varpool_node> (ref->referred))
+ for (j = 0; snode->iterate_reference (j, ref); j++)
+ if (!account_reference_p (snode, ref->referred))
+ ;
+ else if (is_a <varpool_node *> (ref->referred))
{
int index;
- vnode = ipa_ref_varpool_node (ref);
- if (!vnode->definition)
- continue;
- if (!symbol_partitioned_p (vnode) && flag_toplevel_reorder
- && get_symbol_class (vnode) == SYMBOL_PARTITION)
+ vnode = dyn_cast <varpool_node *> (ref->referred);
+ if (!symbol_partitioned_p (vnode)
+ && !vnode->no_reorder
+ && vnode->get_partitioning_class () == SYMBOL_PARTITION)
add_symbol_to_partition (partition, vnode);
index = lto_symtab_encoder_lookup (partition->encoder,
vnode);
{
int index;
- node = ipa_ref_node (ref);
- if (!node->definition)
- continue;
+ node = dyn_cast <cgraph_node *> (ref->referred);
index = lto_symtab_encoder_lookup (partition->encoder,
node);
if (index != LCC_NOT_FOUND
else
cost++;
}
- for (j = 0; ipa_ref_list_referring_iterate (refs, j, ref); j++)
- if (is_a <varpool_node> (ref->referring))
+ for (j = 0; snode->iterate_referring (j, ref); j++)
+ if (!account_reference_p (ref->referring, snode))
+ ;
+ else if (is_a <varpool_node *> (ref->referring))
{
int index;
- vnode = ipa_ref_referring_varpool_node (ref);
+ vnode = dyn_cast <varpool_node *> (ref->referring);
gcc_assert (vnode->definition);
- if (!symbol_partitioned_p (vnode) && flag_toplevel_reorder
- && get_symbol_class (vnode) == SYMBOL_PARTITION)
+ /* It is better to couple variables with their users,
+ because it allows them to be removed. Coupling
+ with objects they refer to only helps to reduce
+ number of symbols promoted to hidden. */
+ if (!symbol_partitioned_p (vnode)
+ && !vnode->no_reorder
+ && !vnode->can_remove_if_no_refs_p ()
+ && vnode->get_partitioning_class () == SYMBOL_PARTITION)
add_symbol_to_partition (partition, vnode);
index = lto_symtab_encoder_lookup (partition->encoder,
vnode);
if (index != LCC_NOT_FOUND
&& index < last_visited_node - 1)
- cost--;
+ cost--, internal++;
else
cost++;
}
{
int index;
- node = ipa_ref_referring_node (ref);
+ node = dyn_cast <cgraph_node *> (ref->referring);
gcc_assert (node->definition);
index = lto_symtab_encoder_lookup (partition->encoder,
node);
if (index != LCC_NOT_FOUND
&& index < last_visited_node - 1)
- cost--;
+ cost--, internal++;
else
cost++;
}
}
- /* If the partition is large enough, start looking for smallest boundary cost. */
- if (partition->insns < partition_size * 3 / 4
- || best_cost == INT_MAX
- || ((!cost
- || (best_internal * (HOST_WIDE_INT) cost
- > (internal * (HOST_WIDE_INT)best_cost)))
- && partition->insns < partition_size * 5 / 4))
+ gcc_assert (cost >= 0 && internal >= 0);
+
+ /* If the partition is large enough, start looking for smallest boundary cost.
+ If partition still seems too small (less than 7/8 of target weight) accept
+ any cost. If partition has right size, optimize for highest internal/cost.
+ Later we stop building partition if its size is 9/8 of the target wight. */
+ if (partition->insns < partition_size * 7 / 8
+ || best_cost == -1
+ || (!cost
+ || ((sreal)best_internal * (sreal) cost
+ < ((sreal) internal * (sreal)best_cost))))
{
best_cost = cost;
best_internal = internal;
+ best_size = partition->insns;
best_i = i;
best_n_nodes = lto_symtab_encoder_size (partition->encoder);
- best_total_size = total_size;
best_varpool_pos = varpool_pos;
}
- if (cgraph_dump_file)
- fprintf (cgraph_dump_file, "Step %i: added %s/%i, size %i, cost %i/%i "
- "best %i/%i, step %i\n", i,
+ if (dump_file)
+ fprintf (dump_file, "Step %i: added %s/%i, size %i, "
+ "cost %" PRId64 "/%" PRId64 " "
+ "best %" PRId64 "/%" PRId64", step %i\n", i,
order[i]->name (), order[i]->order,
partition->insns, cost, internal,
best_cost, best_internal, best_i);
/* Partition is too large, unwind into step when best cost was reached and
start new partition. */
- if (partition->insns > 2 * partition_size)
+ if (partition->insns > 9 * partition_size / 8
+ || partition->insns > max_partition_size)
{
if (best_i != i)
{
- if (cgraph_dump_file)
- fprintf (cgraph_dump_file, "Unwinding %i insertions to step %i\n",
+ if (dump_file)
+ fprintf (dump_file, "Unwinding %i insertions to step %i\n",
i - best_i, best_i);
undo_partition (partition, best_n_nodes);
varpool_pos = best_varpool_pos;
}
+ gcc_assert (best_size == partition->insns);
i = best_i;
+ if (dump_file)
+ fprintf (dump_file,
+ "Partition insns: %i (want %" PRId64 ")\n",
+ partition->insns, partition_size);
/* When we are finished, avoid creating empty partition. */
- while (i < n_nodes - 1 && symbol_partitioned_p (order[i + 1]))
+ while (i < order.length () - 1 && symbol_partitioned_p (order[i + 1]))
i++;
- if (i == n_nodes - 1)
+ if (i == order.length () - 1)
break;
+ total_size -= partition->insns;
partition = new_partition ("");
last_visited_node = 0;
- total_size = best_total_size;
cost = 0;
- if (cgraph_dump_file)
- fprintf (cgraph_dump_file, "New partition\n");
+ if (dump_file)
+ fprintf (dump_file, "New partition\n");
best_n_nodes = 0;
- best_cost = INT_MAX;
+ best_cost = -1;
/* Since the size of partitions is just approximate, update the size after
we finished current one. */
- if (npartitions < PARAM_VALUE (PARAM_LTO_PARTITIONS))
- partition_size = total_size
- / (PARAM_VALUE (PARAM_LTO_PARTITIONS) - npartitions);
+ if (npartitions < n_lto_partitions)
+ partition_size = total_size / (n_lto_partitions - npartitions);
else
- partition_size = INT_MAX;
-
- if (partition_size < PARAM_VALUE (MIN_PARTITION_SIZE))
- partition_size = PARAM_VALUE (MIN_PARTITION_SIZE);
+ /* Watch for overflow. */
+ partition_size = INT_MAX / 16;
+
+ if (dump_file)
+ fprintf (dump_file,
+ "Total size: %" PRId64 " partition_size: %" PRId64 "\n",
+ total_size, partition_size);
+ if (partition_size < param_min_partition_size)
+ partition_size = param_min_partition_size;
npartitions ++;
}
}
+ next_nodes.truncate (0);
+
/* Varables that are not reachable from the code go into last partition. */
- if (flag_toplevel_reorder)
- {
- FOR_EACH_VARIABLE (vnode)
- if (get_symbol_class (vnode) == SYMBOL_PARTITION
- && !symbol_partitioned_p (vnode))
- add_symbol_to_partition (partition, vnode);
- }
- else
+ FOR_EACH_VARIABLE (vnode)
+ if (vnode->get_partitioning_class () == SYMBOL_PARTITION
+ && !symbol_partitioned_p (vnode))
+ next_nodes.safe_push (vnode);
+
+ /* Output remaining ordered symbols. */
+ while (varpool_pos < n_varpool_nodes)
+ next_nodes.safe_push (varpool_order[varpool_pos++]);
+ while (noreorder_pos < (int)noreorder.length ())
+ next_nodes.safe_push (noreorder[noreorder_pos++]);
+ /* For one partition the cost of boundary should be 0 unless we added final
+ symbols here (these are not accounted) or we have accounting bug. */
+ gcc_assert (next_nodes.length () || npartitions != 1 || !best_cost || best_cost == -1);
+ add_sorted_nodes (next_nodes, partition);
+
+ if (dump_file)
{
- while (varpool_pos < n_varpool_nodes)
+ fprintf (dump_file, "\nPartition sizes:\n");
+ unsigned partitions = ltrans_partitions.length ();
+
+ for (unsigned i = 0; i < partitions ; i++)
{
- if (!symbol_partitioned_p (varpool_order[varpool_pos]))
- add_symbol_to_partition (partition, varpool_order[varpool_pos]);
- varpool_pos++;
+ ltrans_partition p = ltrans_partitions[i];
+ fprintf (dump_file, "partition %d contains %d (%2.2f%%)"
+ " symbols and %d (%2.2f%%) insns\n", i, p->symbols,
+ 100.0 * p->symbols / order.length (), p->insns,
+ 100.0 * p->insns / original_total_size);
}
- free (varpool_order);
+
+ fprintf (dump_file, "\n");
}
- free (order);
}
-/* Mangle NODE symbol name into a local name.
- This is necessary to do
- 1) if two or more static vars of same assembler name
- are merged into single ltrans unit.
- 2) if prevoiusly static var was promoted hidden to avoid possible conflict
- with symbols defined out of the LTO world.
-*/
+/* Return true if we must not change the name of the NODE. The name as
+ extracted from the corresponding decl should be passed in NAME. */
static bool
-privatize_symbol_name (symtab_node *node)
+must_not_rename (symtab_node *node, const char *name)
{
- tree decl = node->decl;
- const char *name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl));
-
/* Our renaming machinery do not handle more than one change of assembler name.
We should not need more than one anyway. */
if (node->lto_file_data
&& lto_get_decl_name_mapping (node->lto_file_data, name) != name)
{
- if (cgraph_dump_file)
- fprintf (cgraph_dump_file,
- "Not privatizing symbol name: %s. It privatized already.\n",
- name);
- return false;
+ if (dump_file)
+ fprintf (dump_file,
+ "Not privatizing symbol name: %s. It privatized already.\n",
+ name);
+ return true;
}
/* Avoid mangling of already mangled clones.
??? should have a flag whether a symbol has a 'private' name already,
that are not really clones. */
if (node->unique_name)
{
- if (cgraph_dump_file)
- fprintf (cgraph_dump_file,
- "Not privatizing symbol name: %s. Has unique name.\n",
- name);
- return false;
+ if (dump_file)
+ fprintf (dump_file,
+ "Not privatizing symbol name: %s. Has unique name.\n",
+ name);
+ return true;
+ }
+ return false;
+}
+
+/* If we are an offload compiler, we may have to rewrite symbols to be
+ valid on this target. Return either PTR or a modified version of it. */
+
+static const char *
+maybe_rewrite_identifier (const char *ptr)
+{
+#if defined ACCEL_COMPILER && (defined NO_DOT_IN_LABEL || defined NO_DOLLAR_IN_LABEL)
+#ifndef NO_DOT_IN_LABEL
+ char valid = '.';
+ const char reject[] = "$";
+#elif !defined NO_DOLLAR_IN_LABEL
+ char valid = '$';
+ const char reject[] = ".";
+#else
+ char valid = '_';
+ const char reject[] = ".$";
+#endif
+
+ char *copy = NULL;
+ const char *match = ptr;
+ for (;;)
+ {
+ size_t off = strcspn (match, reject);
+ if (match[off] == '\0')
+ break;
+ if (copy == NULL)
+ {
+ copy = xstrdup (ptr);
+ match = copy;
+ }
+ copy[off] = valid;
+ }
+ return match;
+#else
+ return ptr;
+#endif
+}
+
+/* Ensure that the symbol in NODE is valid for the target, and if not,
+ rewrite it. */
+
+static void
+validize_symbol_for_target (symtab_node *node)
+{
+ tree decl = node->decl;
+ const char *name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl));
+
+ if (must_not_rename (node, name))
+ return;
+
+ const char *name2 = maybe_rewrite_identifier (name);
+ if (name2 != name)
+ {
+ symtab->change_decl_assembler_name (decl, get_identifier (name2));
+ if (node->lto_file_data)
+ lto_record_renamed_decl (node->lto_file_data, name,
+ IDENTIFIER_POINTER
+ (DECL_ASSEMBLER_NAME (decl)));
}
- change_decl_assembler_name (decl, clone_function_name (decl, "lto_priv"));
+}
+
+/* Maps symbol names to unique lto clone counters. */
+static hash_map<const char *, unsigned> *lto_clone_numbers;
+
+/* Helper for privatize_symbol_name. Mangle NODE symbol name
+ represented by DECL. */
+
+static bool
+privatize_symbol_name_1 (symtab_node *node, tree decl)
+{
+ const char *name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl));
+
+ if (must_not_rename (node, name))
+ return false;
+
+ name = maybe_rewrite_identifier (name);
+ unsigned &clone_number = lto_clone_numbers->get_or_insert (name);
+ symtab->change_decl_assembler_name (decl,
+ clone_function_name (
+ name, "lto_priv", clone_number));
+ clone_number++;
+
if (node->lto_file_data)
lto_record_renamed_decl (node->lto_file_data, name,
IDENTIFIER_POINTER
(DECL_ASSEMBLER_NAME (decl)));
- if (cgraph_dump_file)
- fprintf (cgraph_dump_file,
- "Privatizing symbol name: %s -> %s\n",
- name, IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)));
+
+ if (dump_file)
+ fprintf (dump_file,
+ "Privatizing symbol name: %s -> %s\n",
+ name, IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)));
+
+ return true;
+}
+
+/* Mangle NODE symbol name into a local name.
+ This is necessary to do
+ 1) if two or more static vars of same assembler name
+ are merged into single ltrans unit.
+ 2) if previously static var was promoted hidden to avoid possible conflict
+ with symbols defined out of the LTO world. */
+
+static bool
+privatize_symbol_name (symtab_node *node)
+{
+ if (!privatize_symbol_name_1 (node, node->decl))
+ return false;
+
return true;
}
if (DECL_VISIBILITY (node->decl) == VISIBILITY_HIDDEN
&& DECL_VISIBILITY_SPECIFIED (node->decl)
&& TREE_PUBLIC (node->decl))
- return;
+ {
+ validize_symbol_for_target (node);
+ return;
+ }
gcc_checking_assert (!TREE_PUBLIC (node->decl)
&& !DECL_EXTERNAL (node->decl));
TREE_PUBLIC (node->decl) = 1;
DECL_VISIBILITY (node->decl) = VISIBILITY_HIDDEN;
DECL_VISIBILITY_SPECIFIED (node->decl) = true;
- if (cgraph_dump_file)
- fprintf (cgraph_dump_file,
- "Promoting as hidden: %s\n", node->name ());
+ if (dump_file)
+ fprintf (dump_file,
+ "Promoting as hidden: %s (%s)\n", node->name (),
+ IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl)));
+
+ /* Promoting a symbol also promotes all transparent aliases with exception
+ of weakref where the visibility flags are always wrong and set to
+ !PUBLIC. */
+ ipa_ref *ref;
+ for (unsigned i = 0; node->iterate_direct_aliases (i, ref); i++)
+ {
+ struct symtab_node *alias = ref->referring;
+ if (alias->transparent_alias && !alias->weakref)
+ {
+ TREE_PUBLIC (alias->decl) = 1;
+ DECL_VISIBILITY (alias->decl) = VISIBILITY_HIDDEN;
+ DECL_VISIBILITY_SPECIFIED (alias->decl) = true;
+ if (dump_file)
+ fprintf (dump_file,
+ "Promoting alias as hidden: %s\n",
+ IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl)));
+ }
+ gcc_assert (!alias->weakref || TREE_PUBLIC (alias->decl));
+ }
}
-/* Return true if NODE needs named section even if it won't land in the partition
- symbol table.
- FIXME: we should really not use named sections for inline clones and master clones. */
+/* Return true if NODE needs named section even if it won't land in
+ the partition symbol table.
+
+ FIXME: we should really not use named sections for inline clones
+ and master clones. */
static bool
may_need_named_section_p (lto_symtab_encoder_t encoder, symtab_node *node)
{
- struct cgraph_node *cnode = dyn_cast <cgraph_node> (node);
+ struct cgraph_node *cnode = dyn_cast <cgraph_node *> (node);
if (!cnode)
return false;
- if (symtab_real_symbol_p (node))
+ if (node->real_symbol_p ())
return false;
return (!encoder
|| (lto_symtab_encoder_lookup (encoder, node) != LCC_NOT_FOUND
tree name = DECL_ASSEMBLER_NAME (decl);
/* See if this is static symbol. */
- if ((node->externally_visible
+ if (((node->externally_visible && !node->weakref)
/* FIXME: externally_visible is somewhat illogically not set for
external symbols (i.e. those not defined). Remove this test
once this is fixed. */
|| DECL_EXTERNAL (node->decl)
- || !symtab_real_symbol_p (node))
+ || !node->real_symbol_p ())
&& !may_need_named_section_p (encoder, node))
return;
/* Now walk symbols sharing the same name and see if there are any conflicts.
- (all types of symbols counts here, since we can not have static of the
+ (all types of symbols counts here, since we cannot have static of the
same name as external or public symbol.) */
- for (s = symtab_node_for_asm (name);
+ for (s = symtab_node::get_for_asmname (name);
s; s = s->next_sharing_asm_name)
- if ((symtab_real_symbol_p (s) || may_need_named_section_p (encoder, s))
+ if ((s->real_symbol_p () || may_need_named_section_p (encoder, s))
&& s->decl != node->decl
&& (!encoder
|| lto_symtab_encoder_lookup (encoder, s) != LCC_NOT_FOUND))
if (!s)
return;
- if (cgraph_dump_file)
- fprintf (cgraph_dump_file,
+ if (dump_file)
+ fprintf (dump_file,
"Renaming statics with asm name: %s\n", node->name ());
/* Assign every symbol in the set that shares the same ASM name an unique
mangled name. */
- for (s = symtab_node_for_asm (name); s;)
- if (!s->externally_visible
- && ((symtab_real_symbol_p (s)
- && !DECL_EXTERNAL (node->decl)
- && !TREE_PUBLIC (node->decl))
+ for (s = symtab_node::get_for_asmname (name); s;)
+ if ((!s->externally_visible || s->weakref)
+ /* Transparent aliases having same name as target are renamed at a
+ time their target gets new name. Transparent aliases that use
+ separate assembler name require the name to be unique. */
+ && (!s->transparent_alias || !s->definition || s->weakref
+ || !symbol_table::assembler_names_equal_p
+ (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (s->decl)),
+ IDENTIFIER_POINTER
+ (DECL_ASSEMBLER_NAME (s->get_alias_target()->decl))))
+ && ((s->real_symbol_p ()
+ && !DECL_EXTERNAL (s->decl)
+ && !TREE_PUBLIC (s->decl))
|| may_need_named_section_p (encoder, s))
&& (!encoder
|| lto_symtab_encoder_lookup (encoder, s) != LCC_NOT_FOUND))
{
if (privatize_symbol_name (s))
- /* Re-start from beginning since we do not know how many symbols changed a name. */
- s = symtab_node_for_asm (name);
+ /* Re-start from beginning since we do not know how many
+ symbols changed a name. */
+ s = symtab_node::get_for_asmname (name);
else s = s->next_sharing_asm_name;
}
else s = s->next_sharing_asm_name;
gcc_assert (flag_wpa);
+ lto_stream_offload_p = false;
+ select_what_to_stream ();
+
/* First compute boundaries. */
n_sets = ltrans_partitions.length ();
for (i = 0; i < n_sets; i++)
part->encoder = compute_ltrans_boundary (part->encoder);
}
+ lto_clone_numbers = new hash_map<const char *, unsigned>;
+
/* Look at boundaries and promote symbols as needed. */
for (i = 0; i < n_sets; i++)
{
{
symtab_node *node = lsei_node (lsei);
- /* If symbol is static, rename it if its assembler name clash with
- anything else in this unit. */
+ /* If symbol is static, rename it if its assembler name
+ clashes with anything else in this unit. */
rename_statics (encoder, node);
/* No need to promote if symbol already is externally visible ... */
/* ... or if it is part of current partition ... */
|| lto_symtab_encoder_in_partition_p (encoder, node)
/* ... or if we do not partition it. This mean that it will
- appear in every partition refernecing it. */
- || get_symbol_class (node) != SYMBOL_PARTITION)
- continue;
+ appear in every partition referencing it. */
+ || node->get_partitioning_class () != SYMBOL_PARTITION)
+ {
+ validize_symbol_for_target (node);
+ continue;
+ }
promote_symbol (node);
}
}
+ delete lto_clone_numbers;
}
/* Rename statics in the whole unit in the case that
lto_promote_statics_nonwpa (void)
{
symtab_node *node;
+
+ lto_clone_numbers = new hash_map<const char *, unsigned>;
FOR_EACH_SYMBOL (node)
- rename_statics (NULL, node);
+ {
+ rename_statics (NULL, node);
+ validize_symbol_for_target (node);
+ }
+ delete lto_clone_numbers;
}
projects / thirdparty / gcc.git / blobdiff