]>
Commit | Line | Data |
---|---|---|
a66dc285 | 1 | /* LTO partitioning logic routines. |
8d9254fc | 2 | Copyright (C) 2009-2020 Free Software Foundation, Inc. |
a66dc285 JH |
3 | |
4 | This file is part of GCC. | |
5 | ||
6 | GCC is free software; you can redistribute it and/or modify it under | |
7 | the terms of the GNU General Public License as published by the Free | |
8 | Software Foundation; either version 3, or (at your option) any later | |
9 | version. | |
10 | ||
11 | GCC is distributed in the hope that it will be useful, but WITHOUT ANY | |
12 | WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
13 | FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
14 | for more details. | |
15 | ||
16 | You should have received a copy of the GNU General Public License | |
17 | along with GCC; see the file COPYING3. If not see | |
18 | <http://www.gnu.org/licenses/>. */ | |
19 | ||
20 | #include "config.h" | |
21 | #include "system.h" | |
22 | #include "coretypes.h" | |
2adfab87 | 23 | #include "target.h" |
60393bbc | 24 | #include "function.h" |
2fb9a547 | 25 | #include "basic-block.h" |
c7131fb2 | 26 | #include "tree.h" |
8e9055ae | 27 | #include "gimple.h" |
f51ade33 | 28 | #include "alloc-pool.h" |
2adfab87 AM |
29 | #include "stringpool.h" |
30 | #include "cgraph.h" | |
31 | #include "lto-streamer.h" | |
dd912cb8 | 32 | #include "symbol-summary.h" |
8bc5448f | 33 | #include "tree-vrp.h" |
c582198b | 34 | #include "ipa-prop.h" |
27d020cf | 35 | #include "ipa-fnsummary.h" |
a66dc285 | 36 | #include "lto-partition.h" |
ddb0b824 | 37 | #include "sreal.h" |
a66dc285 | 38 | |
9771b263 | 39 | vec<ltrans_partition> ltrans_partitions; |
a66dc285 | 40 | |
5e20cdc9 | 41 | static void add_symbol_to_partition (ltrans_partition part, symtab_node *node); |
c3c445e1 | 42 | |
a66dc285 | 43 | |
b6e33d73 JH |
44 | /* Helper for qsort; compare partitions and return one with smaller order. */ |
45 | ||
46 | static int | |
47 | cmp_partitions_order (const void *a, const void *b) | |
48 | { | |
49 | const struct ltrans_partition_def *pa | |
50 | = *(struct ltrans_partition_def *const *)a; | |
51 | const struct ltrans_partition_def *pb | |
52 | = *(struct ltrans_partition_def *const *)b; | |
53 | int ordera = -1, orderb = -1; | |
54 | ||
55 | if (lto_symtab_encoder_size (pa->encoder)) | |
56 | ordera = lto_symtab_encoder_deref (pa->encoder, 0)->order; | |
57 | if (lto_symtab_encoder_size (pb->encoder)) | |
58 | orderb = lto_symtab_encoder_deref (pb->encoder, 0)->order; | |
59 | return orderb - ordera; | |
60 | } | |
61 | ||
a66dc285 | 62 | /* Create new partition with name NAME. */ |
c3c445e1 | 63 | |
a66dc285 JH |
64 | static ltrans_partition |
65 | new_partition (const char *name) | |
66 | { | |
67 | ltrans_partition part = XCNEW (struct ltrans_partition_def); | |
e75f8f79 | 68 | part->encoder = lto_symtab_encoder_new (false); |
a66dc285 JH |
69 | part->name = name; |
70 | part->insns = 0; | |
faae53f8 | 71 | part->symbols = 0; |
9771b263 | 72 | ltrans_partitions.safe_push (part); |
a66dc285 JH |
73 | return part; |
74 | } | |
75 | ||
76 | /* Free memory used by ltrans datastructures. */ | |
c3c445e1 | 77 | |
a66dc285 JH |
78 | void |
79 | free_ltrans_partitions (void) | |
80 | { | |
81 | unsigned int idx; | |
82 | ltrans_partition part; | |
9771b263 | 83 | for (idx = 0; ltrans_partitions.iterate (idx, &part); idx++) |
a66dc285 | 84 | { |
c3c445e1 | 85 | if (part->initializers_visited) |
6e2830c3 | 86 | delete part->initializers_visited; |
7b99cca4 | 87 | /* Symtab encoder is freed after streaming. */ |
a66dc285 JH |
88 | free (part); |
89 | } | |
9771b263 | 90 | ltrans_partitions.release (); |
a66dc285 JH |
91 | } |
92 | ||
c3c445e1 JH |
93 | /* Return true if symbol is already in some partition. */ |
94 | ||
95 | static inline bool | |
5e20cdc9 | 96 | symbol_partitioned_p (symtab_node *node) |
a66dc285 | 97 | { |
67348ccc | 98 | return node->aux; |
1cdbb3f9 JH |
99 | } |
100 | ||
c3c445e1 | 101 | /* Add references into the partition. */ |
1cdbb3f9 | 102 | static void |
5e20cdc9 | 103 | add_references_to_partition (ltrans_partition part, symtab_node *node) |
1cdbb3f9 JH |
104 | { |
105 | int i; | |
d122681a | 106 | struct ipa_ref *ref = NULL; |
1cdbb3f9 | 107 | |
c3c445e1 | 108 | /* Add all duplicated references to the partition. */ |
d122681a | 109 | for (i = 0; node->iterate_reference (i, ref); i++) |
d52f5295 | 110 | if (ref->referred->get_partitioning_class () == SYMBOL_DUPLICATE) |
c3c445e1 JH |
111 | add_symbol_to_partition (part, ref->referred); |
112 | /* References to a readonly variable may be constant foled into its value. | |
113 | Recursively look into the initializers of the constant variable and add | |
114 | references, too. */ | |
7de90a6c | 115 | else if (is_a <varpool_node *> (ref->referred) |
d5e254e1 | 116 | && (dyn_cast <varpool_node *> (ref->referred) |
31db0fe0 | 117 | ->ctor_useable_for_folding_p ()) |
c3c445e1 | 118 | && !lto_symtab_encoder_in_partition_p (part->encoder, ref->referred)) |
1cdbb3f9 | 119 | { |
c3c445e1 | 120 | if (!part->initializers_visited) |
6e2830c3 TS |
121 | part->initializers_visited = new hash_set<symtab_node *>; |
122 | if (!part->initializers_visited->add (ref->referred)) | |
c3c445e1 | 123 | add_references_to_partition (part, ref->referred); |
1cdbb3f9 | 124 | } |
a66dc285 JH |
125 | } |
126 | ||
c3c445e1 JH |
127 | /* Helper function for add_symbol_to_partition doing the actual dirty work |
128 | of adding NODE to PART. */ | |
a66dc285 JH |
129 | |
130 | static bool | |
5e20cdc9 | 131 | add_symbol_to_partition_1 (ltrans_partition part, symtab_node *node) |
a66dc285 | 132 | { |
d52f5295 | 133 | enum symbol_partitioning_class c = node->get_partitioning_class (); |
e55637b7 | 134 | struct ipa_ref *ref; |
5e20cdc9 | 135 | symtab_node *node1; |
a66dc285 | 136 | |
c3c445e1 | 137 | /* If NODE is already there, we have nothing to do. */ |
67348ccc | 138 | if (lto_symtab_encoder_in_partition_p (part->encoder, node)) |
c3c445e1 JH |
139 | return true; |
140 | ||
141 | /* non-duplicated aliases or tunks of a duplicated symbol needs to be output | |
142 | just once. | |
143 | ||
144 | Be lax about comdats; they may or may not be duplicated and we may | |
145 | end up in need to duplicate keyed comdat because it has unkeyed alias. */ | |
67348ccc | 146 | if (c == SYMBOL_PARTITION && !DECL_COMDAT (node->decl) |
c3c445e1 | 147 | && symbol_partitioned_p (node)) |
1cdbb3f9 JH |
148 | return false; |
149 | ||
c3c445e1 JH |
150 | /* Be sure that we never try to duplicate partitioned symbol |
151 | or add external symbol. */ | |
3a849280 | 152 | gcc_assert (c != SYMBOL_EXTERNAL |
c3c445e1 JH |
153 | && (c == SYMBOL_DUPLICATE || !symbol_partitioned_p (node))); |
154 | ||
faae53f8 ML |
155 | part->symbols++; |
156 | ||
67348ccc | 157 | lto_set_symtab_encoder_in_partition (part->encoder, node); |
a66dc285 | 158 | |
c3c445e1 | 159 | if (symbol_partitioned_p (node)) |
a66dc285 | 160 | { |
67348ccc | 161 | node->in_other_partition = 1; |
29a50dfb JH |
162 | if (dump_file) |
163 | fprintf (dump_file, | |
3dafb85c | 164 | "Symbol node %s now used in multiple partitions\n", |
fec39fa6 | 165 | node->name ()); |
a66dc285 | 166 | } |
67348ccc | 167 | node->aux = (void *)((size_t)node->aux + 1); |
a66dc285 | 168 | |
7de90a6c | 169 | if (cgraph_node *cnode = dyn_cast <cgraph_node *> (node)) |
c3c445e1 | 170 | { |
c3c445e1 | 171 | struct cgraph_edge *e; |
ddb0b824 | 172 | if (!node->alias && c == SYMBOL_PARTITION) |
f658ad30 | 173 | part->insns += ipa_size_summaries->get (cnode)->size; |
c3c445e1 JH |
174 | |
175 | /* Add all inline clones and callees that are duplicated. */ | |
176 | for (e = cnode->callees; e; e = e->next_callee) | |
177 | if (!e->inline_failed) | |
67348ccc | 178 | add_symbol_to_partition_1 (part, e->callee); |
d52f5295 | 179 | else if (e->callee->get_partitioning_class () == SYMBOL_DUPLICATE) |
67348ccc | 180 | add_symbol_to_partition (part, e->callee); |
c3c445e1 JH |
181 | |
182 | /* Add all thunks associated with the function. */ | |
183 | for (e = cnode->callers; e; e = e->next_caller) | |
a62bfab5 | 184 | if (e->caller->thunk.thunk_p && !e->caller->inlined_to) |
67348ccc | 185 | add_symbol_to_partition_1 (part, e->caller); |
c3c445e1 | 186 | } |
a66dc285 | 187 | |
c3c445e1 | 188 | add_references_to_partition (part, node); |
a66dc285 | 189 | |
c3c445e1 | 190 | /* Add all aliases associated with the symbol. */ |
e55637b7 ML |
191 | |
192 | FOR_EACH_ALIAS (node, ref) | |
41c60c6c | 193 | if (!ref->referring->transparent_alias) |
c3c445e1 | 194 | add_symbol_to_partition_1 (part, ref->referring); |
41c60c6c JH |
195 | else |
196 | { | |
197 | struct ipa_ref *ref2; | |
198 | /* We do not need to add transparent aliases if they are not used. | |
199 | However we must add aliases of transparent aliases if they exist. */ | |
200 | FOR_EACH_ALIAS (ref->referring, ref2) | |
201 | { | |
202 | /* Nested transparent aliases are not permitted. */ | |
203 | gcc_checking_assert (!ref2->referring->transparent_alias); | |
204 | add_symbol_to_partition_1 (part, ref2->referring); | |
205 | } | |
206 | } | |
a66dc285 | 207 | |
c3c445e1 | 208 | /* Ensure that SAME_COMDAT_GROUP lists all allways added in a group. */ |
67348ccc DM |
209 | if (node->same_comdat_group) |
210 | for (node1 = node->same_comdat_group; | |
211 | node1 != node; node1 = node1->same_comdat_group) | |
96451279 JH |
212 | if (!node->alias) |
213 | { | |
214 | bool added = add_symbol_to_partition_1 (part, node1); | |
215 | gcc_assert (added); | |
216 | } | |
c3c445e1 | 217 | return true; |
a66dc285 JH |
218 | } |
219 | ||
c3c445e1 JH |
220 | /* If symbol NODE is really part of other symbol's definition (i.e. it is |
221 | internal label, thunk, alias or so), return the outer symbol. | |
222 | When add_symbol_to_partition_1 is called on the outer symbol it must | |
223 | eventually add NODE, too. */ | |
5e20cdc9 DM |
224 | static symtab_node * |
225 | contained_in_symbol (symtab_node *node) | |
c3c445e1 | 226 | { |
41c60c6c JH |
227 | /* There is no need to consider transparent aliases to be part of the |
228 | definition: they are only useful insite the partition they are output | |
229 | and thus we will always see an explicit reference to it. */ | |
230 | if (node->transparent_alias) | |
c3c445e1 | 231 | return node; |
7de90a6c | 232 | if (cgraph_node *cnode = dyn_cast <cgraph_node *> (node)) |
c3c445e1 | 233 | { |
d52f5295 | 234 | cnode = cnode->function_symbol (); |
a62bfab5 ML |
235 | if (cnode->inlined_to) |
236 | cnode = cnode->inlined_to; | |
67348ccc | 237 | return cnode; |
c3c445e1 | 238 | } |
7de90a6c | 239 | else if (varpool_node *vnode = dyn_cast <varpool_node *> (node)) |
9041d2e6 | 240 | return vnode->ultimate_alias_target (); |
c3c445e1 JH |
241 | return node; |
242 | } | |
243 | ||
244 | /* Add symbol NODE to partition. When definition of NODE is part | |
245 | of other symbol definition, add the other symbol, too. */ | |
a66dc285 JH |
246 | |
247 | static void | |
5e20cdc9 | 248 | add_symbol_to_partition (ltrans_partition part, symtab_node *node) |
a66dc285 | 249 | { |
5e20cdc9 | 250 | symtab_node *node1; |
a66dc285 | 251 | |
67914693 | 252 | /* Verify that we do not try to duplicate something that cannot be. */ |
d52f5295 | 253 | gcc_checking_assert (node->get_partitioning_class () == SYMBOL_DUPLICATE |
c3c445e1 | 254 | || !symbol_partitioned_p (node)); |
a66dc285 | 255 | |
c3c445e1 JH |
256 | while ((node1 = contained_in_symbol (node)) != node) |
257 | node = node1; | |
a66dc285 | 258 | |
67914693 | 259 | /* If we have duplicated symbol contained in something we cannot duplicate, |
c3c445e1 JH |
260 | we are very badly screwed. The other way is possible, so we do not |
261 | assert this in add_symbol_to_partition_1. | |
262 | ||
263 | Be lax about comdats; they may or may not be duplicated and we may | |
264 | end up in need to duplicate keyed comdat because it has unkeyed alias. */ | |
9cec31f4 | 265 | |
d52f5295 | 266 | gcc_assert (node->get_partitioning_class () == SYMBOL_DUPLICATE |
67348ccc | 267 | || DECL_COMDAT (node->decl) |
c3c445e1 | 268 | || !symbol_partitioned_p (node)); |
9cec31f4 | 269 | |
c3c445e1 | 270 | add_symbol_to_partition_1 (part, node); |
a66dc285 JH |
271 | } |
272 | ||
273 | /* Undo all additions until number of cgraph nodes in PARITION is N_CGRAPH_NODES | |
274 | and number of varpool nodes is N_VARPOOL_NODES. */ | |
275 | ||
276 | static void | |
7b99cca4 | 277 | undo_partition (ltrans_partition partition, unsigned int n_nodes) |
a66dc285 | 278 | { |
7b99cca4 | 279 | while (lto_symtab_encoder_size (partition->encoder) > (int)n_nodes) |
a66dc285 | 280 | { |
5e20cdc9 | 281 | symtab_node *node = lto_symtab_encoder_deref (partition->encoder, |
7b99cca4 | 282 | n_nodes); |
faae53f8 | 283 | partition->symbols--; |
39434bce | 284 | cgraph_node *cnode; |
c3c445e1 JH |
285 | |
286 | /* After UNDO we no longer know what was visited. */ | |
287 | if (partition->initializers_visited) | |
6e2830c3 | 288 | delete partition->initializers_visited; |
c3c445e1 JH |
289 | partition->initializers_visited = NULL; |
290 | ||
ddb0b824 JH |
291 | if (!node->alias && (cnode = dyn_cast <cgraph_node *> (node)) |
292 | && node->get_partitioning_class () == SYMBOL_PARTITION) | |
f658ad30 | 293 | partition->insns -= ipa_size_summaries->get (cnode)->size; |
7b99cca4 | 294 | lto_symtab_encoder_delete_node (partition->encoder, node); |
67348ccc | 295 | node->aux = (void *)((size_t)node->aux - 1); |
a66dc285 JH |
296 | } |
297 | } | |
298 | ||
a66dc285 JH |
299 | /* Group cgrah nodes by input files. This is used mainly for testing |
300 | right now. */ | |
301 | ||
302 | void | |
303 | lto_1_to_1_map (void) | |
304 | { | |
5e20cdc9 | 305 | symtab_node *node; |
a66dc285 | 306 | struct lto_file_decl_data *file_data; |
39c8aaa4 | 307 | hash_map<lto_file_decl_data *, ltrans_partition> pmap; |
a66dc285 | 308 | ltrans_partition partition; |
a66dc285 JH |
309 | int npartitions = 0; |
310 | ||
c3c445e1 | 311 | FOR_EACH_SYMBOL (node) |
a66dc285 | 312 | { |
d52f5295 | 313 | if (node->get_partitioning_class () != SYMBOL_PARTITION |
c3c445e1 | 314 | || symbol_partitioned_p (node)) |
a66dc285 JH |
315 | continue; |
316 | ||
67348ccc | 317 | file_data = node->lto_file_data; |
a66dc285 JH |
318 | |
319 | if (file_data) | |
320 | { | |
39c8aaa4 TS |
321 | ltrans_partition *slot = &pmap.get_or_insert (file_data); |
322 | if (*slot) | |
323 | partition = *slot; | |
a66dc285 JH |
324 | else |
325 | { | |
326 | partition = new_partition (file_data->file_name); | |
a66dc285 JH |
327 | *slot = partition; |
328 | npartitions++; | |
329 | } | |
330 | } | |
9771b263 DN |
331 | else if (!file_data && ltrans_partitions.length ()) |
332 | partition = ltrans_partitions[0]; | |
a66dc285 JH |
333 | else |
334 | { | |
335 | partition = new_partition (""); | |
39c8aaa4 | 336 | pmap.put (NULL, partition); |
a66dc285 JH |
337 | npartitions++; |
338 | } | |
339 | ||
67348ccc | 340 | add_symbol_to_partition (partition, node); |
a66dc285 | 341 | } |
a66dc285 JH |
342 | |
343 | /* If the cgraph is empty, create one cgraph node set so that there is still | |
344 | an output file for any variables that need to be exported in a DSO. */ | |
345 | if (!npartitions) | |
346 | new_partition ("empty"); | |
347 | ||
b6e33d73 JH |
348 | /* Order partitions by order of symbols because they are linked into binary |
349 | that way. */ | |
350 | ltrans_partitions.qsort (cmp_partitions_order); | |
c3c445e1 JH |
351 | } |
352 | ||
353 | /* Maximal partitioning. Put every new symbol into new partition if possible. */ | |
a66dc285 | 354 | |
c3c445e1 JH |
355 | void |
356 | lto_max_map (void) | |
357 | { | |
5e20cdc9 | 358 | symtab_node *node; |
c3c445e1 JH |
359 | ltrans_partition partition; |
360 | int npartitions = 0; | |
361 | ||
362 | FOR_EACH_SYMBOL (node) | |
363 | { | |
d52f5295 | 364 | if (node->get_partitioning_class () != SYMBOL_PARTITION |
c3c445e1 JH |
365 | || symbol_partitioned_p (node)) |
366 | continue; | |
fec39fa6 | 367 | partition = new_partition (node->asm_name ()); |
67348ccc | 368 | add_symbol_to_partition (partition, node); |
c3c445e1 JH |
369 | npartitions++; |
370 | } | |
371 | if (!npartitions) | |
372 | new_partition ("empty"); | |
a66dc285 JH |
373 | } |
374 | ||
a66dc285 JH |
375 | /* Helper function for qsort; sort nodes by order. */ |
376 | static int | |
6d8fd122 | 377 | node_cmp (const void *pa, const void *pb) |
a66dc285 | 378 | { |
7861b648 AK |
379 | const symtab_node *a = *static_cast<const symtab_node * const *> (pa); |
380 | const symtab_node *b = *static_cast<const symtab_node * const *> (pb); | |
67348ccc | 381 | return b->order - a->order; |
a66dc285 JH |
382 | } |
383 | ||
7861b648 AK |
384 | /* Add all symtab nodes from NEXT_NODE to PARTITION in order. */ |
385 | ||
386 | static void | |
387 | add_sorted_nodes (vec<symtab_node *> &next_nodes, ltrans_partition partition) | |
388 | { | |
389 | unsigned i; | |
390 | symtab_node *node; | |
391 | ||
6d8fd122 | 392 | next_nodes.qsort (node_cmp); |
7861b648 AK |
393 | FOR_EACH_VEC_ELT (next_nodes, i, node) |
394 | if (!symbol_partitioned_p (node)) | |
395 | add_symbol_to_partition (partition, node); | |
396 | } | |
397 | ||
ddb0b824 JH |
398 | /* Return true if we should account reference from N1 to N2 in cost |
399 | of partition boundary. */ | |
400 | ||
401 | bool | |
402 | account_reference_p (symtab_node *n1, symtab_node *n2) | |
403 | { | |
404 | if (cgraph_node *cnode = dyn_cast <cgraph_node *> (n1)) | |
405 | n1 = cnode; | |
ab168044 JH |
406 | /* Do not account references from aliases - they are never split across |
407 | partitions. */ | |
408 | if (n1->alias) | |
409 | return false; | |
ddb0b824 | 410 | /* Do not account recursion - the code below will handle it incorrectly |
ab168044 JH |
411 | otherwise. Do not account references to external symbols: they will |
412 | never become local. Finally do not account references to duplicated | |
413 | symbols: they will be always local. */ | |
ddb0b824 | 414 | if (n1 == n2 |
ab168044 JH |
415 | || !n2->definition |
416 | || n2->get_partitioning_class () != SYMBOL_PARTITION) | |
417 | return false; | |
418 | /* If referring node is external symbol do not account it to boundary | |
419 | cost. Those are added into units only to enable possible constant | |
420 | folding and devirtulization. | |
421 | ||
422 | Here we do not know if it will ever be added to some partition | |
423 | (this is decided by compute_ltrans_boundary) and second it is not | |
424 | that likely that constant folding will actually use the reference. */ | |
425 | if (contained_in_symbol (n1) | |
426 | ->get_partitioning_class () == SYMBOL_EXTERNAL) | |
ddb0b824 JH |
427 | return false; |
428 | return true; | |
429 | } | |
430 | ||
7861b648 | 431 | |
a66dc285 JH |
432 | /* Group cgraph nodes into equally-sized partitions. |
433 | ||
434 | The partitioning algorithm is simple: nodes are taken in predefined order. | |
435 | The order corresponds to the order we want functions to have in the final | |
436 | output. In the future this will be given by function reordering pass, but | |
437 | at the moment we use the topological order, which is a good approximation. | |
438 | ||
439 | The goal is to partition this linear order into intervals (partitions) so | |
440 | that all the partitions have approximately the same size and the number of | |
441 | callgraph or IPA reference edges crossing boundaries is minimal. | |
442 | ||
443 | This is a lot faster (O(n) in size of callgraph) than algorithms doing | |
444 | priority-based graph clustering that are generally O(n^2) and, since | |
445 | WHOPR is designed to make things go well across partitions, it leads | |
446 | to good results. | |
447 | ||
448 | We compute the expected size of a partition as: | |
449 | ||
450 | max (total_size / lto_partitions, min_partition_size) | |
451 | ||
452 | We use dynamic expected size of partition so small programs are partitioned | |
453 | into enough partitions to allow use of multiple CPUs, while large programs | |
454 | are not partitioned too much. Creating too many partitions significantly | |
455 | increases the streaming overhead. | |
456 | ||
457 | In the future, we would like to bound the maximal size of partitions so as | |
458 | to prevent the LTRANS stage from consuming too much memory. At the moment, | |
459 | however, the WPA stage is the most memory intensive for large benchmarks, | |
460 | since too many types and declarations are read into memory. | |
461 | ||
462 | The function implements a simple greedy algorithm. Nodes are being added | |
463 | to the current partition until after 3/4 of the expected partition size is | |
464 | reached. Past this threshold, we keep track of boundary size (number of | |
465 | edges going to other partitions) and continue adding functions until after | |
466 | the current partition has grown to twice the expected partition size. Then | |
467 | the process is undone to the point where the minimal ratio of boundary size | |
468 | and in-partition calls was reached. */ | |
469 | ||
470 | void | |
a2e76867 | 471 | lto_balanced_map (int n_lto_partitions, int max_partition_size) |
a66dc285 | 472 | { |
c525ba9a | 473 | int n_varpool_nodes = 0, varpool_pos = 0, best_varpool_pos = 0; |
74644756 | 474 | auto_vec <cgraph_node *> order (symtab->cgraph_count); |
7861b648 AK |
475 | auto_vec<cgraph_node *> noreorder; |
476 | auto_vec<varpool_node *> varpool_order; | |
a66dc285 | 477 | struct cgraph_node *node; |
ddb0b824 JH |
478 | int64_t original_total_size, total_size = 0; |
479 | int64_t partition_size; | |
a66dc285 | 480 | ltrans_partition partition; |
7b99cca4 | 481 | int last_visited_node = 0; |
2c8326a5 | 482 | varpool_node *vnode; |
ddb0b824 | 483 | int64_t cost = 0, internal = 0; |
74644756 | 484 | unsigned int best_n_nodes = 0, best_i = 0; |
ddb0b824 | 485 | int64_t best_cost = -1, best_internal = 0, best_size = 0; |
a66dc285 JH |
486 | int npartitions; |
487 | int current_order = -1; | |
7861b648 | 488 | int noreorder_pos = 0; |
a66dc285 | 489 | |
65c70e6b | 490 | FOR_EACH_VARIABLE (vnode) |
67348ccc | 491 | gcc_assert (!vnode->aux); |
74644756 | 492 | |
6a49f3c9 | 493 | FOR_EACH_DEFINED_FUNCTION (node) |
d52f5295 | 494 | if (node->get_partitioning_class () == SYMBOL_PARTITION) |
6a49f3c9 | 495 | { |
7861b648 AK |
496 | if (node->no_reorder) |
497 | noreorder.safe_push (node); | |
498 | else | |
74644756 | 499 | order.safe_push (node); |
39434bce | 500 | if (!node->alias) |
f658ad30 | 501 | total_size += ipa_size_summaries->get (node)->size; |
6a49f3c9 | 502 | } |
a66dc285 | 503 | |
faae53f8 ML |
504 | original_total_size = total_size; |
505 | ||
6a49f3c9 JH |
506 | /* Streaming works best when the source units do not cross partition |
507 | boundaries much. This is because importing function from a source | |
508 | unit tends to import a lot of global trees defined there. We should | |
509 | get better about minimizing the function bounday, but until that | |
510 | things works smoother if we order in source order. */ | |
6d8fd122 | 511 | order.qsort (tp_first_run_node_cmp); |
7861b648 | 512 | noreorder.qsort (node_cmp); |
9cec31f4 | 513 | |
29a50dfb | 514 | if (dump_file) |
a66dc285 | 515 | { |
74644756 | 516 | for (unsigned i = 0; i < order.length (); i++) |
59c7b29e JH |
517 | fprintf (dump_file, "Balanced map symbol order:%s:%u\n", |
518 | order[i]->name (), order[i]->tp_first_run); | |
74644756 | 519 | for (unsigned i = 0; i < noreorder.length (); i++) |
59c7b29e JH |
520 | fprintf (dump_file, "Balanced map symbol no_reorder:%s:%u\n", |
521 | noreorder[i]->name (), noreorder[i]->tp_first_run); | |
a66dc285 JH |
522 | } |
523 | ||
7861b648 AK |
524 | /* Collect all variables that should not be reordered. */ |
525 | FOR_EACH_VARIABLE (vnode) | |
526 | if (vnode->get_partitioning_class () == SYMBOL_PARTITION | |
0eaf0bfe | 527 | && vnode->no_reorder) |
7861b648 AK |
528 | varpool_order.safe_push (vnode); |
529 | n_varpool_nodes = varpool_order.length (); | |
6d8fd122 | 530 | varpool_order.qsort (node_cmp); |
7861b648 | 531 | |
a66dc285 | 532 | /* Compute partition size and create the first partition. */ |
028d4092 | 533 | if (param_min_partition_size > max_partition_size) |
ddb0b824 JH |
534 | fatal_error (input_location, "min partition size cannot be greater " |
535 | "than max partition size"); | |
a2e76867 | 536 | |
783dab6b | 537 | partition_size = total_size / n_lto_partitions; |
028d4092 ML |
538 | if (partition_size < param_min_partition_size) |
539 | partition_size = param_min_partition_size; | |
a66dc285 JH |
540 | npartitions = 1; |
541 | partition = new_partition (""); | |
29a50dfb JH |
542 | if (dump_file) |
543 | fprintf (dump_file, "Total unit size: %" PRId64 ", partition size: %" PRId64 "\n", | |
a66dc285 JH |
544 | total_size, partition_size); |
545 | ||
7861b648 AK |
546 | auto_vec<symtab_node *> next_nodes; |
547 | ||
74644756 | 548 | for (unsigned i = 0; i < order.length (); i++) |
a66dc285 | 549 | { |
67348ccc | 550 | if (symbol_partitioned_p (order[i])) |
a66dc285 JH |
551 | continue; |
552 | ||
67348ccc | 553 | current_order = order[i]->order; |
a66dc285 | 554 | |
7861b648 AK |
555 | /* Output noreorder and varpool in program order first. */ |
556 | next_nodes.truncate (0); | |
557 | while (varpool_pos < n_varpool_nodes | |
558 | && varpool_order[varpool_pos]->order < current_order) | |
559 | next_nodes.safe_push (varpool_order[varpool_pos++]); | |
560 | while (noreorder_pos < (int)noreorder.length () | |
561 | && noreorder[noreorder_pos]->order < current_order) | |
ddb0b824 | 562 | next_nodes.safe_push (noreorder[noreorder_pos++]); |
7861b648 | 563 | add_sorted_nodes (next_nodes, partition); |
a66dc285 | 564 | |
0eaf0bfe JH |
565 | if (!symbol_partitioned_p (order[i])) |
566 | add_symbol_to_partition (partition, order[i]); | |
a66dc285 JH |
567 | |
568 | ||
569 | /* Once we added a new node to the partition, we also want to add | |
570 | all referenced variables unless they was already added into some | |
571 | earlier partition. | |
c3c445e1 | 572 | add_symbol_to_partition adds possibly multiple nodes and |
a66dc285 JH |
573 | variables that are needed to satisfy needs of ORDER[i]. |
574 | We remember last visited cgraph and varpool node from last iteration | |
575 | of outer loop that allows us to process every new addition. | |
576 | ||
577 | At the same time we compute size of the boundary into COST. Every | |
578 | callgraph or IPA reference edge leaving the partition contributes into | |
579 | COST. Every edge inside partition was earlier computed as one leaving | |
580 | it and thus we need to subtract it from COST. */ | |
7b99cca4 | 581 | while (last_visited_node < lto_symtab_encoder_size (partition->encoder)) |
a66dc285 | 582 | { |
a66dc285 | 583 | int j; |
d122681a | 584 | struct ipa_ref *ref = NULL; |
5e20cdc9 | 585 | symtab_node *snode = lto_symtab_encoder_deref (partition->encoder, |
7b99cca4 | 586 | last_visited_node); |
a66dc285 | 587 | |
7de90a6c | 588 | if (cgraph_node *node = dyn_cast <cgraph_node *> (snode)) |
a66dc285 JH |
589 | { |
590 | struct cgraph_edge *edge; | |
591 | ||
a66dc285 | 592 | |
7b99cca4 | 593 | last_visited_node++; |
a66dc285 | 594 | |
67348ccc | 595 | gcc_assert (node->definition || node->weakref); |
a66dc285 JH |
596 | |
597 | /* Compute boundary cost of callgraph edges. */ | |
598 | for (edge = node->callees; edge; edge = edge->next_callee) | |
ddb0b824 JH |
599 | /* Inline edges will always end up local. */ |
600 | if (edge->inline_failed | |
601 | && account_reference_p (node, edge->callee)) | |
a66dc285 | 602 | { |
1bad9c18 | 603 | int edge_cost = edge->frequency (); |
7b99cca4 | 604 | int index; |
a66dc285 JH |
605 | |
606 | if (!edge_cost) | |
607 | edge_cost = 1; | |
608 | gcc_assert (edge_cost > 0); | |
7b99cca4 | 609 | index = lto_symtab_encoder_lookup (partition->encoder, |
67348ccc | 610 | edge->callee); |
7b99cca4 JH |
611 | if (index != LCC_NOT_FOUND |
612 | && index < last_visited_node - 1) | |
613 | cost -= edge_cost, internal += edge_cost; | |
a66dc285 JH |
614 | else |
615 | cost += edge_cost; | |
616 | } | |
617 | for (edge = node->callers; edge; edge = edge->next_caller) | |
ddb0b824 JH |
618 | if (edge->inline_failed |
619 | && account_reference_p (edge->caller, node)) | |
a66dc285 | 620 | { |
1bad9c18 | 621 | int edge_cost = edge->frequency (); |
7b99cca4 | 622 | int index; |
a66dc285 | 623 | |
67348ccc | 624 | gcc_assert (edge->caller->definition); |
a66dc285 JH |
625 | if (!edge_cost) |
626 | edge_cost = 1; | |
627 | gcc_assert (edge_cost > 0); | |
7b99cca4 | 628 | index = lto_symtab_encoder_lookup (partition->encoder, |
67348ccc | 629 | edge->caller); |
7b99cca4 JH |
630 | if (index != LCC_NOT_FOUND |
631 | && index < last_visited_node - 1) | |
ddb0b824 | 632 | cost -= edge_cost, internal += edge_cost; |
a66dc285 JH |
633 | else |
634 | cost += edge_cost; | |
635 | } | |
636 | } | |
637 | else | |
ddb0b824 | 638 | last_visited_node++; |
a66dc285 JH |
639 | |
640 | /* Compute boundary cost of IPA REF edges and at the same time look into | |
641 | variables referenced from current partition and try to add them. */ | |
ddb0b824 JH |
642 | for (j = 0; snode->iterate_reference (j, ref); j++) |
643 | if (!account_reference_p (snode, ref->referred)) | |
644 | ; | |
645 | else if (is_a <varpool_node *> (ref->referred)) | |
a66dc285 | 646 | { |
7b99cca4 | 647 | int index; |
a66dc285 | 648 | |
d122681a | 649 | vnode = dyn_cast <varpool_node *> (ref->referred); |
0eaf0bfe | 650 | if (!symbol_partitioned_p (vnode) |
7861b648 | 651 | && !vnode->no_reorder |
d52f5295 | 652 | && vnode->get_partitioning_class () == SYMBOL_PARTITION) |
67348ccc | 653 | add_symbol_to_partition (partition, vnode); |
7b99cca4 | 654 | index = lto_symtab_encoder_lookup (partition->encoder, |
67348ccc | 655 | vnode); |
7b99cca4 JH |
656 | if (index != LCC_NOT_FOUND |
657 | && index < last_visited_node - 1) | |
a66dc285 JH |
658 | cost--, internal++; |
659 | else | |
660 | cost++; | |
661 | } | |
662 | else | |
663 | { | |
7b99cca4 | 664 | int index; |
a66dc285 | 665 | |
d122681a | 666 | node = dyn_cast <cgraph_node *> (ref->referred); |
7b99cca4 | 667 | index = lto_symtab_encoder_lookup (partition->encoder, |
67348ccc | 668 | node); |
7b99cca4 JH |
669 | if (index != LCC_NOT_FOUND |
670 | && index < last_visited_node - 1) | |
a66dc285 JH |
671 | cost--, internal++; |
672 | else | |
673 | cost++; | |
674 | } | |
ddb0b824 JH |
675 | for (j = 0; snode->iterate_referring (j, ref); j++) |
676 | if (!account_reference_p (ref->referring, snode)) | |
677 | ; | |
678 | else if (is_a <varpool_node *> (ref->referring)) | |
a66dc285 | 679 | { |
7b99cca4 | 680 | int index; |
a66dc285 | 681 | |
d122681a | 682 | vnode = dyn_cast <varpool_node *> (ref->referring); |
67348ccc | 683 | gcc_assert (vnode->definition); |
24d3f325 NS |
684 | /* It is better to couple variables with their users, |
685 | because it allows them to be removed. Coupling | |
686 | with objects they refer to only helps to reduce | |
39434bce | 687 | number of symbols promoted to hidden. */ |
0eaf0bfe | 688 | if (!symbol_partitioned_p (vnode) |
7861b648 | 689 | && !vnode->no_reorder |
9041d2e6 | 690 | && !vnode->can_remove_if_no_refs_p () |
d52f5295 | 691 | && vnode->get_partitioning_class () == SYMBOL_PARTITION) |
67348ccc | 692 | add_symbol_to_partition (partition, vnode); |
7b99cca4 | 693 | index = lto_symtab_encoder_lookup (partition->encoder, |
67348ccc | 694 | vnode); |
7b99cca4 JH |
695 | if (index != LCC_NOT_FOUND |
696 | && index < last_visited_node - 1) | |
ddb0b824 | 697 | cost--, internal++; |
a66dc285 JH |
698 | else |
699 | cost++; | |
700 | } | |
701 | else | |
702 | { | |
7b99cca4 | 703 | int index; |
a66dc285 | 704 | |
d122681a | 705 | node = dyn_cast <cgraph_node *> (ref->referring); |
67348ccc | 706 | gcc_assert (node->definition); |
7b99cca4 | 707 | index = lto_symtab_encoder_lookup (partition->encoder, |
67348ccc | 708 | node); |
7b99cca4 JH |
709 | if (index != LCC_NOT_FOUND |
710 | && index < last_visited_node - 1) | |
ddb0b824 | 711 | cost--, internal++; |
a66dc285 JH |
712 | else |
713 | cost++; | |
714 | } | |
715 | } | |
716 | ||
ddb0b824 JH |
717 | gcc_assert (cost >= 0 && internal >= 0); |
718 | ||
719 | /* If the partition is large enough, start looking for smallest boundary cost. | |
720 | If partition still seems too small (less than 7/8 of target weight) accept | |
721 | any cost. If partition has right size, optimize for highest internal/cost. | |
722 | Later we stop building partition if its size is 9/8 of the target wight. */ | |
723 | if (partition->insns < partition_size * 7 / 8 | |
724 | || best_cost == -1 | |
725 | || (!cost | |
726 | || ((sreal)best_internal * (sreal) cost | |
727 | < ((sreal) internal * (sreal)best_cost)))) | |
a66dc285 JH |
728 | { |
729 | best_cost = cost; | |
730 | best_internal = internal; | |
ddb0b824 | 731 | best_size = partition->insns; |
a66dc285 | 732 | best_i = i; |
7b99cca4 | 733 | best_n_nodes = lto_symtab_encoder_size (partition->encoder); |
c525ba9a | 734 | best_varpool_pos = varpool_pos; |
a66dc285 | 735 | } |
29a50dfb JH |
736 | if (dump_file) |
737 | fprintf (dump_file, "Step %i: added %s/%i, size %i, " | |
ddb0b824 JH |
738 | "cost %" PRId64 "/%" PRId64 " " |
739 | "best %" PRId64 "/%" PRId64", step %i\n", i, | |
fec39fa6 | 740 | order[i]->name (), order[i]->order, |
9de04252 | 741 | partition->insns, cost, internal, |
a66dc285 JH |
742 | best_cost, best_internal, best_i); |
743 | /* Partition is too large, unwind into step when best cost was reached and | |
744 | start new partition. */ | |
ddb0b824 | 745 | if (partition->insns > 9 * partition_size / 8 |
a2e76867 | 746 | || partition->insns > max_partition_size) |
a66dc285 JH |
747 | { |
748 | if (best_i != i) | |
749 | { | |
29a50dfb JH |
750 | if (dump_file) |
751 | fprintf (dump_file, "Unwinding %i insertions to step %i\n", | |
a66dc285 | 752 | i - best_i, best_i); |
7b99cca4 | 753 | undo_partition (partition, best_n_nodes); |
c525ba9a | 754 | varpool_pos = best_varpool_pos; |
a66dc285 | 755 | } |
ddb0b824 | 756 | gcc_assert (best_size == partition->insns); |
a66dc285 | 757 | i = best_i; |
29a50dfb JH |
758 | if (dump_file) |
759 | fprintf (dump_file, | |
ddb0b824 JH |
760 | "Partition insns: %i (want %" PRId64 ")\n", |
761 | partition->insns, partition_size); | |
a66dc285 | 762 | /* When we are finished, avoid creating empty partition. */ |
74644756 | 763 | while (i < order.length () - 1 && symbol_partitioned_p (order[i + 1])) |
a66dc285 | 764 | i++; |
74644756 | 765 | if (i == order.length () - 1) |
a66dc285 | 766 | break; |
ddb0b824 | 767 | total_size -= partition->insns; |
a66dc285 | 768 | partition = new_partition (""); |
7b99cca4 | 769 | last_visited_node = 0; |
a66dc285 JH |
770 | cost = 0; |
771 | ||
29a50dfb JH |
772 | if (dump_file) |
773 | fprintf (dump_file, "New partition\n"); | |
a66dc285 | 774 | best_n_nodes = 0; |
ddb0b824 | 775 | best_cost = -1; |
a66dc285 JH |
776 | |
777 | /* Since the size of partitions is just approximate, update the size after | |
778 | we finished current one. */ | |
783dab6b RB |
779 | if (npartitions < n_lto_partitions) |
780 | partition_size = total_size / (n_lto_partitions - npartitions); | |
a66dc285 | 781 | else |
97bb4812 JH |
782 | /* Watch for overflow. */ |
783 | partition_size = INT_MAX / 16; | |
a66dc285 | 784 | |
29a50dfb JH |
785 | if (dump_file) |
786 | fprintf (dump_file, | |
ddb0b824 JH |
787 | "Total size: %" PRId64 " partition_size: %" PRId64 "\n", |
788 | total_size, partition_size); | |
028d4092 ML |
789 | if (partition_size < param_min_partition_size) |
790 | partition_size = param_min_partition_size; | |
a66dc285 JH |
791 | npartitions ++; |
792 | } | |
793 | } | |
794 | ||
7861b648 AK |
795 | next_nodes.truncate (0); |
796 | ||
a66dc285 | 797 | /* Varables that are not reachable from the code go into last partition. */ |
0eaf0bfe JH |
798 | FOR_EACH_VARIABLE (vnode) |
799 | if (vnode->get_partitioning_class () == SYMBOL_PARTITION | |
800 | && !symbol_partitioned_p (vnode)) | |
801 | next_nodes.safe_push (vnode); | |
7861b648 AK |
802 | |
803 | /* Output remaining ordered symbols. */ | |
804 | while (varpool_pos < n_varpool_nodes) | |
805 | next_nodes.safe_push (varpool_order[varpool_pos++]); | |
806 | while (noreorder_pos < (int)noreorder.length ()) | |
807 | next_nodes.safe_push (noreorder[noreorder_pos++]); | |
ddb0b824 JH |
808 | /* For one partition the cost of boundary should be 0 unless we added final |
809 | symbols here (these are not accounted) or we have accounting bug. */ | |
e693ebce | 810 | gcc_assert (next_nodes.length () || npartitions != 1 || !best_cost || best_cost == -1); |
7861b648 AK |
811 | add_sorted_nodes (next_nodes, partition); |
812 | ||
29a50dfb | 813 | if (dump_file) |
faae53f8 | 814 | { |
29a50dfb | 815 | fprintf (dump_file, "\nPartition sizes:\n"); |
faae53f8 ML |
816 | unsigned partitions = ltrans_partitions.length (); |
817 | ||
818 | for (unsigned i = 0; i < partitions ; i++) | |
819 | { | |
820 | ltrans_partition p = ltrans_partitions[i]; | |
29a50dfb | 821 | fprintf (dump_file, "partition %d contains %d (%2.2f%%)" |
faae53f8 | 822 | " symbols and %d (%2.2f%%) insns\n", i, p->symbols, |
74644756 | 823 | 100.0 * p->symbols / order.length (), p->insns, |
faae53f8 ML |
824 | 100.0 * p->insns / original_total_size); |
825 | } | |
826 | ||
29a50dfb | 827 | fprintf (dump_file, "\n"); |
faae53f8 | 828 | } |
a66dc285 JH |
829 | } |
830 | ||
9816367c BS |
831 | /* Return true if we must not change the name of the NODE. The name as |
832 | extracted from the corresponding decl should be passed in NAME. */ | |
64cfa6c0 | 833 | |
8ee05051 | 834 | static bool |
9816367c | 835 | must_not_rename (symtab_node *node, const char *name) |
64cfa6c0 | 836 | { |
64cfa6c0 JH |
837 | /* Our renaming machinery do not handle more than one change of assembler name. |
838 | We should not need more than one anyway. */ | |
67348ccc DM |
839 | if (node->lto_file_data |
840 | && lto_get_decl_name_mapping (node->lto_file_data, name) != name) | |
64cfa6c0 | 841 | { |
29a50dfb JH |
842 | if (dump_file) |
843 | fprintf (dump_file, | |
9816367c BS |
844 | "Not privatizing symbol name: %s. It privatized already.\n", |
845 | name); | |
846 | return true; | |
64cfa6c0 JH |
847 | } |
848 | /* Avoid mangling of already mangled clones. | |
849 | ??? should have a flag whether a symbol has a 'private' name already, | |
850 | since we produce some symbols like that i.e. for global constructors | |
851 | that are not really clones. */ | |
67348ccc | 852 | if (node->unique_name) |
64cfa6c0 | 853 | { |
29a50dfb JH |
854 | if (dump_file) |
855 | fprintf (dump_file, | |
9816367c BS |
856 | "Not privatizing symbol name: %s. Has unique name.\n", |
857 | name); | |
858 | return true; | |
64cfa6c0 | 859 | } |
9816367c BS |
860 | return false; |
861 | } | |
862 | ||
863 | /* If we are an offload compiler, we may have to rewrite symbols to be | |
864 | valid on this target. Return either PTR or a modified version of it. */ | |
865 | ||
866 | static const char * | |
867 | maybe_rewrite_identifier (const char *ptr) | |
868 | { | |
869 | #if defined ACCEL_COMPILER && (defined NO_DOT_IN_LABEL || defined NO_DOLLAR_IN_LABEL) | |
870 | #ifndef NO_DOT_IN_LABEL | |
871 | char valid = '.'; | |
872 | const char reject[] = "$"; | |
873 | #elif !defined NO_DOLLAR_IN_LABEL | |
874 | char valid = '$'; | |
875 | const char reject[] = "."; | |
876 | #else | |
877 | char valid = '_'; | |
878 | const char reject[] = ".$"; | |
879 | #endif | |
880 | ||
881 | char *copy = NULL; | |
882 | const char *match = ptr; | |
883 | for (;;) | |
884 | { | |
885 | size_t off = strcspn (match, reject); | |
886 | if (match[off] == '\0') | |
887 | break; | |
888 | if (copy == NULL) | |
889 | { | |
890 | copy = xstrdup (ptr); | |
891 | match = copy; | |
892 | } | |
893 | copy[off] = valid; | |
894 | } | |
895 | return match; | |
896 | #else | |
897 | return ptr; | |
898 | #endif | |
899 | } | |
900 | ||
901 | /* Ensure that the symbol in NODE is valid for the target, and if not, | |
902 | rewrite it. */ | |
903 | ||
904 | static void | |
905 | validize_symbol_for_target (symtab_node *node) | |
906 | { | |
907 | tree decl = node->decl; | |
908 | const char *name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)); | |
909 | ||
910 | if (must_not_rename (node, name)) | |
911 | return; | |
912 | ||
913 | const char *name2 = maybe_rewrite_identifier (name); | |
914 | if (name2 != name) | |
915 | { | |
916 | symtab->change_decl_assembler_name (decl, get_identifier (name2)); | |
917 | if (node->lto_file_data) | |
918 | lto_record_renamed_decl (node->lto_file_data, name, | |
919 | IDENTIFIER_POINTER | |
920 | (DECL_ASSEMBLER_NAME (decl))); | |
921 | } | |
922 | } | |
923 | ||
ed1b53a0 MP |
924 | /* Maps symbol names to unique lto clone counters. */ |
925 | static hash_map<const char *, unsigned> *lto_clone_numbers; | |
926 | ||
48de5d37 IE |
927 | /* Helper for privatize_symbol_name. Mangle NODE symbol name |
928 | represented by DECL. */ | |
9816367c BS |
929 | |
930 | static bool | |
48de5d37 | 931 | privatize_symbol_name_1 (symtab_node *node, tree decl) |
9816367c | 932 | { |
48de5d37 | 933 | const char *name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)); |
9816367c BS |
934 | |
935 | if (must_not_rename (node, name)) | |
936 | return false; | |
937 | ||
938 | name = maybe_rewrite_identifier (name); | |
ed1b53a0 | 939 | unsigned &clone_number = lto_clone_numbers->get_or_insert (name); |
3dafb85c | 940 | symtab->change_decl_assembler_name (decl, |
ed1b53a0 MP |
941 | clone_function_name ( |
942 | name, "lto_priv", clone_number)); | |
943 | clone_number++; | |
48de5d37 | 944 | |
67348ccc DM |
945 | if (node->lto_file_data) |
946 | lto_record_renamed_decl (node->lto_file_data, name, | |
64cfa6c0 JH |
947 | IDENTIFIER_POINTER |
948 | (DECL_ASSEMBLER_NAME (decl))); | |
48de5d37 | 949 | |
29a50dfb JH |
950 | if (dump_file) |
951 | fprintf (dump_file, | |
48de5d37 IE |
952 | "Privatizing symbol name: %s -> %s\n", |
953 | name, IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl))); | |
954 | ||
955 | return true; | |
956 | } | |
957 | ||
958 | /* Mangle NODE symbol name into a local name. | |
959 | This is necessary to do | |
960 | 1) if two or more static vars of same assembler name | |
961 | are merged into single ltrans unit. | |
962 | 2) if previously static var was promoted hidden to avoid possible conflict | |
963 | with symbols defined out of the LTO world. */ | |
964 | ||
965 | static bool | |
966 | privatize_symbol_name (symtab_node *node) | |
967 | { | |
968 | if (!privatize_symbol_name_1 (node, node->decl)) | |
969 | return false; | |
970 | ||
8ee05051 | 971 | return true; |
64cfa6c0 JH |
972 | } |
973 | ||
a66dc285 JH |
974 | /* Promote variable VNODE to be static. */ |
975 | ||
b4661bfe | 976 | static void |
5e20cdc9 | 977 | promote_symbol (symtab_node *node) |
a66dc285 | 978 | { |
b4661bfe | 979 | /* We already promoted ... */ |
67348ccc DM |
980 | if (DECL_VISIBILITY (node->decl) == VISIBILITY_HIDDEN |
981 | && DECL_VISIBILITY_SPECIFIED (node->decl) | |
982 | && TREE_PUBLIC (node->decl)) | |
9816367c BS |
983 | { |
984 | validize_symbol_for_target (node); | |
985 | return; | |
986 | } | |
a66dc285 | 987 | |
67348ccc DM |
988 | gcc_checking_assert (!TREE_PUBLIC (node->decl) |
989 | && !DECL_EXTERNAL (node->decl)); | |
64cfa6c0 JH |
990 | /* Be sure that newly public symbol does not conflict with anything already |
991 | defined by the non-LTO part. */ | |
992 | privatize_symbol_name (node); | |
67348ccc DM |
993 | TREE_PUBLIC (node->decl) = 1; |
994 | DECL_VISIBILITY (node->decl) = VISIBILITY_HIDDEN; | |
995 | DECL_VISIBILITY_SPECIFIED (node->decl) = true; | |
29a50dfb JH |
996 | if (dump_file) |
997 | fprintf (dump_file, | |
5406184c NS |
998 | "Promoting as hidden: %s (%s)\n", node->name (), |
999 | IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl))); | |
41c60c6c | 1000 | |
5406184c | 1001 | /* Promoting a symbol also promotes all transparent aliases with exception |
41c60c6c JH |
1002 | of weakref where the visibility flags are always wrong and set to |
1003 | !PUBLIC. */ | |
5406184c | 1004 | ipa_ref *ref; |
41c60c6c JH |
1005 | for (unsigned i = 0; node->iterate_direct_aliases (i, ref); i++) |
1006 | { | |
1007 | struct symtab_node *alias = ref->referring; | |
1008 | if (alias->transparent_alias && !alias->weakref) | |
1009 | { | |
1010 | TREE_PUBLIC (alias->decl) = 1; | |
1011 | DECL_VISIBILITY (alias->decl) = VISIBILITY_HIDDEN; | |
1012 | DECL_VISIBILITY_SPECIFIED (alias->decl) = true; | |
29a50dfb JH |
1013 | if (dump_file) |
1014 | fprintf (dump_file, | |
5406184c NS |
1015 | "Promoting alias as hidden: %s\n", |
1016 | IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl))); | |
41c60c6c JH |
1017 | } |
1018 | gcc_assert (!alias->weakref || TREE_PUBLIC (alias->decl)); | |
1019 | } | |
a66dc285 JH |
1020 | } |
1021 | ||
24d3f325 NS |
1022 | /* Return true if NODE needs named section even if it won't land in |
1023 | the partition symbol table. | |
1024 | ||
1025 | FIXME: we should really not use named sections for inline clones | |
1026 | and master clones. */ | |
64cfa6c0 JH |
1027 | |
1028 | static bool | |
5e20cdc9 | 1029 | may_need_named_section_p (lto_symtab_encoder_t encoder, symtab_node *node) |
64cfa6c0 | 1030 | { |
7de90a6c | 1031 | struct cgraph_node *cnode = dyn_cast <cgraph_node *> (node); |
64cfa6c0 JH |
1032 | if (!cnode) |
1033 | return false; | |
d52f5295 | 1034 | if (node->real_symbol_p ()) |
64cfa6c0 | 1035 | return false; |
64cfa6c0 JH |
1036 | return (!encoder |
1037 | || (lto_symtab_encoder_lookup (encoder, node) != LCC_NOT_FOUND | |
1038 | && lto_symtab_encoder_encode_body_p (encoder, | |
1039 | cnode))); | |
1040 | } | |
1041 | ||
1042 | /* If NODE represents a static variable. See if there are other variables | |
1043 | of the same name in partition ENCODER (or in whole compilation unit if | |
1044 | ENCODER is NULL) and if so, mangle the statics. Always mangle all | |
1045 | conflicting statics, so we reduce changes of silently miscompiling | |
9cec31f4 | 1046 | asm statements referring to them by symbol name. */ |
64cfa6c0 JH |
1047 | |
1048 | static void | |
5e20cdc9 | 1049 | rename_statics (lto_symtab_encoder_t encoder, symtab_node *node) |
64cfa6c0 | 1050 | { |
67348ccc | 1051 | tree decl = node->decl; |
5e20cdc9 | 1052 | symtab_node *s; |
64cfa6c0 JH |
1053 | tree name = DECL_ASSEMBLER_NAME (decl); |
1054 | ||
1055 | /* See if this is static symbol. */ | |
41c60c6c | 1056 | if (((node->externally_visible && !node->weakref) |
64cfa6c0 JH |
1057 | /* FIXME: externally_visible is somewhat illogically not set for |
1058 | external symbols (i.e. those not defined). Remove this test | |
1059 | once this is fixed. */ | |
67348ccc | 1060 | || DECL_EXTERNAL (node->decl) |
d52f5295 | 1061 | || !node->real_symbol_p ()) |
08346abd | 1062 | && !may_need_named_section_p (encoder, node)) |
64cfa6c0 JH |
1063 | return; |
1064 | ||
1065 | /* Now walk symbols sharing the same name and see if there are any conflicts. | |
67914693 | 1066 | (all types of symbols counts here, since we cannot have static of the |
64cfa6c0 | 1067 | same name as external or public symbol.) */ |
3dafb85c | 1068 | for (s = symtab_node::get_for_asmname (name); |
67348ccc | 1069 | s; s = s->next_sharing_asm_name) |
d52f5295 | 1070 | if ((s->real_symbol_p () || may_need_named_section_p (encoder, s)) |
67348ccc | 1071 | && s->decl != node->decl |
64cfa6c0 JH |
1072 | && (!encoder |
1073 | || lto_symtab_encoder_lookup (encoder, s) != LCC_NOT_FOUND)) | |
1074 | break; | |
1075 | ||
1076 | /* OK, no confict, so we have nothing to do. */ | |
1077 | if (!s) | |
1078 | return; | |
1079 | ||
29a50dfb JH |
1080 | if (dump_file) |
1081 | fprintf (dump_file, | |
fec39fa6 | 1082 | "Renaming statics with asm name: %s\n", node->name ()); |
64cfa6c0 JH |
1083 | |
1084 | /* Assign every symbol in the set that shares the same ASM name an unique | |
1085 | mangled name. */ | |
3dafb85c | 1086 | for (s = symtab_node::get_for_asmname (name); s;) |
71e54687 JH |
1087 | if ((!s->externally_visible || s->weakref) |
1088 | /* Transparent aliases having same name as target are renamed at a | |
1089 | time their target gets new name. Transparent aliases that use | |
1090 | separate assembler name require the name to be unique. */ | |
1091 | && (!s->transparent_alias || !s->definition || s->weakref | |
1092 | || !symbol_table::assembler_names_equal_p | |
1093 | (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (s->decl)), | |
1094 | IDENTIFIER_POINTER | |
1095 | (DECL_ASSEMBLER_NAME (s->get_alias_target()->decl)))) | |
d52f5295 | 1096 | && ((s->real_symbol_p () |
5803bd01 JH |
1097 | && !DECL_EXTERNAL (s->decl) |
1098 | && !TREE_PUBLIC (s->decl)) | |
64cfa6c0 JH |
1099 | || may_need_named_section_p (encoder, s)) |
1100 | && (!encoder | |
1101 | || lto_symtab_encoder_lookup (encoder, s) != LCC_NOT_FOUND)) | |
1102 | { | |
8ee05051 | 1103 | if (privatize_symbol_name (s)) |
24d3f325 NS |
1104 | /* Re-start from beginning since we do not know how many |
1105 | symbols changed a name. */ | |
3dafb85c | 1106 | s = symtab_node::get_for_asmname (name); |
67348ccc | 1107 | else s = s->next_sharing_asm_name; |
64cfa6c0 | 1108 | } |
67348ccc | 1109 | else s = s->next_sharing_asm_name; |
64cfa6c0 | 1110 | } |
b4661bfe | 1111 | |
a66dc285 JH |
1112 | /* Find out all static decls that need to be promoted to global because |
1113 | of cross file sharing. This function must be run in the WPA mode after | |
1114 | all inlinees are added. */ | |
1115 | ||
1116 | void | |
1117 | lto_promote_cross_file_statics (void) | |
1118 | { | |
a66dc285 | 1119 | unsigned i, n_sets; |
a66dc285 JH |
1120 | |
1121 | gcc_assert (flag_wpa); | |
1122 | ||
837bac8c IV |
1123 | lto_stream_offload_p = false; |
1124 | select_what_to_stream (); | |
1f6be682 | 1125 | |
b4661bfe | 1126 | /* First compute boundaries. */ |
9771b263 | 1127 | n_sets = ltrans_partitions.length (); |
a66dc285 JH |
1128 | for (i = 0; i < n_sets; i++) |
1129 | { | |
1130 | ltrans_partition part | |
9771b263 | 1131 | = ltrans_partitions[i]; |
b4661bfe JH |
1132 | part->encoder = compute_ltrans_boundary (part->encoder); |
1133 | } | |
a66dc285 | 1134 | |
ed1b53a0 MP |
1135 | lto_clone_numbers = new hash_map<const char *, unsigned>; |
1136 | ||
b4661bfe JH |
1137 | /* Look at boundaries and promote symbols as needed. */ |
1138 | for (i = 0; i < n_sets; i++) | |
1139 | { | |
1140 | lto_symtab_encoder_iterator lsei; | |
64cfa6c0 | 1141 | lto_symtab_encoder_t encoder = ltrans_partitions[i]->encoder; |
a66dc285 | 1142 | |
b4661bfe JH |
1143 | for (lsei = lsei_start (encoder); !lsei_end_p (lsei); |
1144 | lsei_next (&lsei)) | |
1145 | { | |
5e20cdc9 | 1146 | symtab_node *node = lsei_node (lsei); |
b4661bfe | 1147 | |
24d3f325 NS |
1148 | /* If symbol is static, rename it if its assembler name |
1149 | clashes with anything else in this unit. */ | |
64cfa6c0 JH |
1150 | rename_statics (encoder, node); |
1151 | ||
b4661bfe | 1152 | /* No need to promote if symbol already is externally visible ... */ |
67348ccc | 1153 | if (node->externally_visible |
b4661bfe JH |
1154 | /* ... or if it is part of current partition ... */ |
1155 | || lto_symtab_encoder_in_partition_p (encoder, node) | |
1156 | /* ... or if we do not partition it. This mean that it will | |
24d3f325 | 1157 | appear in every partition referencing it. */ |
d52f5295 | 1158 | || node->get_partitioning_class () != SYMBOL_PARTITION) |
9816367c BS |
1159 | { |
1160 | validize_symbol_for_target (node); | |
1161 | continue; | |
1162 | } | |
a66dc285 | 1163 | |
b4661bfe JH |
1164 | promote_symbol (node); |
1165 | } | |
a66dc285 | 1166 | } |
ed1b53a0 | 1167 | delete lto_clone_numbers; |
a66dc285 | 1168 | } |
64cfa6c0 JH |
1169 | |
1170 | /* Rename statics in the whole unit in the case that | |
1171 | we do -flto-partition=none. */ | |
1172 | ||
1173 | void | |
1174 | lto_promote_statics_nonwpa (void) | |
1175 | { | |
5e20cdc9 | 1176 | symtab_node *node; |
ed1b53a0 MP |
1177 | |
1178 | lto_clone_numbers = new hash_map<const char *, unsigned>; | |
64cfa6c0 | 1179 | FOR_EACH_SYMBOL (node) |
9816367c BS |
1180 | { |
1181 | rename_statics (NULL, node); | |
1182 | validize_symbol_for_target (node); | |
1183 | } | |
ed1b53a0 | 1184 | delete lto_clone_numbers; |
64cfa6c0 | 1185 | } |