]>
Commit | Line | Data |
---|---|---|
a66dc285 | 1 | /* LTO partitioning logic routines. |
a5544970 | 2 | Copyright (C) 2009-2019 Free Software Foundation, Inc. |
a66dc285 JH |
3 | |
4 | This file is part of GCC. | |
5 | ||
6 | GCC is free software; you can redistribute it and/or modify it under | |
7 | the terms of the GNU General Public License as published by the Free | |
8 | Software Foundation; either version 3, or (at your option) any later | |
9 | version. | |
10 | ||
11 | GCC is distributed in the hope that it will be useful, but WITHOUT ANY | |
12 | WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
13 | FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
14 | for more details. | |
15 | ||
16 | You should have received a copy of the GNU General Public License | |
17 | along with GCC; see the file COPYING3. If not see | |
18 | <http://www.gnu.org/licenses/>. */ | |
19 | ||
20 | #include "config.h" | |
21 | #include "system.h" | |
22 | #include "coretypes.h" | |
2adfab87 | 23 | #include "target.h" |
60393bbc | 24 | #include "function.h" |
2fb9a547 | 25 | #include "basic-block.h" |
c7131fb2 | 26 | #include "tree.h" |
8e9055ae | 27 | #include "gimple.h" |
f51ade33 | 28 | #include "alloc-pool.h" |
2adfab87 AM |
29 | #include "stringpool.h" |
30 | #include "cgraph.h" | |
31 | #include "lto-streamer.h" | |
a66dc285 | 32 | #include "params.h" |
dd912cb8 | 33 | #include "symbol-summary.h" |
8bc5448f | 34 | #include "tree-vrp.h" |
c582198b | 35 | #include "ipa-prop.h" |
27d020cf | 36 | #include "ipa-fnsummary.h" |
a66dc285 | 37 | #include "lto-partition.h" |
ddb0b824 | 38 | #include "sreal.h" |
a66dc285 | 39 | |
9771b263 | 40 | vec<ltrans_partition> ltrans_partitions; |
a66dc285 | 41 | |
5e20cdc9 | 42 | static void add_symbol_to_partition (ltrans_partition part, symtab_node *node); |
c3c445e1 | 43 | |
a66dc285 | 44 | |
b6e33d73 JH |
45 | /* Helper for qsort; compare partitions and return one with smaller order. */ |
46 | ||
47 | static int | |
48 | cmp_partitions_order (const void *a, const void *b) | |
49 | { | |
50 | const struct ltrans_partition_def *pa | |
51 | = *(struct ltrans_partition_def *const *)a; | |
52 | const struct ltrans_partition_def *pb | |
53 | = *(struct ltrans_partition_def *const *)b; | |
54 | int ordera = -1, orderb = -1; | |
55 | ||
56 | if (lto_symtab_encoder_size (pa->encoder)) | |
57 | ordera = lto_symtab_encoder_deref (pa->encoder, 0)->order; | |
58 | if (lto_symtab_encoder_size (pb->encoder)) | |
59 | orderb = lto_symtab_encoder_deref (pb->encoder, 0)->order; | |
60 | return orderb - ordera; | |
61 | } | |
62 | ||
a66dc285 | 63 | /* Create new partition with name NAME. */ |
c3c445e1 | 64 | |
a66dc285 JH |
65 | static ltrans_partition |
66 | new_partition (const char *name) | |
67 | { | |
68 | ltrans_partition part = XCNEW (struct ltrans_partition_def); | |
e75f8f79 | 69 | part->encoder = lto_symtab_encoder_new (false); |
a66dc285 JH |
70 | part->name = name; |
71 | part->insns = 0; | |
faae53f8 | 72 | part->symbols = 0; |
9771b263 | 73 | ltrans_partitions.safe_push (part); |
a66dc285 JH |
74 | return part; |
75 | } | |
76 | ||
77 | /* Free memory used by ltrans datastructures. */ | |
c3c445e1 | 78 | |
a66dc285 JH |
79 | void |
80 | free_ltrans_partitions (void) | |
81 | { | |
82 | unsigned int idx; | |
83 | ltrans_partition part; | |
9771b263 | 84 | for (idx = 0; ltrans_partitions.iterate (idx, &part); idx++) |
a66dc285 | 85 | { |
c3c445e1 | 86 | if (part->initializers_visited) |
6e2830c3 | 87 | delete part->initializers_visited; |
7b99cca4 | 88 | /* Symtab encoder is freed after streaming. */ |
a66dc285 JH |
89 | free (part); |
90 | } | |
9771b263 | 91 | ltrans_partitions.release (); |
a66dc285 JH |
92 | } |
93 | ||
c3c445e1 JH |
94 | /* Return true if symbol is already in some partition. */ |
95 | ||
96 | static inline bool | |
5e20cdc9 | 97 | symbol_partitioned_p (symtab_node *node) |
a66dc285 | 98 | { |
67348ccc | 99 | return node->aux; |
1cdbb3f9 JH |
100 | } |
101 | ||
c3c445e1 | 102 | /* Add references into the partition. */ |
1cdbb3f9 | 103 | static void |
5e20cdc9 | 104 | add_references_to_partition (ltrans_partition part, symtab_node *node) |
1cdbb3f9 JH |
105 | { |
106 | int i; | |
d122681a | 107 | struct ipa_ref *ref = NULL; |
1cdbb3f9 | 108 | |
c3c445e1 | 109 | /* Add all duplicated references to the partition. */ |
d122681a | 110 | for (i = 0; node->iterate_reference (i, ref); i++) |
d52f5295 | 111 | if (ref->referred->get_partitioning_class () == SYMBOL_DUPLICATE) |
c3c445e1 JH |
112 | add_symbol_to_partition (part, ref->referred); |
113 | /* References to a readonly variable may be constant foled into its value. | |
114 | Recursively look into the initializers of the constant variable and add | |
115 | references, too. */ | |
7de90a6c | 116 | else if (is_a <varpool_node *> (ref->referred) |
d5e254e1 | 117 | && (dyn_cast <varpool_node *> (ref->referred) |
31db0fe0 | 118 | ->ctor_useable_for_folding_p ()) |
c3c445e1 | 119 | && !lto_symtab_encoder_in_partition_p (part->encoder, ref->referred)) |
1cdbb3f9 | 120 | { |
c3c445e1 | 121 | if (!part->initializers_visited) |
6e2830c3 TS |
122 | part->initializers_visited = new hash_set<symtab_node *>; |
123 | if (!part->initializers_visited->add (ref->referred)) | |
c3c445e1 | 124 | add_references_to_partition (part, ref->referred); |
1cdbb3f9 | 125 | } |
a66dc285 JH |
126 | } |
127 | ||
c3c445e1 JH |
128 | /* Helper function for add_symbol_to_partition doing the actual dirty work |
129 | of adding NODE to PART. */ | |
a66dc285 JH |
130 | |
131 | static bool | |
5e20cdc9 | 132 | add_symbol_to_partition_1 (ltrans_partition part, symtab_node *node) |
a66dc285 | 133 | { |
d52f5295 | 134 | enum symbol_partitioning_class c = node->get_partitioning_class (); |
e55637b7 | 135 | struct ipa_ref *ref; |
5e20cdc9 | 136 | symtab_node *node1; |
a66dc285 | 137 | |
c3c445e1 | 138 | /* If NODE is already there, we have nothing to do. */ |
67348ccc | 139 | if (lto_symtab_encoder_in_partition_p (part->encoder, node)) |
c3c445e1 JH |
140 | return true; |
141 | ||
142 | /* non-duplicated aliases or tunks of a duplicated symbol needs to be output | |
143 | just once. | |
144 | ||
145 | Be lax about comdats; they may or may not be duplicated and we may | |
146 | end up in need to duplicate keyed comdat because it has unkeyed alias. */ | |
67348ccc | 147 | if (c == SYMBOL_PARTITION && !DECL_COMDAT (node->decl) |
c3c445e1 | 148 | && symbol_partitioned_p (node)) |
1cdbb3f9 JH |
149 | return false; |
150 | ||
c3c445e1 JH |
151 | /* Be sure that we never try to duplicate partitioned symbol |
152 | or add external symbol. */ | |
3a849280 | 153 | gcc_assert (c != SYMBOL_EXTERNAL |
c3c445e1 JH |
154 | && (c == SYMBOL_DUPLICATE || !symbol_partitioned_p (node))); |
155 | ||
faae53f8 ML |
156 | part->symbols++; |
157 | ||
67348ccc | 158 | lto_set_symtab_encoder_in_partition (part->encoder, node); |
a66dc285 | 159 | |
c3c445e1 | 160 | if (symbol_partitioned_p (node)) |
a66dc285 | 161 | { |
67348ccc | 162 | node->in_other_partition = 1; |
29a50dfb JH |
163 | if (dump_file) |
164 | fprintf (dump_file, | |
3dafb85c | 165 | "Symbol node %s now used in multiple partitions\n", |
fec39fa6 | 166 | node->name ()); |
a66dc285 | 167 | } |
67348ccc | 168 | node->aux = (void *)((size_t)node->aux + 1); |
a66dc285 | 169 | |
7de90a6c | 170 | if (cgraph_node *cnode = dyn_cast <cgraph_node *> (node)) |
c3c445e1 | 171 | { |
c3c445e1 | 172 | struct cgraph_edge *e; |
ddb0b824 | 173 | if (!node->alias && c == SYMBOL_PARTITION) |
56f62793 | 174 | part->insns += ipa_fn_summaries->get (cnode)->size; |
c3c445e1 JH |
175 | |
176 | /* Add all inline clones and callees that are duplicated. */ | |
177 | for (e = cnode->callees; e; e = e->next_callee) | |
178 | if (!e->inline_failed) | |
67348ccc | 179 | add_symbol_to_partition_1 (part, e->callee); |
d52f5295 | 180 | else if (e->callee->get_partitioning_class () == SYMBOL_DUPLICATE) |
67348ccc | 181 | add_symbol_to_partition (part, e->callee); |
c3c445e1 JH |
182 | |
183 | /* Add all thunks associated with the function. */ | |
184 | for (e = cnode->callers; e; e = e->next_caller) | |
479f99c9 | 185 | if (e->caller->thunk.thunk_p && !e->caller->global.inlined_to) |
67348ccc | 186 | add_symbol_to_partition_1 (part, e->caller); |
c3c445e1 | 187 | } |
a66dc285 | 188 | |
c3c445e1 | 189 | add_references_to_partition (part, node); |
a66dc285 | 190 | |
c3c445e1 | 191 | /* Add all aliases associated with the symbol. */ |
e55637b7 ML |
192 | |
193 | FOR_EACH_ALIAS (node, ref) | |
41c60c6c | 194 | if (!ref->referring->transparent_alias) |
c3c445e1 | 195 | add_symbol_to_partition_1 (part, ref->referring); |
41c60c6c JH |
196 | else |
197 | { | |
198 | struct ipa_ref *ref2; | |
199 | /* We do not need to add transparent aliases if they are not used. | |
200 | However we must add aliases of transparent aliases if they exist. */ | |
201 | FOR_EACH_ALIAS (ref->referring, ref2) | |
202 | { | |
203 | /* Nested transparent aliases are not permitted. */ | |
204 | gcc_checking_assert (!ref2->referring->transparent_alias); | |
205 | add_symbol_to_partition_1 (part, ref2->referring); | |
206 | } | |
207 | } | |
a66dc285 | 208 | |
c3c445e1 | 209 | /* Ensure that SAME_COMDAT_GROUP lists all allways added in a group. */ |
67348ccc DM |
210 | if (node->same_comdat_group) |
211 | for (node1 = node->same_comdat_group; | |
212 | node1 != node; node1 = node1->same_comdat_group) | |
96451279 JH |
213 | if (!node->alias) |
214 | { | |
215 | bool added = add_symbol_to_partition_1 (part, node1); | |
216 | gcc_assert (added); | |
217 | } | |
c3c445e1 | 218 | return true; |
a66dc285 JH |
219 | } |
220 | ||
c3c445e1 JH |
221 | /* If symbol NODE is really part of other symbol's definition (i.e. it is |
222 | internal label, thunk, alias or so), return the outer symbol. | |
223 | When add_symbol_to_partition_1 is called on the outer symbol it must | |
224 | eventually add NODE, too. */ | |
5e20cdc9 DM |
225 | static symtab_node * |
226 | contained_in_symbol (symtab_node *node) | |
c3c445e1 | 227 | { |
41c60c6c JH |
228 | /* There is no need to consider transparent aliases to be part of the |
229 | definition: they are only useful insite the partition they are output | |
230 | and thus we will always see an explicit reference to it. */ | |
231 | if (node->transparent_alias) | |
c3c445e1 | 232 | return node; |
7de90a6c | 233 | if (cgraph_node *cnode = dyn_cast <cgraph_node *> (node)) |
c3c445e1 | 234 | { |
d52f5295 | 235 | cnode = cnode->function_symbol (); |
c3c445e1 JH |
236 | if (cnode->global.inlined_to) |
237 | cnode = cnode->global.inlined_to; | |
67348ccc | 238 | return cnode; |
c3c445e1 | 239 | } |
7de90a6c | 240 | else if (varpool_node *vnode = dyn_cast <varpool_node *> (node)) |
9041d2e6 | 241 | return vnode->ultimate_alias_target (); |
c3c445e1 JH |
242 | return node; |
243 | } | |
244 | ||
245 | /* Add symbol NODE to partition. When definition of NODE is part | |
246 | of other symbol definition, add the other symbol, too. */ | |
a66dc285 JH |
247 | |
248 | static void | |
5e20cdc9 | 249 | add_symbol_to_partition (ltrans_partition part, symtab_node *node) |
a66dc285 | 250 | { |
5e20cdc9 | 251 | symtab_node *node1; |
a66dc285 | 252 | |
c3c445e1 | 253 | /* Verify that we do not try to duplicate something that can not be. */ |
d52f5295 | 254 | gcc_checking_assert (node->get_partitioning_class () == SYMBOL_DUPLICATE |
c3c445e1 | 255 | || !symbol_partitioned_p (node)); |
a66dc285 | 256 | |
c3c445e1 JH |
257 | while ((node1 = contained_in_symbol (node)) != node) |
258 | node = node1; | |
a66dc285 | 259 | |
c3c445e1 JH |
260 | /* If we have duplicated symbol contained in something we can not duplicate, |
261 | we are very badly screwed. The other way is possible, so we do not | |
262 | assert this in add_symbol_to_partition_1. | |
263 | ||
264 | Be lax about comdats; they may or may not be duplicated and we may | |
265 | end up in need to duplicate keyed comdat because it has unkeyed alias. */ | |
9cec31f4 | 266 | |
d52f5295 | 267 | gcc_assert (node->get_partitioning_class () == SYMBOL_DUPLICATE |
67348ccc | 268 | || DECL_COMDAT (node->decl) |
c3c445e1 | 269 | || !symbol_partitioned_p (node)); |
9cec31f4 | 270 | |
c3c445e1 | 271 | add_symbol_to_partition_1 (part, node); |
a66dc285 JH |
272 | } |
273 | ||
274 | /* Undo all additions until number of cgraph nodes in PARITION is N_CGRAPH_NODES | |
275 | and number of varpool nodes is N_VARPOOL_NODES. */ | |
276 | ||
277 | static void | |
7b99cca4 | 278 | undo_partition (ltrans_partition partition, unsigned int n_nodes) |
a66dc285 | 279 | { |
7b99cca4 | 280 | while (lto_symtab_encoder_size (partition->encoder) > (int)n_nodes) |
a66dc285 | 281 | { |
5e20cdc9 | 282 | symtab_node *node = lto_symtab_encoder_deref (partition->encoder, |
7b99cca4 | 283 | n_nodes); |
faae53f8 | 284 | partition->symbols--; |
39434bce | 285 | cgraph_node *cnode; |
c3c445e1 JH |
286 | |
287 | /* After UNDO we no longer know what was visited. */ | |
288 | if (partition->initializers_visited) | |
6e2830c3 | 289 | delete partition->initializers_visited; |
c3c445e1 JH |
290 | partition->initializers_visited = NULL; |
291 | ||
ddb0b824 JH |
292 | if (!node->alias && (cnode = dyn_cast <cgraph_node *> (node)) |
293 | && node->get_partitioning_class () == SYMBOL_PARTITION) | |
56f62793 | 294 | partition->insns -= ipa_fn_summaries->get (cnode)->size; |
7b99cca4 | 295 | lto_symtab_encoder_delete_node (partition->encoder, node); |
67348ccc | 296 | node->aux = (void *)((size_t)node->aux - 1); |
a66dc285 JH |
297 | } |
298 | } | |
299 | ||
a66dc285 JH |
300 | /* Group cgrah nodes by input files. This is used mainly for testing |
301 | right now. */ | |
302 | ||
303 | void | |
304 | lto_1_to_1_map (void) | |
305 | { | |
5e20cdc9 | 306 | symtab_node *node; |
a66dc285 | 307 | struct lto_file_decl_data *file_data; |
39c8aaa4 | 308 | hash_map<lto_file_decl_data *, ltrans_partition> pmap; |
a66dc285 | 309 | ltrans_partition partition; |
a66dc285 JH |
310 | int npartitions = 0; |
311 | ||
c3c445e1 | 312 | FOR_EACH_SYMBOL (node) |
a66dc285 | 313 | { |
d52f5295 | 314 | if (node->get_partitioning_class () != SYMBOL_PARTITION |
c3c445e1 | 315 | || symbol_partitioned_p (node)) |
a66dc285 JH |
316 | continue; |
317 | ||
67348ccc | 318 | file_data = node->lto_file_data; |
a66dc285 JH |
319 | |
320 | if (file_data) | |
321 | { | |
39c8aaa4 TS |
322 | ltrans_partition *slot = &pmap.get_or_insert (file_data); |
323 | if (*slot) | |
324 | partition = *slot; | |
a66dc285 JH |
325 | else |
326 | { | |
327 | partition = new_partition (file_data->file_name); | |
a66dc285 JH |
328 | *slot = partition; |
329 | npartitions++; | |
330 | } | |
331 | } | |
9771b263 DN |
332 | else if (!file_data && ltrans_partitions.length ()) |
333 | partition = ltrans_partitions[0]; | |
a66dc285 JH |
334 | else |
335 | { | |
336 | partition = new_partition (""); | |
39c8aaa4 | 337 | pmap.put (NULL, partition); |
a66dc285 JH |
338 | npartitions++; |
339 | } | |
340 | ||
67348ccc | 341 | add_symbol_to_partition (partition, node); |
a66dc285 | 342 | } |
a66dc285 JH |
343 | |
344 | /* If the cgraph is empty, create one cgraph node set so that there is still | |
345 | an output file for any variables that need to be exported in a DSO. */ | |
346 | if (!npartitions) | |
347 | new_partition ("empty"); | |
348 | ||
b6e33d73 JH |
349 | /* Order partitions by order of symbols because they are linked into binary |
350 | that way. */ | |
351 | ltrans_partitions.qsort (cmp_partitions_order); | |
c3c445e1 JH |
352 | } |
353 | ||
354 | /* Maximal partitioning. Put every new symbol into new partition if possible. */ | |
a66dc285 | 355 | |
c3c445e1 JH |
356 | void |
357 | lto_max_map (void) | |
358 | { | |
5e20cdc9 | 359 | symtab_node *node; |
c3c445e1 JH |
360 | ltrans_partition partition; |
361 | int npartitions = 0; | |
362 | ||
363 | FOR_EACH_SYMBOL (node) | |
364 | { | |
d52f5295 | 365 | if (node->get_partitioning_class () != SYMBOL_PARTITION |
c3c445e1 JH |
366 | || symbol_partitioned_p (node)) |
367 | continue; | |
fec39fa6 | 368 | partition = new_partition (node->asm_name ()); |
67348ccc | 369 | add_symbol_to_partition (partition, node); |
c3c445e1 JH |
370 | npartitions++; |
371 | } | |
372 | if (!npartitions) | |
373 | new_partition ("empty"); | |
a66dc285 JH |
374 | } |
375 | ||
7861b648 AK |
376 | /* Helper function for qsort; sort nodes by order. noreorder functions must have |
377 | been removed earlier. */ | |
a66dc285 JH |
378 | static int |
379 | node_cmp (const void *pa, const void *pb) | |
380 | { | |
381 | const struct cgraph_node *a = *(const struct cgraph_node * const *) pa; | |
382 | const struct cgraph_node *b = *(const struct cgraph_node * const *) pb; | |
9cec31f4 ML |
383 | |
384 | /* Profile reorder flag enables function reordering based on first execution | |
385 | of a function. All functions with profile are placed in ascending | |
386 | order at the beginning. */ | |
387 | ||
388 | if (flag_profile_reorder_functions) | |
389 | { | |
390 | /* Functions with time profile are sorted in ascending order. */ | |
391 | if (a->tp_first_run && b->tp_first_run) | |
392 | return a->tp_first_run != b->tp_first_run | |
393 | ? a->tp_first_run - b->tp_first_run | |
394 | : a->order - b->order; | |
395 | ||
396 | /* Functions with time profile are sorted before the functions | |
397 | that do not have the profile. */ | |
398 | if (a->tp_first_run || b->tp_first_run) | |
399 | return b->tp_first_run - a->tp_first_run; | |
400 | } | |
401 | ||
67348ccc | 402 | return b->order - a->order; |
a66dc285 JH |
403 | } |
404 | ||
405 | /* Helper function for qsort; sort nodes by order. */ | |
406 | static int | |
407 | varpool_node_cmp (const void *pa, const void *pb) | |
408 | { | |
7861b648 AK |
409 | const symtab_node *a = *static_cast<const symtab_node * const *> (pa); |
410 | const symtab_node *b = *static_cast<const symtab_node * const *> (pb); | |
67348ccc | 411 | return b->order - a->order; |
a66dc285 JH |
412 | } |
413 | ||
7861b648 AK |
414 | /* Add all symtab nodes from NEXT_NODE to PARTITION in order. */ |
415 | ||
416 | static void | |
417 | add_sorted_nodes (vec<symtab_node *> &next_nodes, ltrans_partition partition) | |
418 | { | |
419 | unsigned i; | |
420 | symtab_node *node; | |
421 | ||
422 | next_nodes.qsort (varpool_node_cmp); | |
423 | FOR_EACH_VEC_ELT (next_nodes, i, node) | |
424 | if (!symbol_partitioned_p (node)) | |
425 | add_symbol_to_partition (partition, node); | |
426 | } | |
427 | ||
ddb0b824 JH |
428 | /* Return true if we should account reference from N1 to N2 in cost |
429 | of partition boundary. */ | |
430 | ||
431 | bool | |
432 | account_reference_p (symtab_node *n1, symtab_node *n2) | |
433 | { | |
434 | if (cgraph_node *cnode = dyn_cast <cgraph_node *> (n1)) | |
435 | n1 = cnode; | |
ab168044 JH |
436 | /* Do not account references from aliases - they are never split across |
437 | partitions. */ | |
438 | if (n1->alias) | |
439 | return false; | |
ddb0b824 | 440 | /* Do not account recursion - the code below will handle it incorrectly |
ab168044 JH |
441 | otherwise. Do not account references to external symbols: they will |
442 | never become local. Finally do not account references to duplicated | |
443 | symbols: they will be always local. */ | |
ddb0b824 | 444 | if (n1 == n2 |
ab168044 JH |
445 | || !n2->definition |
446 | || n2->get_partitioning_class () != SYMBOL_PARTITION) | |
447 | return false; | |
448 | /* If referring node is external symbol do not account it to boundary | |
449 | cost. Those are added into units only to enable possible constant | |
450 | folding and devirtulization. | |
451 | ||
452 | Here we do not know if it will ever be added to some partition | |
453 | (this is decided by compute_ltrans_boundary) and second it is not | |
454 | that likely that constant folding will actually use the reference. */ | |
455 | if (contained_in_symbol (n1) | |
456 | ->get_partitioning_class () == SYMBOL_EXTERNAL) | |
ddb0b824 JH |
457 | return false; |
458 | return true; | |
459 | } | |
460 | ||
7861b648 | 461 | |
a66dc285 JH |
462 | /* Group cgraph nodes into equally-sized partitions. |
463 | ||
464 | The partitioning algorithm is simple: nodes are taken in predefined order. | |
465 | The order corresponds to the order we want functions to have in the final | |
466 | output. In the future this will be given by function reordering pass, but | |
467 | at the moment we use the topological order, which is a good approximation. | |
468 | ||
469 | The goal is to partition this linear order into intervals (partitions) so | |
470 | that all the partitions have approximately the same size and the number of | |
471 | callgraph or IPA reference edges crossing boundaries is minimal. | |
472 | ||
473 | This is a lot faster (O(n) in size of callgraph) than algorithms doing | |
474 | priority-based graph clustering that are generally O(n^2) and, since | |
475 | WHOPR is designed to make things go well across partitions, it leads | |
476 | to good results. | |
477 | ||
478 | We compute the expected size of a partition as: | |
479 | ||
480 | max (total_size / lto_partitions, min_partition_size) | |
481 | ||
482 | We use dynamic expected size of partition so small programs are partitioned | |
483 | into enough partitions to allow use of multiple CPUs, while large programs | |
484 | are not partitioned too much. Creating too many partitions significantly | |
485 | increases the streaming overhead. | |
486 | ||
487 | In the future, we would like to bound the maximal size of partitions so as | |
488 | to prevent the LTRANS stage from consuming too much memory. At the moment, | |
489 | however, the WPA stage is the most memory intensive for large benchmarks, | |
490 | since too many types and declarations are read into memory. | |
491 | ||
492 | The function implements a simple greedy algorithm. Nodes are being added | |
493 | to the current partition until after 3/4 of the expected partition size is | |
494 | reached. Past this threshold, we keep track of boundary size (number of | |
495 | edges going to other partitions) and continue adding functions until after | |
496 | the current partition has grown to twice the expected partition size. Then | |
497 | the process is undone to the point where the minimal ratio of boundary size | |
498 | and in-partition calls was reached. */ | |
499 | ||
500 | void | |
a2e76867 | 501 | lto_balanced_map (int n_lto_partitions, int max_partition_size) |
a66dc285 | 502 | { |
c525ba9a | 503 | int n_varpool_nodes = 0, varpool_pos = 0, best_varpool_pos = 0; |
74644756 | 504 | auto_vec <cgraph_node *> order (symtab->cgraph_count); |
7861b648 AK |
505 | auto_vec<cgraph_node *> noreorder; |
506 | auto_vec<varpool_node *> varpool_order; | |
a66dc285 | 507 | struct cgraph_node *node; |
ddb0b824 JH |
508 | int64_t original_total_size, total_size = 0; |
509 | int64_t partition_size; | |
a66dc285 | 510 | ltrans_partition partition; |
7b99cca4 | 511 | int last_visited_node = 0; |
2c8326a5 | 512 | varpool_node *vnode; |
ddb0b824 | 513 | int64_t cost = 0, internal = 0; |
74644756 | 514 | unsigned int best_n_nodes = 0, best_i = 0; |
ddb0b824 | 515 | int64_t best_cost = -1, best_internal = 0, best_size = 0; |
a66dc285 JH |
516 | int npartitions; |
517 | int current_order = -1; | |
7861b648 | 518 | int noreorder_pos = 0; |
a66dc285 | 519 | |
65c70e6b | 520 | FOR_EACH_VARIABLE (vnode) |
67348ccc | 521 | gcc_assert (!vnode->aux); |
74644756 | 522 | |
6a49f3c9 | 523 | FOR_EACH_DEFINED_FUNCTION (node) |
d52f5295 | 524 | if (node->get_partitioning_class () == SYMBOL_PARTITION) |
6a49f3c9 | 525 | { |
7861b648 AK |
526 | if (node->no_reorder) |
527 | noreorder.safe_push (node); | |
528 | else | |
74644756 | 529 | order.safe_push (node); |
39434bce | 530 | if (!node->alias) |
56f62793 | 531 | total_size += ipa_fn_summaries->get (node)->size; |
6a49f3c9 | 532 | } |
a66dc285 | 533 | |
faae53f8 ML |
534 | original_total_size = total_size; |
535 | ||
6a49f3c9 JH |
536 | /* Streaming works best when the source units do not cross partition |
537 | boundaries much. This is because importing function from a source | |
538 | unit tends to import a lot of global trees defined there. We should | |
539 | get better about minimizing the function bounday, but until that | |
540 | things works smoother if we order in source order. */ | |
74644756 | 541 | order.qsort (node_cmp); |
7861b648 | 542 | noreorder.qsort (node_cmp); |
9cec31f4 | 543 | |
29a50dfb | 544 | if (dump_file) |
a66dc285 | 545 | { |
74644756 | 546 | for (unsigned i = 0; i < order.length (); i++) |
29a50dfb | 547 | fprintf (dump_file, "Balanced map symbol order:%s:%u\n", |
7861b648 | 548 | order[i]->name (), order[i]->tp_first_run); |
74644756 | 549 | for (unsigned i = 0; i < noreorder.length (); i++) |
29a50dfb | 550 | fprintf (dump_file, "Balanced map symbol no_reorder:%s:%u\n", |
7861b648 | 551 | noreorder[i]->name (), noreorder[i]->tp_first_run); |
a66dc285 JH |
552 | } |
553 | ||
7861b648 AK |
554 | /* Collect all variables that should not be reordered. */ |
555 | FOR_EACH_VARIABLE (vnode) | |
556 | if (vnode->get_partitioning_class () == SYMBOL_PARTITION | |
0eaf0bfe | 557 | && vnode->no_reorder) |
7861b648 AK |
558 | varpool_order.safe_push (vnode); |
559 | n_varpool_nodes = varpool_order.length (); | |
560 | varpool_order.qsort (varpool_node_cmp); | |
561 | ||
a66dc285 | 562 | /* Compute partition size and create the first partition. */ |
a2e76867 | 563 | if (PARAM_VALUE (MIN_PARTITION_SIZE) > max_partition_size) |
ddb0b824 JH |
564 | fatal_error (input_location, "min partition size cannot be greater " |
565 | "than max partition size"); | |
a2e76867 | 566 | |
783dab6b | 567 | partition_size = total_size / n_lto_partitions; |
a66dc285 JH |
568 | if (partition_size < PARAM_VALUE (MIN_PARTITION_SIZE)) |
569 | partition_size = PARAM_VALUE (MIN_PARTITION_SIZE); | |
570 | npartitions = 1; | |
571 | partition = new_partition (""); | |
29a50dfb JH |
572 | if (dump_file) |
573 | fprintf (dump_file, "Total unit size: %" PRId64 ", partition size: %" PRId64 "\n", | |
a66dc285 JH |
574 | total_size, partition_size); |
575 | ||
7861b648 AK |
576 | auto_vec<symtab_node *> next_nodes; |
577 | ||
74644756 | 578 | for (unsigned i = 0; i < order.length (); i++) |
a66dc285 | 579 | { |
67348ccc | 580 | if (symbol_partitioned_p (order[i])) |
a66dc285 JH |
581 | continue; |
582 | ||
67348ccc | 583 | current_order = order[i]->order; |
a66dc285 | 584 | |
7861b648 AK |
585 | /* Output noreorder and varpool in program order first. */ |
586 | next_nodes.truncate (0); | |
587 | while (varpool_pos < n_varpool_nodes | |
588 | && varpool_order[varpool_pos]->order < current_order) | |
589 | next_nodes.safe_push (varpool_order[varpool_pos++]); | |
590 | while (noreorder_pos < (int)noreorder.length () | |
591 | && noreorder[noreorder_pos]->order < current_order) | |
ddb0b824 | 592 | next_nodes.safe_push (noreorder[noreorder_pos++]); |
7861b648 | 593 | add_sorted_nodes (next_nodes, partition); |
a66dc285 | 594 | |
0eaf0bfe JH |
595 | if (!symbol_partitioned_p (order[i])) |
596 | add_symbol_to_partition (partition, order[i]); | |
a66dc285 JH |
597 | |
598 | ||
599 | /* Once we added a new node to the partition, we also want to add | |
600 | all referenced variables unless they was already added into some | |
601 | earlier partition. | |
c3c445e1 | 602 | add_symbol_to_partition adds possibly multiple nodes and |
a66dc285 JH |
603 | variables that are needed to satisfy needs of ORDER[i]. |
604 | We remember last visited cgraph and varpool node from last iteration | |
605 | of outer loop that allows us to process every new addition. | |
606 | ||
607 | At the same time we compute size of the boundary into COST. Every | |
608 | callgraph or IPA reference edge leaving the partition contributes into | |
609 | COST. Every edge inside partition was earlier computed as one leaving | |
610 | it and thus we need to subtract it from COST. */ | |
7b99cca4 | 611 | while (last_visited_node < lto_symtab_encoder_size (partition->encoder)) |
a66dc285 | 612 | { |
a66dc285 | 613 | int j; |
d122681a | 614 | struct ipa_ref *ref = NULL; |
5e20cdc9 | 615 | symtab_node *snode = lto_symtab_encoder_deref (partition->encoder, |
7b99cca4 | 616 | last_visited_node); |
a66dc285 | 617 | |
7de90a6c | 618 | if (cgraph_node *node = dyn_cast <cgraph_node *> (snode)) |
a66dc285 JH |
619 | { |
620 | struct cgraph_edge *edge; | |
621 | ||
a66dc285 | 622 | |
7b99cca4 | 623 | last_visited_node++; |
a66dc285 | 624 | |
67348ccc | 625 | gcc_assert (node->definition || node->weakref); |
a66dc285 JH |
626 | |
627 | /* Compute boundary cost of callgraph edges. */ | |
628 | for (edge = node->callees; edge; edge = edge->next_callee) | |
ddb0b824 JH |
629 | /* Inline edges will always end up local. */ |
630 | if (edge->inline_failed | |
631 | && account_reference_p (node, edge->callee)) | |
a66dc285 | 632 | { |
1bad9c18 | 633 | int edge_cost = edge->frequency (); |
7b99cca4 | 634 | int index; |
a66dc285 JH |
635 | |
636 | if (!edge_cost) | |
637 | edge_cost = 1; | |
638 | gcc_assert (edge_cost > 0); | |
7b99cca4 | 639 | index = lto_symtab_encoder_lookup (partition->encoder, |
67348ccc | 640 | edge->callee); |
7b99cca4 JH |
641 | if (index != LCC_NOT_FOUND |
642 | && index < last_visited_node - 1) | |
643 | cost -= edge_cost, internal += edge_cost; | |
a66dc285 JH |
644 | else |
645 | cost += edge_cost; | |
646 | } | |
647 | for (edge = node->callers; edge; edge = edge->next_caller) | |
ddb0b824 JH |
648 | if (edge->inline_failed |
649 | && account_reference_p (edge->caller, node)) | |
a66dc285 | 650 | { |
1bad9c18 | 651 | int edge_cost = edge->frequency (); |
7b99cca4 | 652 | int index; |
a66dc285 | 653 | |
67348ccc | 654 | gcc_assert (edge->caller->definition); |
a66dc285 JH |
655 | if (!edge_cost) |
656 | edge_cost = 1; | |
657 | gcc_assert (edge_cost > 0); | |
7b99cca4 | 658 | index = lto_symtab_encoder_lookup (partition->encoder, |
67348ccc | 659 | edge->caller); |
7b99cca4 JH |
660 | if (index != LCC_NOT_FOUND |
661 | && index < last_visited_node - 1) | |
ddb0b824 | 662 | cost -= edge_cost, internal += edge_cost; |
a66dc285 JH |
663 | else |
664 | cost += edge_cost; | |
665 | } | |
666 | } | |
667 | else | |
ddb0b824 | 668 | last_visited_node++; |
a66dc285 JH |
669 | |
670 | /* Compute boundary cost of IPA REF edges and at the same time look into | |
671 | variables referenced from current partition and try to add them. */ | |
ddb0b824 JH |
672 | for (j = 0; snode->iterate_reference (j, ref); j++) |
673 | if (!account_reference_p (snode, ref->referred)) | |
674 | ; | |
675 | else if (is_a <varpool_node *> (ref->referred)) | |
a66dc285 | 676 | { |
7b99cca4 | 677 | int index; |
a66dc285 | 678 | |
d122681a | 679 | vnode = dyn_cast <varpool_node *> (ref->referred); |
0eaf0bfe | 680 | if (!symbol_partitioned_p (vnode) |
7861b648 | 681 | && !vnode->no_reorder |
d52f5295 | 682 | && vnode->get_partitioning_class () == SYMBOL_PARTITION) |
67348ccc | 683 | add_symbol_to_partition (partition, vnode); |
7b99cca4 | 684 | index = lto_symtab_encoder_lookup (partition->encoder, |
67348ccc | 685 | vnode); |
7b99cca4 JH |
686 | if (index != LCC_NOT_FOUND |
687 | && index < last_visited_node - 1) | |
a66dc285 JH |
688 | cost--, internal++; |
689 | else | |
690 | cost++; | |
691 | } | |
692 | else | |
693 | { | |
7b99cca4 | 694 | int index; |
a66dc285 | 695 | |
d122681a | 696 | node = dyn_cast <cgraph_node *> (ref->referred); |
7b99cca4 | 697 | index = lto_symtab_encoder_lookup (partition->encoder, |
67348ccc | 698 | node); |
7b99cca4 JH |
699 | if (index != LCC_NOT_FOUND |
700 | && index < last_visited_node - 1) | |
a66dc285 JH |
701 | cost--, internal++; |
702 | else | |
703 | cost++; | |
704 | } | |
ddb0b824 JH |
705 | for (j = 0; snode->iterate_referring (j, ref); j++) |
706 | if (!account_reference_p (ref->referring, snode)) | |
707 | ; | |
708 | else if (is_a <varpool_node *> (ref->referring)) | |
a66dc285 | 709 | { |
7b99cca4 | 710 | int index; |
a66dc285 | 711 | |
d122681a | 712 | vnode = dyn_cast <varpool_node *> (ref->referring); |
67348ccc | 713 | gcc_assert (vnode->definition); |
24d3f325 NS |
714 | /* It is better to couple variables with their users, |
715 | because it allows them to be removed. Coupling | |
716 | with objects they refer to only helps to reduce | |
39434bce | 717 | number of symbols promoted to hidden. */ |
0eaf0bfe | 718 | if (!symbol_partitioned_p (vnode) |
7861b648 | 719 | && !vnode->no_reorder |
9041d2e6 | 720 | && !vnode->can_remove_if_no_refs_p () |
d52f5295 | 721 | && vnode->get_partitioning_class () == SYMBOL_PARTITION) |
67348ccc | 722 | add_symbol_to_partition (partition, vnode); |
7b99cca4 | 723 | index = lto_symtab_encoder_lookup (partition->encoder, |
67348ccc | 724 | vnode); |
7b99cca4 JH |
725 | if (index != LCC_NOT_FOUND |
726 | && index < last_visited_node - 1) | |
ddb0b824 | 727 | cost--, internal++; |
a66dc285 JH |
728 | else |
729 | cost++; | |
730 | } | |
731 | else | |
732 | { | |
7b99cca4 | 733 | int index; |
a66dc285 | 734 | |
d122681a | 735 | node = dyn_cast <cgraph_node *> (ref->referring); |
67348ccc | 736 | gcc_assert (node->definition); |
7b99cca4 | 737 | index = lto_symtab_encoder_lookup (partition->encoder, |
67348ccc | 738 | node); |
7b99cca4 JH |
739 | if (index != LCC_NOT_FOUND |
740 | && index < last_visited_node - 1) | |
ddb0b824 | 741 | cost--, internal++; |
a66dc285 JH |
742 | else |
743 | cost++; | |
744 | } | |
745 | } | |
746 | ||
ddb0b824 JH |
747 | gcc_assert (cost >= 0 && internal >= 0); |
748 | ||
749 | /* If the partition is large enough, start looking for smallest boundary cost. | |
750 | If partition still seems too small (less than 7/8 of target weight) accept | |
751 | any cost. If partition has right size, optimize for highest internal/cost. | |
752 | Later we stop building partition if its size is 9/8 of the target wight. */ | |
753 | if (partition->insns < partition_size * 7 / 8 | |
754 | || best_cost == -1 | |
755 | || (!cost | |
756 | || ((sreal)best_internal * (sreal) cost | |
757 | < ((sreal) internal * (sreal)best_cost)))) | |
a66dc285 JH |
758 | { |
759 | best_cost = cost; | |
760 | best_internal = internal; | |
ddb0b824 | 761 | best_size = partition->insns; |
a66dc285 | 762 | best_i = i; |
7b99cca4 | 763 | best_n_nodes = lto_symtab_encoder_size (partition->encoder); |
c525ba9a | 764 | best_varpool_pos = varpool_pos; |
a66dc285 | 765 | } |
29a50dfb JH |
766 | if (dump_file) |
767 | fprintf (dump_file, "Step %i: added %s/%i, size %i, " | |
ddb0b824 JH |
768 | "cost %" PRId64 "/%" PRId64 " " |
769 | "best %" PRId64 "/%" PRId64", step %i\n", i, | |
fec39fa6 | 770 | order[i]->name (), order[i]->order, |
9de04252 | 771 | partition->insns, cost, internal, |
a66dc285 JH |
772 | best_cost, best_internal, best_i); |
773 | /* Partition is too large, unwind into step when best cost was reached and | |
774 | start new partition. */ | |
ddb0b824 | 775 | if (partition->insns > 9 * partition_size / 8 |
a2e76867 | 776 | || partition->insns > max_partition_size) |
a66dc285 JH |
777 | { |
778 | if (best_i != i) | |
779 | { | |
29a50dfb JH |
780 | if (dump_file) |
781 | fprintf (dump_file, "Unwinding %i insertions to step %i\n", | |
a66dc285 | 782 | i - best_i, best_i); |
7b99cca4 | 783 | undo_partition (partition, best_n_nodes); |
c525ba9a | 784 | varpool_pos = best_varpool_pos; |
a66dc285 | 785 | } |
ddb0b824 | 786 | gcc_assert (best_size == partition->insns); |
a66dc285 | 787 | i = best_i; |
29a50dfb JH |
788 | if (dump_file) |
789 | fprintf (dump_file, | |
ddb0b824 JH |
790 | "Partition insns: %i (want %" PRId64 ")\n", |
791 | partition->insns, partition_size); | |
a66dc285 | 792 | /* When we are finished, avoid creating empty partition. */ |
74644756 | 793 | while (i < order.length () - 1 && symbol_partitioned_p (order[i + 1])) |
a66dc285 | 794 | i++; |
74644756 | 795 | if (i == order.length () - 1) |
a66dc285 | 796 | break; |
ddb0b824 | 797 | total_size -= partition->insns; |
a66dc285 | 798 | partition = new_partition (""); |
7b99cca4 | 799 | last_visited_node = 0; |
a66dc285 JH |
800 | cost = 0; |
801 | ||
29a50dfb JH |
802 | if (dump_file) |
803 | fprintf (dump_file, "New partition\n"); | |
a66dc285 | 804 | best_n_nodes = 0; |
ddb0b824 | 805 | best_cost = -1; |
a66dc285 JH |
806 | |
807 | /* Since the size of partitions is just approximate, update the size after | |
808 | we finished current one. */ | |
783dab6b RB |
809 | if (npartitions < n_lto_partitions) |
810 | partition_size = total_size / (n_lto_partitions - npartitions); | |
a66dc285 | 811 | else |
97bb4812 JH |
812 | /* Watch for overflow. */ |
813 | partition_size = INT_MAX / 16; | |
a66dc285 | 814 | |
29a50dfb JH |
815 | if (dump_file) |
816 | fprintf (dump_file, | |
ddb0b824 JH |
817 | "Total size: %" PRId64 " partition_size: %" PRId64 "\n", |
818 | total_size, partition_size); | |
a66dc285 JH |
819 | if (partition_size < PARAM_VALUE (MIN_PARTITION_SIZE)) |
820 | partition_size = PARAM_VALUE (MIN_PARTITION_SIZE); | |
821 | npartitions ++; | |
822 | } | |
823 | } | |
824 | ||
7861b648 AK |
825 | next_nodes.truncate (0); |
826 | ||
a66dc285 | 827 | /* Varables that are not reachable from the code go into last partition. */ |
0eaf0bfe JH |
828 | FOR_EACH_VARIABLE (vnode) |
829 | if (vnode->get_partitioning_class () == SYMBOL_PARTITION | |
830 | && !symbol_partitioned_p (vnode)) | |
831 | next_nodes.safe_push (vnode); | |
7861b648 AK |
832 | |
833 | /* Output remaining ordered symbols. */ | |
834 | while (varpool_pos < n_varpool_nodes) | |
835 | next_nodes.safe_push (varpool_order[varpool_pos++]); | |
836 | while (noreorder_pos < (int)noreorder.length ()) | |
837 | next_nodes.safe_push (noreorder[noreorder_pos++]); | |
ddb0b824 JH |
838 | /* For one partition the cost of boundary should be 0 unless we added final |
839 | symbols here (these are not accounted) or we have accounting bug. */ | |
e693ebce | 840 | gcc_assert (next_nodes.length () || npartitions != 1 || !best_cost || best_cost == -1); |
7861b648 AK |
841 | add_sorted_nodes (next_nodes, partition); |
842 | ||
29a50dfb | 843 | if (dump_file) |
faae53f8 | 844 | { |
29a50dfb | 845 | fprintf (dump_file, "\nPartition sizes:\n"); |
faae53f8 ML |
846 | unsigned partitions = ltrans_partitions.length (); |
847 | ||
848 | for (unsigned i = 0; i < partitions ; i++) | |
849 | { | |
850 | ltrans_partition p = ltrans_partitions[i]; | |
29a50dfb | 851 | fprintf (dump_file, "partition %d contains %d (%2.2f%%)" |
faae53f8 | 852 | " symbols and %d (%2.2f%%) insns\n", i, p->symbols, |
74644756 | 853 | 100.0 * p->symbols / order.length (), p->insns, |
faae53f8 ML |
854 | 100.0 * p->insns / original_total_size); |
855 | } | |
856 | ||
29a50dfb | 857 | fprintf (dump_file, "\n"); |
faae53f8 | 858 | } |
a66dc285 JH |
859 | } |
860 | ||
9816367c BS |
861 | /* Return true if we must not change the name of the NODE. The name as |
862 | extracted from the corresponding decl should be passed in NAME. */ | |
64cfa6c0 | 863 | |
8ee05051 | 864 | static bool |
9816367c | 865 | must_not_rename (symtab_node *node, const char *name) |
64cfa6c0 | 866 | { |
64cfa6c0 JH |
867 | /* Our renaming machinery do not handle more than one change of assembler name. |
868 | We should not need more than one anyway. */ | |
67348ccc DM |
869 | if (node->lto_file_data |
870 | && lto_get_decl_name_mapping (node->lto_file_data, name) != name) | |
64cfa6c0 | 871 | { |
29a50dfb JH |
872 | if (dump_file) |
873 | fprintf (dump_file, | |
9816367c BS |
874 | "Not privatizing symbol name: %s. It privatized already.\n", |
875 | name); | |
876 | return true; | |
64cfa6c0 JH |
877 | } |
878 | /* Avoid mangling of already mangled clones. | |
879 | ??? should have a flag whether a symbol has a 'private' name already, | |
880 | since we produce some symbols like that i.e. for global constructors | |
881 | that are not really clones. */ | |
67348ccc | 882 | if (node->unique_name) |
64cfa6c0 | 883 | { |
29a50dfb JH |
884 | if (dump_file) |
885 | fprintf (dump_file, | |
9816367c BS |
886 | "Not privatizing symbol name: %s. Has unique name.\n", |
887 | name); | |
888 | return true; | |
64cfa6c0 | 889 | } |
9816367c BS |
890 | return false; |
891 | } | |
892 | ||
893 | /* If we are an offload compiler, we may have to rewrite symbols to be | |
894 | valid on this target. Return either PTR or a modified version of it. */ | |
895 | ||
896 | static const char * | |
897 | maybe_rewrite_identifier (const char *ptr) | |
898 | { | |
899 | #if defined ACCEL_COMPILER && (defined NO_DOT_IN_LABEL || defined NO_DOLLAR_IN_LABEL) | |
900 | #ifndef NO_DOT_IN_LABEL | |
901 | char valid = '.'; | |
902 | const char reject[] = "$"; | |
903 | #elif !defined NO_DOLLAR_IN_LABEL | |
904 | char valid = '$'; | |
905 | const char reject[] = "."; | |
906 | #else | |
907 | char valid = '_'; | |
908 | const char reject[] = ".$"; | |
909 | #endif | |
910 | ||
911 | char *copy = NULL; | |
912 | const char *match = ptr; | |
913 | for (;;) | |
914 | { | |
915 | size_t off = strcspn (match, reject); | |
916 | if (match[off] == '\0') | |
917 | break; | |
918 | if (copy == NULL) | |
919 | { | |
920 | copy = xstrdup (ptr); | |
921 | match = copy; | |
922 | } | |
923 | copy[off] = valid; | |
924 | } | |
925 | return match; | |
926 | #else | |
927 | return ptr; | |
928 | #endif | |
929 | } | |
930 | ||
931 | /* Ensure that the symbol in NODE is valid for the target, and if not, | |
932 | rewrite it. */ | |
933 | ||
934 | static void | |
935 | validize_symbol_for_target (symtab_node *node) | |
936 | { | |
937 | tree decl = node->decl; | |
938 | const char *name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)); | |
939 | ||
940 | if (must_not_rename (node, name)) | |
941 | return; | |
942 | ||
943 | const char *name2 = maybe_rewrite_identifier (name); | |
944 | if (name2 != name) | |
945 | { | |
946 | symtab->change_decl_assembler_name (decl, get_identifier (name2)); | |
947 | if (node->lto_file_data) | |
948 | lto_record_renamed_decl (node->lto_file_data, name, | |
949 | IDENTIFIER_POINTER | |
950 | (DECL_ASSEMBLER_NAME (decl))); | |
951 | } | |
952 | } | |
953 | ||
ed1b53a0 MP |
954 | /* Maps symbol names to unique lto clone counters. */ |
955 | static hash_map<const char *, unsigned> *lto_clone_numbers; | |
956 | ||
48de5d37 IE |
957 | /* Helper for privatize_symbol_name. Mangle NODE symbol name |
958 | represented by DECL. */ | |
9816367c BS |
959 | |
960 | static bool | |
48de5d37 | 961 | privatize_symbol_name_1 (symtab_node *node, tree decl) |
9816367c | 962 | { |
48de5d37 | 963 | const char *name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)); |
9816367c BS |
964 | |
965 | if (must_not_rename (node, name)) | |
966 | return false; | |
967 | ||
968 | name = maybe_rewrite_identifier (name); | |
ed1b53a0 | 969 | unsigned &clone_number = lto_clone_numbers->get_or_insert (name); |
3dafb85c | 970 | symtab->change_decl_assembler_name (decl, |
ed1b53a0 MP |
971 | clone_function_name ( |
972 | name, "lto_priv", clone_number)); | |
973 | clone_number++; | |
48de5d37 | 974 | |
67348ccc DM |
975 | if (node->lto_file_data) |
976 | lto_record_renamed_decl (node->lto_file_data, name, | |
64cfa6c0 JH |
977 | IDENTIFIER_POINTER |
978 | (DECL_ASSEMBLER_NAME (decl))); | |
48de5d37 | 979 | |
29a50dfb JH |
980 | if (dump_file) |
981 | fprintf (dump_file, | |
48de5d37 IE |
982 | "Privatizing symbol name: %s -> %s\n", |
983 | name, IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl))); | |
984 | ||
985 | return true; | |
986 | } | |
987 | ||
988 | /* Mangle NODE symbol name into a local name. | |
989 | This is necessary to do | |
990 | 1) if two or more static vars of same assembler name | |
991 | are merged into single ltrans unit. | |
992 | 2) if previously static var was promoted hidden to avoid possible conflict | |
993 | with symbols defined out of the LTO world. */ | |
994 | ||
995 | static bool | |
996 | privatize_symbol_name (symtab_node *node) | |
997 | { | |
998 | if (!privatize_symbol_name_1 (node, node->decl)) | |
999 | return false; | |
1000 | ||
8ee05051 | 1001 | return true; |
64cfa6c0 JH |
1002 | } |
1003 | ||
a66dc285 JH |
1004 | /* Promote variable VNODE to be static. */ |
1005 | ||
b4661bfe | 1006 | static void |
5e20cdc9 | 1007 | promote_symbol (symtab_node *node) |
a66dc285 | 1008 | { |
b4661bfe | 1009 | /* We already promoted ... */ |
67348ccc DM |
1010 | if (DECL_VISIBILITY (node->decl) == VISIBILITY_HIDDEN |
1011 | && DECL_VISIBILITY_SPECIFIED (node->decl) | |
1012 | && TREE_PUBLIC (node->decl)) | |
9816367c BS |
1013 | { |
1014 | validize_symbol_for_target (node); | |
1015 | return; | |
1016 | } | |
a66dc285 | 1017 | |
67348ccc DM |
1018 | gcc_checking_assert (!TREE_PUBLIC (node->decl) |
1019 | && !DECL_EXTERNAL (node->decl)); | |
64cfa6c0 JH |
1020 | /* Be sure that newly public symbol does not conflict with anything already |
1021 | defined by the non-LTO part. */ | |
1022 | privatize_symbol_name (node); | |
67348ccc DM |
1023 | TREE_PUBLIC (node->decl) = 1; |
1024 | DECL_VISIBILITY (node->decl) = VISIBILITY_HIDDEN; | |
1025 | DECL_VISIBILITY_SPECIFIED (node->decl) = true; | |
29a50dfb JH |
1026 | if (dump_file) |
1027 | fprintf (dump_file, | |
5406184c NS |
1028 | "Promoting as hidden: %s (%s)\n", node->name (), |
1029 | IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl))); | |
41c60c6c | 1030 | |
5406184c | 1031 | /* Promoting a symbol also promotes all transparent aliases with exception |
41c60c6c JH |
1032 | of weakref where the visibility flags are always wrong and set to |
1033 | !PUBLIC. */ | |
5406184c | 1034 | ipa_ref *ref; |
41c60c6c JH |
1035 | for (unsigned i = 0; node->iterate_direct_aliases (i, ref); i++) |
1036 | { | |
1037 | struct symtab_node *alias = ref->referring; | |
1038 | if (alias->transparent_alias && !alias->weakref) | |
1039 | { | |
1040 | TREE_PUBLIC (alias->decl) = 1; | |
1041 | DECL_VISIBILITY (alias->decl) = VISIBILITY_HIDDEN; | |
1042 | DECL_VISIBILITY_SPECIFIED (alias->decl) = true; | |
29a50dfb JH |
1043 | if (dump_file) |
1044 | fprintf (dump_file, | |
5406184c NS |
1045 | "Promoting alias as hidden: %s\n", |
1046 | IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl))); | |
41c60c6c JH |
1047 | } |
1048 | gcc_assert (!alias->weakref || TREE_PUBLIC (alias->decl)); | |
1049 | } | |
a66dc285 JH |
1050 | } |
1051 | ||
24d3f325 NS |
1052 | /* Return true if NODE needs named section even if it won't land in |
1053 | the partition symbol table. | |
1054 | ||
1055 | FIXME: we should really not use named sections for inline clones | |
1056 | and master clones. */ | |
64cfa6c0 JH |
1057 | |
1058 | static bool | |
5e20cdc9 | 1059 | may_need_named_section_p (lto_symtab_encoder_t encoder, symtab_node *node) |
64cfa6c0 | 1060 | { |
7de90a6c | 1061 | struct cgraph_node *cnode = dyn_cast <cgraph_node *> (node); |
64cfa6c0 JH |
1062 | if (!cnode) |
1063 | return false; | |
d52f5295 | 1064 | if (node->real_symbol_p ()) |
64cfa6c0 | 1065 | return false; |
64cfa6c0 JH |
1066 | return (!encoder |
1067 | || (lto_symtab_encoder_lookup (encoder, node) != LCC_NOT_FOUND | |
1068 | && lto_symtab_encoder_encode_body_p (encoder, | |
1069 | cnode))); | |
1070 | } | |
1071 | ||
1072 | /* If NODE represents a static variable. See if there are other variables | |
1073 | of the same name in partition ENCODER (or in whole compilation unit if | |
1074 | ENCODER is NULL) and if so, mangle the statics. Always mangle all | |
1075 | conflicting statics, so we reduce changes of silently miscompiling | |
9cec31f4 | 1076 | asm statements referring to them by symbol name. */ |
64cfa6c0 JH |
1077 | |
1078 | static void | |
5e20cdc9 | 1079 | rename_statics (lto_symtab_encoder_t encoder, symtab_node *node) |
64cfa6c0 | 1080 | { |
67348ccc | 1081 | tree decl = node->decl; |
5e20cdc9 | 1082 | symtab_node *s; |
64cfa6c0 JH |
1083 | tree name = DECL_ASSEMBLER_NAME (decl); |
1084 | ||
1085 | /* See if this is static symbol. */ | |
41c60c6c | 1086 | if (((node->externally_visible && !node->weakref) |
64cfa6c0 JH |
1087 | /* FIXME: externally_visible is somewhat illogically not set for |
1088 | external symbols (i.e. those not defined). Remove this test | |
1089 | once this is fixed. */ | |
67348ccc | 1090 | || DECL_EXTERNAL (node->decl) |
d52f5295 | 1091 | || !node->real_symbol_p ()) |
08346abd | 1092 | && !may_need_named_section_p (encoder, node)) |
64cfa6c0 JH |
1093 | return; |
1094 | ||
1095 | /* Now walk symbols sharing the same name and see if there are any conflicts. | |
1096 | (all types of symbols counts here, since we can not have static of the | |
1097 | same name as external or public symbol.) */ | |
3dafb85c | 1098 | for (s = symtab_node::get_for_asmname (name); |
67348ccc | 1099 | s; s = s->next_sharing_asm_name) |
d52f5295 | 1100 | if ((s->real_symbol_p () || may_need_named_section_p (encoder, s)) |
67348ccc | 1101 | && s->decl != node->decl |
64cfa6c0 JH |
1102 | && (!encoder |
1103 | || lto_symtab_encoder_lookup (encoder, s) != LCC_NOT_FOUND)) | |
1104 | break; | |
1105 | ||
1106 | /* OK, no confict, so we have nothing to do. */ | |
1107 | if (!s) | |
1108 | return; | |
1109 | ||
29a50dfb JH |
1110 | if (dump_file) |
1111 | fprintf (dump_file, | |
fec39fa6 | 1112 | "Renaming statics with asm name: %s\n", node->name ()); |
64cfa6c0 JH |
1113 | |
1114 | /* Assign every symbol in the set that shares the same ASM name an unique | |
1115 | mangled name. */ | |
3dafb85c | 1116 | for (s = symtab_node::get_for_asmname (name); s;) |
71e54687 JH |
1117 | if ((!s->externally_visible || s->weakref) |
1118 | /* Transparent aliases having same name as target are renamed at a | |
1119 | time their target gets new name. Transparent aliases that use | |
1120 | separate assembler name require the name to be unique. */ | |
1121 | && (!s->transparent_alias || !s->definition || s->weakref | |
1122 | || !symbol_table::assembler_names_equal_p | |
1123 | (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (s->decl)), | |
1124 | IDENTIFIER_POINTER | |
1125 | (DECL_ASSEMBLER_NAME (s->get_alias_target()->decl)))) | |
d52f5295 | 1126 | && ((s->real_symbol_p () |
5803bd01 JH |
1127 | && !DECL_EXTERNAL (s->decl) |
1128 | && !TREE_PUBLIC (s->decl)) | |
64cfa6c0 JH |
1129 | || may_need_named_section_p (encoder, s)) |
1130 | && (!encoder | |
1131 | || lto_symtab_encoder_lookup (encoder, s) != LCC_NOT_FOUND)) | |
1132 | { | |
8ee05051 | 1133 | if (privatize_symbol_name (s)) |
24d3f325 NS |
1134 | /* Re-start from beginning since we do not know how many |
1135 | symbols changed a name. */ | |
3dafb85c | 1136 | s = symtab_node::get_for_asmname (name); |
67348ccc | 1137 | else s = s->next_sharing_asm_name; |
64cfa6c0 | 1138 | } |
67348ccc | 1139 | else s = s->next_sharing_asm_name; |
64cfa6c0 | 1140 | } |
b4661bfe | 1141 | |
a66dc285 JH |
1142 | /* Find out all static decls that need to be promoted to global because |
1143 | of cross file sharing. This function must be run in the WPA mode after | |
1144 | all inlinees are added. */ | |
1145 | ||
1146 | void | |
1147 | lto_promote_cross_file_statics (void) | |
1148 | { | |
a66dc285 | 1149 | unsigned i, n_sets; |
a66dc285 JH |
1150 | |
1151 | gcc_assert (flag_wpa); | |
1152 | ||
837bac8c IV |
1153 | lto_stream_offload_p = false; |
1154 | select_what_to_stream (); | |
1f6be682 | 1155 | |
b4661bfe | 1156 | /* First compute boundaries. */ |
9771b263 | 1157 | n_sets = ltrans_partitions.length (); |
a66dc285 JH |
1158 | for (i = 0; i < n_sets; i++) |
1159 | { | |
1160 | ltrans_partition part | |
9771b263 | 1161 | = ltrans_partitions[i]; |
b4661bfe JH |
1162 | part->encoder = compute_ltrans_boundary (part->encoder); |
1163 | } | |
a66dc285 | 1164 | |
ed1b53a0 MP |
1165 | lto_clone_numbers = new hash_map<const char *, unsigned>; |
1166 | ||
b4661bfe JH |
1167 | /* Look at boundaries and promote symbols as needed. */ |
1168 | for (i = 0; i < n_sets; i++) | |
1169 | { | |
1170 | lto_symtab_encoder_iterator lsei; | |
64cfa6c0 | 1171 | lto_symtab_encoder_t encoder = ltrans_partitions[i]->encoder; |
a66dc285 | 1172 | |
b4661bfe JH |
1173 | for (lsei = lsei_start (encoder); !lsei_end_p (lsei); |
1174 | lsei_next (&lsei)) | |
1175 | { | |
5e20cdc9 | 1176 | symtab_node *node = lsei_node (lsei); |
b4661bfe | 1177 | |
24d3f325 NS |
1178 | /* If symbol is static, rename it if its assembler name |
1179 | clashes with anything else in this unit. */ | |
64cfa6c0 JH |
1180 | rename_statics (encoder, node); |
1181 | ||
b4661bfe | 1182 | /* No need to promote if symbol already is externally visible ... */ |
67348ccc | 1183 | if (node->externally_visible |
b4661bfe JH |
1184 | /* ... or if it is part of current partition ... */ |
1185 | || lto_symtab_encoder_in_partition_p (encoder, node) | |
1186 | /* ... or if we do not partition it. This mean that it will | |
24d3f325 | 1187 | appear in every partition referencing it. */ |
d52f5295 | 1188 | || node->get_partitioning_class () != SYMBOL_PARTITION) |
9816367c BS |
1189 | { |
1190 | validize_symbol_for_target (node); | |
1191 | continue; | |
1192 | } | |
a66dc285 | 1193 | |
b4661bfe JH |
1194 | promote_symbol (node); |
1195 | } | |
a66dc285 | 1196 | } |
ed1b53a0 | 1197 | delete lto_clone_numbers; |
a66dc285 | 1198 | } |
64cfa6c0 JH |
1199 | |
1200 | /* Rename statics in the whole unit in the case that | |
1201 | we do -flto-partition=none. */ | |
1202 | ||
1203 | void | |
1204 | lto_promote_statics_nonwpa (void) | |
1205 | { | |
5e20cdc9 | 1206 | symtab_node *node; |
ed1b53a0 MP |
1207 | |
1208 | lto_clone_numbers = new hash_map<const char *, unsigned>; | |
64cfa6c0 | 1209 | FOR_EACH_SYMBOL (node) |
9816367c BS |
1210 | { |
1211 | rename_statics (NULL, node); | |
1212 | validize_symbol_for_target (node); | |
1213 | } | |
ed1b53a0 | 1214 | delete lto_clone_numbers; |
64cfa6c0 | 1215 | } |