]>
Commit | Line | Data |
---|---|---|
564fe867 | 1 | /* Callgraph clones |
7adcbafe | 2 | Copyright (C) 2003-2022 Free Software Foundation, Inc. |
564fe867 JH |
3 | Contributed by Jan Hubicka |
4 | ||
5 | This file is part of GCC. | |
6 | ||
7 | GCC is free software; you can redistribute it and/or modify it under | |
8 | the terms of the GNU General Public License as published by the Free | |
9 | Software Foundation; either version 3, or (at your option) any later | |
10 | version. | |
11 | ||
12 | GCC is distributed in the hope that it will be useful, but WITHOUT ANY | |
13 | WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
14 | FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
15 | for more details. | |
16 | ||
17 | You should have received a copy of the GNU General Public License | |
18 | along with GCC; see the file COPYING3. If not see | |
19 | <http://www.gnu.org/licenses/>. */ | |
20 | ||
dfea3d6f | 21 | /* This module provide facilities for cloning functions. I.e. creating |
564fe867 JH |
22 | new functions based on existing functions with simple modifications, |
23 | such as replacement of parameters. | |
24 | ||
25 | To allow whole program optimization without actual presence of function | |
26 | bodies, an additional infrastructure is provided for so-called virtual | |
27 | clones | |
28 | ||
29 | A virtual clone in the callgraph is a function that has no | |
30 | associated body, just a description of how to create its body based | |
31 | on a different function (which itself may be a virtual clone). | |
32 | ||
33 | The description of function modifications includes adjustments to | |
34 | the function's signature (which allows, for example, removing or | |
35 | adding function arguments), substitutions to perform on the | |
36 | function body, and, for inlined functions, a pointer to the | |
37 | function that it will be inlined into. | |
38 | ||
39 | It is also possible to redirect any edge of the callgraph from a | |
40 | function to its virtual clone. This implies updating of the call | |
41 | site to adjust for the new function signature. | |
42 | ||
43 | Most of the transformations performed by inter-procedural | |
44 | optimizations can be represented via virtual clones. For | |
45 | instance, a constant propagation pass can produce a virtual clone | |
46 | of the function which replaces one of its arguments by a | |
47 | constant. The inliner can represent its decisions by producing a | |
48 | clone of a function whose body will be later integrated into | |
49 | a given function. | |
50 | ||
51 | Using virtual clones, the program can be easily updated | |
52 | during the Execute stage, solving most of pass interactions | |
53 | problems that would otherwise occur during Transform. | |
54 | ||
55 | Virtual clones are later materialized in the LTRANS stage and | |
56 | turned into real functions. Passes executed after the virtual | |
57 | clone were introduced also perform their Transform stage | |
58 | on new functions, so for a pass there is no significant | |
59 | difference between operating on a real function or a virtual | |
60 | clone introduced before its Execute stage. | |
61 | ||
62 | Optimization passes then work on virtual clones introduced before | |
63 | their Execute stage as if they were real functions. The | |
64 | only difference is that clones are not visible during the | |
65 | Generate Summary stage. */ | |
66 | ||
67 | #include "config.h" | |
68 | #include "system.h" | |
69 | #include "coretypes.h" | |
c7131fb2 | 70 | #include "backend.h" |
957060b5 AM |
71 | #include "target.h" |
72 | #include "rtl.h" | |
c7131fb2 AM |
73 | #include "tree.h" |
74 | #include "gimple.h" | |
40e23961 | 75 | #include "stringpool.h" |
957060b5 | 76 | #include "cgraph.h" |
957060b5 | 77 | #include "lto-streamer.h" |
2fb9a547 | 78 | #include "tree-eh.h" |
442b4905 | 79 | #include "tree-cfg.h" |
564fe867 | 80 | #include "tree-inline.h" |
c24e924f | 81 | #include "dumpfile.h" |
564fe867 | 82 | #include "gimple-pretty-print.h" |
68188fff MJ |
83 | #include "alloc-pool.h" |
84 | #include "symbol-summary.h" | |
85 | #include "tree-vrp.h" | |
86 | #include "ipa-prop.h" | |
87 | #include "ipa-fnsummary.h" | |
67f3791f | 88 | #include "symtab-thunks.h" |
ae7a23a3 | 89 | #include "symtab-clones.h" |
564fe867 | 90 | |
3dafb85c ML |
91 | /* Create clone of edge in the node N represented by CALL_EXPR |
92 | the callgraph. */ | |
93 | ||
94 | cgraph_edge * | |
538dd0b7 | 95 | cgraph_edge::clone (cgraph_node *n, gcall *call_stmt, unsigned stmt_uid, |
1511c8c0 | 96 | profile_count num, profile_count den, |
1bad9c18 | 97 | bool update_original) |
564fe867 | 98 | { |
3dafb85c | 99 | cgraph_edge *new_edge; |
1bad9c18 | 100 | profile_count::adjust_for_ipa_scaling (&num, &den); |
517048ce | 101 | profile_count prof_count = count.apply_scale (num, den); |
564fe867 | 102 | |
3dafb85c | 103 | if (indirect_unknown_callee) |
564fe867 JH |
104 | { |
105 | tree decl; | |
106 | ||
e57872ee JH |
107 | if (call_stmt && (decl = gimple_call_fndecl (call_stmt)) |
108 | /* When the call is speculative, we need to resolve it | |
109 | via cgraph_resolve_speculation and not here. */ | |
3dafb85c | 110 | && !speculative) |
564fe867 | 111 | { |
3dafb85c | 112 | cgraph_node *callee = cgraph_node::get (decl); |
564fe867 | 113 | gcc_checking_assert (callee); |
3187c8a5 | 114 | new_edge = n->create_edge (callee, call_stmt, prof_count, true); |
564fe867 JH |
115 | } |
116 | else | |
117 | { | |
d52f5295 | 118 | new_edge = n->create_indirect_edge (call_stmt, |
3dafb85c | 119 | indirect_info->ecf_flags, |
3187c8a5 | 120 | prof_count, true); |
3dafb85c | 121 | *new_edge->indirect_info = *indirect_info; |
564fe867 JH |
122 | } |
123 | } | |
124 | else | |
125 | { | |
3187c8a5 | 126 | new_edge = n->create_edge (callee, call_stmt, prof_count, true); |
3dafb85c | 127 | if (indirect_info) |
564fe867 JH |
128 | { |
129 | new_edge->indirect_info | |
766090c2 | 130 | = ggc_cleared_alloc<cgraph_indirect_call_info> (); |
3dafb85c | 131 | *new_edge->indirect_info = *indirect_info; |
564fe867 JH |
132 | } |
133 | } | |
134 | ||
3dafb85c ML |
135 | new_edge->inline_failed = inline_failed; |
136 | new_edge->indirect_inlining_edge = indirect_inlining_edge; | |
118aa5e3 JH |
137 | if (!call_stmt) |
138 | new_edge->lto_stmt_uid = stmt_uid; | |
f1ba88b1 | 139 | new_edge->speculative_id = speculative_id; |
564fe867 | 140 | /* Clone flags that depend on call_stmt availability manually. */ |
3dafb85c ML |
141 | new_edge->can_throw_external = can_throw_external; |
142 | new_edge->call_stmt_cannot_inline_p = call_stmt_cannot_inline_p; | |
143 | new_edge->speculative = speculative; | |
f9bb202b | 144 | new_edge->in_polymorphic_cdtor = in_polymorphic_cdtor; |
1bad9c18 JH |
145 | |
146 | /* Update IPA profile. Local profiles need no updating in original. */ | |
517048ce | 147 | if (update_original) |
b49d29d7 JH |
148 | count = count.combine_with_ipa_count_within (count.ipa () |
149 | - new_edge->count.ipa (), | |
150 | caller->count); | |
3dafb85c | 151 | symtab->call_edge_duplication_hooks (this, new_edge); |
564fe867 JH |
152 | return new_edge; |
153 | } | |
154 | ||
610c8ef0 MJ |
155 | /* Set flags of NEW_NODE and its decl. NEW_NODE is a newly created private |
156 | clone or its thunk. */ | |
157 | ||
158 | static void | |
159 | set_new_clone_decl_and_node_flags (cgraph_node *new_node) | |
160 | { | |
161 | DECL_EXTERNAL (new_node->decl) = 0; | |
610c8ef0 MJ |
162 | TREE_PUBLIC (new_node->decl) = 0; |
163 | DECL_COMDAT (new_node->decl) = 0; | |
164 | DECL_WEAK (new_node->decl) = 0; | |
165 | DECL_VIRTUAL_P (new_node->decl) = 0; | |
166 | DECL_STATIC_CONSTRUCTOR (new_node->decl) = 0; | |
167 | DECL_STATIC_DESTRUCTOR (new_node->decl) = 0; | |
fe8e21fd ML |
168 | DECL_SET_IS_OPERATOR_NEW (new_node->decl, 0); |
169 | DECL_SET_IS_OPERATOR_DELETE (new_node->decl, 0); | |
d7a65edb | 170 | DECL_IS_REPLACEABLE_OPERATOR (new_node->decl) = 0; |
610c8ef0 MJ |
171 | |
172 | new_node->externally_visible = 0; | |
87f94429 | 173 | new_node->local = 1; |
610c8ef0 | 174 | new_node->lowered = true; |
75ac95f6 | 175 | new_node->semantic_interposition = 0; |
610c8ef0 MJ |
176 | } |
177 | ||
178 | /* Duplicate thunk THUNK if necessary but make it to refer to NODE. | |
179 | ARGS_TO_SKIP, if non-NULL, determines which parameters should be omitted. | |
180 | Function can return NODE if no thunk is necessary, which can happen when | |
181 | thunk is this_adjusting but we are removing this parameter. */ | |
182 | ||
183 | static cgraph_node * | |
d284e1b8 | 184 | duplicate_thunk_for_node (cgraph_node *thunk, cgraph_node *node) |
610c8ef0 MJ |
185 | { |
186 | cgraph_node *new_thunk, *thunk_of; | |
d52f5295 | 187 | thunk_of = thunk->callees->callee->ultimate_alias_target (); |
610c8ef0 | 188 | |
67f3791f | 189 | if (thunk_of->thunk) |
d284e1b8 | 190 | node = duplicate_thunk_for_node (thunk_of, node); |
610c8ef0 | 191 | |
48fb6d40 | 192 | if (!DECL_ARGUMENTS (thunk->decl)) |
70486010 | 193 | thunk->get_untransformed_body (); |
48fb6d40 | 194 | |
67f3791f | 195 | thunk_info *i = thunk_info::get (thunk); |
3dafb85c | 196 | cgraph_edge *cs; |
610c8ef0 | 197 | for (cs = node->callers; cs; cs = cs->next_caller) |
67f3791f JH |
198 | if (cs->caller->thunk) |
199 | { | |
200 | thunk_info *i2 = thunk_info::get (cs->caller); | |
201 | if (*i2 == *i) | |
202 | return cs->caller; | |
203 | } | |
610c8ef0 MJ |
204 | |
205 | tree new_decl; | |
ae7a23a3 JH |
206 | clone_info *info = clone_info::get (node); |
207 | if (info && info->param_adjustments) | |
610c8ef0 MJ |
208 | { |
209 | /* We do not need to duplicate this_adjusting thunks if we have removed | |
210 | this. */ | |
67f3791f | 211 | if (i->this_adjusting |
ae7a23a3 | 212 | && !info->param_adjustments->first_param_intact_p ()) |
610c8ef0 MJ |
213 | return node; |
214 | ||
ff6686d2 | 215 | new_decl = copy_node (thunk->decl); |
ae7a23a3 | 216 | ipa_param_body_adjustments body_adj (info->param_adjustments, |
ff6686d2 MJ |
217 | new_decl); |
218 | body_adj.modify_formal_parameters (); | |
bec63208 | 219 | } |
ff6686d2 MJ |
220 | else |
221 | new_decl = copy_node (thunk->decl); | |
bec63208 | 222 | |
610c8ef0 MJ |
223 | gcc_checking_assert (!DECL_STRUCT_FUNCTION (new_decl)); |
224 | gcc_checking_assert (!DECL_INITIAL (new_decl)); | |
225 | gcc_checking_assert (!DECL_RESULT (new_decl)); | |
226 | gcc_checking_assert (!DECL_RTL_SET_P (new_decl)); | |
227 | ||
7958186b MP |
228 | DECL_NAME (new_decl) = clone_function_name_numbered (thunk->decl, |
229 | "artificial_thunk"); | |
610c8ef0 | 230 | SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl)); |
610c8ef0 | 231 | |
67124cb6 EB |
232 | /* We need to force DECL_IGNORED_P because the new thunk is created after |
233 | early debug was run. */ | |
234 | DECL_IGNORED_P (new_decl) = 1; | |
235 | ||
d52f5295 | 236 | new_thunk = cgraph_node::create (new_decl); |
610c8ef0 MJ |
237 | set_new_clone_decl_and_node_flags (new_thunk); |
238 | new_thunk->definition = true; | |
87f94429 | 239 | new_thunk->can_change_signature = node->can_change_signature; |
610c8ef0 MJ |
240 | new_thunk->thunk = thunk->thunk; |
241 | new_thunk->unique_name = in_lto_p; | |
242 | new_thunk->former_clone_of = thunk->decl; | |
a2058f58 | 243 | if (info && info->param_adjustments) |
ae7a23a3 JH |
244 | clone_info::get_create (new_thunk)->param_adjustments |
245 | = info->param_adjustments; | |
b74d8dc4 JH |
246 | new_thunk->unit_id = thunk->unit_id; |
247 | new_thunk->merged_comdat = thunk->merged_comdat; | |
248 | new_thunk->merged_extern_inline = thunk->merged_extern_inline; | |
610c8ef0 | 249 | |
1bad9c18 | 250 | cgraph_edge *e = new_thunk->create_edge (node, NULL, new_thunk->count); |
3dafb85c | 251 | symtab->call_edge_duplication_hooks (thunk->callees, e); |
3dafb85c | 252 | symtab->call_cgraph_duplication_hooks (thunk, new_thunk); |
610c8ef0 MJ |
253 | return new_thunk; |
254 | } | |
255 | ||
256 | /* If E does not lead to a thunk, simply redirect it to N. Otherwise create | |
257 | one or more equivalent thunks for N and redirect E to the first in the | |
6a4bad95 MJ |
258 | chain. Note that it is then necessary to call |
259 | n->expand_all_artificial_thunks once all callers are redirected. */ | |
610c8ef0 MJ |
260 | |
261 | void | |
6a4bad95 | 262 | cgraph_edge::redirect_callee_duplicating_thunks (cgraph_node *n) |
610c8ef0 | 263 | { |
6a4bad95 | 264 | cgraph_node *orig_to = callee->ultimate_alias_target (); |
67f3791f | 265 | if (orig_to->thunk) |
d284e1b8 | 266 | n = duplicate_thunk_for_node (orig_to, n); |
610c8ef0 | 267 | |
6a4bad95 MJ |
268 | redirect_callee (n); |
269 | } | |
270 | ||
271 | /* Call expand_thunk on all callers that are thunks and if analyze those nodes | |
272 | that were expanded. */ | |
273 | ||
274 | void | |
275 | cgraph_node::expand_all_artificial_thunks () | |
276 | { | |
277 | cgraph_edge *e; | |
278 | for (e = callers; e;) | |
67f3791f | 279 | if (e->caller->thunk) |
6a4bad95 MJ |
280 | { |
281 | cgraph_node *thunk = e->caller; | |
282 | ||
283 | e = e->next_caller; | |
67f3791f | 284 | if (expand_thunk (thunk, false, false)) |
6a4bad95 | 285 | { |
67f3791f | 286 | thunk->thunk = false; |
6a4bad95 | 287 | thunk->analyze (); |
68188fff MJ |
288 | ipa_analyze_node (thunk); |
289 | inline_analyze_function (thunk); | |
6a4bad95 MJ |
290 | } |
291 | thunk->expand_all_artificial_thunks (); | |
292 | } | |
293 | else | |
294 | e = e->next_caller; | |
610c8ef0 | 295 | } |
564fe867 | 296 | |
0bdad123 ML |
297 | void |
298 | dump_callgraph_transformation (const cgraph_node *original, | |
299 | const cgraph_node *clone, | |
300 | const char *suffix) | |
301 | { | |
302 | if (symtab->ipa_clones_dump_file) | |
303 | { | |
304 | fprintf (symtab->ipa_clones_dump_file, | |
305 | "Callgraph clone;%s;%d;%s;%d;%d;%s;%d;%s;%d;%d;%s\n", | |
306 | original->asm_name (), original->order, | |
307 | DECL_SOURCE_FILE (original->decl), | |
308 | DECL_SOURCE_LINE (original->decl), | |
309 | DECL_SOURCE_COLUMN (original->decl), clone->asm_name (), | |
310 | clone->order, DECL_SOURCE_FILE (clone->decl), | |
311 | DECL_SOURCE_LINE (clone->decl), DECL_SOURCE_COLUMN (clone->decl), | |
312 | suffix); | |
313 | ||
314 | symtab->cloned_nodes.add (original); | |
315 | symtab->cloned_nodes.add (clone); | |
316 | } | |
317 | } | |
318 | ||
34fbe3f0 JH |
319 | /* Turn profile of N to local profile. */ |
320 | ||
321 | static void | |
322 | localize_profile (cgraph_node *n) | |
323 | { | |
324 | n->count = n->count.guessed_local (); | |
325 | for (cgraph_edge *e = n->callees; e; e=e->next_callee) | |
326 | { | |
327 | e->count = e->count.guessed_local (); | |
328 | if (!e->inline_failed) | |
329 | localize_profile (e->callee); | |
330 | } | |
331 | for (cgraph_edge *e = n->indirect_calls; e; e=e->next_callee) | |
332 | e->count = e->count.guessed_local (); | |
333 | } | |
334 | ||
564fe867 JH |
335 | /* Create node representing clone of N executed COUNT times. Decrease |
336 | the execution counts from original node too. | |
337 | The new clone will have decl set to DECL that may or may not be the same | |
338 | as decl of N. | |
339 | ||
340 | When UPDATE_ORIGINAL is true, the counts are subtracted from the original | |
341 | function's profile to reflect the fact that part of execution is handled | |
342 | by node. | |
dfea3d6f | 343 | When CALL_DUPLICATION_HOOK is true, the ipa passes are acknowledged about |
44a60244 MJ |
344 | the new clone. Otherwise the caller is responsible for doing so later. |
345 | ||
346 | If the new node is being inlined into another one, NEW_INLINED_TO should be | |
347 | the outline function the new one is (even indirectly) inlined to. All hooks | |
a62bfab5 | 348 | will see this in node's inlined_to, when invoked. Can be NULL if the |
ff6686d2 MJ |
349 | node is not inlined. |
350 | ||
351 | If PARAM_ADJUSTMENTS is non-NULL, the parameter manipulation information | |
352 | will be overwritten by the new structure. Otherwise the new node will | |
353 | share parameter manipulation information with the original node. */ | |
564fe867 | 354 | |
d52f5295 | 355 | cgraph_node * |
1bad9c18 | 356 | cgraph_node::create_clone (tree new_decl, profile_count prof_count, |
d52f5295 ML |
357 | bool update_original, |
358 | vec<cgraph_edge *> redirect_callers, | |
359 | bool call_duplication_hook, | |
3dafb85c | 360 | cgraph_node *new_inlined_to, |
ff6686d2 MJ |
361 | ipa_param_adjustments *param_adjustments, |
362 | const char *suffix) | |
564fe867 | 363 | { |
3dafb85c ML |
364 | cgraph_node *new_node = symtab->create_empty (); |
365 | cgraph_edge *e; | |
564fe867 | 366 | unsigned i; |
1bad9c18 | 367 | profile_count old_count = count; |
34fbe3f0 | 368 | bool nonzero = count.ipa ().nonzero_p (); |
564fe867 | 369 | |
0bdad123 ML |
370 | if (new_inlined_to) |
371 | dump_callgraph_transformation (this, new_inlined_to, "inlining to"); | |
372 | ||
8f58dbd1 JH |
373 | /* When inlining we scale precisely to prof_count, when cloning we can |
374 | preserve local profile. */ | |
375 | if (!new_inlined_to) | |
376 | prof_count = count.combine_with_ipa_count (prof_count); | |
3995f3a2 | 377 | new_node->count = prof_count; |
1bad9c18 JH |
378 | |
379 | /* Update IPA profile. Local profiles need no updating in original. */ | |
517048ce | 380 | if (update_original) |
b49d29d7 JH |
381 | { |
382 | if (inlined_to) | |
383 | count = count.combine_with_ipa_count_within (count.ipa () | |
384 | - prof_count.ipa (), | |
385 | inlined_to->count); | |
386 | else | |
387 | count = count.combine_with_ipa_count (count.ipa () - prof_count.ipa ()); | |
388 | } | |
ed6ef490 | 389 | new_node->decl = new_decl; |
d52f5295 | 390 | new_node->register_symbol (); |
d52f5295 | 391 | new_node->lto_file_data = lto_file_data; |
d52f5295 ML |
392 | new_node->analyzed = analyzed; |
393 | new_node->definition = definition; | |
87f94429 ML |
394 | new_node->versionable = versionable; |
395 | new_node->can_change_signature = can_change_signature; | |
396 | new_node->redefined_extern_inline = redefined_extern_inline; | |
2a6d372b | 397 | new_node->semantic_interposition = semantic_interposition; |
87f94429 | 398 | new_node->tm_may_enter_irr = tm_may_enter_irr; |
67348ccc | 399 | new_node->externally_visible = false; |
7861b648 | 400 | new_node->no_reorder = no_reorder; |
87f94429 | 401 | new_node->local = true; |
a62bfab5 | 402 | new_node->inlined_to = new_inlined_to; |
d52f5295 | 403 | new_node->rtl = rtl; |
d52f5295 ML |
404 | new_node->frequency = frequency; |
405 | new_node->tp_first_run = tp_first_run; | |
9d6171dc | 406 | new_node->tm_clone = tm_clone; |
0a7246ee | 407 | new_node->icf_merged = icf_merged; |
6bbf39b7 | 408 | new_node->thunk = thunk; |
b74d8dc4 JH |
409 | new_node->unit_id = unit_id; |
410 | new_node->merged_comdat = merged_comdat; | |
411 | new_node->merged_extern_inline = merged_extern_inline; | |
ae7a23a3 | 412 | clone_info *info = clone_info::get (this); |
d284e1b8 | 413 | |
ff6686d2 | 414 | if (param_adjustments) |
ae7a23a3 JH |
415 | clone_info::get_create (new_node)->param_adjustments = param_adjustments; |
416 | else if (info && info->param_adjustments) | |
417 | clone_info::get_create (new_node)->param_adjustments | |
418 | = info->param_adjustments; | |
ed6ef490 | 419 | new_node->split_part = split_part; |
d284e1b8 | 420 | |
9771b263 | 421 | FOR_EACH_VEC_ELT (redirect_callers, i, e) |
564fe867 JH |
422 | { |
423 | /* Redirect calls to the old version node to point to its new | |
9de6f6c3 | 424 | version. The only exception is when the edge was proved to |
dfea3d6f | 425 | be unreachable during the cloning procedure. */ |
9de6f6c3 | 426 | if (!e->callee |
3d78e008 | 427 | || !fndecl_built_in_p (e->callee->decl, BUILT_IN_UNREACHABLE)) |
6a4bad95 | 428 | e->redirect_callee_duplicating_thunks (new_node); |
564fe867 | 429 | } |
6a4bad95 | 430 | new_node->expand_all_artificial_thunks (); |
564fe867 | 431 | |
d52f5295 | 432 | for (e = callees;e; e=e->next_callee) |
1bad9c18 JH |
433 | e->clone (new_node, e->call_stmt, e->lto_stmt_uid, new_node->count, old_count, |
434 | update_original); | |
564fe867 | 435 | |
d52f5295 | 436 | for (e = indirect_calls; e; e = e->next_callee) |
3dafb85c | 437 | e->clone (new_node, e->call_stmt, e->lto_stmt_uid, |
1bad9c18 | 438 | new_node->count, old_count, update_original); |
d52f5295 | 439 | new_node->clone_references (this); |
564fe867 | 440 | |
d52f5295 ML |
441 | new_node->next_sibling_clone = clones; |
442 | if (clones) | |
443 | clones->prev_sibling_clone = new_node; | |
444 | clones = new_node; | |
445 | new_node->clone_of = this; | |
564fe867 JH |
446 | |
447 | if (call_duplication_hook) | |
3dafb85c | 448 | symtab->call_cgraph_duplication_hooks (this, new_node); |
34fbe3f0 JH |
449 | /* With partial train run we do not want to assume that original's |
450 | count is zero whenever we redurect all executed edges to clone. | |
451 | Simply drop profile to local one in this case. */ | |
452 | if (update_original | |
453 | && opt_for_fn (decl, flag_profile_partial_training) | |
454 | && nonzero | |
455 | && count.ipa_p () | |
f7f6be76 JH |
456 | && !count.ipa ().nonzero_p () |
457 | && !inlined_to) | |
34fbe3f0 | 458 | localize_profile (this); |
0bdad123 ML |
459 | |
460 | if (!new_inlined_to) | |
461 | dump_callgraph_transformation (this, new_node, suffix); | |
462 | ||
564fe867 JH |
463 | return new_node; |
464 | } | |
465 | ||
b75255a9 | 466 | static GTY(()) hash_map<const char *, unsigned> *clone_fn_ids; |
564fe867 | 467 | |
7958186b MP |
468 | /* Return a new assembler name for a clone of decl named NAME. Apart |
469 | from the string SUFFIX, the new name will end with a unique (for | |
470 | each NAME) unspecified number. If clone numbering is not needed | |
471 | then the two argument clone_function_name should be used instead. | |
472 | Should not be called directly except for by | |
e53b6e56 | 473 | lto-partition.cc:privatize_symbol_name_1. */ |
9816367c | 474 | |
564fe867 | 475 | tree |
7958186b MP |
476 | clone_function_name_numbered (const char *name, const char *suffix) |
477 | { | |
b75255a9 MP |
478 | /* Initialize the function->counter mapping the first time it's |
479 | needed. */ | |
480 | if (!clone_fn_ids) | |
481 | clone_fn_ids = hash_map<const char *, unsigned int>::create_ggc (64); | |
482 | unsigned int &suffix_counter = clone_fn_ids->get_or_insert ( | |
483 | IDENTIFIER_POINTER (get_identifier (name))); | |
484 | return clone_function_name (name, suffix, suffix_counter++); | |
7958186b MP |
485 | } |
486 | ||
487 | /* Return a new assembler name for a clone of DECL. Apart from string | |
488 | SUFFIX, the new name will end with a unique (for each DECL | |
489 | assembler name) unspecified number. If clone numbering is not | |
490 | needed then the two argument clone_function_name should be used | |
491 | instead. */ | |
492 | ||
493 | tree | |
494 | clone_function_name_numbered (tree decl, const char *suffix) | |
495 | { | |
496 | tree name = DECL_ASSEMBLER_NAME (decl); | |
497 | return clone_function_name_numbered (IDENTIFIER_POINTER (name), | |
498 | suffix); | |
499 | } | |
500 | ||
501 | /* Return a new assembler name for a clone of decl named NAME. Apart | |
502 | from the string SUFFIX, the new name will end with the specified | |
503 | NUMBER. If clone numbering is not needed then the two argument | |
504 | clone_function_name should be used instead. */ | |
505 | ||
506 | tree | |
507 | clone_function_name (const char *name, const char *suffix, | |
508 | unsigned long number) | |
564fe867 | 509 | { |
9816367c | 510 | size_t len = strlen (name); |
564fe867 JH |
511 | char *tmp_name, *prefix; |
512 | ||
513 | prefix = XALLOCAVEC (char, len + strlen (suffix) + 2); | |
9816367c | 514 | memcpy (prefix, name, len); |
564fe867 | 515 | strcpy (prefix + len + 1, suffix); |
f8a1abf8 | 516 | prefix[len] = symbol_table::symbol_suffix_separator (); |
7958186b | 517 | ASM_FORMAT_PRIVATE_NAME (tmp_name, prefix, number); |
564fe867 JH |
518 | return get_identifier (tmp_name); |
519 | } | |
520 | ||
53aedcce MP |
521 | /* Return a new assembler name for a clone of DECL. Apart from the |
522 | string SUFFIX, the new name will end with the specified NUMBER. If | |
523 | clone numbering is not needed then the two argument | |
524 | clone_function_name should be used instead. */ | |
525 | ||
526 | tree | |
527 | clone_function_name (tree decl, const char *suffix, | |
528 | unsigned long number) | |
529 | { | |
530 | return clone_function_name ( | |
531 | IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)), suffix, number); | |
532 | } | |
533 | ||
7958186b MP |
534 | /* Return a new assembler name ending with the string SUFFIX for a |
535 | clone of DECL. */ | |
9816367c BS |
536 | |
537 | tree | |
538 | clone_function_name (tree decl, const char *suffix) | |
539 | { | |
7958186b MP |
540 | tree identifier = DECL_ASSEMBLER_NAME (decl); |
541 | /* For consistency this needs to behave the same way as | |
542 | ASM_FORMAT_PRIVATE_NAME does, but without the final number | |
543 | suffix. */ | |
544 | char *separator = XALLOCAVEC (char, 2); | |
545 | separator[0] = symbol_table::symbol_suffix_separator (); | |
546 | separator[1] = 0; | |
547 | #if defined (NO_DOT_IN_LABEL) && defined (NO_DOLLAR_IN_LABEL) | |
548 | const char *prefix = "__"; | |
549 | #else | |
550 | const char *prefix = ""; | |
551 | #endif | |
552 | char *result = ACONCAT ((prefix, | |
553 | IDENTIFIER_POINTER (identifier), | |
554 | separator, | |
555 | suffix, | |
556 | (char*)0)); | |
557 | return get_identifier (result); | |
9816367c BS |
558 | } |
559 | ||
560 | ||
53aedcce MP |
561 | /* Create callgraph node clone with new declaration. The actual body will be |
562 | copied later at compilation stage. The name of the new clone will be | |
563 | constructed from the name of the original node, SUFFIX and NUM_SUFFIX. | |
564fe867 JH |
564 | |
565 | TODO: after merging in ipa-sra use function call notes instead of args_to_skip | |
566 | bitmap interface. | |
567 | */ | |
3dafb85c | 568 | cgraph_node * |
00dcc88a | 569 | cgraph_node::create_virtual_clone (const vec<cgraph_edge *> &redirect_callers, |
d52f5295 | 570 | vec<ipa_replace_map *, va_gc> *tree_map, |
ff6686d2 MJ |
571 | ipa_param_adjustments *param_adjustments, |
572 | const char * suffix, unsigned num_suffix) | |
564fe867 | 573 | { |
d52f5295 | 574 | tree old_decl = decl; |
3dafb85c | 575 | cgraph_node *new_node = NULL; |
564fe867 | 576 | tree new_decl; |
440a5082 | 577 | size_t len, i; |
3dafb85c | 578 | ipa_replace_map *map; |
440a5082 | 579 | char *name; |
564fe867 | 580 | |
87f94429 | 581 | gcc_checking_assert (versionable); |
ff6686d2 MJ |
582 | /* TODO: It would be nice if we could recognize that param_adjustments do not |
583 | actually perform any changes, but at the moment let's require it simply | |
584 | does not exist. */ | |
87f94429 | 585 | gcc_assert (can_change_signature || !param_adjustments); |
564fe867 JH |
586 | |
587 | /* Make a new FUNCTION_DECL tree node */ | |
ff6686d2 | 588 | if (!param_adjustments) |
564fe867 JH |
589 | new_decl = copy_node (old_decl); |
590 | else | |
ff6686d2 | 591 | new_decl = param_adjustments->adjust_decl (old_decl); |
49bde175 JH |
592 | |
593 | /* These pointers represent function body and will be populated only when clone | |
594 | is materialized. */ | |
595 | gcc_assert (new_decl != old_decl); | |
564fe867 | 596 | DECL_STRUCT_FUNCTION (new_decl) = NULL; |
49bde175 JH |
597 | DECL_ARGUMENTS (new_decl) = NULL; |
598 | DECL_INITIAL (new_decl) = NULL; | |
599 | DECL_RESULT (new_decl) = NULL; | |
67914693 | 600 | /* We cannot do DECL_RESULT (new_decl) = NULL; here because of LTO partitioning |
49bde175 | 601 | sometimes storing only clone decl instead of original. */ |
564fe867 JH |
602 | |
603 | /* Generate a new name for the new version. */ | |
440a5082 EB |
604 | len = IDENTIFIER_LENGTH (DECL_NAME (old_decl)); |
605 | name = XALLOCAVEC (char, len + strlen (suffix) + 2); | |
606 | memcpy (name, IDENTIFIER_POINTER (DECL_NAME (old_decl)), len); | |
607 | strcpy (name + len + 1, suffix); | |
608 | name[len] = '.'; | |
609 | DECL_NAME (new_decl) = get_identifier (name); | |
53aedcce MP |
610 | SET_DECL_ASSEMBLER_NAME (new_decl, |
611 | clone_function_name (old_decl, suffix, num_suffix)); | |
564fe867 JH |
612 | SET_DECL_RTL (new_decl, NULL); |
613 | ||
1bad9c18 | 614 | new_node = create_clone (new_decl, count, false, |
ff6686d2 MJ |
615 | redirect_callers, false, NULL, param_adjustments, |
616 | suffix); | |
d52f5295 | 617 | |
564fe867 JH |
618 | /* Update the properties. |
619 | Make clone visible only within this translation unit. Make sure | |
620 | that is not weak also. | |
621 | ??? We cannot use COMDAT linkage because there is no | |
622 | ABI support for this. */ | |
610c8ef0 | 623 | set_new_clone_decl_and_node_flags (new_node); |
6cf67b62 | 624 | new_node->ipcp_clone = ipcp_clone; |
ae7a23a3 JH |
625 | if (tree_map) |
626 | clone_info::get_create (new_node)->tree_map = tree_map; | |
4ab26ee0 | 627 | if (!implicit_section) |
8b9a92f7 | 628 | new_node->set_section (*this); |
702d8703 JH |
629 | |
630 | /* Clones of global symbols or symbols with unique names are unique. */ | |
631 | if ((TREE_PUBLIC (old_decl) | |
632 | && !DECL_EXTERNAL (old_decl) | |
633 | && !DECL_WEAK (old_decl) | |
634 | && !DECL_COMDAT (old_decl)) | |
635 | || in_lto_p) | |
67348ccc | 636 | new_node->unique_name = true; |
9771b263 | 637 | FOR_EACH_VEC_SAFE_ELT (tree_map, i, map) |
13586172 MJ |
638 | { |
639 | tree repl = map->new_tree; | |
640 | if (map->force_load_ref) | |
641 | { | |
642 | gcc_assert (TREE_CODE (repl) == ADDR_EXPR); | |
643 | repl = get_base_address (TREE_OPERAND (repl, 0)); | |
644 | } | |
645 | new_node->maybe_create_reference (repl, NULL); | |
646 | } | |
d284e1b8 | 647 | |
d52f5295 | 648 | if (ipa_transforms_to_apply.exists ()) |
ca860d03 | 649 | new_node->ipa_transforms_to_apply |
d52f5295 | 650 | = ipa_transforms_to_apply.copy (); |
564fe867 | 651 | |
3dafb85c | 652 | symtab->call_cgraph_duplication_hooks (this, new_node); |
564fe867 JH |
653 | |
654 | return new_node; | |
655 | } | |
656 | ||
d52f5295 | 657 | /* callgraph node being removed from symbol table; see if its entry can be |
ab4664eb JH |
658 | replaced by other inline clone. |
659 | INFO is clone info to attach to the new root. */ | |
d52f5295 | 660 | cgraph_node * |
ab4664eb | 661 | cgraph_node::find_replacement (clone_info *info) |
564fe867 | 662 | { |
3dafb85c | 663 | cgraph_node *next_inline_clone, *replacement; |
564fe867 | 664 | |
d52f5295 | 665 | for (next_inline_clone = clones; |
564fe867 | 666 | next_inline_clone |
d52f5295 | 667 | && next_inline_clone->decl != decl; |
564fe867 JH |
668 | next_inline_clone = next_inline_clone->next_sibling_clone) |
669 | ; | |
670 | ||
671 | /* If there is inline clone of the node being removed, we need | |
672 | to put it into the position of removed node and reorganize all | |
673 | other clones to be based on it. */ | |
674 | if (next_inline_clone) | |
675 | { | |
3dafb85c ML |
676 | cgraph_node *n; |
677 | cgraph_node *new_clones; | |
564fe867 JH |
678 | |
679 | replacement = next_inline_clone; | |
680 | ||
681 | /* Unlink inline clone from the list of clones of removed node. */ | |
682 | if (next_inline_clone->next_sibling_clone) | |
683 | next_inline_clone->next_sibling_clone->prev_sibling_clone | |
684 | = next_inline_clone->prev_sibling_clone; | |
685 | if (next_inline_clone->prev_sibling_clone) | |
686 | { | |
d52f5295 | 687 | gcc_assert (clones != next_inline_clone); |
564fe867 JH |
688 | next_inline_clone->prev_sibling_clone->next_sibling_clone |
689 | = next_inline_clone->next_sibling_clone; | |
690 | } | |
691 | else | |
692 | { | |
d52f5295 ML |
693 | gcc_assert (clones == next_inline_clone); |
694 | clones = next_inline_clone->next_sibling_clone; | |
564fe867 JH |
695 | } |
696 | ||
d52f5295 ML |
697 | new_clones = clones; |
698 | clones = NULL; | |
564fe867 JH |
699 | |
700 | /* Copy clone info. */ | |
ae7a23a3 JH |
701 | if (info) |
702 | *clone_info::get_create (next_inline_clone) = *info; | |
564fe867 JH |
703 | |
704 | /* Now place it into clone tree at same level at NODE. */ | |
d52f5295 | 705 | next_inline_clone->clone_of = clone_of; |
564fe867 JH |
706 | next_inline_clone->prev_sibling_clone = NULL; |
707 | next_inline_clone->next_sibling_clone = NULL; | |
d52f5295 | 708 | if (clone_of) |
564fe867 | 709 | { |
d52f5295 ML |
710 | if (clone_of->clones) |
711 | clone_of->clones->prev_sibling_clone = next_inline_clone; | |
712 | next_inline_clone->next_sibling_clone = clone_of->clones; | |
713 | clone_of->clones = next_inline_clone; | |
564fe867 JH |
714 | } |
715 | ||
716 | /* Merge the clone list. */ | |
717 | if (new_clones) | |
718 | { | |
719 | if (!next_inline_clone->clones) | |
720 | next_inline_clone->clones = new_clones; | |
721 | else | |
722 | { | |
723 | n = next_inline_clone->clones; | |
724 | while (n->next_sibling_clone) | |
d52f5295 | 725 | n = n->next_sibling_clone; |
564fe867 JH |
726 | n->next_sibling_clone = new_clones; |
727 | new_clones->prev_sibling_clone = n; | |
728 | } | |
729 | } | |
730 | ||
731 | /* Update clone_of pointers. */ | |
732 | n = new_clones; | |
733 | while (n) | |
734 | { | |
735 | n->clone_of = next_inline_clone; | |
736 | n = n->next_sibling_clone; | |
737 | } | |
3c56d8d8 ML |
738 | |
739 | /* Update order in order to be able to find a LTO section | |
740 | with function body. */ | |
741 | replacement->order = order; | |
742 | ||
564fe867 JH |
743 | return replacement; |
744 | } | |
745 | else | |
746 | return NULL; | |
747 | } | |
748 | ||
749 | /* Like cgraph_set_call_stmt but walk the clone tree and update all | |
042ae7d2 JH |
750 | clones sharing the same function body. |
751 | When WHOLE_SPECULATIVE_EDGES is true, all three components of | |
752 | speculative edge gets updated. Otherwise we update only direct | |
753 | call. */ | |
564fe867 JH |
754 | |
755 | void | |
355fe088 | 756 | cgraph_node::set_call_stmt_including_clones (gimple *old_stmt, |
538dd0b7 | 757 | gcall *new_stmt, |
d52f5295 | 758 | bool update_speculative) |
564fe867 | 759 | { |
3dafb85c | 760 | cgraph_node *node; |
27c5a177 | 761 | cgraph_edge *master_edge = get_edge (old_stmt); |
564fe867 | 762 | |
27c5a177 MJ |
763 | if (master_edge) |
764 | cgraph_edge::set_call_stmt (master_edge, new_stmt, update_speculative); | |
564fe867 | 765 | |
d52f5295 | 766 | node = clones; |
564fe867 | 767 | if (node) |
d52f5295 | 768 | while (node != this) |
564fe867 | 769 | { |
3dafb85c | 770 | cgraph_edge *edge = node->get_edge (old_stmt); |
564fe867 | 771 | if (edge) |
042ae7d2 | 772 | { |
27c5a177 MJ |
773 | edge = cgraph_edge::set_call_stmt (edge, new_stmt, |
774 | update_speculative); | |
042ae7d2 JH |
775 | /* If UPDATE_SPECULATIVE is false, it means that we are turning |
776 | speculative call into a real code sequence. Update the | |
777 | callgraph edges. */ | |
778 | if (edge->speculative && !update_speculative) | |
779 | { | |
845bb366 JH |
780 | cgraph_edge *indirect = edge->speculative_call_indirect_edge (); |
781 | ||
782 | for (cgraph_edge *next, *direct | |
783 | = edge->first_speculative_call_target (); | |
784 | direct; | |
785 | direct = next) | |
786 | { | |
787 | next = direct->next_speculative_call_target (); | |
788 | direct->speculative_call_target_ref ()->speculative = false; | |
789 | direct->speculative = false; | |
790 | } | |
042ae7d2 | 791 | indirect->speculative = false; |
042ae7d2 JH |
792 | } |
793 | } | |
564fe867 JH |
794 | if (node->clones) |
795 | node = node->clones; | |
796 | else if (node->next_sibling_clone) | |
797 | node = node->next_sibling_clone; | |
798 | else | |
799 | { | |
d52f5295 | 800 | while (node != this && !node->next_sibling_clone) |
564fe867 | 801 | node = node->clone_of; |
d52f5295 | 802 | if (node != this) |
564fe867 JH |
803 | node = node->next_sibling_clone; |
804 | } | |
805 | } | |
806 | } | |
807 | ||
808 | /* Like cgraph_create_edge walk the clone tree and update all clones sharing | |
809 | same function body. If clones already have edge for OLD_STMT; only | |
810 | update the edge same way as cgraph_set_call_stmt_including_clones does. | |
811 | ||
812 | TODO: COUNT and LOOP_DEPTH should be properly distributed based on relative | |
813 | frequencies of the clones. */ | |
814 | ||
815 | void | |
3dafb85c | 816 | cgraph_node::create_edge_including_clones (cgraph_node *callee, |
355fe088 | 817 | gimple *old_stmt, gcall *stmt, |
3995f3a2 | 818 | profile_count count, |
d52f5295 | 819 | cgraph_inline_failed_t reason) |
564fe867 | 820 | { |
3dafb85c | 821 | cgraph_node *node; |
564fe867 | 822 | |
d52f5295 | 823 | if (!get_edge (stmt)) |
564fe867 | 824 | { |
27c5a177 | 825 | cgraph_edge *edge = create_edge (callee, stmt, count); |
564fe867 JH |
826 | edge->inline_failed = reason; |
827 | } | |
828 | ||
d52f5295 | 829 | node = clones; |
564fe867 | 830 | if (node) |
d52f5295 | 831 | while (node != this) |
ec6a1e35 | 832 | /* Thunk clones do not get updated while copying inline function body. */ |
67f3791f | 833 | if (!node->thunk) |
ec6a1e35 JH |
834 | { |
835 | cgraph_edge *edge = node->get_edge (old_stmt); | |
836 | ||
837 | /* It is possible that clones already contain the edge while | |
838 | master didn't. Either we promoted indirect call into direct | |
839 | call in the clone or we are processing clones of unreachable | |
840 | master where edges has been removed. */ | |
841 | if (edge) | |
27c5a177 | 842 | edge = cgraph_edge::set_call_stmt (edge, stmt); |
ec6a1e35 JH |
843 | else if (! node->get_edge (stmt)) |
844 | { | |
1bad9c18 | 845 | edge = node->create_edge (callee, stmt, count); |
ec6a1e35 JH |
846 | edge->inline_failed = reason; |
847 | } | |
564fe867 | 848 | |
ec6a1e35 JH |
849 | if (node->clones) |
850 | node = node->clones; | |
851 | else if (node->next_sibling_clone) | |
852 | node = node->next_sibling_clone; | |
853 | else | |
854 | { | |
855 | while (node != this && !node->next_sibling_clone) | |
856 | node = node->clone_of; | |
857 | if (node != this) | |
858 | node = node->next_sibling_clone; | |
859 | } | |
860 | } | |
564fe867 JH |
861 | } |
862 | ||
863 | /* Remove the node from cgraph and all inline clones inlined into it. | |
864 | Skip however removal of FORBIDDEN_NODE and return true if it needs to be | |
865 | removed. This allows to call the function from outer loop walking clone | |
866 | tree. */ | |
867 | ||
868 | bool | |
d52f5295 | 869 | cgraph_node::remove_symbol_and_inline_clones (cgraph_node *forbidden_node) |
564fe867 | 870 | { |
3dafb85c | 871 | cgraph_edge *e, *next; |
564fe867 JH |
872 | bool found = false; |
873 | ||
d52f5295 | 874 | if (this == forbidden_node) |
39f9719e | 875 | { |
27c5a177 | 876 | cgraph_edge::remove (callers); |
39f9719e JH |
877 | return true; |
878 | } | |
d52f5295 | 879 | for (e = callees; e; e = next) |
564fe867 JH |
880 | { |
881 | next = e->next_callee; | |
882 | if (!e->inline_failed) | |
d52f5295 | 883 | found |= e->callee->remove_symbol_and_inline_clones (forbidden_node); |
564fe867 | 884 | } |
d52f5295 | 885 | remove (); |
564fe867 JH |
886 | return found; |
887 | } | |
888 | ||
889 | /* The edges representing the callers of the NEW_VERSION node were | |
890 | fixed by cgraph_function_versioning (), now the call_expr in their | |
891 | respective tree code should be updated to call the NEW_VERSION. */ | |
892 | ||
893 | static void | |
3dafb85c | 894 | update_call_expr (cgraph_node *new_version) |
564fe867 | 895 | { |
3dafb85c | 896 | cgraph_edge *e; |
564fe867 JH |
897 | |
898 | gcc_assert (new_version); | |
899 | ||
900 | /* Update the call expr on the edges to call the new version. */ | |
901 | for (e = new_version->callers; e; e = e->next_caller) | |
902 | { | |
3dafb85c | 903 | function *inner_function = DECL_STRUCT_FUNCTION (e->caller->decl); |
67348ccc | 904 | gimple_call_set_fndecl (e->call_stmt, new_version->decl); |
564fe867 JH |
905 | maybe_clean_eh_stmt_fn (inner_function, e->call_stmt); |
906 | } | |
907 | } | |
908 | ||
909 | ||
910 | /* Create a new cgraph node which is the new version of | |
d52f5295 | 911 | callgraph node. REDIRECT_CALLERS holds the callers |
564fe867 | 912 | edges which should be redirected to point to |
d52f5295 | 913 | NEW_VERSION. ALL the callees edges of the node |
564fe867 JH |
914 | are cloned to the new version node. Return the new |
915 | version node. | |
916 | ||
917 | If non-NULL BLOCK_TO_COPY determine what basic blocks | |
918 | was copied to prevent duplications of calls that are dead | |
919 | in the clone. */ | |
920 | ||
d52f5295 ML |
921 | cgraph_node * |
922 | cgraph_node::create_version_clone (tree new_decl, | |
923 | vec<cgraph_edge *> redirect_callers, | |
0bdad123 ML |
924 | bitmap bbs_to_copy, |
925 | const char *suffix) | |
564fe867 | 926 | { |
3dafb85c ML |
927 | cgraph_node *new_version; |
928 | cgraph_edge *e; | |
564fe867 JH |
929 | unsigned i; |
930 | ||
d52f5295 | 931 | new_version = cgraph_node::create (new_decl); |
564fe867 | 932 | |
d52f5295 ML |
933 | new_version->analyzed = analyzed; |
934 | new_version->definition = definition; | |
935 | new_version->local = local; | |
67348ccc | 936 | new_version->externally_visible = false; |
7861b648 | 937 | new_version->no_reorder = no_reorder; |
87f94429 | 938 | new_version->local = new_version->definition; |
a62bfab5 | 939 | new_version->inlined_to = inlined_to; |
d52f5295 ML |
940 | new_version->rtl = rtl; |
941 | new_version->count = count; | |
b74d8dc4 JH |
942 | new_version->unit_id = unit_id; |
943 | new_version->merged_comdat = merged_comdat; | |
944 | new_version->merged_extern_inline = merged_extern_inline; | |
564fe867 | 945 | |
d52f5295 | 946 | for (e = callees; e; e=e->next_callee) |
564fe867 JH |
947 | if (!bbs_to_copy |
948 | || bitmap_bit_p (bbs_to_copy, gimple_bb (e->call_stmt)->index)) | |
3dafb85c | 949 | e->clone (new_version, e->call_stmt, |
1511c8c0 | 950 | e->lto_stmt_uid, count, count, |
3dafb85c | 951 | true); |
d52f5295 | 952 | for (e = indirect_calls; e; e=e->next_callee) |
564fe867 JH |
953 | if (!bbs_to_copy |
954 | || bitmap_bit_p (bbs_to_copy, gimple_bb (e->call_stmt)->index)) | |
3dafb85c | 955 | e->clone (new_version, e->call_stmt, |
1511c8c0 | 956 | e->lto_stmt_uid, count, count, |
3dafb85c | 957 | true); |
9771b263 | 958 | FOR_EACH_VEC_ELT (redirect_callers, i, e) |
564fe867 JH |
959 | { |
960 | /* Redirect calls to the old version node to point to its new | |
961 | version. */ | |
3dafb85c | 962 | e->redirect_callee (new_version); |
564fe867 JH |
963 | } |
964 | ||
0bdad123 ML |
965 | dump_callgraph_transformation (this, new_version, suffix); |
966 | ||
564fe867 JH |
967 | return new_version; |
968 | } | |
969 | ||
970 | /* Perform function versioning. | |
971 | Function versioning includes copying of the tree and | |
972 | a callgraph update (creating a new cgraph node and updating | |
973 | its callees and callers). | |
974 | ||
975 | REDIRECT_CALLERS varray includes the edges to be redirected | |
976 | to the new version. | |
977 | ||
978 | TREE_MAP is a mapping of tree nodes we want to replace with | |
979 | new ones (according to results of prior analysis). | |
564fe867 JH |
980 | |
981 | If non-NULL ARGS_TO_SKIP determine function parameters to remove | |
982 | from new version. | |
983 | If SKIP_RETURN is true, the new version will return void. | |
984 | If non-NULL BLOCK_TO_COPY determine what basic blocks to copy. | |
985 | If non_NULL NEW_ENTRY determine new entry BB of the clone. | |
986 | ||
5928bc2e ML |
987 | If TARGET_ATTRIBUTES is non-null, when creating a new declaration, |
988 | add the attributes to DECL_ATTRIBUTES. And call valid_attribute_p | |
989 | that will promote value of the attribute DECL_FUNCTION_SPECIFIC_TARGET | |
990 | of the declaration. | |
991 | ||
bfc9250e ML |
992 | If VERSION_DECL is set true, use clone_function_name_numbered for the |
993 | function clone. Otherwise, use clone_function_name. | |
994 | ||
564fe867 JH |
995 | Return the new version's cgraph node. */ |
996 | ||
d52f5295 ML |
997 | cgraph_node * |
998 | cgraph_node::create_version_clone_with_body | |
999 | (vec<cgraph_edge *> redirect_callers, | |
ff6686d2 MJ |
1000 | vec<ipa_replace_map *, va_gc> *tree_map, |
1001 | ipa_param_adjustments *param_adjustments, | |
1002 | bitmap bbs_to_copy, basic_block new_entry_block, const char *suffix, | |
bfc9250e | 1003 | tree target_attributes, bool version_decl) |
564fe867 | 1004 | { |
d52f5295 | 1005 | tree old_decl = decl; |
3dafb85c | 1006 | cgraph_node *new_version_node = NULL; |
564fe867 JH |
1007 | tree new_decl; |
1008 | ||
1009 | if (!tree_versionable_function_p (old_decl)) | |
1010 | return NULL; | |
1011 | ||
ff6686d2 | 1012 | /* TODO: Restore an assert that we do not change signature if |
87f94429 | 1013 | can_change_signature is false. We cannot just check that |
ff6686d2 MJ |
1014 | param_adjustments is NULL because unfortunately ipa-split removes return |
1015 | values from such functions. */ | |
564fe867 JH |
1016 | |
1017 | /* Make a new FUNCTION_DECL tree node for the new version. */ | |
ff6686d2 MJ |
1018 | if (param_adjustments) |
1019 | new_decl = param_adjustments->adjust_decl (old_decl); | |
564fe867 | 1020 | else |
ff6686d2 | 1021 | new_decl = copy_node (old_decl); |
564fe867 JH |
1022 | |
1023 | /* Generate a new name for the new version. */ | |
bfc9250e ML |
1024 | tree fnname = (version_decl ? clone_function_name_numbered (old_decl, suffix) |
1025 | : clone_function_name (old_decl, suffix)); | |
1026 | DECL_NAME (new_decl) = fnname; | |
1027 | SET_DECL_ASSEMBLER_NAME (new_decl, fnname); | |
564fe867 JH |
1028 | SET_DECL_RTL (new_decl, NULL); |
1029 | ||
97ae2126 JH |
1030 | DECL_VIRTUAL_P (new_decl) = 0; |
1031 | ||
5928bc2e ML |
1032 | if (target_attributes) |
1033 | { | |
1034 | DECL_ATTRIBUTES (new_decl) = target_attributes; | |
1035 | ||
1036 | location_t saved_loc = input_location; | |
1037 | tree v = TREE_VALUE (target_attributes); | |
1038 | input_location = DECL_SOURCE_LOCATION (new_decl); | |
cc2a672a | 1039 | bool r = targetm.target_option.valid_attribute_p (new_decl, NULL, v, 1); |
5928bc2e ML |
1040 | input_location = saved_loc; |
1041 | if (!r) | |
1042 | return NULL; | |
1043 | } | |
1044 | ||
564fe867 | 1045 | /* When the old decl was a con-/destructor make sure the clone isn't. */ |
c3284718 RS |
1046 | DECL_STATIC_CONSTRUCTOR (new_decl) = 0; |
1047 | DECL_STATIC_DESTRUCTOR (new_decl) = 0; | |
fe8e21fd ML |
1048 | DECL_SET_IS_OPERATOR_NEW (new_decl, 0); |
1049 | DECL_SET_IS_OPERATOR_DELETE (new_decl, 0); | |
d7a65edb | 1050 | DECL_IS_REPLACEABLE_OPERATOR (new_decl) = 0; |
564fe867 JH |
1051 | |
1052 | /* Create the new version's call-graph node. | |
1053 | and update the edges of the new node. */ | |
d52f5295 | 1054 | new_version_node = create_version_clone (new_decl, redirect_callers, |
0bdad123 | 1055 | bbs_to_copy, suffix); |
564fe867 | 1056 | |
d52f5295 | 1057 | if (ipa_transforms_to_apply.exists ()) |
c6d43074 | 1058 | new_version_node->ipa_transforms_to_apply |
d52f5295 | 1059 | = ipa_transforms_to_apply.copy (); |
564fe867 | 1060 | /* Copy the OLD_VERSION_NODE function tree to the new version. */ |
ff6686d2 MJ |
1061 | tree_function_versioning (old_decl, new_decl, tree_map, param_adjustments, |
1062 | false, bbs_to_copy, new_entry_block); | |
564fe867 JH |
1063 | |
1064 | /* Update the new version's properties. | |
1065 | Make The new version visible only within this translation unit. Make sure | |
1066 | that is not weak also. | |
1067 | ??? We cannot use COMDAT linkage because there is no | |
1068 | ABI support for this. */ | |
d52f5295 | 1069 | new_version_node->make_decl_local (); |
67348ccc DM |
1070 | DECL_VIRTUAL_P (new_version_node->decl) = 0; |
1071 | new_version_node->externally_visible = 0; | |
87f94429 | 1072 | new_version_node->local = 1; |
564fe867 | 1073 | new_version_node->lowered = true; |
4ab26ee0 | 1074 | if (!implicit_section) |
8b9a92f7 | 1075 | new_version_node->set_section (*this); |
702d8703 JH |
1076 | /* Clones of global symbols or symbols with unique names are unique. */ |
1077 | if ((TREE_PUBLIC (old_decl) | |
1078 | && !DECL_EXTERNAL (old_decl) | |
1079 | && !DECL_WEAK (old_decl) | |
1080 | && !DECL_COMDAT (old_decl)) | |
1081 | || in_lto_p) | |
67348ccc | 1082 | new_version_node->unique_name = true; |
564fe867 JH |
1083 | |
1084 | /* Update the call_expr on the edges to call the new version node. */ | |
1085 | update_call_expr (new_version_node); | |
1086 | ||
cbb4e4ca | 1087 | symtab->call_cgraph_insertion_hooks (new_version_node); |
564fe867 JH |
1088 | return new_version_node; |
1089 | } | |
1090 | ||
d0b1b67a MJ |
1091 | /* Remove the node from the tree of virtual and inline clones and make it a |
1092 | standalone node - not a clone any more. */ | |
1093 | ||
1094 | void cgraph_node::remove_from_clone_tree () | |
1095 | { | |
1096 | if (next_sibling_clone) | |
1097 | next_sibling_clone->prev_sibling_clone = prev_sibling_clone; | |
1098 | if (prev_sibling_clone) | |
1099 | prev_sibling_clone->next_sibling_clone = next_sibling_clone; | |
1100 | else | |
1101 | clone_of->clones = next_sibling_clone; | |
1102 | next_sibling_clone = NULL; | |
1103 | prev_sibling_clone = NULL; | |
1104 | clone_of = NULL; | |
1105 | } | |
1106 | ||
564fe867 JH |
1107 | /* Given virtual clone, turn it into actual clone. */ |
1108 | ||
564fe867 | 1109 | void |
0e590b68 | 1110 | cgraph_node::materialize_clone () |
564fe867 | 1111 | { |
ae7a23a3 | 1112 | clone_info *info = clone_info::get (this); |
0e590b68 JH |
1113 | clone_of->get_untransformed_body (); |
1114 | former_clone_of = clone_of->decl; | |
1115 | if (clone_of->former_clone_of) | |
1116 | former_clone_of = clone_of->former_clone_of; | |
3dafb85c | 1117 | if (symtab->dump_file) |
564fe867 | 1118 | { |
0e590b68 JH |
1119 | fprintf (symtab->dump_file, "cloning %s to %s\n", |
1120 | clone_of->dump_name (), | |
1121 | dump_name ()); | |
25a0d08b | 1122 | if (info && info->tree_map) |
564fe867 | 1123 | { |
0e590b68 JH |
1124 | fprintf (symtab->dump_file, " replace map:"); |
1125 | for (unsigned int i = 0; | |
ae7a23a3 | 1126 | i < vec_safe_length (info->tree_map); |
0e590b68 | 1127 | i++) |
564fe867 | 1128 | { |
0e590b68 | 1129 | ipa_replace_map *replace_info; |
ae7a23a3 | 1130 | replace_info = (*info->tree_map)[i]; |
0e590b68 JH |
1131 | fprintf (symtab->dump_file, "%s %i -> ", |
1132 | i ? "," : "", replace_info->parm_num); | |
1133 | print_generic_expr (symtab->dump_file, | |
1134 | replace_info->new_tree); | |
564fe867 | 1135 | } |
0e590b68 | 1136 | fprintf (symtab->dump_file, "\n"); |
564fe867 | 1137 | } |
25a0d08b | 1138 | if (info && info->param_adjustments) |
ae7a23a3 | 1139 | info->param_adjustments->dump (symtab->dump_file); |
564fe867 | 1140 | } |
783dc02d | 1141 | clear_stmts_in_references (); |
0e590b68 JH |
1142 | /* Copy the OLD_VERSION_NODE function tree to the new version. */ |
1143 | tree_function_versioning (clone_of->decl, decl, | |
ae7a23a3 JH |
1144 | info ? info->tree_map : NULL, |
1145 | info ? info->param_adjustments : NULL, | |
0e590b68 | 1146 | true, NULL, NULL); |
3dafb85c | 1147 | if (symtab->dump_file) |
0e590b68 JH |
1148 | { |
1149 | dump_function_to_file (clone_of->decl, symtab->dump_file, | |
1150 | dump_flags); | |
1151 | dump_function_to_file (decl, symtab->dump_file, dump_flags); | |
1152 | } | |
b2b29377 | 1153 | |
0e590b68 JH |
1154 | cgraph_node *this_clone_of = clone_of; |
1155 | /* Function is no longer clone. */ | |
1156 | remove_from_clone_tree (); | |
1157 | if (!this_clone_of->analyzed && !this_clone_of->clones) | |
d7145b4b | 1158 | this_clone_of->release_body (); |
564fe867 JH |
1159 | } |
1160 | ||
1161 | #include "gt-cgraphclones.h" |