]>
Commit | Line | Data |
---|---|---|
99c67f24 | 1 | /* Inlining decision heuristics. |
711789cc | 2 | Copyright (C) 2003-2013 Free Software Foundation, Inc. |
99c67f24 | 3 | Contributed by Jan Hubicka |
4 | ||
5 | This file is part of GCC. | |
6 | ||
7 | GCC is free software; you can redistribute it and/or modify it under | |
8 | the terms of the GNU General Public License as published by the Free | |
9 | Software Foundation; either version 3, or (at your option) any later | |
10 | version. | |
11 | ||
12 | GCC is distributed in the hope that it will be useful, but WITHOUT ANY | |
13 | WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
14 | FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
15 | for more details. | |
16 | ||
17 | You should have received a copy of the GNU General Public License | |
18 | along with GCC; see the file COPYING3. If not see | |
19 | <http://www.gnu.org/licenses/>. */ | |
20 | ||
21 | /* Analysis used by the inliner and other passes limiting code size growth. | |
22 | ||
23 | We estimate for each function | |
24 | - function body size | |
c7b2cc59 | 25 | - average function execution time |
99c67f24 | 26 | - inlining size benefit (that is how much of function body size |
27 | and its call sequence is expected to disappear by inlining) | |
28 | - inlining time benefit | |
29 | - function frame size | |
30 | For each call | |
c7b2cc59 | 31 | - call statement size and time |
99c67f24 | 32 | |
33 | inlinie_summary datastructures store above information locally (i.e. | |
34 | parameters of the function itself) and globally (i.e. parameters of | |
35 | the function created by applying all the inline decisions already | |
36 | present in the callgraph). | |
37 | ||
a41f2a28 | 38 | We provide accestor to the inline_summary datastructure and |
99c67f24 | 39 | basic logic updating the parameters when inlining is performed. |
40 | ||
a41f2a28 | 41 | The summaries are context sensitive. Context means |
42 | 1) partial assignment of known constant values of operands | |
43 | 2) whether function is inlined into the call or not. | |
44 | It is easy to add more variants. To represent function size and time | |
45 | that depends on context (i.e. it is known to be optimized away when | |
46 | context is known either by inlining or from IP-CP and clonning), | |
47 | we use predicates. Predicates are logical formulas in | |
48 | conjunctive-disjunctive form consisting of clauses. Clauses are bitmaps | |
49 | specifying what conditions must be true. Conditions are simple test | |
50 | of the form described above. | |
51 | ||
52 | In order to make predicate (possibly) true, all of its clauses must | |
53 | be (possibly) true. To make clause (possibly) true, one of conditions | |
54 | it mentions must be (possibly) true. There are fixed bounds on | |
55 | number of clauses and conditions and all the manipulation functions | |
56 | are conservative in positive direction. I.e. we may lose precision | |
57 | by thinking that predicate may be true even when it is not. | |
58 | ||
59 | estimate_edge_size and estimate_edge_growth can be used to query | |
60 | function size/time in the given context. inline_merge_summary merges | |
61 | properties of caller and callee after inlining. | |
62 | ||
99c67f24 | 63 | Finally pass_inline_parameters is exported. This is used to drive |
64 | computation of function parameters used by the early inliner. IPA | |
65 | inlined performs analysis via its analyze_function method. */ | |
66 | ||
67 | #include "config.h" | |
68 | #include "system.h" | |
69 | #include "coretypes.h" | |
70 | #include "tm.h" | |
71 | #include "tree.h" | |
9ed99284 | 72 | #include "stor-layout.h" |
73 | #include "stringpool.h" | |
74 | #include "print-tree.h" | |
99c67f24 | 75 | #include "tree-inline.h" |
76 | #include "langhooks.h" | |
77 | #include "flags.h" | |
99c67f24 | 78 | #include "diagnostic.h" |
79 | #include "gimple-pretty-print.h" | |
99c67f24 | 80 | #include "params.h" |
81 | #include "tree-pass.h" | |
82 | #include "coverage.h" | |
83 | #include "ggc.h" | |
073c1fd5 | 84 | #include "gimple.h" |
dcf1a1ec | 85 | #include "gimple-iterator.h" |
073c1fd5 | 86 | #include "gimple-ssa.h" |
87 | #include "tree-cfg.h" | |
88 | #include "tree-phinodes.h" | |
89 | #include "ssa-iterators.h" | |
90 | #include "tree-ssanames.h" | |
05d9c18a | 91 | #include "tree-ssa-loop-niter.h" |
073c1fd5 | 92 | #include "tree-ssa-loop.h" |
99c67f24 | 93 | #include "ipa-prop.h" |
c7b2cc59 | 94 | #include "lto-streamer.h" |
2541503d | 95 | #include "data-streamer.h" |
96 | #include "tree-streamer.h" | |
99c67f24 | 97 | #include "ipa-inline.h" |
6a18c0be | 98 | #include "alloc-pool.h" |
6b42039a | 99 | #include "cfgloop.h" |
7c07aa3d | 100 | #include "tree-scalar-evolution.h" |
6eaf903b | 101 | #include "ipa-utils.h" |
d037099f | 102 | #include "cilk.h" |
e797f49f | 103 | #include "cfgexpand.h" |
99c67f24 | 104 | |
a41f2a28 | 105 | /* Estimate runtime of function can easilly run into huge numbers with many |
5aebd70d | 106 | nested loops. Be sure we can compute time * INLINE_SIZE_SCALE * 2 in an |
107 | integer. For anything larger we use gcov_type. */ | |
39273ba1 | 108 | #define MAX_TIME 500000 |
a41f2a28 | 109 | |
110 | /* Number of bits in integer, but we really want to be stable across different | |
111 | hosts. */ | |
112 | #define NUM_CONDITIONS 32 | |
113 | ||
114 | enum predicate_conditions | |
115 | { | |
116 | predicate_false_condition = 0, | |
117 | predicate_not_inlined_condition = 1, | |
118 | predicate_first_dynamic_condition = 2 | |
119 | }; | |
120 | ||
121 | /* Special condition code we use to represent test that operand is compile time | |
122 | constant. */ | |
123 | #define IS_NOT_CONSTANT ERROR_MARK | |
eb4ae064 | 124 | /* Special condition code we use to represent test that operand is not changed |
125 | across invocation of the function. When operand IS_NOT_CONSTANT it is always | |
126 | CHANGED, however i.e. loop invariants can be NOT_CHANGED given percentage | |
127 | of executions even when they are not compile time constants. */ | |
128 | #define CHANGED IDENTIFIER_NODE | |
99c67f24 | 129 | |
130 | /* Holders of ipa cgraph hooks: */ | |
131 | static struct cgraph_node_hook_list *function_insertion_hook_holder; | |
c7b2cc59 | 132 | static struct cgraph_node_hook_list *node_removal_hook_holder; |
133 | static struct cgraph_2node_hook_list *node_duplication_hook_holder; | |
0835ad03 | 134 | static struct cgraph_2edge_hook_list *edge_duplication_hook_holder; |
a41f2a28 | 135 | static struct cgraph_edge_hook_list *edge_removal_hook_holder; |
c7b2cc59 | 136 | static void inline_node_removal_hook (struct cgraph_node *, void *); |
137 | static void inline_node_duplication_hook (struct cgraph_node *, | |
138 | struct cgraph_node *, void *); | |
0835ad03 | 139 | static void inline_edge_removal_hook (struct cgraph_edge *, void *); |
140 | static void inline_edge_duplication_hook (struct cgraph_edge *, | |
e876d531 | 141 | struct cgraph_edge *, void *); |
c7b2cc59 | 142 | |
a41f2a28 | 143 | /* VECtor holding inline summaries. |
144 | In GGC memory because conditions might point to constant trees. */ | |
f1f41a6c | 145 | vec<inline_summary_t, va_gc> *inline_summary_vec; |
146 | vec<inline_edge_summary_t> inline_edge_summary_vec; | |
a41f2a28 | 147 | |
148 | /* Cached node/edge growths. */ | |
f1f41a6c | 149 | vec<int> node_growth_cache; |
150 | vec<edge_growth_cache_entry> edge_growth_cache; | |
a41f2a28 | 151 | |
6a18c0be | 152 | /* Edge predicates goes here. */ |
153 | static alloc_pool edge_predicate_pool; | |
a41f2a28 | 154 | |
155 | /* Return true predicate (tautology). | |
156 | We represent it by empty list of clauses. */ | |
157 | ||
158 | static inline struct predicate | |
159 | true_predicate (void) | |
160 | { | |
161 | struct predicate p; | |
563fae60 | 162 | p.clause[0] = 0; |
a41f2a28 | 163 | return p; |
164 | } | |
165 | ||
166 | ||
167 | /* Return predicate testing single condition number COND. */ | |
168 | ||
169 | static inline struct predicate | |
170 | single_cond_predicate (int cond) | |
171 | { | |
172 | struct predicate p; | |
563fae60 | 173 | p.clause[0] = 1 << cond; |
174 | p.clause[1] = 0; | |
a41f2a28 | 175 | return p; |
176 | } | |
177 | ||
178 | ||
179 | /* Return false predicate. First clause require false condition. */ | |
180 | ||
181 | static inline struct predicate | |
182 | false_predicate (void) | |
183 | { | |
184 | return single_cond_predicate (predicate_false_condition); | |
185 | } | |
186 | ||
187 | ||
6a18c0be | 188 | /* Return true if P is (false). */ |
189 | ||
190 | static inline bool | |
191 | true_predicate_p (struct predicate *p) | |
192 | { | |
193 | return !p->clause[0]; | |
194 | } | |
195 | ||
196 | ||
197 | /* Return true if P is (false). */ | |
198 | ||
199 | static inline bool | |
200 | false_predicate_p (struct predicate *p) | |
201 | { | |
202 | if (p->clause[0] == (1 << predicate_false_condition)) | |
203 | { | |
204 | gcc_checking_assert (!p->clause[1] | |
205 | && p->clause[0] == 1 << predicate_false_condition); | |
206 | return true; | |
207 | } | |
208 | return false; | |
209 | } | |
210 | ||
211 | ||
a41f2a28 | 212 | /* Return predicate that is set true when function is not inlined. */ |
e876d531 | 213 | |
a41f2a28 | 214 | static inline struct predicate |
215 | not_inlined_predicate (void) | |
216 | { | |
217 | return single_cond_predicate (predicate_not_inlined_condition); | |
218 | } | |
219 | ||
a4f60e55 | 220 | /* Simple description of whether a memory load or a condition refers to a load |
221 | from an aggregate and if so, how and where from in the aggregate. | |
222 | Individual fields have the same meaning like fields with the same name in | |
223 | struct condition. */ | |
a41f2a28 | 224 | |
a4f60e55 | 225 | struct agg_position_info |
226 | { | |
227 | HOST_WIDE_INT offset; | |
228 | bool agg_contents; | |
229 | bool by_ref; | |
230 | }; | |
231 | ||
232 | /* Add condition to condition list CONDS. AGGPOS describes whether the used | |
233 | oprand is loaded from an aggregate and where in the aggregate it is. It can | |
234 | be NULL, which means this not a load from an aggregate. */ | |
a41f2a28 | 235 | |
236 | static struct predicate | |
237 | add_condition (struct inline_summary *summary, int operand_num, | |
a4f60e55 | 238 | struct agg_position_info *aggpos, |
a41f2a28 | 239 | enum tree_code code, tree val) |
240 | { | |
241 | int i; | |
242 | struct condition *c; | |
243 | struct condition new_cond; | |
a4f60e55 | 244 | HOST_WIDE_INT offset; |
245 | bool agg_contents, by_ref; | |
a41f2a28 | 246 | |
a4f60e55 | 247 | if (aggpos) |
248 | { | |
249 | offset = aggpos->offset; | |
250 | agg_contents = aggpos->agg_contents; | |
251 | by_ref = aggpos->by_ref; | |
252 | } | |
253 | else | |
254 | { | |
255 | offset = 0; | |
256 | agg_contents = false; | |
257 | by_ref = false; | |
258 | } | |
259 | ||
260 | gcc_checking_assert (operand_num >= 0); | |
f1f41a6c | 261 | for (i = 0; vec_safe_iterate (summary->conds, i, &c); i++) |
a41f2a28 | 262 | { |
263 | if (c->operand_num == operand_num | |
264 | && c->code == code | |
a4f60e55 | 265 | && c->val == val |
266 | && c->agg_contents == agg_contents | |
267 | && (!agg_contents || (c->offset == offset && c->by_ref == by_ref))) | |
e876d531 | 268 | return single_cond_predicate (i + predicate_first_dynamic_condition); |
a41f2a28 | 269 | } |
270 | /* Too many conditions. Give up and return constant true. */ | |
271 | if (i == NUM_CONDITIONS - predicate_first_dynamic_condition) | |
272 | return true_predicate (); | |
273 | ||
274 | new_cond.operand_num = operand_num; | |
275 | new_cond.code = code; | |
276 | new_cond.val = val; | |
a4f60e55 | 277 | new_cond.agg_contents = agg_contents; |
278 | new_cond.by_ref = by_ref; | |
279 | new_cond.offset = offset; | |
f1f41a6c | 280 | vec_safe_push (summary->conds, new_cond); |
a41f2a28 | 281 | return single_cond_predicate (i + predicate_first_dynamic_condition); |
282 | } | |
283 | ||
284 | ||
905aa3bd | 285 | /* Add clause CLAUSE into the predicate P. */ |
a41f2a28 | 286 | |
287 | static inline void | |
94646c9c | 288 | add_clause (conditions conditions, struct predicate *p, clause_t clause) |
a41f2a28 | 289 | { |
290 | int i; | |
905aa3bd | 291 | int i2; |
5cb1b112 | 292 | int insert_here = -1; |
94646c9c | 293 | int c1, c2; |
6a18c0be | 294 | |
a41f2a28 | 295 | /* True clause. */ |
296 | if (!clause) | |
297 | return; | |
298 | ||
905aa3bd | 299 | /* False clause makes the whole predicate false. Kill the other variants. */ |
6a18c0be | 300 | if (clause == (1 << predicate_false_condition)) |
a41f2a28 | 301 | { |
302 | p->clause[0] = (1 << predicate_false_condition); | |
303 | p->clause[1] = 0; | |
6a18c0be | 304 | return; |
a41f2a28 | 305 | } |
6a18c0be | 306 | if (false_predicate_p (p)) |
307 | return; | |
905aa3bd | 308 | |
309 | /* No one should be sily enough to add false into nontrivial clauses. */ | |
310 | gcc_checking_assert (!(clause & (1 << predicate_false_condition))); | |
311 | ||
312 | /* Look where to insert the clause. At the same time prune out | |
313 | clauses of P that are implied by the new clause and thus | |
314 | redundant. */ | |
315 | for (i = 0, i2 = 0; i <= MAX_CLAUSES; i++) | |
a41f2a28 | 316 | { |
905aa3bd | 317 | p->clause[i2] = p->clause[i]; |
318 | ||
a41f2a28 | 319 | if (!p->clause[i]) |
320 | break; | |
905aa3bd | 321 | |
322 | /* If p->clause[i] implies clause, there is nothing to add. */ | |
323 | if ((p->clause[i] & clause) == p->clause[i]) | |
324 | { | |
fb3c587e | 325 | /* We had nothing to add, none of clauses should've become |
326 | redundant. */ | |
905aa3bd | 327 | gcc_checking_assert (i == i2); |
328 | return; | |
329 | } | |
330 | ||
331 | if (p->clause[i] < clause && insert_here < 0) | |
332 | insert_here = i2; | |
333 | ||
334 | /* If clause implies p->clause[i], then p->clause[i] becomes redundant. | |
e876d531 | 335 | Otherwise the p->clause[i] has to stay. */ |
905aa3bd | 336 | if ((p->clause[i] & clause) != clause) |
337 | i2++; | |
a41f2a28 | 338 | } |
94646c9c | 339 | |
340 | /* Look for clauses that are obviously true. I.e. | |
341 | op0 == 5 || op0 != 5. */ | |
342 | for (c1 = predicate_first_dynamic_condition; c1 < NUM_CONDITIONS; c1++) | |
eb4ae064 | 343 | { |
344 | condition *cc1; | |
345 | if (!(clause & (1 << c1))) | |
346 | continue; | |
f1f41a6c | 347 | cc1 = &(*conditions)[c1 - predicate_first_dynamic_condition]; |
eb4ae064 | 348 | /* We have no way to represent !CHANGED and !IS_NOT_CONSTANT |
e876d531 | 349 | and thus there is no point for looking for them. */ |
350 | if (cc1->code == CHANGED || cc1->code == IS_NOT_CONSTANT) | |
eb4ae064 | 351 | continue; |
8249d427 | 352 | for (c2 = c1 + 1; c2 < NUM_CONDITIONS; c2++) |
eb4ae064 | 353 | if (clause & (1 << c2)) |
354 | { | |
e876d531 | 355 | condition *cc1 = |
356 | &(*conditions)[c1 - predicate_first_dynamic_condition]; | |
357 | condition *cc2 = | |
358 | &(*conditions)[c2 - predicate_first_dynamic_condition]; | |
eb4ae064 | 359 | if (cc1->operand_num == cc2->operand_num |
360 | && cc1->val == cc2->val | |
361 | && cc2->code != IS_NOT_CONSTANT | |
362 | && cc2->code != CHANGED | |
e876d531 | 363 | && cc1->code == invert_tree_comparison |
364 | (cc2->code, | |
365 | HONOR_NANS (TYPE_MODE (TREE_TYPE (cc1->val))))) | |
eb4ae064 | 366 | return; |
367 | } | |
368 | } | |
e876d531 | 369 | |
94646c9c | 370 | |
905aa3bd | 371 | /* We run out of variants. Be conservative in positive direction. */ |
372 | if (i2 == MAX_CLAUSES) | |
a41f2a28 | 373 | return; |
905aa3bd | 374 | /* Keep clauses in decreasing order. This makes equivalence testing easy. */ |
375 | p->clause[i2 + 1] = 0; | |
5cb1b112 | 376 | if (insert_here >= 0) |
e876d531 | 377 | for (; i2 > insert_here; i2--) |
905aa3bd | 378 | p->clause[i2] = p->clause[i2 - 1]; |
5cb1b112 | 379 | else |
905aa3bd | 380 | insert_here = i2; |
a41f2a28 | 381 | p->clause[insert_here] = clause; |
382 | } | |
383 | ||
384 | ||
385 | /* Return P & P2. */ | |
386 | ||
387 | static struct predicate | |
94646c9c | 388 | and_predicates (conditions conditions, |
389 | struct predicate *p, struct predicate *p2) | |
a41f2a28 | 390 | { |
391 | struct predicate out = *p; | |
392 | int i; | |
6a18c0be | 393 | |
905aa3bd | 394 | /* Avoid busy work. */ |
395 | if (false_predicate_p (p2) || true_predicate_p (p)) | |
396 | return *p2; | |
397 | if (false_predicate_p (p) || true_predicate_p (p2)) | |
398 | return *p; | |
399 | ||
400 | /* See how far predicates match. */ | |
401 | for (i = 0; p->clause[i] && p->clause[i] == p2->clause[i]; i++) | |
402 | { | |
403 | gcc_checking_assert (i < MAX_CLAUSES); | |
404 | } | |
e876d531 | 405 | |
905aa3bd | 406 | /* Combine the predicates rest. */ |
407 | for (; p2->clause[i]; i++) | |
5cb1b112 | 408 | { |
409 | gcc_checking_assert (i < MAX_CLAUSES); | |
94646c9c | 410 | add_clause (conditions, &out, p2->clause[i]); |
5cb1b112 | 411 | } |
a41f2a28 | 412 | return out; |
413 | } | |
414 | ||
415 | ||
905aa3bd | 416 | /* Return true if predicates are obviously equal. */ |
417 | ||
418 | static inline bool | |
419 | predicates_equal_p (struct predicate *p, struct predicate *p2) | |
420 | { | |
421 | int i; | |
422 | for (i = 0; p->clause[i]; i++) | |
423 | { | |
424 | gcc_checking_assert (i < MAX_CLAUSES); | |
e876d531 | 425 | gcc_checking_assert (p->clause[i] > p->clause[i + 1]); |
fb3c587e | 426 | gcc_checking_assert (!p2->clause[i] |
e876d531 | 427 | || p2->clause[i] > p2->clause[i + 1]); |
905aa3bd | 428 | if (p->clause[i] != p2->clause[i]) |
e876d531 | 429 | return false; |
905aa3bd | 430 | } |
431 | return !p2->clause[i]; | |
432 | } | |
433 | ||
434 | ||
a41f2a28 | 435 | /* Return P | P2. */ |
436 | ||
437 | static struct predicate | |
e876d531 | 438 | or_predicates (conditions conditions, |
439 | struct predicate *p, struct predicate *p2) | |
a41f2a28 | 440 | { |
441 | struct predicate out = true_predicate (); | |
e876d531 | 442 | int i, j; |
6a18c0be | 443 | |
905aa3bd | 444 | /* Avoid busy work. */ |
445 | if (false_predicate_p (p2) || true_predicate_p (p)) | |
6a18c0be | 446 | return *p; |
905aa3bd | 447 | if (false_predicate_p (p) || true_predicate_p (p2)) |
6a18c0be | 448 | return *p2; |
905aa3bd | 449 | if (predicates_equal_p (p, p2)) |
450 | return *p; | |
451 | ||
452 | /* OK, combine the predicates. */ | |
a41f2a28 | 453 | for (i = 0; p->clause[i]; i++) |
454 | for (j = 0; p2->clause[j]; j++) | |
5cb1b112 | 455 | { |
e876d531 | 456 | gcc_checking_assert (i < MAX_CLAUSES && j < MAX_CLAUSES); |
457 | add_clause (conditions, &out, p->clause[i] | p2->clause[j]); | |
5cb1b112 | 458 | } |
a41f2a28 | 459 | return out; |
460 | } | |
461 | ||
462 | ||
fb3c587e | 463 | /* Having partial truth assignment in POSSIBLE_TRUTHS, return false |
464 | if predicate P is known to be false. */ | |
a41f2a28 | 465 | |
466 | static bool | |
6a18c0be | 467 | evaluate_predicate (struct predicate *p, clause_t possible_truths) |
a41f2a28 | 468 | { |
469 | int i; | |
470 | ||
471 | /* True remains true. */ | |
6a18c0be | 472 | if (true_predicate_p (p)) |
a41f2a28 | 473 | return true; |
474 | ||
6a18c0be | 475 | gcc_assert (!(possible_truths & (1 << predicate_false_condition))); |
476 | ||
a41f2a28 | 477 | /* See if we can find clause we can disprove. */ |
478 | for (i = 0; p->clause[i]; i++) | |
5cb1b112 | 479 | { |
480 | gcc_checking_assert (i < MAX_CLAUSES); | |
481 | if (!(p->clause[i] & possible_truths)) | |
e876d531 | 482 | return false; |
5cb1b112 | 483 | } |
a41f2a28 | 484 | return true; |
485 | } | |
486 | ||
eb4ae064 | 487 | /* Return the probability in range 0...REG_BR_PROB_BASE that the predicated |
488 | instruction will be recomputed per invocation of the inlined call. */ | |
489 | ||
490 | static int | |
491 | predicate_probability (conditions conds, | |
492 | struct predicate *p, clause_t possible_truths, | |
f1f41a6c | 493 | vec<inline_param_summary_t> inline_param_summary) |
eb4ae064 | 494 | { |
495 | int i; | |
496 | int combined_prob = REG_BR_PROB_BASE; | |
497 | ||
498 | /* True remains true. */ | |
499 | if (true_predicate_p (p)) | |
500 | return REG_BR_PROB_BASE; | |
501 | ||
502 | if (false_predicate_p (p)) | |
503 | return 0; | |
504 | ||
505 | gcc_assert (!(possible_truths & (1 << predicate_false_condition))); | |
506 | ||
507 | /* See if we can find clause we can disprove. */ | |
508 | for (i = 0; p->clause[i]; i++) | |
509 | { | |
510 | gcc_checking_assert (i < MAX_CLAUSES); | |
511 | if (!(p->clause[i] & possible_truths)) | |
512 | return 0; | |
513 | else | |
514 | { | |
515 | int this_prob = 0; | |
516 | int i2; | |
f1f41a6c | 517 | if (!inline_param_summary.exists ()) |
eb4ae064 | 518 | return REG_BR_PROB_BASE; |
519 | for (i2 = 0; i2 < NUM_CONDITIONS; i2++) | |
520 | if ((p->clause[i] & possible_truths) & (1 << i2)) | |
521 | { | |
522 | if (i2 >= predicate_first_dynamic_condition) | |
523 | { | |
e876d531 | 524 | condition *c = |
525 | &(*conds)[i2 - predicate_first_dynamic_condition]; | |
eb4ae064 | 526 | if (c->code == CHANGED |
e876d531 | 527 | && (c->operand_num < |
528 | (int) inline_param_summary.length ())) | |
eb4ae064 | 529 | { |
e876d531 | 530 | int iprob = |
531 | inline_param_summary[c->operand_num].change_prob; | |
eb4ae064 | 532 | this_prob = MAX (this_prob, iprob); |
533 | } | |
534 | else | |
535 | this_prob = REG_BR_PROB_BASE; | |
e876d531 | 536 | } |
537 | else | |
538 | this_prob = REG_BR_PROB_BASE; | |
eb4ae064 | 539 | } |
540 | combined_prob = MIN (this_prob, combined_prob); | |
541 | if (!combined_prob) | |
e876d531 | 542 | return 0; |
eb4ae064 | 543 | } |
544 | } | |
545 | return combined_prob; | |
546 | } | |
547 | ||
a41f2a28 | 548 | |
549 | /* Dump conditional COND. */ | |
550 | ||
551 | static void | |
552 | dump_condition (FILE *f, conditions conditions, int cond) | |
553 | { | |
554 | condition *c; | |
555 | if (cond == predicate_false_condition) | |
556 | fprintf (f, "false"); | |
557 | else if (cond == predicate_not_inlined_condition) | |
558 | fprintf (f, "not inlined"); | |
559 | else | |
560 | { | |
f1f41a6c | 561 | c = &(*conditions)[cond - predicate_first_dynamic_condition]; |
a41f2a28 | 562 | fprintf (f, "op%i", c->operand_num); |
a4f60e55 | 563 | if (c->agg_contents) |
564 | fprintf (f, "[%soffset: " HOST_WIDE_INT_PRINT_DEC "]", | |
565 | c->by_ref ? "ref " : "", c->offset); | |
a41f2a28 | 566 | if (c->code == IS_NOT_CONSTANT) |
567 | { | |
568 | fprintf (f, " not constant"); | |
569 | return; | |
570 | } | |
eb4ae064 | 571 | if (c->code == CHANGED) |
572 | { | |
573 | fprintf (f, " changed"); | |
574 | return; | |
575 | } | |
a41f2a28 | 576 | fprintf (f, " %s ", op_symbol_code (c->code)); |
577 | print_generic_expr (f, c->val, 1); | |
578 | } | |
579 | } | |
580 | ||
581 | ||
582 | /* Dump clause CLAUSE. */ | |
583 | ||
584 | static void | |
585 | dump_clause (FILE *f, conditions conds, clause_t clause) | |
586 | { | |
587 | int i; | |
588 | bool found = false; | |
589 | fprintf (f, "("); | |
590 | if (!clause) | |
591 | fprintf (f, "true"); | |
592 | for (i = 0; i < NUM_CONDITIONS; i++) | |
593 | if (clause & (1 << i)) | |
594 | { | |
595 | if (found) | |
596 | fprintf (f, " || "); | |
597 | found = true; | |
e876d531 | 598 | dump_condition (f, conds, i); |
a41f2a28 | 599 | } |
600 | fprintf (f, ")"); | |
601 | } | |
602 | ||
603 | ||
604 | /* Dump predicate PREDICATE. */ | |
605 | ||
606 | static void | |
607 | dump_predicate (FILE *f, conditions conds, struct predicate *pred) | |
608 | { | |
609 | int i; | |
6a18c0be | 610 | if (true_predicate_p (pred)) |
a41f2a28 | 611 | dump_clause (f, conds, 0); |
612 | else | |
613 | for (i = 0; pred->clause[i]; i++) | |
614 | { | |
615 | if (i) | |
616 | fprintf (f, " && "); | |
e876d531 | 617 | dump_clause (f, conds, pred->clause[i]); |
a41f2a28 | 618 | } |
619 | fprintf (f, "\n"); | |
620 | } | |
621 | ||
622 | ||
eb7c606e | 623 | /* Dump inline hints. */ |
624 | void | |
625 | dump_inline_hints (FILE *f, inline_hints hints) | |
626 | { | |
627 | if (!hints) | |
628 | return; | |
629 | fprintf (f, "inline hints:"); | |
630 | if (hints & INLINE_HINT_indirect_call) | |
631 | { | |
632 | hints &= ~INLINE_HINT_indirect_call; | |
633 | fprintf (f, " indirect_call"); | |
634 | } | |
7c07aa3d | 635 | if (hints & INLINE_HINT_loop_iterations) |
636 | { | |
637 | hints &= ~INLINE_HINT_loop_iterations; | |
638 | fprintf (f, " loop_iterations"); | |
639 | } | |
3716ee8f | 640 | if (hints & INLINE_HINT_loop_stride) |
641 | { | |
642 | hints &= ~INLINE_HINT_loop_stride; | |
643 | fprintf (f, " loop_stride"); | |
644 | } | |
41d39f38 | 645 | if (hints & INLINE_HINT_same_scc) |
646 | { | |
647 | hints &= ~INLINE_HINT_same_scc; | |
648 | fprintf (f, " same_scc"); | |
649 | } | |
650 | if (hints & INLINE_HINT_in_scc) | |
651 | { | |
652 | hints &= ~INLINE_HINT_in_scc; | |
653 | fprintf (f, " in_scc"); | |
654 | } | |
3172b7bf | 655 | if (hints & INLINE_HINT_cross_module) |
656 | { | |
657 | hints &= ~INLINE_HINT_cross_module; | |
658 | fprintf (f, " cross_module"); | |
659 | } | |
660 | if (hints & INLINE_HINT_declared_inline) | |
661 | { | |
662 | hints &= ~INLINE_HINT_declared_inline; | |
663 | fprintf (f, " declared_inline"); | |
664 | } | |
be343a9c | 665 | if (hints & INLINE_HINT_array_index) |
666 | { | |
667 | hints &= ~INLINE_HINT_array_index; | |
668 | fprintf (f, " array_index"); | |
669 | } | |
eb7c606e | 670 | gcc_assert (!hints); |
671 | } | |
672 | ||
673 | ||
a41f2a28 | 674 | /* Record SIZE and TIME under condition PRED into the inline summary. */ |
675 | ||
676 | static void | |
fb3c587e | 677 | account_size_time (struct inline_summary *summary, int size, int time, |
678 | struct predicate *pred) | |
a41f2a28 | 679 | { |
680 | size_time_entry *e; | |
681 | bool found = false; | |
682 | int i; | |
683 | ||
6a18c0be | 684 | if (false_predicate_p (pred)) |
a41f2a28 | 685 | return; |
686 | ||
687 | /* We need to create initial empty unconitional clause, but otherwie | |
688 | we don't need to account empty times and sizes. */ | |
8bae3ea4 | 689 | if (!size && !time && summary->entry) |
a41f2a28 | 690 | return; |
691 | ||
692 | /* Watch overflow that might result from insane profiles. */ | |
693 | if (time > MAX_TIME * INLINE_TIME_SCALE) | |
694 | time = MAX_TIME * INLINE_TIME_SCALE; | |
695 | gcc_assert (time >= 0); | |
696 | ||
f1f41a6c | 697 | for (i = 0; vec_safe_iterate (summary->entry, i, &e); i++) |
a41f2a28 | 698 | if (predicates_equal_p (&e->predicate, pred)) |
699 | { | |
700 | found = true; | |
e876d531 | 701 | break; |
a41f2a28 | 702 | } |
563fae60 | 703 | if (i == 256) |
a41f2a28 | 704 | { |
705 | i = 0; | |
706 | found = true; | |
f1f41a6c | 707 | e = &(*summary->entry)[0]; |
a41f2a28 | 708 | gcc_assert (!e->predicate.clause[0]); |
563fae60 | 709 | if (dump_file && (dump_flags & TDF_DETAILS)) |
e876d531 | 710 | fprintf (dump_file, |
711 | "\t\tReached limit on number of entries, " | |
712 | "ignoring the predicate."); | |
a41f2a28 | 713 | } |
714 | if (dump_file && (dump_flags & TDF_DETAILS) && (time || size)) | |
715 | { | |
e876d531 | 716 | fprintf (dump_file, |
717 | "\t\tAccounting size:%3.2f, time:%3.2f on %spredicate:", | |
718 | ((double) size) / INLINE_SIZE_SCALE, | |
719 | ((double) time) / INLINE_TIME_SCALE, found ? "" : "new "); | |
a41f2a28 | 720 | dump_predicate (dump_file, summary->conds, pred); |
721 | } | |
722 | if (!found) | |
723 | { | |
724 | struct size_time_entry new_entry; | |
725 | new_entry.size = size; | |
726 | new_entry.time = time; | |
727 | new_entry.predicate = *pred; | |
f1f41a6c | 728 | vec_safe_push (summary->entry, new_entry); |
a41f2a28 | 729 | } |
730 | else | |
731 | { | |
732 | e->size += size; | |
733 | e->time += time; | |
734 | if (e->time > MAX_TIME * INLINE_TIME_SCALE) | |
735 | e->time = MAX_TIME * INLINE_TIME_SCALE; | |
736 | } | |
737 | } | |
738 | ||
6a18c0be | 739 | /* Set predicate for edge E. */ |
740 | ||
741 | static void | |
742 | edge_set_predicate (struct cgraph_edge *e, struct predicate *predicate) | |
743 | { | |
744 | struct inline_edge_summary *es = inline_edge_summary (e); | |
745 | if (predicate && !true_predicate_p (predicate)) | |
746 | { | |
747 | if (!es->predicate) | |
e876d531 | 748 | es->predicate = (struct predicate *) pool_alloc (edge_predicate_pool); |
6a18c0be | 749 | *es->predicate = *predicate; |
750 | } | |
751 | else | |
752 | { | |
753 | if (es->predicate) | |
e876d531 | 754 | pool_free (edge_predicate_pool, es->predicate); |
6a18c0be | 755 | es->predicate = NULL; |
756 | } | |
757 | } | |
758 | ||
3716ee8f | 759 | /* Set predicate for hint *P. */ |
760 | ||
761 | static void | |
762 | set_hint_predicate (struct predicate **p, struct predicate new_predicate) | |
763 | { | |
e876d531 | 764 | if (false_predicate_p (&new_predicate) || true_predicate_p (&new_predicate)) |
3716ee8f | 765 | { |
766 | if (*p) | |
767 | pool_free (edge_predicate_pool, *p); | |
768 | *p = NULL; | |
769 | } | |
770 | else | |
771 | { | |
772 | if (!*p) | |
e876d531 | 773 | *p = (struct predicate *) pool_alloc (edge_predicate_pool); |
3716ee8f | 774 | **p = new_predicate; |
775 | } | |
776 | } | |
777 | ||
a41f2a28 | 778 | |
8bae3ea4 | 779 | /* KNOWN_VALS is partial mapping of parameters of NODE to constant values. |
a4f60e55 | 780 | KNOWN_AGGS is a vector of aggreggate jump functions for each parameter. |
781 | Return clause of possible truths. When INLINE_P is true, assume that we are | |
782 | inlining. | |
eb4ae064 | 783 | |
784 | ERROR_MARK means compile time invariant. */ | |
8bae3ea4 | 785 | |
786 | static clause_t | |
787 | evaluate_conditions_for_known_args (struct cgraph_node *node, | |
e876d531 | 788 | bool inline_p, |
789 | vec<tree> known_vals, | |
790 | vec<ipa_agg_jump_function_p> | |
791 | known_aggs) | |
8bae3ea4 | 792 | { |
793 | clause_t clause = inline_p ? 0 : 1 << predicate_not_inlined_condition; | |
794 | struct inline_summary *info = inline_summary (node); | |
795 | int i; | |
796 | struct condition *c; | |
797 | ||
f1f41a6c | 798 | for (i = 0; vec_safe_iterate (info->conds, i, &c); i++) |
8bae3ea4 | 799 | { |
92ceb887 | 800 | tree val; |
8bae3ea4 | 801 | tree res; |
802 | ||
a4f60e55 | 803 | /* We allow call stmt to have fewer arguments than the callee function |
e876d531 | 804 | (especially for K&R style programs). So bound check here (we assume |
805 | known_aggs vector, if non-NULL, has the same length as | |
806 | known_vals). */ | |
f1f41a6c | 807 | gcc_checking_assert (!known_aggs.exists () |
808 | || (known_vals.length () == known_aggs.length ())); | |
809 | if (c->operand_num >= (int) known_vals.length ()) | |
a4f60e55 | 810 | { |
811 | clause |= 1 << (i + predicate_first_dynamic_condition); | |
812 | continue; | |
813 | } | |
92ceb887 | 814 | |
a4f60e55 | 815 | if (c->agg_contents) |
816 | { | |
817 | struct ipa_agg_jump_function *agg; | |
818 | ||
819 | if (c->code == CHANGED | |
820 | && !c->by_ref | |
e876d531 | 821 | && (known_vals[c->operand_num] == error_mark_node)) |
a4f60e55 | 822 | continue; |
823 | ||
f1f41a6c | 824 | if (known_aggs.exists ()) |
a4f60e55 | 825 | { |
f1f41a6c | 826 | agg = known_aggs[c->operand_num]; |
a4f60e55 | 827 | val = ipa_find_agg_cst_for_param (agg, c->offset, c->by_ref); |
828 | } | |
829 | else | |
830 | val = NULL_TREE; | |
831 | } | |
832 | else | |
833 | { | |
f1f41a6c | 834 | val = known_vals[c->operand_num]; |
a4f60e55 | 835 | if (val == error_mark_node && c->code != CHANGED) |
836 | val = NULL_TREE; | |
837 | } | |
eb4ae064 | 838 | |
8bae3ea4 | 839 | if (!val) |
840 | { | |
841 | clause |= 1 << (i + predicate_first_dynamic_condition); | |
842 | continue; | |
843 | } | |
eb4ae064 | 844 | if (c->code == IS_NOT_CONSTANT || c->code == CHANGED) |
8bae3ea4 | 845 | continue; |
846 | res = fold_binary_to_constant (c->code, boolean_type_node, val, c->val); | |
e876d531 | 847 | if (res && integer_zerop (res)) |
8bae3ea4 | 848 | continue; |
849 | clause |= 1 << (i + predicate_first_dynamic_condition); | |
850 | } | |
851 | return clause; | |
852 | } | |
853 | ||
854 | ||
a41f2a28 | 855 | /* Work out what conditions might be true at invocation of E. */ |
856 | ||
20da2013 | 857 | static void |
858 | evaluate_properties_for_edge (struct cgraph_edge *e, bool inline_p, | |
e876d531 | 859 | clause_t *clause_ptr, |
860 | vec<tree> *known_vals_ptr, | |
861 | vec<tree> *known_binfos_ptr, | |
862 | vec<ipa_agg_jump_function_p> *known_aggs_ptr) | |
a41f2a28 | 863 | { |
e876d531 | 864 | struct cgraph_node *callee = |
865 | cgraph_function_or_thunk_node (e->callee, NULL); | |
82626cb0 | 866 | struct inline_summary *info = inline_summary (callee); |
1e094109 | 867 | vec<tree> known_vals = vNULL; |
868 | vec<ipa_agg_jump_function_p> known_aggs = vNULL; | |
a41f2a28 | 869 | |
20da2013 | 870 | if (clause_ptr) |
871 | *clause_ptr = inline_p ? 0 : 1 << predicate_not_inlined_condition; | |
872 | if (known_vals_ptr) | |
f1f41a6c | 873 | known_vals_ptr->create (0); |
20da2013 | 874 | if (known_binfos_ptr) |
f1f41a6c | 875 | known_binfos_ptr->create (0); |
20da2013 | 876 | |
f1f41a6c | 877 | if (ipa_node_params_vector.exists () |
e67e73bd | 878 | && !e->call_stmt_cannot_inline_p |
e876d531 | 879 | && ((clause_ptr && info->conds) || known_vals_ptr || known_binfos_ptr)) |
a41f2a28 | 880 | { |
881 | struct ipa_node_params *parms_info; | |
882 | struct ipa_edge_args *args = IPA_EDGE_REF (e); | |
eb4ae064 | 883 | struct inline_edge_summary *es = inline_edge_summary (e); |
a41f2a28 | 884 | int i, count = ipa_get_cs_argument_count (args); |
a41f2a28 | 885 | |
886 | if (e->caller->global.inlined_to) | |
e876d531 | 887 | parms_info = IPA_NODE_REF (e->caller->global.inlined_to); |
a41f2a28 | 888 | else |
e876d531 | 889 | parms_info = IPA_NODE_REF (e->caller); |
a41f2a28 | 890 | |
20da2013 | 891 | if (count && (info->conds || known_vals_ptr)) |
f1f41a6c | 892 | known_vals.safe_grow_cleared (count); |
a4f60e55 | 893 | if (count && (info->conds || known_aggs_ptr)) |
f1f41a6c | 894 | known_aggs.safe_grow_cleared (count); |
20da2013 | 895 | if (count && known_binfos_ptr) |
f1f41a6c | 896 | known_binfos_ptr->safe_grow_cleared (count); |
20da2013 | 897 | |
a41f2a28 | 898 | for (i = 0; i < count; i++) |
899 | { | |
a4f60e55 | 900 | struct ipa_jump_func *jf = ipa_get_ith_jump_func (args, i); |
901 | tree cst = ipa_value_from_jfunc (parms_info, jf); | |
93f713da | 902 | if (cst) |
20da2013 | 903 | { |
f1f41a6c | 904 | if (known_vals.exists () && TREE_CODE (cst) != TREE_BINFO) |
905 | known_vals[i] = cst; | |
e876d531 | 906 | else if (known_binfos_ptr != NULL |
907 | && TREE_CODE (cst) == TREE_BINFO) | |
f1f41a6c | 908 | (*known_binfos_ptr)[i] = cst; |
20da2013 | 909 | } |
f1f41a6c | 910 | else if (inline_p && !es->param[i].change_prob) |
911 | known_vals[i] = error_mark_node; | |
a4f60e55 | 912 | /* TODO: When IPA-CP starts propagating and merging aggregate jump |
913 | functions, use its knowledge of the caller too, just like the | |
914 | scalar case above. */ | |
f1f41a6c | 915 | known_aggs[i] = &jf->agg; |
a41f2a28 | 916 | } |
a41f2a28 | 917 | } |
a41f2a28 | 918 | |
e67e73bd | 919 | if (clause_ptr) |
920 | *clause_ptr = evaluate_conditions_for_known_args (callee, inline_p, | |
a4f60e55 | 921 | known_vals, known_aggs); |
e67e73bd | 922 | |
923 | if (known_vals_ptr) | |
924 | *known_vals_ptr = known_vals; | |
925 | else | |
f1f41a6c | 926 | known_vals.release (); |
a4f60e55 | 927 | |
928 | if (known_aggs_ptr) | |
929 | *known_aggs_ptr = known_aggs; | |
930 | else | |
f1f41a6c | 931 | known_aggs.release (); |
a41f2a28 | 932 | } |
933 | ||
c7b2cc59 | 934 | |
935 | /* Allocate the inline summary vector or resize it to cover all cgraph nodes. */ | |
936 | ||
937 | static void | |
938 | inline_summary_alloc (void) | |
939 | { | |
940 | if (!node_removal_hook_holder) | |
941 | node_removal_hook_holder = | |
942 | cgraph_add_node_removal_hook (&inline_node_removal_hook, NULL); | |
0835ad03 | 943 | if (!edge_removal_hook_holder) |
944 | edge_removal_hook_holder = | |
945 | cgraph_add_edge_removal_hook (&inline_edge_removal_hook, NULL); | |
c7b2cc59 | 946 | if (!node_duplication_hook_holder) |
947 | node_duplication_hook_holder = | |
948 | cgraph_add_node_duplication_hook (&inline_node_duplication_hook, NULL); | |
0835ad03 | 949 | if (!edge_duplication_hook_holder) |
950 | edge_duplication_hook_holder = | |
951 | cgraph_add_edge_duplication_hook (&inline_edge_duplication_hook, NULL); | |
c7b2cc59 | 952 | |
f1f41a6c | 953 | if (vec_safe_length (inline_summary_vec) <= (unsigned) cgraph_max_uid) |
954 | vec_safe_grow_cleared (inline_summary_vec, cgraph_max_uid + 1); | |
955 | if (inline_edge_summary_vec.length () <= (unsigned) cgraph_edge_max_uid) | |
956 | inline_edge_summary_vec.safe_grow_cleared (cgraph_edge_max_uid + 1); | |
6a18c0be | 957 | if (!edge_predicate_pool) |
fb3c587e | 958 | edge_predicate_pool = create_alloc_pool ("edge predicates", |
e876d531 | 959 | sizeof (struct predicate), 10); |
c7b2cc59 | 960 | } |
961 | ||
3b9dd281 | 962 | /* We are called multiple time for given function; clear |
963 | data from previous run so they are not cumulated. */ | |
964 | ||
965 | static void | |
966 | reset_inline_edge_summary (struct cgraph_edge *e) | |
967 | { | |
e876d531 | 968 | if (e->uid < (int) inline_edge_summary_vec.length ()) |
d10a25bb | 969 | { |
970 | struct inline_edge_summary *es = inline_edge_summary (e); | |
3b9dd281 | 971 | |
563fae60 | 972 | es->call_stmt_size = es->call_stmt_time = 0; |
d10a25bb | 973 | if (es->predicate) |
974 | pool_free (edge_predicate_pool, es->predicate); | |
975 | es->predicate = NULL; | |
f1f41a6c | 976 | es->param.release (); |
d10a25bb | 977 | } |
3b9dd281 | 978 | } |
979 | ||
980 | /* We are called multiple time for given function; clear | |
981 | data from previous run so they are not cumulated. */ | |
982 | ||
983 | static void | |
984 | reset_inline_summary (struct cgraph_node *node) | |
985 | { | |
986 | struct inline_summary *info = inline_summary (node); | |
987 | struct cgraph_edge *e; | |
988 | ||
989 | info->self_size = info->self_time = 0; | |
990 | info->estimated_stack_size = 0; | |
991 | info->estimated_self_stack_size = 0; | |
992 | info->stack_frame_offset = 0; | |
993 | info->size = 0; | |
994 | info->time = 0; | |
3172b7bf | 995 | info->growth = 0; |
41d39f38 | 996 | info->scc_no = 0; |
7c07aa3d | 997 | if (info->loop_iterations) |
998 | { | |
999 | pool_free (edge_predicate_pool, info->loop_iterations); | |
1000 | info->loop_iterations = NULL; | |
1001 | } | |
3716ee8f | 1002 | if (info->loop_stride) |
1003 | { | |
1004 | pool_free (edge_predicate_pool, info->loop_stride); | |
1005 | info->loop_stride = NULL; | |
1006 | } | |
be343a9c | 1007 | if (info->array_index) |
1008 | { | |
1009 | pool_free (edge_predicate_pool, info->array_index); | |
1010 | info->array_index = NULL; | |
1011 | } | |
f1f41a6c | 1012 | vec_free (info->conds); |
1013 | vec_free (info->entry); | |
3b9dd281 | 1014 | for (e = node->callees; e; e = e->next_callee) |
1015 | reset_inline_edge_summary (e); | |
1016 | for (e = node->indirect_calls; e; e = e->next_callee) | |
1017 | reset_inline_edge_summary (e); | |
1018 | } | |
1019 | ||
c7b2cc59 | 1020 | /* Hook that is called by cgraph.c when a node is removed. */ |
1021 | ||
1022 | static void | |
e876d531 | 1023 | inline_node_removal_hook (struct cgraph_node *node, |
1024 | void *data ATTRIBUTE_UNUSED) | |
c7b2cc59 | 1025 | { |
cbd7f5a0 | 1026 | struct inline_summary *info; |
e876d531 | 1027 | if (vec_safe_length (inline_summary_vec) <= (unsigned) node->uid) |
c7b2cc59 | 1028 | return; |
cbd7f5a0 | 1029 | info = inline_summary (node); |
3b9dd281 | 1030 | reset_inline_summary (node); |
cbd7f5a0 | 1031 | memset (info, 0, sizeof (inline_summary_t)); |
c7b2cc59 | 1032 | } |
1033 | ||
3716ee8f | 1034 | /* Remap predicate P of former function to be predicate of duplicated functoin. |
1035 | POSSIBLE_TRUTHS is clause of possible truths in the duplicated node, | |
1036 | INFO is inline summary of the duplicated node. */ | |
1037 | ||
1038 | static struct predicate | |
1039 | remap_predicate_after_duplication (struct predicate *p, | |
1040 | clause_t possible_truths, | |
1041 | struct inline_summary *info) | |
1042 | { | |
1043 | struct predicate new_predicate = true_predicate (); | |
1044 | int j; | |
1045 | for (j = 0; p->clause[j]; j++) | |
1046 | if (!(possible_truths & p->clause[j])) | |
1047 | { | |
1048 | new_predicate = false_predicate (); | |
1049 | break; | |
1050 | } | |
1051 | else | |
1052 | add_clause (info->conds, &new_predicate, | |
1053 | possible_truths & p->clause[j]); | |
1054 | return new_predicate; | |
1055 | } | |
1056 | ||
1057 | /* Same as remap_predicate_after_duplication but handle hint predicate *P. | |
1058 | Additionally care about allocating new memory slot for updated predicate | |
1059 | and set it to NULL when it becomes true or false (and thus uninteresting). | |
1060 | */ | |
1061 | ||
1062 | static void | |
1063 | remap_hint_predicate_after_duplication (struct predicate **p, | |
1064 | clause_t possible_truths, | |
1065 | struct inline_summary *info) | |
1066 | { | |
1067 | struct predicate new_predicate; | |
1068 | ||
1069 | if (!*p) | |
1070 | return; | |
1071 | ||
1072 | new_predicate = remap_predicate_after_duplication (*p, | |
e876d531 | 1073 | possible_truths, info); |
3716ee8f | 1074 | /* We do not want to free previous predicate; it is used by node origin. */ |
1075 | *p = NULL; | |
1076 | set_hint_predicate (p, new_predicate); | |
1077 | } | |
1078 | ||
0835ad03 | 1079 | |
c7b2cc59 | 1080 | /* Hook that is called by cgraph.c when a node is duplicated. */ |
1081 | ||
1082 | static void | |
e876d531 | 1083 | inline_node_duplication_hook (struct cgraph_node *src, |
1084 | struct cgraph_node *dst, | |
c7b2cc59 | 1085 | ATTRIBUTE_UNUSED void *data) |
1086 | { | |
cbd7f5a0 | 1087 | struct inline_summary *info; |
c7b2cc59 | 1088 | inline_summary_alloc (); |
cbd7f5a0 | 1089 | info = inline_summary (dst); |
e876d531 | 1090 | memcpy (info, inline_summary (src), sizeof (struct inline_summary)); |
8bae3ea4 | 1091 | /* TODO: as an optimization, we may avoid copying conditions |
1092 | that are known to be false or true. */ | |
f1f41a6c | 1093 | info->conds = vec_safe_copy (info->conds); |
8bae3ea4 | 1094 | |
1095 | /* When there are any replacements in the function body, see if we can figure | |
1096 | out that something was optimized out. */ | |
e876d531 | 1097 | if (ipa_node_params_vector.exists () && dst->clone.tree_map) |
8bae3ea4 | 1098 | { |
f1f41a6c | 1099 | vec<size_time_entry, va_gc> *entry = info->entry; |
8bae3ea4 | 1100 | /* Use SRC parm info since it may not be copied yet. */ |
1101 | struct ipa_node_params *parms_info = IPA_NODE_REF (src); | |
1e094109 | 1102 | vec<tree> known_vals = vNULL; |
8bae3ea4 | 1103 | int count = ipa_get_param_count (parms_info); |
e876d531 | 1104 | int i, j; |
8bae3ea4 | 1105 | clause_t possible_truths; |
1106 | struct predicate true_pred = true_predicate (); | |
1107 | size_time_entry *e; | |
1108 | int optimized_out_size = 0; | |
8bae3ea4 | 1109 | bool inlined_to_p = false; |
1110 | struct cgraph_edge *edge; | |
1111 | ||
839c5aac | 1112 | info->entry = 0; |
f1f41a6c | 1113 | known_vals.safe_grow_cleared (count); |
8bae3ea4 | 1114 | for (i = 0; i < count; i++) |
e876d531 | 1115 | { |
8bae3ea4 | 1116 | struct ipa_replace_map *r; |
1117 | ||
f1f41a6c | 1118 | for (j = 0; vec_safe_iterate (dst->clone.tree_map, j, &r); j++) |
8bae3ea4 | 1119 | { |
09ab6335 | 1120 | if (((!r->old_tree && r->parm_num == i) |
1121 | || (r->old_tree && r->old_tree == ipa_get_param (parms_info, i))) | |
1122 | && r->replace_p && !r->ref_p) | |
8bae3ea4 | 1123 | { |
f1f41a6c | 1124 | known_vals[i] = r->new_tree; |
8bae3ea4 | 1125 | break; |
1126 | } | |
1127 | } | |
1128 | } | |
a4f60e55 | 1129 | possible_truths = evaluate_conditions_for_known_args (dst, false, |
e876d531 | 1130 | known_vals, |
1131 | vNULL); | |
f1f41a6c | 1132 | known_vals.release (); |
8bae3ea4 | 1133 | |
1134 | account_size_time (info, 0, 0, &true_pred); | |
1135 | ||
1136 | /* Remap size_time vectors. | |
e876d531 | 1137 | Simplify the predicate by prunning out alternatives that are known |
1138 | to be false. | |
1139 | TODO: as on optimization, we can also eliminate conditions known | |
1140 | to be true. */ | |
f1f41a6c | 1141 | for (i = 0; vec_safe_iterate (entry, i, &e); i++) |
8bae3ea4 | 1142 | { |
3716ee8f | 1143 | struct predicate new_predicate; |
1144 | new_predicate = remap_predicate_after_duplication (&e->predicate, | |
1145 | possible_truths, | |
1146 | info); | |
8bae3ea4 | 1147 | if (false_predicate_p (&new_predicate)) |
18b64b34 | 1148 | optimized_out_size += e->size; |
8bae3ea4 | 1149 | else |
1150 | account_size_time (info, e->size, e->time, &new_predicate); | |
1151 | } | |
1152 | ||
fb3c587e | 1153 | /* Remap edge predicates with the same simplification as above. |
e876d531 | 1154 | Also copy constantness arrays. */ |
8bae3ea4 | 1155 | for (edge = dst->callees; edge; edge = edge->next_callee) |
1156 | { | |
3716ee8f | 1157 | struct predicate new_predicate; |
8bae3ea4 | 1158 | struct inline_edge_summary *es = inline_edge_summary (edge); |
1159 | ||
1160 | if (!edge->inline_failed) | |
1161 | inlined_to_p = true; | |
1162 | if (!es->predicate) | |
1163 | continue; | |
3716ee8f | 1164 | new_predicate = remap_predicate_after_duplication (es->predicate, |
1165 | possible_truths, | |
1166 | info); | |
8bae3ea4 | 1167 | if (false_predicate_p (&new_predicate) |
1168 | && !false_predicate_p (es->predicate)) | |
1169 | { | |
1170 | optimized_out_size += es->call_stmt_size * INLINE_SIZE_SCALE; | |
8bae3ea4 | 1171 | edge->frequency = 0; |
1172 | } | |
7c07aa3d | 1173 | edge_set_predicate (edge, &new_predicate); |
8bae3ea4 | 1174 | } |
1175 | ||
fb3c587e | 1176 | /* Remap indirect edge predicates with the same simplificaiton as above. |
e876d531 | 1177 | Also copy constantness arrays. */ |
8bae3ea4 | 1178 | for (edge = dst->indirect_calls; edge; edge = edge->next_callee) |
1179 | { | |
3716ee8f | 1180 | struct predicate new_predicate; |
8bae3ea4 | 1181 | struct inline_edge_summary *es = inline_edge_summary (edge); |
1182 | ||
3716ee8f | 1183 | gcc_checking_assert (edge->inline_failed); |
8bae3ea4 | 1184 | if (!es->predicate) |
1185 | continue; | |
3716ee8f | 1186 | new_predicate = remap_predicate_after_duplication (es->predicate, |
1187 | possible_truths, | |
1188 | info); | |
8bae3ea4 | 1189 | if (false_predicate_p (&new_predicate) |
1190 | && !false_predicate_p (es->predicate)) | |
1191 | { | |
1192 | optimized_out_size += es->call_stmt_size * INLINE_SIZE_SCALE; | |
8bae3ea4 | 1193 | edge->frequency = 0; |
1194 | } | |
7c07aa3d | 1195 | edge_set_predicate (edge, &new_predicate); |
1196 | } | |
3716ee8f | 1197 | remap_hint_predicate_after_duplication (&info->loop_iterations, |
e876d531 | 1198 | possible_truths, info); |
3716ee8f | 1199 | remap_hint_predicate_after_duplication (&info->loop_stride, |
e876d531 | 1200 | possible_truths, info); |
be343a9c | 1201 | remap_hint_predicate_after_duplication (&info->array_index, |
e876d531 | 1202 | possible_truths, info); |
8bae3ea4 | 1203 | |
1204 | /* If inliner or someone after inliner will ever start producing | |
e876d531 | 1205 | non-trivial clones, we will get trouble with lack of information |
1206 | about updating self sizes, because size vectors already contains | |
1207 | sizes of the calees. */ | |
1208 | gcc_assert (!inlined_to_p || !optimized_out_size); | |
8bae3ea4 | 1209 | } |
1210 | else | |
7c07aa3d | 1211 | { |
f1f41a6c | 1212 | info->entry = vec_safe_copy (info->entry); |
7c07aa3d | 1213 | if (info->loop_iterations) |
1214 | { | |
1215 | predicate p = *info->loop_iterations; | |
3716ee8f | 1216 | info->loop_iterations = NULL; |
1217 | set_hint_predicate (&info->loop_iterations, p); | |
1218 | } | |
1219 | if (info->loop_stride) | |
1220 | { | |
1221 | predicate p = *info->loop_stride; | |
1222 | info->loop_stride = NULL; | |
1223 | set_hint_predicate (&info->loop_stride, p); | |
7c07aa3d | 1224 | } |
be343a9c | 1225 | if (info->array_index) |
1226 | { | |
1227 | predicate p = *info->array_index; | |
1228 | info->array_index = NULL; | |
1229 | set_hint_predicate (&info->array_index, p); | |
1230 | } | |
7c07aa3d | 1231 | } |
18b64b34 | 1232 | inline_update_overall_summary (dst); |
a41f2a28 | 1233 | } |
1234 | ||
1235 | ||
0835ad03 | 1236 | /* Hook that is called by cgraph.c when a node is duplicated. */ |
1237 | ||
1238 | static void | |
e876d531 | 1239 | inline_edge_duplication_hook (struct cgraph_edge *src, |
1240 | struct cgraph_edge *dst, | |
0835ad03 | 1241 | ATTRIBUTE_UNUSED void *data) |
1242 | { | |
1243 | struct inline_edge_summary *info; | |
6a18c0be | 1244 | struct inline_edge_summary *srcinfo; |
0835ad03 | 1245 | inline_summary_alloc (); |
1246 | info = inline_edge_summary (dst); | |
6a18c0be | 1247 | srcinfo = inline_edge_summary (src); |
e876d531 | 1248 | memcpy (info, srcinfo, sizeof (struct inline_edge_summary)); |
6a18c0be | 1249 | info->predicate = NULL; |
1250 | edge_set_predicate (dst, srcinfo->predicate); | |
f1f41a6c | 1251 | info->param = srcinfo->param.copy (); |
0835ad03 | 1252 | } |
1253 | ||
1254 | ||
a41f2a28 | 1255 | /* Keep edge cache consistent across edge removal. */ |
1256 | ||
1257 | static void | |
e876d531 | 1258 | inline_edge_removal_hook (struct cgraph_edge *edge, |
1259 | void *data ATTRIBUTE_UNUSED) | |
a41f2a28 | 1260 | { |
f1f41a6c | 1261 | if (edge_growth_cache.exists ()) |
0835ad03 | 1262 | reset_edge_growth_cache (edge); |
d10a25bb | 1263 | reset_inline_edge_summary (edge); |
a41f2a28 | 1264 | } |
1265 | ||
1266 | ||
1267 | /* Initialize growth caches. */ | |
1268 | ||
1269 | void | |
1270 | initialize_growth_caches (void) | |
1271 | { | |
a41f2a28 | 1272 | if (cgraph_edge_max_uid) |
f1f41a6c | 1273 | edge_growth_cache.safe_grow_cleared (cgraph_edge_max_uid); |
a41f2a28 | 1274 | if (cgraph_max_uid) |
f1f41a6c | 1275 | node_growth_cache.safe_grow_cleared (cgraph_max_uid); |
a41f2a28 | 1276 | } |
1277 | ||
1278 | ||
1279 | /* Free growth caches. */ | |
1280 | ||
1281 | void | |
1282 | free_growth_caches (void) | |
1283 | { | |
f1f41a6c | 1284 | edge_growth_cache.release (); |
1285 | node_growth_cache.release (); | |
c7b2cc59 | 1286 | } |
1287 | ||
a41f2a28 | 1288 | |
0835ad03 | 1289 | /* Dump edge summaries associated to NODE and recursively to all clones. |
1290 | Indent by INDENT. */ | |
1291 | ||
1292 | static void | |
e876d531 | 1293 | dump_inline_edge_summary (FILE *f, int indent, struct cgraph_node *node, |
6a18c0be | 1294 | struct inline_summary *info) |
0835ad03 | 1295 | { |
1296 | struct cgraph_edge *edge; | |
1297 | for (edge = node->callees; edge; edge = edge->next_callee) | |
1298 | { | |
1299 | struct inline_edge_summary *es = inline_edge_summary (edge); | |
e876d531 | 1300 | struct cgraph_node *callee = |
1301 | cgraph_function_or_thunk_node (edge->callee, NULL); | |
eb4ae064 | 1302 | int i; |
1303 | ||
e876d531 | 1304 | fprintf (f, |
1305 | "%*s%s/%i %s\n%*s loop depth:%2i freq:%4i size:%2i" | |
1306 | " time: %2i callee size:%2i stack:%2i", | |
f1c8b4d7 | 1307 | indent, "", callee->name (), callee->order, |
e876d531 | 1308 | !edge->inline_failed |
1309 | ? "inlined" : cgraph_inline_failed_string (edge-> inline_failed), | |
1310 | indent, "", es->loop_depth, edge->frequency, | |
1311 | es->call_stmt_size, es->call_stmt_time, | |
1312 | (int) inline_summary (callee)->size / INLINE_SIZE_SCALE, | |
1313 | (int) inline_summary (callee)->estimated_stack_size); | |
eb4ae064 | 1314 | |
6a18c0be | 1315 | if (es->predicate) |
1316 | { | |
1317 | fprintf (f, " predicate: "); | |
1318 | dump_predicate (f, info->conds, es->predicate); | |
1319 | } | |
1320 | else | |
e876d531 | 1321 | fprintf (f, "\n"); |
f1f41a6c | 1322 | if (es->param.exists ()) |
e876d531 | 1323 | for (i = 0; i < (int) es->param.length (); i++) |
eb4ae064 | 1324 | { |
f1f41a6c | 1325 | int prob = es->param[i].change_prob; |
eb4ae064 | 1326 | |
1327 | if (!prob) | |
1328 | fprintf (f, "%*s op%i is compile time invariant\n", | |
1329 | indent + 2, "", i); | |
1330 | else if (prob != REG_BR_PROB_BASE) | |
1331 | fprintf (f, "%*s op%i change %f%% of time\n", indent + 2, "", i, | |
1332 | prob * 100.0 / REG_BR_PROB_BASE); | |
1333 | } | |
0835ad03 | 1334 | if (!edge->inline_failed) |
0a0ca4d6 | 1335 | { |
e876d531 | 1336 | fprintf (f, "%*sStack frame offset %i, callee self size %i," |
eb4ae064 | 1337 | " callee size %i\n", |
e876d531 | 1338 | indent + 2, "", |
1339 | (int) inline_summary (callee)->stack_frame_offset, | |
1340 | (int) inline_summary (callee)->estimated_self_stack_size, | |
1341 | (int) inline_summary (callee)->estimated_stack_size); | |
1342 | dump_inline_edge_summary (f, indent + 2, callee, info); | |
0a0ca4d6 | 1343 | } |
0835ad03 | 1344 | } |
1345 | for (edge = node->indirect_calls; edge; edge = edge->next_callee) | |
1346 | { | |
1347 | struct inline_edge_summary *es = inline_edge_summary (edge); | |
fb3c587e | 1348 | fprintf (f, "%*sindirect call loop depth:%2i freq:%4i size:%2i" |
eb4ae064 | 1349 | " time: %2i", |
0835ad03 | 1350 | indent, "", |
e876d531 | 1351 | es->loop_depth, |
1352 | edge->frequency, es->call_stmt_size, es->call_stmt_time); | |
6a18c0be | 1353 | if (es->predicate) |
1354 | { | |
1355 | fprintf (f, "predicate: "); | |
1356 | dump_predicate (f, info->conds, es->predicate); | |
1357 | } | |
1358 | else | |
fb3c587e | 1359 | fprintf (f, "\n"); |
0835ad03 | 1360 | } |
1361 | } | |
1362 | ||
1363 | ||
0a0ca4d6 | 1364 | void |
e876d531 | 1365 | dump_inline_summary (FILE *f, struct cgraph_node *node) |
c7b2cc59 | 1366 | { |
02774f2d | 1367 | if (node->definition) |
c7b2cc59 | 1368 | { |
1369 | struct inline_summary *s = inline_summary (node); | |
a41f2a28 | 1370 | size_time_entry *e; |
1371 | int i; | |
f1c8b4d7 | 1372 | fprintf (f, "Inline summary for %s/%i", node->name (), |
02774f2d | 1373 | node->order); |
1374 | if (DECL_DISREGARD_INLINE_LIMITS (node->decl)) | |
cbd7f5a0 | 1375 | fprintf (f, " always_inline"); |
1376 | if (s->inlinable) | |
1377 | fprintf (f, " inlinable"); | |
e876d531 | 1378 | fprintf (f, "\n self time: %i\n", s->self_time); |
cbd7f5a0 | 1379 | fprintf (f, " global time: %i\n", s->time); |
e876d531 | 1380 | fprintf (f, " self size: %i\n", s->self_size); |
4869c23f | 1381 | fprintf (f, " global size: %i\n", s->size); |
c7b2cc59 | 1382 | fprintf (f, " self stack: %i\n", |
a41f2a28 | 1383 | (int) s->estimated_self_stack_size); |
e876d531 | 1384 | fprintf (f, " global stack: %i\n", (int) s->estimated_stack_size); |
3172b7bf | 1385 | if (s->growth) |
e876d531 | 1386 | fprintf (f, " estimated growth:%i\n", (int) s->growth); |
db2db13c | 1387 | if (s->scc_no) |
e876d531 | 1388 | fprintf (f, " In SCC: %i\n", (int) s->scc_no); |
f1f41a6c | 1389 | for (i = 0; vec_safe_iterate (s->entry, i, &e); i++) |
a41f2a28 | 1390 | { |
1391 | fprintf (f, " size:%f, time:%f, predicate:", | |
1392 | (double) e->size / INLINE_SIZE_SCALE, | |
1393 | (double) e->time / INLINE_TIME_SCALE); | |
1394 | dump_predicate (f, s->conds, &e->predicate); | |
1395 | } | |
7c07aa3d | 1396 | if (s->loop_iterations) |
1397 | { | |
1398 | fprintf (f, " loop iterations:"); | |
1399 | dump_predicate (f, s->conds, s->loop_iterations); | |
1400 | } | |
3716ee8f | 1401 | if (s->loop_stride) |
1402 | { | |
1403 | fprintf (f, " loop stride:"); | |
1404 | dump_predicate (f, s->conds, s->loop_stride); | |
1405 | } | |
be343a9c | 1406 | if (s->array_index) |
1407 | { | |
1408 | fprintf (f, " array index:"); | |
1409 | dump_predicate (f, s->conds, s->array_index); | |
1410 | } | |
0835ad03 | 1411 | fprintf (f, " calls:\n"); |
6a18c0be | 1412 | dump_inline_edge_summary (f, 4, node, s); |
a41f2a28 | 1413 | fprintf (f, "\n"); |
c7b2cc59 | 1414 | } |
1415 | } | |
1416 | ||
0a0ca4d6 | 1417 | DEBUG_FUNCTION void |
c7b2cc59 | 1418 | debug_inline_summary (struct cgraph_node *node) |
1419 | { | |
1420 | dump_inline_summary (stderr, node); | |
1421 | } | |
1422 | ||
1423 | void | |
1424 | dump_inline_summaries (FILE *f) | |
1425 | { | |
1426 | struct cgraph_node *node; | |
1427 | ||
7c455d87 | 1428 | FOR_EACH_DEFINED_FUNCTION (node) |
1429 | if (!node->global.inlined_to) | |
c7b2cc59 | 1430 | dump_inline_summary (f, node); |
1431 | } | |
99c67f24 | 1432 | |
cbd7f5a0 | 1433 | /* Give initial reasons why inlining would fail on EDGE. This gets either |
1434 | nullified or usually overwritten by more precise reasons later. */ | |
1435 | ||
1436 | void | |
1437 | initialize_inline_failed (struct cgraph_edge *e) | |
1438 | { | |
1439 | struct cgraph_node *callee = e->callee; | |
1440 | ||
1441 | if (e->indirect_unknown_callee) | |
1442 | e->inline_failed = CIF_INDIRECT_UNKNOWN_CALL; | |
02774f2d | 1443 | else if (!callee->definition) |
cbd7f5a0 | 1444 | e->inline_failed = CIF_BODY_NOT_AVAILABLE; |
1445 | else if (callee->local.redefined_extern_inline) | |
1446 | e->inline_failed = CIF_REDEFINED_EXTERN_INLINE; | |
f883da84 | 1447 | else if (e->call_stmt_cannot_inline_p) |
cbd7f5a0 | 1448 | e->inline_failed = CIF_MISMATCHED_ARGUMENTS; |
d037099f | 1449 | else if (cfun && fn_contains_cilk_spawn_p (cfun)) |
1450 | /* We can't inline if the function is spawing a function. */ | |
1451 | e->inline_failed = CIF_FUNCTION_NOT_INLINABLE; | |
cbd7f5a0 | 1452 | else |
1453 | e->inline_failed = CIF_FUNCTION_NOT_CONSIDERED; | |
1454 | } | |
1455 | ||
94646c9c | 1456 | /* Callback of walk_aliased_vdefs. Flags that it has been invoked to the |
1457 | boolean variable pointed to by DATA. */ | |
1458 | ||
1459 | static bool | |
1460 | mark_modified (ao_ref *ao ATTRIBUTE_UNUSED, tree vdef ATTRIBUTE_UNUSED, | |
e876d531 | 1461 | void *data) |
94646c9c | 1462 | { |
1463 | bool *b = (bool *) data; | |
1464 | *b = true; | |
1465 | return true; | |
1466 | } | |
1467 | ||
a4f60e55 | 1468 | /* If OP refers to value of function parameter, return the corresponding |
1469 | parameter. */ | |
94646c9c | 1470 | |
1471 | static tree | |
a4f60e55 | 1472 | unmodified_parm_1 (gimple stmt, tree op) |
94646c9c | 1473 | { |
1474 | /* SSA_NAME referring to parm default def? */ | |
1475 | if (TREE_CODE (op) == SSA_NAME | |
1476 | && SSA_NAME_IS_DEFAULT_DEF (op) | |
1477 | && TREE_CODE (SSA_NAME_VAR (op)) == PARM_DECL) | |
1478 | return SSA_NAME_VAR (op); | |
1479 | /* Non-SSA parm reference? */ | |
1480 | if (TREE_CODE (op) == PARM_DECL) | |
1481 | { | |
1482 | bool modified = false; | |
1483 | ||
1484 | ao_ref refd; | |
1485 | ao_ref_init (&refd, op); | |
1486 | walk_aliased_vdefs (&refd, gimple_vuse (stmt), mark_modified, &modified, | |
1487 | NULL); | |
1488 | if (!modified) | |
1489 | return op; | |
1490 | } | |
a4f60e55 | 1491 | return NULL_TREE; |
1492 | } | |
1493 | ||
1494 | /* If OP refers to value of function parameter, return the corresponding | |
1495 | parameter. Also traverse chains of SSA register assignments. */ | |
1496 | ||
1497 | static tree | |
1498 | unmodified_parm (gimple stmt, tree op) | |
1499 | { | |
1500 | tree res = unmodified_parm_1 (stmt, op); | |
1501 | if (res) | |
1502 | return res; | |
1503 | ||
94646c9c | 1504 | if (TREE_CODE (op) == SSA_NAME |
1505 | && !SSA_NAME_IS_DEFAULT_DEF (op) | |
1506 | && gimple_assign_single_p (SSA_NAME_DEF_STMT (op))) | |
1507 | return unmodified_parm (SSA_NAME_DEF_STMT (op), | |
1508 | gimple_assign_rhs1 (SSA_NAME_DEF_STMT (op))); | |
a4f60e55 | 1509 | return NULL_TREE; |
1510 | } | |
1511 | ||
1512 | /* If OP refers to a value of a function parameter or value loaded from an | |
1513 | aggregate passed to a parameter (either by value or reference), return TRUE | |
1514 | and store the number of the parameter to *INDEX_P and information whether | |
1515 | and how it has been loaded from an aggregate into *AGGPOS. INFO describes | |
1516 | the function parameters, STMT is the statement in which OP is used or | |
1517 | loaded. */ | |
1518 | ||
1519 | static bool | |
1520 | unmodified_parm_or_parm_agg_item (struct ipa_node_params *info, | |
1521 | gimple stmt, tree op, int *index_p, | |
1522 | struct agg_position_info *aggpos) | |
1523 | { | |
1524 | tree res = unmodified_parm_1 (stmt, op); | |
1525 | ||
1526 | gcc_checking_assert (aggpos); | |
1527 | if (res) | |
1528 | { | |
1529 | *index_p = ipa_get_param_decl_index (info, res); | |
1530 | if (*index_p < 0) | |
1531 | return false; | |
1532 | aggpos->agg_contents = false; | |
1533 | aggpos->by_ref = false; | |
1534 | return true; | |
1535 | } | |
1536 | ||
1537 | if (TREE_CODE (op) == SSA_NAME) | |
1538 | { | |
1539 | if (SSA_NAME_IS_DEFAULT_DEF (op) | |
1540 | || !gimple_assign_single_p (SSA_NAME_DEF_STMT (op))) | |
1541 | return false; | |
1542 | stmt = SSA_NAME_DEF_STMT (op); | |
1543 | op = gimple_assign_rhs1 (stmt); | |
1544 | if (!REFERENCE_CLASS_P (op)) | |
1545 | return unmodified_parm_or_parm_agg_item (info, stmt, op, index_p, | |
1546 | aggpos); | |
1547 | } | |
1548 | ||
1549 | aggpos->agg_contents = true; | |
1550 | return ipa_load_from_parm_agg (info, stmt, op, index_p, &aggpos->offset, | |
1551 | &aggpos->by_ref); | |
94646c9c | 1552 | } |
1553 | ||
99c67f24 | 1554 | /* See if statement might disappear after inlining. |
1555 | 0 - means not eliminated | |
1556 | 1 - half of statements goes away | |
1557 | 2 - for sure it is eliminated. | |
1558 | We are not terribly sophisticated, basically looking for simple abstraction | |
1559 | penalty wrappers. */ | |
1560 | ||
1561 | static int | |
1562 | eliminated_by_inlining_prob (gimple stmt) | |
1563 | { | |
1564 | enum gimple_code code = gimple_code (stmt); | |
11f20fba | 1565 | enum tree_code rhs_code; |
94646c9c | 1566 | |
1567 | if (!optimize) | |
1568 | return 0; | |
1569 | ||
99c67f24 | 1570 | switch (code) |
1571 | { | |
e876d531 | 1572 | case GIMPLE_RETURN: |
1573 | return 2; | |
1574 | case GIMPLE_ASSIGN: | |
1575 | if (gimple_num_ops (stmt) != 2) | |
99c67f24 | 1576 | return 0; |
e876d531 | 1577 | |
1578 | rhs_code = gimple_assign_rhs_code (stmt); | |
1579 | ||
1580 | /* Casts of parameters, loads from parameters passed by reference | |
1581 | and stores to return value or parameters are often free after | |
1582 | inlining dua to SRA and further combining. | |
1583 | Assume that half of statements goes away. */ | |
1584 | if (rhs_code == CONVERT_EXPR | |
1585 | || rhs_code == NOP_EXPR | |
1586 | || rhs_code == VIEW_CONVERT_EXPR | |
1587 | || rhs_code == ADDR_EXPR | |
1588 | || gimple_assign_rhs_class (stmt) == GIMPLE_SINGLE_RHS) | |
1589 | { | |
1590 | tree rhs = gimple_assign_rhs1 (stmt); | |
1591 | tree lhs = gimple_assign_lhs (stmt); | |
1592 | tree inner_rhs = get_base_address (rhs); | |
1593 | tree inner_lhs = get_base_address (lhs); | |
1594 | bool rhs_free = false; | |
1595 | bool lhs_free = false; | |
1596 | ||
1597 | if (!inner_rhs) | |
1598 | inner_rhs = rhs; | |
1599 | if (!inner_lhs) | |
1600 | inner_lhs = lhs; | |
1601 | ||
1602 | /* Reads of parameter are expected to be free. */ | |
1603 | if (unmodified_parm (stmt, inner_rhs)) | |
1604 | rhs_free = true; | |
1605 | /* Match expressions of form &this->field. Those will most likely | |
1606 | combine with something upstream after inlining. */ | |
1607 | else if (TREE_CODE (inner_rhs) == ADDR_EXPR) | |
1608 | { | |
1609 | tree op = get_base_address (TREE_OPERAND (inner_rhs, 0)); | |
1610 | if (TREE_CODE (op) == PARM_DECL) | |
1611 | rhs_free = true; | |
1612 | else if (TREE_CODE (op) == MEM_REF | |
1613 | && unmodified_parm (stmt, TREE_OPERAND (op, 0))) | |
1614 | rhs_free = true; | |
1615 | } | |
1616 | ||
1617 | /* When parameter is not SSA register because its address is taken | |
1618 | and it is just copied into one, the statement will be completely | |
1619 | free after inlining (we will copy propagate backward). */ | |
1620 | if (rhs_free && is_gimple_reg (lhs)) | |
1621 | return 2; | |
1622 | ||
1623 | /* Reads of parameters passed by reference | |
1624 | expected to be free (i.e. optimized out after inlining). */ | |
1625 | if (TREE_CODE (inner_rhs) == MEM_REF | |
1626 | && unmodified_parm (stmt, TREE_OPERAND (inner_rhs, 0))) | |
1627 | rhs_free = true; | |
1628 | ||
1629 | /* Copying parameter passed by reference into gimple register is | |
1630 | probably also going to copy propagate, but we can't be quite | |
1631 | sure. */ | |
1632 | if (rhs_free && is_gimple_reg (lhs)) | |
1633 | lhs_free = true; | |
1634 | ||
1635 | /* Writes to parameters, parameters passed by value and return value | |
1636 | (either dirrectly or passed via invisible reference) are free. | |
1637 | ||
1638 | TODO: We ought to handle testcase like | |
1639 | struct a {int a,b;}; | |
1640 | struct a | |
1641 | retrurnsturct (void) | |
1642 | { | |
1643 | struct a a ={1,2}; | |
1644 | return a; | |
1645 | } | |
1646 | ||
1647 | This translate into: | |
1648 | ||
1649 | retrurnsturct () | |
1650 | { | |
1651 | int a$b; | |
1652 | int a$a; | |
1653 | struct a a; | |
1654 | struct a D.2739; | |
1655 | ||
1656 | <bb 2>: | |
1657 | D.2739.a = 1; | |
1658 | D.2739.b = 2; | |
1659 | return D.2739; | |
1660 | ||
1661 | } | |
1662 | For that we either need to copy ipa-split logic detecting writes | |
1663 | to return value. */ | |
1664 | if (TREE_CODE (inner_lhs) == PARM_DECL | |
1665 | || TREE_CODE (inner_lhs) == RESULT_DECL | |
1666 | || (TREE_CODE (inner_lhs) == MEM_REF | |
1667 | && (unmodified_parm (stmt, TREE_OPERAND (inner_lhs, 0)) | |
1668 | || (TREE_CODE (TREE_OPERAND (inner_lhs, 0)) == SSA_NAME | |
1669 | && SSA_NAME_VAR (TREE_OPERAND (inner_lhs, 0)) | |
1670 | && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND | |
1671 | (inner_lhs, | |
1672 | 0))) == RESULT_DECL)))) | |
1673 | lhs_free = true; | |
1674 | if (lhs_free | |
1675 | && (is_gimple_reg (rhs) || is_gimple_min_invariant (rhs))) | |
1676 | rhs_free = true; | |
1677 | if (lhs_free && rhs_free) | |
1678 | return 1; | |
1679 | } | |
1680 | return 0; | |
1681 | default: | |
1682 | return 0; | |
99c67f24 | 1683 | } |
1684 | } | |
1685 | ||
1686 | ||
905aa3bd | 1687 | /* If BB ends by a conditional we can turn into predicates, attach corresponding |
1688 | predicates to the CFG edges. */ | |
a41f2a28 | 1689 | |
905aa3bd | 1690 | static void |
1691 | set_cond_stmt_execution_predicate (struct ipa_node_params *info, | |
e876d531 | 1692 | struct inline_summary *summary, |
1693 | basic_block bb) | |
a41f2a28 | 1694 | { |
a41f2a28 | 1695 | gimple last; |
1696 | tree op; | |
1697 | int index; | |
a4f60e55 | 1698 | struct agg_position_info aggpos; |
905aa3bd | 1699 | enum tree_code code, inverted_code; |
1700 | edge e; | |
1701 | edge_iterator ei; | |
1702 | gimple set_stmt; | |
1703 | tree op2; | |
a41f2a28 | 1704 | |
905aa3bd | 1705 | last = last_stmt (bb); |
e876d531 | 1706 | if (!last || gimple_code (last) != GIMPLE_COND) |
905aa3bd | 1707 | return; |
a41f2a28 | 1708 | if (!is_gimple_ip_invariant (gimple_cond_rhs (last))) |
905aa3bd | 1709 | return; |
a41f2a28 | 1710 | op = gimple_cond_lhs (last); |
1711 | /* TODO: handle conditionals like | |
1712 | var = op0 < 4; | |
905aa3bd | 1713 | if (var != 0). */ |
a4f60e55 | 1714 | if (unmodified_parm_or_parm_agg_item (info, last, op, &index, &aggpos)) |
905aa3bd | 1715 | { |
905aa3bd | 1716 | code = gimple_cond_code (last); |
fb3c587e | 1717 | inverted_code |
e876d531 | 1718 | = invert_tree_comparison (code, |
1719 | HONOR_NANS (TYPE_MODE (TREE_TYPE (op)))); | |
905aa3bd | 1720 | |
1721 | FOR_EACH_EDGE (e, ei, bb->succs) | |
1722 | { | |
a4f60e55 | 1723 | struct predicate p = add_condition (summary, index, &aggpos, |
905aa3bd | 1724 | e->flags & EDGE_TRUE_VALUE |
1725 | ? code : inverted_code, | |
1726 | gimple_cond_rhs (last)); | |
1727 | e->aux = pool_alloc (edge_predicate_pool); | |
e876d531 | 1728 | *(struct predicate *) e->aux = p; |
905aa3bd | 1729 | } |
1730 | } | |
1731 | ||
94646c9c | 1732 | if (TREE_CODE (op) != SSA_NAME) |
1733 | return; | |
905aa3bd | 1734 | /* Special case |
1735 | if (builtin_constant_p (op)) | |
e876d531 | 1736 | constant_code |
905aa3bd | 1737 | else |
e876d531 | 1738 | nonconstant_code. |
905aa3bd | 1739 | Here we can predicate nonconstant_code. We can't |
1740 | really handle constant_code since we have no predicate | |
1741 | for this and also the constant code is not known to be | |
1742 | optimized away when inliner doen't see operand is constant. | |
1743 | Other optimizers might think otherwise. */ | |
a4f60e55 | 1744 | if (gimple_cond_code (last) != NE_EXPR |
1745 | || !integer_zerop (gimple_cond_rhs (last))) | |
1746 | return; | |
905aa3bd | 1747 | set_stmt = SSA_NAME_DEF_STMT (op); |
1748 | if (!gimple_call_builtin_p (set_stmt, BUILT_IN_CONSTANT_P) | |
1749 | || gimple_call_num_args (set_stmt) != 1) | |
1750 | return; | |
1751 | op2 = gimple_call_arg (set_stmt, 0); | |
e876d531 | 1752 | if (!unmodified_parm_or_parm_agg_item |
1753 | (info, set_stmt, op2, &index, &aggpos)) | |
905aa3bd | 1754 | return; |
e876d531 | 1755 | FOR_EACH_EDGE (e, ei, bb->succs) if (e->flags & EDGE_FALSE_VALUE) |
1756 | { | |
1757 | struct predicate p = add_condition (summary, index, &aggpos, | |
1758 | IS_NOT_CONSTANT, NULL_TREE); | |
1759 | e->aux = pool_alloc (edge_predicate_pool); | |
1760 | *(struct predicate *) e->aux = p; | |
1761 | } | |
905aa3bd | 1762 | } |
1763 | ||
1764 | ||
1765 | /* If BB ends by a switch we can turn into predicates, attach corresponding | |
1766 | predicates to the CFG edges. */ | |
1767 | ||
1768 | static void | |
1769 | set_switch_stmt_execution_predicate (struct ipa_node_params *info, | |
e876d531 | 1770 | struct inline_summary *summary, |
1771 | basic_block bb) | |
905aa3bd | 1772 | { |
1773 | gimple last; | |
1774 | tree op; | |
1775 | int index; | |
a4f60e55 | 1776 | struct agg_position_info aggpos; |
905aa3bd | 1777 | edge e; |
1778 | edge_iterator ei; | |
1779 | size_t n; | |
1780 | size_t case_idx; | |
1781 | ||
1782 | last = last_stmt (bb); | |
e876d531 | 1783 | if (!last || gimple_code (last) != GIMPLE_SWITCH) |
905aa3bd | 1784 | return; |
1785 | op = gimple_switch_index (last); | |
a4f60e55 | 1786 | if (!unmodified_parm_or_parm_agg_item (info, last, op, &index, &aggpos)) |
905aa3bd | 1787 | return; |
a41f2a28 | 1788 | |
905aa3bd | 1789 | FOR_EACH_EDGE (e, ei, bb->succs) |
1790 | { | |
1791 | e->aux = pool_alloc (edge_predicate_pool); | |
e876d531 | 1792 | *(struct predicate *) e->aux = false_predicate (); |
905aa3bd | 1793 | } |
e876d531 | 1794 | n = gimple_switch_num_labels (last); |
905aa3bd | 1795 | for (case_idx = 0; case_idx < n; ++case_idx) |
1796 | { | |
1797 | tree cl = gimple_switch_label (last, case_idx); | |
1798 | tree min, max; | |
1799 | struct predicate p; | |
a41f2a28 | 1800 | |
905aa3bd | 1801 | e = find_edge (bb, label_to_block (CASE_LABEL (cl))); |
1802 | min = CASE_LOW (cl); | |
1803 | max = CASE_HIGH (cl); | |
1804 | ||
1805 | /* For default we might want to construct predicate that none | |
e876d531 | 1806 | of cases is met, but it is bit hard to do not having negations |
1807 | of conditionals handy. */ | |
905aa3bd | 1808 | if (!min && !max) |
1809 | p = true_predicate (); | |
1810 | else if (!max) | |
a4f60e55 | 1811 | p = add_condition (summary, index, &aggpos, EQ_EXPR, min); |
905aa3bd | 1812 | else |
1813 | { | |
1814 | struct predicate p1, p2; | |
a4f60e55 | 1815 | p1 = add_condition (summary, index, &aggpos, GE_EXPR, min); |
1816 | p2 = add_condition (summary, index, &aggpos, LE_EXPR, max); | |
94646c9c | 1817 | p = and_predicates (summary->conds, &p1, &p2); |
905aa3bd | 1818 | } |
e876d531 | 1819 | *(struct predicate *) e->aux |
1820 | = or_predicates (summary->conds, &p, (struct predicate *) e->aux); | |
905aa3bd | 1821 | } |
1822 | } | |
1823 | ||
1824 | ||
1825 | /* For each BB in NODE attach to its AUX pointer predicate under | |
1826 | which it is executable. */ | |
1827 | ||
1828 | static void | |
1829 | compute_bb_predicates (struct cgraph_node *node, | |
1830 | struct ipa_node_params *parms_info, | |
1831 | struct inline_summary *summary) | |
1832 | { | |
02774f2d | 1833 | struct function *my_function = DECL_STRUCT_FUNCTION (node->decl); |
905aa3bd | 1834 | bool done = false; |
1835 | basic_block bb; | |
1836 | ||
1837 | FOR_EACH_BB_FN (bb, my_function) | |
1838 | { | |
1839 | set_cond_stmt_execution_predicate (parms_info, summary, bb); | |
1840 | set_switch_stmt_execution_predicate (parms_info, summary, bb); | |
1841 | } | |
1842 | ||
1843 | /* Entry block is always executable. */ | |
fb3c587e | 1844 | ENTRY_BLOCK_PTR_FOR_FUNCTION (my_function)->aux |
1845 | = pool_alloc (edge_predicate_pool); | |
e876d531 | 1846 | *(struct predicate *) ENTRY_BLOCK_PTR_FOR_FUNCTION (my_function)->aux |
905aa3bd | 1847 | = true_predicate (); |
1848 | ||
1849 | /* A simple dataflow propagation of predicates forward in the CFG. | |
1850 | TODO: work in reverse postorder. */ | |
1851 | while (!done) | |
1852 | { | |
1853 | done = true; | |
1854 | FOR_EACH_BB_FN (bb, my_function) | |
1855 | { | |
e876d531 | 1856 | struct predicate p = false_predicate (); |
1857 | edge e; | |
1858 | edge_iterator ei; | |
905aa3bd | 1859 | FOR_EACH_EDGE (e, ei, bb->preds) |
1860 | { | |
1861 | if (e->src->aux) | |
1862 | { | |
fb3c587e | 1863 | struct predicate this_bb_predicate |
e876d531 | 1864 | = *(struct predicate *) e->src->aux; |
905aa3bd | 1865 | if (e->aux) |
fb3c587e | 1866 | this_bb_predicate |
e876d531 | 1867 | = and_predicates (summary->conds, &this_bb_predicate, |
1868 | (struct predicate *) e->aux); | |
94646c9c | 1869 | p = or_predicates (summary->conds, &p, &this_bb_predicate); |
905aa3bd | 1870 | if (true_predicate_p (&p)) |
1871 | break; | |
1872 | } | |
1873 | } | |
1874 | if (false_predicate_p (&p)) | |
1875 | gcc_assert (!bb->aux); | |
1876 | else | |
1877 | { | |
1878 | if (!bb->aux) | |
1879 | { | |
1880 | done = false; | |
1881 | bb->aux = pool_alloc (edge_predicate_pool); | |
e876d531 | 1882 | *((struct predicate *) bb->aux) = p; |
905aa3bd | 1883 | } |
e876d531 | 1884 | else if (!predicates_equal_p (&p, (struct predicate *) bb->aux)) |
905aa3bd | 1885 | { |
1886 | done = false; | |
e876d531 | 1887 | *((struct predicate *) bb->aux) = p; |
905aa3bd | 1888 | } |
1889 | } | |
1890 | } | |
1891 | } | |
a41f2a28 | 1892 | } |
1893 | ||
0b50fa0e | 1894 | |
1895 | /* We keep info about constantness of SSA names. */ | |
1896 | ||
1897 | typedef struct predicate predicate_t; | |
7c07aa3d | 1898 | /* Return predicate specifying when the STMT might have result that is not |
1899 | a compile time constant. */ | |
1900 | ||
1901 | static struct predicate | |
1902 | will_be_nonconstant_expr_predicate (struct ipa_node_params *info, | |
e876d531 | 1903 | struct inline_summary *summary, |
1904 | tree expr, | |
1905 | vec<predicate_t> nonconstant_names) | |
7c07aa3d | 1906 | { |
1907 | tree parm; | |
1908 | int index; | |
1909 | ||
1910 | while (UNARY_CLASS_P (expr)) | |
1911 | expr = TREE_OPERAND (expr, 0); | |
1912 | ||
1913 | parm = unmodified_parm (NULL, expr); | |
e876d531 | 1914 | if (parm && (index = ipa_get_param_decl_index (info, parm)) >= 0) |
7c07aa3d | 1915 | return add_condition (summary, index, NULL, CHANGED, NULL_TREE); |
1916 | if (is_gimple_min_invariant (expr)) | |
1917 | return false_predicate (); | |
1918 | if (TREE_CODE (expr) == SSA_NAME) | |
f1f41a6c | 1919 | return nonconstant_names[SSA_NAME_VERSION (expr)]; |
e876d531 | 1920 | if (BINARY_CLASS_P (expr) || COMPARISON_CLASS_P (expr)) |
3716ee8f | 1921 | { |
1922 | struct predicate p1 = will_be_nonconstant_expr_predicate | |
e876d531 | 1923 | (info, summary, TREE_OPERAND (expr, 0), |
1924 | nonconstant_names); | |
3716ee8f | 1925 | struct predicate p2; |
1926 | if (true_predicate_p (&p1)) | |
1927 | return p1; | |
1928 | p2 = will_be_nonconstant_expr_predicate (info, summary, | |
1929 | TREE_OPERAND (expr, 1), | |
1930 | nonconstant_names); | |
1931 | return or_predicates (summary->conds, &p1, &p2); | |
1932 | } | |
1933 | else if (TREE_CODE (expr) == COND_EXPR) | |
7c07aa3d | 1934 | { |
3716ee8f | 1935 | struct predicate p1 = will_be_nonconstant_expr_predicate |
e876d531 | 1936 | (info, summary, TREE_OPERAND (expr, 0), |
1937 | nonconstant_names); | |
7c07aa3d | 1938 | struct predicate p2; |
1939 | if (true_predicate_p (&p1)) | |
1940 | return p1; | |
3716ee8f | 1941 | p2 = will_be_nonconstant_expr_predicate (info, summary, |
1942 | TREE_OPERAND (expr, 1), | |
1943 | nonconstant_names); | |
1944 | if (true_predicate_p (&p2)) | |
1945 | return p2; | |
1946 | p1 = or_predicates (summary->conds, &p1, &p2); | |
1947 | p2 = will_be_nonconstant_expr_predicate (info, summary, | |
1948 | TREE_OPERAND (expr, 2), | |
1949 | nonconstant_names); | |
7c07aa3d | 1950 | return or_predicates (summary->conds, &p1, &p2); |
1951 | } | |
1952 | else | |
1953 | { | |
1954 | debug_tree (expr); | |
1955 | gcc_unreachable (); | |
1956 | } | |
1957 | return false_predicate (); | |
1958 | } | |
0b50fa0e | 1959 | |
1960 | ||
fb3c587e | 1961 | /* Return predicate specifying when the STMT might have result that is not |
1962 | a compile time constant. */ | |
0b50fa0e | 1963 | |
a41f2a28 | 1964 | static struct predicate |
1965 | will_be_nonconstant_predicate (struct ipa_node_params *info, | |
1966 | struct inline_summary *summary, | |
0b50fa0e | 1967 | gimple stmt, |
f1f41a6c | 1968 | vec<predicate_t> nonconstant_names) |
a41f2a28 | 1969 | { |
1970 | struct predicate p = true_predicate (); | |
1971 | ssa_op_iter iter; | |
1972 | tree use; | |
1973 | struct predicate op_non_const; | |
8e22665e | 1974 | bool is_load; |
a4f60e55 | 1975 | int base_index; |
1976 | struct agg_position_info aggpos; | |
a41f2a28 | 1977 | |
1978 | /* What statments might be optimized away | |
1979 | when their arguments are constant | |
905aa3bd | 1980 | TODO: also trivial builtins. |
1981 | builtin_constant_p is already handled later. */ | |
a41f2a28 | 1982 | if (gimple_code (stmt) != GIMPLE_ASSIGN |
1983 | && gimple_code (stmt) != GIMPLE_COND | |
1984 | && gimple_code (stmt) != GIMPLE_SWITCH) | |
1985 | return p; | |
1986 | ||
8e22665e | 1987 | /* Stores will stay anyway. */ |
3172b7bf | 1988 | if (gimple_store_p (stmt)) |
a41f2a28 | 1989 | return p; |
1990 | ||
3172b7bf | 1991 | is_load = gimple_assign_load_p (stmt); |
1992 | ||
8e22665e | 1993 | /* Loads can be optimized when the value is known. */ |
1994 | if (is_load) | |
1995 | { | |
a4f60e55 | 1996 | tree op; |
8e22665e | 1997 | gcc_assert (gimple_assign_single_p (stmt)); |
a4f60e55 | 1998 | op = gimple_assign_rhs1 (stmt); |
1999 | if (!unmodified_parm_or_parm_agg_item (info, stmt, op, &base_index, | |
2000 | &aggpos)) | |
8e22665e | 2001 | return p; |
2002 | } | |
a4f60e55 | 2003 | else |
2004 | base_index = -1; | |
8e22665e | 2005 | |
a41f2a28 | 2006 | /* See if we understand all operands before we start |
2007 | adding conditionals. */ | |
2008 | FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE) | |
2009 | { | |
94646c9c | 2010 | tree parm = unmodified_parm (stmt, use); |
0b50fa0e | 2011 | /* For arguments we can build a condition. */ |
94646c9c | 2012 | if (parm && ipa_get_param_decl_index (info, parm) >= 0) |
0b50fa0e | 2013 | continue; |
94646c9c | 2014 | if (TREE_CODE (use) != SSA_NAME) |
2015 | return p; | |
0b50fa0e | 2016 | /* If we know when operand is constant, |
2017 | we still can say something useful. */ | |
f1f41a6c | 2018 | if (!true_predicate_p (&nonconstant_names[SSA_NAME_VERSION (use)])) |
0b50fa0e | 2019 | continue; |
2020 | return p; | |
a41f2a28 | 2021 | } |
a4f60e55 | 2022 | |
8e22665e | 2023 | if (is_load) |
e876d531 | 2024 | op_non_const = |
2025 | add_condition (summary, base_index, &aggpos, CHANGED, NULL); | |
a4f60e55 | 2026 | else |
2027 | op_non_const = false_predicate (); | |
a41f2a28 | 2028 | FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE) |
2029 | { | |
94646c9c | 2030 | tree parm = unmodified_parm (stmt, use); |
a4f60e55 | 2031 | int index; |
2032 | ||
e876d531 | 2033 | if (parm && (index = ipa_get_param_decl_index (info, parm)) >= 0) |
a4f60e55 | 2034 | { |
2035 | if (index != base_index) | |
2036 | p = add_condition (summary, index, NULL, CHANGED, NULL_TREE); | |
2037 | else | |
2038 | continue; | |
2039 | } | |
0b50fa0e | 2040 | else |
f1f41a6c | 2041 | p = nonconstant_names[SSA_NAME_VERSION (use)]; |
94646c9c | 2042 | op_non_const = or_predicates (summary->conds, &p, &op_non_const); |
a41f2a28 | 2043 | } |
0b50fa0e | 2044 | if (gimple_code (stmt) == GIMPLE_ASSIGN |
2045 | && TREE_CODE (gimple_assign_lhs (stmt)) == SSA_NAME) | |
f1f41a6c | 2046 | nonconstant_names[SSA_NAME_VERSION (gimple_assign_lhs (stmt))] |
e876d531 | 2047 | = op_non_const; |
a41f2a28 | 2048 | return op_non_const; |
2049 | } | |
2050 | ||
eb4ae064 | 2051 | struct record_modified_bb_info |
2052 | { | |
2053 | bitmap bb_set; | |
2054 | gimple stmt; | |
2055 | }; | |
2056 | ||
2057 | /* Callback of walk_aliased_vdefs. Records basic blocks where the value may be | |
2058 | set except for info->stmt. */ | |
2059 | ||
2060 | static bool | |
e876d531 | 2061 | record_modified (ao_ref *ao ATTRIBUTE_UNUSED, tree vdef, void *data) |
eb4ae064 | 2062 | { |
e876d531 | 2063 | struct record_modified_bb_info *info = |
2064 | (struct record_modified_bb_info *) data; | |
eb4ae064 | 2065 | if (SSA_NAME_DEF_STMT (vdef) == info->stmt) |
2066 | return false; | |
2067 | bitmap_set_bit (info->bb_set, | |
2068 | SSA_NAME_IS_DEFAULT_DEF (vdef) | |
e876d531 | 2069 | ? ENTRY_BLOCK_PTR->index |
2070 | : gimple_bb (SSA_NAME_DEF_STMT (vdef))->index); | |
eb4ae064 | 2071 | return false; |
2072 | } | |
2073 | ||
2074 | /* Return probability (based on REG_BR_PROB_BASE) that I-th parameter of STMT | |
2075 | will change since last invocation of STMT. | |
2076 | ||
2077 | Value 0 is reserved for compile time invariants. | |
2078 | For common parameters it is REG_BR_PROB_BASE. For loop invariants it | |
2079 | ought to be REG_BR_PROB_BASE / estimated_iters. */ | |
2080 | ||
2081 | static int | |
2082 | param_change_prob (gimple stmt, int i) | |
2083 | { | |
2084 | tree op = gimple_call_arg (stmt, i); | |
2085 | basic_block bb = gimple_bb (stmt); | |
2086 | tree base; | |
2087 | ||
e876d531 | 2088 | /* Global invariants neve change. */ |
eb4ae064 | 2089 | if (is_gimple_min_invariant (op)) |
2090 | return 0; | |
2091 | /* We would have to do non-trivial analysis to really work out what | |
2092 | is the probability of value to change (i.e. when init statement | |
2093 | is in a sibling loop of the call). | |
2094 | ||
2095 | We do an conservative estimate: when call is executed N times more often | |
2096 | than the statement defining value, we take the frequency 1/N. */ | |
2097 | if (TREE_CODE (op) == SSA_NAME) | |
2098 | { | |
2099 | int init_freq; | |
2100 | ||
2101 | if (!bb->frequency) | |
2102 | return REG_BR_PROB_BASE; | |
2103 | ||
2104 | if (SSA_NAME_IS_DEFAULT_DEF (op)) | |
2105 | init_freq = ENTRY_BLOCK_PTR->frequency; | |
2106 | else | |
2107 | init_freq = gimple_bb (SSA_NAME_DEF_STMT (op))->frequency; | |
2108 | ||
2109 | if (!init_freq) | |
2110 | init_freq = 1; | |
2111 | if (init_freq < bb->frequency) | |
f9d4b7f4 | 2112 | return MAX (GCOV_COMPUTE_SCALE (init_freq, bb->frequency), 1); |
eb4ae064 | 2113 | else |
e876d531 | 2114 | return REG_BR_PROB_BASE; |
eb4ae064 | 2115 | } |
2116 | ||
2117 | base = get_base_address (op); | |
2118 | if (base) | |
2119 | { | |
2120 | ao_ref refd; | |
2121 | int max; | |
2122 | struct record_modified_bb_info info; | |
2123 | bitmap_iterator bi; | |
2124 | unsigned index; | |
df8d3e89 | 2125 | tree init = ctor_for_folding (base); |
eb4ae064 | 2126 | |
df8d3e89 | 2127 | if (init != error_mark_node) |
eb4ae064 | 2128 | return 0; |
2129 | if (!bb->frequency) | |
2130 | return REG_BR_PROB_BASE; | |
2131 | ao_ref_init (&refd, op); | |
2132 | info.stmt = stmt; | |
2133 | info.bb_set = BITMAP_ALLOC (NULL); | |
2134 | walk_aliased_vdefs (&refd, gimple_vuse (stmt), record_modified, &info, | |
2135 | NULL); | |
2136 | if (bitmap_bit_p (info.bb_set, bb->index)) | |
2137 | { | |
e876d531 | 2138 | BITMAP_FREE (info.bb_set); |
eb4ae064 | 2139 | return REG_BR_PROB_BASE; |
2140 | } | |
2141 | ||
2142 | /* Assume that every memory is initialized at entry. | |
e876d531 | 2143 | TODO: Can we easilly determine if value is always defined |
2144 | and thus we may skip entry block? */ | |
eb4ae064 | 2145 | if (ENTRY_BLOCK_PTR->frequency) |
2146 | max = ENTRY_BLOCK_PTR->frequency; | |
2147 | else | |
2148 | max = 1; | |
2149 | ||
2150 | EXECUTE_IF_SET_IN_BITMAP (info.bb_set, 0, index, bi) | |
2151 | max = MIN (max, BASIC_BLOCK (index)->frequency); | |
e876d531 | 2152 | |
eb4ae064 | 2153 | BITMAP_FREE (info.bb_set); |
2154 | if (max < bb->frequency) | |
f9d4b7f4 | 2155 | return MAX (GCOV_COMPUTE_SCALE (max, bb->frequency), 1); |
eb4ae064 | 2156 | else |
e876d531 | 2157 | return REG_BR_PROB_BASE; |
eb4ae064 | 2158 | } |
2159 | return REG_BR_PROB_BASE; | |
2160 | } | |
2161 | ||
32691791 | 2162 | /* Find whether a basic block BB is the final block of a (half) diamond CFG |
2163 | sub-graph and if the predicate the condition depends on is known. If so, | |
2164 | return true and store the pointer the predicate in *P. */ | |
2165 | ||
2166 | static bool | |
2167 | phi_result_unknown_predicate (struct ipa_node_params *info, | |
2168 | struct inline_summary *summary, basic_block bb, | |
2169 | struct predicate *p, | |
f1f41a6c | 2170 | vec<predicate_t> nonconstant_names) |
32691791 | 2171 | { |
2172 | edge e; | |
2173 | edge_iterator ei; | |
2174 | basic_block first_bb = NULL; | |
2175 | gimple stmt; | |
2176 | ||
2177 | if (single_pred_p (bb)) | |
2178 | { | |
2179 | *p = false_predicate (); | |
2180 | return true; | |
2181 | } | |
2182 | ||
2183 | FOR_EACH_EDGE (e, ei, bb->preds) | |
2184 | { | |
2185 | if (single_succ_p (e->src)) | |
2186 | { | |
2187 | if (!single_pred_p (e->src)) | |
2188 | return false; | |
2189 | if (!first_bb) | |
2190 | first_bb = single_pred (e->src); | |
2191 | else if (single_pred (e->src) != first_bb) | |
2192 | return false; | |
2193 | } | |
2194 | else | |
2195 | { | |
2196 | if (!first_bb) | |
2197 | first_bb = e->src; | |
2198 | else if (e->src != first_bb) | |
2199 | return false; | |
2200 | } | |
2201 | } | |
2202 | ||
2203 | if (!first_bb) | |
2204 | return false; | |
2205 | ||
2206 | stmt = last_stmt (first_bb); | |
2207 | if (!stmt | |
2208 | || gimple_code (stmt) != GIMPLE_COND | |
2209 | || !is_gimple_ip_invariant (gimple_cond_rhs (stmt))) | |
2210 | return false; | |
2211 | ||
2212 | *p = will_be_nonconstant_expr_predicate (info, summary, | |
2213 | gimple_cond_lhs (stmt), | |
2214 | nonconstant_names); | |
2215 | if (true_predicate_p (p)) | |
2216 | return false; | |
2217 | else | |
2218 | return true; | |
2219 | } | |
2220 | ||
2221 | /* Given a PHI statement in a function described by inline properties SUMMARY | |
2222 | and *P being the predicate describing whether the selected PHI argument is | |
2223 | known, store a predicate for the result of the PHI statement into | |
2224 | NONCONSTANT_NAMES, if possible. */ | |
2225 | ||
2226 | static void | |
2227 | predicate_for_phi_result (struct inline_summary *summary, gimple phi, | |
2228 | struct predicate *p, | |
f1f41a6c | 2229 | vec<predicate_t> nonconstant_names) |
32691791 | 2230 | { |
2231 | unsigned i; | |
2232 | ||
2233 | for (i = 0; i < gimple_phi_num_args (phi); i++) | |
2234 | { | |
2235 | tree arg = gimple_phi_arg (phi, i)->def; | |
2236 | if (!is_gimple_min_invariant (arg)) | |
2237 | { | |
2238 | gcc_assert (TREE_CODE (arg) == SSA_NAME); | |
2239 | *p = or_predicates (summary->conds, p, | |
f1f41a6c | 2240 | &nonconstant_names[SSA_NAME_VERSION (arg)]); |
32691791 | 2241 | if (true_predicate_p (p)) |
2242 | return; | |
2243 | } | |
2244 | } | |
2245 | ||
2246 | if (dump_file && (dump_flags & TDF_DETAILS)) | |
2247 | { | |
2248 | fprintf (dump_file, "\t\tphi predicate: "); | |
2249 | dump_predicate (dump_file, summary->conds, p); | |
2250 | } | |
f1f41a6c | 2251 | nonconstant_names[SSA_NAME_VERSION (gimple_phi_result (phi))] = *p; |
32691791 | 2252 | } |
a41f2a28 | 2253 | |
be343a9c | 2254 | /* Return predicate specifying when array index in access OP becomes non-constant. */ |
2255 | ||
2256 | static struct predicate | |
2257 | array_index_predicate (struct inline_summary *info, | |
e876d531 | 2258 | vec< predicate_t> nonconstant_names, tree op) |
be343a9c | 2259 | { |
2260 | struct predicate p = false_predicate (); | |
2261 | while (handled_component_p (op)) | |
2262 | { | |
e876d531 | 2263 | if (TREE_CODE (op) == ARRAY_REF || TREE_CODE (op) == ARRAY_RANGE_REF) |
2264 | { | |
be343a9c | 2265 | if (TREE_CODE (TREE_OPERAND (op, 1)) == SSA_NAME) |
e876d531 | 2266 | p = or_predicates (info->conds, &p, |
2267 | &nonconstant_names[SSA_NAME_VERSION | |
2268 | (TREE_OPERAND (op, 1))]); | |
2269 | } | |
be343a9c | 2270 | op = TREE_OPERAND (op, 0); |
2271 | } | |
2272 | return p; | |
2273 | } | |
2274 | ||
caa0f772 | 2275 | /* For a typical usage of __builtin_expect (a<b, 1), we |
2276 | may introduce an extra relation stmt: | |
2277 | With the builtin, we have | |
2278 | t1 = a <= b; | |
2279 | t2 = (long int) t1; | |
2280 | t3 = __builtin_expect (t2, 1); | |
2281 | if (t3 != 0) | |
2282 | goto ... | |
2283 | Without the builtin, we have | |
2284 | if (a<=b) | |
2285 | goto... | |
2286 | This affects the size/time estimation and may have | |
2287 | an impact on the earlier inlining. | |
2288 | Here find this pattern and fix it up later. */ | |
2289 | ||
2290 | static gimple | |
2291 | find_foldable_builtin_expect (basic_block bb) | |
2292 | { | |
2293 | gimple_stmt_iterator bsi; | |
2294 | ||
2295 | for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi)) | |
2296 | { | |
2297 | gimple stmt = gsi_stmt (bsi); | |
2298 | if (gimple_call_builtin_p (stmt, BUILT_IN_EXPECT)) | |
2299 | { | |
2300 | tree var = gimple_call_lhs (stmt); | |
2301 | tree arg = gimple_call_arg (stmt, 0); | |
2302 | use_operand_p use_p; | |
2303 | gimple use_stmt; | |
2304 | bool match = false; | |
2305 | bool done = false; | |
2306 | ||
2307 | if (!var || !arg) | |
2308 | continue; | |
2309 | gcc_assert (TREE_CODE (var) == SSA_NAME); | |
2310 | ||
2311 | while (TREE_CODE (arg) == SSA_NAME) | |
2312 | { | |
2313 | gimple stmt_tmp = SSA_NAME_DEF_STMT (arg); | |
2314 | if (!is_gimple_assign (stmt_tmp)) | |
2315 | break; | |
2316 | switch (gimple_assign_rhs_code (stmt_tmp)) | |
2317 | { | |
2318 | case LT_EXPR: | |
2319 | case LE_EXPR: | |
2320 | case GT_EXPR: | |
2321 | case GE_EXPR: | |
2322 | case EQ_EXPR: | |
2323 | case NE_EXPR: | |
2324 | match = true; | |
2325 | done = true; | |
2326 | break; | |
2327 | case NOP_EXPR: | |
2328 | break; | |
2329 | default: | |
2330 | done = true; | |
2331 | break; | |
2332 | } | |
2333 | if (done) | |
2334 | break; | |
2335 | arg = gimple_assign_rhs1 (stmt_tmp); | |
2336 | } | |
2337 | ||
2338 | if (match && single_imm_use (var, &use_p, &use_stmt) | |
2339 | && gimple_code (use_stmt) == GIMPLE_COND) | |
2340 | return use_stmt; | |
2341 | } | |
2342 | } | |
2343 | return NULL; | |
2344 | } | |
2345 | ||
a41f2a28 | 2346 | /* Compute function body size parameters for NODE. |
2347 | When EARLY is true, we compute only simple summaries without | |
2348 | non-trivial predicates to drive the early inliner. */ | |
99c67f24 | 2349 | |
2350 | static void | |
a41f2a28 | 2351 | estimate_function_body_sizes (struct cgraph_node *node, bool early) |
99c67f24 | 2352 | { |
2353 | gcov_type time = 0; | |
99c67f24 | 2354 | /* Estimate static overhead for function prologue/epilogue and alignment. */ |
2355 | int size = 2; | |
2356 | /* Benefits are scaled by probability of elimination that is in range | |
2357 | <0,2>. */ | |
99c67f24 | 2358 | basic_block bb; |
2359 | gimple_stmt_iterator bsi; | |
02774f2d | 2360 | struct function *my_function = DECL_STRUCT_FUNCTION (node->decl); |
99c67f24 | 2361 | int freq; |
a41f2a28 | 2362 | struct inline_summary *info = inline_summary (node); |
2363 | struct predicate bb_predicate; | |
0b50fa0e | 2364 | struct ipa_node_params *parms_info = NULL; |
1e094109 | 2365 | vec<predicate_t> nonconstant_names = vNULL; |
563fae60 | 2366 | int nblocks, n; |
2367 | int *order; | |
be343a9c | 2368 | predicate array_index = true_predicate (); |
caa0f772 | 2369 | gimple fix_builtin_expect_stmt; |
a41f2a28 | 2370 | |
f1f41a6c | 2371 | info->conds = NULL; |
2372 | info->entry = NULL; | |
a41f2a28 | 2373 | |
4b209fe7 | 2374 | if (optimize && !early) |
2375 | { | |
2376 | calculate_dominance_info (CDI_DOMINATORS); | |
2377 | loop_optimizer_init (LOOPS_NORMAL | LOOPS_HAVE_RECORDED_EXITS); | |
f228b1b9 | 2378 | |
f1f41a6c | 2379 | if (ipa_node_params_vector.exists ()) |
f228b1b9 | 2380 | { |
2381 | parms_info = IPA_NODE_REF (node); | |
e876d531 | 2382 | nonconstant_names.safe_grow_cleared |
2383 | (SSANAMES (my_function)->length ()); | |
f228b1b9 | 2384 | } |
4b209fe7 | 2385 | } |
99c67f24 | 2386 | |
2387 | if (dump_file) | |
a41f2a28 | 2388 | fprintf (dump_file, "\nAnalyzing function body size: %s\n", |
f1c8b4d7 | 2389 | node->name ()); |
99c67f24 | 2390 | |
a41f2a28 | 2391 | /* When we run into maximal number of entries, we assign everything to the |
2392 | constant truth case. Be sure to have it in list. */ | |
2393 | bb_predicate = true_predicate (); | |
2394 | account_size_time (info, 0, 0, &bb_predicate); | |
2395 | ||
2396 | bb_predicate = not_inlined_predicate (); | |
2397 | account_size_time (info, 2 * INLINE_SIZE_SCALE, 0, &bb_predicate); | |
2398 | ||
99c67f24 | 2399 | gcc_assert (my_function && my_function->cfg); |
905aa3bd | 2400 | if (parms_info) |
2401 | compute_bb_predicates (node, parms_info, info); | |
563fae60 | 2402 | gcc_assert (cfun == my_function); |
a28770e1 | 2403 | order = XNEWVEC (int, n_basic_blocks_for_fn (cfun)); |
563fae60 | 2404 | nblocks = pre_and_rev_post_order_compute (NULL, order, false); |
2405 | for (n = 0; n < nblocks; n++) | |
99c67f24 | 2406 | { |
563fae60 | 2407 | bb = BASIC_BLOCK (order[n]); |
02774f2d | 2408 | freq = compute_call_stmt_bb_frequency (node->decl, bb); |
a41f2a28 | 2409 | |
2410 | /* TODO: Obviously predicates can be propagated down across CFG. */ | |
2411 | if (parms_info) | |
2412 | { | |
905aa3bd | 2413 | if (bb->aux) |
e876d531 | 2414 | bb_predicate = *(struct predicate *) bb->aux; |
905aa3bd | 2415 | else |
2416 | bb_predicate = false_predicate (); | |
a41f2a28 | 2417 | } |
2418 | else | |
2419 | bb_predicate = true_predicate (); | |
2420 | ||
2421 | if (dump_file && (dump_flags & TDF_DETAILS)) | |
2422 | { | |
2423 | fprintf (dump_file, "\n BB %i predicate:", bb->index); | |
2424 | dump_predicate (dump_file, info->conds, &bb_predicate); | |
2425 | } | |
32691791 | 2426 | |
f1f41a6c | 2427 | if (parms_info && nonconstant_names.exists ()) |
32691791 | 2428 | { |
2429 | struct predicate phi_predicate; | |
2430 | bool first_phi = true; | |
2431 | ||
2432 | for (bsi = gsi_start_phis (bb); !gsi_end_p (bsi); gsi_next (&bsi)) | |
2433 | { | |
2434 | if (first_phi | |
2435 | && !phi_result_unknown_predicate (parms_info, info, bb, | |
2436 | &phi_predicate, | |
2437 | nonconstant_names)) | |
2438 | break; | |
2439 | first_phi = false; | |
2440 | if (dump_file && (dump_flags & TDF_DETAILS)) | |
2441 | { | |
2442 | fprintf (dump_file, " "); | |
2443 | print_gimple_stmt (dump_file, gsi_stmt (bsi), 0, 0); | |
2444 | } | |
2445 | predicate_for_phi_result (info, gsi_stmt (bsi), &phi_predicate, | |
2446 | nonconstant_names); | |
2447 | } | |
2448 | } | |
2449 | ||
caa0f772 | 2450 | fix_builtin_expect_stmt = find_foldable_builtin_expect (bb); |
2451 | ||
99c67f24 | 2452 | for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi)) |
2453 | { | |
2454 | gimple stmt = gsi_stmt (bsi); | |
2455 | int this_size = estimate_num_insns (stmt, &eni_size_weights); | |
2456 | int this_time = estimate_num_insns (stmt, &eni_time_weights); | |
2457 | int prob; | |
905aa3bd | 2458 | struct predicate will_be_nonconstant; |
99c67f24 | 2459 | |
caa0f772 | 2460 | /* This relation stmt should be folded after we remove |
2461 | buildin_expect call. Adjust the cost here. */ | |
2462 | if (stmt == fix_builtin_expect_stmt) | |
2463 | { | |
2464 | this_size--; | |
2465 | this_time--; | |
2466 | } | |
2467 | ||
99c67f24 | 2468 | if (dump_file && (dump_flags & TDF_DETAILS)) |
2469 | { | |
a41f2a28 | 2470 | fprintf (dump_file, " "); |
99c67f24 | 2471 | print_gimple_stmt (dump_file, stmt, 0, 0); |
a41f2a28 | 2472 | fprintf (dump_file, "\t\tfreq:%3.2f size:%3i time:%3i\n", |
e876d531 | 2473 | ((double) freq) / CGRAPH_FREQ_BASE, this_size, |
2474 | this_time); | |
99c67f24 | 2475 | } |
c7b2cc59 | 2476 | |
f1f41a6c | 2477 | if (gimple_assign_load_p (stmt) && nonconstant_names.exists ()) |
be343a9c | 2478 | { |
2479 | struct predicate this_array_index; | |
e876d531 | 2480 | this_array_index = |
2481 | array_index_predicate (info, nonconstant_names, | |
2482 | gimple_assign_rhs1 (stmt)); | |
be343a9c | 2483 | if (!false_predicate_p (&this_array_index)) |
e876d531 | 2484 | array_index = |
2485 | and_predicates (info->conds, &array_index, | |
2486 | &this_array_index); | |
be343a9c | 2487 | } |
f1f41a6c | 2488 | if (gimple_store_p (stmt) && nonconstant_names.exists ()) |
be343a9c | 2489 | { |
2490 | struct predicate this_array_index; | |
e876d531 | 2491 | this_array_index = |
2492 | array_index_predicate (info, nonconstant_names, | |
2493 | gimple_get_lhs (stmt)); | |
be343a9c | 2494 | if (!false_predicate_p (&this_array_index)) |
e876d531 | 2495 | array_index = |
2496 | and_predicates (info->conds, &array_index, | |
2497 | &this_array_index); | |
be343a9c | 2498 | } |
e876d531 | 2499 | |
be343a9c | 2500 | |
c7b2cc59 | 2501 | if (is_gimple_call (stmt)) |
2502 | { | |
2503 | struct cgraph_edge *edge = cgraph_edge (node, stmt); | |
0835ad03 | 2504 | struct inline_edge_summary *es = inline_edge_summary (edge); |
2505 | ||
0b50fa0e | 2506 | /* Special case: results of BUILT_IN_CONSTANT_P will be always |
e876d531 | 2507 | resolved as constant. We however don't want to optimize |
2508 | out the cgraph edges. */ | |
f1f41a6c | 2509 | if (nonconstant_names.exists () |
0b50fa0e | 2510 | && gimple_call_builtin_p (stmt, BUILT_IN_CONSTANT_P) |
2511 | && gimple_call_lhs (stmt) | |
2512 | && TREE_CODE (gimple_call_lhs (stmt)) == SSA_NAME) | |
2513 | { | |
2514 | struct predicate false_p = false_predicate (); | |
f1f41a6c | 2515 | nonconstant_names[SSA_NAME_VERSION (gimple_call_lhs (stmt))] |
e876d531 | 2516 | = false_p; |
eb4ae064 | 2517 | } |
f1f41a6c | 2518 | if (ipa_node_params_vector.exists ()) |
eb4ae064 | 2519 | { |
e876d531 | 2520 | int count = gimple_call_num_args (stmt); |
eb4ae064 | 2521 | int i; |
2522 | ||
2523 | if (count) | |
f1f41a6c | 2524 | es->param.safe_grow_cleared (count); |
eb4ae064 | 2525 | for (i = 0; i < count; i++) |
2526 | { | |
2527 | int prob = param_change_prob (stmt, i); | |
2528 | gcc_assert (prob >= 0 && prob <= REG_BR_PROB_BASE); | |
f1f41a6c | 2529 | es->param[i].change_prob = prob; |
eb4ae064 | 2530 | } |
0b50fa0e | 2531 | } |
2532 | ||
0835ad03 | 2533 | es->call_stmt_size = this_size; |
2534 | es->call_stmt_time = this_time; | |
6b42039a | 2535 | es->loop_depth = bb_loop_depth (bb); |
6a18c0be | 2536 | edge_set_predicate (edge, &bb_predicate); |
c7b2cc59 | 2537 | } |
2538 | ||
905aa3bd | 2539 | /* TODO: When conditional jump or swithc is known to be constant, but |
e876d531 | 2540 | we did not translate it into the predicates, we really can account |
905aa3bd | 2541 | just maximum of the possible paths. */ |
2542 | if (parms_info) | |
2543 | will_be_nonconstant | |
e876d531 | 2544 | = will_be_nonconstant_predicate (parms_info, info, |
2545 | stmt, nonconstant_names); | |
a41f2a28 | 2546 | if (this_time || this_size) |
2547 | { | |
a41f2a28 | 2548 | struct predicate p; |
2549 | ||
2550 | this_time *= freq; | |
c7b2cc59 | 2551 | |
a41f2a28 | 2552 | prob = eliminated_by_inlining_prob (stmt); |
2553 | if (prob == 1 && dump_file && (dump_flags & TDF_DETAILS)) | |
e876d531 | 2554 | fprintf (dump_file, |
2555 | "\t\t50%% will be eliminated by inlining\n"); | |
a41f2a28 | 2556 | if (prob == 2 && dump_file && (dump_flags & TDF_DETAILS)) |
d4db9dfd | 2557 | fprintf (dump_file, "\t\tWill be eliminated by inlining\n"); |
a41f2a28 | 2558 | |
2559 | if (parms_info) | |
fb3c587e | 2560 | p = and_predicates (info->conds, &bb_predicate, |
2561 | &will_be_nonconstant); | |
a41f2a28 | 2562 | else |
2563 | p = true_predicate (); | |
c7b2cc59 | 2564 | |
18b64b34 | 2565 | if (!false_predicate_p (&p)) |
2566 | { | |
2567 | time += this_time; | |
2568 | size += this_size; | |
3c49142d | 2569 | if (time > MAX_TIME * INLINE_TIME_SCALE) |
2570 | time = MAX_TIME * INLINE_TIME_SCALE; | |
18b64b34 | 2571 | } |
2572 | ||
a41f2a28 | 2573 | /* We account everything but the calls. Calls have their own |
e876d531 | 2574 | size/time info attached to cgraph edges. This is necessary |
2575 | in order to make the cost disappear after inlining. */ | |
a41f2a28 | 2576 | if (!is_gimple_call (stmt)) |
2577 | { | |
2578 | if (prob) | |
2579 | { | |
2580 | struct predicate ip = not_inlined_predicate (); | |
94646c9c | 2581 | ip = and_predicates (info->conds, &ip, &p); |
a41f2a28 | 2582 | account_size_time (info, this_size * prob, |
2583 | this_time * prob, &ip); | |
2584 | } | |
2585 | if (prob != 2) | |
2586 | account_size_time (info, this_size * (2 - prob), | |
2587 | this_time * (2 - prob), &p); | |
2588 | } | |
c7b2cc59 | 2589 | |
a41f2a28 | 2590 | gcc_assert (time >= 0); |
2591 | gcc_assert (size >= 0); | |
2592 | } | |
99c67f24 | 2593 | } |
2594 | } | |
be343a9c | 2595 | set_hint_predicate (&inline_summary (node)->array_index, array_index); |
99c67f24 | 2596 | time = (time + CGRAPH_FREQ_BASE / 2) / CGRAPH_FREQ_BASE; |
99c67f24 | 2597 | if (time > MAX_TIME) |
2598 | time = MAX_TIME; | |
563fae60 | 2599 | free (order); |
7c07aa3d | 2600 | |
f1f41a6c | 2601 | if (!early && nonconstant_names.exists ()) |
7c07aa3d | 2602 | { |
2603 | struct loop *loop; | |
7c07aa3d | 2604 | predicate loop_iterations = true_predicate (); |
3716ee8f | 2605 | predicate loop_stride = true_predicate (); |
7c07aa3d | 2606 | |
7c07aa3d | 2607 | if (dump_file && (dump_flags & TDF_DETAILS)) |
2608 | flow_loops_dump (dump_file, NULL, 0); | |
2609 | scev_initialize (); | |
f21d4d00 | 2610 | FOR_EACH_LOOP (loop, 0) |
7c07aa3d | 2611 | { |
e876d531 | 2612 | vec<edge> exits; |
2613 | edge ex; | |
3716ee8f | 2614 | unsigned int j, i; |
7c07aa3d | 2615 | struct tree_niter_desc niter_desc; |
3716ee8f | 2616 | basic_block *body = get_loop_body (loop); |
e876d531 | 2617 | bb_predicate = *(struct predicate *) loop->header->aux; |
7c07aa3d | 2618 | |
2619 | exits = get_loop_exit_edges (loop); | |
e876d531 | 2620 | FOR_EACH_VEC_ELT (exits, j, ex) |
7c07aa3d | 2621 | if (number_of_iterations_exit (loop, ex, &niter_desc, false) |
2622 | && !is_gimple_min_invariant (niter_desc.niter)) | |
e876d531 | 2623 | { |
2624 | predicate will_be_nonconstant | |
2625 | = will_be_nonconstant_expr_predicate (parms_info, info, | |
2626 | niter_desc.niter, | |
2627 | nonconstant_names); | |
2628 | if (!true_predicate_p (&will_be_nonconstant)) | |
2629 | will_be_nonconstant = and_predicates (info->conds, | |
2630 | &bb_predicate, | |
2631 | &will_be_nonconstant); | |
2632 | if (!true_predicate_p (&will_be_nonconstant) | |
2633 | && !false_predicate_p (&will_be_nonconstant)) | |
2634 | /* This is slightly inprecise. We may want to represent each | |
2635 | loop with independent predicate. */ | |
2636 | loop_iterations = | |
2637 | and_predicates (info->conds, &loop_iterations, | |
2638 | &will_be_nonconstant); | |
2639 | } | |
2640 | exits.release (); | |
3716ee8f | 2641 | |
e876d531 | 2642 | for (i = 0; i < loop->num_nodes; i++) |
3716ee8f | 2643 | { |
2644 | gimple_stmt_iterator gsi; | |
e876d531 | 2645 | bb_predicate = *(struct predicate *) body[i]->aux; |
2646 | for (gsi = gsi_start_bb (body[i]); !gsi_end_p (gsi); | |
2647 | gsi_next (&gsi)) | |
3716ee8f | 2648 | { |
2649 | gimple stmt = gsi_stmt (gsi); | |
2650 | affine_iv iv; | |
2651 | ssa_op_iter iter; | |
2652 | tree use; | |
2653 | ||
2654 | FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE) | |
e876d531 | 2655 | { |
2656 | predicate will_be_nonconstant; | |
2657 | ||
2658 | if (!simple_iv | |
2659 | (loop, loop_containing_stmt (stmt), use, &iv, true) | |
2660 | || is_gimple_min_invariant (iv.step)) | |
2661 | continue; | |
2662 | will_be_nonconstant | |
2663 | = will_be_nonconstant_expr_predicate (parms_info, info, | |
2664 | iv.step, | |
2665 | nonconstant_names); | |
2666 | if (!true_predicate_p (&will_be_nonconstant)) | |
3716ee8f | 2667 | will_be_nonconstant |
e876d531 | 2668 | = and_predicates (info->conds, |
2669 | &bb_predicate, | |
2670 | &will_be_nonconstant); | |
2671 | if (!true_predicate_p (&will_be_nonconstant) | |
2672 | && !false_predicate_p (&will_be_nonconstant)) | |
2673 | /* This is slightly inprecise. We may want to represent | |
2674 | each loop with independent predicate. */ | |
2675 | loop_stride = | |
2676 | and_predicates (info->conds, &loop_stride, | |
2677 | &will_be_nonconstant); | |
2678 | } | |
3716ee8f | 2679 | } |
2680 | } | |
2681 | free (body); | |
7c07aa3d | 2682 | } |
e876d531 | 2683 | set_hint_predicate (&inline_summary (node)->loop_iterations, |
2684 | loop_iterations); | |
3716ee8f | 2685 | set_hint_predicate (&inline_summary (node)->loop_stride, loop_stride); |
7c07aa3d | 2686 | scev_finalize (); |
7c07aa3d | 2687 | } |
be343a9c | 2688 | FOR_ALL_BB_FN (bb, my_function) |
2689 | { | |
2690 | edge e; | |
2691 | edge_iterator ei; | |
2692 | ||
2693 | if (bb->aux) | |
2694 | pool_free (edge_predicate_pool, bb->aux); | |
2695 | bb->aux = NULL; | |
2696 | FOR_EACH_EDGE (e, ei, bb->succs) | |
2697 | { | |
2698 | if (e->aux) | |
2699 | pool_free (edge_predicate_pool, e->aux); | |
2700 | e->aux = NULL; | |
2701 | } | |
2702 | } | |
99c67f24 | 2703 | inline_summary (node)->self_time = time; |
2704 | inline_summary (node)->self_size = size; | |
f1f41a6c | 2705 | nonconstant_names.release (); |
4b209fe7 | 2706 | if (optimize && !early) |
2707 | { | |
2708 | loop_optimizer_finalize (); | |
2709 | free_dominance_info (CDI_DOMINATORS); | |
2710 | } | |
a41f2a28 | 2711 | if (dump_file) |
2712 | { | |
2713 | fprintf (dump_file, "\n"); | |
2714 | dump_inline_summary (dump_file, node); | |
2715 | } | |
99c67f24 | 2716 | } |
2717 | ||
2718 | ||
a41f2a28 | 2719 | /* Compute parameters of functions used by inliner. |
2720 | EARLY is true when we compute parameters for the early inliner */ | |
99c67f24 | 2721 | |
2722 | void | |
a41f2a28 | 2723 | compute_inline_parameters (struct cgraph_node *node, bool early) |
99c67f24 | 2724 | { |
2725 | HOST_WIDE_INT self_stack_size; | |
2726 | struct cgraph_edge *e; | |
cbd7f5a0 | 2727 | struct inline_summary *info; |
99c67f24 | 2728 | |
2729 | gcc_assert (!node->global.inlined_to); | |
2730 | ||
c7b2cc59 | 2731 | inline_summary_alloc (); |
2732 | ||
cbd7f5a0 | 2733 | info = inline_summary (node); |
3b9dd281 | 2734 | reset_inline_summary (node); |
cbd7f5a0 | 2735 | |
91bf9d9a | 2736 | /* FIXME: Thunks are inlinable, but tree-inline don't know how to do that. |
2737 | Once this happen, we will need to more curefully predict call | |
2738 | statement size. */ | |
2739 | if (node->thunk.thunk_p) | |
2740 | { | |
2741 | struct inline_edge_summary *es = inline_edge_summary (node->callees); | |
2742 | struct predicate t = true_predicate (); | |
2743 | ||
c8d92fc1 | 2744 | info->inlinable = 0; |
91bf9d9a | 2745 | node->callees->call_stmt_cannot_inline_p = true; |
2746 | node->local.can_change_signature = false; | |
2747 | es->call_stmt_time = 1; | |
2748 | es->call_stmt_size = 1; | |
2749 | account_size_time (info, 0, 0, &t); | |
2750 | return; | |
2751 | } | |
2752 | ||
8e22665e | 2753 | /* Even is_gimple_min_invariant rely on current_function_decl. */ |
02774f2d | 2754 | push_cfun (DECL_STRUCT_FUNCTION (node->decl)); |
8e22665e | 2755 | |
99c67f24 | 2756 | /* Estimate the stack size for the function if we're optimizing. */ |
2757 | self_stack_size = optimize ? estimated_stack_frame_size (node) : 0; | |
cbd7f5a0 | 2758 | info->estimated_self_stack_size = self_stack_size; |
2759 | info->estimated_stack_size = self_stack_size; | |
2760 | info->stack_frame_offset = 0; | |
99c67f24 | 2761 | |
2762 | /* Can this function be inlined at all? */ | |
26051fcf | 2763 | if (!optimize && !lookup_attribute ("always_inline", |
02774f2d | 2764 | DECL_ATTRIBUTES (node->decl))) |
26051fcf | 2765 | info->inlinable = false; |
2766 | else | |
02774f2d | 2767 | info->inlinable = tree_inlinable_function_p (node->decl); |
99c67f24 | 2768 | |
982ffd8d | 2769 | /* Type attributes can use parameter indices to describe them. */ |
02774f2d | 2770 | if (TYPE_ATTRIBUTES (TREE_TYPE (node->decl))) |
982ffd8d | 2771 | node->local.can_change_signature = false; |
99c67f24 | 2772 | else |
2773 | { | |
982ffd8d | 2774 | /* Otherwise, inlinable functions always can change signature. */ |
2775 | if (info->inlinable) | |
2776 | node->local.can_change_signature = true; | |
2777 | else | |
2778 | { | |
2779 | /* Functions calling builtin_apply can not change signature. */ | |
2780 | for (e = node->callees; e; e = e->next_callee) | |
2781 | { | |
02774f2d | 2782 | tree cdecl = e->callee->decl; |
982ffd8d | 2783 | if (DECL_BUILT_IN (cdecl) |
2784 | && DECL_BUILT_IN_CLASS (cdecl) == BUILT_IN_NORMAL | |
2785 | && (DECL_FUNCTION_CODE (cdecl) == BUILT_IN_APPLY_ARGS | |
2786 | || DECL_FUNCTION_CODE (cdecl) == BUILT_IN_VA_START)) | |
2787 | break; | |
2788 | } | |
2789 | node->local.can_change_signature = !e; | |
2790 | } | |
99c67f24 | 2791 | } |
a41f2a28 | 2792 | estimate_function_body_sizes (node, early); |
c7b2cc59 | 2793 | |
99c67f24 | 2794 | /* Inlining characteristics are maintained by the cgraph_mark_inline. */ |
cbd7f5a0 | 2795 | info->time = info->self_time; |
2796 | info->size = info->self_size; | |
cbd7f5a0 | 2797 | info->stack_frame_offset = 0; |
2798 | info->estimated_stack_size = info->estimated_self_stack_size; | |
18b64b34 | 2799 | #ifdef ENABLE_CHECKING |
2800 | inline_update_overall_summary (node); | |
e876d531 | 2801 | gcc_assert (info->time == info->self_time && info->size == info->self_size); |
18b64b34 | 2802 | #endif |
2803 | ||
8e22665e | 2804 | pop_cfun (); |
99c67f24 | 2805 | } |
2806 | ||
2807 | ||
2808 | /* Compute parameters of functions used by inliner using | |
2809 | current_function_decl. */ | |
2810 | ||
2811 | static unsigned int | |
2812 | compute_inline_parameters_for_current (void) | |
2813 | { | |
a41f2a28 | 2814 | compute_inline_parameters (cgraph_get_node (current_function_decl), true); |
99c67f24 | 2815 | return 0; |
2816 | } | |
2817 | ||
cbe8bda8 | 2818 | namespace { |
2819 | ||
2820 | const pass_data pass_data_inline_parameters = | |
99c67f24 | 2821 | { |
cbe8bda8 | 2822 | GIMPLE_PASS, /* type */ |
2823 | "inline_param", /* name */ | |
2824 | OPTGROUP_INLINE, /* optinfo_flags */ | |
2825 | false, /* has_gate */ | |
2826 | true, /* has_execute */ | |
2827 | TV_INLINE_PARAMETERS, /* tv_id */ | |
2828 | 0, /* properties_required */ | |
2829 | 0, /* properties_provided */ | |
2830 | 0, /* properties_destroyed */ | |
2831 | 0, /* todo_flags_start */ | |
2832 | 0, /* todo_flags_finish */ | |
99c67f24 | 2833 | }; |
2834 | ||
cbe8bda8 | 2835 | class pass_inline_parameters : public gimple_opt_pass |
2836 | { | |
2837 | public: | |
9af5ce0c | 2838 | pass_inline_parameters (gcc::context *ctxt) |
2839 | : gimple_opt_pass (pass_data_inline_parameters, ctxt) | |
cbe8bda8 | 2840 | {} |
2841 | ||
2842 | /* opt_pass methods: */ | |
ae84f584 | 2843 | opt_pass * clone () { return new pass_inline_parameters (m_ctxt); } |
cbe8bda8 | 2844 | unsigned int execute () { |
2845 | return compute_inline_parameters_for_current (); | |
2846 | } | |
2847 | ||
2848 | }; // class pass_inline_parameters | |
2849 | ||
2850 | } // anon namespace | |
2851 | ||
2852 | gimple_opt_pass * | |
2853 | make_pass_inline_parameters (gcc::context *ctxt) | |
2854 | { | |
2855 | return new pass_inline_parameters (ctxt); | |
2856 | } | |
2857 | ||
99c67f24 | 2858 | |
20da2013 | 2859 | /* Estimate benefit devirtualizing indirect edge IE, provided KNOWN_VALS and |
2860 | KNOWN_BINFOS. */ | |
2861 | ||
eb7c606e | 2862 | static bool |
20da2013 | 2863 | estimate_edge_devirt_benefit (struct cgraph_edge *ie, |
18b64b34 | 2864 | int *size, int *time, |
f1f41a6c | 2865 | vec<tree> known_vals, |
2866 | vec<tree> known_binfos, | |
2867 | vec<ipa_agg_jump_function_p> known_aggs) | |
20da2013 | 2868 | { |
2869 | tree target; | |
eb7c606e | 2870 | struct cgraph_node *callee; |
2871 | struct inline_summary *isummary; | |
20da2013 | 2872 | |
f1f41a6c | 2873 | if (!known_vals.exists () && !known_binfos.exists ()) |
eb7c606e | 2874 | return false; |
18b64b34 | 2875 | if (!flag_indirect_inlining) |
2876 | return false; | |
20da2013 | 2877 | |
a4f60e55 | 2878 | target = ipa_get_indirect_edge_target (ie, known_vals, known_binfos, |
2879 | known_aggs); | |
20da2013 | 2880 | if (!target) |
eb7c606e | 2881 | return false; |
20da2013 | 2882 | |
2883 | /* Account for difference in cost between indirect and direct calls. */ | |
18b64b34 | 2884 | *size -= (eni_size_weights.indirect_call_cost - eni_size_weights.call_cost); |
2885 | *time -= (eni_time_weights.indirect_call_cost - eni_time_weights.call_cost); | |
2886 | gcc_checking_assert (*time >= 0); | |
2887 | gcc_checking_assert (*size >= 0); | |
5dcaa672 | 2888 | |
20da2013 | 2889 | callee = cgraph_get_node (target); |
02774f2d | 2890 | if (!callee || !callee->definition) |
eb7c606e | 2891 | return false; |
20da2013 | 2892 | isummary = inline_summary (callee); |
eb7c606e | 2893 | return isummary->inlinable; |
20da2013 | 2894 | } |
2895 | ||
18b64b34 | 2896 | /* Increase SIZE and TIME for size and time needed to handle edge E. */ |
2897 | ||
2898 | static inline void | |
2899 | estimate_edge_size_and_time (struct cgraph_edge *e, int *size, int *time, | |
2900 | int prob, | |
f1f41a6c | 2901 | vec<tree> known_vals, |
2902 | vec<tree> known_binfos, | |
2903 | vec<ipa_agg_jump_function_p> known_aggs, | |
18b64b34 | 2904 | inline_hints *hints) |
18b64b34 | 2905 | { |
2906 | struct inline_edge_summary *es = inline_edge_summary (e); | |
2907 | int call_size = es->call_stmt_size; | |
2908 | int call_time = es->call_stmt_time; | |
2909 | if (!e->callee | |
2910 | && estimate_edge_devirt_benefit (e, &call_size, &call_time, | |
2911 | known_vals, known_binfos, known_aggs) | |
e876d531 | 2912 | && hints && cgraph_maybe_hot_edge_p (e)) |
18b64b34 | 2913 | *hints |= INLINE_HINT_indirect_call; |
2914 | *size += call_size * INLINE_SIZE_SCALE; | |
70074000 | 2915 | *time += apply_probability ((gcov_type) call_time, prob) |
e876d531 | 2916 | * e->frequency * (INLINE_TIME_SCALE / CGRAPH_FREQ_BASE); |
18b64b34 | 2917 | if (*time > MAX_TIME * INLINE_TIME_SCALE) |
2918 | *time = MAX_TIME * INLINE_TIME_SCALE; | |
2919 | } | |
2920 | ||
2921 | ||
20da2013 | 2922 | |
2923 | /* Increase SIZE and TIME for size and time needed to handle all calls in NODE. | |
2924 | POSSIBLE_TRUTHS, KNOWN_VALS and KNOWN_BINFOS describe context of the call | |
2925 | site. */ | |
a41f2a28 | 2926 | |
2927 | static void | |
6a18c0be | 2928 | estimate_calls_size_and_time (struct cgraph_node *node, int *size, int *time, |
eb7c606e | 2929 | inline_hints *hints, |
20da2013 | 2930 | clause_t possible_truths, |
f1f41a6c | 2931 | vec<tree> known_vals, |
2932 | vec<tree> known_binfos, | |
2933 | vec<ipa_agg_jump_function_p> known_aggs) | |
a41f2a28 | 2934 | { |
2935 | struct cgraph_edge *e; | |
2936 | for (e = node->callees; e; e = e->next_callee) | |
6a18c0be | 2937 | { |
2938 | struct inline_edge_summary *es = inline_edge_summary (e); | |
e876d531 | 2939 | if (!es->predicate |
2940 | || evaluate_predicate (es->predicate, possible_truths)) | |
6a18c0be | 2941 | { |
2942 | if (e->inline_failed) | |
eb4ae064 | 2943 | { |
2944 | /* Predicates of calls shall not use NOT_CHANGED codes, | |
e876d531 | 2945 | sowe do not need to compute probabilities. */ |
18b64b34 | 2946 | estimate_edge_size_and_time (e, size, time, REG_BR_PROB_BASE, |
e876d531 | 2947 | known_vals, known_binfos, |
2948 | known_aggs, hints); | |
eb4ae064 | 2949 | } |
6a18c0be | 2950 | else |
eb7c606e | 2951 | estimate_calls_size_and_time (e->callee, size, time, hints, |
20da2013 | 2952 | possible_truths, |
e876d531 | 2953 | known_vals, known_binfos, |
2954 | known_aggs); | |
6a18c0be | 2955 | } |
2956 | } | |
a41f2a28 | 2957 | for (e = node->indirect_calls; e; e = e->next_callee) |
6a18c0be | 2958 | { |
2959 | struct inline_edge_summary *es = inline_edge_summary (e); | |
e876d531 | 2960 | if (!es->predicate |
2961 | || evaluate_predicate (es->predicate, possible_truths)) | |
18b64b34 | 2962 | estimate_edge_size_and_time (e, size, time, REG_BR_PROB_BASE, |
2963 | known_vals, known_binfos, known_aggs, | |
2964 | hints); | |
6a18c0be | 2965 | } |
a41f2a28 | 2966 | } |
2967 | ||
2968 | ||
8bae3ea4 | 2969 | /* Estimate size and time needed to execute NODE assuming |
20da2013 | 2970 | POSSIBLE_TRUTHS clause, and KNOWN_VALS and KNOWN_BINFOS information |
2971 | about NODE's arguments. */ | |
99c67f24 | 2972 | |
a41f2a28 | 2973 | static void |
8bae3ea4 | 2974 | estimate_node_size_and_time (struct cgraph_node *node, |
2975 | clause_t possible_truths, | |
f1f41a6c | 2976 | vec<tree> known_vals, |
2977 | vec<tree> known_binfos, | |
2978 | vec<ipa_agg_jump_function_p> known_aggs, | |
e876d531 | 2979 | int *ret_size, int *ret_time, |
eb7c606e | 2980 | inline_hints *ret_hints, |
f1f41a6c | 2981 | vec<inline_param_summary_t> |
e876d531 | 2982 | inline_param_summary) |
99c67f24 | 2983 | { |
8bae3ea4 | 2984 | struct inline_summary *info = inline_summary (node); |
a41f2a28 | 2985 | size_time_entry *e; |
18b64b34 | 2986 | int size = 0; |
2987 | int time = 0; | |
eb7c606e | 2988 | inline_hints hints = 0; |
a41f2a28 | 2989 | int i; |
2990 | ||
e876d531 | 2991 | if (dump_file && (dump_flags & TDF_DETAILS)) |
a41f2a28 | 2992 | { |
2993 | bool found = false; | |
8bae3ea4 | 2994 | fprintf (dump_file, " Estimating body: %s/%i\n" |
f1c8b4d7 | 2995 | " Known to be false: ", node->name (), |
02774f2d | 2996 | node->order); |
a41f2a28 | 2997 | |
2998 | for (i = predicate_not_inlined_condition; | |
2999 | i < (predicate_first_dynamic_condition | |
e876d531 | 3000 | + (int) vec_safe_length (info->conds)); i++) |
8bae3ea4 | 3001 | if (!(possible_truths & (1 << i))) |
a41f2a28 | 3002 | { |
3003 | if (found) | |
3004 | fprintf (dump_file, ", "); | |
3005 | found = true; | |
e876d531 | 3006 | dump_condition (dump_file, info->conds, i); |
a41f2a28 | 3007 | } |
3008 | } | |
3009 | ||
f1f41a6c | 3010 | for (i = 0; vec_safe_iterate (info->entry, i, &e); i++) |
8bae3ea4 | 3011 | if (evaluate_predicate (&e->predicate, possible_truths)) |
eb4ae064 | 3012 | { |
3013 | size += e->size; | |
18b64b34 | 3014 | gcc_checking_assert (e->time >= 0); |
e876d531 | 3015 | gcc_checking_assert (time >= 0); |
f1f41a6c | 3016 | if (!inline_param_summary.exists ()) |
eb4ae064 | 3017 | time += e->time; |
3018 | else | |
3019 | { | |
3020 | int prob = predicate_probability (info->conds, | |
3021 | &e->predicate, | |
3022 | possible_truths, | |
3023 | inline_param_summary); | |
18b64b34 | 3024 | gcc_checking_assert (prob >= 0); |
3025 | gcc_checking_assert (prob <= REG_BR_PROB_BASE); | |
70074000 | 3026 | time += apply_probability ((gcov_type) e->time, prob); |
eb4ae064 | 3027 | } |
e876d531 | 3028 | if (time > MAX_TIME * INLINE_TIME_SCALE) |
3029 | time = MAX_TIME * INLINE_TIME_SCALE; | |
3030 | gcc_checking_assert (time >= 0); | |
3031 | ||
eb4ae064 | 3032 | } |
18b64b34 | 3033 | gcc_checking_assert (size >= 0); |
3034 | gcc_checking_assert (time >= 0); | |
cbd7f5a0 | 3035 | |
7c07aa3d | 3036 | if (info->loop_iterations |
3037 | && !evaluate_predicate (info->loop_iterations, possible_truths)) | |
e876d531 | 3038 | hints |= INLINE_HINT_loop_iterations; |
3716ee8f | 3039 | if (info->loop_stride |
3040 | && !evaluate_predicate (info->loop_stride, possible_truths)) | |
e876d531 | 3041 | hints |= INLINE_HINT_loop_stride; |
be343a9c | 3042 | if (info->array_index |
3043 | && !evaluate_predicate (info->array_index, possible_truths)) | |
e876d531 | 3044 | hints |= INLINE_HINT_array_index; |
41d39f38 | 3045 | if (info->scc_no) |
3046 | hints |= INLINE_HINT_in_scc; | |
02774f2d | 3047 | if (DECL_DECLARED_INLINE_P (node->decl)) |
3172b7bf | 3048 | hints |= INLINE_HINT_declared_inline; |
a41f2a28 | 3049 | |
eb7c606e | 3050 | estimate_calls_size_and_time (node, &size, &time, &hints, possible_truths, |
a4f60e55 | 3051 | known_vals, known_binfos, known_aggs); |
18b64b34 | 3052 | gcc_checking_assert (size >= 0); |
3053 | gcc_checking_assert (time >= 0); | |
3054 | time = RDIV (time, INLINE_TIME_SCALE); | |
3055 | size = RDIV (size, INLINE_SIZE_SCALE); | |
a41f2a28 | 3056 | |
e876d531 | 3057 | if (dump_file && (dump_flags & TDF_DETAILS)) |
3058 | fprintf (dump_file, "\n size:%i time:%i\n", (int) size, (int) time); | |
a41f2a28 | 3059 | if (ret_time) |
3060 | *ret_time = time; | |
3061 | if (ret_size) | |
3062 | *ret_size = size; | |
eb7c606e | 3063 | if (ret_hints) |
3064 | *ret_hints = hints; | |
a41f2a28 | 3065 | return; |
3066 | } | |
3067 | ||
3068 | ||
93f713da | 3069 | /* Estimate size and time needed to execute callee of EDGE assuming that |
3070 | parameters known to be constant at caller of EDGE are propagated. | |
20da2013 | 3071 | KNOWN_VALS and KNOWN_BINFOS are vectors of assumed known constant values |
3072 | and types for parameters. */ | |
8bae3ea4 | 3073 | |
3074 | void | |
3075 | estimate_ipcp_clone_size_and_time (struct cgraph_node *node, | |
e876d531 | 3076 | vec<tree> known_vals, |
3077 | vec<tree> known_binfos, | |
3078 | vec<ipa_agg_jump_function_p> known_aggs, | |
3079 | int *ret_size, int *ret_time, | |
3080 | inline_hints *hints) | |
8bae3ea4 | 3081 | { |
93f713da | 3082 | clause_t clause; |
3083 | ||
803a7988 | 3084 | clause = evaluate_conditions_for_known_args (node, false, known_vals, |
3085 | known_aggs); | |
3086 | estimate_node_size_and_time (node, clause, known_vals, known_binfos, | |
1e094109 | 3087 | known_aggs, ret_size, ret_time, hints, vNULL); |
8bae3ea4 | 3088 | } |
3089 | ||
eb4ae064 | 3090 | /* Translate all conditions from callee representation into caller |
3091 | representation and symbolically evaluate predicate P into new predicate. | |
6a18c0be | 3092 | |
a4f60e55 | 3093 | INFO is inline_summary of function we are adding predicate into, CALLEE_INFO |
3094 | is summary of function predicate P is from. OPERAND_MAP is array giving | |
3095 | callee formal IDs the caller formal IDs. POSSSIBLE_TRUTHS is clausule of all | |
3096 | callee conditions that may be true in caller context. TOPLEV_PREDICATE is | |
3097 | predicate under which callee is executed. OFFSET_MAP is an array of of | |
3098 | offsets that need to be added to conditions, negative offset means that | |
3099 | conditions relying on values passed by reference have to be discarded | |
3100 | because they might not be preserved (and should be considered offset zero | |
3101 | for other purposes). */ | |
a41f2a28 | 3102 | |
3103 | static struct predicate | |
eb4ae064 | 3104 | remap_predicate (struct inline_summary *info, |
3105 | struct inline_summary *callee_info, | |
a41f2a28 | 3106 | struct predicate *p, |
f1f41a6c | 3107 | vec<int> operand_map, |
3108 | vec<int> offset_map, | |
e876d531 | 3109 | clause_t possible_truths, struct predicate *toplev_predicate) |
a41f2a28 | 3110 | { |
3111 | int i; | |
3112 | struct predicate out = true_predicate (); | |
3113 | ||
3114 | /* True predicate is easy. */ | |
6a18c0be | 3115 | if (true_predicate_p (p)) |
3116 | return *toplev_predicate; | |
a41f2a28 | 3117 | for (i = 0; p->clause[i]; i++) |
3118 | { | |
3119 | clause_t clause = p->clause[i]; | |
3120 | int cond; | |
3121 | struct predicate clause_predicate = false_predicate (); | |
3122 | ||
5cb1b112 | 3123 | gcc_assert (i < MAX_CLAUSES); |
3124 | ||
e876d531 | 3125 | for (cond = 0; cond < NUM_CONDITIONS; cond++) |
a41f2a28 | 3126 | /* Do we have condition we can't disprove? */ |
3127 | if (clause & possible_truths & (1 << cond)) | |
3128 | { | |
3129 | struct predicate cond_predicate; | |
3130 | /* Work out if the condition can translate to predicate in the | |
3131 | inlined function. */ | |
3132 | if (cond >= predicate_first_dynamic_condition) | |
3133 | { | |
e876d531 | 3134 | struct condition *c; |
3135 | ||
3136 | c = &(*callee_info->conds)[cond | |
3137 | - | |
3138 | predicate_first_dynamic_condition]; | |
3139 | /* See if we can remap condition operand to caller's operand. | |
3140 | Otherwise give up. */ | |
3141 | if (!operand_map.exists () | |
3142 | || (int) operand_map.length () <= c->operand_num | |
3143 | || operand_map[c->operand_num] == -1 | |
3144 | /* TODO: For non-aggregate conditions, adding an offset is | |
3145 | basically an arithmetic jump function processing which | |
3146 | we should support in future. */ | |
3147 | || ((!c->agg_contents || !c->by_ref) | |
3148 | && offset_map[c->operand_num] > 0) | |
3149 | || (c->agg_contents && c->by_ref | |
3150 | && offset_map[c->operand_num] < 0)) | |
3151 | cond_predicate = true_predicate (); | |
3152 | else | |
3153 | { | |
3154 | struct agg_position_info ap; | |
3155 | HOST_WIDE_INT offset_delta = offset_map[c->operand_num]; | |
3156 | if (offset_delta < 0) | |
3157 | { | |
3158 | gcc_checking_assert (!c->agg_contents || !c->by_ref); | |
3159 | offset_delta = 0; | |
3160 | } | |
3161 | gcc_assert (!c->agg_contents | |
3162 | || c->by_ref || offset_delta == 0); | |
3163 | ap.offset = c->offset + offset_delta; | |
3164 | ap.agg_contents = c->agg_contents; | |
3165 | ap.by_ref = c->by_ref; | |
3166 | cond_predicate = add_condition (info, | |
3167 | operand_map[c->operand_num], | |
3168 | &ap, c->code, c->val); | |
3169 | } | |
a41f2a28 | 3170 | } |
3171 | /* Fixed conditions remains same, construct single | |
3172 | condition predicate. */ | |
3173 | else | |
3174 | { | |
3175 | cond_predicate.clause[0] = 1 << cond; | |
3176 | cond_predicate.clause[1] = 0; | |
3177 | } | |
94646c9c | 3178 | clause_predicate = or_predicates (info->conds, &clause_predicate, |
3179 | &cond_predicate); | |
a41f2a28 | 3180 | } |
94646c9c | 3181 | out = and_predicates (info->conds, &out, &clause_predicate); |
a41f2a28 | 3182 | } |
94646c9c | 3183 | return and_predicates (info->conds, &out, toplev_predicate); |
a41f2a28 | 3184 | } |
3185 | ||
3186 | ||
0835ad03 | 3187 | /* Update summary information of inline clones after inlining. |
3188 | Compute peak stack usage. */ | |
3189 | ||
3190 | static void | |
e876d531 | 3191 | inline_update_callee_summaries (struct cgraph_node *node, int depth) |
0835ad03 | 3192 | { |
3193 | struct cgraph_edge *e; | |
3194 | struct inline_summary *callee_info = inline_summary (node); | |
3195 | struct inline_summary *caller_info = inline_summary (node->callers->caller); | |
3196 | HOST_WIDE_INT peak; | |
3197 | ||
3198 | callee_info->stack_frame_offset | |
3199 | = caller_info->stack_frame_offset | |
e876d531 | 3200 | + caller_info->estimated_self_stack_size; |
0835ad03 | 3201 | peak = callee_info->stack_frame_offset |
e876d531 | 3202 | + callee_info->estimated_self_stack_size; |
3203 | if (inline_summary (node->global.inlined_to)->estimated_stack_size < peak) | |
3204 | inline_summary (node->global.inlined_to)->estimated_stack_size = peak; | |
6eaf903b | 3205 | ipa_propagate_frequency (node); |
0835ad03 | 3206 | for (e = node->callees; e; e = e->next_callee) |
3207 | { | |
3208 | if (!e->inline_failed) | |
3209 | inline_update_callee_summaries (e->callee, depth); | |
3210 | inline_edge_summary (e)->loop_depth += depth; | |
3211 | } | |
3212 | for (e = node->indirect_calls; e; e = e->next_callee) | |
3213 | inline_edge_summary (e)->loop_depth += depth; | |
3214 | } | |
3215 | ||
eb4ae064 | 3216 | /* Update change_prob of EDGE after INLINED_EDGE has been inlined. |
3217 | When functoin A is inlined in B and A calls C with parameter that | |
3218 | changes with probability PROB1 and C is known to be passthroug | |
3219 | of argument if B that change with probability PROB2, the probability | |
3220 | of change is now PROB1*PROB2. */ | |
3221 | ||
3222 | static void | |
3223 | remap_edge_change_prob (struct cgraph_edge *inlined_edge, | |
3224 | struct cgraph_edge *edge) | |
3225 | { | |
f1f41a6c | 3226 | if (ipa_node_params_vector.exists ()) |
eb4ae064 | 3227 | { |
3228 | int i; | |
3229 | struct ipa_edge_args *args = IPA_EDGE_REF (edge); | |
3230 | struct inline_edge_summary *es = inline_edge_summary (edge); | |
3231 | struct inline_edge_summary *inlined_es | |
e876d531 | 3232 | = inline_edge_summary (inlined_edge); |
eb4ae064 | 3233 | |
3234 | for (i = 0; i < ipa_get_cs_argument_count (args); i++) | |
3235 | { | |
3236 | struct ipa_jump_func *jfunc = ipa_get_ith_jump_func (args, i); | |
3237 | if (jfunc->type == IPA_JF_PASS_THROUGH | |
4fa83f96 | 3238 | && (ipa_get_jf_pass_through_formal_id (jfunc) |
f1f41a6c | 3239 | < (int) inlined_es->param.length ())) |
eb4ae064 | 3240 | { |
4fa83f96 | 3241 | int jf_formal_id = ipa_get_jf_pass_through_formal_id (jfunc); |
f1f41a6c | 3242 | int prob1 = es->param[i].change_prob; |
3243 | int prob2 = inlined_es->param[jf_formal_id].change_prob; | |
f9d4b7f4 | 3244 | int prob = combine_probabilities (prob1, prob2); |
eb4ae064 | 3245 | |
3246 | if (prob1 && prob2 && !prob) | |
3247 | prob = 1; | |
3248 | ||
f1f41a6c | 3249 | es->param[i].change_prob = prob; |
eb4ae064 | 3250 | } |
3251 | } | |
e876d531 | 3252 | } |
eb4ae064 | 3253 | } |
3254 | ||
3255 | /* Update edge summaries of NODE after INLINED_EDGE has been inlined. | |
3256 | ||
3257 | Remap predicates of callees of NODE. Rest of arguments match | |
3258 | remap_predicate. | |
0835ad03 | 3259 | |
eb4ae064 | 3260 | Also update change probabilities. */ |
6a18c0be | 3261 | |
3262 | static void | |
e876d531 | 3263 | remap_edge_summaries (struct cgraph_edge *inlined_edge, |
3264 | struct cgraph_node *node, | |
3265 | struct inline_summary *info, | |
3266 | struct inline_summary *callee_info, | |
3267 | vec<int> operand_map, | |
3268 | vec<int> offset_map, | |
3269 | clause_t possible_truths, | |
3270 | struct predicate *toplev_predicate) | |
6a18c0be | 3271 | { |
3272 | struct cgraph_edge *e; | |
3273 | for (e = node->callees; e; e = e->next_callee) | |
3274 | { | |
3275 | struct inline_edge_summary *es = inline_edge_summary (e); | |
3276 | struct predicate p; | |
eb4ae064 | 3277 | |
a226c368 | 3278 | if (e->inline_failed) |
6a18c0be | 3279 | { |
eb4ae064 | 3280 | remap_edge_change_prob (inlined_edge, e); |
3281 | ||
a226c368 | 3282 | if (es->predicate) |
6a18c0be | 3283 | { |
a226c368 | 3284 | p = remap_predicate (info, callee_info, |
a4f60e55 | 3285 | es->predicate, operand_map, offset_map, |
e876d531 | 3286 | possible_truths, toplev_predicate); |
a226c368 | 3287 | edge_set_predicate (e, &p); |
eb4ae064 | 3288 | /* TODO: We should remove the edge for code that will be |
e876d531 | 3289 | optimized out, but we need to keep verifiers and tree-inline |
3290 | happy. Make it cold for now. */ | |
a226c368 | 3291 | if (false_predicate_p (&p)) |
3292 | { | |
3293 | e->count = 0; | |
3294 | e->frequency = 0; | |
3295 | } | |
6a18c0be | 3296 | } |
a226c368 | 3297 | else |
3298 | edge_set_predicate (e, toplev_predicate); | |
6a18c0be | 3299 | } |
a226c368 | 3300 | else |
eb4ae064 | 3301 | remap_edge_summaries (inlined_edge, e->callee, info, callee_info, |
a4f60e55 | 3302 | operand_map, offset_map, possible_truths, |
3303 | toplev_predicate); | |
6a18c0be | 3304 | } |
3305 | for (e = node->indirect_calls; e; e = e->next_callee) | |
3306 | { | |
3307 | struct inline_edge_summary *es = inline_edge_summary (e); | |
3308 | struct predicate p; | |
eb4ae064 | 3309 | |
3310 | remap_edge_change_prob (inlined_edge, e); | |
6a18c0be | 3311 | if (es->predicate) |
3312 | { | |
3313 | p = remap_predicate (info, callee_info, | |
a4f60e55 | 3314 | es->predicate, operand_map, offset_map, |
3315 | possible_truths, toplev_predicate); | |
6a18c0be | 3316 | edge_set_predicate (e, &p); |
eb4ae064 | 3317 | /* TODO: We should remove the edge for code that will be optimized |
3318 | out, but we need to keep verifiers and tree-inline happy. | |
6a18c0be | 3319 | Make it cold for now. */ |
3320 | if (false_predicate_p (&p)) | |
3321 | { | |
3322 | e->count = 0; | |
3323 | e->frequency = 0; | |
3324 | } | |
3325 | } | |
144eea3a | 3326 | else |
3327 | edge_set_predicate (e, toplev_predicate); | |
6a18c0be | 3328 | } |
3329 | } | |
3330 | ||
3716ee8f | 3331 | /* Same as remap_predicate, but set result into hint *HINT. */ |
3332 | ||
3333 | static void | |
3334 | remap_hint_predicate (struct inline_summary *info, | |
3335 | struct inline_summary *callee_info, | |
3336 | struct predicate **hint, | |
f1f41a6c | 3337 | vec<int> operand_map, |
3338 | vec<int> offset_map, | |
3716ee8f | 3339 | clause_t possible_truths, |
3340 | struct predicate *toplev_predicate) | |
3341 | { | |
3342 | predicate p; | |
3343 | ||
3344 | if (!*hint) | |
3345 | return; | |
3346 | p = remap_predicate (info, callee_info, | |
3347 | *hint, | |
3348 | operand_map, offset_map, | |
e876d531 | 3349 | possible_truths, toplev_predicate); |
3350 | if (!false_predicate_p (&p) && !true_predicate_p (&p)) | |
3716ee8f | 3351 | { |
3352 | if (!*hint) | |
3353 | set_hint_predicate (hint, p); | |
3354 | else | |
e876d531 | 3355 | **hint = and_predicates (info->conds, *hint, &p); |
3716ee8f | 3356 | } |
3357 | } | |
6a18c0be | 3358 | |
a41f2a28 | 3359 | /* We inlined EDGE. Update summary of the function we inlined into. */ |
3360 | ||
3361 | void | |
3362 | inline_merge_summary (struct cgraph_edge *edge) | |
3363 | { | |
3364 | struct inline_summary *callee_info = inline_summary (edge->callee); | |
3365 | struct cgraph_node *to = (edge->caller->global.inlined_to | |
3366 | ? edge->caller->global.inlined_to : edge->caller); | |
3367 | struct inline_summary *info = inline_summary (to); | |
3368 | clause_t clause = 0; /* not_inline is known to be false. */ | |
3369 | size_time_entry *e; | |
1e094109 | 3370 | vec<int> operand_map = vNULL; |
3371 | vec<int> offset_map = vNULL; | |
a41f2a28 | 3372 | int i; |
6a18c0be | 3373 | struct predicate toplev_predicate; |
a226c368 | 3374 | struct predicate true_p = true_predicate (); |
6a18c0be | 3375 | struct inline_edge_summary *es = inline_edge_summary (edge); |
3376 | ||
3377 | if (es->predicate) | |
3378 | toplev_predicate = *es->predicate; | |
3379 | else | |
3380 | toplev_predicate = true_predicate (); | |
a41f2a28 | 3381 | |
f1f41a6c | 3382 | if (ipa_node_params_vector.exists () && callee_info->conds) |
a41f2a28 | 3383 | { |
3384 | struct ipa_edge_args *args = IPA_EDGE_REF (edge); | |
3385 | int count = ipa_get_cs_argument_count (args); | |
3386 | int i; | |
3387 | ||
a4f60e55 | 3388 | evaluate_properties_for_edge (edge, true, &clause, NULL, NULL, NULL); |
a226c368 | 3389 | if (count) |
a4f60e55 | 3390 | { |
f1f41a6c | 3391 | operand_map.safe_grow_cleared (count); |
3392 | offset_map.safe_grow_cleared (count); | |
a4f60e55 | 3393 | } |
a41f2a28 | 3394 | for (i = 0; i < count; i++) |
3395 | { | |
3396 | struct ipa_jump_func *jfunc = ipa_get_ith_jump_func (args, i); | |
3397 | int map = -1; | |
a4f60e55 | 3398 | |
a41f2a28 | 3399 | /* TODO: handle non-NOPs when merging. */ |
a4f60e55 | 3400 | if (jfunc->type == IPA_JF_PASS_THROUGH) |
3401 | { | |
3402 | if (ipa_get_jf_pass_through_operation (jfunc) == NOP_EXPR) | |
3403 | map = ipa_get_jf_pass_through_formal_id (jfunc); | |
3404 | if (!ipa_get_jf_pass_through_agg_preserved (jfunc)) | |
f1f41a6c | 3405 | offset_map[i] = -1; |
a4f60e55 | 3406 | } |
3407 | else if (jfunc->type == IPA_JF_ANCESTOR) | |
3408 | { | |
3409 | HOST_WIDE_INT offset = ipa_get_jf_ancestor_offset (jfunc); | |
3410 | if (offset >= 0 && offset < INT_MAX) | |
3411 | { | |
3412 | map = ipa_get_jf_ancestor_formal_id (jfunc); | |
3413 | if (!ipa_get_jf_ancestor_agg_preserved (jfunc)) | |
3414 | offset = -1; | |
f1f41a6c | 3415 | offset_map[i] = offset; |
a4f60e55 | 3416 | } |
3417 | } | |
f1f41a6c | 3418 | operand_map[i] = map; |
5cb1b112 | 3419 | gcc_assert (map < ipa_get_param_count (IPA_NODE_REF (to))); |
a41f2a28 | 3420 | } |
3421 | } | |
f1f41a6c | 3422 | for (i = 0; vec_safe_iterate (callee_info->entry, i, &e); i++) |
a41f2a28 | 3423 | { |
3424 | struct predicate p = remap_predicate (info, callee_info, | |
a4f60e55 | 3425 | &e->predicate, operand_map, |
3426 | offset_map, clause, | |
6a18c0be | 3427 | &toplev_predicate); |
eb4ae064 | 3428 | if (!false_predicate_p (&p)) |
3429 | { | |
e876d531 | 3430 | gcov_type add_time = ((gcov_type) e->time * edge->frequency |
eb4ae064 | 3431 | + CGRAPH_FREQ_BASE / 2) / CGRAPH_FREQ_BASE; |
3432 | int prob = predicate_probability (callee_info->conds, | |
3433 | &e->predicate, | |
3434 | clause, es->param); | |
70074000 | 3435 | add_time = apply_probability ((gcov_type) add_time, prob); |
eb4ae064 | 3436 | if (add_time > MAX_TIME * INLINE_TIME_SCALE) |
3437 | add_time = MAX_TIME * INLINE_TIME_SCALE; | |
3438 | if (prob != REG_BR_PROB_BASE | |
3439 | && dump_file && (dump_flags & TDF_DETAILS)) | |
3440 | { | |
3441 | fprintf (dump_file, "\t\tScaling time by probability:%f\n", | |
e876d531 | 3442 | (double) prob / REG_BR_PROB_BASE); |
eb4ae064 | 3443 | } |
3444 | account_size_time (info, e->size, add_time, &p); | |
3445 | } | |
3446 | } | |
3447 | remap_edge_summaries (edge, edge->callee, info, callee_info, operand_map, | |
a4f60e55 | 3448 | offset_map, clause, &toplev_predicate); |
3716ee8f | 3449 | remap_hint_predicate (info, callee_info, |
3450 | &callee_info->loop_iterations, | |
e876d531 | 3451 | operand_map, offset_map, clause, &toplev_predicate); |
3716ee8f | 3452 | remap_hint_predicate (info, callee_info, |
3453 | &callee_info->loop_stride, | |
e876d531 | 3454 | operand_map, offset_map, clause, &toplev_predicate); |
be343a9c | 3455 | remap_hint_predicate (info, callee_info, |
3456 | &callee_info->array_index, | |
e876d531 | 3457 | operand_map, offset_map, clause, &toplev_predicate); |
0835ad03 | 3458 | |
3459 | inline_update_callee_summaries (edge->callee, | |
3460 | inline_edge_summary (edge)->loop_depth); | |
3461 | ||
a226c368 | 3462 | /* We do not maintain predicates of inlined edges, free it. */ |
3463 | edge_set_predicate (edge, &true_p); | |
eb4ae064 | 3464 | /* Similarly remove param summaries. */ |
f1f41a6c | 3465 | es->param.release (); |
3466 | operand_map.release (); | |
3467 | offset_map.release (); | |
6331b6fa | 3468 | } |
3469 | ||
3470 | /* For performance reasons inline_merge_summary is not updating overall size | |
3471 | and time. Recompute it. */ | |
a226c368 | 3472 | |
6331b6fa | 3473 | void |
3474 | inline_update_overall_summary (struct cgraph_node *node) | |
3475 | { | |
3476 | struct inline_summary *info = inline_summary (node); | |
3477 | size_time_entry *e; | |
3478 | int i; | |
3479 | ||
3480 | info->size = 0; | |
3481 | info->time = 0; | |
f1f41a6c | 3482 | for (i = 0; vec_safe_iterate (info->entry, i, &e); i++) |
3c49142d | 3483 | { |
3484 | info->size += e->size, info->time += e->time; | |
3485 | if (info->time > MAX_TIME * INLINE_TIME_SCALE) | |
e876d531 | 3486 | info->time = MAX_TIME * INLINE_TIME_SCALE; |
3c49142d | 3487 | } |
eb7c606e | 3488 | estimate_calls_size_and_time (node, &info->size, &info->time, NULL, |
e876d531 | 3489 | ~(clause_t) (1 << predicate_false_condition), |
1e094109 | 3490 | vNULL, vNULL, vNULL); |
a41f2a28 | 3491 | info->time = (info->time + INLINE_TIME_SCALE / 2) / INLINE_TIME_SCALE; |
3492 | info->size = (info->size + INLINE_SIZE_SCALE / 2) / INLINE_SIZE_SCALE; | |
3493 | } | |
3494 | ||
3172b7bf | 3495 | /* Return hints derrived from EDGE. */ |
3496 | int | |
3497 | simple_edge_hints (struct cgraph_edge *edge) | |
3498 | { | |
3499 | int hints = 0; | |
3500 | struct cgraph_node *to = (edge->caller->global.inlined_to | |
e876d531 | 3501 | ? edge->caller->global.inlined_to : edge->caller); |
3172b7bf | 3502 | if (inline_summary (to)->scc_no |
3503 | && inline_summary (to)->scc_no == inline_summary (edge->callee)->scc_no | |
3504 | && !cgraph_edge_recursive_p (edge)) | |
3505 | hints |= INLINE_HINT_same_scc; | |
3506 | ||
02774f2d | 3507 | if (to->lto_file_data && edge->callee->lto_file_data |
3508 | && to->lto_file_data != edge->callee->lto_file_data) | |
3172b7bf | 3509 | hints |= INLINE_HINT_cross_module; |
3510 | ||
3511 | return hints; | |
3512 | } | |
3513 | ||
a41f2a28 | 3514 | /* Estimate the time cost for the caller when inlining EDGE. |
3515 | Only to be called via estimate_edge_time, that handles the | |
3516 | caching mechanism. | |
3517 | ||
3518 | When caching, also update the cache entry. Compute both time and | |
3519 | size, since we always need both metrics eventually. */ | |
3520 | ||
3521 | int | |
3522 | do_estimate_edge_time (struct cgraph_edge *edge) | |
3523 | { | |
3524 | int time; | |
3525 | int size; | |
eb7c606e | 3526 | inline_hints hints; |
20da2013 | 3527 | struct cgraph_node *callee; |
3528 | clause_t clause; | |
f1f41a6c | 3529 | vec<tree> known_vals; |
3530 | vec<tree> known_binfos; | |
3531 | vec<ipa_agg_jump_function_p> known_aggs; | |
0835ad03 | 3532 | struct inline_edge_summary *es = inline_edge_summary (edge); |
a41f2a28 | 3533 | |
20da2013 | 3534 | callee = cgraph_function_or_thunk_node (edge->callee, NULL); |
3535 | ||
a41f2a28 | 3536 | gcc_checking_assert (edge->inline_failed); |
20da2013 | 3537 | evaluate_properties_for_edge (edge, true, |
a4f60e55 | 3538 | &clause, &known_vals, &known_binfos, |
3539 | &known_aggs); | |
20da2013 | 3540 | estimate_node_size_and_time (callee, clause, known_vals, known_binfos, |
eb7c606e | 3541 | known_aggs, &size, &time, &hints, es->param); |
f1f41a6c | 3542 | known_vals.release (); |
3543 | known_binfos.release (); | |
3544 | known_aggs.release (); | |
3172b7bf | 3545 | gcc_checking_assert (size >= 0); |
3546 | gcc_checking_assert (time >= 0); | |
a41f2a28 | 3547 | |
3548 | /* When caching, update the cache entry. */ | |
f1f41a6c | 3549 | if (edge_growth_cache.exists ()) |
a41f2a28 | 3550 | { |
e876d531 | 3551 | if ((int) edge_growth_cache.length () <= edge->uid) |
f1f41a6c | 3552 | edge_growth_cache.safe_grow_cleared (cgraph_edge_max_uid); |
3553 | edge_growth_cache[edge->uid].time = time + (time >= 0); | |
a41f2a28 | 3554 | |
f1f41a6c | 3555 | edge_growth_cache[edge->uid].size = size + (size >= 0); |
3172b7bf | 3556 | hints |= simple_edge_hints (edge); |
f1f41a6c | 3557 | edge_growth_cache[edge->uid].hints = hints + 1; |
a41f2a28 | 3558 | } |
3172b7bf | 3559 | return time; |
a41f2a28 | 3560 | } |
3561 | ||
3562 | ||
6c2c7775 | 3563 | /* Return estimated callee growth after inlining EDGE. |
a41f2a28 | 3564 | Only to be called via estimate_edge_size. */ |
3565 | ||
3566 | int | |
6c2c7775 | 3567 | do_estimate_edge_size (struct cgraph_edge *edge) |
a41f2a28 | 3568 | { |
3569 | int size; | |
82626cb0 | 3570 | struct cgraph_node *callee; |
20da2013 | 3571 | clause_t clause; |
f1f41a6c | 3572 | vec<tree> known_vals; |
3573 | vec<tree> known_binfos; | |
3574 | vec<ipa_agg_jump_function_p> known_aggs; | |
a41f2a28 | 3575 | |
3576 | /* When we do caching, use do_estimate_edge_time to populate the entry. */ | |
3577 | ||
f1f41a6c | 3578 | if (edge_growth_cache.exists ()) |
a41f2a28 | 3579 | { |
3580 | do_estimate_edge_time (edge); | |
f1f41a6c | 3581 | size = edge_growth_cache[edge->uid].size; |
a41f2a28 | 3582 | gcc_checking_assert (size); |
3583 | return size - (size > 0); | |
3584 | } | |
20da2013 | 3585 | |
82626cb0 | 3586 | callee = cgraph_function_or_thunk_node (edge->callee, NULL); |
a41f2a28 | 3587 | |
3588 | /* Early inliner runs without caching, go ahead and do the dirty work. */ | |
3589 | gcc_checking_assert (edge->inline_failed); | |
20da2013 | 3590 | evaluate_properties_for_edge (edge, true, |
a4f60e55 | 3591 | &clause, &known_vals, &known_binfos, |
3592 | &known_aggs); | |
20da2013 | 3593 | estimate_node_size_and_time (callee, clause, known_vals, known_binfos, |
1e094109 | 3594 | known_aggs, &size, NULL, NULL, vNULL); |
f1f41a6c | 3595 | known_vals.release (); |
3596 | known_binfos.release (); | |
3597 | known_aggs.release (); | |
6c2c7775 | 3598 | return size; |
99c67f24 | 3599 | } |
3600 | ||
3601 | ||
eb7c606e | 3602 | /* Estimate the growth of the caller when inlining EDGE. |
3603 | Only to be called via estimate_edge_size. */ | |
3604 | ||
3605 | inline_hints | |
3606 | do_estimate_edge_hints (struct cgraph_edge *edge) | |
3607 | { | |
3608 | inline_hints hints; | |
3609 | struct cgraph_node *callee; | |
3610 | clause_t clause; | |
f1f41a6c | 3611 | vec<tree> known_vals; |
3612 | vec<tree> known_binfos; | |
3613 | vec<ipa_agg_jump_function_p> known_aggs; | |
eb7c606e | 3614 | |
3615 | /* When we do caching, use do_estimate_edge_time to populate the entry. */ | |
3616 | ||
f1f41a6c | 3617 | if (edge_growth_cache.exists ()) |
eb7c606e | 3618 | { |
3619 | do_estimate_edge_time (edge); | |
f1f41a6c | 3620 | hints = edge_growth_cache[edge->uid].hints; |
eb7c606e | 3621 | gcc_checking_assert (hints); |
3622 | return hints - 1; | |
3623 | } | |
3624 | ||
3625 | callee = cgraph_function_or_thunk_node (edge->callee, NULL); | |
3626 | ||
3627 | /* Early inliner runs without caching, go ahead and do the dirty work. */ | |
3628 | gcc_checking_assert (edge->inline_failed); | |
3629 | evaluate_properties_for_edge (edge, true, | |
3630 | &clause, &known_vals, &known_binfos, | |
3631 | &known_aggs); | |
3632 | estimate_node_size_and_time (callee, clause, known_vals, known_binfos, | |
1e094109 | 3633 | known_aggs, NULL, NULL, &hints, vNULL); |
f1f41a6c | 3634 | known_vals.release (); |
3635 | known_binfos.release (); | |
3636 | known_aggs.release (); | |
3172b7bf | 3637 | hints |= simple_edge_hints (edge); |
eb7c606e | 3638 | return hints; |
3639 | } | |
3640 | ||
3641 | ||
99c67f24 | 3642 | /* Estimate self time of the function NODE after inlining EDGE. */ |
3643 | ||
3644 | int | |
3645 | estimate_time_after_inlining (struct cgraph_node *node, | |
3646 | struct cgraph_edge *edge) | |
3647 | { | |
905aa3bd | 3648 | struct inline_edge_summary *es = inline_edge_summary (edge); |
3649 | if (!es->predicate || !false_predicate_p (es->predicate)) | |
3650 | { | |
e876d531 | 3651 | gcov_type time = |
3652 | inline_summary (node)->time + estimate_edge_time (edge); | |
905aa3bd | 3653 | if (time < 0) |
3654 | time = 0; | |
3655 | if (time > MAX_TIME) | |
3656 | time = MAX_TIME; | |
3657 | return time; | |
3658 | } | |
3659 | return inline_summary (node)->time; | |
99c67f24 | 3660 | } |
3661 | ||
3662 | ||
3663 | /* Estimate the size of NODE after inlining EDGE which should be an | |
3664 | edge to either NODE or a call inlined into NODE. */ | |
3665 | ||
3666 | int | |
3667 | estimate_size_after_inlining (struct cgraph_node *node, | |
c7b2cc59 | 3668 | struct cgraph_edge *edge) |
99c67f24 | 3669 | { |
905aa3bd | 3670 | struct inline_edge_summary *es = inline_edge_summary (edge); |
3671 | if (!es->predicate || !false_predicate_p (es->predicate)) | |
3672 | { | |
3673 | int size = inline_summary (node)->size + estimate_edge_growth (edge); | |
3674 | gcc_assert (size >= 0); | |
3675 | return size; | |
3676 | } | |
3677 | return inline_summary (node)->size; | |
99c67f24 | 3678 | } |
3679 | ||
3680 | ||
82626cb0 | 3681 | struct growth_data |
3682 | { | |
ce1f57d6 | 3683 | struct cgraph_node *node; |
82626cb0 | 3684 | bool self_recursive; |
3685 | int growth; | |
3686 | }; | |
99c67f24 | 3687 | |
82626cb0 | 3688 | |
3689 | /* Worker for do_estimate_growth. Collect growth for all callers. */ | |
3690 | ||
3691 | static bool | |
3692 | do_estimate_growth_1 (struct cgraph_node *node, void *data) | |
99c67f24 | 3693 | { |
99c67f24 | 3694 | struct cgraph_edge *e; |
82626cb0 | 3695 | struct growth_data *d = (struct growth_data *) data; |
99c67f24 | 3696 | |
99c67f24 | 3697 | for (e = node->callers; e; e = e->next_caller) |
3698 | { | |
4869c23f | 3699 | gcc_checking_assert (e->inline_failed); |
3700 | ||
ce1f57d6 | 3701 | if (e->caller == d->node |
4869c23f | 3702 | || (e->caller->global.inlined_to |
ce1f57d6 | 3703 | && e->caller->global.inlined_to == d->node)) |
e876d531 | 3704 | d->self_recursive = true; |
82626cb0 | 3705 | d->growth += estimate_edge_growth (e); |
4869c23f | 3706 | } |
82626cb0 | 3707 | return false; |
3708 | } | |
3709 | ||
3710 | ||
3711 | /* Estimate the growth caused by inlining NODE into all callees. */ | |
3712 | ||
3713 | int | |
3714 | do_estimate_growth (struct cgraph_node *node) | |
3715 | { | |
ce1f57d6 | 3716 | struct growth_data d = { node, 0, false }; |
82626cb0 | 3717 | struct inline_summary *info = inline_summary (node); |
3718 | ||
3719 | cgraph_for_node_and_aliases (node, do_estimate_growth_1, &d, true); | |
4869c23f | 3720 | |
3721 | /* For self recursive functions the growth estimation really should be | |
3722 | infinity. We don't want to return very large values because the growth | |
3723 | plays various roles in badness computation fractions. Be sure to not | |
3724 | return zero or negative growths. */ | |
82626cb0 | 3725 | if (d.self_recursive) |
3726 | d.growth = d.growth < info->size ? info->size : d.growth; | |
02774f2d | 3727 | else if (DECL_EXTERNAL (node->decl)) |
3172b7bf | 3728 | ; |
4869c23f | 3729 | else |
3730 | { | |
3172b7bf | 3731 | if (cgraph_will_be_removed_from_program_if_no_direct_calls (node)) |
82626cb0 | 3732 | d.growth -= info->size; |
fb3c587e | 3733 | /* COMDAT functions are very often not shared across multiple units |
e876d531 | 3734 | since they come from various template instantiations. |
3735 | Take this into account. */ | |
02774f2d | 3736 | else if (DECL_COMDAT (node->decl) |
e876d531 | 3737 | && cgraph_can_remove_if_no_direct_calls_p (node)) |
82626cb0 | 3738 | d.growth -= (info->size |
fb3c587e | 3739 | * (100 - PARAM_VALUE (PARAM_COMDAT_SHARING_PROBABILITY)) |
3740 | + 50) / 100; | |
99c67f24 | 3741 | } |
99c67f24 | 3742 | |
f1f41a6c | 3743 | if (node_growth_cache.exists ()) |
a41f2a28 | 3744 | { |
e876d531 | 3745 | if ((int) node_growth_cache.length () <= node->uid) |
f1f41a6c | 3746 | node_growth_cache.safe_grow_cleared (cgraph_max_uid); |
3747 | node_growth_cache[node->uid] = d.growth + (d.growth >= 0); | |
a41f2a28 | 3748 | } |
82626cb0 | 3749 | return d.growth; |
99c67f24 | 3750 | } |
3751 | ||
c7b2cc59 | 3752 | |
99c67f24 | 3753 | /* This function performs intraprocedural analysis in NODE that is required to |
3754 | inline indirect calls. */ | |
c7b2cc59 | 3755 | |
99c67f24 | 3756 | static void |
3757 | inline_indirect_intraprocedural_analysis (struct cgraph_node *node) | |
3758 | { | |
3759 | ipa_analyze_node (node); | |
3760 | if (dump_file && (dump_flags & TDF_DETAILS)) | |
3761 | { | |
3762 | ipa_print_node_params (dump_file, node); | |
3763 | ipa_print_node_jump_functions (dump_file, node); | |
3764 | } | |
3765 | } | |
3766 | ||
3767 | ||
3768 | /* Note function body size. */ | |
3769 | ||
3770 | static void | |
3771 | inline_analyze_function (struct cgraph_node *node) | |
3772 | { | |
02774f2d | 3773 | push_cfun (DECL_STRUCT_FUNCTION (node->decl)); |
99c67f24 | 3774 | |
a41f2a28 | 3775 | if (dump_file) |
3776 | fprintf (dump_file, "\nAnalyzing function: %s/%u\n", | |
f1c8b4d7 | 3777 | node->name (), node->order); |
a226c368 | 3778 | if (optimize && !node->thunk.thunk_p) |
99c67f24 | 3779 | inline_indirect_intraprocedural_analysis (node); |
a41f2a28 | 3780 | compute_inline_parameters (node, false); |
26051fcf | 3781 | if (!optimize) |
3782 | { | |
3783 | struct cgraph_edge *e; | |
3784 | for (e = node->callees; e; e = e->next_callee) | |
3785 | { | |
3786 | if (e->inline_failed == CIF_FUNCTION_NOT_CONSIDERED) | |
3787 | e->inline_failed = CIF_FUNCTION_NOT_OPTIMIZED; | |
3788 | e->call_stmt_cannot_inline_p = true; | |
3789 | } | |
3790 | for (e = node->indirect_calls; e; e = e->next_callee) | |
3791 | { | |
3792 | if (e->inline_failed == CIF_FUNCTION_NOT_CONSIDERED) | |
3793 | e->inline_failed = CIF_FUNCTION_NOT_OPTIMIZED; | |
3794 | e->call_stmt_cannot_inline_p = true; | |
3795 | } | |
3796 | } | |
99c67f24 | 3797 | |
99c67f24 | 3798 | pop_cfun (); |
3799 | } | |
3800 | ||
3801 | ||
3802 | /* Called when new function is inserted to callgraph late. */ | |
3803 | ||
3804 | static void | |
3805 | add_new_function (struct cgraph_node *node, void *data ATTRIBUTE_UNUSED) | |
3806 | { | |
3807 | inline_analyze_function (node); | |
3808 | } | |
3809 | ||
3810 | ||
3811 | /* Note function body size. */ | |
3812 | ||
3813 | void | |
3814 | inline_generate_summary (void) | |
3815 | { | |
3816 | struct cgraph_node *node; | |
3817 | ||
2fe870c5 | 3818 | /* When not optimizing, do not bother to analyze. Inlining is still done |
3819 | because edge redirection needs to happen there. */ | |
3820 | if (!optimize && !flag_lto && !flag_wpa) | |
3821 | return; | |
3822 | ||
99c67f24 | 3823 | function_insertion_hook_holder = |
e876d531 | 3824 | cgraph_add_function_insertion_hook (&add_new_function, NULL); |
99c67f24 | 3825 | |
a226c368 | 3826 | ipa_register_cgraph_hooks (); |
3b9dd281 | 3827 | inline_free_summary (); |
99c67f24 | 3828 | |
91bf9d9a | 3829 | FOR_EACH_DEFINED_FUNCTION (node) |
02774f2d | 3830 | if (!node->alias) |
99c67f24 | 3831 | inline_analyze_function (node); |
99c67f24 | 3832 | } |
3833 | ||
3834 | ||
6a18c0be | 3835 | /* Read predicate from IB. */ |
3836 | ||
3837 | static struct predicate | |
3838 | read_predicate (struct lto_input_block *ib) | |
3839 | { | |
3840 | struct predicate out; | |
3841 | clause_t clause; | |
3842 | int k = 0; | |
3843 | ||
e876d531 | 3844 | do |
6a18c0be | 3845 | { |
905aa3bd | 3846 | gcc_assert (k <= MAX_CLAUSES); |
7f385784 | 3847 | clause = out.clause[k++] = streamer_read_uhwi (ib); |
6a18c0be | 3848 | } |
3849 | while (clause); | |
72cb6720 | 3850 | |
3851 | /* Zero-initialize the remaining clauses in OUT. */ | |
3852 | while (k <= MAX_CLAUSES) | |
3853 | out.clause[k++] = 0; | |
3854 | ||
6a18c0be | 3855 | return out; |
3856 | } | |
3857 | ||
3858 | ||
0835ad03 | 3859 | /* Write inline summary for edge E to OB. */ |
3860 | ||
3861 | static void | |
3862 | read_inline_edge_summary (struct lto_input_block *ib, struct cgraph_edge *e) | |
3863 | { | |
3864 | struct inline_edge_summary *es = inline_edge_summary (e); | |
6a18c0be | 3865 | struct predicate p; |
eb4ae064 | 3866 | int length, i; |
6a18c0be | 3867 | |
7f385784 | 3868 | es->call_stmt_size = streamer_read_uhwi (ib); |
3869 | es->call_stmt_time = streamer_read_uhwi (ib); | |
3870 | es->loop_depth = streamer_read_uhwi (ib); | |
6a18c0be | 3871 | p = read_predicate (ib); |
3872 | edge_set_predicate (e, &p); | |
eb4ae064 | 3873 | length = streamer_read_uhwi (ib); |
3874 | if (length) | |
3875 | { | |
f1f41a6c | 3876 | es->param.safe_grow_cleared (length); |
eb4ae064 | 3877 | for (i = 0; i < length; i++) |
e876d531 | 3878 | es->param[i].change_prob = streamer_read_uhwi (ib); |
eb4ae064 | 3879 | } |
0835ad03 | 3880 | } |
3881 | ||
3882 | ||
a41f2a28 | 3883 | /* Stream in inline summaries from the section. */ |
3884 | ||
3885 | static void | |
3886 | inline_read_section (struct lto_file_decl_data *file_data, const char *data, | |
3887 | size_t len) | |
3888 | { | |
3889 | const struct lto_function_header *header = | |
3890 | (const struct lto_function_header *) data; | |
949e5786 | 3891 | const int cfg_offset = sizeof (struct lto_function_header); |
3892 | const int main_offset = cfg_offset + header->cfg_size; | |
3893 | const int string_offset = main_offset + header->main_size; | |
a41f2a28 | 3894 | struct data_in *data_in; |
3895 | struct lto_input_block ib; | |
3896 | unsigned int i, count2, j; | |
3897 | unsigned int f_count; | |
3898 | ||
3899 | LTO_INIT_INPUT_BLOCK (ib, (const char *) data + main_offset, 0, | |
3900 | header->main_size); | |
3901 | ||
3902 | data_in = | |
3903 | lto_data_in_create (file_data, (const char *) data + string_offset, | |
1e094109 | 3904 | header->string_size, vNULL); |
7f385784 | 3905 | f_count = streamer_read_uhwi (&ib); |
a41f2a28 | 3906 | for (i = 0; i < f_count; i++) |
3907 | { | |
3908 | unsigned int index; | |
3909 | struct cgraph_node *node; | |
3910 | struct inline_summary *info; | |
70225339 | 3911 | lto_symtab_encoder_t encoder; |
a41f2a28 | 3912 | struct bitpack_d bp; |
0835ad03 | 3913 | struct cgraph_edge *e; |
7c07aa3d | 3914 | predicate p; |
a41f2a28 | 3915 | |
7f385784 | 3916 | index = streamer_read_uhwi (&ib); |
70225339 | 3917 | encoder = file_data->symtab_node_encoder; |
3918 | node = cgraph (lto_symtab_encoder_deref (encoder, index)); | |
a41f2a28 | 3919 | info = inline_summary (node); |
3920 | ||
3921 | info->estimated_stack_size | |
7f385784 | 3922 | = info->estimated_self_stack_size = streamer_read_uhwi (&ib); |
3923 | info->size = info->self_size = streamer_read_uhwi (&ib); | |
3924 | info->time = info->self_time = streamer_read_uhwi (&ib); | |
a41f2a28 | 3925 | |
7f385784 | 3926 | bp = streamer_read_bitpack (&ib); |
a41f2a28 | 3927 | info->inlinable = bp_unpack_value (&bp, 1); |
a41f2a28 | 3928 | |
7f385784 | 3929 | count2 = streamer_read_uhwi (&ib); |
a41f2a28 | 3930 | gcc_assert (!info->conds); |
3931 | for (j = 0; j < count2; j++) | |
3932 | { | |
3933 | struct condition c; | |
7f385784 | 3934 | c.operand_num = streamer_read_uhwi (&ib); |
3935 | c.code = (enum tree_code) streamer_read_uhwi (&ib); | |
515cf651 | 3936 | c.val = stream_read_tree (&ib, data_in); |
a4f60e55 | 3937 | bp = streamer_read_bitpack (&ib); |
3938 | c.agg_contents = bp_unpack_value (&bp, 1); | |
3939 | c.by_ref = bp_unpack_value (&bp, 1); | |
3940 | if (c.agg_contents) | |
3941 | c.offset = streamer_read_uhwi (&ib); | |
f1f41a6c | 3942 | vec_safe_push (info->conds, c); |
a41f2a28 | 3943 | } |
7f385784 | 3944 | count2 = streamer_read_uhwi (&ib); |
a41f2a28 | 3945 | gcc_assert (!info->entry); |
3946 | for (j = 0; j < count2; j++) | |
3947 | { | |
3948 | struct size_time_entry e; | |
a41f2a28 | 3949 | |
7f385784 | 3950 | e.size = streamer_read_uhwi (&ib); |
3951 | e.time = streamer_read_uhwi (&ib); | |
6a18c0be | 3952 | e.predicate = read_predicate (&ib); |
a41f2a28 | 3953 | |
f1f41a6c | 3954 | vec_safe_push (info->entry, e); |
a41f2a28 | 3955 | } |
e876d531 | 3956 | |
7c07aa3d | 3957 | p = read_predicate (&ib); |
3716ee8f | 3958 | set_hint_predicate (&info->loop_iterations, p); |
3959 | p = read_predicate (&ib); | |
3960 | set_hint_predicate (&info->loop_stride, p); | |
be343a9c | 3961 | p = read_predicate (&ib); |
3962 | set_hint_predicate (&info->array_index, p); | |
0835ad03 | 3963 | for (e = node->callees; e; e = e->next_callee) |
3964 | read_inline_edge_summary (&ib, e); | |
3965 | for (e = node->indirect_calls; e; e = e->next_callee) | |
3966 | read_inline_edge_summary (&ib, e); | |
a41f2a28 | 3967 | } |
3968 | ||
3969 | lto_free_section_data (file_data, LTO_section_inline_summary, NULL, data, | |
3970 | len); | |
3971 | lto_data_in_delete (data_in); | |
3972 | } | |
3973 | ||
3974 | ||
99c67f24 | 3975 | /* Read inline summary. Jump functions are shared among ipa-cp |
3976 | and inliner, so when ipa-cp is active, we don't need to write them | |
3977 | twice. */ | |
3978 | ||
3979 | void | |
3980 | inline_read_summary (void) | |
3981 | { | |
c7b2cc59 | 3982 | struct lto_file_decl_data **file_data_vec = lto_get_file_decl_data (); |
3983 | struct lto_file_decl_data *file_data; | |
3984 | unsigned int j = 0; | |
3985 | ||
3986 | inline_summary_alloc (); | |
3987 | ||
3988 | while ((file_data = file_data_vec[j++])) | |
3989 | { | |
3990 | size_t len; | |
eb4ae064 | 3991 | const char *data = lto_get_section_data (file_data, |
3992 | LTO_section_inline_summary, | |
3993 | NULL, &len); | |
a41f2a28 | 3994 | if (data) |
e876d531 | 3995 | inline_read_section (file_data, data, len); |
c7b2cc59 | 3996 | else |
eb4ae064 | 3997 | /* Fatal error here. We do not want to support compiling ltrans units |
3998 | with different version of compiler or different flags than the WPA | |
3999 | unit, so this should never happen. */ | |
c7b2cc59 | 4000 | fatal_error ("ipa inline summary is missing in input file"); |
4001 | } | |
a226c368 | 4002 | if (optimize) |
99c67f24 | 4003 | { |
4004 | ipa_register_cgraph_hooks (); | |
4005 | if (!flag_ipa_cp) | |
e876d531 | 4006 | ipa_prop_read_jump_functions (); |
99c67f24 | 4007 | } |
4008 | function_insertion_hook_holder = | |
e876d531 | 4009 | cgraph_add_function_insertion_hook (&add_new_function, NULL); |
99c67f24 | 4010 | } |
4011 | ||
6a18c0be | 4012 | |
4013 | /* Write predicate P to OB. */ | |
4014 | ||
4015 | static void | |
4016 | write_predicate (struct output_block *ob, struct predicate *p) | |
4017 | { | |
4018 | int j; | |
4019 | if (p) | |
4020 | for (j = 0; p->clause[j]; j++) | |
4021 | { | |
e876d531 | 4022 | gcc_assert (j < MAX_CLAUSES); |
4023 | streamer_write_uhwi (ob, p->clause[j]); | |
6a18c0be | 4024 | } |
7f385784 | 4025 | streamer_write_uhwi (ob, 0); |
6a18c0be | 4026 | } |
4027 | ||
4028 | ||
0835ad03 | 4029 | /* Write inline summary for edge E to OB. */ |
4030 | ||
4031 | static void | |
4032 | write_inline_edge_summary (struct output_block *ob, struct cgraph_edge *e) | |
4033 | { | |
4034 | struct inline_edge_summary *es = inline_edge_summary (e); | |
eb4ae064 | 4035 | int i; |
4036 | ||
7f385784 | 4037 | streamer_write_uhwi (ob, es->call_stmt_size); |
4038 | streamer_write_uhwi (ob, es->call_stmt_time); | |
4039 | streamer_write_uhwi (ob, es->loop_depth); | |
6a18c0be | 4040 | write_predicate (ob, es->predicate); |
f1f41a6c | 4041 | streamer_write_uhwi (ob, es->param.length ()); |
e876d531 | 4042 | for (i = 0; i < (int) es->param.length (); i++) |
f1f41a6c | 4043 | streamer_write_uhwi (ob, es->param[i].change_prob); |
0835ad03 | 4044 | } |
4045 | ||
99c67f24 | 4046 | |
4047 | /* Write inline summary for node in SET. | |
4048 | Jump functions are shared among ipa-cp and inliner, so when ipa-cp is | |
4049 | active, we don't need to write them twice. */ | |
4050 | ||
4051 | void | |
eab36a5a | 4052 | inline_write_summary (void) |
99c67f24 | 4053 | { |
c7b2cc59 | 4054 | struct cgraph_node *node; |
a41f2a28 | 4055 | struct output_block *ob = create_output_block (LTO_section_inline_summary); |
70225339 | 4056 | lto_symtab_encoder_t encoder = ob->decl_state->symtab_node_encoder; |
c7b2cc59 | 4057 | unsigned int count = 0; |
4058 | int i; | |
4059 | ||
70225339 | 4060 | for (i = 0; i < lto_symtab_encoder_size (encoder); i++) |
2dc9831f | 4061 | { |
452659af | 4062 | symtab_node *snode = lto_symtab_encoder_deref (encoder, i); |
2dc9831f | 4063 | cgraph_node *cnode = dyn_cast <cgraph_node> (snode); |
02774f2d | 4064 | if (cnode && cnode->definition && !cnode->alias) |
2dc9831f | 4065 | count++; |
4066 | } | |
7f385784 | 4067 | streamer_write_uhwi (ob, count); |
c7b2cc59 | 4068 | |
70225339 | 4069 | for (i = 0; i < lto_symtab_encoder_size (encoder); i++) |
c7b2cc59 | 4070 | { |
452659af | 4071 | symtab_node *snode = lto_symtab_encoder_deref (encoder, i); |
2dc9831f | 4072 | cgraph_node *cnode = dyn_cast <cgraph_node> (snode); |
02774f2d | 4073 | if (cnode && (node = cnode)->definition && !node->alias) |
c7b2cc59 | 4074 | { |
4075 | struct inline_summary *info = inline_summary (node); | |
cbd7f5a0 | 4076 | struct bitpack_d bp; |
0835ad03 | 4077 | struct cgraph_edge *edge; |
a41f2a28 | 4078 | int i; |
4079 | size_time_entry *e; | |
4080 | struct condition *c; | |
e876d531 | 4081 | |
4082 | streamer_write_uhwi (ob, | |
4083 | lto_symtab_encoder_encode (encoder, | |
02774f2d | 4084 | |
e876d531 | 4085 | node)); |
7f385784 | 4086 | streamer_write_hwi (ob, info->estimated_self_stack_size); |
4087 | streamer_write_hwi (ob, info->self_size); | |
4088 | streamer_write_hwi (ob, info->self_time); | |
cbd7f5a0 | 4089 | bp = bitpack_create (ob->main_stream); |
4090 | bp_pack_value (&bp, info->inlinable, 1); | |
7f385784 | 4091 | streamer_write_bitpack (&bp); |
f1f41a6c | 4092 | streamer_write_uhwi (ob, vec_safe_length (info->conds)); |
4093 | for (i = 0; vec_safe_iterate (info->conds, i, &c); i++) | |
a41f2a28 | 4094 | { |
7f385784 | 4095 | streamer_write_uhwi (ob, c->operand_num); |
4096 | streamer_write_uhwi (ob, c->code); | |
515cf651 | 4097 | stream_write_tree (ob, c->val, true); |
a4f60e55 | 4098 | bp = bitpack_create (ob->main_stream); |
4099 | bp_pack_value (&bp, c->agg_contents, 1); | |
4100 | bp_pack_value (&bp, c->by_ref, 1); | |
4101 | streamer_write_bitpack (&bp); | |
4102 | if (c->agg_contents) | |
e876d531 | 4103 | streamer_write_uhwi (ob, c->offset); |
a41f2a28 | 4104 | } |
f1f41a6c | 4105 | streamer_write_uhwi (ob, vec_safe_length (info->entry)); |
4106 | for (i = 0; vec_safe_iterate (info->entry, i, &e); i++) | |
a41f2a28 | 4107 | { |
7f385784 | 4108 | streamer_write_uhwi (ob, e->size); |
4109 | streamer_write_uhwi (ob, e->time); | |
6a18c0be | 4110 | write_predicate (ob, &e->predicate); |
a41f2a28 | 4111 | } |
7c07aa3d | 4112 | write_predicate (ob, info->loop_iterations); |
3716ee8f | 4113 | write_predicate (ob, info->loop_stride); |
be343a9c | 4114 | write_predicate (ob, info->array_index); |
0835ad03 | 4115 | for (edge = node->callees; edge; edge = edge->next_callee) |
4116 | write_inline_edge_summary (ob, edge); | |
4117 | for (edge = node->indirect_calls; edge; edge = edge->next_callee) | |
4118 | write_inline_edge_summary (ob, edge); | |
c7b2cc59 | 4119 | } |
4120 | } | |
7f385784 | 4121 | streamer_write_char_stream (ob->main_stream, 0); |
a41f2a28 | 4122 | produce_asm (ob, NULL); |
4123 | destroy_output_block (ob); | |
c7b2cc59 | 4124 | |
a226c368 | 4125 | if (optimize && !flag_ipa_cp) |
eab36a5a | 4126 | ipa_prop_write_jump_functions (); |
99c67f24 | 4127 | } |
4128 | ||
c7b2cc59 | 4129 | |
99c67f24 | 4130 | /* Release inline summary. */ |
4131 | ||
4132 | void | |
4133 | inline_free_summary (void) | |
4134 | { | |
3b9dd281 | 4135 | struct cgraph_node *node; |
f1f41a6c | 4136 | if (!inline_edge_summary_vec.exists ()) |
f8bfd7f7 | 4137 | return; |
3b9dd281 | 4138 | FOR_EACH_DEFINED_FUNCTION (node) |
4139 | reset_inline_summary (node); | |
c7b2cc59 | 4140 | if (function_insertion_hook_holder) |
4141 | cgraph_remove_function_insertion_hook (function_insertion_hook_holder); | |
4142 | function_insertion_hook_holder = NULL; | |
4143 | if (node_removal_hook_holder) | |
4144 | cgraph_remove_node_removal_hook (node_removal_hook_holder); | |
3b9dd281 | 4145 | node_removal_hook_holder = NULL; |
0835ad03 | 4146 | if (edge_removal_hook_holder) |
4147 | cgraph_remove_edge_removal_hook (edge_removal_hook_holder); | |
3b9dd281 | 4148 | edge_removal_hook_holder = NULL; |
c7b2cc59 | 4149 | if (node_duplication_hook_holder) |
4150 | cgraph_remove_node_duplication_hook (node_duplication_hook_holder); | |
3b9dd281 | 4151 | node_duplication_hook_holder = NULL; |
0835ad03 | 4152 | if (edge_duplication_hook_holder) |
4153 | cgraph_remove_edge_duplication_hook (edge_duplication_hook_holder); | |
3b9dd281 | 4154 | edge_duplication_hook_holder = NULL; |
f1f41a6c | 4155 | vec_free (inline_summary_vec); |
4156 | inline_edge_summary_vec.release (); | |
6a18c0be | 4157 | if (edge_predicate_pool) |
4158 | free_alloc_pool (edge_predicate_pool); | |
4159 | edge_predicate_pool = 0; | |
99c67f24 | 4160 | } |