1 /* Inlining decision heuristics.
2 Copyright (C) 2003, 2004, 2007, 2008, 2009, 2010, 2011
3 Free Software Foundation, Inc.
4 Contributed by Jan Hubicka
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
22 /* Analysis used by the inliner and other passes limiting code size growth.
24 We estimate for each function
26 - average function execution time
27 - inlining size benefit (that is how much of function body size
28 and its call sequence is expected to disappear by inlining)
29 - inlining time benefit
32 - call statement size and time
34 inlinie_summary datastructures store above information locally (i.e.
35 parameters of the function itself) and globally (i.e. parameters of
36 the function created by applying all the inline decisions already
37 present in the callgraph).
39 We provide accestor to the inline_summary datastructure and
40 basic logic updating the parameters when inlining is performed.
42 The summaries are context sensitive. Context means
43 1) partial assignment of known constant values of operands
44 2) whether function is inlined into the call or not.
45 It is easy to add more variants. To represent function size and time
46 that depends on context (i.e. it is known to be optimized away when
47 context is known either by inlining or from IP-CP and clonning),
48 we use predicates. Predicates are logical formulas in
49 conjunctive-disjunctive form consisting of clauses. Clauses are bitmaps
50 specifying what conditions must be true. Conditions are simple test
51 of the form described above.
53 In order to make predicate (possibly) true, all of its clauses must
54 be (possibly) true. To make clause (possibly) true, one of conditions
55 it mentions must be (possibly) true. There are fixed bounds on
56 number of clauses and conditions and all the manipulation functions
57 are conservative in positive direction. I.e. we may lose precision
58 by thinking that predicate may be true even when it is not.
60 estimate_edge_size and estimate_edge_growth can be used to query
61 function size/time in the given context. inline_merge_summary merges
62 properties of caller and callee after inlining.
64 Finally pass_inline_parameters is exported. This is used to drive
65 computation of function parameters used by the early inliner. IPA
66 inlined performs analysis via its analyze_function method. */
70 #include "coretypes.h"
73 #include "tree-inline.h"
74 #include "langhooks.h"
77 #include "diagnostic.h"
78 #include "gimple-pretty-print.h"
80 #include "tree-pass.h"
83 #include "tree-flow.h"
85 #include "lto-streamer.h"
86 #include "data-streamer.h"
87 #include "tree-streamer.h"
88 #include "ipa-inline.h"
89 #include "alloc-pool.h"
92 #include "tree-scalar-evolution.h"
94 /* Estimate runtime of function can easilly run into huge numbers with many
95 nested loops. Be sure we can compute time * INLINE_SIZE_SCALE * 2 in an
96 integer. For anything larger we use gcov_type. */
97 #define MAX_TIME 500000
99 /* Number of bits in integer, but we really want to be stable across different
101 #define NUM_CONDITIONS 32
103 enum predicate_conditions
105 predicate_false_condition
= 0,
106 predicate_not_inlined_condition
= 1,
107 predicate_first_dynamic_condition
= 2
110 /* Special condition code we use to represent test that operand is compile time
112 #define IS_NOT_CONSTANT ERROR_MARK
113 /* Special condition code we use to represent test that operand is not changed
114 across invocation of the function. When operand IS_NOT_CONSTANT it is always
115 CHANGED, however i.e. loop invariants can be NOT_CHANGED given percentage
116 of executions even when they are not compile time constants. */
117 #define CHANGED IDENTIFIER_NODE
119 /* Holders of ipa cgraph hooks: */
120 static struct cgraph_node_hook_list
*function_insertion_hook_holder
;
121 static struct cgraph_node_hook_list
*node_removal_hook_holder
;
122 static struct cgraph_2node_hook_list
*node_duplication_hook_holder
;
123 static struct cgraph_2edge_hook_list
*edge_duplication_hook_holder
;
124 static struct cgraph_edge_hook_list
*edge_removal_hook_holder
;
125 static void inline_node_removal_hook (struct cgraph_node
*, void *);
126 static void inline_node_duplication_hook (struct cgraph_node
*,
127 struct cgraph_node
*, void *);
128 static void inline_edge_removal_hook (struct cgraph_edge
*, void *);
129 static void inline_edge_duplication_hook (struct cgraph_edge
*,
130 struct cgraph_edge
*,
133 /* VECtor holding inline summaries.
134 In GGC memory because conditions might point to constant trees. */
135 VEC(inline_summary_t
,gc
) *inline_summary_vec
;
136 VEC(inline_edge_summary_t
,heap
) *inline_edge_summary_vec
;
138 /* Cached node/edge growths. */
139 VEC(int,heap
) *node_growth_cache
;
140 VEC(edge_growth_cache_entry
,heap
) *edge_growth_cache
;
142 /* Edge predicates goes here. */
143 static alloc_pool edge_predicate_pool
;
145 /* Return true predicate (tautology).
146 We represent it by empty list of clauses. */
148 static inline struct predicate
149 true_predicate (void)
157 /* Return predicate testing single condition number COND. */
159 static inline struct predicate
160 single_cond_predicate (int cond
)
163 p
.clause
[0]=1 << cond
;
169 /* Return false predicate. First clause require false condition. */
171 static inline struct predicate
172 false_predicate (void)
174 return single_cond_predicate (predicate_false_condition
);
178 /* Return true if P is (false). */
181 true_predicate_p (struct predicate
*p
)
183 return !p
->clause
[0];
187 /* Return true if P is (false). */
190 false_predicate_p (struct predicate
*p
)
192 if (p
->clause
[0] == (1 << predicate_false_condition
))
194 gcc_checking_assert (!p
->clause
[1]
195 && p
->clause
[0] == 1 << predicate_false_condition
);
202 /* Return predicate that is set true when function is not inlined. */
203 static inline struct predicate
204 not_inlined_predicate (void)
206 return single_cond_predicate (predicate_not_inlined_condition
);
209 /* Simple description of whether a memory load or a condition refers to a load
210 from an aggregate and if so, how and where from in the aggregate.
211 Individual fields have the same meaning like fields with the same name in
214 struct agg_position_info
216 HOST_WIDE_INT offset
;
221 /* Add condition to condition list CONDS. AGGPOS describes whether the used
222 oprand is loaded from an aggregate and where in the aggregate it is. It can
223 be NULL, which means this not a load from an aggregate. */
225 static struct predicate
226 add_condition (struct inline_summary
*summary
, int operand_num
,
227 struct agg_position_info
*aggpos
,
228 enum tree_code code
, tree val
)
232 struct condition new_cond
;
233 HOST_WIDE_INT offset
;
234 bool agg_contents
, by_ref
;
238 offset
= aggpos
->offset
;
239 agg_contents
= aggpos
->agg_contents
;
240 by_ref
= aggpos
->by_ref
;
245 agg_contents
= false;
249 gcc_checking_assert (operand_num
>= 0);
250 for (i
= 0; VEC_iterate (condition
, summary
->conds
, i
, c
); i
++)
252 if (c
->operand_num
== operand_num
255 && c
->agg_contents
== agg_contents
256 && (!agg_contents
|| (c
->offset
== offset
&& c
->by_ref
== by_ref
)))
257 return single_cond_predicate (i
+ predicate_first_dynamic_condition
);
259 /* Too many conditions. Give up and return constant true. */
260 if (i
== NUM_CONDITIONS
- predicate_first_dynamic_condition
)
261 return true_predicate ();
263 new_cond
.operand_num
= operand_num
;
264 new_cond
.code
= code
;
266 new_cond
.agg_contents
= agg_contents
;
267 new_cond
.by_ref
= by_ref
;
268 new_cond
.offset
= offset
;
269 VEC_safe_push (condition
, gc
, summary
->conds
, new_cond
);
270 return single_cond_predicate (i
+ predicate_first_dynamic_condition
);
274 /* Add clause CLAUSE into the predicate P. */
277 add_clause (conditions conditions
, struct predicate
*p
, clause_t clause
)
281 int insert_here
= -1;
288 /* False clause makes the whole predicate false. Kill the other variants. */
289 if (clause
== (1 << predicate_false_condition
))
291 p
->clause
[0] = (1 << predicate_false_condition
);
295 if (false_predicate_p (p
))
298 /* No one should be sily enough to add false into nontrivial clauses. */
299 gcc_checking_assert (!(clause
& (1 << predicate_false_condition
)));
301 /* Look where to insert the clause. At the same time prune out
302 clauses of P that are implied by the new clause and thus
304 for (i
= 0, i2
= 0; i
<= MAX_CLAUSES
; i
++)
306 p
->clause
[i2
] = p
->clause
[i
];
311 /* If p->clause[i] implies clause, there is nothing to add. */
312 if ((p
->clause
[i
] & clause
) == p
->clause
[i
])
314 /* We had nothing to add, none of clauses should've become
316 gcc_checking_assert (i
== i2
);
320 if (p
->clause
[i
] < clause
&& insert_here
< 0)
323 /* If clause implies p->clause[i], then p->clause[i] becomes redundant.
324 Otherwise the p->clause[i] has to stay. */
325 if ((p
->clause
[i
] & clause
) != clause
)
329 /* Look for clauses that are obviously true. I.e.
330 op0 == 5 || op0 != 5. */
331 for (c1
= predicate_first_dynamic_condition
; c1
< NUM_CONDITIONS
; c1
++)
334 if (!(clause
& (1 << c1
)))
336 cc1
= &VEC_index (condition
,
338 c1
- predicate_first_dynamic_condition
);
339 /* We have no way to represent !CHANGED and !IS_NOT_CONSTANT
340 and thus there is no point for looking for them. */
341 if (cc1
->code
== CHANGED
342 || cc1
->code
== IS_NOT_CONSTANT
)
344 for (c2
= c1
+ 1; c2
<= NUM_CONDITIONS
; c2
++)
345 if (clause
& (1 << c2
))
347 condition
*cc1
= &VEC_index (condition
,
349 c1
- predicate_first_dynamic_condition
);
350 condition
*cc2
= &VEC_index (condition
,
352 c2
- predicate_first_dynamic_condition
);
353 if (cc1
->operand_num
== cc2
->operand_num
354 && cc1
->val
== cc2
->val
355 && cc2
->code
!= IS_NOT_CONSTANT
356 && cc2
->code
!= CHANGED
357 && cc1
->code
== invert_tree_comparison
359 HONOR_NANS (TYPE_MODE (TREE_TYPE (cc1
->val
)))))
365 /* We run out of variants. Be conservative in positive direction. */
366 if (i2
== MAX_CLAUSES
)
368 /* Keep clauses in decreasing order. This makes equivalence testing easy. */
369 p
->clause
[i2
+ 1] = 0;
370 if (insert_here
>= 0)
371 for (;i2
> insert_here
; i2
--)
372 p
->clause
[i2
] = p
->clause
[i2
- 1];
375 p
->clause
[insert_here
] = clause
;
381 static struct predicate
382 and_predicates (conditions conditions
,
383 struct predicate
*p
, struct predicate
*p2
)
385 struct predicate out
= *p
;
388 /* Avoid busy work. */
389 if (false_predicate_p (p2
) || true_predicate_p (p
))
391 if (false_predicate_p (p
) || true_predicate_p (p2
))
394 /* See how far predicates match. */
395 for (i
= 0; p
->clause
[i
] && p
->clause
[i
] == p2
->clause
[i
]; i
++)
397 gcc_checking_assert (i
< MAX_CLAUSES
);
400 /* Combine the predicates rest. */
401 for (; p2
->clause
[i
]; i
++)
403 gcc_checking_assert (i
< MAX_CLAUSES
);
404 add_clause (conditions
, &out
, p2
->clause
[i
]);
410 /* Return true if predicates are obviously equal. */
413 predicates_equal_p (struct predicate
*p
, struct predicate
*p2
)
416 for (i
= 0; p
->clause
[i
]; i
++)
418 gcc_checking_assert (i
< MAX_CLAUSES
);
419 gcc_checking_assert (p
->clause
[i
] > p
->clause
[i
+ 1]);
420 gcc_checking_assert (!p2
->clause
[i
]
421 || p2
->clause
[i
] > p2
->clause
[i
+ 1]);
422 if (p
->clause
[i
] != p2
->clause
[i
])
425 return !p2
->clause
[i
];
431 static struct predicate
432 or_predicates (conditions conditions
, struct predicate
*p
, struct predicate
*p2
)
434 struct predicate out
= true_predicate ();
437 /* Avoid busy work. */
438 if (false_predicate_p (p2
) || true_predicate_p (p
))
440 if (false_predicate_p (p
) || true_predicate_p (p2
))
442 if (predicates_equal_p (p
, p2
))
445 /* OK, combine the predicates. */
446 for (i
= 0; p
->clause
[i
]; i
++)
447 for (j
= 0; p2
->clause
[j
]; j
++)
449 gcc_checking_assert (i
< MAX_CLAUSES
&& j
< MAX_CLAUSES
);
450 add_clause (conditions
, &out
, p
->clause
[i
] | p2
->clause
[j
]);
456 /* Having partial truth assignment in POSSIBLE_TRUTHS, return false
457 if predicate P is known to be false. */
460 evaluate_predicate (struct predicate
*p
, clause_t possible_truths
)
464 /* True remains true. */
465 if (true_predicate_p (p
))
468 gcc_assert (!(possible_truths
& (1 << predicate_false_condition
)));
470 /* See if we can find clause we can disprove. */
471 for (i
= 0; p
->clause
[i
]; i
++)
473 gcc_checking_assert (i
< MAX_CLAUSES
);
474 if (!(p
->clause
[i
] & possible_truths
))
480 /* Return the probability in range 0...REG_BR_PROB_BASE that the predicated
481 instruction will be recomputed per invocation of the inlined call. */
484 predicate_probability (conditions conds
,
485 struct predicate
*p
, clause_t possible_truths
,
486 VEC (inline_param_summary_t
, heap
) *inline_param_summary
)
489 int combined_prob
= REG_BR_PROB_BASE
;
491 /* True remains true. */
492 if (true_predicate_p (p
))
493 return REG_BR_PROB_BASE
;
495 if (false_predicate_p (p
))
498 gcc_assert (!(possible_truths
& (1 << predicate_false_condition
)));
500 /* See if we can find clause we can disprove. */
501 for (i
= 0; p
->clause
[i
]; i
++)
503 gcc_checking_assert (i
< MAX_CLAUSES
);
504 if (!(p
->clause
[i
] & possible_truths
))
510 if (!inline_param_summary
)
511 return REG_BR_PROB_BASE
;
512 for (i2
= 0; i2
< NUM_CONDITIONS
; i2
++)
513 if ((p
->clause
[i
] & possible_truths
) & (1 << i2
))
515 if (i2
>= predicate_first_dynamic_condition
)
517 condition
*c
= &VEC_index
519 i2
- predicate_first_dynamic_condition
);
520 if (c
->code
== CHANGED
522 < (int) VEC_length (inline_param_summary_t
,
523 inline_param_summary
)))
525 int iprob
= VEC_index (inline_param_summary_t
,
526 inline_param_summary
,
527 c
->operand_num
).change_prob
;
528 this_prob
= MAX (this_prob
, iprob
);
531 this_prob
= REG_BR_PROB_BASE
;
534 this_prob
= REG_BR_PROB_BASE
;
536 combined_prob
= MIN (this_prob
, combined_prob
);
541 return combined_prob
;
545 /* Dump conditional COND. */
548 dump_condition (FILE *f
, conditions conditions
, int cond
)
551 if (cond
== predicate_false_condition
)
552 fprintf (f
, "false");
553 else if (cond
== predicate_not_inlined_condition
)
554 fprintf (f
, "not inlined");
557 c
= &VEC_index (condition
, conditions
,
558 cond
- predicate_first_dynamic_condition
);
559 fprintf (f
, "op%i", c
->operand_num
);
561 fprintf (f
, "[%soffset: " HOST_WIDE_INT_PRINT_DEC
"]",
562 c
->by_ref
? "ref " : "", c
->offset
);
563 if (c
->code
== IS_NOT_CONSTANT
)
565 fprintf (f
, " not constant");
568 if (c
->code
== CHANGED
)
570 fprintf (f
, " changed");
573 fprintf (f
, " %s ", op_symbol_code (c
->code
));
574 print_generic_expr (f
, c
->val
, 1);
579 /* Dump clause CLAUSE. */
582 dump_clause (FILE *f
, conditions conds
, clause_t clause
)
589 for (i
= 0; i
< NUM_CONDITIONS
; i
++)
590 if (clause
& (1 << i
))
595 dump_condition (f
, conds
, i
);
601 /* Dump predicate PREDICATE. */
604 dump_predicate (FILE *f
, conditions conds
, struct predicate
*pred
)
607 if (true_predicate_p (pred
))
608 dump_clause (f
, conds
, 0);
610 for (i
= 0; pred
->clause
[i
]; i
++)
614 dump_clause (f
, conds
, pred
->clause
[i
]);
620 /* Dump inline hints. */
622 dump_inline_hints (FILE *f
, inline_hints hints
)
626 fprintf (f
, "inline hints:");
627 if (hints
& INLINE_HINT_indirect_call
)
629 hints
&= ~INLINE_HINT_indirect_call
;
630 fprintf (f
, " indirect_call");
632 if (hints
& INLINE_HINT_loop_iterations
)
634 hints
&= ~INLINE_HINT_loop_iterations
;
635 fprintf (f
, " loop_iterations");
641 /* Record SIZE and TIME under condition PRED into the inline summary. */
644 account_size_time (struct inline_summary
*summary
, int size
, int time
,
645 struct predicate
*pred
)
651 if (false_predicate_p (pred
))
654 /* We need to create initial empty unconitional clause, but otherwie
655 we don't need to account empty times and sizes. */
656 if (!size
&& !time
&& summary
->entry
)
659 /* Watch overflow that might result from insane profiles. */
660 if (time
> MAX_TIME
* INLINE_TIME_SCALE
)
661 time
= MAX_TIME
* INLINE_TIME_SCALE
;
662 gcc_assert (time
>= 0);
664 for (i
= 0; VEC_iterate (size_time_entry
, summary
->entry
, i
, e
); i
++)
665 if (predicates_equal_p (&e
->predicate
, pred
))
674 e
= &VEC_index (size_time_entry
, summary
->entry
, 0);
675 gcc_assert (!e
->predicate
.clause
[0]);
677 if (dump_file
&& (dump_flags
& TDF_DETAILS
) && (time
|| size
))
679 fprintf (dump_file
, "\t\tAccounting size:%3.2f, time:%3.2f on %spredicate:",
680 ((double)size
) / INLINE_SIZE_SCALE
,
681 ((double)time
) / INLINE_TIME_SCALE
,
682 found
? "" : "new ");
683 dump_predicate (dump_file
, summary
->conds
, pred
);
687 struct size_time_entry new_entry
;
688 new_entry
.size
= size
;
689 new_entry
.time
= time
;
690 new_entry
.predicate
= *pred
;
691 VEC_safe_push (size_time_entry
, gc
, summary
->entry
, new_entry
);
697 if (e
->time
> MAX_TIME
* INLINE_TIME_SCALE
)
698 e
->time
= MAX_TIME
* INLINE_TIME_SCALE
;
702 /* Set predicate for edge E. */
705 edge_set_predicate (struct cgraph_edge
*e
, struct predicate
*predicate
)
707 struct inline_edge_summary
*es
= inline_edge_summary (e
);
708 if (predicate
&& !true_predicate_p (predicate
))
711 es
->predicate
= (struct predicate
*)pool_alloc (edge_predicate_pool
);
712 *es
->predicate
= *predicate
;
717 pool_free (edge_predicate_pool
, es
->predicate
);
718 es
->predicate
= NULL
;
723 /* KNOWN_VALS is partial mapping of parameters of NODE to constant values.
724 KNOWN_AGGS is a vector of aggreggate jump functions for each parameter.
725 Return clause of possible truths. When INLINE_P is true, assume that we are
728 ERROR_MARK means compile time invariant. */
731 evaluate_conditions_for_known_args (struct cgraph_node
*node
,
733 VEC (tree
, heap
) *known_vals
,
734 VEC (ipa_agg_jump_function_p
, heap
) *known_aggs
)
736 clause_t clause
= inline_p
? 0 : 1 << predicate_not_inlined_condition
;
737 struct inline_summary
*info
= inline_summary (node
);
741 for (i
= 0; VEC_iterate (condition
, info
->conds
, i
, c
); i
++)
746 /* We allow call stmt to have fewer arguments than the callee function
747 (especially for K&R style programs). So bound check here (we assume
748 known_aggs vector, if non-NULL, has the same length as
750 gcc_checking_assert (!known_aggs
751 || (VEC_length (tree
, known_vals
)
752 == VEC_length (ipa_agg_jump_function_p
,
754 if (c
->operand_num
>= (int) VEC_length (tree
, known_vals
))
756 clause
|= 1 << (i
+ predicate_first_dynamic_condition
);
762 struct ipa_agg_jump_function
*agg
;
764 if (c
->code
== CHANGED
766 && (VEC_index (tree
, known_vals
, c
->operand_num
)
772 agg
= VEC_index (ipa_agg_jump_function_p
, known_aggs
,
774 val
= ipa_find_agg_cst_for_param (agg
, c
->offset
, c
->by_ref
);
781 val
= VEC_index (tree
, known_vals
, c
->operand_num
);
782 if (val
== error_mark_node
&& c
->code
!= CHANGED
)
788 clause
|= 1 << (i
+ predicate_first_dynamic_condition
);
791 if (c
->code
== IS_NOT_CONSTANT
|| c
->code
== CHANGED
)
793 res
= fold_binary_to_constant (c
->code
, boolean_type_node
, val
, c
->val
);
795 && integer_zerop (res
))
797 clause
|= 1 << (i
+ predicate_first_dynamic_condition
);
803 /* Work out what conditions might be true at invocation of E. */
806 evaluate_properties_for_edge (struct cgraph_edge
*e
, bool inline_p
,
807 clause_t
*clause_ptr
,
808 VEC (tree
, heap
) **known_vals_ptr
,
809 VEC (tree
, heap
) **known_binfos_ptr
,
810 VEC (ipa_agg_jump_function_p
, heap
) **known_aggs_ptr
)
812 struct cgraph_node
*callee
= cgraph_function_or_thunk_node (e
->callee
, NULL
);
813 struct inline_summary
*info
= inline_summary (callee
);
814 VEC (tree
, heap
) *known_vals
= NULL
;
815 VEC (ipa_agg_jump_function_p
, heap
) *known_aggs
= NULL
;
818 *clause_ptr
= inline_p
? 0 : 1 << predicate_not_inlined_condition
;
820 *known_vals_ptr
= NULL
;
821 if (known_binfos_ptr
)
822 *known_binfos_ptr
= NULL
;
824 if (ipa_node_params_vector
825 && !e
->call_stmt_cannot_inline_p
826 && ((clause_ptr
&& info
->conds
) || known_vals_ptr
|| known_binfos_ptr
))
828 struct ipa_node_params
*parms_info
;
829 struct ipa_edge_args
*args
= IPA_EDGE_REF (e
);
830 struct inline_edge_summary
*es
= inline_edge_summary (e
);
831 int i
, count
= ipa_get_cs_argument_count (args
);
833 if (e
->caller
->global
.inlined_to
)
834 parms_info
= IPA_NODE_REF (e
->caller
->global
.inlined_to
);
836 parms_info
= IPA_NODE_REF (e
->caller
);
838 if (count
&& (info
->conds
|| known_vals_ptr
))
839 VEC_safe_grow_cleared (tree
, heap
, known_vals
, count
);
840 if (count
&& (info
->conds
|| known_aggs_ptr
))
841 VEC_safe_grow_cleared (ipa_agg_jump_function_p
, heap
, known_aggs
,
843 if (count
&& known_binfos_ptr
)
844 VEC_safe_grow_cleared (tree
, heap
, *known_binfos_ptr
, count
);
846 for (i
= 0; i
< count
; i
++)
848 struct ipa_jump_func
*jf
= ipa_get_ith_jump_func (args
, i
);
849 tree cst
= ipa_value_from_jfunc (parms_info
, jf
);
852 if (known_vals
&& TREE_CODE (cst
) != TREE_BINFO
)
853 VEC_replace (tree
, known_vals
, i
, cst
);
854 else if (known_binfos_ptr
!= NULL
&& TREE_CODE (cst
) == TREE_BINFO
)
855 VEC_replace (tree
, *known_binfos_ptr
, i
, cst
);
858 && !VEC_index (inline_param_summary_t
,
861 VEC_replace (tree
, known_vals
, i
, error_mark_node
);
862 /* TODO: When IPA-CP starts propagating and merging aggregate jump
863 functions, use its knowledge of the caller too, just like the
864 scalar case above. */
865 VEC_replace (ipa_agg_jump_function_p
, known_aggs
, i
, &jf
->agg
);
870 *clause_ptr
= evaluate_conditions_for_known_args (callee
, inline_p
,
871 known_vals
, known_aggs
);
874 *known_vals_ptr
= known_vals
;
876 VEC_free (tree
, heap
, known_vals
);
879 *known_aggs_ptr
= known_aggs
;
881 VEC_free (ipa_agg_jump_function_p
, heap
, known_aggs
);
885 /* Allocate the inline summary vector or resize it to cover all cgraph nodes. */
888 inline_summary_alloc (void)
890 if (!node_removal_hook_holder
)
891 node_removal_hook_holder
=
892 cgraph_add_node_removal_hook (&inline_node_removal_hook
, NULL
);
893 if (!edge_removal_hook_holder
)
894 edge_removal_hook_holder
=
895 cgraph_add_edge_removal_hook (&inline_edge_removal_hook
, NULL
);
896 if (!node_duplication_hook_holder
)
897 node_duplication_hook_holder
=
898 cgraph_add_node_duplication_hook (&inline_node_duplication_hook
, NULL
);
899 if (!edge_duplication_hook_holder
)
900 edge_duplication_hook_holder
=
901 cgraph_add_edge_duplication_hook (&inline_edge_duplication_hook
, NULL
);
903 if (VEC_length (inline_summary_t
, inline_summary_vec
)
904 <= (unsigned) cgraph_max_uid
)
905 VEC_safe_grow_cleared (inline_summary_t
, gc
,
906 inline_summary_vec
, cgraph_max_uid
+ 1);
907 if (VEC_length (inline_edge_summary_t
, inline_edge_summary_vec
)
908 <= (unsigned) cgraph_edge_max_uid
)
909 VEC_safe_grow_cleared (inline_edge_summary_t
, heap
,
910 inline_edge_summary_vec
, cgraph_edge_max_uid
+ 1);
911 if (!edge_predicate_pool
)
912 edge_predicate_pool
= create_alloc_pool ("edge predicates",
913 sizeof (struct predicate
),
917 /* We are called multiple time for given function; clear
918 data from previous run so they are not cumulated. */
921 reset_inline_edge_summary (struct cgraph_edge
*e
)
924 < (int)VEC_length (inline_edge_summary_t
, inline_edge_summary_vec
))
926 struct inline_edge_summary
*es
= inline_edge_summary (e
);
928 es
->call_stmt_size
= es
->call_stmt_time
=0;
930 pool_free (edge_predicate_pool
, es
->predicate
);
931 es
->predicate
= NULL
;
932 VEC_free (inline_param_summary_t
, heap
, es
->param
);
936 /* We are called multiple time for given function; clear
937 data from previous run so they are not cumulated. */
940 reset_inline_summary (struct cgraph_node
*node
)
942 struct inline_summary
*info
= inline_summary (node
);
943 struct cgraph_edge
*e
;
945 info
->self_size
= info
->self_time
= 0;
946 info
->estimated_stack_size
= 0;
947 info
->estimated_self_stack_size
= 0;
948 info
->stack_frame_offset
= 0;
951 if (info
->loop_iterations
)
953 pool_free (edge_predicate_pool
, info
->loop_iterations
);
954 info
->loop_iterations
= NULL
;
956 VEC_free (condition
, gc
, info
->conds
);
957 VEC_free (size_time_entry
,gc
, info
->entry
);
958 for (e
= node
->callees
; e
; e
= e
->next_callee
)
959 reset_inline_edge_summary (e
);
960 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
961 reset_inline_edge_summary (e
);
964 /* Hook that is called by cgraph.c when a node is removed. */
967 inline_node_removal_hook (struct cgraph_node
*node
, void *data ATTRIBUTE_UNUSED
)
969 struct inline_summary
*info
;
970 if (VEC_length (inline_summary_t
, inline_summary_vec
)
971 <= (unsigned)node
->uid
)
973 info
= inline_summary (node
);
974 reset_inline_summary (node
);
975 memset (info
, 0, sizeof (inline_summary_t
));
979 /* Hook that is called by cgraph.c when a node is duplicated. */
982 inline_node_duplication_hook (struct cgraph_node
*src
, struct cgraph_node
*dst
,
983 ATTRIBUTE_UNUSED
void *data
)
985 struct inline_summary
*info
;
986 inline_summary_alloc ();
987 info
= inline_summary (dst
);
988 memcpy (info
, inline_summary (src
),
989 sizeof (struct inline_summary
));
990 /* TODO: as an optimization, we may avoid copying conditions
991 that are known to be false or true. */
992 info
->conds
= VEC_copy (condition
, gc
, info
->conds
);
994 /* When there are any replacements in the function body, see if we can figure
995 out that something was optimized out. */
996 if (ipa_node_params_vector
&& dst
->clone
.tree_map
)
998 VEC(size_time_entry
,gc
) *entry
= info
->entry
;
999 /* Use SRC parm info since it may not be copied yet. */
1000 struct ipa_node_params
*parms_info
= IPA_NODE_REF (src
);
1001 VEC (tree
, heap
) *known_vals
= NULL
;
1002 int count
= ipa_get_param_count (parms_info
);
1004 clause_t possible_truths
;
1005 struct predicate true_pred
= true_predicate ();
1007 int optimized_out_size
= 0;
1008 gcov_type optimized_out_time
= 0;
1009 bool inlined_to_p
= false;
1010 struct cgraph_edge
*edge
;
1013 VEC_safe_grow_cleared (tree
, heap
, known_vals
, count
);
1014 for (i
= 0; i
< count
; i
++)
1016 tree t
= ipa_get_param (parms_info
, i
);
1017 struct ipa_replace_map
*r
;
1020 VEC_iterate (ipa_replace_map_p
, dst
->clone
.tree_map
, j
, r
);
1023 if (r
->old_tree
== t
1027 VEC_replace (tree
, known_vals
, i
, r
->new_tree
);
1032 possible_truths
= evaluate_conditions_for_known_args (dst
, false,
1034 VEC_free (tree
, heap
, known_vals
);
1036 account_size_time (info
, 0, 0, &true_pred
);
1038 /* Remap size_time vectors.
1039 Simplify the predicate by prunning out alternatives that are known
1041 TODO: as on optimization, we can also eliminate conditions known
1043 for (i
= 0; VEC_iterate (size_time_entry
, entry
, i
, e
); i
++)
1045 struct predicate new_predicate
= true_predicate ();
1046 for (j
= 0; e
->predicate
.clause
[j
]; j
++)
1047 if (!(possible_truths
& e
->predicate
.clause
[j
]))
1049 new_predicate
= false_predicate ();
1053 add_clause (info
->conds
, &new_predicate
,
1054 possible_truths
& e
->predicate
.clause
[j
]);
1055 if (false_predicate_p (&new_predicate
))
1057 optimized_out_size
+= e
->size
;
1058 optimized_out_time
+= e
->time
;
1061 account_size_time (info
, e
->size
, e
->time
, &new_predicate
);
1064 /* Remap edge predicates with the same simplification as above.
1065 Also copy constantness arrays. */
1066 for (edge
= dst
->callees
; edge
; edge
= edge
->next_callee
)
1068 struct predicate new_predicate
= true_predicate ();
1069 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1071 if (!edge
->inline_failed
)
1072 inlined_to_p
= true;
1075 for (j
= 0; es
->predicate
->clause
[j
]; j
++)
1076 if (!(possible_truths
& es
->predicate
->clause
[j
]))
1078 new_predicate
= false_predicate ();
1082 add_clause (info
->conds
, &new_predicate
,
1083 possible_truths
& es
->predicate
->clause
[j
]);
1084 if (false_predicate_p (&new_predicate
)
1085 && !false_predicate_p (es
->predicate
))
1087 optimized_out_size
+= es
->call_stmt_size
* INLINE_SIZE_SCALE
;
1088 optimized_out_time
+= (es
->call_stmt_time
1089 * (INLINE_TIME_SCALE
/ CGRAPH_FREQ_BASE
)
1091 edge
->frequency
= 0;
1093 edge_set_predicate (edge
, &new_predicate
);
1096 /* Remap indirect edge predicates with the same simplificaiton as above.
1097 Also copy constantness arrays. */
1098 for (edge
= dst
->indirect_calls
; edge
; edge
= edge
->next_callee
)
1100 struct predicate new_predicate
= true_predicate ();
1101 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1103 if (!edge
->inline_failed
)
1104 inlined_to_p
= true;
1107 for (j
= 0; es
->predicate
->clause
[j
]; j
++)
1108 if (!(possible_truths
& es
->predicate
->clause
[j
]))
1110 new_predicate
= false_predicate ();
1114 add_clause (info
->conds
, &new_predicate
,
1115 possible_truths
& es
->predicate
->clause
[j
]);
1116 if (false_predicate_p (&new_predicate
)
1117 && !false_predicate_p (es
->predicate
))
1119 optimized_out_size
+= es
->call_stmt_size
* INLINE_SIZE_SCALE
;
1120 optimized_out_time
+= (es
->call_stmt_time
1121 * (INLINE_TIME_SCALE
/ CGRAPH_FREQ_BASE
)
1123 edge
->frequency
= 0;
1125 edge_set_predicate (edge
, &new_predicate
);
1127 if (info
->loop_iterations
)
1129 struct predicate new_predicate
= true_predicate ();
1131 for (j
= 0; info
->loop_iterations
->clause
[j
]; j
++)
1132 if (!(possible_truths
& info
->loop_iterations
->clause
[j
]))
1134 new_predicate
= false_predicate ();
1138 add_clause (info
->conds
, &new_predicate
,
1139 possible_truths
& info
->loop_iterations
->clause
[j
]);
1140 if (false_predicate_p (&new_predicate
)
1141 || true_predicate_p (&new_predicate
))
1142 info
->loop_iterations
= NULL
;
1145 info
->loop_iterations
= (struct predicate
*)pool_alloc (edge_predicate_pool
);
1146 *info
->loop_iterations
= new_predicate
;
1150 /* If inliner or someone after inliner will ever start producing
1151 non-trivial clones, we will get trouble with lack of information
1152 about updating self sizes, because size vectors already contains
1153 sizes of the calees. */
1154 gcc_assert (!inlined_to_p
1155 || (!optimized_out_size
&& !optimized_out_time
));
1157 info
->size
-= optimized_out_size
/ INLINE_SIZE_SCALE
;
1158 info
->self_size
-= optimized_out_size
/ INLINE_SIZE_SCALE
;
1159 gcc_assert (info
->size
> 0);
1160 gcc_assert (info
->self_size
> 0);
1162 optimized_out_time
/= INLINE_TIME_SCALE
;
1163 if (optimized_out_time
> MAX_TIME
)
1164 optimized_out_time
= MAX_TIME
;
1165 info
->time
-= optimized_out_time
;
1166 info
->self_time
-= optimized_out_time
;
1169 if (info
->self_time
< 0)
1170 info
->self_time
= 0;
1174 info
->entry
= VEC_copy (size_time_entry
, gc
, info
->entry
);
1175 if (info
->loop_iterations
)
1177 predicate p
= *info
->loop_iterations
;
1178 info
->loop_iterations
= (struct predicate
*)pool_alloc (edge_predicate_pool
);
1179 *info
->loop_iterations
= p
;
1185 /* Hook that is called by cgraph.c when a node is duplicated. */
1188 inline_edge_duplication_hook (struct cgraph_edge
*src
, struct cgraph_edge
*dst
,
1189 ATTRIBUTE_UNUSED
void *data
)
1191 struct inline_edge_summary
*info
;
1192 struct inline_edge_summary
*srcinfo
;
1193 inline_summary_alloc ();
1194 info
= inline_edge_summary (dst
);
1195 srcinfo
= inline_edge_summary (src
);
1196 memcpy (info
, srcinfo
,
1197 sizeof (struct inline_edge_summary
));
1198 info
->predicate
= NULL
;
1199 edge_set_predicate (dst
, srcinfo
->predicate
);
1200 info
->param
= VEC_copy (inline_param_summary_t
, heap
, srcinfo
->param
);
1204 /* Keep edge cache consistent across edge removal. */
1207 inline_edge_removal_hook (struct cgraph_edge
*edge
, void *data ATTRIBUTE_UNUSED
)
1209 if (edge_growth_cache
)
1210 reset_edge_growth_cache (edge
);
1211 reset_inline_edge_summary (edge
);
1215 /* Initialize growth caches. */
1218 initialize_growth_caches (void)
1220 if (cgraph_edge_max_uid
)
1221 VEC_safe_grow_cleared (edge_growth_cache_entry
, heap
, edge_growth_cache
,
1222 cgraph_edge_max_uid
);
1224 VEC_safe_grow_cleared (int, heap
, node_growth_cache
, cgraph_max_uid
);
1228 /* Free growth caches. */
1231 free_growth_caches (void)
1233 VEC_free (edge_growth_cache_entry
, heap
, edge_growth_cache
);
1234 edge_growth_cache
= 0;
1235 VEC_free (int, heap
, node_growth_cache
);
1236 node_growth_cache
= 0;
1240 /* Dump edge summaries associated to NODE and recursively to all clones.
1241 Indent by INDENT. */
1244 dump_inline_edge_summary (FILE * f
, int indent
, struct cgraph_node
*node
,
1245 struct inline_summary
*info
)
1247 struct cgraph_edge
*edge
;
1248 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
1250 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1251 struct cgraph_node
*callee
= cgraph_function_or_thunk_node (edge
->callee
, NULL
);
1254 fprintf (f
, "%*s%s/%i %s\n%*s loop depth:%2i freq:%4i size:%2i time: %2i callee size:%2i stack:%2i",
1255 indent
, "", cgraph_node_name (callee
),
1257 !edge
->inline_failed
? "inlined"
1258 : cgraph_inline_failed_string (edge
->inline_failed
),
1264 (int)inline_summary (callee
)->size
/ INLINE_SIZE_SCALE
,
1265 (int)inline_summary (callee
)->estimated_stack_size
);
1269 fprintf (f
, " predicate: ");
1270 dump_predicate (f
, info
->conds
, es
->predicate
);
1275 for (i
= 0; i
< (int)VEC_length (inline_param_summary_t
, es
->param
);
1278 int prob
= VEC_index (inline_param_summary_t
,
1279 es
->param
, i
).change_prob
;
1282 fprintf (f
, "%*s op%i is compile time invariant\n",
1284 else if (prob
!= REG_BR_PROB_BASE
)
1285 fprintf (f
, "%*s op%i change %f%% of time\n", indent
+ 2, "", i
,
1286 prob
* 100.0 / REG_BR_PROB_BASE
);
1288 if (!edge
->inline_failed
)
1290 fprintf (f
, "%*sStack frame offset %i, callee self size %i,"
1291 " callee size %i\n",
1293 (int)inline_summary (callee
)->stack_frame_offset
,
1294 (int)inline_summary (callee
)->estimated_self_stack_size
,
1295 (int)inline_summary (callee
)->estimated_stack_size
);
1296 dump_inline_edge_summary (f
, indent
+2, callee
, info
);
1299 for (edge
= node
->indirect_calls
; edge
; edge
= edge
->next_callee
)
1301 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
1302 fprintf (f
, "%*sindirect call loop depth:%2i freq:%4i size:%2i"
1308 es
->call_stmt_time
);
1311 fprintf (f
, "predicate: ");
1312 dump_predicate (f
, info
->conds
, es
->predicate
);
1321 dump_inline_summary (FILE * f
, struct cgraph_node
*node
)
1325 struct inline_summary
*s
= inline_summary (node
);
1328 fprintf (f
, "Inline summary for %s/%i", cgraph_node_name (node
),
1330 if (DECL_DISREGARD_INLINE_LIMITS (node
->symbol
.decl
))
1331 fprintf (f
, " always_inline");
1333 fprintf (f
, " inlinable");
1334 fprintf (f
, "\n self time: %i\n",
1336 fprintf (f
, " global time: %i\n", s
->time
);
1337 fprintf (f
, " self size: %i\n",
1339 fprintf (f
, " global size: %i\n", s
->size
);
1340 fprintf (f
, " self stack: %i\n",
1341 (int) s
->estimated_self_stack_size
);
1342 fprintf (f
, " global stack: %i\n",
1343 (int) s
->estimated_stack_size
);
1345 VEC_iterate (size_time_entry
, s
->entry
, i
, e
);
1348 fprintf (f
, " size:%f, time:%f, predicate:",
1349 (double) e
->size
/ INLINE_SIZE_SCALE
,
1350 (double) e
->time
/ INLINE_TIME_SCALE
);
1351 dump_predicate (f
, s
->conds
, &e
->predicate
);
1353 if (s
->loop_iterations
)
1355 fprintf (f
, " loop iterations:");
1356 dump_predicate (f
, s
->conds
, s
->loop_iterations
);
1358 fprintf (f
, " calls:\n");
1359 dump_inline_edge_summary (f
, 4, node
, s
);
1365 debug_inline_summary (struct cgraph_node
*node
)
1367 dump_inline_summary (stderr
, node
);
1371 dump_inline_summaries (FILE *f
)
1373 struct cgraph_node
*node
;
1375 FOR_EACH_DEFINED_FUNCTION (node
)
1376 if (!node
->global
.inlined_to
)
1377 dump_inline_summary (f
, node
);
1380 /* Give initial reasons why inlining would fail on EDGE. This gets either
1381 nullified or usually overwritten by more precise reasons later. */
1384 initialize_inline_failed (struct cgraph_edge
*e
)
1386 struct cgraph_node
*callee
= e
->callee
;
1388 if (e
->indirect_unknown_callee
)
1389 e
->inline_failed
= CIF_INDIRECT_UNKNOWN_CALL
;
1390 else if (!callee
->analyzed
)
1391 e
->inline_failed
= CIF_BODY_NOT_AVAILABLE
;
1392 else if (callee
->local
.redefined_extern_inline
)
1393 e
->inline_failed
= CIF_REDEFINED_EXTERN_INLINE
;
1394 else if (e
->call_stmt_cannot_inline_p
)
1395 e
->inline_failed
= CIF_MISMATCHED_ARGUMENTS
;
1397 e
->inline_failed
= CIF_FUNCTION_NOT_CONSIDERED
;
1400 /* Callback of walk_aliased_vdefs. Flags that it has been invoked to the
1401 boolean variable pointed to by DATA. */
1404 mark_modified (ao_ref
*ao ATTRIBUTE_UNUSED
, tree vdef ATTRIBUTE_UNUSED
,
1407 bool *b
= (bool *) data
;
1412 /* If OP refers to value of function parameter, return the corresponding
1416 unmodified_parm_1 (gimple stmt
, tree op
)
1418 /* SSA_NAME referring to parm default def? */
1419 if (TREE_CODE (op
) == SSA_NAME
1420 && SSA_NAME_IS_DEFAULT_DEF (op
)
1421 && TREE_CODE (SSA_NAME_VAR (op
)) == PARM_DECL
)
1422 return SSA_NAME_VAR (op
);
1423 /* Non-SSA parm reference? */
1424 if (TREE_CODE (op
) == PARM_DECL
)
1426 bool modified
= false;
1429 ao_ref_init (&refd
, op
);
1430 walk_aliased_vdefs (&refd
, gimple_vuse (stmt
), mark_modified
, &modified
,
1438 /* If OP refers to value of function parameter, return the corresponding
1439 parameter. Also traverse chains of SSA register assignments. */
1442 unmodified_parm (gimple stmt
, tree op
)
1444 tree res
= unmodified_parm_1 (stmt
, op
);
1448 if (TREE_CODE (op
) == SSA_NAME
1449 && !SSA_NAME_IS_DEFAULT_DEF (op
)
1450 && gimple_assign_single_p (SSA_NAME_DEF_STMT (op
)))
1451 return unmodified_parm (SSA_NAME_DEF_STMT (op
),
1452 gimple_assign_rhs1 (SSA_NAME_DEF_STMT (op
)));
1456 /* If OP refers to a value of a function parameter or value loaded from an
1457 aggregate passed to a parameter (either by value or reference), return TRUE
1458 and store the number of the parameter to *INDEX_P and information whether
1459 and how it has been loaded from an aggregate into *AGGPOS. INFO describes
1460 the function parameters, STMT is the statement in which OP is used or
1464 unmodified_parm_or_parm_agg_item (struct ipa_node_params
*info
,
1465 gimple stmt
, tree op
, int *index_p
,
1466 struct agg_position_info
*aggpos
)
1468 tree res
= unmodified_parm_1 (stmt
, op
);
1470 gcc_checking_assert (aggpos
);
1473 *index_p
= ipa_get_param_decl_index (info
, res
);
1476 aggpos
->agg_contents
= false;
1477 aggpos
->by_ref
= false;
1481 if (TREE_CODE (op
) == SSA_NAME
)
1483 if (SSA_NAME_IS_DEFAULT_DEF (op
)
1484 || !gimple_assign_single_p (SSA_NAME_DEF_STMT (op
)))
1486 stmt
= SSA_NAME_DEF_STMT (op
);
1487 op
= gimple_assign_rhs1 (stmt
);
1488 if (!REFERENCE_CLASS_P (op
))
1489 return unmodified_parm_or_parm_agg_item (info
, stmt
, op
, index_p
,
1493 aggpos
->agg_contents
= true;
1494 return ipa_load_from_parm_agg (info
, stmt
, op
, index_p
, &aggpos
->offset
,
1498 /* See if statement might disappear after inlining.
1499 0 - means not eliminated
1500 1 - half of statements goes away
1501 2 - for sure it is eliminated.
1502 We are not terribly sophisticated, basically looking for simple abstraction
1503 penalty wrappers. */
1506 eliminated_by_inlining_prob (gimple stmt
)
1508 enum gimple_code code
= gimple_code (stmt
);
1518 if (gimple_num_ops (stmt
) != 2)
1521 /* Casts of parameters, loads from parameters passed by reference
1522 and stores to return value or parameters are often free after
1523 inlining dua to SRA and further combining.
1524 Assume that half of statements goes away. */
1525 if (gimple_assign_rhs_code (stmt
) == CONVERT_EXPR
1526 || gimple_assign_rhs_code (stmt
) == NOP_EXPR
1527 || gimple_assign_rhs_code (stmt
) == VIEW_CONVERT_EXPR
1528 || gimple_assign_rhs_class (stmt
) == GIMPLE_SINGLE_RHS
)
1530 tree rhs
= gimple_assign_rhs1 (stmt
);
1531 tree lhs
= gimple_assign_lhs (stmt
);
1532 tree inner_rhs
= get_base_address (rhs
);
1533 tree inner_lhs
= get_base_address (lhs
);
1534 bool rhs_free
= false;
1535 bool lhs_free
= false;
1542 /* Reads of parameter are expected to be free. */
1543 if (unmodified_parm (stmt
, inner_rhs
))
1546 /* When parameter is not SSA register because its address is taken
1547 and it is just copied into one, the statement will be completely
1548 free after inlining (we will copy propagate backward). */
1549 if (rhs_free
&& is_gimple_reg (lhs
))
1552 /* Reads of parameters passed by reference
1553 expected to be free (i.e. optimized out after inlining). */
1554 if (TREE_CODE(inner_rhs
) == MEM_REF
1555 && unmodified_parm (stmt
, TREE_OPERAND (inner_rhs
, 0)))
1558 /* Copying parameter passed by reference into gimple register is
1559 probably also going to copy propagate, but we can't be quite
1561 if (rhs_free
&& is_gimple_reg (lhs
))
1564 /* Writes to parameters, parameters passed by value and return value
1565 (either dirrectly or passed via invisible reference) are free.
1567 TODO: We ought to handle testcase like
1568 struct a {int a,b;};
1570 retrurnsturct (void)
1576 This translate into:
1591 For that we either need to copy ipa-split logic detecting writes
1593 if (TREE_CODE (inner_lhs
) == PARM_DECL
1594 || TREE_CODE (inner_lhs
) == RESULT_DECL
1595 || (TREE_CODE(inner_lhs
) == MEM_REF
1596 && (unmodified_parm (stmt
, TREE_OPERAND (inner_lhs
, 0))
1597 || (TREE_CODE (TREE_OPERAND (inner_lhs
, 0)) == SSA_NAME
1598 && SSA_NAME_VAR (TREE_OPERAND (inner_lhs
, 0))
1599 && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND
1600 (inner_lhs
, 0))) == RESULT_DECL
))))
1603 && (is_gimple_reg (rhs
) || is_gimple_min_invariant (rhs
)))
1605 if (lhs_free
&& rhs_free
)
1615 /* If BB ends by a conditional we can turn into predicates, attach corresponding
1616 predicates to the CFG edges. */
1619 set_cond_stmt_execution_predicate (struct ipa_node_params
*info
,
1620 struct inline_summary
*summary
,
1626 struct agg_position_info aggpos
;
1627 enum tree_code code
, inverted_code
;
1633 last
= last_stmt (bb
);
1635 || gimple_code (last
) != GIMPLE_COND
)
1637 if (!is_gimple_ip_invariant (gimple_cond_rhs (last
)))
1639 op
= gimple_cond_lhs (last
);
1640 /* TODO: handle conditionals like
1643 if (unmodified_parm_or_parm_agg_item (info
, last
, op
, &index
, &aggpos
))
1645 code
= gimple_cond_code (last
);
1647 = invert_tree_comparison (code
,
1648 HONOR_NANS (TYPE_MODE (TREE_TYPE (op
))));
1650 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1652 struct predicate p
= add_condition (summary
, index
, &aggpos
,
1653 e
->flags
& EDGE_TRUE_VALUE
1654 ? code
: inverted_code
,
1655 gimple_cond_rhs (last
));
1656 e
->aux
= pool_alloc (edge_predicate_pool
);
1657 *(struct predicate
*)e
->aux
= p
;
1661 if (TREE_CODE (op
) != SSA_NAME
)
1664 if (builtin_constant_p (op))
1668 Here we can predicate nonconstant_code. We can't
1669 really handle constant_code since we have no predicate
1670 for this and also the constant code is not known to be
1671 optimized away when inliner doen't see operand is constant.
1672 Other optimizers might think otherwise. */
1673 if (gimple_cond_code (last
) != NE_EXPR
1674 || !integer_zerop (gimple_cond_rhs (last
)))
1676 set_stmt
= SSA_NAME_DEF_STMT (op
);
1677 if (!gimple_call_builtin_p (set_stmt
, BUILT_IN_CONSTANT_P
)
1678 || gimple_call_num_args (set_stmt
) != 1)
1680 op2
= gimple_call_arg (set_stmt
, 0);
1681 if (!unmodified_parm_or_parm_agg_item (info
, set_stmt
, op2
, &index
, &aggpos
))
1683 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1684 if (e
->flags
& EDGE_FALSE_VALUE
)
1686 struct predicate p
= add_condition (summary
, index
, &aggpos
,
1687 IS_NOT_CONSTANT
, NULL_TREE
);
1688 e
->aux
= pool_alloc (edge_predicate_pool
);
1689 *(struct predicate
*)e
->aux
= p
;
1694 /* If BB ends by a switch we can turn into predicates, attach corresponding
1695 predicates to the CFG edges. */
1698 set_switch_stmt_execution_predicate (struct ipa_node_params
*info
,
1699 struct inline_summary
*summary
,
1705 struct agg_position_info aggpos
;
1711 last
= last_stmt (bb
);
1713 || gimple_code (last
) != GIMPLE_SWITCH
)
1715 op
= gimple_switch_index (last
);
1716 if (!unmodified_parm_or_parm_agg_item (info
, last
, op
, &index
, &aggpos
))
1719 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1721 e
->aux
= pool_alloc (edge_predicate_pool
);
1722 *(struct predicate
*)e
->aux
= false_predicate ();
1724 n
= gimple_switch_num_labels(last
);
1725 for (case_idx
= 0; case_idx
< n
; ++case_idx
)
1727 tree cl
= gimple_switch_label (last
, case_idx
);
1731 e
= find_edge (bb
, label_to_block (CASE_LABEL (cl
)));
1732 min
= CASE_LOW (cl
);
1733 max
= CASE_HIGH (cl
);
1735 /* For default we might want to construct predicate that none
1736 of cases is met, but it is bit hard to do not having negations
1737 of conditionals handy. */
1739 p
= true_predicate ();
1741 p
= add_condition (summary
, index
, &aggpos
, EQ_EXPR
, min
);
1744 struct predicate p1
, p2
;
1745 p1
= add_condition (summary
, index
, &aggpos
, GE_EXPR
, min
);
1746 p2
= add_condition (summary
, index
, &aggpos
, LE_EXPR
, max
);
1747 p
= and_predicates (summary
->conds
, &p1
, &p2
);
1749 *(struct predicate
*)e
->aux
1750 = or_predicates (summary
->conds
, &p
, (struct predicate
*)e
->aux
);
1755 /* For each BB in NODE attach to its AUX pointer predicate under
1756 which it is executable. */
1759 compute_bb_predicates (struct cgraph_node
*node
,
1760 struct ipa_node_params
*parms_info
,
1761 struct inline_summary
*summary
)
1763 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->symbol
.decl
);
1767 FOR_EACH_BB_FN (bb
, my_function
)
1769 set_cond_stmt_execution_predicate (parms_info
, summary
, bb
);
1770 set_switch_stmt_execution_predicate (parms_info
, summary
, bb
);
1773 /* Entry block is always executable. */
1774 ENTRY_BLOCK_PTR_FOR_FUNCTION (my_function
)->aux
1775 = pool_alloc (edge_predicate_pool
);
1776 *(struct predicate
*)ENTRY_BLOCK_PTR_FOR_FUNCTION (my_function
)->aux
1777 = true_predicate ();
1779 /* A simple dataflow propagation of predicates forward in the CFG.
1780 TODO: work in reverse postorder. */
1784 FOR_EACH_BB_FN (bb
, my_function
)
1786 struct predicate p
= false_predicate ();
1789 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
1793 struct predicate this_bb_predicate
1794 = *(struct predicate
*)e
->src
->aux
;
1797 = and_predicates (summary
->conds
, &this_bb_predicate
,
1798 (struct predicate
*)e
->aux
);
1799 p
= or_predicates (summary
->conds
, &p
, &this_bb_predicate
);
1800 if (true_predicate_p (&p
))
1804 if (false_predicate_p (&p
))
1805 gcc_assert (!bb
->aux
);
1811 bb
->aux
= pool_alloc (edge_predicate_pool
);
1812 *((struct predicate
*)bb
->aux
) = p
;
1814 else if (!predicates_equal_p (&p
, (struct predicate
*)bb
->aux
))
1817 *((struct predicate
*)bb
->aux
) = p
;
1825 /* We keep info about constantness of SSA names. */
1827 typedef struct predicate predicate_t
;
1828 DEF_VEC_O (predicate_t
);
1829 DEF_VEC_ALLOC_O (predicate_t
, heap
);
1830 /* Return predicate specifying when the STMT might have result that is not
1831 a compile time constant. */
1833 static struct predicate
1834 will_be_nonconstant_expr_predicate (struct ipa_node_params
*info
,
1835 struct inline_summary
*summary
,
1837 VEC (predicate_t
, heap
) *nonconstant_names
)
1842 while (UNARY_CLASS_P (expr
))
1843 expr
= TREE_OPERAND (expr
, 0);
1845 parm
= unmodified_parm (NULL
, expr
);
1847 && (index
= ipa_get_param_decl_index (info
, parm
)) >= 0)
1848 return add_condition (summary
, index
, NULL
, CHANGED
, NULL_TREE
);
1849 if (is_gimple_min_invariant (expr
))
1850 return false_predicate ();
1851 if (TREE_CODE (expr
) == SSA_NAME
)
1852 return VEC_index (predicate_t
, nonconstant_names
,
1853 SSA_NAME_VERSION (expr
));
1854 if (BINARY_CLASS_P (expr
))
1856 struct predicate p1
= will_be_nonconstant_expr_predicate (info
, summary
, TREE_OPERAND (expr
, 0), nonconstant_names
);
1857 struct predicate p2
;
1858 if (true_predicate_p (&p1
))
1860 p2
= will_be_nonconstant_expr_predicate (info
, summary
, TREE_OPERAND (expr
, 0), nonconstant_names
);
1861 return or_predicates (summary
->conds
, &p1
, &p2
);
1868 return false_predicate ();
1872 /* Return predicate specifying when the STMT might have result that is not
1873 a compile time constant. */
1875 static struct predicate
1876 will_be_nonconstant_predicate (struct ipa_node_params
*info
,
1877 struct inline_summary
*summary
,
1879 VEC (predicate_t
, heap
) *nonconstant_names
)
1881 struct predicate p
= true_predicate ();
1884 struct predicate op_non_const
;
1887 struct agg_position_info aggpos
;
1889 /* What statments might be optimized away
1890 when their arguments are constant
1891 TODO: also trivial builtins.
1892 builtin_constant_p is already handled later. */
1893 if (gimple_code (stmt
) != GIMPLE_ASSIGN
1894 && gimple_code (stmt
) != GIMPLE_COND
1895 && gimple_code (stmt
) != GIMPLE_SWITCH
)
1898 /* Stores will stay anyway. */
1899 if (gimple_vdef (stmt
))
1902 is_load
= gimple_vuse (stmt
) != NULL
;
1903 /* Loads can be optimized when the value is known. */
1907 gcc_assert (gimple_assign_single_p (stmt
));
1908 op
= gimple_assign_rhs1 (stmt
);
1909 if (!unmodified_parm_or_parm_agg_item (info
, stmt
, op
, &base_index
,
1916 /* See if we understand all operands before we start
1917 adding conditionals. */
1918 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
1920 tree parm
= unmodified_parm (stmt
, use
);
1921 /* For arguments we can build a condition. */
1922 if (parm
&& ipa_get_param_decl_index (info
, parm
) >= 0)
1924 if (TREE_CODE (use
) != SSA_NAME
)
1926 /* If we know when operand is constant,
1927 we still can say something useful. */
1928 if (!true_predicate_p (&VEC_index (predicate_t
, nonconstant_names
,
1929 SSA_NAME_VERSION (use
))))
1935 op_non_const
= add_condition (summary
, base_index
, &aggpos
, CHANGED
, NULL
);
1937 op_non_const
= false_predicate ();
1938 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
1940 tree parm
= unmodified_parm (stmt
, use
);
1944 && (index
= ipa_get_param_decl_index (info
, parm
)) >= 0)
1946 if (index
!= base_index
)
1947 p
= add_condition (summary
, index
, NULL
, CHANGED
, NULL_TREE
);
1952 p
= VEC_index (predicate_t
, nonconstant_names
,
1953 SSA_NAME_VERSION (use
));
1954 op_non_const
= or_predicates (summary
->conds
, &p
, &op_non_const
);
1956 if (gimple_code (stmt
) == GIMPLE_ASSIGN
1957 && TREE_CODE (gimple_assign_lhs (stmt
)) == SSA_NAME
)
1958 VEC_replace (predicate_t
, nonconstant_names
,
1959 SSA_NAME_VERSION (gimple_assign_lhs (stmt
)), op_non_const
);
1960 return op_non_const
;
1963 struct record_modified_bb_info
1969 /* Callback of walk_aliased_vdefs. Records basic blocks where the value may be
1970 set except for info->stmt. */
1973 record_modified (ao_ref
*ao ATTRIBUTE_UNUSED
, tree vdef
,
1976 struct record_modified_bb_info
*info
= (struct record_modified_bb_info
*) data
;
1977 if (SSA_NAME_DEF_STMT (vdef
) == info
->stmt
)
1979 bitmap_set_bit (info
->bb_set
,
1980 SSA_NAME_IS_DEFAULT_DEF (vdef
)
1981 ? ENTRY_BLOCK_PTR
->index
: gimple_bb (SSA_NAME_DEF_STMT (vdef
))->index
);
1985 /* Return probability (based on REG_BR_PROB_BASE) that I-th parameter of STMT
1986 will change since last invocation of STMT.
1988 Value 0 is reserved for compile time invariants.
1989 For common parameters it is REG_BR_PROB_BASE. For loop invariants it
1990 ought to be REG_BR_PROB_BASE / estimated_iters. */
1993 param_change_prob (gimple stmt
, int i
)
1995 tree op
= gimple_call_arg (stmt
, i
);
1996 basic_block bb
= gimple_bb (stmt
);
1999 if (is_gimple_min_invariant (op
))
2001 /* We would have to do non-trivial analysis to really work out what
2002 is the probability of value to change (i.e. when init statement
2003 is in a sibling loop of the call).
2005 We do an conservative estimate: when call is executed N times more often
2006 than the statement defining value, we take the frequency 1/N. */
2007 if (TREE_CODE (op
) == SSA_NAME
)
2012 return REG_BR_PROB_BASE
;
2014 if (SSA_NAME_IS_DEFAULT_DEF (op
))
2015 init_freq
= ENTRY_BLOCK_PTR
->frequency
;
2017 init_freq
= gimple_bb (SSA_NAME_DEF_STMT (op
))->frequency
;
2021 if (init_freq
< bb
->frequency
)
2022 return MAX ((init_freq
* REG_BR_PROB_BASE
+
2023 bb
->frequency
/ 2) / bb
->frequency
, 1);
2025 return REG_BR_PROB_BASE
;
2028 base
= get_base_address (op
);
2033 struct record_modified_bb_info info
;
2037 if (const_value_known_p (base
))
2040 return REG_BR_PROB_BASE
;
2041 ao_ref_init (&refd
, op
);
2043 info
.bb_set
= BITMAP_ALLOC (NULL
);
2044 walk_aliased_vdefs (&refd
, gimple_vuse (stmt
), record_modified
, &info
,
2046 if (bitmap_bit_p (info
.bb_set
, bb
->index
))
2048 BITMAP_FREE (info
.bb_set
);
2049 return REG_BR_PROB_BASE
;
2052 /* Assume that every memory is initialized at entry.
2053 TODO: Can we easilly determine if value is always defined
2054 and thus we may skip entry block? */
2055 if (ENTRY_BLOCK_PTR
->frequency
)
2056 max
= ENTRY_BLOCK_PTR
->frequency
;
2060 EXECUTE_IF_SET_IN_BITMAP (info
.bb_set
, 0, index
, bi
)
2061 max
= MIN (max
, BASIC_BLOCK (index
)->frequency
);
2063 BITMAP_FREE (info
.bb_set
);
2064 if (max
< bb
->frequency
)
2065 return MAX ((max
* REG_BR_PROB_BASE
+
2066 bb
->frequency
/ 2) / bb
->frequency
, 1);
2068 return REG_BR_PROB_BASE
;
2070 return REG_BR_PROB_BASE
;
2073 /* Find whether a basic block BB is the final block of a (half) diamond CFG
2074 sub-graph and if the predicate the condition depends on is known. If so,
2075 return true and store the pointer the predicate in *P. */
2078 phi_result_unknown_predicate (struct ipa_node_params
*info
,
2079 struct inline_summary
*summary
, basic_block bb
,
2080 struct predicate
*p
,
2081 VEC (predicate_t
, heap
) *nonconstant_names
)
2085 basic_block first_bb
= NULL
;
2088 if (single_pred_p (bb
))
2090 *p
= false_predicate ();
2094 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
2096 if (single_succ_p (e
->src
))
2098 if (!single_pred_p (e
->src
))
2101 first_bb
= single_pred (e
->src
);
2102 else if (single_pred (e
->src
) != first_bb
)
2109 else if (e
->src
!= first_bb
)
2117 stmt
= last_stmt (first_bb
);
2119 || gimple_code (stmt
) != GIMPLE_COND
2120 || !is_gimple_ip_invariant (gimple_cond_rhs (stmt
)))
2123 *p
= will_be_nonconstant_expr_predicate (info
, summary
,
2124 gimple_cond_lhs (stmt
),
2126 if (true_predicate_p (p
))
2132 /* Given a PHI statement in a function described by inline properties SUMMARY
2133 and *P being the predicate describing whether the selected PHI argument is
2134 known, store a predicate for the result of the PHI statement into
2135 NONCONSTANT_NAMES, if possible. */
2138 predicate_for_phi_result (struct inline_summary
*summary
, gimple phi
,
2139 struct predicate
*p
,
2140 VEC (predicate_t
, heap
) *nonconstant_names
)
2144 for (i
= 0; i
< gimple_phi_num_args (phi
); i
++)
2146 tree arg
= gimple_phi_arg (phi
, i
)->def
;
2147 if (!is_gimple_min_invariant (arg
))
2149 gcc_assert (TREE_CODE (arg
) == SSA_NAME
);
2150 *p
= or_predicates (summary
->conds
, p
,
2151 &VEC_index (predicate_t
, nonconstant_names
,
2152 SSA_NAME_VERSION (arg
)));
2153 if (true_predicate_p (p
))
2158 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2160 fprintf (dump_file
, "\t\tphi predicate: ");
2161 dump_predicate (dump_file
, summary
->conds
, p
);
2163 VEC_replace (predicate_t
, nonconstant_names
,
2164 SSA_NAME_VERSION (gimple_phi_result (phi
)), *p
);
2167 /* Compute function body size parameters for NODE.
2168 When EARLY is true, we compute only simple summaries without
2169 non-trivial predicates to drive the early inliner. */
2172 estimate_function_body_sizes (struct cgraph_node
*node
, bool early
)
2175 /* Estimate static overhead for function prologue/epilogue and alignment. */
2177 /* Benefits are scaled by probability of elimination that is in range
2180 gimple_stmt_iterator bsi
;
2181 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->symbol
.decl
);
2183 struct inline_summary
*info
= inline_summary (node
);
2184 struct predicate bb_predicate
;
2185 struct ipa_node_params
*parms_info
= NULL
;
2186 VEC (predicate_t
, heap
) *nonconstant_names
= NULL
;
2191 if (optimize
&& !early
)
2193 calculate_dominance_info (CDI_DOMINATORS
);
2194 loop_optimizer_init (LOOPS_NORMAL
| LOOPS_HAVE_RECORDED_EXITS
);
2196 if (ipa_node_params_vector
)
2198 parms_info
= IPA_NODE_REF (node
);
2199 VEC_safe_grow_cleared (predicate_t
, heap
, nonconstant_names
,
2200 VEC_length (tree
, SSANAMES (my_function
)));
2205 fprintf (dump_file
, "\nAnalyzing function body size: %s\n",
2206 cgraph_node_name (node
));
2208 /* When we run into maximal number of entries, we assign everything to the
2209 constant truth case. Be sure to have it in list. */
2210 bb_predicate
= true_predicate ();
2211 account_size_time (info
, 0, 0, &bb_predicate
);
2213 bb_predicate
= not_inlined_predicate ();
2214 account_size_time (info
, 2 * INLINE_SIZE_SCALE
, 0, &bb_predicate
);
2216 gcc_assert (my_function
&& my_function
->cfg
);
2218 compute_bb_predicates (node
, parms_info
, info
);
2219 FOR_EACH_BB_FN (bb
, my_function
)
2221 freq
= compute_call_stmt_bb_frequency (node
->symbol
.decl
, bb
);
2223 /* TODO: Obviously predicates can be propagated down across CFG. */
2227 bb_predicate
= *(struct predicate
*)bb
->aux
;
2229 bb_predicate
= false_predicate ();
2232 bb_predicate
= true_predicate ();
2234 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2236 fprintf (dump_file
, "\n BB %i predicate:", bb
->index
);
2237 dump_predicate (dump_file
, info
->conds
, &bb_predicate
);
2240 if (parms_info
&& nonconstant_names
)
2242 struct predicate phi_predicate
;
2243 bool first_phi
= true;
2245 for (bsi
= gsi_start_phis (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
2248 && !phi_result_unknown_predicate (parms_info
, info
, bb
,
2253 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2255 fprintf (dump_file
, " ");
2256 print_gimple_stmt (dump_file
, gsi_stmt (bsi
), 0, 0);
2258 predicate_for_phi_result (info
, gsi_stmt (bsi
), &phi_predicate
,
2263 for (bsi
= gsi_start_bb (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
2265 gimple stmt
= gsi_stmt (bsi
);
2266 int this_size
= estimate_num_insns (stmt
, &eni_size_weights
);
2267 int this_time
= estimate_num_insns (stmt
, &eni_time_weights
);
2269 struct predicate will_be_nonconstant
;
2271 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2273 fprintf (dump_file
, " ");
2274 print_gimple_stmt (dump_file
, stmt
, 0, 0);
2275 fprintf (dump_file
, "\t\tfreq:%3.2f size:%3i time:%3i\n",
2276 ((double)freq
)/CGRAPH_FREQ_BASE
, this_size
, this_time
);
2279 if (is_gimple_call (stmt
))
2281 struct cgraph_edge
*edge
= cgraph_edge (node
, stmt
);
2282 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
2284 /* Special case: results of BUILT_IN_CONSTANT_P will be always
2285 resolved as constant. We however don't want to optimize
2286 out the cgraph edges. */
2287 if (nonconstant_names
2288 && gimple_call_builtin_p (stmt
, BUILT_IN_CONSTANT_P
)
2289 && gimple_call_lhs (stmt
)
2290 && TREE_CODE (gimple_call_lhs (stmt
)) == SSA_NAME
)
2292 struct predicate false_p
= false_predicate ();
2293 VEC_replace (predicate_t
, nonconstant_names
,
2294 SSA_NAME_VERSION (gimple_call_lhs (stmt
)),
2297 if (ipa_node_params_vector
)
2299 int count
= gimple_call_num_args (stmt
);
2303 VEC_safe_grow_cleared (inline_param_summary_t
, heap
,
2305 for (i
= 0; i
< count
; i
++)
2307 int prob
= param_change_prob (stmt
, i
);
2308 gcc_assert (prob
>= 0 && prob
<= REG_BR_PROB_BASE
);
2309 VEC_index (inline_param_summary_t
,
2310 es
->param
, i
).change_prob
= prob
;
2314 es
->call_stmt_size
= this_size
;
2315 es
->call_stmt_time
= this_time
;
2316 es
->loop_depth
= bb_loop_depth (bb
);
2317 edge_set_predicate (edge
, &bb_predicate
);
2320 /* TODO: When conditional jump or swithc is known to be constant, but
2321 we did not translate it into the predicates, we really can account
2322 just maximum of the possible paths. */
2325 = will_be_nonconstant_predicate (parms_info
, info
,
2326 stmt
, nonconstant_names
);
2327 if (this_time
|| this_size
)
2335 prob
= eliminated_by_inlining_prob (stmt
);
2336 if (prob
== 1 && dump_file
&& (dump_flags
& TDF_DETAILS
))
2337 fprintf (dump_file
, "\t\t50%% will be eliminated by inlining\n");
2338 if (prob
== 2 && dump_file
&& (dump_flags
& TDF_DETAILS
))
2339 fprintf (dump_file
, "\t\tWill be eliminated by inlining\n");
2342 p
= and_predicates (info
->conds
, &bb_predicate
,
2343 &will_be_nonconstant
);
2345 p
= true_predicate ();
2347 /* We account everything but the calls. Calls have their own
2348 size/time info attached to cgraph edges. This is necessary
2349 in order to make the cost disappear after inlining. */
2350 if (!is_gimple_call (stmt
))
2354 struct predicate ip
= not_inlined_predicate ();
2355 ip
= and_predicates (info
->conds
, &ip
, &p
);
2356 account_size_time (info
, this_size
* prob
,
2357 this_time
* prob
, &ip
);
2360 account_size_time (info
, this_size
* (2 - prob
),
2361 this_time
* (2 - prob
), &p
);
2364 gcc_assert (time
>= 0);
2365 gcc_assert (size
>= 0);
2369 FOR_ALL_BB_FN (bb
, my_function
)
2375 pool_free (edge_predicate_pool
, bb
->aux
);
2377 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2380 pool_free (edge_predicate_pool
, e
->aux
);
2384 time
= (time
+ CGRAPH_FREQ_BASE
/ 2) / CGRAPH_FREQ_BASE
;
2385 if (time
> MAX_TIME
)
2388 if (!early
&& nonconstant_names
)
2392 predicate loop_iterations
= true_predicate ();
2394 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2395 flow_loops_dump (dump_file
, NULL
, 0);
2397 FOR_EACH_LOOP (li
, loop
, 0)
2399 VEC (edge
, heap
) *exits
;
2402 struct tree_niter_desc niter_desc
;
2404 exits
= get_loop_exit_edges (loop
);
2405 FOR_EACH_VEC_ELT (edge
, exits
, j
, ex
)
2406 if (number_of_iterations_exit (loop
, ex
, &niter_desc
, false)
2407 && !is_gimple_min_invariant (niter_desc
.niter
))
2409 predicate will_be_nonconstant
2410 = will_be_nonconstant_expr_predicate (parms_info
, info
,
2411 niter_desc
.niter
, nonconstant_names
);
2412 if (!true_predicate_p (&will_be_nonconstant
)
2413 && !false_predicate_p (&will_be_nonconstant
))
2414 /* This is slightly inprecise. We may want to represent each loop with
2415 independent predicate. */
2416 loop_iterations
= and_predicates (info
->conds
, &loop_iterations
, &will_be_nonconstant
);
2418 VEC_free (edge
, heap
, exits
);
2420 if (!true_predicate_p (&loop_iterations
))
2422 inline_summary (node
)->loop_iterations
= (struct predicate
*)pool_alloc (edge_predicate_pool
);
2423 *inline_summary (node
)->loop_iterations
= loop_iterations
;
2427 inline_summary (node
)->self_time
= time
;
2428 inline_summary (node
)->self_size
= size
;
2429 VEC_free (predicate_t
, heap
, nonconstant_names
);
2430 if (optimize
&& !early
)
2432 loop_optimizer_finalize ();
2433 free_dominance_info (CDI_DOMINATORS
);
2437 fprintf (dump_file
, "\n");
2438 dump_inline_summary (dump_file
, node
);
2443 /* Compute parameters of functions used by inliner.
2444 EARLY is true when we compute parameters for the early inliner */
2447 compute_inline_parameters (struct cgraph_node
*node
, bool early
)
2449 HOST_WIDE_INT self_stack_size
;
2450 struct cgraph_edge
*e
;
2451 struct inline_summary
*info
;
2452 tree old_decl
= current_function_decl
;
2454 gcc_assert (!node
->global
.inlined_to
);
2456 inline_summary_alloc ();
2458 info
= inline_summary (node
);
2459 reset_inline_summary (node
);
2461 /* FIXME: Thunks are inlinable, but tree-inline don't know how to do that.
2462 Once this happen, we will need to more curefully predict call
2464 if (node
->thunk
.thunk_p
)
2466 struct inline_edge_summary
*es
= inline_edge_summary (node
->callees
);
2467 struct predicate t
= true_predicate ();
2469 info
->inlinable
= 0;
2470 node
->callees
->call_stmt_cannot_inline_p
= true;
2471 node
->local
.can_change_signature
= false;
2472 es
->call_stmt_time
= 1;
2473 es
->call_stmt_size
= 1;
2474 account_size_time (info
, 0, 0, &t
);
2478 /* Even is_gimple_min_invariant rely on current_function_decl. */
2479 current_function_decl
= node
->symbol
.decl
;
2480 push_cfun (DECL_STRUCT_FUNCTION (node
->symbol
.decl
));
2482 /* Estimate the stack size for the function if we're optimizing. */
2483 self_stack_size
= optimize
? estimated_stack_frame_size (node
) : 0;
2484 info
->estimated_self_stack_size
= self_stack_size
;
2485 info
->estimated_stack_size
= self_stack_size
;
2486 info
->stack_frame_offset
= 0;
2488 /* Can this function be inlined at all? */
2489 info
->inlinable
= tree_inlinable_function_p (node
->symbol
.decl
);
2491 /* Type attributes can use parameter indices to describe them. */
2492 if (TYPE_ATTRIBUTES (TREE_TYPE (node
->symbol
.decl
)))
2493 node
->local
.can_change_signature
= false;
2496 /* Otherwise, inlinable functions always can change signature. */
2497 if (info
->inlinable
)
2498 node
->local
.can_change_signature
= true;
2501 /* Functions calling builtin_apply can not change signature. */
2502 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2504 tree
cdecl = e
->callee
->symbol
.decl
;
2505 if (DECL_BUILT_IN (cdecl)
2506 && DECL_BUILT_IN_CLASS (cdecl) == BUILT_IN_NORMAL
2507 && (DECL_FUNCTION_CODE (cdecl) == BUILT_IN_APPLY_ARGS
2508 || DECL_FUNCTION_CODE (cdecl) == BUILT_IN_VA_START
))
2511 node
->local
.can_change_signature
= !e
;
2514 estimate_function_body_sizes (node
, early
);
2516 /* Inlining characteristics are maintained by the cgraph_mark_inline. */
2517 info
->time
= info
->self_time
;
2518 info
->size
= info
->self_size
;
2519 info
->stack_frame_offset
= 0;
2520 info
->estimated_stack_size
= info
->estimated_self_stack_size
;
2521 current_function_decl
= old_decl
;
2526 /* Compute parameters of functions used by inliner using
2527 current_function_decl. */
2530 compute_inline_parameters_for_current (void)
2532 compute_inline_parameters (cgraph_get_node (current_function_decl
), true);
2536 struct gimple_opt_pass pass_inline_parameters
=
2540 "inline_param", /* name */
2542 compute_inline_parameters_for_current
,/* execute */
2545 0, /* static_pass_number */
2546 TV_INLINE_PARAMETERS
, /* tv_id */
2547 0, /* properties_required */
2548 0, /* properties_provided */
2549 0, /* properties_destroyed */
2550 0, /* todo_flags_start */
2551 0 /* todo_flags_finish */
2556 /* Increase SIZE and TIME for size and time needed to handle edge E. */
2559 estimate_edge_size_and_time (struct cgraph_edge
*e
, int *size
, int *time
,
2562 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2563 *size
+= es
->call_stmt_size
* INLINE_SIZE_SCALE
;
2564 *time
+= (es
->call_stmt_time
* prob
/ REG_BR_PROB_BASE
2565 * e
->frequency
* (INLINE_TIME_SCALE
/ CGRAPH_FREQ_BASE
));
2566 if (*time
> MAX_TIME
* INLINE_TIME_SCALE
)
2567 *time
= MAX_TIME
* INLINE_TIME_SCALE
;
2571 /* Estimate benefit devirtualizing indirect edge IE, provided KNOWN_VALS and
2575 estimate_edge_devirt_benefit (struct cgraph_edge
*ie
,
2576 int *size
, int *time
, int prob
,
2577 VEC (tree
, heap
) *known_vals
,
2578 VEC (tree
, heap
) *known_binfos
,
2579 VEC (ipa_agg_jump_function_p
, heap
) *known_aggs
)
2582 int time_diff
, size_diff
;
2583 struct cgraph_node
*callee
;
2584 struct inline_summary
*isummary
;
2586 if (!known_vals
&& !known_binfos
)
2589 target
= ipa_get_indirect_edge_target (ie
, known_vals
, known_binfos
,
2594 /* Account for difference in cost between indirect and direct calls. */
2595 size_diff
= ((eni_size_weights
.indirect_call_cost
- eni_size_weights
.call_cost
)
2596 * INLINE_SIZE_SCALE
);
2598 time_diff
= ((eni_time_weights
.indirect_call_cost
- eni_time_weights
.call_cost
)
2599 * INLINE_TIME_SCALE
* prob
/ REG_BR_PROB_BASE
);
2602 callee
= cgraph_get_node (target
);
2603 if (!callee
|| !callee
->analyzed
)
2605 isummary
= inline_summary (callee
);
2606 return isummary
->inlinable
;
2610 /* Increase SIZE and TIME for size and time needed to handle all calls in NODE.
2611 POSSIBLE_TRUTHS, KNOWN_VALS and KNOWN_BINFOS describe context of the call
2615 estimate_calls_size_and_time (struct cgraph_node
*node
, int *size
, int *time
,
2616 inline_hints
*hints
,
2617 clause_t possible_truths
,
2618 VEC (tree
, heap
) *known_vals
,
2619 VEC (tree
, heap
) *known_binfos
,
2620 VEC (ipa_agg_jump_function_p
, heap
) *known_aggs
)
2622 struct cgraph_edge
*e
;
2623 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2625 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2626 if (!es
->predicate
|| evaluate_predicate (es
->predicate
, possible_truths
))
2628 if (e
->inline_failed
)
2630 /* Predicates of calls shall not use NOT_CHANGED codes,
2631 sowe do not need to compute probabilities. */
2632 estimate_edge_size_and_time (e
, size
, time
, REG_BR_PROB_BASE
);
2635 estimate_calls_size_and_time (e
->callee
, size
, time
, hints
,
2637 known_vals
, known_binfos
, known_aggs
);
2640 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
2642 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2643 if (!es
->predicate
|| evaluate_predicate (es
->predicate
, possible_truths
))
2645 estimate_edge_size_and_time (e
, size
, time
, REG_BR_PROB_BASE
);
2646 if (estimate_edge_devirt_benefit (e
, size
, time
, REG_BR_PROB_BASE
,
2647 known_vals
, known_binfos
, known_aggs
)
2649 && cgraph_maybe_hot_edge_p (e
))
2650 *hints
|= INLINE_HINT_indirect_call
;
2656 /* Estimate size and time needed to execute NODE assuming
2657 POSSIBLE_TRUTHS clause, and KNOWN_VALS and KNOWN_BINFOS information
2658 about NODE's arguments. */
2661 estimate_node_size_and_time (struct cgraph_node
*node
,
2662 clause_t possible_truths
,
2663 VEC (tree
, heap
) *known_vals
,
2664 VEC (tree
, heap
) *known_binfos
,
2665 VEC (ipa_agg_jump_function_p
, heap
) *known_aggs
,
2666 int *ret_size
, int *ret_time
,
2667 inline_hints
*ret_hints
,
2668 VEC (inline_param_summary_t
, heap
)
2669 *inline_param_summary
)
2671 struct inline_summary
*info
= inline_summary (node
);
2673 int size
= 0, time
= 0;
2674 inline_hints hints
= 0;
2678 && (dump_flags
& TDF_DETAILS
))
2681 fprintf (dump_file
, " Estimating body: %s/%i\n"
2682 " Known to be false: ",
2683 cgraph_node_name (node
),
2686 for (i
= predicate_not_inlined_condition
;
2687 i
< (predicate_first_dynamic_condition
2688 + (int)VEC_length (condition
, info
->conds
)); i
++)
2689 if (!(possible_truths
& (1 << i
)))
2692 fprintf (dump_file
, ", ");
2694 dump_condition (dump_file
, info
->conds
, i
);
2698 for (i
= 0; VEC_iterate (size_time_entry
, info
->entry
, i
, e
); i
++)
2699 if (evaluate_predicate (&e
->predicate
, possible_truths
))
2702 if (!inline_param_summary
)
2706 int prob
= predicate_probability (info
->conds
,
2709 inline_param_summary
);
2710 time
+= e
->time
* prob
/ REG_BR_PROB_BASE
;
2715 if (info
->loop_iterations
2716 && !evaluate_predicate (info
->loop_iterations
, possible_truths
))
2717 hints
|=INLINE_HINT_loop_iterations
;
2719 if (time
> MAX_TIME
* INLINE_TIME_SCALE
)
2720 time
= MAX_TIME
* INLINE_TIME_SCALE
;
2722 estimate_calls_size_and_time (node
, &size
, &time
, &hints
, possible_truths
,
2723 known_vals
, known_binfos
, known_aggs
);
2724 time
= (time
+ INLINE_TIME_SCALE
/ 2) / INLINE_TIME_SCALE
;
2725 size
= (size
+ INLINE_SIZE_SCALE
/ 2) / INLINE_SIZE_SCALE
;
2729 && (dump_flags
& TDF_DETAILS
))
2730 fprintf (dump_file
, "\n size:%i time:%i\n", size
, time
);
2741 /* Estimate size and time needed to execute callee of EDGE assuming that
2742 parameters known to be constant at caller of EDGE are propagated.
2743 KNOWN_VALS and KNOWN_BINFOS are vectors of assumed known constant values
2744 and types for parameters. */
2747 estimate_ipcp_clone_size_and_time (struct cgraph_node
*node
,
2748 VEC (tree
, heap
) *known_vals
,
2749 VEC (tree
, heap
) *known_binfos
,
2750 int *ret_size
, int *ret_time
)
2754 clause
= evaluate_conditions_for_known_args (node
, false, known_vals
, NULL
);
2755 estimate_node_size_and_time (node
, clause
, known_vals
, known_binfos
, NULL
,
2756 ret_size
, ret_time
, NULL
,
2760 /* Translate all conditions from callee representation into caller
2761 representation and symbolically evaluate predicate P into new predicate.
2763 INFO is inline_summary of function we are adding predicate into, CALLEE_INFO
2764 is summary of function predicate P is from. OPERAND_MAP is array giving
2765 callee formal IDs the caller formal IDs. POSSSIBLE_TRUTHS is clausule of all
2766 callee conditions that may be true in caller context. TOPLEV_PREDICATE is
2767 predicate under which callee is executed. OFFSET_MAP is an array of of
2768 offsets that need to be added to conditions, negative offset means that
2769 conditions relying on values passed by reference have to be discarded
2770 because they might not be preserved (and should be considered offset zero
2771 for other purposes). */
2773 static struct predicate
2774 remap_predicate (struct inline_summary
*info
,
2775 struct inline_summary
*callee_info
,
2776 struct predicate
*p
,
2777 VEC (int, heap
) *operand_map
,
2778 VEC (int, heap
) *offset_map
,
2779 clause_t possible_truths
,
2780 struct predicate
*toplev_predicate
)
2783 struct predicate out
= true_predicate ();
2785 /* True predicate is easy. */
2786 if (true_predicate_p (p
))
2787 return *toplev_predicate
;
2788 for (i
= 0; p
->clause
[i
]; i
++)
2790 clause_t clause
= p
->clause
[i
];
2792 struct predicate clause_predicate
= false_predicate ();
2794 gcc_assert (i
< MAX_CLAUSES
);
2796 for (cond
= 0; cond
< NUM_CONDITIONS
; cond
++)
2797 /* Do we have condition we can't disprove? */
2798 if (clause
& possible_truths
& (1 << cond
))
2800 struct predicate cond_predicate
;
2801 /* Work out if the condition can translate to predicate in the
2802 inlined function. */
2803 if (cond
>= predicate_first_dynamic_condition
)
2805 struct condition
*c
;
2807 c
= &VEC_index (condition
, callee_info
->conds
,
2808 cond
- predicate_first_dynamic_condition
);
2809 /* See if we can remap condition operand to caller's operand.
2810 Otherwise give up. */
2812 || (int)VEC_length (int, operand_map
) <= c
->operand_num
2813 || VEC_index (int, operand_map
, c
->operand_num
) == -1
2814 /* TODO: For non-aggregate conditions, adding an offset is
2815 basically an arithmetic jump function processing which
2816 we should support in future. */
2817 || ((!c
->agg_contents
|| !c
->by_ref
)
2818 && VEC_index (int, offset_map
, c
->operand_num
) > 0)
2819 || (c
->agg_contents
&& c
->by_ref
2820 && VEC_index (int, offset_map
, c
->operand_num
) < 0))
2821 cond_predicate
= true_predicate ();
2824 struct agg_position_info ap
;
2825 HOST_WIDE_INT offset_delta
= VEC_index (int, offset_map
,
2827 if (offset_delta
< 0)
2829 gcc_checking_assert (!c
->agg_contents
|| !c
->by_ref
);
2832 gcc_assert (!c
->agg_contents
2834 || offset_delta
== 0);
2835 ap
.offset
= c
->offset
+ offset_delta
;
2836 ap
.agg_contents
= c
->agg_contents
;
2837 ap
.by_ref
= c
->by_ref
;
2838 cond_predicate
= add_condition (info
,
2842 &ap
, c
->code
, c
->val
);
2845 /* Fixed conditions remains same, construct single
2846 condition predicate. */
2849 cond_predicate
.clause
[0] = 1 << cond
;
2850 cond_predicate
.clause
[1] = 0;
2852 clause_predicate
= or_predicates (info
->conds
, &clause_predicate
,
2855 out
= and_predicates (info
->conds
, &out
, &clause_predicate
);
2857 return and_predicates (info
->conds
, &out
, toplev_predicate
);
2861 /* Update summary information of inline clones after inlining.
2862 Compute peak stack usage. */
2865 inline_update_callee_summaries (struct cgraph_node
*node
,
2868 struct cgraph_edge
*e
;
2869 struct inline_summary
*callee_info
= inline_summary (node
);
2870 struct inline_summary
*caller_info
= inline_summary (node
->callers
->caller
);
2873 callee_info
->stack_frame_offset
2874 = caller_info
->stack_frame_offset
2875 + caller_info
->estimated_self_stack_size
;
2876 peak
= callee_info
->stack_frame_offset
2877 + callee_info
->estimated_self_stack_size
;
2878 if (inline_summary (node
->global
.inlined_to
)->estimated_stack_size
2880 inline_summary (node
->global
.inlined_to
)->estimated_stack_size
= peak
;
2881 cgraph_propagate_frequency (node
);
2882 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2884 if (!e
->inline_failed
)
2885 inline_update_callee_summaries (e
->callee
, depth
);
2886 inline_edge_summary (e
)->loop_depth
+= depth
;
2888 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
2889 inline_edge_summary (e
)->loop_depth
+= depth
;
2892 /* Update change_prob of EDGE after INLINED_EDGE has been inlined.
2893 When functoin A is inlined in B and A calls C with parameter that
2894 changes with probability PROB1 and C is known to be passthroug
2895 of argument if B that change with probability PROB2, the probability
2896 of change is now PROB1*PROB2. */
2899 remap_edge_change_prob (struct cgraph_edge
*inlined_edge
,
2900 struct cgraph_edge
*edge
)
2902 if (ipa_node_params_vector
)
2905 struct ipa_edge_args
*args
= IPA_EDGE_REF (edge
);
2906 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
2907 struct inline_edge_summary
*inlined_es
2908 = inline_edge_summary (inlined_edge
);
2910 for (i
= 0; i
< ipa_get_cs_argument_count (args
); i
++)
2912 struct ipa_jump_func
*jfunc
= ipa_get_ith_jump_func (args
, i
);
2913 if (jfunc
->type
== IPA_JF_PASS_THROUGH
2914 && (ipa_get_jf_pass_through_formal_id (jfunc
)
2915 < (int) VEC_length (inline_param_summary_t
,
2916 inlined_es
->param
)))
2918 int jf_formal_id
= ipa_get_jf_pass_through_formal_id (jfunc
);
2919 int prob1
= VEC_index (inline_param_summary_t
,
2920 es
->param
, i
).change_prob
;
2921 int prob2
= VEC_index
2922 (inline_param_summary_t
,
2923 inlined_es
->param
, jf_formal_id
).change_prob
;
2924 int prob
= ((prob1
* prob2
+ REG_BR_PROB_BASE
/ 2)
2925 / REG_BR_PROB_BASE
);
2927 if (prob1
&& prob2
&& !prob
)
2930 VEC_index (inline_param_summary_t
,
2931 es
->param
, i
).change_prob
= prob
;
2937 /* Update edge summaries of NODE after INLINED_EDGE has been inlined.
2939 Remap predicates of callees of NODE. Rest of arguments match
2942 Also update change probabilities. */
2945 remap_edge_summaries (struct cgraph_edge
*inlined_edge
,
2946 struct cgraph_node
*node
,
2947 struct inline_summary
*info
,
2948 struct inline_summary
*callee_info
,
2949 VEC (int, heap
) *operand_map
,
2950 VEC (int, heap
) *offset_map
,
2951 clause_t possible_truths
,
2952 struct predicate
*toplev_predicate
)
2954 struct cgraph_edge
*e
;
2955 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2957 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2960 if (e
->inline_failed
)
2962 remap_edge_change_prob (inlined_edge
, e
);
2966 p
= remap_predicate (info
, callee_info
,
2967 es
->predicate
, operand_map
, offset_map
,
2970 edge_set_predicate (e
, &p
);
2971 /* TODO: We should remove the edge for code that will be
2972 optimized out, but we need to keep verifiers and tree-inline
2973 happy. Make it cold for now. */
2974 if (false_predicate_p (&p
))
2981 edge_set_predicate (e
, toplev_predicate
);
2984 remap_edge_summaries (inlined_edge
, e
->callee
, info
, callee_info
,
2985 operand_map
, offset_map
, possible_truths
,
2988 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
2990 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2993 remap_edge_change_prob (inlined_edge
, e
);
2996 p
= remap_predicate (info
, callee_info
,
2997 es
->predicate
, operand_map
, offset_map
,
2998 possible_truths
, toplev_predicate
);
2999 edge_set_predicate (e
, &p
);
3000 /* TODO: We should remove the edge for code that will be optimized
3001 out, but we need to keep verifiers and tree-inline happy.
3002 Make it cold for now. */
3003 if (false_predicate_p (&p
))
3010 edge_set_predicate (e
, toplev_predicate
);
3015 /* We inlined EDGE. Update summary of the function we inlined into. */
3018 inline_merge_summary (struct cgraph_edge
*edge
)
3020 struct inline_summary
*callee_info
= inline_summary (edge
->callee
);
3021 struct cgraph_node
*to
= (edge
->caller
->global
.inlined_to
3022 ? edge
->caller
->global
.inlined_to
: edge
->caller
);
3023 struct inline_summary
*info
= inline_summary (to
);
3024 clause_t clause
= 0; /* not_inline is known to be false. */
3026 VEC (int, heap
) *operand_map
= NULL
;
3027 VEC (int, heap
) *offset_map
= NULL
;
3029 struct predicate toplev_predicate
;
3030 struct predicate true_p
= true_predicate ();
3031 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3034 toplev_predicate
= *es
->predicate
;
3036 toplev_predicate
= true_predicate ();
3038 if (ipa_node_params_vector
&& callee_info
->conds
)
3040 struct ipa_edge_args
*args
= IPA_EDGE_REF (edge
);
3041 int count
= ipa_get_cs_argument_count (args
);
3044 evaluate_properties_for_edge (edge
, true, &clause
, NULL
, NULL
, NULL
);
3047 VEC_safe_grow_cleared (int, heap
, operand_map
, count
);
3048 VEC_safe_grow_cleared (int, heap
, offset_map
, count
);
3050 for (i
= 0; i
< count
; i
++)
3052 struct ipa_jump_func
*jfunc
= ipa_get_ith_jump_func (args
, i
);
3055 /* TODO: handle non-NOPs when merging. */
3056 if (jfunc
->type
== IPA_JF_PASS_THROUGH
)
3058 if (ipa_get_jf_pass_through_operation (jfunc
) == NOP_EXPR
)
3059 map
= ipa_get_jf_pass_through_formal_id (jfunc
);
3060 if (!ipa_get_jf_pass_through_agg_preserved (jfunc
))
3061 VEC_replace (int, offset_map
, i
, -1);
3063 else if (jfunc
->type
== IPA_JF_ANCESTOR
)
3065 HOST_WIDE_INT offset
= ipa_get_jf_ancestor_offset (jfunc
);
3066 if (offset
>= 0 && offset
< INT_MAX
)
3068 map
= ipa_get_jf_ancestor_formal_id (jfunc
);
3069 if (!ipa_get_jf_ancestor_agg_preserved (jfunc
))
3071 VEC_replace (int, offset_map
, i
, offset
);
3074 VEC_replace (int, operand_map
, i
, map
);
3075 gcc_assert (map
< ipa_get_param_count (IPA_NODE_REF (to
)));
3078 for (i
= 0; VEC_iterate (size_time_entry
, callee_info
->entry
, i
, e
); i
++)
3080 struct predicate p
= remap_predicate (info
, callee_info
,
3081 &e
->predicate
, operand_map
,
3084 if (!false_predicate_p (&p
))
3086 gcov_type add_time
= ((gcov_type
)e
->time
* edge
->frequency
3087 + CGRAPH_FREQ_BASE
/ 2) / CGRAPH_FREQ_BASE
;
3088 int prob
= predicate_probability (callee_info
->conds
,
3091 add_time
= add_time
* prob
/ REG_BR_PROB_BASE
;
3092 if (add_time
> MAX_TIME
* INLINE_TIME_SCALE
)
3093 add_time
= MAX_TIME
* INLINE_TIME_SCALE
;
3094 if (prob
!= REG_BR_PROB_BASE
3095 && dump_file
&& (dump_flags
& TDF_DETAILS
))
3097 fprintf (dump_file
, "\t\tScaling time by probability:%f\n",
3098 (double)prob
/ REG_BR_PROB_BASE
);
3100 account_size_time (info
, e
->size
, add_time
, &p
);
3103 remap_edge_summaries (edge
, edge
->callee
, info
, callee_info
, operand_map
,
3104 offset_map
, clause
, &toplev_predicate
);
3105 if (callee_info
->loop_iterations
)
3107 predicate p
= remap_predicate (info
, callee_info
,
3108 callee_info
->loop_iterations
,
3109 operand_map
, offset_map
,
3112 if (!false_predicate_p (&p
)
3113 && !true_predicate_p (&p
))
3115 if (!info
->loop_iterations
)
3117 info
->loop_iterations
3118 = (struct predicate
*)pool_alloc (edge_predicate_pool
);
3119 *info
->loop_iterations
= p
;
3122 *info
->loop_iterations
= and_predicates (info
->conds
,
3123 info
->loop_iterations
,
3128 inline_update_callee_summaries (edge
->callee
,
3129 inline_edge_summary (edge
)->loop_depth
);
3131 /* We do not maintain predicates of inlined edges, free it. */
3132 edge_set_predicate (edge
, &true_p
);
3133 /* Similarly remove param summaries. */
3134 VEC_free (inline_param_summary_t
, heap
, es
->param
);
3135 VEC_free (int, heap
, operand_map
);
3136 VEC_free (int, heap
, offset_map
);
3139 /* For performance reasons inline_merge_summary is not updating overall size
3140 and time. Recompute it. */
3143 inline_update_overall_summary (struct cgraph_node
*node
)
3145 struct inline_summary
*info
= inline_summary (node
);
3151 for (i
= 0; VEC_iterate (size_time_entry
, info
->entry
, i
, e
); i
++)
3152 info
->size
+= e
->size
, info
->time
+= e
->time
;
3153 estimate_calls_size_and_time (node
, &info
->size
, &info
->time
, NULL
,
3154 ~(clause_t
)(1 << predicate_false_condition
),
3156 info
->time
= (info
->time
+ INLINE_TIME_SCALE
/ 2) / INLINE_TIME_SCALE
;
3157 info
->size
= (info
->size
+ INLINE_SIZE_SCALE
/ 2) / INLINE_SIZE_SCALE
;
3160 /* Estimate the time cost for the caller when inlining EDGE.
3161 Only to be called via estimate_edge_time, that handles the
3164 When caching, also update the cache entry. Compute both time and
3165 size, since we always need both metrics eventually. */
3168 do_estimate_edge_time (struct cgraph_edge
*edge
)
3174 struct cgraph_node
*callee
;
3176 VEC (tree
, heap
) *known_vals
;
3177 VEC (tree
, heap
) *known_binfos
;
3178 VEC (ipa_agg_jump_function_p
, heap
) *known_aggs
;
3179 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3181 callee
= cgraph_function_or_thunk_node (edge
->callee
, NULL
);
3183 gcc_checking_assert (edge
->inline_failed
);
3184 evaluate_properties_for_edge (edge
, true,
3185 &clause
, &known_vals
, &known_binfos
,
3187 estimate_node_size_and_time (callee
, clause
, known_vals
, known_binfos
,
3188 known_aggs
, &size
, &time
, &hints
, es
->param
);
3189 VEC_free (tree
, heap
, known_vals
);
3190 VEC_free (tree
, heap
, known_binfos
);
3191 VEC_free (ipa_agg_jump_function_p
, heap
, known_aggs
);
3193 ret
= (((gcov_type
)time
3194 - es
->call_stmt_time
) * edge
->frequency
3195 + CGRAPH_FREQ_BASE
/ 2) / CGRAPH_FREQ_BASE
;
3197 /* When caching, update the cache entry. */
3198 if (edge_growth_cache
)
3201 if ((int)VEC_length (edge_growth_cache_entry
, edge_growth_cache
)
3203 VEC_safe_grow_cleared (edge_growth_cache_entry
, heap
, edge_growth_cache
,
3204 cgraph_edge_max_uid
);
3205 VEC_index (edge_growth_cache_entry
, edge_growth_cache
, edge
->uid
).time
3208 ret_size
= size
- es
->call_stmt_size
;
3209 gcc_checking_assert (es
->call_stmt_size
);
3210 VEC_index (edge_growth_cache_entry
, edge_growth_cache
, edge
->uid
).size
3211 = ret_size
+ (ret_size
>= 0);
3212 VEC_index (edge_growth_cache_entry
, edge_growth_cache
, edge
->uid
).hints
3219 /* Estimate the growth of the caller when inlining EDGE.
3220 Only to be called via estimate_edge_size. */
3223 do_estimate_edge_growth (struct cgraph_edge
*edge
)
3226 struct cgraph_node
*callee
;
3228 VEC (tree
, heap
) *known_vals
;
3229 VEC (tree
, heap
) *known_binfos
;
3230 VEC (ipa_agg_jump_function_p
, heap
) *known_aggs
;
3232 /* When we do caching, use do_estimate_edge_time to populate the entry. */
3234 if (edge_growth_cache
)
3236 do_estimate_edge_time (edge
);
3237 size
= VEC_index (edge_growth_cache_entry
,
3240 gcc_checking_assert (size
);
3241 return size
- (size
> 0);
3244 callee
= cgraph_function_or_thunk_node (edge
->callee
, NULL
);
3246 /* Early inliner runs without caching, go ahead and do the dirty work. */
3247 gcc_checking_assert (edge
->inline_failed
);
3248 evaluate_properties_for_edge (edge
, true,
3249 &clause
, &known_vals
, &known_binfos
,
3251 estimate_node_size_and_time (callee
, clause
, known_vals
, known_binfos
,
3252 known_aggs
, &size
, NULL
, NULL
, NULL
);
3253 VEC_free (tree
, heap
, known_vals
);
3254 VEC_free (tree
, heap
, known_binfos
);
3255 VEC_free (ipa_agg_jump_function_p
, heap
, known_aggs
);
3256 gcc_checking_assert (inline_edge_summary (edge
)->call_stmt_size
);
3257 return size
- inline_edge_summary (edge
)->call_stmt_size
;
3261 /* Estimate the growth of the caller when inlining EDGE.
3262 Only to be called via estimate_edge_size. */
3265 do_estimate_edge_hints (struct cgraph_edge
*edge
)
3268 struct cgraph_node
*callee
;
3270 VEC (tree
, heap
) *known_vals
;
3271 VEC (tree
, heap
) *known_binfos
;
3272 VEC (ipa_agg_jump_function_p
, heap
) *known_aggs
;
3274 /* When we do caching, use do_estimate_edge_time to populate the entry. */
3276 if (edge_growth_cache
)
3278 do_estimate_edge_time (edge
);
3279 hints
= VEC_index (edge_growth_cache_entry
,
3282 gcc_checking_assert (hints
);
3286 callee
= cgraph_function_or_thunk_node (edge
->callee
, NULL
);
3288 /* Early inliner runs without caching, go ahead and do the dirty work. */
3289 gcc_checking_assert (edge
->inline_failed
);
3290 evaluate_properties_for_edge (edge
, true,
3291 &clause
, &known_vals
, &known_binfos
,
3293 estimate_node_size_and_time (callee
, clause
, known_vals
, known_binfos
,
3294 known_aggs
, NULL
, NULL
, &hints
, NULL
);
3295 VEC_free (tree
, heap
, known_vals
);
3296 VEC_free (tree
, heap
, known_binfos
);
3297 VEC_free (ipa_agg_jump_function_p
, heap
, known_aggs
);
3302 /* Estimate self time of the function NODE after inlining EDGE. */
3305 estimate_time_after_inlining (struct cgraph_node
*node
,
3306 struct cgraph_edge
*edge
)
3308 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3309 if (!es
->predicate
|| !false_predicate_p (es
->predicate
))
3311 gcov_type time
= inline_summary (node
)->time
+ estimate_edge_time (edge
);
3314 if (time
> MAX_TIME
)
3318 return inline_summary (node
)->time
;
3322 /* Estimate the size of NODE after inlining EDGE which should be an
3323 edge to either NODE or a call inlined into NODE. */
3326 estimate_size_after_inlining (struct cgraph_node
*node
,
3327 struct cgraph_edge
*edge
)
3329 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3330 if (!es
->predicate
|| !false_predicate_p (es
->predicate
))
3332 int size
= inline_summary (node
)->size
+ estimate_edge_growth (edge
);
3333 gcc_assert (size
>= 0);
3336 return inline_summary (node
)->size
;
3342 bool self_recursive
;
3347 /* Worker for do_estimate_growth. Collect growth for all callers. */
3350 do_estimate_growth_1 (struct cgraph_node
*node
, void *data
)
3352 struct cgraph_edge
*e
;
3353 struct growth_data
*d
= (struct growth_data
*) data
;
3355 for (e
= node
->callers
; e
; e
= e
->next_caller
)
3357 gcc_checking_assert (e
->inline_failed
);
3359 if (e
->caller
== node
3360 || (e
->caller
->global
.inlined_to
3361 && e
->caller
->global
.inlined_to
== node
))
3362 d
->self_recursive
= true;
3363 d
->growth
+= estimate_edge_growth (e
);
3369 /* Estimate the growth caused by inlining NODE into all callees. */
3372 do_estimate_growth (struct cgraph_node
*node
)
3374 struct growth_data d
= {0, false};
3375 struct inline_summary
*info
= inline_summary (node
);
3377 cgraph_for_node_and_aliases (node
, do_estimate_growth_1
, &d
, true);
3379 /* For self recursive functions the growth estimation really should be
3380 infinity. We don't want to return very large values because the growth
3381 plays various roles in badness computation fractions. Be sure to not
3382 return zero or negative growths. */
3383 if (d
.self_recursive
)
3384 d
.growth
= d
.growth
< info
->size
? info
->size
: d
.growth
;
3387 if (!DECL_EXTERNAL (node
->symbol
.decl
)
3388 && cgraph_will_be_removed_from_program_if_no_direct_calls (node
))
3389 d
.growth
-= info
->size
;
3390 /* COMDAT functions are very often not shared across multiple units
3391 since they come from various template instantiations.
3392 Take this into account. */
3393 else if (DECL_COMDAT (node
->symbol
.decl
)
3394 && cgraph_can_remove_if_no_direct_calls_p (node
))
3395 d
.growth
-= (info
->size
3396 * (100 - PARAM_VALUE (PARAM_COMDAT_SHARING_PROBABILITY
))
3400 if (node_growth_cache
)
3402 if ((int)VEC_length (int, node_growth_cache
) <= node
->uid
)
3403 VEC_safe_grow_cleared (int, heap
, node_growth_cache
, cgraph_max_uid
);
3404 VEC_replace (int, node_growth_cache
, node
->uid
,
3405 d
.growth
+ (d
.growth
>= 0));
3411 /* This function performs intraprocedural analysis in NODE that is required to
3412 inline indirect calls. */
3415 inline_indirect_intraprocedural_analysis (struct cgraph_node
*node
)
3417 ipa_analyze_node (node
);
3418 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
3420 ipa_print_node_params (dump_file
, node
);
3421 ipa_print_node_jump_functions (dump_file
, node
);
3426 /* Note function body size. */
3429 inline_analyze_function (struct cgraph_node
*node
)
3431 push_cfun (DECL_STRUCT_FUNCTION (node
->symbol
.decl
));
3432 current_function_decl
= node
->symbol
.decl
;
3435 fprintf (dump_file
, "\nAnalyzing function: %s/%u\n",
3436 cgraph_node_name (node
), node
->uid
);
3437 if (optimize
&& !node
->thunk
.thunk_p
)
3438 inline_indirect_intraprocedural_analysis (node
);
3439 compute_inline_parameters (node
, false);
3441 current_function_decl
= NULL
;
3446 /* Called when new function is inserted to callgraph late. */
3449 add_new_function (struct cgraph_node
*node
, void *data ATTRIBUTE_UNUSED
)
3451 inline_analyze_function (node
);
3455 /* Note function body size. */
3458 inline_generate_summary (void)
3460 struct cgraph_node
*node
;
3462 function_insertion_hook_holder
=
3463 cgraph_add_function_insertion_hook (&add_new_function
, NULL
);
3465 ipa_register_cgraph_hooks ();
3466 inline_free_summary ();
3468 FOR_EACH_DEFINED_FUNCTION (node
)
3470 inline_analyze_function (node
);
3474 /* Read predicate from IB. */
3476 static struct predicate
3477 read_predicate (struct lto_input_block
*ib
)
3479 struct predicate out
;
3485 gcc_assert (k
<= MAX_CLAUSES
);
3486 clause
= out
.clause
[k
++] = streamer_read_uhwi (ib
);
3490 /* Zero-initialize the remaining clauses in OUT. */
3491 while (k
<= MAX_CLAUSES
)
3492 out
.clause
[k
++] = 0;
3498 /* Write inline summary for edge E to OB. */
3501 read_inline_edge_summary (struct lto_input_block
*ib
, struct cgraph_edge
*e
)
3503 struct inline_edge_summary
*es
= inline_edge_summary (e
);
3507 es
->call_stmt_size
= streamer_read_uhwi (ib
);
3508 es
->call_stmt_time
= streamer_read_uhwi (ib
);
3509 es
->loop_depth
= streamer_read_uhwi (ib
);
3510 p
= read_predicate (ib
);
3511 edge_set_predicate (e
, &p
);
3512 length
= streamer_read_uhwi (ib
);
3515 VEC_safe_grow_cleared (inline_param_summary_t
, heap
, es
->param
, length
);
3516 for (i
= 0; i
< length
; i
++)
3517 VEC_index (inline_param_summary_t
, es
->param
, i
).change_prob
3518 = streamer_read_uhwi (ib
);
3523 /* Stream in inline summaries from the section. */
3526 inline_read_section (struct lto_file_decl_data
*file_data
, const char *data
,
3529 const struct lto_function_header
*header
=
3530 (const struct lto_function_header
*) data
;
3531 const int cfg_offset
= sizeof (struct lto_function_header
);
3532 const int main_offset
= cfg_offset
+ header
->cfg_size
;
3533 const int string_offset
= main_offset
+ header
->main_size
;
3534 struct data_in
*data_in
;
3535 struct lto_input_block ib
;
3536 unsigned int i
, count2
, j
;
3537 unsigned int f_count
;
3539 LTO_INIT_INPUT_BLOCK (ib
, (const char *) data
+ main_offset
, 0,
3543 lto_data_in_create (file_data
, (const char *) data
+ string_offset
,
3544 header
->string_size
, NULL
);
3545 f_count
= streamer_read_uhwi (&ib
);
3546 for (i
= 0; i
< f_count
; i
++)
3549 struct cgraph_node
*node
;
3550 struct inline_summary
*info
;
3551 lto_symtab_encoder_t encoder
;
3552 struct bitpack_d bp
;
3553 struct cgraph_edge
*e
;
3556 index
= streamer_read_uhwi (&ib
);
3557 encoder
= file_data
->symtab_node_encoder
;
3558 node
= cgraph (lto_symtab_encoder_deref (encoder
, index
));
3559 info
= inline_summary (node
);
3561 info
->estimated_stack_size
3562 = info
->estimated_self_stack_size
= streamer_read_uhwi (&ib
);
3563 info
->size
= info
->self_size
= streamer_read_uhwi (&ib
);
3564 info
->time
= info
->self_time
= streamer_read_uhwi (&ib
);
3566 bp
= streamer_read_bitpack (&ib
);
3567 info
->inlinable
= bp_unpack_value (&bp
, 1);
3569 count2
= streamer_read_uhwi (&ib
);
3570 gcc_assert (!info
->conds
);
3571 for (j
= 0; j
< count2
; j
++)
3574 c
.operand_num
= streamer_read_uhwi (&ib
);
3575 c
.code
= (enum tree_code
) streamer_read_uhwi (&ib
);
3576 c
.val
= stream_read_tree (&ib
, data_in
);
3577 bp
= streamer_read_bitpack (&ib
);
3578 c
.agg_contents
= bp_unpack_value (&bp
, 1);
3579 c
.by_ref
= bp_unpack_value (&bp
, 1);
3581 c
.offset
= streamer_read_uhwi (&ib
);
3582 VEC_safe_push (condition
, gc
, info
->conds
, c
);
3584 count2
= streamer_read_uhwi (&ib
);
3585 gcc_assert (!info
->entry
);
3586 for (j
= 0; j
< count2
; j
++)
3588 struct size_time_entry e
;
3590 e
.size
= streamer_read_uhwi (&ib
);
3591 e
.time
= streamer_read_uhwi (&ib
);
3592 e
.predicate
= read_predicate (&ib
);
3594 VEC_safe_push (size_time_entry
, gc
, info
->entry
, e
);
3597 p
= read_predicate (&ib
);
3598 if (!true_predicate_p (&p
))
3600 info
->loop_iterations
= (struct predicate
*)pool_alloc (edge_predicate_pool
);
3601 *info
->loop_iterations
= p
;
3603 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3604 read_inline_edge_summary (&ib
, e
);
3605 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3606 read_inline_edge_summary (&ib
, e
);
3609 lto_free_section_data (file_data
, LTO_section_inline_summary
, NULL
, data
,
3611 lto_data_in_delete (data_in
);
3615 /* Read inline summary. Jump functions are shared among ipa-cp
3616 and inliner, so when ipa-cp is active, we don't need to write them
3620 inline_read_summary (void)
3622 struct lto_file_decl_data
**file_data_vec
= lto_get_file_decl_data ();
3623 struct lto_file_decl_data
*file_data
;
3626 inline_summary_alloc ();
3628 while ((file_data
= file_data_vec
[j
++]))
3631 const char *data
= lto_get_section_data (file_data
,
3632 LTO_section_inline_summary
,
3635 inline_read_section (file_data
, data
, len
);
3637 /* Fatal error here. We do not want to support compiling ltrans units
3638 with different version of compiler or different flags than the WPA
3639 unit, so this should never happen. */
3640 fatal_error ("ipa inline summary is missing in input file");
3644 ipa_register_cgraph_hooks ();
3646 ipa_prop_read_jump_functions ();
3648 function_insertion_hook_holder
=
3649 cgraph_add_function_insertion_hook (&add_new_function
, NULL
);
3653 /* Write predicate P to OB. */
3656 write_predicate (struct output_block
*ob
, struct predicate
*p
)
3660 for (j
= 0; p
->clause
[j
]; j
++)
3662 gcc_assert (j
< MAX_CLAUSES
);
3663 streamer_write_uhwi (ob
, p
->clause
[j
]);
3665 streamer_write_uhwi (ob
, 0);
3669 /* Write inline summary for edge E to OB. */
3672 write_inline_edge_summary (struct output_block
*ob
, struct cgraph_edge
*e
)
3674 struct inline_edge_summary
*es
= inline_edge_summary (e
);
3677 streamer_write_uhwi (ob
, es
->call_stmt_size
);
3678 streamer_write_uhwi (ob
, es
->call_stmt_time
);
3679 streamer_write_uhwi (ob
, es
->loop_depth
);
3680 write_predicate (ob
, es
->predicate
);
3681 streamer_write_uhwi (ob
, VEC_length (inline_param_summary_t
, es
->param
));
3682 for (i
= 0; i
< (int)VEC_length (inline_param_summary_t
, es
->param
); i
++)
3683 streamer_write_uhwi (ob
, VEC_index (inline_param_summary_t
,
3684 es
->param
, i
).change_prob
);
3688 /* Write inline summary for node in SET.
3689 Jump functions are shared among ipa-cp and inliner, so when ipa-cp is
3690 active, we don't need to write them twice. */
3693 inline_write_summary (void)
3695 struct cgraph_node
*node
;
3697 struct output_block
*ob
= create_output_block (LTO_section_inline_summary
);
3698 lto_symtab_encoder_t encoder
= ob
->decl_state
->symtab_node_encoder
;
3699 unsigned int count
= 0;
3702 for (i
= 0; i
< lto_symtab_encoder_size (encoder
); i
++)
3703 if (symtab_function_p (snode
= lto_symtab_encoder_deref (encoder
, i
))
3704 && cgraph (snode
)->analyzed
)
3706 streamer_write_uhwi (ob
, count
);
3708 for (i
= 0; i
< lto_symtab_encoder_size (encoder
); i
++)
3710 if (symtab_function_p (snode
= lto_symtab_encoder_deref (encoder
, i
))
3711 && (node
= cgraph (snode
))->analyzed
)
3713 struct inline_summary
*info
= inline_summary (node
);
3714 struct bitpack_d bp
;
3715 struct cgraph_edge
*edge
;
3718 struct condition
*c
;
3720 streamer_write_uhwi (ob
, lto_symtab_encoder_encode (encoder
, (symtab_node
)node
));
3721 streamer_write_hwi (ob
, info
->estimated_self_stack_size
);
3722 streamer_write_hwi (ob
, info
->self_size
);
3723 streamer_write_hwi (ob
, info
->self_time
);
3724 bp
= bitpack_create (ob
->main_stream
);
3725 bp_pack_value (&bp
, info
->inlinable
, 1);
3726 streamer_write_bitpack (&bp
);
3727 streamer_write_uhwi (ob
, VEC_length (condition
, info
->conds
));
3728 for (i
= 0; VEC_iterate (condition
, info
->conds
, i
, c
); i
++)
3730 streamer_write_uhwi (ob
, c
->operand_num
);
3731 streamer_write_uhwi (ob
, c
->code
);
3732 stream_write_tree (ob
, c
->val
, true);
3733 bp
= bitpack_create (ob
->main_stream
);
3734 bp_pack_value (&bp
, c
->agg_contents
, 1);
3735 bp_pack_value (&bp
, c
->by_ref
, 1);
3736 streamer_write_bitpack (&bp
);
3737 if (c
->agg_contents
)
3738 streamer_write_uhwi (ob
, c
->offset
);
3740 streamer_write_uhwi (ob
, VEC_length (size_time_entry
, info
->entry
));
3742 VEC_iterate (size_time_entry
, info
->entry
, i
, e
);
3745 streamer_write_uhwi (ob
, e
->size
);
3746 streamer_write_uhwi (ob
, e
->time
);
3747 write_predicate (ob
, &e
->predicate
);
3749 write_predicate (ob
, info
->loop_iterations
);
3750 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
3751 write_inline_edge_summary (ob
, edge
);
3752 for (edge
= node
->indirect_calls
; edge
; edge
= edge
->next_callee
)
3753 write_inline_edge_summary (ob
, edge
);
3756 streamer_write_char_stream (ob
->main_stream
, 0);
3757 produce_asm (ob
, NULL
);
3758 destroy_output_block (ob
);
3760 if (optimize
&& !flag_ipa_cp
)
3761 ipa_prop_write_jump_functions ();
3765 /* Release inline summary. */
3768 inline_free_summary (void)
3770 struct cgraph_node
*node
;
3771 if (inline_edge_summary_vec
== NULL
)
3773 FOR_EACH_DEFINED_FUNCTION (node
)
3774 reset_inline_summary (node
);
3775 if (function_insertion_hook_holder
)
3776 cgraph_remove_function_insertion_hook (function_insertion_hook_holder
);
3777 function_insertion_hook_holder
= NULL
;
3778 if (node_removal_hook_holder
)
3779 cgraph_remove_node_removal_hook (node_removal_hook_holder
);
3780 node_removal_hook_holder
= NULL
;
3781 if (edge_removal_hook_holder
)
3782 cgraph_remove_edge_removal_hook (edge_removal_hook_holder
);
3783 edge_removal_hook_holder
= NULL
;
3784 if (node_duplication_hook_holder
)
3785 cgraph_remove_node_duplication_hook (node_duplication_hook_holder
);
3786 node_duplication_hook_holder
= NULL
;
3787 if (edge_duplication_hook_holder
)
3788 cgraph_remove_edge_duplication_hook (edge_duplication_hook_holder
);
3789 edge_duplication_hook_holder
= NULL
;
3790 VEC_free (inline_summary_t
, gc
, inline_summary_vec
);
3791 inline_summary_vec
= NULL
;
3792 VEC_free (inline_edge_summary_t
, heap
, inline_edge_summary_vec
);
3793 inline_edge_summary_vec
= NULL
;
3794 if (edge_predicate_pool
)
3795 free_alloc_pool (edge_predicate_pool
);
3796 edge_predicate_pool
= 0;