1 /* Function summary pass.
2 Copyright (C) 2003-2019 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* Analysis of function bodies used by inter-procedural passes
23 We estimate for each function
24 - function body size and size after specializing into given context
25 - average function execution time in a given context
28 - call statement size, time and how often the parameters change
30 ipa_fn_summary data structures store above information locally (i.e.
31 parameters of the function itself) and globally (i.e. parameters of
32 the function created by applying all the inline decisions already
33 present in the callgraph).
35 We provide access to the ipa_fn_summary data structure and
36 basic logic updating the parameters when inlining is performed.
38 The summaries are context sensitive. Context means
39 1) partial assignment of known constant values of operands
40 2) whether function is inlined into the call or not.
41 It is easy to add more variants. To represent function size and time
42 that depends on context (i.e. it is known to be optimized away when
43 context is known either by inlining or from IP-CP and cloning),
46 estimate_edge_size_and_time can be used to query
47 function size/time in the given context. ipa_merge_fn_summary_after_inlining merges
48 properties of caller and callee after inlining.
50 Finally pass_inline_parameters is exported. This is used to drive
51 computation of function parameters used by the early inliner. IPA
52 inlined performs analysis via its analyze_function method. */
56 #include "coretypes.h"
60 #include "alloc-pool.h"
61 #include "tree-pass.h"
63 #include "tree-streamer.h"
65 #include "diagnostic.h"
66 #include "fold-const.h"
67 #include "print-tree.h"
68 #include "tree-inline.h"
69 #include "gimple-pretty-print.h"
72 #include "gimple-iterator.h"
74 #include "tree-ssa-loop-niter.h"
75 #include "tree-ssa-loop.h"
76 #include "symbol-summary.h"
78 #include "ipa-fnsummary.h"
80 #include "tree-scalar-evolution.h"
81 #include "ipa-utils.h"
82 #include "cfgexpand.h"
84 #include "stringpool.h"
88 fast_function_summary
<ipa_fn_summary
*, va_gc
> *ipa_fn_summaries
;
89 fast_function_summary
<ipa_size_summary
*, va_heap
> *ipa_size_summaries
;
90 fast_call_summary
<ipa_call_summary
*, va_heap
> *ipa_call_summaries
;
92 /* Edge predicates goes here. */
93 static object_allocator
<predicate
> edge_predicate_pool ("edge predicates");
98 ipa_dump_hints (FILE *f
, ipa_hints hints
)
102 fprintf (f
, "IPA hints:");
103 if (hints
& INLINE_HINT_indirect_call
)
105 hints
&= ~INLINE_HINT_indirect_call
;
106 fprintf (f
, " indirect_call");
108 if (hints
& INLINE_HINT_loop_iterations
)
110 hints
&= ~INLINE_HINT_loop_iterations
;
111 fprintf (f
, " loop_iterations");
113 if (hints
& INLINE_HINT_loop_stride
)
115 hints
&= ~INLINE_HINT_loop_stride
;
116 fprintf (f
, " loop_stride");
118 if (hints
& INLINE_HINT_same_scc
)
120 hints
&= ~INLINE_HINT_same_scc
;
121 fprintf (f
, " same_scc");
123 if (hints
& INLINE_HINT_in_scc
)
125 hints
&= ~INLINE_HINT_in_scc
;
126 fprintf (f
, " in_scc");
128 if (hints
& INLINE_HINT_cross_module
)
130 hints
&= ~INLINE_HINT_cross_module
;
131 fprintf (f
, " cross_module");
133 if (hints
& INLINE_HINT_declared_inline
)
135 hints
&= ~INLINE_HINT_declared_inline
;
136 fprintf (f
, " declared_inline");
138 if (hints
& INLINE_HINT_known_hot
)
140 hints
&= ~INLINE_HINT_known_hot
;
141 fprintf (f
, " known_hot");
147 /* Record SIZE and TIME to SUMMARY.
148 The accounted code will be executed when EXEC_PRED is true.
149 When NONCONST_PRED is false the code will evaulate to constant and
150 will get optimized out in specialized clones of the function. */
153 ipa_fn_summary::account_size_time (int size
, sreal time
,
154 const predicate
&exec_pred
,
155 const predicate
&nonconst_pred_in
)
160 predicate nonconst_pred
;
162 if (exec_pred
== false)
165 nonconst_pred
= nonconst_pred_in
& exec_pred
;
167 if (nonconst_pred
== false)
170 /* We need to create initial empty unconitional clause, but otherwie
171 we don't need to account empty times and sizes. */
172 if (!size
&& time
== 0 && size_time_table
)
175 gcc_assert (time
>= 0);
177 for (i
= 0; vec_safe_iterate (size_time_table
, i
, &e
); i
++)
178 if (e
->exec_predicate
== exec_pred
179 && e
->nonconst_predicate
== nonconst_pred
)
188 e
= &(*size_time_table
)[0];
189 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
191 "\t\tReached limit on number of entries, "
192 "ignoring the predicate.");
194 if (dump_file
&& (dump_flags
& TDF_DETAILS
) && (time
!= 0 || size
))
197 "\t\tAccounting size:%3.2f, time:%3.2f on %spredicate exec:",
198 ((double) size
) / ipa_fn_summary::size_scale
,
199 (time
.to_double ()), found
? "" : "new ");
200 exec_pred
.dump (dump_file
, conds
, 0);
201 if (exec_pred
!= nonconst_pred
)
203 fprintf (dump_file
, " nonconst:");
204 nonconst_pred
.dump (dump_file
, conds
);
207 fprintf (dump_file
, "\n");
211 class size_time_entry new_entry
;
212 new_entry
.size
= size
;
213 new_entry
.time
= time
;
214 new_entry
.exec_predicate
= exec_pred
;
215 new_entry
.nonconst_predicate
= nonconst_pred
;
216 vec_safe_push (size_time_table
, new_entry
);
225 /* We proved E to be unreachable, redirect it to __bultin_unreachable. */
227 static struct cgraph_edge
*
228 redirect_to_unreachable (struct cgraph_edge
*e
)
230 struct cgraph_node
*callee
= !e
->inline_failed
? e
->callee
: NULL
;
231 struct cgraph_node
*target
= cgraph_node::get_create
232 (builtin_decl_implicit (BUILT_IN_UNREACHABLE
));
235 e
= e
->resolve_speculation (target
->decl
);
237 e
->make_direct (target
);
239 e
->redirect_callee (target
);
240 class ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
241 e
->inline_failed
= CIF_UNREACHABLE
;
242 e
->count
= profile_count::zero ();
243 es
->call_stmt_size
= 0;
244 es
->call_stmt_time
= 0;
246 callee
->remove_symbol_and_inline_clones ();
250 /* Set predicate for edge E. */
253 edge_set_predicate (struct cgraph_edge
*e
, predicate
*predicate
)
255 /* If the edge is determined to be never executed, redirect it
256 to BUILTIN_UNREACHABLE to make it clear to IPA passes the call will
258 if (predicate
&& *predicate
== false
259 /* When handling speculative edges, we need to do the redirection
260 just once. Do it always on the direct edge, so we do not
261 attempt to resolve speculation while duplicating the edge. */
262 && (!e
->speculative
|| e
->callee
))
263 e
= redirect_to_unreachable (e
);
265 class ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
266 if (predicate
&& *predicate
!= true)
269 es
->predicate
= edge_predicate_pool
.allocate ();
270 *es
->predicate
= *predicate
;
275 edge_predicate_pool
.remove (es
->predicate
);
276 es
->predicate
= NULL
;
280 /* Set predicate for hint *P. */
283 set_hint_predicate (predicate
**p
, predicate new_predicate
)
285 if (new_predicate
== false || new_predicate
== true)
288 edge_predicate_pool
.remove (*p
);
294 *p
= edge_predicate_pool
.allocate ();
300 /* Compute what conditions may or may not hold given invormation about
301 parameters. RET_CLAUSE returns truths that may hold in a specialized copy,
302 whie RET_NONSPEC_CLAUSE returns truths that may hold in an nonspecialized
303 copy when called in a given context. It is a bitmask of conditions. Bit
304 0 means that condition is known to be false, while bit 1 means that condition
305 may or may not be true. These differs - for example NOT_INLINED condition
306 is always false in the second and also builtin_constant_p tests cannot use
307 the fact that parameter is indeed a constant.
309 KNOWN_VALS is partial mapping of parameters of NODE to constant values.
310 KNOWN_AGGS is a vector of aggreggate jump functions for each parameter.
311 Return clause of possible truths. When INLINE_P is true, assume that we are
314 ERROR_MARK means compile time invariant. */
317 evaluate_conditions_for_known_args (struct cgraph_node
*node
,
319 vec
<tree
> known_vals
,
320 vec
<ipa_agg_jump_function_p
>
322 clause_t
*ret_clause
,
323 clause_t
*ret_nonspec_clause
)
325 clause_t clause
= inline_p
? 0 : 1 << predicate::not_inlined_condition
;
326 clause_t nonspec_clause
= 1 << predicate::not_inlined_condition
;
327 class ipa_fn_summary
*info
= ipa_fn_summaries
->get (node
);
331 for (i
= 0; vec_safe_iterate (info
->conds
, i
, &c
); i
++)
336 struct expr_eval_op
*op
;
338 /* We allow call stmt to have fewer arguments than the callee function
339 (especially for K&R style programs). So bound check here (we assume
340 known_aggs vector, if non-NULL, has the same length as
342 gcc_checking_assert (!known_aggs
.exists ()
343 || (known_vals
.length () == known_aggs
.length ()));
344 if (c
->operand_num
>= (int) known_vals
.length ())
346 clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
347 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
353 struct ipa_agg_jump_function
*agg
;
355 if (c
->code
== predicate::changed
357 && (known_vals
[c
->operand_num
] == error_mark_node
))
360 if (known_aggs
.exists ())
362 agg
= known_aggs
[c
->operand_num
];
363 val
= ipa_find_agg_cst_for_param (agg
, known_vals
[c
->operand_num
],
364 c
->offset
, c
->by_ref
);
371 val
= known_vals
[c
->operand_num
];
372 if (val
== error_mark_node
&& c
->code
!= predicate::changed
)
378 clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
379 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
382 if (c
->code
== predicate::changed
)
384 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
388 if (TYPE_SIZE (c
->type
) != TYPE_SIZE (TREE_TYPE (val
)))
390 clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
391 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
394 if (c
->code
== predicate::is_not_constant
)
396 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
400 val
= fold_unary (VIEW_CONVERT_EXPR
, c
->type
, val
);
401 for (j
= 0; vec_safe_iterate (c
->param_ops
, j
, &op
); j
++)
406 val
= fold_unary (op
->code
, op
->type
, val
);
407 else if (!op
->val
[1])
408 val
= fold_binary (op
->code
, op
->type
,
409 op
->index
? op
->val
[0] : val
,
410 op
->index
? val
: op
->val
[0]);
411 else if (op
->index
== 0)
412 val
= fold_ternary (op
->code
, op
->type
,
413 val
, op
->val
[0], op
->val
[1]);
414 else if (op
->index
== 1)
415 val
= fold_ternary (op
->code
, op
->type
,
416 op
->val
[0], val
, op
->val
[1]);
417 else if (op
->index
== 2)
418 val
= fold_ternary (op
->code
, op
->type
,
419 op
->val
[0], op
->val
[1], val
);
425 ? fold_binary_to_constant (c
->code
, boolean_type_node
, val
, c
->val
)
428 if (res
&& integer_zerop (res
))
431 clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
432 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
434 *ret_clause
= clause
;
435 if (ret_nonspec_clause
)
436 *ret_nonspec_clause
= nonspec_clause
;
440 /* Work out what conditions might be true at invocation of E. */
443 evaluate_properties_for_edge (struct cgraph_edge
*e
, bool inline_p
,
444 clause_t
*clause_ptr
,
445 clause_t
*nonspec_clause_ptr
,
446 vec
<tree
> *known_vals_ptr
,
447 vec
<ipa_polymorphic_call_context
>
449 vec
<ipa_agg_jump_function_p
> *known_aggs_ptr
)
451 struct cgraph_node
*callee
= e
->callee
->ultimate_alias_target ();
452 class ipa_fn_summary
*info
= ipa_fn_summaries
->get (callee
);
453 vec
<tree
> known_vals
= vNULL
;
454 vec
<ipa_agg_jump_function_p
> known_aggs
= vNULL
;
455 class ipa_edge_args
*args
;
458 *clause_ptr
= inline_p
? 0 : 1 << predicate::not_inlined_condition
;
460 known_vals_ptr
->create (0);
461 if (known_contexts_ptr
)
462 known_contexts_ptr
->create (0);
464 if (ipa_node_params_sum
465 && !e
->call_stmt_cannot_inline_p
466 && ((clause_ptr
&& info
->conds
) || known_vals_ptr
|| known_contexts_ptr
)
467 && (args
= IPA_EDGE_REF (e
)) != NULL
)
469 class ipa_node_params
*caller_parms_info
, *callee_pi
;
470 class ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
471 int i
, count
= ipa_get_cs_argument_count (args
);
473 if (e
->caller
->inlined_to
)
474 caller_parms_info
= IPA_NODE_REF (e
->caller
->inlined_to
);
476 caller_parms_info
= IPA_NODE_REF (e
->caller
);
477 callee_pi
= IPA_NODE_REF (callee
);
479 if (count
&& (info
->conds
|| known_vals_ptr
))
480 known_vals
.safe_grow_cleared (count
);
481 if (count
&& (info
->conds
|| known_aggs_ptr
))
482 known_aggs
.safe_grow_cleared (count
);
483 if (count
&& known_contexts_ptr
)
484 known_contexts_ptr
->safe_grow_cleared (count
);
487 for (i
= 0; i
< count
; i
++)
489 struct ipa_jump_func
*jf
= ipa_get_ith_jump_func (args
, i
);
490 tree cst
= ipa_value_from_jfunc (caller_parms_info
, jf
,
491 ipa_get_type (callee_pi
, i
));
493 if (!cst
&& e
->call_stmt
494 && i
< (int)gimple_call_num_args (e
->call_stmt
))
496 cst
= gimple_call_arg (e
->call_stmt
, i
);
497 if (!is_gimple_min_invariant (cst
))
502 gcc_checking_assert (TREE_CODE (cst
) != TREE_BINFO
);
503 if (known_vals
.exists ())
506 else if (inline_p
&& !es
->param
[i
].change_prob
)
507 known_vals
[i
] = error_mark_node
;
509 if (known_contexts_ptr
)
510 (*known_contexts_ptr
)[i
]
511 = ipa_context_from_jfunc (caller_parms_info
, e
, i
, jf
);
512 /* TODO: When IPA-CP starts propagating and merging aggregate jump
513 functions, use its knowledge of the caller too, just like the
514 scalar case above. */
515 known_aggs
[i
] = &jf
->agg
;
518 gcc_assert (callee
->thunk
.thunk_p
);
520 else if (e
->call_stmt
&& !e
->call_stmt_cannot_inline_p
521 && ((clause_ptr
&& info
->conds
) || known_vals_ptr
))
523 int i
, count
= (int)gimple_call_num_args (e
->call_stmt
);
525 if (count
&& (info
->conds
|| known_vals_ptr
))
526 known_vals
.safe_grow_cleared (count
);
527 for (i
= 0; i
< count
; i
++)
529 tree cst
= gimple_call_arg (e
->call_stmt
, i
);
530 if (!is_gimple_min_invariant (cst
))
537 evaluate_conditions_for_known_args (callee
, inline_p
,
538 known_vals
, known_aggs
, clause_ptr
,
542 *known_vals_ptr
= known_vals
;
544 known_vals
.release ();
547 *known_aggs_ptr
= known_aggs
;
549 known_aggs
.release ();
553 /* Allocate the function summary. */
556 ipa_fn_summary_alloc (void)
558 gcc_checking_assert (!ipa_fn_summaries
);
559 ipa_size_summaries
= new fast_function_summary
<ipa_size_summary
*, va_heap
>
561 ipa_fn_summaries
= ipa_fn_summary_t::create_ggc (symtab
);
562 ipa_call_summaries
= new ipa_call_summary_t (symtab
);
565 ipa_call_summary::~ipa_call_summary ()
568 edge_predicate_pool
.remove (predicate
);
573 ipa_fn_summary::~ipa_fn_summary ()
576 edge_predicate_pool
.remove (loop_iterations
);
578 edge_predicate_pool
.remove (loop_stride
);
580 vec_free (size_time_table
);
584 ipa_fn_summary_t::remove_callees (cgraph_node
*node
)
587 for (e
= node
->callees
; e
; e
= e
->next_callee
)
588 ipa_call_summaries
->remove (e
);
589 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
590 ipa_call_summaries
->remove (e
);
593 /* Same as remap_predicate_after_duplication but handle hint predicate *P.
594 Additionally care about allocating new memory slot for updated predicate
595 and set it to NULL when it becomes true or false (and thus uninteresting).
599 remap_hint_predicate_after_duplication (predicate
**p
,
600 clause_t possible_truths
)
602 predicate new_predicate
;
607 new_predicate
= (*p
)->remap_after_duplication (possible_truths
);
608 /* We do not want to free previous predicate; it is used by node origin. */
610 set_hint_predicate (p
, new_predicate
);
614 /* Hook that is called by cgraph.c when a node is duplicated. */
616 ipa_fn_summary_t::duplicate (cgraph_node
*src
,
619 ipa_fn_summary
*info
)
621 new (info
) ipa_fn_summary (*ipa_fn_summaries
->get (src
));
622 /* TODO: as an optimization, we may avoid copying conditions
623 that are known to be false or true. */
624 info
->conds
= vec_safe_copy (info
->conds
);
626 /* When there are any replacements in the function body, see if we can figure
627 out that something was optimized out. */
628 if (ipa_node_params_sum
&& dst
->clone
.tree_map
)
630 vec
<size_time_entry
, va_gc
> *entry
= info
->size_time_table
;
631 /* Use SRC parm info since it may not be copied yet. */
632 class ipa_node_params
*parms_info
= IPA_NODE_REF (src
);
633 vec
<tree
> known_vals
= vNULL
;
634 int count
= ipa_get_param_count (parms_info
);
636 clause_t possible_truths
;
637 predicate true_pred
= true;
639 int optimized_out_size
= 0;
640 bool inlined_to_p
= false;
641 struct cgraph_edge
*edge
, *next
;
643 info
->size_time_table
= 0;
644 known_vals
.safe_grow_cleared (count
);
645 for (i
= 0; i
< count
; i
++)
647 struct ipa_replace_map
*r
;
649 for (j
= 0; vec_safe_iterate (dst
->clone
.tree_map
, j
, &r
); j
++)
651 if (r
->parm_num
== i
)
653 known_vals
[i
] = r
->new_tree
;
658 evaluate_conditions_for_known_args (dst
, false,
662 /* We are going to specialize,
663 so ignore nonspec truths. */
665 known_vals
.release ();
667 info
->account_size_time (0, 0, true_pred
, true_pred
);
669 /* Remap size_time vectors.
670 Simplify the predicate by prunning out alternatives that are known
672 TODO: as on optimization, we can also eliminate conditions known
674 for (i
= 0; vec_safe_iterate (entry
, i
, &e
); i
++)
676 predicate new_exec_pred
;
677 predicate new_nonconst_pred
;
678 new_exec_pred
= e
->exec_predicate
.remap_after_duplication
680 new_nonconst_pred
= e
->nonconst_predicate
.remap_after_duplication
682 if (new_exec_pred
== false || new_nonconst_pred
== false)
683 optimized_out_size
+= e
->size
;
685 info
->account_size_time (e
->size
, e
->time
, new_exec_pred
,
689 /* Remap edge predicates with the same simplification as above.
690 Also copy constantness arrays. */
691 for (edge
= dst
->callees
; edge
; edge
= next
)
693 predicate new_predicate
;
694 class ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
695 next
= edge
->next_callee
;
697 if (!edge
->inline_failed
)
701 new_predicate
= es
->predicate
->remap_after_duplication
703 if (new_predicate
== false && *es
->predicate
!= false)
704 optimized_out_size
+= es
->call_stmt_size
* ipa_fn_summary::size_scale
;
705 edge_set_predicate (edge
, &new_predicate
);
708 /* Remap indirect edge predicates with the same simplificaiton as above.
709 Also copy constantness arrays. */
710 for (edge
= dst
->indirect_calls
; edge
; edge
= next
)
712 predicate new_predicate
;
713 class ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
714 next
= edge
->next_callee
;
716 gcc_checking_assert (edge
->inline_failed
);
719 new_predicate
= es
->predicate
->remap_after_duplication
721 if (new_predicate
== false && *es
->predicate
!= false)
722 optimized_out_size
+= es
->call_stmt_size
* ipa_fn_summary::size_scale
;
723 edge_set_predicate (edge
, &new_predicate
);
725 remap_hint_predicate_after_duplication (&info
->loop_iterations
,
727 remap_hint_predicate_after_duplication (&info
->loop_stride
,
730 /* If inliner or someone after inliner will ever start producing
731 non-trivial clones, we will get trouble with lack of information
732 about updating self sizes, because size vectors already contains
733 sizes of the calees. */
734 gcc_assert (!inlined_to_p
|| !optimized_out_size
);
738 info
->size_time_table
= vec_safe_copy (info
->size_time_table
);
739 if (info
->loop_iterations
)
741 predicate p
= *info
->loop_iterations
;
742 info
->loop_iterations
= NULL
;
743 set_hint_predicate (&info
->loop_iterations
, p
);
745 if (info
->loop_stride
)
747 predicate p
= *info
->loop_stride
;
748 info
->loop_stride
= NULL
;
749 set_hint_predicate (&info
->loop_stride
, p
);
752 if (!dst
->inlined_to
)
753 ipa_update_overall_fn_summary (dst
);
757 /* Hook that is called by cgraph.c when a node is duplicated. */
760 ipa_call_summary_t::duplicate (struct cgraph_edge
*src
,
761 struct cgraph_edge
*dst
,
762 class ipa_call_summary
*srcinfo
,
763 class ipa_call_summary
*info
)
765 new (info
) ipa_call_summary (*srcinfo
);
766 info
->predicate
= NULL
;
767 edge_set_predicate (dst
, srcinfo
->predicate
);
768 info
->param
= srcinfo
->param
.copy ();
769 if (!dst
->indirect_unknown_callee
&& src
->indirect_unknown_callee
)
771 info
->call_stmt_size
-= (eni_size_weights
.indirect_call_cost
772 - eni_size_weights
.call_cost
);
773 info
->call_stmt_time
-= (eni_time_weights
.indirect_call_cost
774 - eni_time_weights
.call_cost
);
778 /* Dump edge summaries associated to NODE and recursively to all clones.
782 dump_ipa_call_summary (FILE *f
, int indent
, struct cgraph_node
*node
,
783 class ipa_fn_summary
*info
)
785 struct cgraph_edge
*edge
;
786 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
788 class ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
789 struct cgraph_node
*callee
= edge
->callee
->ultimate_alias_target ();
793 "%*s%s/%i %s\n%*s freq:%4.2f",
794 indent
, "", callee
->name (), callee
->order
,
796 ? "inlined" : cgraph_inline_failed_string (edge
-> inline_failed
),
797 indent
, "", edge
->sreal_frequency ().to_double ());
800 fprintf (f
, " loop depth:%2i size:%2i time: %2i",
801 es
->loop_depth
, es
->call_stmt_size
, es
->call_stmt_time
);
803 ipa_fn_summary
*s
= ipa_fn_summaries
->get (callee
);
804 ipa_size_summary
*ss
= ipa_size_summaries
->get (callee
);
806 fprintf (f
, " callee size:%2i stack:%2i",
807 (int) (ss
->size
/ ipa_fn_summary::size_scale
),
808 (int) s
->estimated_stack_size
);
810 if (es
&& es
->predicate
)
812 fprintf (f
, " predicate: ");
813 es
->predicate
->dump (f
, info
->conds
);
817 if (es
&& es
->param
.exists ())
818 for (i
= 0; i
< (int) es
->param
.length (); i
++)
820 int prob
= es
->param
[i
].change_prob
;
823 fprintf (f
, "%*s op%i is compile time invariant\n",
825 else if (prob
!= REG_BR_PROB_BASE
)
826 fprintf (f
, "%*s op%i change %f%% of time\n", indent
+ 2, "", i
,
827 prob
* 100.0 / REG_BR_PROB_BASE
);
829 if (!edge
->inline_failed
)
831 ipa_size_summary
*ss
= ipa_size_summaries
->get (callee
);
832 fprintf (f
, "%*sStack frame offset %i, callee self size %i\n",
834 (int) ipa_get_stack_frame_offset (callee
),
835 (int) ss
->estimated_self_stack_size
);
836 dump_ipa_call_summary (f
, indent
+ 2, callee
, info
);
839 for (edge
= node
->indirect_calls
; edge
; edge
= edge
->next_callee
)
841 class ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
842 fprintf (f
, "%*sindirect call loop depth:%2i freq:%4.2f size:%2i"
846 edge
->sreal_frequency ().to_double (), es
->call_stmt_size
,
850 fprintf (f
, "predicate: ");
851 es
->predicate
->dump (f
, info
->conds
);
860 ipa_dump_fn_summary (FILE *f
, struct cgraph_node
*node
)
862 if (node
->definition
)
864 class ipa_fn_summary
*s
= ipa_fn_summaries
->get (node
);
865 class ipa_size_summary
*ss
= ipa_size_summaries
->get (node
);
870 fprintf (f
, "IPA function summary for %s", node
->dump_name ());
871 if (DECL_DISREGARD_INLINE_LIMITS (node
->decl
))
872 fprintf (f
, " always_inline");
874 fprintf (f
, " inlinable");
875 if (s
->fp_expressions
)
876 fprintf (f
, " fp_expression");
877 fprintf (f
, "\n global time: %f\n", s
->time
.to_double ());
878 fprintf (f
, " self size: %i\n", ss
->self_size
);
879 fprintf (f
, " global size: %i\n", ss
->size
);
880 fprintf (f
, " min size: %i\n", s
->min_size
);
881 fprintf (f
, " self stack: %i\n",
882 (int) ss
->estimated_self_stack_size
);
883 fprintf (f
, " global stack: %i\n", (int) s
->estimated_stack_size
);
885 fprintf (f
, " estimated growth:%i\n", (int) s
->growth
);
887 fprintf (f
, " In SCC: %i\n", (int) s
->scc_no
);
888 for (i
= 0; vec_safe_iterate (s
->size_time_table
, i
, &e
); i
++)
890 fprintf (f
, " size:%f, time:%f",
891 (double) e
->size
/ ipa_fn_summary::size_scale
,
892 e
->time
.to_double ());
893 if (e
->exec_predicate
!= true)
895 fprintf (f
, ", executed if:");
896 e
->exec_predicate
.dump (f
, s
->conds
, 0);
898 if (e
->exec_predicate
!= e
->nonconst_predicate
)
900 fprintf (f
, ", nonconst if:");
901 e
->nonconst_predicate
.dump (f
, s
->conds
, 0);
905 if (s
->loop_iterations
)
907 fprintf (f
, " loop iterations:");
908 s
->loop_iterations
->dump (f
, s
->conds
);
912 fprintf (f
, " loop stride:");
913 s
->loop_stride
->dump (f
, s
->conds
);
915 fprintf (f
, " calls:\n");
916 dump_ipa_call_summary (f
, 4, node
, s
);
920 fprintf (f
, "IPA summary for %s is missing.\n", node
->dump_name ());
925 ipa_debug_fn_summary (struct cgraph_node
*node
)
927 ipa_dump_fn_summary (stderr
, node
);
931 ipa_dump_fn_summaries (FILE *f
)
933 struct cgraph_node
*node
;
935 FOR_EACH_DEFINED_FUNCTION (node
)
936 if (!node
->inlined_to
)
937 ipa_dump_fn_summary (f
, node
);
940 /* Callback of walk_aliased_vdefs. Flags that it has been invoked to the
941 boolean variable pointed to by DATA. */
944 mark_modified (ao_ref
*ao ATTRIBUTE_UNUSED
, tree vdef ATTRIBUTE_UNUSED
,
947 bool *b
= (bool *) data
;
952 /* If OP refers to value of function parameter, return the corresponding
953 parameter. If non-NULL, the size of the memory load (or the SSA_NAME of the
954 PARM_DECL) will be stored to *SIZE_P in that case too. */
957 unmodified_parm_1 (ipa_func_body_info
*fbi
, gimple
*stmt
, tree op
,
960 /* SSA_NAME referring to parm default def? */
961 if (TREE_CODE (op
) == SSA_NAME
962 && SSA_NAME_IS_DEFAULT_DEF (op
)
963 && TREE_CODE (SSA_NAME_VAR (op
)) == PARM_DECL
)
966 *size_p
= tree_to_poly_int64 (TYPE_SIZE (TREE_TYPE (op
)));
967 return SSA_NAME_VAR (op
);
969 /* Non-SSA parm reference? */
970 if (TREE_CODE (op
) == PARM_DECL
)
972 bool modified
= false;
975 ao_ref_init (&refd
, op
);
976 int walked
= walk_aliased_vdefs (&refd
, gimple_vuse (stmt
),
977 mark_modified
, &modified
, NULL
, NULL
,
978 fbi
->aa_walk_budget
+ 1);
981 fbi
->aa_walk_budget
= 0;
987 *size_p
= tree_to_poly_int64 (TYPE_SIZE (TREE_TYPE (op
)));
994 /* If OP refers to value of function parameter, return the corresponding
995 parameter. Also traverse chains of SSA register assignments. If non-NULL,
996 the size of the memory load (or the SSA_NAME of the PARM_DECL) will be
997 stored to *SIZE_P in that case too. */
1000 unmodified_parm (ipa_func_body_info
*fbi
, gimple
*stmt
, tree op
,
1003 tree res
= unmodified_parm_1 (fbi
, stmt
, op
, size_p
);
1007 if (TREE_CODE (op
) == SSA_NAME
1008 && !SSA_NAME_IS_DEFAULT_DEF (op
)
1009 && gimple_assign_single_p (SSA_NAME_DEF_STMT (op
)))
1010 return unmodified_parm (fbi
, SSA_NAME_DEF_STMT (op
),
1011 gimple_assign_rhs1 (SSA_NAME_DEF_STMT (op
)),
1016 /* If OP refers to a value of a function parameter or value loaded from an
1017 aggregate passed to a parameter (either by value or reference), return TRUE
1018 and store the number of the parameter to *INDEX_P, the access size into
1019 *SIZE_P, and information whether and how it has been loaded from an
1020 aggregate into *AGGPOS. INFO describes the function parameters, STMT is the
1021 statement in which OP is used or loaded. */
1024 unmodified_parm_or_parm_agg_item (struct ipa_func_body_info
*fbi
,
1025 gimple
*stmt
, tree op
, int *index_p
,
1027 struct agg_position_info
*aggpos
)
1029 tree res
= unmodified_parm_1 (fbi
, stmt
, op
, size_p
);
1031 gcc_checking_assert (aggpos
);
1034 *index_p
= ipa_get_param_decl_index (fbi
->info
, res
);
1037 aggpos
->agg_contents
= false;
1038 aggpos
->by_ref
= false;
1042 if (TREE_CODE (op
) == SSA_NAME
)
1044 if (SSA_NAME_IS_DEFAULT_DEF (op
)
1045 || !gimple_assign_single_p (SSA_NAME_DEF_STMT (op
)))
1047 stmt
= SSA_NAME_DEF_STMT (op
);
1048 op
= gimple_assign_rhs1 (stmt
);
1049 if (!REFERENCE_CLASS_P (op
))
1050 return unmodified_parm_or_parm_agg_item (fbi
, stmt
, op
, index_p
, size_p
,
1054 aggpos
->agg_contents
= true;
1055 return ipa_load_from_parm_agg (fbi
, fbi
->info
->descriptors
,
1056 stmt
, op
, index_p
, &aggpos
->offset
,
1057 size_p
, &aggpos
->by_ref
);
1060 /* See if statement might disappear after inlining.
1061 0 - means not eliminated
1062 1 - half of statements goes away
1063 2 - for sure it is eliminated.
1064 We are not terribly sophisticated, basically looking for simple abstraction
1065 penalty wrappers. */
1068 eliminated_by_inlining_prob (ipa_func_body_info
*fbi
, gimple
*stmt
)
1070 enum gimple_code code
= gimple_code (stmt
);
1071 enum tree_code rhs_code
;
1081 if (gimple_num_ops (stmt
) != 2)
1084 rhs_code
= gimple_assign_rhs_code (stmt
);
1086 /* Casts of parameters, loads from parameters passed by reference
1087 and stores to return value or parameters are often free after
1088 inlining dua to SRA and further combining.
1089 Assume that half of statements goes away. */
1090 if (CONVERT_EXPR_CODE_P (rhs_code
)
1091 || rhs_code
== VIEW_CONVERT_EXPR
1092 || rhs_code
== ADDR_EXPR
1093 || gimple_assign_rhs_class (stmt
) == GIMPLE_SINGLE_RHS
)
1095 tree rhs
= gimple_assign_rhs1 (stmt
);
1096 tree lhs
= gimple_assign_lhs (stmt
);
1097 tree inner_rhs
= get_base_address (rhs
);
1098 tree inner_lhs
= get_base_address (lhs
);
1099 bool rhs_free
= false;
1100 bool lhs_free
= false;
1107 /* Reads of parameter are expected to be free. */
1108 if (unmodified_parm (fbi
, stmt
, inner_rhs
, NULL
))
1110 /* Match expressions of form &this->field. Those will most likely
1111 combine with something upstream after inlining. */
1112 else if (TREE_CODE (inner_rhs
) == ADDR_EXPR
)
1114 tree op
= get_base_address (TREE_OPERAND (inner_rhs
, 0));
1115 if (TREE_CODE (op
) == PARM_DECL
)
1117 else if (TREE_CODE (op
) == MEM_REF
1118 && unmodified_parm (fbi
, stmt
, TREE_OPERAND (op
, 0),
1123 /* When parameter is not SSA register because its address is taken
1124 and it is just copied into one, the statement will be completely
1125 free after inlining (we will copy propagate backward). */
1126 if (rhs_free
&& is_gimple_reg (lhs
))
1129 /* Reads of parameters passed by reference
1130 expected to be free (i.e. optimized out after inlining). */
1131 if (TREE_CODE (inner_rhs
) == MEM_REF
1132 && unmodified_parm (fbi
, stmt
, TREE_OPERAND (inner_rhs
, 0), NULL
))
1135 /* Copying parameter passed by reference into gimple register is
1136 probably also going to copy propagate, but we can't be quite
1138 if (rhs_free
&& is_gimple_reg (lhs
))
1141 /* Writes to parameters, parameters passed by value and return value
1142 (either dirrectly or passed via invisible reference) are free.
1144 TODO: We ought to handle testcase like
1145 struct a {int a,b;};
1147 retrurnsturct (void)
1153 This translate into:
1168 For that we either need to copy ipa-split logic detecting writes
1170 if (TREE_CODE (inner_lhs
) == PARM_DECL
1171 || TREE_CODE (inner_lhs
) == RESULT_DECL
1172 || (TREE_CODE (inner_lhs
) == MEM_REF
1173 && (unmodified_parm (fbi
, stmt
, TREE_OPERAND (inner_lhs
, 0),
1175 || (TREE_CODE (TREE_OPERAND (inner_lhs
, 0)) == SSA_NAME
1176 && SSA_NAME_VAR (TREE_OPERAND (inner_lhs
, 0))
1177 && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND
1179 0))) == RESULT_DECL
))))
1182 && (is_gimple_reg (rhs
) || is_gimple_min_invariant (rhs
)))
1184 if (lhs_free
&& rhs_free
)
1193 /* Analyze EXPR if it represents a series of simple operations performed on
1194 a function parameter and return true if so. FBI, STMT, EXPR, INDEX_P and
1195 AGGPOS have the same meaning like in unmodified_parm_or_parm_agg_item.
1196 Type of the parameter or load from an aggregate via the parameter is
1197 stored in *TYPE_P. Operations on the parameter are recorded to
1198 PARAM_OPS_P if it is not NULL. */
1201 decompose_param_expr (struct ipa_func_body_info
*fbi
,
1202 gimple
*stmt
, tree expr
,
1203 int *index_p
, tree
*type_p
,
1204 struct agg_position_info
*aggpos
,
1205 expr_eval_ops
*param_ops_p
= NULL
)
1207 int op_limit
= param_ipa_max_param_expr_ops
;
1211 *param_ops_p
= NULL
;
1215 expr_eval_op eval_op
;
1217 unsigned cst_count
= 0;
1219 if (unmodified_parm_or_parm_agg_item (fbi
, stmt
, expr
, index_p
, NULL
,
1222 tree type
= TREE_TYPE (expr
);
1224 if (aggpos
->agg_contents
)
1226 /* Stop if containing bit-field. */
1227 if (TREE_CODE (expr
) == BIT_FIELD_REF
1228 || contains_bitfld_component_ref_p (expr
))
1236 if (TREE_CODE (expr
) != SSA_NAME
|| SSA_NAME_IS_DEFAULT_DEF (expr
))
1239 if (!is_gimple_assign (stmt
= SSA_NAME_DEF_STMT (expr
)))
1242 switch (gimple_assign_rhs_class (stmt
))
1244 case GIMPLE_SINGLE_RHS
:
1245 expr
= gimple_assign_rhs1 (stmt
);
1248 case GIMPLE_UNARY_RHS
:
1252 case GIMPLE_BINARY_RHS
:
1256 case GIMPLE_TERNARY_RHS
:
1264 /* Stop if expression is too complex. */
1265 if (op_count
++ == op_limit
)
1270 eval_op
.code
= gimple_assign_rhs_code (stmt
);
1271 eval_op
.type
= TREE_TYPE (gimple_assign_lhs (stmt
));
1272 eval_op
.val
[0] = NULL_TREE
;
1273 eval_op
.val
[1] = NULL_TREE
;
1277 for (unsigned i
= 0; i
< rhs_count
; i
++)
1279 tree op
= gimple_op (stmt
, i
+ 1);
1281 gcc_assert (op
&& !TYPE_P (op
));
1282 if (is_gimple_ip_invariant (op
))
1284 if (++cst_count
== rhs_count
)
1287 eval_op
.val
[cst_count
- 1] = op
;
1291 /* Found a non-constant operand, and record its index in rhs
1298 /* Found more than one non-constant operands. */
1304 vec_safe_insert (*param_ops_p
, 0, eval_op
);
1307 /* Failed to decompose, free resource and return. */
1310 vec_free (*param_ops_p
);
1315 /* If BB ends by a conditional we can turn into predicates, attach corresponding
1316 predicates to the CFG edges. */
1319 set_cond_stmt_execution_predicate (struct ipa_func_body_info
*fbi
,
1320 class ipa_fn_summary
*summary
,
1321 class ipa_node_params
*params_summary
,
1327 struct agg_position_info aggpos
;
1328 enum tree_code code
, inverted_code
;
1333 expr_eval_ops param_ops
;
1335 last
= last_stmt (bb
);
1336 if (!last
|| gimple_code (last
) != GIMPLE_COND
)
1338 if (!is_gimple_ip_invariant (gimple_cond_rhs (last
)))
1340 op
= gimple_cond_lhs (last
);
1342 if (decompose_param_expr (fbi
, last
, op
, &index
, ¶m_type
, &aggpos
,
1345 code
= gimple_cond_code (last
);
1346 inverted_code
= invert_tree_comparison (code
, HONOR_NANS (op
));
1348 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1350 enum tree_code this_code
= (e
->flags
& EDGE_TRUE_VALUE
1351 ? code
: inverted_code
);
1352 /* invert_tree_comparison will return ERROR_MARK on FP
1353 comparsions that are not EQ/NE instead of returning proper
1354 unordered one. Be sure it is not confused with NON_CONSTANT.
1356 And if the edge's target is the final block of diamond CFG graph
1357 of this conditional statement, we do not need to compute
1358 predicate for the edge because the final block's predicate must
1359 be at least as that of the first block of the statement. */
1360 if (this_code
!= ERROR_MARK
1361 && !dominated_by_p (CDI_POST_DOMINATORS
, bb
, e
->dest
))
1364 = add_condition (summary
, params_summary
, index
,
1365 param_type
, &aggpos
,
1366 this_code
, gimple_cond_rhs (last
), param_ops
);
1367 e
->aux
= edge_predicate_pool
.allocate ();
1368 *(predicate
*) e
->aux
= p
;
1371 vec_free (param_ops
);
1374 if (TREE_CODE (op
) != SSA_NAME
)
1377 if (builtin_constant_p (op))
1381 Here we can predicate nonconstant_code. We can't
1382 really handle constant_code since we have no predicate
1383 for this and also the constant code is not known to be
1384 optimized away when inliner doen't see operand is constant.
1385 Other optimizers might think otherwise. */
1386 if (gimple_cond_code (last
) != NE_EXPR
1387 || !integer_zerop (gimple_cond_rhs (last
)))
1389 set_stmt
= SSA_NAME_DEF_STMT (op
);
1390 if (!gimple_call_builtin_p (set_stmt
, BUILT_IN_CONSTANT_P
)
1391 || gimple_call_num_args (set_stmt
) != 1)
1393 op2
= gimple_call_arg (set_stmt
, 0);
1394 if (!decompose_param_expr (fbi
, set_stmt
, op2
, &index
, ¶m_type
, &aggpos
))
1396 FOR_EACH_EDGE (e
, ei
, bb
->succs
) if (e
->flags
& EDGE_FALSE_VALUE
)
1398 predicate p
= add_condition (summary
, params_summary
, index
,
1399 param_type
, &aggpos
,
1400 predicate::is_not_constant
, NULL_TREE
);
1401 e
->aux
= edge_predicate_pool
.allocate ();
1402 *(predicate
*) e
->aux
= p
;
1407 /* If BB ends by a switch we can turn into predicates, attach corresponding
1408 predicates to the CFG edges. */
1411 set_switch_stmt_execution_predicate (struct ipa_func_body_info
*fbi
,
1412 class ipa_fn_summary
*summary
,
1413 class ipa_node_params
*params_summary
,
1419 struct agg_position_info aggpos
;
1425 expr_eval_ops param_ops
;
1427 lastg
= last_stmt (bb
);
1428 if (!lastg
|| gimple_code (lastg
) != GIMPLE_SWITCH
)
1430 gswitch
*last
= as_a
<gswitch
*> (lastg
);
1431 op
= gimple_switch_index (last
);
1432 if (!decompose_param_expr (fbi
, last
, op
, &index
, ¶m_type
, &aggpos
,
1436 auto_vec
<std::pair
<tree
, tree
> > ranges
;
1437 tree type
= TREE_TYPE (op
);
1438 int bound_limit
= param_ipa_max_switch_predicate_bounds
;
1439 int bound_count
= 0;
1440 wide_int vr_wmin
, vr_wmax
;
1441 value_range_kind vr_type
= get_range_info (op
, &vr_wmin
, &vr_wmax
);
1443 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1445 e
->aux
= edge_predicate_pool
.allocate ();
1446 *(predicate
*) e
->aux
= false;
1449 e
= gimple_switch_edge (cfun
, last
, 0);
1450 /* Set BOUND_COUNT to maximum count to bypass computing predicate for
1451 default case if its target basic block is in convergence point of all
1452 switch cases, which can be determined by checking whether it
1453 post-dominates the switch statement. */
1454 if (dominated_by_p (CDI_POST_DOMINATORS
, bb
, e
->dest
))
1455 bound_count
= INT_MAX
;
1457 n
= gimple_switch_num_labels (last
);
1458 for (case_idx
= 1; case_idx
< n
; ++case_idx
)
1460 tree cl
= gimple_switch_label (last
, case_idx
);
1461 tree min
= CASE_LOW (cl
);
1462 tree max
= CASE_HIGH (cl
);
1465 e
= gimple_switch_edge (cfun
, last
, case_idx
);
1467 /* The case value might not have same type as switch expression,
1468 extend the value based on the expression type. */
1469 if (TREE_TYPE (min
) != type
)
1470 min
= wide_int_to_tree (type
, wi::to_wide (min
));
1474 else if (TREE_TYPE (max
) != type
)
1475 max
= wide_int_to_tree (type
, wi::to_wide (max
));
1477 /* The case's target basic block is in convergence point of all switch
1478 cases, its predicate should be at least as that of the switch
1480 if (dominated_by_p (CDI_POST_DOMINATORS
, bb
, e
->dest
))
1482 else if (min
== max
)
1483 p
= add_condition (summary
, params_summary
, index
, param_type
,
1484 &aggpos
, EQ_EXPR
, min
, param_ops
);
1488 p1
= add_condition (summary
, params_summary
, index
, param_type
,
1489 &aggpos
, GE_EXPR
, min
, param_ops
);
1490 p2
= add_condition (summary
, params_summary
,index
, param_type
,
1491 &aggpos
, LE_EXPR
, max
, param_ops
);
1494 *(class predicate
*) e
->aux
1495 = p
.or_with (summary
->conds
, *(class predicate
*) e
->aux
);
1497 /* If there are too many disjoint case ranges, predicate for default
1498 case might become too complicated. So add a limit here. */
1499 if (bound_count
> bound_limit
)
1502 bool new_range
= true;
1504 if (!ranges
.is_empty ())
1506 wide_int curr_wmin
= wi::to_wide (min
);
1507 wide_int last_wmax
= wi::to_wide (ranges
.last ().second
);
1509 /* Merge case ranges if they are continuous. */
1510 if (curr_wmin
== last_wmax
+ 1)
1512 else if (vr_type
== VR_ANTI_RANGE
)
1514 /* If two disjoint case ranges can be connected by anti-range
1515 of switch index, combine them to one range. */
1516 if (wi::lt_p (vr_wmax
, curr_wmin
- 1, TYPE_SIGN (type
)))
1517 vr_type
= VR_UNDEFINED
;
1518 else if (wi::le_p (vr_wmin
, last_wmax
+ 1, TYPE_SIGN (type
)))
1523 /* Create/extend a case range. And we count endpoints of range set,
1524 this number nearly equals to number of conditions that we will create
1525 for predicate of default case. */
1528 bound_count
+= (min
== max
) ? 1 : 2;
1529 ranges
.safe_push (std::make_pair (min
, max
));
1533 bound_count
+= (ranges
.last ().first
== ranges
.last ().second
);
1534 ranges
.last ().second
= max
;
1538 e
= gimple_switch_edge (cfun
, last
, 0);
1539 if (bound_count
> bound_limit
)
1541 *(class predicate
*) e
->aux
= true;
1542 vec_free (param_ops
);
1546 predicate p_seg
= true;
1547 predicate p_all
= false;
1549 if (vr_type
!= VR_RANGE
)
1551 vr_wmin
= wi::to_wide (TYPE_MIN_VALUE (type
));
1552 vr_wmax
= wi::to_wide (TYPE_MAX_VALUE (type
));
1555 /* Construct predicate to represent default range set that is negation of
1556 all case ranges. Case range is classified as containing single/non-single
1557 values. Suppose a piece of case ranges in the following.
1559 [D1...D2] [S1] ... [Sn] [D3...D4]
1561 To represent default case's range sets between two non-single value
1562 case ranges (From D2 to D3), we construct predicate as:
1564 D2 < x < D3 && x != S1 && ... && x != Sn
1566 for (size_t i
= 0; i
< ranges
.length (); i
++)
1568 tree min
= ranges
[i
].first
;
1569 tree max
= ranges
[i
].second
;
1572 p_seg
&= add_condition (summary
, params_summary
, index
,
1573 param_type
, &aggpos
, NE_EXPR
,
1577 /* Do not create sub-predicate for range that is beyond low bound
1579 if (wi::lt_p (vr_wmin
, wi::to_wide (min
), TYPE_SIGN (type
)))
1581 p_seg
&= add_condition (summary
, params_summary
, index
,
1582 param_type
, &aggpos
,
1583 LT_EXPR
, min
, param_ops
);
1584 p_all
= p_all
.or_with (summary
->conds
, p_seg
);
1587 /* Do not create sub-predicate for range that is beyond up bound
1589 if (wi::le_p (vr_wmax
, wi::to_wide (max
), TYPE_SIGN (type
)))
1595 p_seg
= add_condition (summary
, params_summary
, index
,
1596 param_type
, &aggpos
, GT_EXPR
,
1601 p_all
= p_all
.or_with (summary
->conds
, p_seg
);
1602 *(class predicate
*) e
->aux
1603 = p_all
.or_with (summary
->conds
, *(class predicate
*) e
->aux
);
1605 vec_free (param_ops
);
1609 /* For each BB in NODE attach to its AUX pointer predicate under
1610 which it is executable. */
1613 compute_bb_predicates (struct ipa_func_body_info
*fbi
,
1614 struct cgraph_node
*node
,
1615 class ipa_fn_summary
*summary
,
1616 class ipa_node_params
*params_summary
)
1618 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->decl
);
1622 FOR_EACH_BB_FN (bb
, my_function
)
1624 set_cond_stmt_execution_predicate (fbi
, summary
, params_summary
, bb
);
1625 set_switch_stmt_execution_predicate (fbi
, summary
, params_summary
, bb
);
1628 /* Entry block is always executable. */
1629 ENTRY_BLOCK_PTR_FOR_FN (my_function
)->aux
1630 = edge_predicate_pool
.allocate ();
1631 *(predicate
*) ENTRY_BLOCK_PTR_FOR_FN (my_function
)->aux
= true;
1633 /* A simple dataflow propagation of predicates forward in the CFG.
1634 TODO: work in reverse postorder. */
1638 FOR_EACH_BB_FN (bb
, my_function
)
1640 predicate p
= false;
1643 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
1647 predicate this_bb_predicate
1648 = *(predicate
*) e
->src
->aux
;
1650 this_bb_predicate
&= (*(class predicate
*) e
->aux
);
1651 p
= p
.or_with (summary
->conds
, this_bb_predicate
);
1658 basic_block pdom_bb
;
1663 bb
->aux
= edge_predicate_pool
.allocate ();
1664 *((predicate
*) bb
->aux
) = p
;
1666 else if (p
!= *(predicate
*) bb
->aux
)
1668 /* This OR operation is needed to ensure monotonous data flow
1669 in the case we hit the limit on number of clauses and the
1670 and/or operations above give approximate answers. */
1671 p
= p
.or_with (summary
->conds
, *(predicate
*)bb
->aux
);
1672 if (p
!= *(predicate
*) bb
->aux
)
1675 *((predicate
*) bb
->aux
) = p
;
1679 /* For switch/if statement, we can OR-combine predicates of all
1680 its cases/branches to get predicate for basic block in their
1681 convergence point, but sometimes this will generate very
1682 complicated predicate. Actually, we can get simplified
1683 predicate in another way by using the fact that predicate
1684 for a basic block must also hold true for its post dominators.
1685 To be specific, basic block in convergence point of
1686 conditional statement should include predicate of the
1688 pdom_bb
= get_immediate_dominator (CDI_POST_DOMINATORS
, bb
);
1689 if (pdom_bb
== EXIT_BLOCK_PTR_FOR_FN (my_function
) || !pdom_bb
)
1691 else if (!pdom_bb
->aux
)
1694 pdom_bb
->aux
= edge_predicate_pool
.allocate ();
1695 *((predicate
*) pdom_bb
->aux
) = p
;
1697 else if (p
!= *(predicate
*) pdom_bb
->aux
)
1699 p
= p
.or_with (summary
->conds
, *(predicate
*)pdom_bb
->aux
);
1700 if (p
!= *(predicate
*) pdom_bb
->aux
)
1703 *((predicate
*) pdom_bb
->aux
) = p
;
1712 /* Return predicate specifying when the STMT might have result that is not
1713 a compile time constant. */
1716 will_be_nonconstant_expr_predicate (ipa_func_body_info
*fbi
,
1717 class ipa_fn_summary
*summary
,
1718 class ipa_node_params
*params_summary
,
1720 vec
<predicate
> nonconstant_names
)
1725 while (UNARY_CLASS_P (expr
))
1726 expr
= TREE_OPERAND (expr
, 0);
1728 parm
= unmodified_parm (fbi
, NULL
, expr
, NULL
);
1729 if (parm
&& (index
= ipa_get_param_decl_index (fbi
->info
, parm
)) >= 0)
1730 return add_condition (summary
, params_summary
, index
, TREE_TYPE (parm
), NULL
,
1731 predicate::changed
, NULL_TREE
);
1732 if (is_gimple_min_invariant (expr
))
1734 if (TREE_CODE (expr
) == SSA_NAME
)
1735 return nonconstant_names
[SSA_NAME_VERSION (expr
)];
1736 if (BINARY_CLASS_P (expr
) || COMPARISON_CLASS_P (expr
))
1739 = will_be_nonconstant_expr_predicate (fbi
, summary
,
1741 TREE_OPERAND (expr
, 0),
1747 = will_be_nonconstant_expr_predicate (fbi
, summary
,
1749 TREE_OPERAND (expr
, 1),
1751 return p1
.or_with (summary
->conds
, p2
);
1753 else if (TREE_CODE (expr
) == COND_EXPR
)
1756 = will_be_nonconstant_expr_predicate (fbi
, summary
,
1758 TREE_OPERAND (expr
, 0),
1764 = will_be_nonconstant_expr_predicate (fbi
, summary
,
1766 TREE_OPERAND (expr
, 1),
1770 p1
= p1
.or_with (summary
->conds
, p2
);
1771 p2
= will_be_nonconstant_expr_predicate (fbi
, summary
,
1773 TREE_OPERAND (expr
, 2),
1775 return p2
.or_with (summary
->conds
, p1
);
1777 else if (TREE_CODE (expr
) == CALL_EXPR
)
1788 /* Return predicate specifying when the STMT might have result that is not
1789 a compile time constant. */
1792 will_be_nonconstant_predicate (struct ipa_func_body_info
*fbi
,
1793 class ipa_fn_summary
*summary
,
1794 class ipa_node_params
*params_summary
,
1796 vec
<predicate
> nonconstant_names
)
1801 tree param_type
= NULL_TREE
;
1802 predicate op_non_const
;
1805 struct agg_position_info aggpos
;
1807 /* What statments might be optimized away
1808 when their arguments are constant. */
1809 if (gimple_code (stmt
) != GIMPLE_ASSIGN
1810 && gimple_code (stmt
) != GIMPLE_COND
1811 && gimple_code (stmt
) != GIMPLE_SWITCH
1812 && (gimple_code (stmt
) != GIMPLE_CALL
1813 || !(gimple_call_flags (stmt
) & ECF_CONST
)))
1816 /* Stores will stay anyway. */
1817 if (gimple_store_p (stmt
))
1820 is_load
= gimple_assign_load_p (stmt
);
1822 /* Loads can be optimized when the value is known. */
1825 tree op
= gimple_assign_rhs1 (stmt
);
1826 if (!decompose_param_expr (fbi
, stmt
, op
, &base_index
, ¶m_type
,
1833 /* See if we understand all operands before we start
1834 adding conditionals. */
1835 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
1837 tree parm
= unmodified_parm (fbi
, stmt
, use
, NULL
);
1838 /* For arguments we can build a condition. */
1839 if (parm
&& ipa_get_param_decl_index (fbi
->info
, parm
) >= 0)
1841 if (TREE_CODE (use
) != SSA_NAME
)
1843 /* If we know when operand is constant,
1844 we still can say something useful. */
1845 if (nonconstant_names
[SSA_NAME_VERSION (use
)] != true)
1852 add_condition (summary
, params_summary
,
1853 base_index
, param_type
, &aggpos
,
1854 predicate::changed
, NULL_TREE
);
1856 op_non_const
= false;
1857 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
1859 tree parm
= unmodified_parm (fbi
, stmt
, use
, NULL
);
1862 if (parm
&& (index
= ipa_get_param_decl_index (fbi
->info
, parm
)) >= 0)
1864 if (index
!= base_index
)
1865 p
= add_condition (summary
, params_summary
, index
,
1866 TREE_TYPE (parm
), NULL
,
1867 predicate::changed
, NULL_TREE
);
1872 p
= nonconstant_names
[SSA_NAME_VERSION (use
)];
1873 op_non_const
= p
.or_with (summary
->conds
, op_non_const
);
1875 if ((gimple_code (stmt
) == GIMPLE_ASSIGN
|| gimple_code (stmt
) == GIMPLE_CALL
)
1876 && gimple_op (stmt
, 0)
1877 && TREE_CODE (gimple_op (stmt
, 0)) == SSA_NAME
)
1878 nonconstant_names
[SSA_NAME_VERSION (gimple_op (stmt
, 0))]
1880 return op_non_const
;
1883 struct record_modified_bb_info
1890 /* Value is initialized in INIT_BB and used in USE_BB. We want to copute
1891 probability how often it changes between USE_BB.
1892 INIT_BB->count/USE_BB->count is an estimate, but if INIT_BB
1893 is in different loop nest, we can do better.
1894 This is all just estimate. In theory we look for minimal cut separating
1895 INIT_BB and USE_BB, but we only want to anticipate loop invariant motion
1899 get_minimal_bb (basic_block init_bb
, basic_block use_bb
)
1901 class loop
*l
= find_common_loop (init_bb
->loop_father
, use_bb
->loop_father
);
1902 if (l
&& l
->header
->count
< init_bb
->count
)
1907 /* Callback of walk_aliased_vdefs. Records basic blocks where the value may be
1908 set except for info->stmt. */
1911 record_modified (ao_ref
*ao ATTRIBUTE_UNUSED
, tree vdef
, void *data
)
1913 struct record_modified_bb_info
*info
=
1914 (struct record_modified_bb_info
*) data
;
1915 if (SSA_NAME_DEF_STMT (vdef
) == info
->stmt
)
1917 if (gimple_clobber_p (SSA_NAME_DEF_STMT (vdef
)))
1919 bitmap_set_bit (info
->bb_set
,
1920 SSA_NAME_IS_DEFAULT_DEF (vdef
)
1921 ? ENTRY_BLOCK_PTR_FOR_FN (cfun
)->index
1923 (gimple_bb (SSA_NAME_DEF_STMT (vdef
)),
1924 gimple_bb (info
->stmt
))->index
);
1927 fprintf (dump_file
, " Param ");
1928 print_generic_expr (dump_file
, info
->op
, TDF_SLIM
);
1929 fprintf (dump_file
, " changed at bb %i, minimal: %i stmt: ",
1930 gimple_bb (SSA_NAME_DEF_STMT (vdef
))->index
,
1932 (gimple_bb (SSA_NAME_DEF_STMT (vdef
)),
1933 gimple_bb (info
->stmt
))->index
);
1934 print_gimple_stmt (dump_file
, SSA_NAME_DEF_STMT (vdef
), 0);
1939 /* Return probability (based on REG_BR_PROB_BASE) that I-th parameter of STMT
1940 will change since last invocation of STMT.
1942 Value 0 is reserved for compile time invariants.
1943 For common parameters it is REG_BR_PROB_BASE. For loop invariants it
1944 ought to be REG_BR_PROB_BASE / estimated_iters. */
1947 param_change_prob (ipa_func_body_info
*fbi
, gimple
*stmt
, int i
)
1949 tree op
= gimple_call_arg (stmt
, i
);
1950 basic_block bb
= gimple_bb (stmt
);
1952 if (TREE_CODE (op
) == WITH_SIZE_EXPR
)
1953 op
= TREE_OPERAND (op
, 0);
1955 tree base
= get_base_address (op
);
1957 /* Global invariants never change. */
1958 if (is_gimple_min_invariant (base
))
1961 /* We would have to do non-trivial analysis to really work out what
1962 is the probability of value to change (i.e. when init statement
1963 is in a sibling loop of the call).
1965 We do an conservative estimate: when call is executed N times more often
1966 than the statement defining value, we take the frequency 1/N. */
1967 if (TREE_CODE (base
) == SSA_NAME
)
1969 profile_count init_count
;
1971 if (!bb
->count
.nonzero_p ())
1972 return REG_BR_PROB_BASE
;
1974 if (SSA_NAME_IS_DEFAULT_DEF (base
))
1975 init_count
= ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
;
1977 init_count
= get_minimal_bb
1978 (gimple_bb (SSA_NAME_DEF_STMT (base
)),
1979 gimple_bb (stmt
))->count
;
1981 if (init_count
< bb
->count
)
1982 return MAX ((init_count
.to_sreal_scale (bb
->count
)
1983 * REG_BR_PROB_BASE
).to_int (), 1);
1984 return REG_BR_PROB_BASE
;
1989 profile_count max
= ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
;
1990 struct record_modified_bb_info info
;
1991 tree init
= ctor_for_folding (base
);
1993 if (init
!= error_mark_node
)
1995 if (!bb
->count
.nonzero_p ())
1996 return REG_BR_PROB_BASE
;
1999 fprintf (dump_file
, " Analyzing param change probability of ");
2000 print_generic_expr (dump_file
, op
, TDF_SLIM
);
2001 fprintf (dump_file
, "\n");
2003 ao_ref_init (&refd
, op
);
2006 info
.bb_set
= BITMAP_ALLOC (NULL
);
2008 = walk_aliased_vdefs (&refd
, gimple_vuse (stmt
), record_modified
, &info
,
2009 NULL
, NULL
, fbi
->aa_walk_budget
);
2010 if (walked
< 0 || bitmap_bit_p (info
.bb_set
, bb
->index
))
2015 fprintf (dump_file
, " Ran out of AA walking budget.\n");
2017 fprintf (dump_file
, " Set in same BB as used.\n");
2019 BITMAP_FREE (info
.bb_set
);
2020 return REG_BR_PROB_BASE
;
2025 /* Lookup the most frequent update of the value and believe that
2026 it dominates all the other; precise analysis here is difficult. */
2027 EXECUTE_IF_SET_IN_BITMAP (info
.bb_set
, 0, index
, bi
)
2028 max
= max
.max (BASIC_BLOCK_FOR_FN (cfun
, index
)->count
);
2031 fprintf (dump_file
, " Set with count ");
2032 max
.dump (dump_file
);
2033 fprintf (dump_file
, " and used with count ");
2034 bb
->count
.dump (dump_file
);
2035 fprintf (dump_file
, " freq %f\n",
2036 max
.to_sreal_scale (bb
->count
).to_double ());
2039 BITMAP_FREE (info
.bb_set
);
2040 if (max
< bb
->count
)
2041 return MAX ((max
.to_sreal_scale (bb
->count
)
2042 * REG_BR_PROB_BASE
).to_int (), 1);
2043 return REG_BR_PROB_BASE
;
2047 /* Find whether a basic block BB is the final block of a (half) diamond CFG
2048 sub-graph and if the predicate the condition depends on is known. If so,
2049 return true and store the pointer the predicate in *P. */
2052 phi_result_unknown_predicate (ipa_func_body_info
*fbi
,
2053 ipa_fn_summary
*summary
,
2054 class ipa_node_params
*params_summary
,
2057 vec
<predicate
> nonconstant_names
)
2061 basic_block first_bb
= NULL
;
2064 if (single_pred_p (bb
))
2070 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
2072 if (single_succ_p (e
->src
))
2074 if (!single_pred_p (e
->src
))
2077 first_bb
= single_pred (e
->src
);
2078 else if (single_pred (e
->src
) != first_bb
)
2085 else if (e
->src
!= first_bb
)
2093 stmt
= last_stmt (first_bb
);
2095 || gimple_code (stmt
) != GIMPLE_COND
2096 || !is_gimple_ip_invariant (gimple_cond_rhs (stmt
)))
2099 *p
= will_be_nonconstant_expr_predicate (fbi
, summary
, params_summary
,
2100 gimple_cond_lhs (stmt
),
2108 /* Given a PHI statement in a function described by inline properties SUMMARY
2109 and *P being the predicate describing whether the selected PHI argument is
2110 known, store a predicate for the result of the PHI statement into
2111 NONCONSTANT_NAMES, if possible. */
2114 predicate_for_phi_result (class ipa_fn_summary
*summary
, gphi
*phi
,
2116 vec
<predicate
> nonconstant_names
)
2120 for (i
= 0; i
< gimple_phi_num_args (phi
); i
++)
2122 tree arg
= gimple_phi_arg (phi
, i
)->def
;
2123 if (!is_gimple_min_invariant (arg
))
2125 gcc_assert (TREE_CODE (arg
) == SSA_NAME
);
2126 *p
= p
->or_with (summary
->conds
,
2127 nonconstant_names
[SSA_NAME_VERSION (arg
)]);
2133 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2135 fprintf (dump_file
, "\t\tphi predicate: ");
2136 p
->dump (dump_file
, summary
->conds
);
2138 nonconstant_names
[SSA_NAME_VERSION (gimple_phi_result (phi
))] = *p
;
2141 /* For a typical usage of __builtin_expect (a<b, 1), we
2142 may introduce an extra relation stmt:
2143 With the builtin, we have
2146 t3 = __builtin_expect (t2, 1);
2149 Without the builtin, we have
2152 This affects the size/time estimation and may have
2153 an impact on the earlier inlining.
2154 Here find this pattern and fix it up later. */
2157 find_foldable_builtin_expect (basic_block bb
)
2159 gimple_stmt_iterator bsi
;
2161 for (bsi
= gsi_start_bb (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
2163 gimple
*stmt
= gsi_stmt (bsi
);
2164 if (gimple_call_builtin_p (stmt
, BUILT_IN_EXPECT
)
2165 || gimple_call_builtin_p (stmt
, BUILT_IN_EXPECT_WITH_PROBABILITY
)
2166 || gimple_call_internal_p (stmt
, IFN_BUILTIN_EXPECT
))
2168 tree var
= gimple_call_lhs (stmt
);
2169 tree arg
= gimple_call_arg (stmt
, 0);
2170 use_operand_p use_p
;
2177 gcc_assert (TREE_CODE (var
) == SSA_NAME
);
2179 while (TREE_CODE (arg
) == SSA_NAME
)
2181 gimple
*stmt_tmp
= SSA_NAME_DEF_STMT (arg
);
2182 if (!is_gimple_assign (stmt_tmp
))
2184 switch (gimple_assign_rhs_code (stmt_tmp
))
2203 arg
= gimple_assign_rhs1 (stmt_tmp
);
2206 if (match
&& single_imm_use (var
, &use_p
, &use_stmt
)
2207 && gimple_code (use_stmt
) == GIMPLE_COND
)
2214 /* Return true when the basic blocks contains only clobbers followed by RESX.
2215 Such BBs are kept around to make removal of dead stores possible with
2216 presence of EH and will be optimized out by optimize_clobbers later in the
2219 NEED_EH is used to recurse in case the clobber has non-EH predecestors
2220 that can be clobber only, too.. When it is false, the RESX is not necessary
2221 on the end of basic block. */
2224 clobber_only_eh_bb_p (basic_block bb
, bool need_eh
= true)
2226 gimple_stmt_iterator gsi
= gsi_last_bb (bb
);
2232 if (gsi_end_p (gsi
))
2234 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_RESX
)
2238 else if (!single_succ_p (bb
))
2241 for (; !gsi_end_p (gsi
); gsi_prev (&gsi
))
2243 gimple
*stmt
= gsi_stmt (gsi
);
2244 if (is_gimple_debug (stmt
))
2246 if (gimple_clobber_p (stmt
))
2248 if (gimple_code (stmt
) == GIMPLE_LABEL
)
2253 /* See if all predecestors are either throws or clobber only BBs. */
2254 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
2255 if (!(e
->flags
& EDGE_EH
)
2256 && !clobber_only_eh_bb_p (e
->src
, false))
2262 /* Return true if STMT compute a floating point expression that may be affected
2263 by -ffast-math and similar flags. */
2266 fp_expression_p (gimple
*stmt
)
2271 FOR_EACH_SSA_TREE_OPERAND (op
, stmt
, i
, SSA_OP_DEF
|SSA_OP_USE
)
2272 if (FLOAT_TYPE_P (TREE_TYPE (op
)))
2277 /* Analyze function body for NODE.
2278 EARLY indicates run from early optimization pipeline. */
2281 analyze_function_body (struct cgraph_node
*node
, bool early
)
2283 sreal time
= param_uninlined_function_time
;
2284 /* Estimate static overhead for function prologue/epilogue and alignment. */
2285 int size
= param_uninlined_function_insns
;
2286 /* Benefits are scaled by probability of elimination that is in range
2289 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->decl
);
2291 class ipa_fn_summary
*info
= ipa_fn_summaries
->get_create (node
);
2292 class ipa_node_params
*params_summary
= early
? NULL
: IPA_NODE_REF (node
);
2293 predicate bb_predicate
;
2294 struct ipa_func_body_info fbi
;
2295 vec
<predicate
> nonconstant_names
= vNULL
;
2298 gimple
*fix_builtin_expect_stmt
;
2300 gcc_assert (my_function
&& my_function
->cfg
);
2301 gcc_assert (cfun
== my_function
);
2303 memset(&fbi
, 0, sizeof(fbi
));
2304 vec_free (info
->conds
);
2306 vec_free (info
->size_time_table
);
2307 info
->size_time_table
= NULL
;
2309 /* When optimizing and analyzing for IPA inliner, initialize loop optimizer
2310 so we can produce proper inline hints.
2312 When optimizing and analyzing for early inliner, initialize node params
2313 so we can produce correct BB predicates. */
2315 if (opt_for_fn (node
->decl
, optimize
))
2317 calculate_dominance_info (CDI_DOMINATORS
);
2318 calculate_dominance_info (CDI_POST_DOMINATORS
);
2320 loop_optimizer_init (LOOPS_NORMAL
| LOOPS_HAVE_RECORDED_EXITS
);
2323 ipa_check_create_node_params ();
2324 ipa_initialize_node_params (node
);
2327 if (ipa_node_params_sum
)
2330 fbi
.info
= IPA_NODE_REF (node
);
2331 fbi
.bb_infos
= vNULL
;
2332 fbi
.bb_infos
.safe_grow_cleared (last_basic_block_for_fn (cfun
));
2333 fbi
.param_count
= count_formal_params (node
->decl
);
2334 fbi
.aa_walk_budget
= param_ipa_max_aa_steps
;
2336 nonconstant_names
.safe_grow_cleared
2337 (SSANAMES (my_function
)->length ());
2342 fprintf (dump_file
, "\nAnalyzing function body size: %s\n",
2345 /* When we run into maximal number of entries, we assign everything to the
2346 constant truth case. Be sure to have it in list. */
2347 bb_predicate
= true;
2348 info
->account_size_time (0, 0, bb_predicate
, bb_predicate
);
2350 bb_predicate
= predicate::not_inlined ();
2351 info
->account_size_time (param_uninlined_function_insns
2352 * ipa_fn_summary::size_scale
,
2353 param_uninlined_function_time
,
2358 compute_bb_predicates (&fbi
, node
, info
, params_summary
);
2359 order
= XNEWVEC (int, n_basic_blocks_for_fn (cfun
));
2360 nblocks
= pre_and_rev_post_order_compute (NULL
, order
, false);
2361 for (n
= 0; n
< nblocks
; n
++)
2363 bb
= BASIC_BLOCK_FOR_FN (cfun
, order
[n
]);
2364 freq
= bb
->count
.to_sreal_scale (ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
);
2365 if (clobber_only_eh_bb_p (bb
))
2367 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2368 fprintf (dump_file
, "\n Ignoring BB %i;"
2369 " it will be optimized away by cleanup_clobbers\n",
2374 /* TODO: Obviously predicates can be propagated down across CFG. */
2378 bb_predicate
= *(predicate
*) bb
->aux
;
2380 bb_predicate
= false;
2383 bb_predicate
= true;
2385 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2387 fprintf (dump_file
, "\n BB %i predicate:", bb
->index
);
2388 bb_predicate
.dump (dump_file
, info
->conds
);
2391 if (fbi
.info
&& nonconstant_names
.exists ())
2393 predicate phi_predicate
;
2394 bool first_phi
= true;
2396 for (gphi_iterator bsi
= gsi_start_phis (bb
); !gsi_end_p (bsi
);
2400 && !phi_result_unknown_predicate (&fbi
, info
,
2407 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2409 fprintf (dump_file
, " ");
2410 print_gimple_stmt (dump_file
, gsi_stmt (bsi
), 0);
2412 predicate_for_phi_result (info
, bsi
.phi (), &phi_predicate
,
2417 fix_builtin_expect_stmt
= find_foldable_builtin_expect (bb
);
2419 for (gimple_stmt_iterator bsi
= gsi_start_nondebug_bb (bb
);
2420 !gsi_end_p (bsi
); gsi_next_nondebug (&bsi
))
2422 gimple
*stmt
= gsi_stmt (bsi
);
2423 int this_size
= estimate_num_insns (stmt
, &eni_size_weights
);
2424 int this_time
= estimate_num_insns (stmt
, &eni_time_weights
);
2426 predicate will_be_nonconstant
;
2428 /* This relation stmt should be folded after we remove
2429 buildin_expect call. Adjust the cost here. */
2430 if (stmt
== fix_builtin_expect_stmt
)
2436 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2438 fprintf (dump_file
, " ");
2439 print_gimple_stmt (dump_file
, stmt
, 0);
2440 fprintf (dump_file
, "\t\tfreq:%3.2f size:%3i time:%3i\n",
2441 freq
.to_double (), this_size
,
2445 if (is_gimple_call (stmt
)
2446 && !gimple_call_internal_p (stmt
))
2448 struct cgraph_edge
*edge
= node
->get_edge (stmt
);
2449 ipa_call_summary
*es
= ipa_call_summaries
->get_create (edge
);
2451 /* Special case: results of BUILT_IN_CONSTANT_P will be always
2452 resolved as constant. We however don't want to optimize
2453 out the cgraph edges. */
2454 if (nonconstant_names
.exists ()
2455 && gimple_call_builtin_p (stmt
, BUILT_IN_CONSTANT_P
)
2456 && gimple_call_lhs (stmt
)
2457 && TREE_CODE (gimple_call_lhs (stmt
)) == SSA_NAME
)
2459 predicate false_p
= false;
2460 nonconstant_names
[SSA_NAME_VERSION (gimple_call_lhs (stmt
))]
2463 if (ipa_node_params_sum
)
2465 int count
= gimple_call_num_args (stmt
);
2469 es
->param
.safe_grow_cleared (count
);
2470 for (i
= 0; i
< count
; i
++)
2472 int prob
= param_change_prob (&fbi
, stmt
, i
);
2473 gcc_assert (prob
>= 0 && prob
<= REG_BR_PROB_BASE
);
2474 es
->param
[i
].change_prob
= prob
;
2478 es
->call_stmt_size
= this_size
;
2479 es
->call_stmt_time
= this_time
;
2480 es
->loop_depth
= bb_loop_depth (bb
);
2481 edge_set_predicate (edge
, &bb_predicate
);
2482 if (edge
->speculative
)
2484 cgraph_edge
*direct
, *indirect
;
2486 edge
->speculative_call_info (direct
, indirect
, ref
);
2487 gcc_assert (direct
== edge
);
2488 ipa_call_summary
*es2
2489 = ipa_call_summaries
->get_create (indirect
);
2490 ipa_call_summaries
->duplicate (edge
, indirect
,
2495 /* TODO: When conditional jump or swithc is known to be constant, but
2496 we did not translate it into the predicates, we really can account
2497 just maximum of the possible paths. */
2500 = will_be_nonconstant_predicate (&fbi
, info
, params_summary
,
2501 stmt
, nonconstant_names
);
2503 will_be_nonconstant
= true;
2504 if (this_time
|| this_size
)
2506 sreal final_time
= (sreal
)this_time
* freq
;
2508 prob
= eliminated_by_inlining_prob (&fbi
, stmt
);
2509 if (prob
== 1 && dump_file
&& (dump_flags
& TDF_DETAILS
))
2511 "\t\t50%% will be eliminated by inlining\n");
2512 if (prob
== 2 && dump_file
&& (dump_flags
& TDF_DETAILS
))
2513 fprintf (dump_file
, "\t\tWill be eliminated by inlining\n");
2515 class predicate p
= bb_predicate
& will_be_nonconstant
;
2517 /* We can ignore statement when we proved it is never going
2518 to happen, but we cannot do that for call statements
2519 because edges are accounted specially. */
2521 if (*(is_gimple_call (stmt
) ? &bb_predicate
: &p
) != false)
2527 /* We account everything but the calls. Calls have their own
2528 size/time info attached to cgraph edges. This is necessary
2529 in order to make the cost disappear after inlining. */
2530 if (!is_gimple_call (stmt
))
2534 predicate ip
= bb_predicate
& predicate::not_inlined ();
2535 info
->account_size_time (this_size
* prob
,
2536 (final_time
* prob
) / 2, ip
,
2540 info
->account_size_time (this_size
* (2 - prob
),
2541 (final_time
* (2 - prob
) / 2),
2546 if (!info
->fp_expressions
&& fp_expression_p (stmt
))
2548 info
->fp_expressions
= true;
2550 fprintf (dump_file
, " fp_expression set\n");
2554 /* Account cost of address calculations in the statements. */
2555 for (unsigned int i
= 0; i
< gimple_num_ops (stmt
); i
++)
2557 for (tree op
= gimple_op (stmt
, i
);
2558 op
&& handled_component_p (op
);
2559 op
= TREE_OPERAND (op
, 0))
2560 if ((TREE_CODE (op
) == ARRAY_REF
2561 || TREE_CODE (op
) == ARRAY_RANGE_REF
)
2562 && TREE_CODE (TREE_OPERAND (op
, 1)) == SSA_NAME
)
2564 predicate p
= bb_predicate
;
2566 p
= p
& will_be_nonconstant_expr_predicate
2567 (&fbi
, info
, params_summary
,
2568 TREE_OPERAND (op
, 1),
2576 "\t\tAccounting address calculation.\n");
2577 info
->account_size_time (ipa_fn_summary::size_scale
,
2589 if (nonconstant_names
.exists () && !early
)
2592 predicate loop_iterations
= true;
2593 predicate loop_stride
= true;
2595 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2596 flow_loops_dump (dump_file
, NULL
, 0);
2598 FOR_EACH_LOOP (loop
, 0)
2603 class tree_niter_desc niter_desc
;
2604 bb_predicate
= *(predicate
*) loop
->header
->aux
;
2606 exits
= get_loop_exit_edges (loop
);
2607 FOR_EACH_VEC_ELT (exits
, j
, ex
)
2608 if (number_of_iterations_exit (loop
, ex
, &niter_desc
, false)
2609 && !is_gimple_min_invariant (niter_desc
.niter
))
2611 predicate will_be_nonconstant
2612 = will_be_nonconstant_expr_predicate (&fbi
, info
,
2616 if (will_be_nonconstant
!= true)
2617 will_be_nonconstant
= bb_predicate
& will_be_nonconstant
;
2618 if (will_be_nonconstant
!= true
2619 && will_be_nonconstant
!= false)
2620 /* This is slightly inprecise. We may want to represent each
2621 loop with independent predicate. */
2622 loop_iterations
&= will_be_nonconstant
;
2627 /* To avoid quadratic behavior we analyze stride predicates only
2628 with respect to the containing loop. Thus we simply iterate
2629 over all defs in the outermost loop body. */
2630 for (loop
= loops_for_fn (cfun
)->tree_root
->inner
;
2631 loop
!= NULL
; loop
= loop
->next
)
2633 basic_block
*body
= get_loop_body (loop
);
2634 for (unsigned i
= 0; i
< loop
->num_nodes
; i
++)
2636 gimple_stmt_iterator gsi
;
2637 bb_predicate
= *(predicate
*) body
[i
]->aux
;
2638 for (gsi
= gsi_start_bb (body
[i
]); !gsi_end_p (gsi
);
2641 gimple
*stmt
= gsi_stmt (gsi
);
2643 if (!is_gimple_assign (stmt
))
2646 tree def
= gimple_assign_lhs (stmt
);
2647 if (TREE_CODE (def
) != SSA_NAME
)
2651 if (!simple_iv (loop_containing_stmt (stmt
),
2652 loop_containing_stmt (stmt
),
2654 || is_gimple_min_invariant (iv
.step
))
2657 predicate will_be_nonconstant
2658 = will_be_nonconstant_expr_predicate (&fbi
, info
,
2662 if (will_be_nonconstant
!= true)
2663 will_be_nonconstant
= bb_predicate
& will_be_nonconstant
;
2664 if (will_be_nonconstant
!= true
2665 && will_be_nonconstant
!= false)
2666 /* This is slightly inprecise. We may want to represent
2667 each loop with independent predicate. */
2668 loop_stride
= loop_stride
& will_be_nonconstant
;
2673 ipa_fn_summary
*s
= ipa_fn_summaries
->get (node
);
2674 set_hint_predicate (&s
->loop_iterations
, loop_iterations
);
2675 set_hint_predicate (&s
->loop_stride
, loop_stride
);
2678 FOR_ALL_BB_FN (bb
, my_function
)
2684 edge_predicate_pool
.remove ((predicate
*)bb
->aux
);
2686 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2689 edge_predicate_pool
.remove ((predicate
*) e
->aux
);
2693 ipa_fn_summary
*s
= ipa_fn_summaries
->get (node
);
2694 ipa_size_summary
*ss
= ipa_size_summaries
->get (node
);
2696 ss
->self_size
= size
;
2697 nonconstant_names
.release ();
2698 ipa_release_body_info (&fbi
);
2699 if (opt_for_fn (node
->decl
, optimize
))
2702 loop_optimizer_finalize ();
2703 else if (!ipa_edge_args_sum
)
2704 ipa_free_all_node_params ();
2705 free_dominance_info (CDI_DOMINATORS
);
2706 free_dominance_info (CDI_POST_DOMINATORS
);
2710 fprintf (dump_file
, "\n");
2711 ipa_dump_fn_summary (dump_file
, node
);
2716 /* Compute function summary.
2717 EARLY is true when we compute parameters during early opts. */
2720 compute_fn_summary (struct cgraph_node
*node
, bool early
)
2722 HOST_WIDE_INT self_stack_size
;
2723 struct cgraph_edge
*e
;
2725 gcc_assert (!node
->inlined_to
);
2727 if (!ipa_fn_summaries
)
2728 ipa_fn_summary_alloc ();
2730 /* Create a new ipa_fn_summary. */
2731 ((ipa_fn_summary_t
*)ipa_fn_summaries
)->remove_callees (node
);
2732 ipa_fn_summaries
->remove (node
);
2733 class ipa_fn_summary
*info
= ipa_fn_summaries
->get_create (node
);
2734 class ipa_size_summary
*size_info
= ipa_size_summaries
->get_create (node
);
2736 /* Estimate the stack size for the function if we're optimizing. */
2737 self_stack_size
= optimize
&& !node
->thunk
.thunk_p
2738 ? estimated_stack_frame_size (node
) : 0;
2739 size_info
->estimated_self_stack_size
= self_stack_size
;
2740 info
->estimated_stack_size
= self_stack_size
;
2742 if (node
->thunk
.thunk_p
)
2744 ipa_call_summary
*es
= ipa_call_summaries
->get_create (node
->callees
);
2747 node
->can_change_signature
= false;
2748 es
->call_stmt_size
= eni_size_weights
.call_cost
;
2749 es
->call_stmt_time
= eni_time_weights
.call_cost
;
2750 info
->account_size_time (ipa_fn_summary::size_scale
2751 * param_uninlined_function_thunk_insns
,
2752 param_uninlined_function_thunk_time
, t
, t
);
2753 t
= predicate::not_inlined ();
2754 info
->account_size_time (2 * ipa_fn_summary::size_scale
, 0, t
, t
);
2755 ipa_update_overall_fn_summary (node
);
2756 size_info
->self_size
= size_info
->size
;
2757 if (stdarg_p (TREE_TYPE (node
->decl
)))
2759 info
->inlinable
= false;
2760 node
->callees
->inline_failed
= CIF_VARIADIC_THUNK
;
2763 info
->inlinable
= true;
2767 /* Even is_gimple_min_invariant rely on current_function_decl. */
2768 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
2770 /* Can this function be inlined at all? */
2771 if (!opt_for_fn (node
->decl
, optimize
)
2772 && !lookup_attribute ("always_inline",
2773 DECL_ATTRIBUTES (node
->decl
)))
2774 info
->inlinable
= false;
2776 info
->inlinable
= tree_inlinable_function_p (node
->decl
);
2778 /* Type attributes can use parameter indices to describe them. */
2779 if (TYPE_ATTRIBUTES (TREE_TYPE (node
->decl
))
2780 /* Likewise for #pragma omp declare simd functions or functions
2781 with simd attribute. */
2782 || lookup_attribute ("omp declare simd",
2783 DECL_ATTRIBUTES (node
->decl
)))
2784 node
->can_change_signature
= false;
2787 /* Otherwise, inlinable functions always can change signature. */
2788 if (info
->inlinable
)
2789 node
->can_change_signature
= true;
2792 /* Functions calling builtin_apply cannot change signature. */
2793 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2795 tree
cdecl = e
->callee
->decl
;
2796 if (fndecl_built_in_p (cdecl, BUILT_IN_APPLY_ARGS
)
2797 || fndecl_built_in_p (cdecl, BUILT_IN_VA_START
))
2800 node
->can_change_signature
= !e
;
2803 analyze_function_body (node
, early
);
2806 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2807 if (e
->callee
->comdat_local_p ())
2809 node
->calls_comdat_local
= (e
!= NULL
);
2811 /* Inlining characteristics are maintained by the cgraph_mark_inline. */
2812 size_info
->size
= size_info
->self_size
;
2813 info
->estimated_stack_size
= size_info
->estimated_self_stack_size
;
2815 /* Code above should compute exactly the same result as
2816 ipa_update_overall_fn_summary but because computation happens in
2817 different order the roundoff errors result in slight changes. */
2818 ipa_update_overall_fn_summary (node
);
2819 /* In LTO mode we may have speculative edges set. */
2820 gcc_assert (in_lto_p
|| size_info
->size
== size_info
->self_size
);
2824 /* Compute parameters of functions used by inliner using
2825 current_function_decl. */
2828 compute_fn_summary_for_current (void)
2830 compute_fn_summary (cgraph_node::get (current_function_decl
), true);
2834 /* Estimate benefit devirtualizing indirect edge IE, provided KNOWN_VALS,
2835 KNOWN_CONTEXTS and KNOWN_AGGS. */
2838 estimate_edge_devirt_benefit (struct cgraph_edge
*ie
,
2839 int *size
, int *time
,
2840 vec
<tree
> known_vals
,
2841 vec
<ipa_polymorphic_call_context
> known_contexts
,
2842 vec
<ipa_agg_jump_function_p
> known_aggs
)
2845 struct cgraph_node
*callee
;
2846 class ipa_fn_summary
*isummary
;
2847 enum availability avail
;
2850 if (!known_vals
.exists () && !known_contexts
.exists ())
2852 if (!opt_for_fn (ie
->caller
->decl
, flag_indirect_inlining
))
2855 target
= ipa_get_indirect_edge_target (ie
, known_vals
, known_contexts
,
2856 known_aggs
, &speculative
);
2857 if (!target
|| speculative
)
2860 /* Account for difference in cost between indirect and direct calls. */
2861 *size
-= (eni_size_weights
.indirect_call_cost
- eni_size_weights
.call_cost
);
2862 *time
-= (eni_time_weights
.indirect_call_cost
- eni_time_weights
.call_cost
);
2863 gcc_checking_assert (*time
>= 0);
2864 gcc_checking_assert (*size
>= 0);
2866 callee
= cgraph_node::get (target
);
2867 if (!callee
|| !callee
->definition
)
2869 callee
= callee
->function_symbol (&avail
);
2870 if (avail
< AVAIL_AVAILABLE
)
2872 isummary
= ipa_fn_summaries
->get (callee
);
2873 if (isummary
== NULL
)
2876 return isummary
->inlinable
;
2879 /* Increase SIZE, MIN_SIZE (if non-NULL) and TIME for size and time needed to
2880 handle edge E with probability PROB.
2881 Set HINTS if edge may be devirtualized.
2882 KNOWN_VALS, KNOWN_AGGS and KNOWN_CONTEXTS describe context of the call
2886 estimate_edge_size_and_time (struct cgraph_edge
*e
, int *size
, int *min_size
,
2889 vec
<tree
> known_vals
,
2890 vec
<ipa_polymorphic_call_context
> known_contexts
,
2891 vec
<ipa_agg_jump_function_p
> known_aggs
,
2894 class ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
2895 int call_size
= es
->call_stmt_size
;
2896 int call_time
= es
->call_stmt_time
;
2898 if (!e
->callee
&& hints
&& e
->maybe_hot_p ()
2899 && estimate_edge_devirt_benefit (e
, &call_size
, &call_time
,
2900 known_vals
, known_contexts
, known_aggs
))
2901 *hints
|= INLINE_HINT_indirect_call
;
2902 cur_size
= call_size
* ipa_fn_summary::size_scale
;
2905 *min_size
+= cur_size
;
2908 else if (prob
== REG_BR_PROB_BASE
)
2909 *time
+= ((sreal
)call_time
) * e
->sreal_frequency ();
2911 *time
+= ((sreal
)call_time
* prob
) * e
->sreal_frequency ();
2916 /* Increase SIZE, MIN_SIZE and TIME for size and time needed to handle all
2917 calls in NODE. POSSIBLE_TRUTHS, KNOWN_VALS, KNOWN_AGGS and KNOWN_CONTEXTS
2918 describe context of the call site. */
2921 estimate_calls_size_and_time (struct cgraph_node
*node
, int *size
,
2922 int *min_size
, sreal
*time
,
2924 clause_t possible_truths
,
2925 vec
<tree
> known_vals
,
2926 vec
<ipa_polymorphic_call_context
> known_contexts
,
2927 vec
<ipa_agg_jump_function_p
> known_aggs
)
2929 struct cgraph_edge
*e
;
2930 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2932 if (!e
->inline_failed
)
2934 gcc_checking_assert (!ipa_call_summaries
->get (e
));
2935 estimate_calls_size_and_time (e
->callee
, size
, min_size
, time
,
2938 known_vals
, known_contexts
,
2942 class ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
2944 /* Do not care about zero sized builtins. */
2945 if (!es
->call_stmt_size
)
2947 gcc_checking_assert (!es
->call_stmt_time
);
2951 || es
->predicate
->evaluate (possible_truths
))
2953 /* Predicates of calls shall not use NOT_CHANGED codes,
2954 sowe do not need to compute probabilities. */
2955 estimate_edge_size_and_time (e
, size
,
2956 es
->predicate
? NULL
: min_size
,
2957 time
, REG_BR_PROB_BASE
,
2958 known_vals
, known_contexts
,
2962 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
2964 class ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
2966 || es
->predicate
->evaluate (possible_truths
))
2967 estimate_edge_size_and_time (e
, size
,
2968 es
->predicate
? NULL
: min_size
,
2969 time
, REG_BR_PROB_BASE
,
2970 known_vals
, known_contexts
, known_aggs
,
2975 /* Default constructor for ipa call context.
2976 Memory alloction of known_vals, known_contexts
2977 and known_aggs vectors is owned by the caller, but can
2978 be release by ipa_call_context::release.
2980 inline_param_summary is owned by the caller. */
2981 ipa_call_context::ipa_call_context (cgraph_node
*node
,
2982 clause_t possible_truths
,
2983 clause_t nonspec_possible_truths
,
2984 vec
<tree
> known_vals
,
2985 vec
<ipa_polymorphic_call_context
>
2987 vec
<ipa_agg_jump_function_p
> known_aggs
,
2988 vec
<inline_param_summary
>
2989 inline_param_summary
)
2990 : m_node (node
), m_possible_truths (possible_truths
),
2991 m_nonspec_possible_truths (nonspec_possible_truths
),
2992 m_inline_param_summary (inline_param_summary
),
2993 m_known_vals (known_vals
),
2994 m_known_contexts (known_contexts
),
2995 m_known_aggs (known_aggs
)
2999 /* Set THIS to be a duplicate of CTX. Copy all relevant info. */
3002 ipa_call_context::duplicate_from (const ipa_call_context
&ctx
)
3004 m_node
= ctx
.m_node
;
3005 m_possible_truths
= ctx
.m_possible_truths
;
3006 m_nonspec_possible_truths
= ctx
.m_nonspec_possible_truths
;
3007 class ipa_node_params
*params_summary
= IPA_NODE_REF (m_node
);
3008 unsigned int nargs
= params_summary
3009 ? ipa_get_param_count (params_summary
) : 0;
3011 m_inline_param_summary
= vNULL
;
3012 /* Copy the info only if there is at least one useful entry. */
3013 if (ctx
.m_inline_param_summary
.exists ())
3015 unsigned int n
= MIN (ctx
.m_inline_param_summary
.length (), nargs
);
3017 for (unsigned int i
= 0; i
< n
; i
++)
3018 if (ipa_is_param_used_by_ipa_predicates (params_summary
, i
)
3019 && !ctx
.m_inline_param_summary
[i
].useless_p ())
3021 m_inline_param_summary
3022 = ctx
.m_inline_param_summary
.copy ();
3026 m_known_vals
= vNULL
;
3027 if (ctx
.m_known_vals
.exists ())
3029 unsigned int n
= MIN (ctx
.m_known_vals
.length (), nargs
);
3031 for (unsigned int i
= 0; i
< n
; i
++)
3032 if (ipa_is_param_used_by_indirect_call (params_summary
, i
)
3033 && ctx
.m_known_vals
[i
])
3035 m_known_vals
= ctx
.m_known_vals
.copy ();
3040 m_known_contexts
= vNULL
;
3041 if (ctx
.m_known_contexts
.exists ())
3043 unsigned int n
= MIN (ctx
.m_known_contexts
.length (), nargs
);
3045 for (unsigned int i
= 0; i
< n
; i
++)
3046 if (ipa_is_param_used_by_polymorphic_call (params_summary
, i
)
3047 && !ctx
.m_known_contexts
[i
].useless_p ())
3049 m_known_contexts
= ctx
.m_known_contexts
.copy ();
3054 m_known_aggs
= vNULL
;
3055 if (ctx
.m_known_aggs
.exists ())
3057 unsigned int n
= MIN (ctx
.m_known_aggs
.length (), nargs
);
3059 for (unsigned int i
= 0; i
< n
; i
++)
3060 if (ipa_is_param_used_by_indirect_call (params_summary
, i
)
3061 && ctx
.m_known_aggs
[i
])
3063 m_known_aggs
= ctx
.m_known_aggs
.copy ();
3069 /* Release memory used by known_vals/contexts/aggs vectors.
3070 If ALL is true release also inline_param_summary.
3071 This happens when context was previously duplciated to be stored
3075 ipa_call_context::release (bool all
)
3077 /* See if context is initialized at first place. */
3080 m_known_vals
.release ();
3081 m_known_contexts
.release ();
3082 m_known_aggs
.release ();
3084 m_inline_param_summary
.release ();
3087 /* Return true if CTX describes the same call context as THIS. */
3090 ipa_call_context::equal_to (const ipa_call_context
&ctx
)
3092 if (m_node
!= ctx
.m_node
3093 || m_possible_truths
!= ctx
.m_possible_truths
3094 || m_nonspec_possible_truths
!= ctx
.m_nonspec_possible_truths
)
3097 class ipa_node_params
*params_summary
= IPA_NODE_REF (m_node
);
3098 unsigned int nargs
= params_summary
3099 ? ipa_get_param_count (params_summary
) : 0;
3101 if (m_inline_param_summary
.exists () || ctx
.m_inline_param_summary
.exists ())
3103 for (unsigned int i
= 0; i
< nargs
; i
++)
3105 if (!ipa_is_param_used_by_ipa_predicates (params_summary
, i
))
3107 if (i
>= m_inline_param_summary
.length ()
3108 || m_inline_param_summary
[i
].useless_p ())
3110 if (i
< ctx
.m_inline_param_summary
.length ()
3111 && !ctx
.m_inline_param_summary
[i
].useless_p ())
3115 if (i
>= ctx
.m_inline_param_summary
.length ()
3116 || ctx
.m_inline_param_summary
[i
].useless_p ())
3118 if (i
< m_inline_param_summary
.length ()
3119 && !m_inline_param_summary
[i
].useless_p ())
3123 if (!m_inline_param_summary
[i
].equal_to
3124 (ctx
.m_inline_param_summary
[i
]))
3128 if (m_known_vals
.exists () || ctx
.m_known_vals
.exists ())
3130 for (unsigned int i
= 0; i
< nargs
; i
++)
3132 if (!ipa_is_param_used_by_indirect_call (params_summary
, i
))
3134 if (i
>= m_known_vals
.length () || !m_known_vals
[i
])
3136 if (i
< ctx
.m_known_vals
.length () && ctx
.m_known_vals
[i
])
3140 if (i
>= ctx
.m_known_vals
.length () || !ctx
.m_known_vals
[i
])
3142 if (i
< m_known_vals
.length () && m_known_vals
[i
])
3146 if (m_known_vals
[i
] != ctx
.m_known_vals
[i
])
3150 if (m_known_contexts
.exists () || ctx
.m_known_contexts
.exists ())
3152 for (unsigned int i
= 0; i
< nargs
; i
++)
3154 if (!ipa_is_param_used_by_polymorphic_call (params_summary
, i
))
3156 if (i
>= m_known_contexts
.length ()
3157 || m_known_contexts
[i
].useless_p ())
3159 if (i
< ctx
.m_known_contexts
.length ()
3160 && !ctx
.m_known_contexts
[i
].useless_p ())
3164 if (i
>= ctx
.m_known_contexts
.length ()
3165 || ctx
.m_known_contexts
[i
].useless_p ())
3167 if (i
< m_known_contexts
.length ()
3168 && !m_known_contexts
[i
].useless_p ())
3172 if (!m_known_contexts
[i
].equal_to
3173 (ctx
.m_known_contexts
[i
]))
3177 if (m_known_aggs
.exists () || ctx
.m_known_aggs
.exists ())
3179 for (unsigned int i
= 0; i
< nargs
; i
++)
3181 if (!ipa_is_param_used_by_indirect_call (params_summary
, i
))
3183 if (i
>= m_known_aggs
.length () || !m_known_aggs
[i
])
3185 if (i
< ctx
.m_known_aggs
.length () && ctx
.m_known_aggs
[i
])
3189 if (i
>= ctx
.m_known_aggs
.length () || !ctx
.m_known_aggs
[i
])
3191 if (i
< m_known_aggs
.length () && m_known_aggs
[i
])
3195 if (m_known_aggs
[i
] != ctx
.m_known_aggs
[i
])
3202 /* Estimate size and time needed to execute call in the given context.
3203 Additionally detemine hints determined by the context. Finally compute
3204 minimal size needed for the call that is independent on the call context and
3205 can be used for fast estimates. Return the values in RET_SIZE,
3206 RET_MIN_SIZE, RET_TIME and RET_HINTS. */
3209 ipa_call_context::estimate_size_and_time (int *ret_size
,
3212 sreal
*ret_nonspecialized_time
,
3213 ipa_hints
*ret_hints
)
3215 class ipa_fn_summary
*info
= ipa_fn_summaries
->get (m_node
);
3220 ipa_hints hints
= 0;
3223 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
3226 fprintf (dump_file
, " Estimating body: %s/%i\n"
3227 " Known to be false: ", m_node
->name (),
3230 for (i
= predicate::not_inlined_condition
;
3231 i
< (predicate::first_dynamic_condition
3232 + (int) vec_safe_length (info
->conds
)); i
++)
3233 if (!(m_possible_truths
& (1 << i
)))
3236 fprintf (dump_file
, ", ");
3238 dump_condition (dump_file
, info
->conds
, i
);
3242 estimate_calls_size_and_time (m_node
, &size
, &min_size
,
3243 ret_time
? &time
: NULL
,
3244 ret_hints
? &hints
: NULL
, m_possible_truths
,
3245 m_known_vals
, m_known_contexts
, m_known_aggs
);
3247 sreal nonspecialized_time
= time
;
3249 min_size
+= (*info
->size_time_table
)[0].size
;
3250 for (i
= 0; vec_safe_iterate (info
->size_time_table
, i
, &e
); i
++)
3252 bool exec
= e
->exec_predicate
.evaluate (m_nonspec_possible_truths
);
3254 /* Because predicates are conservative, it can happen that nonconst is 1
3258 bool nonconst
= e
->nonconst_predicate
.evaluate (m_possible_truths
);
3260 gcc_checking_assert (e
->time
>= 0);
3261 gcc_checking_assert (time
>= 0);
3263 /* We compute specialized size only because size of nonspecialized
3264 copy is context independent.
3266 The difference between nonspecialized execution and specialized is
3267 that nonspecialized is not going to have optimized out computations
3268 known to be constant in a specialized setting. */
3273 nonspecialized_time
+= e
->time
;
3276 else if (!m_inline_param_summary
.exists ())
3283 int prob
= e
->nonconst_predicate
.probability
3284 (info
->conds
, m_possible_truths
,
3285 m_inline_param_summary
);
3286 gcc_checking_assert (prob
>= 0);
3287 gcc_checking_assert (prob
<= REG_BR_PROB_BASE
);
3288 time
+= e
->time
* prob
/ REG_BR_PROB_BASE
;
3290 gcc_checking_assert (time
>= 0);
3293 gcc_checking_assert ((*info
->size_time_table
)[0].exec_predicate
== true);
3294 gcc_checking_assert ((*info
->size_time_table
)[0].nonconst_predicate
== true);
3295 gcc_checking_assert (min_size
>= 0);
3296 gcc_checking_assert (size
>= 0);
3297 gcc_checking_assert (time
>= 0);
3298 /* nonspecialized_time should be always bigger than specialized time.
3299 Roundoff issues however may get into the way. */
3300 gcc_checking_assert ((nonspecialized_time
- time
* 99 / 100) >= -1);
3302 /* Roundoff issues may make specialized time bigger than nonspecialized
3303 time. We do not really want that to happen because some heurstics
3304 may get confused by seeing negative speedups. */
3305 if (time
> nonspecialized_time
)
3306 time
= nonspecialized_time
;
3310 if (info
->loop_iterations
3311 && !info
->loop_iterations
->evaluate (m_possible_truths
))
3312 hints
|= INLINE_HINT_loop_iterations
;
3313 if (info
->loop_stride
3314 && !info
->loop_stride
->evaluate (m_possible_truths
))
3315 hints
|= INLINE_HINT_loop_stride
;
3317 hints
|= INLINE_HINT_in_scc
;
3318 if (DECL_DECLARED_INLINE_P (m_node
->decl
))
3319 hints
|= INLINE_HINT_declared_inline
;
3322 size
= RDIV (size
, ipa_fn_summary::size_scale
);
3323 min_size
= RDIV (min_size
, ipa_fn_summary::size_scale
);
3325 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
3326 fprintf (dump_file
, "\n size:%i time:%f nonspec time:%f\n", (int) size
,
3327 time
.to_double (), nonspecialized_time
.to_double ());
3330 if (ret_nonspecialized_time
)
3331 *ret_nonspecialized_time
= nonspecialized_time
;
3335 *ret_min_size
= min_size
;
3342 /* Estimate size and time needed to execute callee of EDGE assuming that
3343 parameters known to be constant at caller of EDGE are propagated.
3344 KNOWN_VALS and KNOWN_CONTEXTS are vectors of assumed known constant values
3345 and types for parameters. */
3348 estimate_ipcp_clone_size_and_time (struct cgraph_node
*node
,
3349 vec
<tree
> known_vals
,
3350 vec
<ipa_polymorphic_call_context
>
3352 vec
<ipa_agg_jump_function_p
> known_aggs
,
3353 int *ret_size
, sreal
*ret_time
,
3354 sreal
*ret_nonspec_time
,
3357 clause_t clause
, nonspec_clause
;
3359 evaluate_conditions_for_known_args (node
, false, known_vals
, known_aggs
,
3360 &clause
, &nonspec_clause
);
3361 ipa_call_context
ctx (node
, clause
, nonspec_clause
,
3362 known_vals
, known_contexts
,
3364 ctx
.estimate_size_and_time (ret_size
, NULL
, ret_time
,
3365 ret_nonspec_time
, hints
);
3368 /* Return stack frame offset where frame of NODE is supposed to start inside
3369 of the function it is inlined to.
3370 Return 0 for functions that are not inlined. */
3373 ipa_get_stack_frame_offset (struct cgraph_node
*node
)
3375 HOST_WIDE_INT offset
= 0;
3376 if (!node
->inlined_to
)
3378 node
= node
->callers
->caller
;
3381 offset
+= ipa_size_summaries
->get (node
)->estimated_self_stack_size
;
3382 if (!node
->inlined_to
)
3384 node
= node
->callers
->caller
;
3389 /* Update summary information of inline clones after inlining.
3390 Compute peak stack usage. */
3393 inline_update_callee_summaries (struct cgraph_node
*node
, int depth
)
3395 struct cgraph_edge
*e
;
3397 ipa_propagate_frequency (node
);
3398 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3400 if (!e
->inline_failed
)
3401 inline_update_callee_summaries (e
->callee
, depth
);
3403 ipa_call_summaries
->get (e
)->loop_depth
+= depth
;
3405 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3406 ipa_call_summaries
->get (e
)->loop_depth
+= depth
;
3409 /* Update change_prob of EDGE after INLINED_EDGE has been inlined.
3410 When function A is inlined in B and A calls C with parameter that
3411 changes with probability PROB1 and C is known to be passthroug
3412 of argument if B that change with probability PROB2, the probability
3413 of change is now PROB1*PROB2. */
3416 remap_edge_change_prob (struct cgraph_edge
*inlined_edge
,
3417 struct cgraph_edge
*edge
)
3419 if (ipa_node_params_sum
)
3422 class ipa_edge_args
*args
= IPA_EDGE_REF (edge
);
3425 class ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
3426 class ipa_call_summary
*inlined_es
3427 = ipa_call_summaries
->get (inlined_edge
);
3429 if (es
->param
.length () == 0)
3432 for (i
= 0; i
< ipa_get_cs_argument_count (args
); i
++)
3434 struct ipa_jump_func
*jfunc
= ipa_get_ith_jump_func (args
, i
);
3435 if (jfunc
->type
== IPA_JF_PASS_THROUGH
3436 || jfunc
->type
== IPA_JF_ANCESTOR
)
3438 int id
= jfunc
->type
== IPA_JF_PASS_THROUGH
3439 ? ipa_get_jf_pass_through_formal_id (jfunc
)
3440 : ipa_get_jf_ancestor_formal_id (jfunc
);
3441 if (id
< (int) inlined_es
->param
.length ())
3443 int prob1
= es
->param
[i
].change_prob
;
3444 int prob2
= inlined_es
->param
[id
].change_prob
;
3445 int prob
= combine_probabilities (prob1
, prob2
);
3447 if (prob1
&& prob2
&& !prob
)
3450 es
->param
[i
].change_prob
= prob
;
3457 /* Update edge summaries of NODE after INLINED_EDGE has been inlined.
3459 Remap predicates of callees of NODE. Rest of arguments match
3462 Also update change probabilities. */
3465 remap_edge_summaries (struct cgraph_edge
*inlined_edge
,
3466 struct cgraph_node
*node
,
3467 class ipa_fn_summary
*info
,
3468 class ipa_node_params
*params_summary
,
3469 class ipa_fn_summary
*callee_info
,
3470 vec
<int> operand_map
,
3471 vec
<int> offset_map
,
3472 clause_t possible_truths
,
3473 predicate
*toplev_predicate
)
3475 struct cgraph_edge
*e
, *next
;
3476 for (e
= node
->callees
; e
; e
= next
)
3479 next
= e
->next_callee
;
3481 if (e
->inline_failed
)
3483 class ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
3484 remap_edge_change_prob (inlined_edge
, e
);
3488 p
= es
->predicate
->remap_after_inlining
3489 (info
, params_summary
,
3490 callee_info
, operand_map
,
3491 offset_map
, possible_truths
,
3493 edge_set_predicate (e
, &p
);
3496 edge_set_predicate (e
, toplev_predicate
);
3499 remap_edge_summaries (inlined_edge
, e
->callee
, info
,
3500 params_summary
, callee_info
,
3501 operand_map
, offset_map
, possible_truths
,
3504 for (e
= node
->indirect_calls
; e
; e
= next
)
3506 class ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
3508 next
= e
->next_callee
;
3510 remap_edge_change_prob (inlined_edge
, e
);
3513 p
= es
->predicate
->remap_after_inlining
3514 (info
, params_summary
,
3515 callee_info
, operand_map
, offset_map
,
3516 possible_truths
, *toplev_predicate
);
3517 edge_set_predicate (e
, &p
);
3520 edge_set_predicate (e
, toplev_predicate
);
3524 /* Same as remap_predicate, but set result into hint *HINT. */
3527 remap_hint_predicate (class ipa_fn_summary
*info
,
3528 class ipa_node_params
*params_summary
,
3529 class ipa_fn_summary
*callee_info
,
3531 vec
<int> operand_map
,
3532 vec
<int> offset_map
,
3533 clause_t possible_truths
,
3534 predicate
*toplev_predicate
)
3540 p
= (*hint
)->remap_after_inlining
3541 (info
, params_summary
, callee_info
,
3542 operand_map
, offset_map
,
3543 possible_truths
, *toplev_predicate
);
3544 if (p
!= false && p
!= true)
3547 set_hint_predicate (hint
, p
);
3553 /* We inlined EDGE. Update summary of the function we inlined into. */
3556 ipa_merge_fn_summary_after_inlining (struct cgraph_edge
*edge
)
3558 ipa_fn_summary
*callee_info
= ipa_fn_summaries
->get (edge
->callee
);
3559 struct cgraph_node
*to
= (edge
->caller
->inlined_to
3560 ? edge
->caller
->inlined_to
: edge
->caller
);
3561 class ipa_fn_summary
*info
= ipa_fn_summaries
->get (to
);
3562 clause_t clause
= 0; /* not_inline is known to be false. */
3564 auto_vec
<int, 8> operand_map
;
3565 auto_vec
<int, 8> offset_map
;
3567 predicate toplev_predicate
;
3568 class ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
3569 class ipa_node_params
*params_summary
= (ipa_node_params_sum
3570 ? IPA_NODE_REF (to
) : NULL
);
3573 toplev_predicate
= *es
->predicate
;
3575 toplev_predicate
= true;
3577 info
->fp_expressions
|= callee_info
->fp_expressions
;
3579 if (callee_info
->conds
)
3580 evaluate_properties_for_edge (edge
, true, &clause
, NULL
, NULL
, NULL
, NULL
);
3581 if (ipa_node_params_sum
&& callee_info
->conds
)
3583 class ipa_edge_args
*args
= IPA_EDGE_REF (edge
);
3584 int count
= args
? ipa_get_cs_argument_count (args
) : 0;
3589 operand_map
.safe_grow_cleared (count
);
3590 offset_map
.safe_grow_cleared (count
);
3592 for (i
= 0; i
< count
; i
++)
3594 struct ipa_jump_func
*jfunc
= ipa_get_ith_jump_func (args
, i
);
3597 /* TODO: handle non-NOPs when merging. */
3598 if (jfunc
->type
== IPA_JF_PASS_THROUGH
)
3600 if (ipa_get_jf_pass_through_operation (jfunc
) == NOP_EXPR
)
3601 map
= ipa_get_jf_pass_through_formal_id (jfunc
);
3602 if (!ipa_get_jf_pass_through_agg_preserved (jfunc
))
3605 else if (jfunc
->type
== IPA_JF_ANCESTOR
)
3607 HOST_WIDE_INT offset
= ipa_get_jf_ancestor_offset (jfunc
);
3608 if (offset
>= 0 && offset
< INT_MAX
)
3610 map
= ipa_get_jf_ancestor_formal_id (jfunc
);
3611 if (!ipa_get_jf_ancestor_agg_preserved (jfunc
))
3613 offset_map
[i
] = offset
;
3616 operand_map
[i
] = map
;
3617 gcc_assert (map
< ipa_get_param_count (params_summary
));
3620 sreal freq
= edge
->sreal_frequency ();
3621 for (i
= 0; vec_safe_iterate (callee_info
->size_time_table
, i
, &e
); i
++)
3624 p
= e
->exec_predicate
.remap_after_inlining
3625 (info
, params_summary
,
3626 callee_info
, operand_map
,
3629 predicate nonconstp
;
3630 nonconstp
= e
->nonconst_predicate
.remap_after_inlining
3631 (info
, params_summary
,
3632 callee_info
, operand_map
,
3635 if (p
!= false && nonconstp
!= false)
3637 sreal add_time
= ((sreal
)e
->time
* freq
);
3638 int prob
= e
->nonconst_predicate
.probability (callee_info
->conds
,
3640 add_time
= add_time
* prob
/ REG_BR_PROB_BASE
;
3641 if (prob
!= REG_BR_PROB_BASE
3642 && dump_file
&& (dump_flags
& TDF_DETAILS
))
3644 fprintf (dump_file
, "\t\tScaling time by probability:%f\n",
3645 (double) prob
/ REG_BR_PROB_BASE
);
3647 info
->account_size_time (e
->size
, add_time
, p
, nonconstp
);
3650 remap_edge_summaries (edge
, edge
->callee
, info
, params_summary
,
3651 callee_info
, operand_map
,
3652 offset_map
, clause
, &toplev_predicate
);
3653 remap_hint_predicate (info
, params_summary
, callee_info
,
3654 &callee_info
->loop_iterations
,
3655 operand_map
, offset_map
, clause
, &toplev_predicate
);
3656 remap_hint_predicate (info
, params_summary
, callee_info
,
3657 &callee_info
->loop_stride
,
3658 operand_map
, offset_map
, clause
, &toplev_predicate
);
3660 HOST_WIDE_INT stack_frame_offset
= ipa_get_stack_frame_offset (edge
->callee
);
3661 HOST_WIDE_INT peak
= stack_frame_offset
+ callee_info
->estimated_stack_size
;
3663 if (info
->estimated_stack_size
< peak
)
3664 info
->estimated_stack_size
= peak
;
3666 inline_update_callee_summaries (edge
->callee
, es
->loop_depth
);
3668 /* Free summaries that are not maintained for inline clones/edges. */
3669 ipa_call_summaries
->remove (edge
);
3670 ipa_fn_summaries
->remove (edge
->callee
);
3671 ipa_remove_from_growth_caches (edge
);
3674 /* For performance reasons ipa_merge_fn_summary_after_inlining is not updating
3675 overall size and time. Recompute it. */
3678 ipa_update_overall_fn_summary (struct cgraph_node
*node
)
3680 class ipa_fn_summary
*info
= ipa_fn_summaries
->get (node
);
3681 class ipa_size_summary
*size_info
= ipa_size_summaries
->get (node
);
3685 size_info
->size
= 0;
3687 for (i
= 0; vec_safe_iterate (info
->size_time_table
, i
, &e
); i
++)
3689 size_info
->size
+= e
->size
;
3690 info
->time
+= e
->time
;
3692 info
->min_size
= (*info
->size_time_table
)[0].size
;
3693 estimate_calls_size_and_time (node
, &size_info
->size
, &info
->min_size
,
3695 ~(clause_t
) (1 << predicate::false_condition
),
3696 vNULL
, vNULL
, vNULL
);
3697 size_info
->size
= RDIV (size_info
->size
, ipa_fn_summary::size_scale
);
3698 info
->min_size
= RDIV (info
->min_size
, ipa_fn_summary::size_scale
);
3702 /* This function performs intraprocedural analysis in NODE that is required to
3703 inline indirect calls. */
3706 inline_indirect_intraprocedural_analysis (struct cgraph_node
*node
)
3708 ipa_analyze_node (node
);
3709 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
3711 ipa_print_node_params (dump_file
, node
);
3712 ipa_print_node_jump_functions (dump_file
, node
);
3717 /* Note function body size. */
3720 inline_analyze_function (struct cgraph_node
*node
)
3722 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
3725 fprintf (dump_file
, "\nAnalyzing function: %s/%u\n",
3726 node
->name (), node
->order
);
3727 if (opt_for_fn (node
->decl
, optimize
) && !node
->thunk
.thunk_p
)
3728 inline_indirect_intraprocedural_analysis (node
);
3729 compute_fn_summary (node
, false);
3732 struct cgraph_edge
*e
;
3733 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3734 e
->inline_failed
= CIF_FUNCTION_NOT_OPTIMIZED
;
3735 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3736 e
->inline_failed
= CIF_FUNCTION_NOT_OPTIMIZED
;
3743 /* Called when new function is inserted to callgraph late. */
3746 ipa_fn_summary_t::insert (struct cgraph_node
*node
, ipa_fn_summary
*)
3748 inline_analyze_function (node
);
3751 /* Note function body size. */
3754 ipa_fn_summary_generate (void)
3756 struct cgraph_node
*node
;
3758 FOR_EACH_DEFINED_FUNCTION (node
)
3759 if (DECL_STRUCT_FUNCTION (node
->decl
))
3760 node
->versionable
= tree_versionable_function_p (node
->decl
);
3762 ipa_fn_summary_alloc ();
3764 ipa_fn_summaries
->enable_insertion_hook ();
3766 ipa_register_cgraph_hooks ();
3768 FOR_EACH_DEFINED_FUNCTION (node
)
3770 && (flag_generate_lto
|| flag_generate_offload
|| flag_wpa
3771 || opt_for_fn (node
->decl
, optimize
)))
3772 inline_analyze_function (node
);
3776 /* Write inline summary for edge E to OB. */
3779 read_ipa_call_summary (class lto_input_block
*ib
, struct cgraph_edge
*e
,
3782 class ipa_call_summary
*es
= prevails
3783 ? ipa_call_summaries
->get_create (e
) : NULL
;
3787 int size
= streamer_read_uhwi (ib
);
3788 int time
= streamer_read_uhwi (ib
);
3789 int depth
= streamer_read_uhwi (ib
);
3793 es
->call_stmt_size
= size
;
3794 es
->call_stmt_time
= time
;
3795 es
->loop_depth
= depth
;
3798 bitpack_d bp
= streamer_read_bitpack (ib
);
3800 es
->is_return_callee_uncaptured
= bp_unpack_value (&bp
, 1);
3802 bp_unpack_value (&bp
, 1);
3806 edge_set_predicate (e
, &p
);
3807 length
= streamer_read_uhwi (ib
);
3808 if (length
&& es
&& e
->possibly_call_in_translation_unit_p ())
3810 es
->param
.safe_grow_cleared (length
);
3811 for (i
= 0; i
< length
; i
++)
3812 es
->param
[i
].change_prob
= streamer_read_uhwi (ib
);
3816 for (i
= 0; i
< length
; i
++)
3817 streamer_read_uhwi (ib
);
3822 /* Stream in inline summaries from the section. */
3825 inline_read_section (struct lto_file_decl_data
*file_data
, const char *data
,
3828 const struct lto_function_header
*header
=
3829 (const struct lto_function_header
*) data
;
3830 const int cfg_offset
= sizeof (struct lto_function_header
);
3831 const int main_offset
= cfg_offset
+ header
->cfg_size
;
3832 const int string_offset
= main_offset
+ header
->main_size
;
3833 class data_in
*data_in
;
3834 unsigned int i
, count2
, j
;
3835 unsigned int f_count
;
3837 lto_input_block
ib ((const char *) data
+ main_offset
, header
->main_size
,
3838 file_data
->mode_table
);
3841 lto_data_in_create (file_data
, (const char *) data
+ string_offset
,
3842 header
->string_size
, vNULL
);
3843 f_count
= streamer_read_uhwi (&ib
);
3844 for (i
= 0; i
< f_count
; i
++)
3847 struct cgraph_node
*node
;
3848 class ipa_fn_summary
*info
;
3849 class ipa_node_params
*params_summary
;
3850 class ipa_size_summary
*size_info
;
3851 lto_symtab_encoder_t encoder
;
3852 struct bitpack_d bp
;
3853 struct cgraph_edge
*e
;
3856 index
= streamer_read_uhwi (&ib
);
3857 encoder
= file_data
->symtab_node_encoder
;
3858 node
= dyn_cast
<cgraph_node
*> (lto_symtab_encoder_deref (encoder
,
3860 info
= node
->prevailing_p () ? ipa_fn_summaries
->get_create (node
) : NULL
;
3861 params_summary
= node
->prevailing_p () ? IPA_NODE_REF (node
) : NULL
;
3862 size_info
= node
->prevailing_p ()
3863 ? ipa_size_summaries
->get_create (node
) : NULL
;
3865 int stack_size
= streamer_read_uhwi (&ib
);
3866 int size
= streamer_read_uhwi (&ib
);
3867 sreal time
= sreal::stream_in (&ib
);
3871 info
->estimated_stack_size
3872 = size_info
->estimated_self_stack_size
= stack_size
;
3873 size_info
->size
= size_info
->self_size
= size
;
3877 bp
= streamer_read_bitpack (&ib
);
3880 info
->inlinable
= bp_unpack_value (&bp
, 1);
3881 info
->fp_expressions
= bp_unpack_value (&bp
, 1);
3885 bp_unpack_value (&bp
, 1);
3886 bp_unpack_value (&bp
, 1);
3889 count2
= streamer_read_uhwi (&ib
);
3890 gcc_assert (!info
|| !info
->conds
);
3892 vec_safe_reserve_exact (info
->conds
, count2
);
3893 for (j
= 0; j
< count2
; j
++)
3896 unsigned int k
, count3
;
3897 c
.operand_num
= streamer_read_uhwi (&ib
);
3898 c
.code
= (enum tree_code
) streamer_read_uhwi (&ib
);
3899 c
.type
= stream_read_tree (&ib
, data_in
);
3900 c
.val
= stream_read_tree (&ib
, data_in
);
3901 bp
= streamer_read_bitpack (&ib
);
3902 c
.agg_contents
= bp_unpack_value (&bp
, 1);
3903 c
.by_ref
= bp_unpack_value (&bp
, 1);
3905 c
.offset
= streamer_read_uhwi (&ib
);
3906 count3
= streamer_read_uhwi (&ib
);
3909 vec_safe_reserve_exact (c
.param_ops
, count3
);
3911 ipa_set_param_used_by_ipa_predicates
3912 (params_summary
, c
.operand_num
, true);
3913 for (k
= 0; k
< count3
; k
++)
3915 struct expr_eval_op op
;
3916 enum gimple_rhs_class rhs_class
;
3917 op
.code
= (enum tree_code
) streamer_read_uhwi (&ib
);
3918 op
.type
= stream_read_tree (&ib
, data_in
);
3919 switch (rhs_class
= get_gimple_rhs_class (op
.code
))
3921 case GIMPLE_UNARY_RHS
:
3923 op
.val
[0] = NULL_TREE
;
3924 op
.val
[1] = NULL_TREE
;
3927 case GIMPLE_BINARY_RHS
:
3928 case GIMPLE_TERNARY_RHS
:
3929 bp
= streamer_read_bitpack (&ib
);
3930 op
.index
= bp_unpack_value (&bp
, 2);
3931 op
.val
[0] = stream_read_tree (&ib
, data_in
);
3932 if (rhs_class
== GIMPLE_BINARY_RHS
)
3933 op
.val
[1] = NULL_TREE
;
3935 op
.val
[1] = stream_read_tree (&ib
, data_in
);
3939 fatal_error (UNKNOWN_LOCATION
,
3940 "invalid fnsummary in LTO stream");
3943 c
.param_ops
->quick_push (op
);
3946 info
->conds
->quick_push (c
);
3948 count2
= streamer_read_uhwi (&ib
);
3949 gcc_assert (!info
|| !info
->size_time_table
);
3951 vec_safe_reserve_exact (info
->size_time_table
, count2
);
3952 for (j
= 0; j
< count2
; j
++)
3954 class size_time_entry e
;
3956 e
.size
= streamer_read_uhwi (&ib
);
3957 e
.time
= sreal::stream_in (&ib
);
3958 e
.exec_predicate
.stream_in (&ib
);
3959 e
.nonconst_predicate
.stream_in (&ib
);
3962 info
->size_time_table
->quick_push (e
);
3967 set_hint_predicate (&info
->loop_iterations
, p
);
3970 set_hint_predicate (&info
->loop_stride
, p
);
3971 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3972 read_ipa_call_summary (&ib
, e
, info
!= NULL
);
3973 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3974 read_ipa_call_summary (&ib
, e
, info
!= NULL
);
3977 lto_free_section_data (file_data
, LTO_section_ipa_fn_summary
, NULL
, data
,
3979 lto_data_in_delete (data_in
);
3983 /* Read inline summary. Jump functions are shared among ipa-cp
3984 and inliner, so when ipa-cp is active, we don't need to write them
3988 ipa_fn_summary_read (void)
3990 struct lto_file_decl_data
**file_data_vec
= lto_get_file_decl_data ();
3991 struct lto_file_decl_data
*file_data
;
3994 ipa_fn_summary_alloc ();
3996 while ((file_data
= file_data_vec
[j
++]))
4000 = lto_get_summary_section_data (file_data
, LTO_section_ipa_fn_summary
,
4003 inline_read_section (file_data
, data
, len
);
4005 /* Fatal error here. We do not want to support compiling ltrans units
4006 with different version of compiler or different flags than the WPA
4007 unit, so this should never happen. */
4008 fatal_error (input_location
,
4009 "ipa inline summary is missing in input file");
4011 ipa_register_cgraph_hooks ();
4013 ipa_prop_read_jump_functions ();
4015 gcc_assert (ipa_fn_summaries
);
4016 ipa_fn_summaries
->enable_insertion_hook ();
4020 /* Write inline summary for edge E to OB. */
4023 write_ipa_call_summary (struct output_block
*ob
, struct cgraph_edge
*e
)
4025 class ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
4028 streamer_write_uhwi (ob
, es
->call_stmt_size
);
4029 streamer_write_uhwi (ob
, es
->call_stmt_time
);
4030 streamer_write_uhwi (ob
, es
->loop_depth
);
4032 bitpack_d bp
= bitpack_create (ob
->main_stream
);
4033 bp_pack_value (&bp
, es
->is_return_callee_uncaptured
, 1);
4034 streamer_write_bitpack (&bp
);
4037 es
->predicate
->stream_out (ob
);
4039 streamer_write_uhwi (ob
, 0);
4040 streamer_write_uhwi (ob
, es
->param
.length ());
4041 for (i
= 0; i
< (int) es
->param
.length (); i
++)
4042 streamer_write_uhwi (ob
, es
->param
[i
].change_prob
);
4046 /* Write inline summary for node in SET.
4047 Jump functions are shared among ipa-cp and inliner, so when ipa-cp is
4048 active, we don't need to write them twice. */
4051 ipa_fn_summary_write (void)
4053 struct output_block
*ob
= create_output_block (LTO_section_ipa_fn_summary
);
4054 lto_symtab_encoder_t encoder
= ob
->decl_state
->symtab_node_encoder
;
4055 unsigned int count
= 0;
4058 for (i
= 0; i
< lto_symtab_encoder_size (encoder
); i
++)
4060 symtab_node
*snode
= lto_symtab_encoder_deref (encoder
, i
);
4061 cgraph_node
*cnode
= dyn_cast
<cgraph_node
*> (snode
);
4062 if (cnode
&& cnode
->definition
&& !cnode
->alias
)
4065 streamer_write_uhwi (ob
, count
);
4067 for (i
= 0; i
< lto_symtab_encoder_size (encoder
); i
++)
4069 symtab_node
*snode
= lto_symtab_encoder_deref (encoder
, i
);
4070 cgraph_node
*cnode
= dyn_cast
<cgraph_node
*> (snode
);
4071 if (cnode
&& cnode
->definition
&& !cnode
->alias
)
4073 class ipa_fn_summary
*info
= ipa_fn_summaries
->get (cnode
);
4074 class ipa_size_summary
*size_info
= ipa_size_summaries
->get (cnode
);
4075 struct bitpack_d bp
;
4076 struct cgraph_edge
*edge
;
4079 struct condition
*c
;
4081 streamer_write_uhwi (ob
, lto_symtab_encoder_encode (encoder
, cnode
));
4082 streamer_write_hwi (ob
, size_info
->estimated_self_stack_size
);
4083 streamer_write_hwi (ob
, size_info
->self_size
);
4084 info
->time
.stream_out (ob
);
4085 bp
= bitpack_create (ob
->main_stream
);
4086 bp_pack_value (&bp
, info
->inlinable
, 1);
4087 bp_pack_value (&bp
, false, 1);
4088 bp_pack_value (&bp
, info
->fp_expressions
, 1);
4089 streamer_write_bitpack (&bp
);
4090 streamer_write_uhwi (ob
, vec_safe_length (info
->conds
));
4091 for (i
= 0; vec_safe_iterate (info
->conds
, i
, &c
); i
++)
4094 struct expr_eval_op
*op
;
4096 streamer_write_uhwi (ob
, c
->operand_num
);
4097 streamer_write_uhwi (ob
, c
->code
);
4098 stream_write_tree (ob
, c
->type
, true);
4099 stream_write_tree (ob
, c
->val
, true);
4100 bp
= bitpack_create (ob
->main_stream
);
4101 bp_pack_value (&bp
, c
->agg_contents
, 1);
4102 bp_pack_value (&bp
, c
->by_ref
, 1);
4103 streamer_write_bitpack (&bp
);
4104 if (c
->agg_contents
)
4105 streamer_write_uhwi (ob
, c
->offset
);
4106 streamer_write_uhwi (ob
, vec_safe_length (c
->param_ops
));
4107 for (j
= 0; vec_safe_iterate (c
->param_ops
, j
, &op
); j
++)
4109 streamer_write_uhwi (ob
, op
->code
);
4110 stream_write_tree (ob
, op
->type
, true);
4113 bp
= bitpack_create (ob
->main_stream
);
4114 bp_pack_value (&bp
, op
->index
, 2);
4115 streamer_write_bitpack (&bp
);
4116 stream_write_tree (ob
, op
->val
[0], true);
4118 stream_write_tree (ob
, op
->val
[1], true);
4122 streamer_write_uhwi (ob
, vec_safe_length (info
->size_time_table
));
4123 for (i
= 0; vec_safe_iterate (info
->size_time_table
, i
, &e
); i
++)
4125 streamer_write_uhwi (ob
, e
->size
);
4126 e
->time
.stream_out (ob
);
4127 e
->exec_predicate
.stream_out (ob
);
4128 e
->nonconst_predicate
.stream_out (ob
);
4130 if (info
->loop_iterations
)
4131 info
->loop_iterations
->stream_out (ob
);
4133 streamer_write_uhwi (ob
, 0);
4134 if (info
->loop_stride
)
4135 info
->loop_stride
->stream_out (ob
);
4137 streamer_write_uhwi (ob
, 0);
4138 for (edge
= cnode
->callees
; edge
; edge
= edge
->next_callee
)
4139 write_ipa_call_summary (ob
, edge
);
4140 for (edge
= cnode
->indirect_calls
; edge
; edge
= edge
->next_callee
)
4141 write_ipa_call_summary (ob
, edge
);
4144 streamer_write_char_stream (ob
->main_stream
, 0);
4145 produce_asm (ob
, NULL
);
4146 destroy_output_block (ob
);
4149 ipa_prop_write_jump_functions ();
4153 /* Release function summary. */
4156 ipa_free_fn_summary (void)
4158 if (!ipa_call_summaries
)
4160 ggc_delete (ipa_fn_summaries
);
4161 ipa_fn_summaries
= NULL
;
4162 delete ipa_call_summaries
;
4163 ipa_call_summaries
= NULL
;
4164 edge_predicate_pool
.release ();
4165 /* During IPA this is one of largest datastructures to release. */
4170 /* Release function summary. */
4173 ipa_free_size_summary (void)
4175 if (!ipa_size_summaries
)
4177 delete ipa_size_summaries
;
4178 ipa_size_summaries
= NULL
;
4183 const pass_data pass_data_local_fn_summary
=
4185 GIMPLE_PASS
, /* type */
4186 "local-fnsummary", /* name */
4187 OPTGROUP_INLINE
, /* optinfo_flags */
4188 TV_INLINE_PARAMETERS
, /* tv_id */
4189 0, /* properties_required */
4190 0, /* properties_provided */
4191 0, /* properties_destroyed */
4192 0, /* todo_flags_start */
4193 0, /* todo_flags_finish */
4196 class pass_local_fn_summary
: public gimple_opt_pass
4199 pass_local_fn_summary (gcc::context
*ctxt
)
4200 : gimple_opt_pass (pass_data_local_fn_summary
, ctxt
)
4203 /* opt_pass methods: */
4204 opt_pass
* clone () { return new pass_local_fn_summary (m_ctxt
); }
4205 virtual unsigned int execute (function
*)
4207 return compute_fn_summary_for_current ();
4210 }; // class pass_local_fn_summary
4215 make_pass_local_fn_summary (gcc::context
*ctxt
)
4217 return new pass_local_fn_summary (ctxt
);
4221 /* Free inline summary. */
4225 const pass_data pass_data_ipa_free_fn_summary
=
4227 SIMPLE_IPA_PASS
, /* type */
4228 "free-fnsummary", /* name */
4229 OPTGROUP_NONE
, /* optinfo_flags */
4230 TV_IPA_FREE_INLINE_SUMMARY
, /* tv_id */
4231 0, /* properties_required */
4232 0, /* properties_provided */
4233 0, /* properties_destroyed */
4234 0, /* todo_flags_start */
4235 0, /* todo_flags_finish */
4238 class pass_ipa_free_fn_summary
: public simple_ipa_opt_pass
4241 pass_ipa_free_fn_summary (gcc::context
*ctxt
)
4242 : simple_ipa_opt_pass (pass_data_ipa_free_fn_summary
, ctxt
),
4246 /* opt_pass methods: */
4247 opt_pass
*clone () { return new pass_ipa_free_fn_summary (m_ctxt
); }
4248 void set_pass_param (unsigned int n
, bool param
)
4250 gcc_assert (n
== 0);
4253 virtual bool gate (function
*) { return true; }
4254 virtual unsigned int execute (function
*)
4256 ipa_free_fn_summary ();
4258 ipa_free_size_summary ();
4264 }; // class pass_ipa_free_fn_summary
4268 simple_ipa_opt_pass
*
4269 make_pass_ipa_free_fn_summary (gcc::context
*ctxt
)
4271 return new pass_ipa_free_fn_summary (ctxt
);
4276 const pass_data pass_data_ipa_fn_summary
=
4278 IPA_PASS
, /* type */
4279 "fnsummary", /* name */
4280 OPTGROUP_INLINE
, /* optinfo_flags */
4281 TV_IPA_FNSUMMARY
, /* tv_id */
4282 0, /* properties_required */
4283 0, /* properties_provided */
4284 0, /* properties_destroyed */
4285 0, /* todo_flags_start */
4286 ( TODO_dump_symtab
), /* todo_flags_finish */
4289 class pass_ipa_fn_summary
: public ipa_opt_pass_d
4292 pass_ipa_fn_summary (gcc::context
*ctxt
)
4293 : ipa_opt_pass_d (pass_data_ipa_fn_summary
, ctxt
,
4294 ipa_fn_summary_generate
, /* generate_summary */
4295 ipa_fn_summary_write
, /* write_summary */
4296 ipa_fn_summary_read
, /* read_summary */
4297 NULL
, /* write_optimization_summary */
4298 NULL
, /* read_optimization_summary */
4299 NULL
, /* stmt_fixup */
4300 0, /* function_transform_todo_flags_start */
4301 NULL
, /* function_transform */
4302 NULL
) /* variable_transform */
4305 /* opt_pass methods: */
4306 virtual unsigned int execute (function
*) { return 0; }
4308 }; // class pass_ipa_fn_summary
4313 make_pass_ipa_fn_summary (gcc::context
*ctxt
)
4315 return new pass_ipa_fn_summary (ctxt
);
4318 /* Reset all state within ipa-fnsummary.c so that we can rerun the compiler
4319 within the same process. For use by toplev::finalize. */
4322 ipa_fnsummary_c_finalize (void)
4324 ipa_free_fn_summary ();