]> git.ipfire.org Git - thirdparty/gcc.git/blame - gcc/ipa-fnsummary.c
vec: add exact argument for various grow functions.
[thirdparty/gcc.git] / gcc / ipa-fnsummary.c
CommitLineData
27d020cf 1/* Function summary pass.
8d9254fc 2 Copyright (C) 2003-2020 Free Software Foundation, Inc.
27d020cf
JH
3 Contributed by Jan Hubicka
4
5This file is part of GCC.
6
7GCC is free software; you can redistribute it and/or modify it under
8the terms of the GNU General Public License as published by the Free
9Software Foundation; either version 3, or (at your option) any later
10version.
11
12GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13WARRANTY; without even the implied warranty of MERCHANTABILITY or
14FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15for more details.
16
17You should have received a copy of the GNU General Public License
18along with GCC; see the file COPYING3. If not see
19<http://www.gnu.org/licenses/>. */
20
21/* Analysis of function bodies used by inter-procedural passes
22
23 We estimate for each function
24 - function body size and size after specializing into given context
25 - average function execution time in a given context
26 - function frame size
27 For each call
28 - call statement size, time and how often the parameters change
29
0bceb671 30 ipa_fn_summary data structures store above information locally (i.e.
27d020cf
JH
31 parameters of the function itself) and globally (i.e. parameters of
32 the function created by applying all the inline decisions already
33 present in the callgraph).
34
0bceb671 35 We provide access to the ipa_fn_summary data structure and
27d020cf
JH
36 basic logic updating the parameters when inlining is performed.
37
38 The summaries are context sensitive. Context means
39 1) partial assignment of known constant values of operands
40 2) whether function is inlined into the call or not.
41 It is easy to add more variants. To represent function size and time
42 that depends on context (i.e. it is known to be optimized away when
43 context is known either by inlining or from IP-CP and cloning),
44 we use predicates.
45
46 estimate_edge_size_and_time can be used to query
0bceb671 47 function size/time in the given context. ipa_merge_fn_summary_after_inlining merges
27d020cf
JH
48 properties of caller and callee after inlining.
49
50 Finally pass_inline_parameters is exported. This is used to drive
51 computation of function parameters used by the early inliner. IPA
52 inlined performs analysis via its analyze_function method. */
53
54#include "config.h"
85245bda 55#define INCLUDE_VECTOR
27d020cf
JH
56#include "system.h"
57#include "coretypes.h"
58#include "backend.h"
59#include "tree.h"
60#include "gimple.h"
61#include "alloc-pool.h"
62#include "tree-pass.h"
63#include "ssa.h"
64#include "tree-streamer.h"
65#include "cgraph.h"
66#include "diagnostic.h"
67#include "fold-const.h"
68#include "print-tree.h"
69#include "tree-inline.h"
70#include "gimple-pretty-print.h"
27d020cf
JH
71#include "cfganal.h"
72#include "gimple-iterator.h"
73#include "tree-cfg.h"
74#include "tree-ssa-loop-niter.h"
75#include "tree-ssa-loop.h"
76#include "symbol-summary.h"
77#include "ipa-prop.h"
78#include "ipa-fnsummary.h"
79#include "cfgloop.h"
80#include "tree-scalar-evolution.h"
81#include "ipa-utils.h"
27d020cf
JH
82#include "cfgexpand.h"
83#include "gimplify.h"
314e6352
ML
84#include "stringpool.h"
85#include "attribs.h"
ac0573de 86#include "tree-into-ssa.h"
27d020cf
JH
87
88/* Summaries. */
db30281f 89fast_function_summary <ipa_fn_summary *, va_gc> *ipa_fn_summaries;
f658ad30 90fast_function_summary <ipa_size_summary *, va_heap> *ipa_size_summaries;
db30281f 91fast_call_summary <ipa_call_summary *, va_heap> *ipa_call_summaries;
27d020cf
JH
92
93/* Edge predicates goes here. */
94static object_allocator<predicate> edge_predicate_pool ("edge predicates");
95
96
0bceb671 97/* Dump IPA hints. */
27d020cf 98void
0bceb671 99ipa_dump_hints (FILE *f, ipa_hints hints)
27d020cf
JH
100{
101 if (!hints)
102 return;
0bceb671 103 fprintf (f, "IPA hints:");
27d020cf
JH
104 if (hints & INLINE_HINT_indirect_call)
105 {
106 hints &= ~INLINE_HINT_indirect_call;
107 fprintf (f, " indirect_call");
108 }
109 if (hints & INLINE_HINT_loop_iterations)
110 {
111 hints &= ~INLINE_HINT_loop_iterations;
112 fprintf (f, " loop_iterations");
113 }
114 if (hints & INLINE_HINT_loop_stride)
115 {
116 hints &= ~INLINE_HINT_loop_stride;
117 fprintf (f, " loop_stride");
118 }
119 if (hints & INLINE_HINT_same_scc)
120 {
121 hints &= ~INLINE_HINT_same_scc;
122 fprintf (f, " same_scc");
123 }
124 if (hints & INLINE_HINT_in_scc)
125 {
126 hints &= ~INLINE_HINT_in_scc;
127 fprintf (f, " in_scc");
128 }
129 if (hints & INLINE_HINT_cross_module)
130 {
131 hints &= ~INLINE_HINT_cross_module;
132 fprintf (f, " cross_module");
133 }
134 if (hints & INLINE_HINT_declared_inline)
135 {
136 hints &= ~INLINE_HINT_declared_inline;
137 fprintf (f, " declared_inline");
138 }
27d020cf
JH
139 if (hints & INLINE_HINT_known_hot)
140 {
141 hints &= ~INLINE_HINT_known_hot;
142 fprintf (f, " known_hot");
143 }
144 gcc_assert (!hints);
145}
146
147
148/* Record SIZE and TIME to SUMMARY.
149 The accounted code will be executed when EXEC_PRED is true.
956d615d 150 When NONCONST_PRED is false the code will evaluate to constant and
070e3489
JH
151 will get optimized out in specialized clones of the function.
152 If CALL is true account to call_size_time_table rather than
153 size_time_table. */
27d020cf
JH
154
155void
0bceb671 156ipa_fn_summary::account_size_time (int size, sreal time,
27d020cf 157 const predicate &exec_pred,
070e3489
JH
158 const predicate &nonconst_pred_in,
159 bool call)
27d020cf
JH
160{
161 size_time_entry *e;
162 bool found = false;
163 int i;
164 predicate nonconst_pred;
070e3489
JH
165 vec<size_time_entry, va_gc> *table = call
166 ? call_size_time_table : size_time_table;
27d020cf
JH
167
168 if (exec_pred == false)
169 return;
170
171 nonconst_pred = nonconst_pred_in & exec_pred;
172
173 if (nonconst_pred == false)
174 return;
175
956d615d 176 /* We need to create initial empty unconditional clause, but otherwise
27d020cf 177 we don't need to account empty times and sizes. */
070e3489 178 if (!size && time == 0 && table)
27d020cf
JH
179 return;
180
956d615d 181 /* Only for calls we are unaccounting what we previously recorded. */
d2bcf46c 182 gcc_checking_assert (time >= 0 || call);
27d020cf 183
070e3489 184 for (i = 0; vec_safe_iterate (table, i, &e); i++)
27d020cf
JH
185 if (e->exec_predicate == exec_pred
186 && e->nonconst_predicate == nonconst_pred)
187 {
188 found = true;
189 break;
190 }
070e3489 191 if (i == max_size_time_table_size)
27d020cf
JH
192 {
193 i = 0;
194 found = true;
070e3489 195 e = &(*table)[0];
27d020cf
JH
196 if (dump_file && (dump_flags & TDF_DETAILS))
197 fprintf (dump_file,
198 "\t\tReached limit on number of entries, "
199 "ignoring the predicate.");
200 }
201 if (dump_file && (dump_flags & TDF_DETAILS) && (time != 0 || size))
202 {
203 fprintf (dump_file,
204 "\t\tAccounting size:%3.2f, time:%3.2f on %spredicate exec:",
0bceb671 205 ((double) size) / ipa_fn_summary::size_scale,
27d020cf
JH
206 (time.to_double ()), found ? "" : "new ");
207 exec_pred.dump (dump_file, conds, 0);
208 if (exec_pred != nonconst_pred)
209 {
210 fprintf (dump_file, " nonconst:");
211 nonconst_pred.dump (dump_file, conds);
212 }
213 else
214 fprintf (dump_file, "\n");
215 }
216 if (!found)
217 {
99b1c316 218 class size_time_entry new_entry;
27d020cf
JH
219 new_entry.size = size;
220 new_entry.time = time;
221 new_entry.exec_predicate = exec_pred;
222 new_entry.nonconst_predicate = nonconst_pred;
070e3489
JH
223 if (call)
224 vec_safe_push (call_size_time_table, new_entry);
225 else
226 vec_safe_push (size_time_table, new_entry);
27d020cf
JH
227 }
228 else
229 {
230 e->size += size;
231 e->time += time;
dd86c8da 232 /* FIXME: PR bootstrap/92653 gcc_checking_assert (e->time >= -1); */
d2bcf46c
JH
233 /* Tolerate small roundoff issues. */
234 if (e->time < 0)
235 e->time = 0;
27d020cf
JH
236 }
237}
238
956d615d 239/* We proved E to be unreachable, redirect it to __builtin_unreachable. */
27d020cf
JH
240
241static struct cgraph_edge *
242redirect_to_unreachable (struct cgraph_edge *e)
243{
244 struct cgraph_node *callee = !e->inline_failed ? e->callee : NULL;
245 struct cgraph_node *target = cgraph_node::get_create
246 (builtin_decl_implicit (BUILT_IN_UNREACHABLE));
247
248 if (e->speculative)
27c5a177 249 e = cgraph_edge::resolve_speculation (e, target->decl);
27d020cf 250 else if (!e->callee)
27c5a177 251 e = cgraph_edge::make_direct (e, target);
27d020cf
JH
252 else
253 e->redirect_callee (target);
99b1c316 254 class ipa_call_summary *es = ipa_call_summaries->get (e);
27d020cf 255 e->inline_failed = CIF_UNREACHABLE;
3995f3a2 256 e->count = profile_count::zero ();
27d020cf
JH
257 es->call_stmt_size = 0;
258 es->call_stmt_time = 0;
259 if (callee)
260 callee->remove_symbol_and_inline_clones ();
261 return e;
262}
263
264/* Set predicate for edge E. */
265
266static void
267edge_set_predicate (struct cgraph_edge *e, predicate *predicate)
268{
269 /* If the edge is determined to be never executed, redirect it
0bceb671
JH
270 to BUILTIN_UNREACHABLE to make it clear to IPA passes the call will
271 be optimized out. */
27d020cf
JH
272 if (predicate && *predicate == false
273 /* When handling speculative edges, we need to do the redirection
274 just once. Do it always on the direct edge, so we do not
275 attempt to resolve speculation while duplicating the edge. */
276 && (!e->speculative || e->callee))
277 e = redirect_to_unreachable (e);
278
99b1c316 279 class ipa_call_summary *es = ipa_call_summaries->get (e);
27d020cf
JH
280 if (predicate && *predicate != true)
281 {
282 if (!es->predicate)
283 es->predicate = edge_predicate_pool.allocate ();
284 *es->predicate = *predicate;
285 }
286 else
287 {
288 if (es->predicate)
289 edge_predicate_pool.remove (es->predicate);
290 es->predicate = NULL;
291 }
292}
293
294/* Set predicate for hint *P. */
295
296static void
297set_hint_predicate (predicate **p, predicate new_predicate)
298{
299 if (new_predicate == false || new_predicate == true)
300 {
301 if (*p)
302 edge_predicate_pool.remove (*p);
303 *p = NULL;
304 }
305 else
306 {
307 if (!*p)
308 *p = edge_predicate_pool.allocate ();
309 **p = new_predicate;
310 }
311}
312
313
956d615d 314/* Compute what conditions may or may not hold given information about
27d020cf 315 parameters. RET_CLAUSE returns truths that may hold in a specialized copy,
956d615d 316 while RET_NONSPEC_CLAUSE returns truths that may hold in an nonspecialized
27d020cf
JH
317 copy when called in a given context. It is a bitmask of conditions. Bit
318 0 means that condition is known to be false, while bit 1 means that condition
319 may or may not be true. These differs - for example NOT_INLINED condition
67914693 320 is always false in the second and also builtin_constant_p tests cannot use
27d020cf
JH
321 the fact that parameter is indeed a constant.
322
323 KNOWN_VALS is partial mapping of parameters of NODE to constant values.
956d615d 324 KNOWN_AGGS is a vector of aggregate known offset/value set for each
eb270950
FX
325 parameter. Return clause of possible truths. When INLINE_P is true, assume
326 that we are inlining.
27d020cf
JH
327
328 ERROR_MARK means compile time invariant. */
329
330static void
331evaluate_conditions_for_known_args (struct cgraph_node *node,
332 bool inline_p,
333 vec<tree> known_vals,
82c4b78d 334 vec<value_range> known_value_ranges,
eb270950 335 vec<ipa_agg_value_set> known_aggs,
27d020cf
JH
336 clause_t *ret_clause,
337 clause_t *ret_nonspec_clause)
338{
339 clause_t clause = inline_p ? 0 : 1 << predicate::not_inlined_condition;
340 clause_t nonspec_clause = 1 << predicate::not_inlined_condition;
99b1c316 341 class ipa_fn_summary *info = ipa_fn_summaries->get (node);
27d020cf
JH
342 int i;
343 struct condition *c;
344
345 for (i = 0; vec_safe_iterate (info->conds, i, &c); i++)
346 {
b0d55476 347 tree val = NULL;
27d020cf 348 tree res;
4307a485
FX
349 int j;
350 struct expr_eval_op *op;
27d020cf
JH
351
352 /* We allow call stmt to have fewer arguments than the callee function
353 (especially for K&R style programs). So bound check here (we assume
354 known_aggs vector, if non-NULL, has the same length as
355 known_vals). */
b0d55476 356 gcc_checking_assert (!known_aggs.length () || !known_vals.length ()
27d020cf 357 || (known_vals.length () == known_aggs.length ()));
27d020cf
JH
358
359 if (c->agg_contents)
360 {
eb270950 361 struct ipa_agg_value_set *agg;
27d020cf
JH
362
363 if (c->code == predicate::changed
364 && !c->by_ref
b0d55476 365 && c->operand_num < (int)known_vals.length ()
27d020cf
JH
366 && (known_vals[c->operand_num] == error_mark_node))
367 continue;
368
b0d55476 369 if (c->operand_num < (int)known_aggs.length ())
27d020cf 370 {
eb270950 371 agg = &known_aggs[c->operand_num];
b0d55476
JH
372 val = ipa_find_agg_cst_for_param (agg,
373 c->operand_num
374 < (int) known_vals.length ()
375 ? known_vals[c->operand_num]
376 : NULL,
27d020cf
JH
377 c->offset, c->by_ref);
378 }
379 else
380 val = NULL_TREE;
381 }
b0d55476 382 else if (c->operand_num < (int) known_vals.length ())
27d020cf
JH
383 {
384 val = known_vals[c->operand_num];
385 if (val == error_mark_node && c->code != predicate::changed)
386 val = NULL_TREE;
387 }
388
68718e8e
JH
389 if (!val
390 && (c->code == predicate::changed
391 || c->code == predicate::is_not_constant))
27d020cf
JH
392 {
393 clause |= 1 << (i + predicate::first_dynamic_condition);
394 nonspec_clause |= 1 << (i + predicate::first_dynamic_condition);
395 continue;
396 }
397 if (c->code == predicate::changed)
398 {
399 nonspec_clause |= 1 << (i + predicate::first_dynamic_condition);
400 continue;
401 }
402
27d020cf
JH
403 if (c->code == predicate::is_not_constant)
404 {
405 nonspec_clause |= 1 << (i + predicate::first_dynamic_condition);
406 continue;
407 }
408
68718e8e 409 if (val && TYPE_SIZE (c->type) == TYPE_SIZE (TREE_TYPE (val)))
4307a485 410 {
68718e8e
JH
411 if (c->type != TREE_TYPE (val))
412 val = fold_unary (VIEW_CONVERT_EXPR, c->type, val);
413 for (j = 0; vec_safe_iterate (c->param_ops, j, &op); j++)
414 {
415 if (!val)
416 break;
417 if (!op->val[0])
418 val = fold_unary (op->code, op->type, val);
419 else if (!op->val[1])
420 val = fold_binary (op->code, op->type,
421 op->index ? op->val[0] : val,
422 op->index ? val : op->val[0]);
423 else if (op->index == 0)
424 val = fold_ternary (op->code, op->type,
425 val, op->val[0], op->val[1]);
426 else if (op->index == 1)
427 val = fold_ternary (op->code, op->type,
428 op->val[0], val, op->val[1]);
429 else if (op->index == 2)
430 val = fold_ternary (op->code, op->type,
431 op->val[0], op->val[1], val);
432 else
433 val = NULL_TREE;
434 }
435
436 res = val
437 ? fold_binary_to_constant (c->code, boolean_type_node, val, c->val)
438 : NULL;
439
440 if (res && integer_zerop (res))
441 continue;
442 if (res && integer_onep (res))
443 {
444 clause |= 1 << (i + predicate::first_dynamic_condition);
445 nonspec_clause |= 1 << (i + predicate::first_dynamic_condition);
446 continue;
447 }
4307a485 448 }
82c4b78d 449 if (c->operand_num < (int) known_value_ranges.length ()
68718e8e
JH
450 && !c->agg_contents
451 && !known_value_ranges[c->operand_num].undefined_p ()
452 && !known_value_ranges[c->operand_num].varying_p ()
453 && TYPE_SIZE (c->type)
454 == TYPE_SIZE (known_value_ranges[c->operand_num].type ())
455 && (!val || TREE_CODE (val) != INTEGER_CST))
456 {
457 value_range vr = known_value_ranges[c->operand_num];
458 if (!useless_type_conversion_p (c->type, vr.type ()))
459 {
460 value_range res;
461 range_fold_unary_expr (&res, NOP_EXPR,
462 c->type, &vr, vr.type ());
463 vr = res;
464 }
465 tree type = c->type;
4307a485 466
68718e8e
JH
467 for (j = 0; vec_safe_iterate (c->param_ops, j, &op); j++)
468 {
469 if (vr.varying_p () || vr.undefined_p ())
470 break;
27d020cf 471
68718e8e
JH
472 value_range res;
473 if (!op->val[0])
474 range_fold_unary_expr (&res, op->code, op->type, &vr, type);
475 else if (!op->val[1])
476 {
477 value_range op0 (op->val[0], op->val[0]);
478 range_fold_binary_expr (&res, op->code, op->type,
479 op->index ? &op0 : &vr,
480 op->index ? &vr : &op0);
481 }
482 else
483 gcc_unreachable ();
484 type = op->type;
485 vr = res;
486 }
487 if (!vr.varying_p () && !vr.undefined_p ())
488 {
489 value_range res;
490 value_range val_vr (c->val, c->val);
491 range_fold_binary_expr (&res, c->code, boolean_type_node,
492 &vr,
493 &val_vr);
494 if (res.zero_p ())
495 continue;
496 }
497 }
27d020cf
JH
498
499 clause |= 1 << (i + predicate::first_dynamic_condition);
500 nonspec_clause |= 1 << (i + predicate::first_dynamic_condition);
501 }
502 *ret_clause = clause;
503 if (ret_nonspec_clause)
504 *ret_nonspec_clause = nonspec_clause;
505}
506
2523d721
JH
507/* Return true if VRP will be exectued on the function.
508 We do not want to anticipate optimizations that will not happen.
509
510 FIXME: This can be confused with -fdisable and debug counters and thus
511 it should not be used for correctness (only to make heuristics work).
512 This means that inliner should do its own optimizations of expressions
513 that it predicts to be constant so wrong code can not be triggered by
514 builtin_constant_p. */
515
516static bool
517vrp_will_run_p (struct cgraph_node *node)
518{
519 return (opt_for_fn (node->decl, optimize)
520 && !opt_for_fn (node->decl, optimize_debug)
521 && opt_for_fn (node->decl, flag_tree_vrp));
522}
523
524/* Similarly about FRE. */
525
526static bool
527fre_will_run_p (struct cgraph_node *node)
528{
529 return (opt_for_fn (node->decl, optimize)
530 && !opt_for_fn (node->decl, optimize_debug)
531 && opt_for_fn (node->decl, flag_tree_fre));
532}
27d020cf 533
b0d55476
JH
534/* Work out what conditions might be true at invocation of E.
535 Compute costs for inlined edge if INLINE_P is true.
536
956d615d 537 Return in CLAUSE_PTR the evaluated conditions and in NONSPEC_CLAUSE_PTR
b0d55476
JH
538 (if non-NULL) conditions evaluated for nonspecialized clone called
539 in a given context.
540
541 KNOWN_VALS_PTR and KNOWN_AGGS_PTR must be non-NULL and will be filled by
956d615d 542 known constant and aggregate values of parameters.
b0d55476
JH
543
544 KNOWN_CONTEXT_PTR, if non-NULL, will be filled by polymorphic call contexts
545 of parameter used by a polymorphic call. */
27d020cf
JH
546
547void
548evaluate_properties_for_edge (struct cgraph_edge *e, bool inline_p,
549 clause_t *clause_ptr,
550 clause_t *nonspec_clause_ptr,
551 vec<tree> *known_vals_ptr,
552 vec<ipa_polymorphic_call_context>
553 *known_contexts_ptr,
eb270950 554 vec<ipa_agg_value_set> *known_aggs_ptr)
27d020cf
JH
555{
556 struct cgraph_node *callee = e->callee->ultimate_alias_target ();
99b1c316 557 class ipa_fn_summary *info = ipa_fn_summaries->get (callee);
82c4b78d 558 auto_vec<value_range, 32> known_value_ranges;
a33c028e 559 class ipa_edge_args *args;
27d020cf
JH
560
561 if (clause_ptr)
562 *clause_ptr = inline_p ? 0 : 1 << predicate::not_inlined_condition;
27d020cf
JH
563
564 if (ipa_node_params_sum
565 && !e->call_stmt_cannot_inline_p
b0d55476 566 && (info->conds || known_contexts_ptr)
a33c028e 567 && (args = IPA_EDGE_REF (e)) != NULL)
27d020cf 568 {
eb270950 569 struct cgraph_node *caller;
b0d55476 570 class ipa_node_params *caller_parms_info, *callee_pi = NULL;
99b1c316 571 class ipa_call_summary *es = ipa_call_summaries->get (e);
27d020cf
JH
572 int i, count = ipa_get_cs_argument_count (args);
573
b0d55476
JH
574 if (count)
575 {
576 if (e->caller->inlined_to)
577 caller = e->caller->inlined_to;
578 else
579 caller = e->caller;
580 caller_parms_info = IPA_NODE_REF (caller);
581 callee_pi = IPA_NODE_REF (callee);
582
583 /* Watch for thunks. */
584 if (callee_pi)
585 /* Watch for variadic functions. */
586 count = MIN (count, ipa_get_param_count (callee_pi));
587 }
27d020cf 588
6cf67b62
JH
589 if (callee_pi)
590 for (i = 0; i < count; i++)
591 {
592 struct ipa_jump_func *jf = ipa_get_ith_jump_func (args, i);
6cf67b62 593
b0d55476
JH
594 if (ipa_is_param_used_by_indirect_call (callee_pi, i)
595 || ipa_is_param_used_by_ipa_predicates (callee_pi, i))
6cf67b62 596 {
b0d55476
JH
597 /* Determine if we know constant value of the parameter. */
598 tree cst = ipa_value_from_jfunc (caller_parms_info, jf,
599 ipa_get_type (callee_pi, i));
600
601 if (!cst && e->call_stmt
602 && i < (int)gimple_call_num_args (e->call_stmt))
603 {
604 cst = gimple_call_arg (e->call_stmt, i);
605 if (!is_gimple_min_invariant (cst))
606 cst = NULL;
607 }
608 if (cst)
609 {
610 gcc_checking_assert (TREE_CODE (cst) != TREE_BINFO);
611 if (!known_vals_ptr->length ())
cb3874dc 612 vec_safe_grow_cleared (known_vals_ptr, count, true);
b0d55476
JH
613 (*known_vals_ptr)[i] = cst;
614 }
615 else if (inline_p && !es->param[i].change_prob)
616 {
617 if (!known_vals_ptr->length ())
cb3874dc 618 vec_safe_grow_cleared (known_vals_ptr, count, true);
b0d55476
JH
619 (*known_vals_ptr)[i] = error_mark_node;
620 }
621
622 /* If we failed to get simple constant, try value range. */
623 if ((!cst || TREE_CODE (cst) != INTEGER_CST)
2523d721 624 && vrp_will_run_p (caller)
b0d55476
JH
625 && ipa_is_param_used_by_ipa_predicates (callee_pi, i))
626 {
627 value_range vr
628 = ipa_value_range_from_jfunc (caller_parms_info, e, jf,
629 ipa_get_type (callee_pi,
630 i));
631 if (!vr.undefined_p () && !vr.varying_p ())
632 {
82c4b78d 633 if (!known_value_ranges.length ())
4ba9fb0a 634 {
cb3874dc 635 known_value_ranges.safe_grow (count, true);
4ba9fb0a 636 for (int i = 0; i < count; ++i)
82c4b78d 637 new (&known_value_ranges[i]) value_range ();
4ba9fb0a 638 }
b0d55476
JH
639 known_value_ranges[i] = vr;
640 }
641 }
642
643 /* Determine known aggregate values. */
21e28527 644 if (fre_will_run_p (caller))
b0d55476 645 {
2523d721
JH
646 ipa_agg_value_set agg
647 = ipa_agg_value_set_from_jfunc (caller_parms_info,
648 caller, &jf->agg);
649 if (agg.items.length ())
650 {
651 if (!known_aggs_ptr->length ())
cb3874dc 652 vec_safe_grow_cleared (known_aggs_ptr, count, true);
2523d721
JH
653 (*known_aggs_ptr)[i] = agg;
654 }
b0d55476 655 }
6cf67b62 656 }
b0d55476
JH
657
658 /* For calls used in polymorphic calls we further determine
659 polymorphic call context. */
660 if (known_contexts_ptr
661 && ipa_is_param_used_by_polymorphic_call (callee_pi, i))
6cf67b62 662 {
b0d55476
JH
663 ipa_polymorphic_call_context
664 ctx = ipa_context_from_jfunc (caller_parms_info, e, i, jf);
665 if (!ctx.useless_p ())
666 {
667 if (!known_contexts_ptr->length ())
cb3874dc 668 known_contexts_ptr->safe_grow_cleared (count, true);
b0d55476
JH
669 (*known_contexts_ptr)[i]
670 = ipa_context_from_jfunc (caller_parms_info, e, i, jf);
671 }
672 }
6cf67b62
JH
673 }
674 else
b0d55476 675 gcc_assert (!count || callee->thunk.thunk_p);
27d020cf 676 }
b0d55476 677 else if (e->call_stmt && !e->call_stmt_cannot_inline_p && info->conds)
27d020cf
JH
678 {
679 int i, count = (int)gimple_call_num_args (e->call_stmt);
680
27d020cf
JH
681 for (i = 0; i < count; i++)
682 {
683 tree cst = gimple_call_arg (e->call_stmt, i);
684 if (!is_gimple_min_invariant (cst))
685 cst = NULL;
686 if (cst)
b0d55476
JH
687 {
688 if (!known_vals_ptr->length ())
cb3874dc 689 vec_safe_grow_cleared (known_vals_ptr, count, true);
b0d55476
JH
690 (*known_vals_ptr)[i] = cst;
691 }
27d020cf
JH
692 }
693 }
694
695 evaluate_conditions_for_known_args (callee, inline_p,
b0d55476 696 *known_vals_ptr,
68718e8e 697 known_value_ranges,
b0d55476
JH
698 *known_aggs_ptr,
699 clause_ptr,
27d020cf 700 nonspec_clause_ptr);
27d020cf
JH
701}
702
703
0bceb671 704/* Allocate the function summary. */
27d020cf
JH
705
706static void
0bceb671 707ipa_fn_summary_alloc (void)
27d020cf 708{
0bceb671 709 gcc_checking_assert (!ipa_fn_summaries);
44fca832 710 ipa_size_summaries = new ipa_size_summary_t (symtab);
7237f93e 711 ipa_fn_summaries = ipa_fn_summary_t::create_ggc (symtab);
db30281f 712 ipa_call_summaries = new ipa_call_summary_t (symtab);
27d020cf
JH
713}
714
56f62793 715ipa_call_summary::~ipa_call_summary ()
27d020cf 716{
27d020cf
JH
717 if (predicate)
718 edge_predicate_pool.remove (predicate);
56f62793 719
27d020cf
JH
720 param.release ();
721}
722
56f62793 723ipa_fn_summary::~ipa_fn_summary ()
27d020cf 724{
27d020cf 725 if (loop_iterations)
56f62793 726 edge_predicate_pool.remove (loop_iterations);
27d020cf 727 if (loop_stride)
56f62793 728 edge_predicate_pool.remove (loop_stride);
27d020cf
JH
729 vec_free (conds);
730 vec_free (size_time_table);
070e3489 731 vec_free (call_size_time_table);
27d020cf
JH
732}
733
27d020cf 734void
56f62793 735ipa_fn_summary_t::remove_callees (cgraph_node *node)
27d020cf 736{
56f62793
ML
737 cgraph_edge *e;
738 for (e = node->callees; e; e = e->next_callee)
739 ipa_call_summaries->remove (e);
740 for (e = node->indirect_calls; e; e = e->next_callee)
741 ipa_call_summaries->remove (e);
27d020cf
JH
742}
743
744/* Same as remap_predicate_after_duplication but handle hint predicate *P.
745 Additionally care about allocating new memory slot for updated predicate
746 and set it to NULL when it becomes true or false (and thus uninteresting).
747 */
748
749static void
750remap_hint_predicate_after_duplication (predicate **p,
751 clause_t possible_truths)
752{
753 predicate new_predicate;
754
755 if (!*p)
756 return;
757
758 new_predicate = (*p)->remap_after_duplication (possible_truths);
759 /* We do not want to free previous predicate; it is used by node origin. */
760 *p = NULL;
761 set_hint_predicate (p, new_predicate);
762}
763
764
765/* Hook that is called by cgraph.c when a node is duplicated. */
766void
0bceb671 767ipa_fn_summary_t::duplicate (cgraph_node *src,
27d020cf 768 cgraph_node *dst,
0bceb671
JH
769 ipa_fn_summary *,
770 ipa_fn_summary *info)
27d020cf 771{
56f62793 772 new (info) ipa_fn_summary (*ipa_fn_summaries->get (src));
27d020cf
JH
773 /* TODO: as an optimization, we may avoid copying conditions
774 that are known to be false or true. */
775 info->conds = vec_safe_copy (info->conds);
776
777 /* When there are any replacements in the function body, see if we can figure
778 out that something was optimized out. */
779 if (ipa_node_params_sum && dst->clone.tree_map)
780 {
781 vec<size_time_entry, va_gc> *entry = info->size_time_table;
782 /* Use SRC parm info since it may not be copied yet. */
99b1c316 783 class ipa_node_params *parms_info = IPA_NODE_REF (src);
27d020cf
JH
784 vec<tree> known_vals = vNULL;
785 int count = ipa_get_param_count (parms_info);
786 int i, j;
787 clause_t possible_truths;
788 predicate true_pred = true;
789 size_time_entry *e;
790 int optimized_out_size = 0;
791 bool inlined_to_p = false;
792 struct cgraph_edge *edge, *next;
793
794 info->size_time_table = 0;
cb3874dc 795 known_vals.safe_grow_cleared (count, true);
27d020cf
JH
796 for (i = 0; i < count; i++)
797 {
798 struct ipa_replace_map *r;
799
800 for (j = 0; vec_safe_iterate (dst->clone.tree_map, j, &r); j++)
801 {
ff6686d2 802 if (r->parm_num == i)
27d020cf
JH
803 {
804 known_vals[i] = r->new_tree;
805 break;
806 }
807 }
808 }
809 evaluate_conditions_for_known_args (dst, false,
810 known_vals,
82c4b78d 811 vNULL,
68718e8e 812 vNULL,
27d020cf
JH
813 &possible_truths,
814 /* We are going to specialize,
815 so ignore nonspec truths. */
816 NULL);
817 known_vals.release ();
818
819 info->account_size_time (0, 0, true_pred, true_pred);
820
821 /* Remap size_time vectors.
956d615d 822 Simplify the predicate by pruning out alternatives that are known
27d020cf
JH
823 to be false.
824 TODO: as on optimization, we can also eliminate conditions known
825 to be true. */
826 for (i = 0; vec_safe_iterate (entry, i, &e); i++)
827 {
828 predicate new_exec_pred;
829 predicate new_nonconst_pred;
830 new_exec_pred = e->exec_predicate.remap_after_duplication
831 (possible_truths);
832 new_nonconst_pred = e->nonconst_predicate.remap_after_duplication
833 (possible_truths);
834 if (new_exec_pred == false || new_nonconst_pred == false)
835 optimized_out_size += e->size;
836 else
837 info->account_size_time (e->size, e->time, new_exec_pred,
838 new_nonconst_pred);
839 }
840
841 /* Remap edge predicates with the same simplification as above.
842 Also copy constantness arrays. */
843 for (edge = dst->callees; edge; edge = next)
844 {
845 predicate new_predicate;
7237f93e 846 class ipa_call_summary *es = ipa_call_summaries->get (edge);
27d020cf
JH
847 next = edge->next_callee;
848
849 if (!edge->inline_failed)
850 inlined_to_p = true;
851 if (!es->predicate)
852 continue;
853 new_predicate = es->predicate->remap_after_duplication
854 (possible_truths);
855 if (new_predicate == false && *es->predicate != false)
0bceb671 856 optimized_out_size += es->call_stmt_size * ipa_fn_summary::size_scale;
27d020cf
JH
857 edge_set_predicate (edge, &new_predicate);
858 }
859
956d615d 860 /* Remap indirect edge predicates with the same simplification as above.
27d020cf
JH
861 Also copy constantness arrays. */
862 for (edge = dst->indirect_calls; edge; edge = next)
863 {
864 predicate new_predicate;
7237f93e 865 class ipa_call_summary *es = ipa_call_summaries->get (edge);
27d020cf
JH
866 next = edge->next_callee;
867
868 gcc_checking_assert (edge->inline_failed);
869 if (!es->predicate)
870 continue;
871 new_predicate = es->predicate->remap_after_duplication
872 (possible_truths);
873 if (new_predicate == false && *es->predicate != false)
0bceb671 874 optimized_out_size += es->call_stmt_size * ipa_fn_summary::size_scale;
27d020cf
JH
875 edge_set_predicate (edge, &new_predicate);
876 }
877 remap_hint_predicate_after_duplication (&info->loop_iterations,
878 possible_truths);
879 remap_hint_predicate_after_duplication (&info->loop_stride,
880 possible_truths);
27d020cf
JH
881
882 /* If inliner or someone after inliner will ever start producing
883 non-trivial clones, we will get trouble with lack of information
884 about updating self sizes, because size vectors already contains
956d615d 885 sizes of the callees. */
27d020cf
JH
886 gcc_assert (!inlined_to_p || !optimized_out_size);
887 }
888 else
889 {
890 info->size_time_table = vec_safe_copy (info->size_time_table);
891 if (info->loop_iterations)
892 {
893 predicate p = *info->loop_iterations;
894 info->loop_iterations = NULL;
895 set_hint_predicate (&info->loop_iterations, p);
896 }
897 if (info->loop_stride)
898 {
899 predicate p = *info->loop_stride;
900 info->loop_stride = NULL;
901 set_hint_predicate (&info->loop_stride, p);
902 }
27d020cf 903 }
a62bfab5 904 if (!dst->inlined_to)
0bceb671 905 ipa_update_overall_fn_summary (dst);
27d020cf
JH
906}
907
908
909/* Hook that is called by cgraph.c when a node is duplicated. */
910
911void
912ipa_call_summary_t::duplicate (struct cgraph_edge *src,
913 struct cgraph_edge *dst,
99b1c316
MS
914 class ipa_call_summary *srcinfo,
915 class ipa_call_summary *info)
27d020cf 916{
56f62793 917 new (info) ipa_call_summary (*srcinfo);
27d020cf
JH
918 info->predicate = NULL;
919 edge_set_predicate (dst, srcinfo->predicate);
920 info->param = srcinfo->param.copy ();
921 if (!dst->indirect_unknown_callee && src->indirect_unknown_callee)
922 {
923 info->call_stmt_size -= (eni_size_weights.indirect_call_cost
924 - eni_size_weights.call_cost);
925 info->call_stmt_time -= (eni_time_weights.indirect_call_cost
926 - eni_time_weights.call_cost);
927 }
928}
929
27d020cf
JH
930/* Dump edge summaries associated to NODE and recursively to all clones.
931 Indent by INDENT. */
932
933static void
934dump_ipa_call_summary (FILE *f, int indent, struct cgraph_node *node,
99b1c316 935 class ipa_fn_summary *info)
27d020cf
JH
936{
937 struct cgraph_edge *edge;
938 for (edge = node->callees; edge; edge = edge->next_callee)
939 {
99b1c316 940 class ipa_call_summary *es = ipa_call_summaries->get (edge);
27d020cf
JH
941 struct cgraph_node *callee = edge->callee->ultimate_alias_target ();
942 int i;
943
944 fprintf (f,
d597b944
ML
945 "%*s%s %s\n%*s freq:%4.2f",
946 indent, "", callee->dump_name (),
27d020cf
JH
947 !edge->inline_failed
948 ? "inlined" : cgraph_inline_failed_string (edge-> inline_failed),
7237f93e
JH
949 indent, "", edge->sreal_frequency ().to_double ());
950
b74d8dc4
JH
951 if (cross_module_call_p (edge))
952 fprintf (f, " cross module");
953
7237f93e
JH
954 if (es)
955 fprintf (f, " loop depth:%2i size:%2i time: %2i",
956 es->loop_depth, es->call_stmt_size, es->call_stmt_time);
56f62793
ML
957
958 ipa_fn_summary *s = ipa_fn_summaries->get (callee);
f658ad30 959 ipa_size_summary *ss = ipa_size_summaries->get (callee);
56f62793 960 if (s != NULL)
f658ad30
JH
961 fprintf (f, " callee size:%2i stack:%2i",
962 (int) (ss->size / ipa_fn_summary::size_scale),
56f62793 963 (int) s->estimated_stack_size);
27d020cf 964
7237f93e 965 if (es && es->predicate)
27d020cf
JH
966 {
967 fprintf (f, " predicate: ");
968 es->predicate->dump (f, info->conds);
969 }
970 else
971 fprintf (f, "\n");
7237f93e 972 if (es && es->param.exists ())
27d020cf
JH
973 for (i = 0; i < (int) es->param.length (); i++)
974 {
975 int prob = es->param[i].change_prob;
976
977 if (!prob)
978 fprintf (f, "%*s op%i is compile time invariant\n",
979 indent + 2, "", i);
980 else if (prob != REG_BR_PROB_BASE)
981 fprintf (f, "%*s op%i change %f%% of time\n", indent + 2, "", i,
982 prob * 100.0 / REG_BR_PROB_BASE);
983 }
984 if (!edge->inline_failed)
985 {
f658ad30
JH
986 ipa_size_summary *ss = ipa_size_summaries->get (callee);
987 fprintf (f, "%*sStack frame offset %i, callee self size %i\n",
27d020cf 988 indent + 2, "",
f658ad30
JH
989 (int) ipa_get_stack_frame_offset (callee),
990 (int) ss->estimated_self_stack_size);
27d020cf
JH
991 dump_ipa_call_summary (f, indent + 2, callee, info);
992 }
993 }
994 for (edge = node->indirect_calls; edge; edge = edge->next_callee)
995 {
99b1c316 996 class ipa_call_summary *es = ipa_call_summaries->get (edge);
41f0e819 997 fprintf (f, "%*sindirect call loop depth:%2i freq:%4.2f size:%2i"
27d020cf
JH
998 " time: %2i",
999 indent, "",
1000 es->loop_depth,
41f0e819
JH
1001 edge->sreal_frequency ().to_double (), es->call_stmt_size,
1002 es->call_stmt_time);
27d020cf
JH
1003 if (es->predicate)
1004 {
1005 fprintf (f, "predicate: ");
1006 es->predicate->dump (f, info->conds);
1007 }
1008 else
1009 fprintf (f, "\n");
1010 }
1011}
1012
1013
1014void
0bceb671 1015ipa_dump_fn_summary (FILE *f, struct cgraph_node *node)
27d020cf
JH
1016{
1017 if (node->definition)
1018 {
99b1c316 1019 class ipa_fn_summary *s = ipa_fn_summaries->get (node);
f658ad30 1020 class ipa_size_summary *ss = ipa_size_summaries->get (node);
56f62793 1021 if (s != NULL)
27d020cf 1022 {
56f62793
ML
1023 size_time_entry *e;
1024 int i;
1025 fprintf (f, "IPA function summary for %s", node->dump_name ());
1026 if (DECL_DISREGARD_INLINE_LIMITS (node->decl))
1027 fprintf (f, " always_inline");
1028 if (s->inlinable)
1029 fprintf (f, " inlinable");
1030 if (s->fp_expressions)
1031 fprintf (f, " fp_expression");
1032 fprintf (f, "\n global time: %f\n", s->time.to_double ());
f658ad30
JH
1033 fprintf (f, " self size: %i\n", ss->self_size);
1034 fprintf (f, " global size: %i\n", ss->size);
56f62793
ML
1035 fprintf (f, " min size: %i\n", s->min_size);
1036 fprintf (f, " self stack: %i\n",
f658ad30 1037 (int) ss->estimated_self_stack_size);
56f62793
ML
1038 fprintf (f, " global stack: %i\n", (int) s->estimated_stack_size);
1039 if (s->growth)
1040 fprintf (f, " estimated growth:%i\n", (int) s->growth);
1041 if (s->scc_no)
1042 fprintf (f, " In SCC: %i\n", (int) s->scc_no);
1043 for (i = 0; vec_safe_iterate (s->size_time_table, i, &e); i++)
1044 {
1045 fprintf (f, " size:%f, time:%f",
1046 (double) e->size / ipa_fn_summary::size_scale,
1047 e->time.to_double ());
1048 if (e->exec_predicate != true)
1049 {
1050 fprintf (f, ", executed if:");
1051 e->exec_predicate.dump (f, s->conds, 0);
1052 }
1053 if (e->exec_predicate != e->nonconst_predicate)
1054 {
1055 fprintf (f, ", nonconst if:");
1056 e->nonconst_predicate.dump (f, s->conds, 0);
1057 }
1058 fprintf (f, "\n");
1059 }
1060 if (s->loop_iterations)
27d020cf 1061 {
56f62793
ML
1062 fprintf (f, " loop iterations:");
1063 s->loop_iterations->dump (f, s->conds);
27d020cf 1064 }
56f62793 1065 if (s->loop_stride)
27d020cf 1066 {
56f62793
ML
1067 fprintf (f, " loop stride:");
1068 s->loop_stride->dump (f, s->conds);
27d020cf 1069 }
56f62793
ML
1070 fprintf (f, " calls:\n");
1071 dump_ipa_call_summary (f, 4, node, s);
27d020cf
JH
1072 fprintf (f, "\n");
1073 }
56f62793
ML
1074 else
1075 fprintf (f, "IPA summary for %s is missing.\n", node->dump_name ());
27d020cf
JH
1076 }
1077}
1078
1079DEBUG_FUNCTION void
0bceb671 1080ipa_debug_fn_summary (struct cgraph_node *node)
27d020cf 1081{
0bceb671 1082 ipa_dump_fn_summary (stderr, node);
27d020cf
JH
1083}
1084
1085void
0bceb671 1086ipa_dump_fn_summaries (FILE *f)
27d020cf
JH
1087{
1088 struct cgraph_node *node;
1089
1090 FOR_EACH_DEFINED_FUNCTION (node)
a62bfab5 1091 if (!node->inlined_to)
0bceb671 1092 ipa_dump_fn_summary (f, node);
27d020cf
JH
1093}
1094
1095/* Callback of walk_aliased_vdefs. Flags that it has been invoked to the
1096 boolean variable pointed to by DATA. */
1097
1098static bool
1099mark_modified (ao_ref *ao ATTRIBUTE_UNUSED, tree vdef ATTRIBUTE_UNUSED,
1100 void *data)
1101{
1102 bool *b = (bool *) data;
1103 *b = true;
1104 return true;
1105}
1106
1107/* If OP refers to value of function parameter, return the corresponding
1108 parameter. If non-NULL, the size of the memory load (or the SSA_NAME of the
1109 PARM_DECL) will be stored to *SIZE_P in that case too. */
1110
1111static tree
c628d1c3 1112unmodified_parm_1 (ipa_func_body_info *fbi, gimple *stmt, tree op,
86003645 1113 poly_int64 *size_p)
27d020cf
JH
1114{
1115 /* SSA_NAME referring to parm default def? */
1116 if (TREE_CODE (op) == SSA_NAME
1117 && SSA_NAME_IS_DEFAULT_DEF (op)
1118 && TREE_CODE (SSA_NAME_VAR (op)) == PARM_DECL)
1119 {
1120 if (size_p)
86003645 1121 *size_p = tree_to_poly_int64 (TYPE_SIZE (TREE_TYPE (op)));
27d020cf
JH
1122 return SSA_NAME_VAR (op);
1123 }
1124 /* Non-SSA parm reference? */
1125 if (TREE_CODE (op) == PARM_DECL)
1126 {
1127 bool modified = false;
1128
1129 ao_ref refd;
1130 ao_ref_init (&refd, op);
c628d1c3
MJ
1131 int walked = walk_aliased_vdefs (&refd, gimple_vuse (stmt),
1132 mark_modified, &modified, NULL, NULL,
1133 fbi->aa_walk_budget + 1);
1134 if (walked < 0)
1135 {
1136 fbi->aa_walk_budget = 0;
1137 return NULL_TREE;
1138 }
27d020cf
JH
1139 if (!modified)
1140 {
1141 if (size_p)
86003645 1142 *size_p = tree_to_poly_int64 (TYPE_SIZE (TREE_TYPE (op)));
27d020cf
JH
1143 return op;
1144 }
1145 }
1146 return NULL_TREE;
1147}
1148
1149/* If OP refers to value of function parameter, return the corresponding
1150 parameter. Also traverse chains of SSA register assignments. If non-NULL,
1151 the size of the memory load (or the SSA_NAME of the PARM_DECL) will be
1152 stored to *SIZE_P in that case too. */
1153
1154static tree
c628d1c3 1155unmodified_parm (ipa_func_body_info *fbi, gimple *stmt, tree op,
86003645 1156 poly_int64 *size_p)
27d020cf 1157{
c628d1c3 1158 tree res = unmodified_parm_1 (fbi, stmt, op, size_p);
27d020cf
JH
1159 if (res)
1160 return res;
1161
1162 if (TREE_CODE (op) == SSA_NAME
1163 && !SSA_NAME_IS_DEFAULT_DEF (op)
1164 && gimple_assign_single_p (SSA_NAME_DEF_STMT (op)))
c628d1c3 1165 return unmodified_parm (fbi, SSA_NAME_DEF_STMT (op),
27d020cf
JH
1166 gimple_assign_rhs1 (SSA_NAME_DEF_STMT (op)),
1167 size_p);
1168 return NULL_TREE;
1169}
1170
1171/* If OP refers to a value of a function parameter or value loaded from an
1172 aggregate passed to a parameter (either by value or reference), return TRUE
1173 and store the number of the parameter to *INDEX_P, the access size into
1174 *SIZE_P, and information whether and how it has been loaded from an
1175 aggregate into *AGGPOS. INFO describes the function parameters, STMT is the
1176 statement in which OP is used or loaded. */
1177
1178static bool
1179unmodified_parm_or_parm_agg_item (struct ipa_func_body_info *fbi,
1180 gimple *stmt, tree op, int *index_p,
86003645 1181 poly_int64 *size_p,
27d020cf
JH
1182 struct agg_position_info *aggpos)
1183{
c628d1c3 1184 tree res = unmodified_parm_1 (fbi, stmt, op, size_p);
27d020cf
JH
1185
1186 gcc_checking_assert (aggpos);
1187 if (res)
1188 {
1189 *index_p = ipa_get_param_decl_index (fbi->info, res);
1190 if (*index_p < 0)
1191 return false;
1192 aggpos->agg_contents = false;
1193 aggpos->by_ref = false;
1194 return true;
1195 }
1196
1197 if (TREE_CODE (op) == SSA_NAME)
1198 {
1199 if (SSA_NAME_IS_DEFAULT_DEF (op)
1200 || !gimple_assign_single_p (SSA_NAME_DEF_STMT (op)))
1201 return false;
1202 stmt = SSA_NAME_DEF_STMT (op);
1203 op = gimple_assign_rhs1 (stmt);
1204 if (!REFERENCE_CLASS_P (op))
1205 return unmodified_parm_or_parm_agg_item (fbi, stmt, op, index_p, size_p,
1206 aggpos);
1207 }
1208
1209 aggpos->agg_contents = true;
1210 return ipa_load_from_parm_agg (fbi, fbi->info->descriptors,
1211 stmt, op, index_p, &aggpos->offset,
1212 size_p, &aggpos->by_ref);
1213}
1214
1215/* See if statement might disappear after inlining.
1216 0 - means not eliminated
1217 1 - half of statements goes away
1218 2 - for sure it is eliminated.
1219 We are not terribly sophisticated, basically looking for simple abstraction
1220 penalty wrappers. */
1221
1222static int
c628d1c3 1223eliminated_by_inlining_prob (ipa_func_body_info *fbi, gimple *stmt)
27d020cf
JH
1224{
1225 enum gimple_code code = gimple_code (stmt);
1226 enum tree_code rhs_code;
1227
1228 if (!optimize)
1229 return 0;
1230
1231 switch (code)
1232 {
1233 case GIMPLE_RETURN:
1234 return 2;
1235 case GIMPLE_ASSIGN:
1236 if (gimple_num_ops (stmt) != 2)
1237 return 0;
1238
1239 rhs_code = gimple_assign_rhs_code (stmt);
1240
1241 /* Casts of parameters, loads from parameters passed by reference
1242 and stores to return value or parameters are often free after
956d615d 1243 inlining due to SRA and further combining.
27d020cf
JH
1244 Assume that half of statements goes away. */
1245 if (CONVERT_EXPR_CODE_P (rhs_code)
1246 || rhs_code == VIEW_CONVERT_EXPR
1247 || rhs_code == ADDR_EXPR
1248 || gimple_assign_rhs_class (stmt) == GIMPLE_SINGLE_RHS)
1249 {
1250 tree rhs = gimple_assign_rhs1 (stmt);
1251 tree lhs = gimple_assign_lhs (stmt);
1252 tree inner_rhs = get_base_address (rhs);
1253 tree inner_lhs = get_base_address (lhs);
1254 bool rhs_free = false;
1255 bool lhs_free = false;
1256
1257 if (!inner_rhs)
1258 inner_rhs = rhs;
1259 if (!inner_lhs)
1260 inner_lhs = lhs;
1261
1262 /* Reads of parameter are expected to be free. */
c628d1c3 1263 if (unmodified_parm (fbi, stmt, inner_rhs, NULL))
27d020cf
JH
1264 rhs_free = true;
1265 /* Match expressions of form &this->field. Those will most likely
1266 combine with something upstream after inlining. */
1267 else if (TREE_CODE (inner_rhs) == ADDR_EXPR)
1268 {
1269 tree op = get_base_address (TREE_OPERAND (inner_rhs, 0));
1270 if (TREE_CODE (op) == PARM_DECL)
1271 rhs_free = true;
1272 else if (TREE_CODE (op) == MEM_REF
c628d1c3
MJ
1273 && unmodified_parm (fbi, stmt, TREE_OPERAND (op, 0),
1274 NULL))
27d020cf
JH
1275 rhs_free = true;
1276 }
1277
1278 /* When parameter is not SSA register because its address is taken
1279 and it is just copied into one, the statement will be completely
1280 free after inlining (we will copy propagate backward). */
1281 if (rhs_free && is_gimple_reg (lhs))
1282 return 2;
1283
1284 /* Reads of parameters passed by reference
1285 expected to be free (i.e. optimized out after inlining). */
1286 if (TREE_CODE (inner_rhs) == MEM_REF
c628d1c3 1287 && unmodified_parm (fbi, stmt, TREE_OPERAND (inner_rhs, 0), NULL))
27d020cf
JH
1288 rhs_free = true;
1289
1290 /* Copying parameter passed by reference into gimple register is
1291 probably also going to copy propagate, but we can't be quite
1292 sure. */
1293 if (rhs_free && is_gimple_reg (lhs))
1294 lhs_free = true;
1295
1296 /* Writes to parameters, parameters passed by value and return value
956d615d 1297 (either directly or passed via invisible reference) are free.
27d020cf
JH
1298
1299 TODO: We ought to handle testcase like
1300 struct a {int a,b;};
1301 struct a
956d615d 1302 returnstruct (void)
27d020cf
JH
1303 {
1304 struct a a ={1,2};
1305 return a;
1306 }
1307
1308 This translate into:
1309
956d615d 1310 returnstruct ()
27d020cf
JH
1311 {
1312 int a$b;
1313 int a$a;
1314 struct a a;
1315 struct a D.2739;
1316
1317 <bb 2>:
1318 D.2739.a = 1;
1319 D.2739.b = 2;
1320 return D.2739;
1321
1322 }
1323 For that we either need to copy ipa-split logic detecting writes
1324 to return value. */
1325 if (TREE_CODE (inner_lhs) == PARM_DECL
1326 || TREE_CODE (inner_lhs) == RESULT_DECL
1327 || (TREE_CODE (inner_lhs) == MEM_REF
c628d1c3
MJ
1328 && (unmodified_parm (fbi, stmt, TREE_OPERAND (inner_lhs, 0),
1329 NULL)
27d020cf
JH
1330 || (TREE_CODE (TREE_OPERAND (inner_lhs, 0)) == SSA_NAME
1331 && SSA_NAME_VAR (TREE_OPERAND (inner_lhs, 0))
1332 && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND
1333 (inner_lhs,
1334 0))) == RESULT_DECL))))
1335 lhs_free = true;
1336 if (lhs_free
1337 && (is_gimple_reg (rhs) || is_gimple_min_invariant (rhs)))
1338 rhs_free = true;
1339 if (lhs_free && rhs_free)
1340 return 1;
1341 }
1342 return 0;
1343 default:
1344 return 0;
1345 }
1346}
1347
4307a485
FX
1348/* Analyze EXPR if it represents a series of simple operations performed on
1349 a function parameter and return true if so. FBI, STMT, EXPR, INDEX_P and
1350 AGGPOS have the same meaning like in unmodified_parm_or_parm_agg_item.
1351 Type of the parameter or load from an aggregate via the parameter is
1352 stored in *TYPE_P. Operations on the parameter are recorded to
1353 PARAM_OPS_P if it is not NULL. */
1354
1355static bool
1356decompose_param_expr (struct ipa_func_body_info *fbi,
1357 gimple *stmt, tree expr,
1358 int *index_p, tree *type_p,
1359 struct agg_position_info *aggpos,
1360 expr_eval_ops *param_ops_p = NULL)
1361{
fdfd7f53 1362 int op_limit = opt_for_fn (fbi->node->decl, param_ipa_max_param_expr_ops);
4307a485
FX
1363 int op_count = 0;
1364
1365 if (param_ops_p)
1366 *param_ops_p = NULL;
1367
1368 while (true)
1369 {
1370 expr_eval_op eval_op;
1371 unsigned rhs_count;
1372 unsigned cst_count = 0;
1373
1374 if (unmodified_parm_or_parm_agg_item (fbi, stmt, expr, index_p, NULL,
1375 aggpos))
1376 {
1377 tree type = TREE_TYPE (expr);
1378
1379 if (aggpos->agg_contents)
1380 {
1381 /* Stop if containing bit-field. */
1382 if (TREE_CODE (expr) == BIT_FIELD_REF
1383 || contains_bitfld_component_ref_p (expr))
1384 break;
1385 }
1386
1387 *type_p = type;
1388 return true;
1389 }
1390
1391 if (TREE_CODE (expr) != SSA_NAME || SSA_NAME_IS_DEFAULT_DEF (expr))
1392 break;
1393
1394 if (!is_gimple_assign (stmt = SSA_NAME_DEF_STMT (expr)))
1395 break;
1396
1397 switch (gimple_assign_rhs_class (stmt))
1398 {
1399 case GIMPLE_SINGLE_RHS:
1400 expr = gimple_assign_rhs1 (stmt);
1401 continue;
1402
1403 case GIMPLE_UNARY_RHS:
1404 rhs_count = 1;
1405 break;
1406
1407 case GIMPLE_BINARY_RHS:
1408 rhs_count = 2;
1409 break;
1410
1411 case GIMPLE_TERNARY_RHS:
1412 rhs_count = 3;
1413 break;
1414
1415 default:
1416 goto fail;
1417 }
1418
1419 /* Stop if expression is too complex. */
1420 if (op_count++ == op_limit)
1421 break;
1422
1423 if (param_ops_p)
1424 {
1425 eval_op.code = gimple_assign_rhs_code (stmt);
1426 eval_op.type = TREE_TYPE (gimple_assign_lhs (stmt));
1427 eval_op.val[0] = NULL_TREE;
1428 eval_op.val[1] = NULL_TREE;
1429 }
1430
1431 expr = NULL_TREE;
1432 for (unsigned i = 0; i < rhs_count; i++)
1433 {
1434 tree op = gimple_op (stmt, i + 1);
1435
1436 gcc_assert (op && !TYPE_P (op));
1437 if (is_gimple_ip_invariant (op))
1438 {
1439 if (++cst_count == rhs_count)
1440 goto fail;
1441
1442 eval_op.val[cst_count - 1] = op;
1443 }
1444 else if (!expr)
1445 {
1446 /* Found a non-constant operand, and record its index in rhs
1447 operands. */
1448 eval_op.index = i;
1449 expr = op;
1450 }
1451 else
1452 {
1453 /* Found more than one non-constant operands. */
1454 goto fail;
1455 }
1456 }
1457
1458 if (param_ops_p)
1459 vec_safe_insert (*param_ops_p, 0, eval_op);
1460 }
1461
1462 /* Failed to decompose, free resource and return. */
1463fail:
1464 if (param_ops_p)
1465 vec_free (*param_ops_p);
1466
1467 return false;
1468}
27d020cf
JH
1469
1470/* If BB ends by a conditional we can turn into predicates, attach corresponding
1471 predicates to the CFG edges. */
1472
1473static void
1474set_cond_stmt_execution_predicate (struct ipa_func_body_info *fbi,
99b1c316 1475 class ipa_fn_summary *summary,
40a777e8 1476 class ipa_node_params *params_summary,
27d020cf
JH
1477 basic_block bb)
1478{
1479 gimple *last;
4307a485 1480 tree op, op2;
27d020cf 1481 int index;
27d020cf
JH
1482 struct agg_position_info aggpos;
1483 enum tree_code code, inverted_code;
1484 edge e;
1485 edge_iterator ei;
1486 gimple *set_stmt;
4307a485
FX
1487 tree param_type;
1488 expr_eval_ops param_ops;
27d020cf
JH
1489
1490 last = last_stmt (bb);
1491 if (!last || gimple_code (last) != GIMPLE_COND)
1492 return;
1493 if (!is_gimple_ip_invariant (gimple_cond_rhs (last)))
1494 return;
1495 op = gimple_cond_lhs (last);
4307a485
FX
1496
1497 if (decompose_param_expr (fbi, last, op, &index, &param_type, &aggpos,
1498 &param_ops))
27d020cf
JH
1499 {
1500 code = gimple_cond_code (last);
1501 inverted_code = invert_tree_comparison (code, HONOR_NANS (op));
1502
1503 FOR_EACH_EDGE (e, ei, bb->succs)
1504 {
1505 enum tree_code this_code = (e->flags & EDGE_TRUE_VALUE
1506 ? code : inverted_code);
1507 /* invert_tree_comparison will return ERROR_MARK on FP
956d615d 1508 comparisons that are not EQ/NE instead of returning proper
efe12656
FX
1509 unordered one. Be sure it is not confused with NON_CONSTANT.
1510
1511 And if the edge's target is the final block of diamond CFG graph
1512 of this conditional statement, we do not need to compute
1513 predicate for the edge because the final block's predicate must
1514 be at least as that of the first block of the statement. */
1515 if (this_code != ERROR_MARK
1516 && !dominated_by_p (CDI_POST_DOMINATORS, bb, e->dest))
27d020cf
JH
1517 {
1518 predicate p
40a777e8
JH
1519 = add_condition (summary, params_summary, index,
1520 param_type, &aggpos,
4307a485 1521 this_code, gimple_cond_rhs (last), param_ops);
27d020cf
JH
1522 e->aux = edge_predicate_pool.allocate ();
1523 *(predicate *) e->aux = p;
1524 }
1525 }
4307a485 1526 vec_free (param_ops);
27d020cf
JH
1527 }
1528
1529 if (TREE_CODE (op) != SSA_NAME)
1530 return;
1531 /* Special case
1532 if (builtin_constant_p (op))
1533 constant_code
1534 else
1535 nonconstant_code.
1536 Here we can predicate nonconstant_code. We can't
1537 really handle constant_code since we have no predicate
1538 for this and also the constant code is not known to be
956d615d 1539 optimized away when inliner doesn't see operand is constant.
27d020cf
JH
1540 Other optimizers might think otherwise. */
1541 if (gimple_cond_code (last) != NE_EXPR
1542 || !integer_zerop (gimple_cond_rhs (last)))
1543 return;
1544 set_stmt = SSA_NAME_DEF_STMT (op);
1545 if (!gimple_call_builtin_p (set_stmt, BUILT_IN_CONSTANT_P)
1546 || gimple_call_num_args (set_stmt) != 1)
1547 return;
1548 op2 = gimple_call_arg (set_stmt, 0);
4307a485 1549 if (!decompose_param_expr (fbi, set_stmt, op2, &index, &param_type, &aggpos))
27d020cf
JH
1550 return;
1551 FOR_EACH_EDGE (e, ei, bb->succs) if (e->flags & EDGE_FALSE_VALUE)
1552 {
40a777e8
JH
1553 predicate p = add_condition (summary, params_summary, index,
1554 param_type, &aggpos,
27d020cf
JH
1555 predicate::is_not_constant, NULL_TREE);
1556 e->aux = edge_predicate_pool.allocate ();
1557 *(predicate *) e->aux = p;
1558 }
1559}
1560
1561
1562/* If BB ends by a switch we can turn into predicates, attach corresponding
1563 predicates to the CFG edges. */
1564
1565static void
1566set_switch_stmt_execution_predicate (struct ipa_func_body_info *fbi,
99b1c316 1567 class ipa_fn_summary *summary,
40a777e8 1568 class ipa_node_params *params_summary,
27d020cf
JH
1569 basic_block bb)
1570{
1571 gimple *lastg;
1572 tree op;
1573 int index;
27d020cf
JH
1574 struct agg_position_info aggpos;
1575 edge e;
1576 edge_iterator ei;
1577 size_t n;
1578 size_t case_idx;
4307a485
FX
1579 tree param_type;
1580 expr_eval_ops param_ops;
27d020cf
JH
1581
1582 lastg = last_stmt (bb);
1583 if (!lastg || gimple_code (lastg) != GIMPLE_SWITCH)
1584 return;
1585 gswitch *last = as_a <gswitch *> (lastg);
1586 op = gimple_switch_index (last);
4307a485
FX
1587 if (!decompose_param_expr (fbi, last, op, &index, &param_type, &aggpos,
1588 &param_ops))
27d020cf
JH
1589 return;
1590
351e7c3b
FX
1591 auto_vec<std::pair<tree, tree> > ranges;
1592 tree type = TREE_TYPE (op);
fdfd7f53
ML
1593 int bound_limit = opt_for_fn (fbi->node->decl,
1594 param_ipa_max_switch_predicate_bounds);
351e7c3b
FX
1595 int bound_count = 0;
1596 wide_int vr_wmin, vr_wmax;
1597 value_range_kind vr_type = get_range_info (op, &vr_wmin, &vr_wmax);
1598
27d020cf
JH
1599 FOR_EACH_EDGE (e, ei, bb->succs)
1600 {
1601 e->aux = edge_predicate_pool.allocate ();
1602 *(predicate *) e->aux = false;
1603 }
351e7c3b 1604
efe12656
FX
1605 e = gimple_switch_edge (cfun, last, 0);
1606 /* Set BOUND_COUNT to maximum count to bypass computing predicate for
1607 default case if its target basic block is in convergence point of all
1608 switch cases, which can be determined by checking whether it
1609 post-dominates the switch statement. */
1610 if (dominated_by_p (CDI_POST_DOMINATORS, bb, e->dest))
1611 bound_count = INT_MAX;
1612
27d020cf 1613 n = gimple_switch_num_labels (last);
351e7c3b 1614 for (case_idx = 1; case_idx < n; ++case_idx)
27d020cf
JH
1615 {
1616 tree cl = gimple_switch_label (last, case_idx);
efe12656
FX
1617 tree min = CASE_LOW (cl);
1618 tree max = CASE_HIGH (cl);
27d020cf
JH
1619 predicate p;
1620
4307a485
FX
1621 e = gimple_switch_edge (cfun, last, case_idx);
1622
efe12656
FX
1623 /* The case value might not have same type as switch expression,
1624 extend the value based on the expression type. */
1625 if (TREE_TYPE (min) != type)
1626 min = wide_int_to_tree (type, wi::to_wide (min));
27d020cf 1627
351e7c3b 1628 if (!max)
efe12656
FX
1629 max = min;
1630 else if (TREE_TYPE (max) != type)
1631 max = wide_int_to_tree (type, wi::to_wide (max));
1632
1633 /* The case's target basic block is in convergence point of all switch
1634 cases, its predicate should be at least as that of the switch
1635 statement. */
1636 if (dominated_by_p (CDI_POST_DOMINATORS, bb, e->dest))
1637 p = true;
1638 else if (min == max)
40a777e8
JH
1639 p = add_condition (summary, params_summary, index, param_type,
1640 &aggpos, EQ_EXPR, min, param_ops);
27d020cf
JH
1641 else
1642 {
1643 predicate p1, p2;
40a777e8
JH
1644 p1 = add_condition (summary, params_summary, index, param_type,
1645 &aggpos, GE_EXPR, min, param_ops);
1646 p2 = add_condition (summary, params_summary,index, param_type,
1647 &aggpos, LE_EXPR, max, param_ops);
27d020cf
JH
1648 p = p1 & p2;
1649 }
99b1c316
MS
1650 *(class predicate *) e->aux
1651 = p.or_with (summary->conds, *(class predicate *) e->aux);
351e7c3b
FX
1652
1653 /* If there are too many disjoint case ranges, predicate for default
1654 case might become too complicated. So add a limit here. */
1655 if (bound_count > bound_limit)
1656 continue;
1657
1658 bool new_range = true;
1659
1660 if (!ranges.is_empty ())
1661 {
1662 wide_int curr_wmin = wi::to_wide (min);
1663 wide_int last_wmax = wi::to_wide (ranges.last ().second);
1664
1665 /* Merge case ranges if they are continuous. */
1666 if (curr_wmin == last_wmax + 1)
1667 new_range = false;
1668 else if (vr_type == VR_ANTI_RANGE)
1669 {
1670 /* If two disjoint case ranges can be connected by anti-range
1671 of switch index, combine them to one range. */
1672 if (wi::lt_p (vr_wmax, curr_wmin - 1, TYPE_SIGN (type)))
1673 vr_type = VR_UNDEFINED;
1674 else if (wi::le_p (vr_wmin, last_wmax + 1, TYPE_SIGN (type)))
1675 new_range = false;
1676 }
1677 }
1678
351e7c3b
FX
1679 /* Create/extend a case range. And we count endpoints of range set,
1680 this number nearly equals to number of conditions that we will create
1681 for predicate of default case. */
1682 if (new_range)
1683 {
1684 bound_count += (min == max) ? 1 : 2;
1685 ranges.safe_push (std::make_pair (min, max));
1686 }
1687 else
1688 {
1689 bound_count += (ranges.last ().first == ranges.last ().second);
1690 ranges.last ().second = max;
1691 }
1692 }
1693
1694 e = gimple_switch_edge (cfun, last, 0);
1695 if (bound_count > bound_limit)
1696 {
1697 *(class predicate *) e->aux = true;
4307a485 1698 vec_free (param_ops);
351e7c3b 1699 return;
27d020cf 1700 }
351e7c3b
FX
1701
1702 predicate p_seg = true;
1703 predicate p_all = false;
1704
1705 if (vr_type != VR_RANGE)
1706 {
1707 vr_wmin = wi::to_wide (TYPE_MIN_VALUE (type));
1708 vr_wmax = wi::to_wide (TYPE_MAX_VALUE (type));
1709 }
1710
1711 /* Construct predicate to represent default range set that is negation of
1712 all case ranges. Case range is classified as containing single/non-single
1713 values. Suppose a piece of case ranges in the following.
1714
1715 [D1...D2] [S1] ... [Sn] [D3...D4]
1716
1717 To represent default case's range sets between two non-single value
1718 case ranges (From D2 to D3), we construct predicate as:
1719
1720 D2 < x < D3 && x != S1 && ... && x != Sn
1721 */
1722 for (size_t i = 0; i < ranges.length (); i++)
1723 {
1724 tree min = ranges[i].first;
1725 tree max = ranges[i].second;
1726
1727 if (min == max)
40a777e8
JH
1728 p_seg &= add_condition (summary, params_summary, index,
1729 param_type, &aggpos, NE_EXPR,
4307a485 1730 min, param_ops);
351e7c3b
FX
1731 else
1732 {
1733 /* Do not create sub-predicate for range that is beyond low bound
1734 of switch index. */
1735 if (wi::lt_p (vr_wmin, wi::to_wide (min), TYPE_SIGN (type)))
1736 {
40a777e8
JH
1737 p_seg &= add_condition (summary, params_summary, index,
1738 param_type, &aggpos,
4307a485 1739 LT_EXPR, min, param_ops);
351e7c3b
FX
1740 p_all = p_all.or_with (summary->conds, p_seg);
1741 }
1742
1743 /* Do not create sub-predicate for range that is beyond up bound
1744 of switch index. */
1745 if (wi::le_p (vr_wmax, wi::to_wide (max), TYPE_SIGN (type)))
1746 {
1747 p_seg = false;
1748 break;
1749 }
1750
40a777e8
JH
1751 p_seg = add_condition (summary, params_summary, index,
1752 param_type, &aggpos, GT_EXPR,
4307a485 1753 max, param_ops);
351e7c3b
FX
1754 }
1755 }
1756
1757 p_all = p_all.or_with (summary->conds, p_seg);
1758 *(class predicate *) e->aux
1759 = p_all.or_with (summary->conds, *(class predicate *) e->aux);
4307a485
FX
1760
1761 vec_free (param_ops);
27d020cf
JH
1762}
1763
1764
1765/* For each BB in NODE attach to its AUX pointer predicate under
1766 which it is executable. */
1767
1768static void
1769compute_bb_predicates (struct ipa_func_body_info *fbi,
1770 struct cgraph_node *node,
40a777e8
JH
1771 class ipa_fn_summary *summary,
1772 class ipa_node_params *params_summary)
27d020cf
JH
1773{
1774 struct function *my_function = DECL_STRUCT_FUNCTION (node->decl);
1775 bool done = false;
1776 basic_block bb;
1777
1778 FOR_EACH_BB_FN (bb, my_function)
1779 {
40a777e8
JH
1780 set_cond_stmt_execution_predicate (fbi, summary, params_summary, bb);
1781 set_switch_stmt_execution_predicate (fbi, summary, params_summary, bb);
27d020cf
JH
1782 }
1783
1784 /* Entry block is always executable. */
1785 ENTRY_BLOCK_PTR_FOR_FN (my_function)->aux
1786 = edge_predicate_pool.allocate ();
1787 *(predicate *) ENTRY_BLOCK_PTR_FOR_FN (my_function)->aux = true;
1788
1789 /* A simple dataflow propagation of predicates forward in the CFG.
1790 TODO: work in reverse postorder. */
1791 while (!done)
1792 {
1793 done = true;
1794 FOR_EACH_BB_FN (bb, my_function)
1795 {
1796 predicate p = false;
1797 edge e;
1798 edge_iterator ei;
1799 FOR_EACH_EDGE (e, ei, bb->preds)
1800 {
1801 if (e->src->aux)
1802 {
1803 predicate this_bb_predicate
1804 = *(predicate *) e->src->aux;
1805 if (e->aux)
99b1c316 1806 this_bb_predicate &= (*(class predicate *) e->aux);
27d020cf
JH
1807 p = p.or_with (summary->conds, this_bb_predicate);
1808 if (p == true)
1809 break;
1810 }
1811 }
efe12656 1812 if (p != false)
27d020cf 1813 {
efe12656
FX
1814 basic_block pdom_bb;
1815
27d020cf
JH
1816 if (!bb->aux)
1817 {
1818 done = false;
1819 bb->aux = edge_predicate_pool.allocate ();
1820 *((predicate *) bb->aux) = p;
1821 }
1822 else if (p != *(predicate *) bb->aux)
1823 {
1824 /* This OR operation is needed to ensure monotonous data flow
1825 in the case we hit the limit on number of clauses and the
1826 and/or operations above give approximate answers. */
1827 p = p.or_with (summary->conds, *(predicate *)bb->aux);
1828 if (p != *(predicate *) bb->aux)
1829 {
1830 done = false;
1831 *((predicate *) bb->aux) = p;
1832 }
1833 }
efe12656
FX
1834
1835 /* For switch/if statement, we can OR-combine predicates of all
1836 its cases/branches to get predicate for basic block in their
1837 convergence point, but sometimes this will generate very
1838 complicated predicate. Actually, we can get simplified
1839 predicate in another way by using the fact that predicate
1840 for a basic block must also hold true for its post dominators.
1841 To be specific, basic block in convergence point of
1842 conditional statement should include predicate of the
1843 statement. */
1844 pdom_bb = get_immediate_dominator (CDI_POST_DOMINATORS, bb);
1845 if (pdom_bb == EXIT_BLOCK_PTR_FOR_FN (my_function) || !pdom_bb)
1846 ;
1847 else if (!pdom_bb->aux)
1848 {
1849 done = false;
1850 pdom_bb->aux = edge_predicate_pool.allocate ();
1851 *((predicate *) pdom_bb->aux) = p;
1852 }
1853 else if (p != *(predicate *) pdom_bb->aux)
1854 {
1855 p = p.or_with (summary->conds, *(predicate *)pdom_bb->aux);
1856 if (p != *(predicate *) pdom_bb->aux)
1857 {
1858 done = false;
1859 *((predicate *) pdom_bb->aux) = p;
1860 }
1861 }
27d020cf
JH
1862 }
1863 }
1864 }
1865}
1866
1867
1868/* Return predicate specifying when the STMT might have result that is not
1869 a compile time constant. */
1870
1871static predicate
c628d1c3 1872will_be_nonconstant_expr_predicate (ipa_func_body_info *fbi,
99b1c316 1873 class ipa_fn_summary *summary,
40a777e8 1874 class ipa_node_params *params_summary,
27d020cf
JH
1875 tree expr,
1876 vec<predicate> nonconstant_names)
1877{
1878 tree parm;
1879 int index;
27d020cf
JH
1880
1881 while (UNARY_CLASS_P (expr))
1882 expr = TREE_OPERAND (expr, 0);
1883
4307a485 1884 parm = unmodified_parm (fbi, NULL, expr, NULL);
c628d1c3 1885 if (parm && (index = ipa_get_param_decl_index (fbi->info, parm)) >= 0)
40a777e8 1886 return add_condition (summary, params_summary, index, TREE_TYPE (parm), NULL,
4307a485 1887 predicate::changed, NULL_TREE);
27d020cf
JH
1888 if (is_gimple_min_invariant (expr))
1889 return false;
1890 if (TREE_CODE (expr) == SSA_NAME)
1891 return nonconstant_names[SSA_NAME_VERSION (expr)];
1892 if (BINARY_CLASS_P (expr) || COMPARISON_CLASS_P (expr))
1893 {
c628d1c3
MJ
1894 predicate p1
1895 = will_be_nonconstant_expr_predicate (fbi, summary,
40a777e8 1896 params_summary,
c628d1c3
MJ
1897 TREE_OPERAND (expr, 0),
1898 nonconstant_names);
27d020cf
JH
1899 if (p1 == true)
1900 return p1;
1901
c628d1c3
MJ
1902 predicate p2
1903 = will_be_nonconstant_expr_predicate (fbi, summary,
40a777e8 1904 params_summary,
c628d1c3
MJ
1905 TREE_OPERAND (expr, 1),
1906 nonconstant_names);
27d020cf
JH
1907 return p1.or_with (summary->conds, p2);
1908 }
1909 else if (TREE_CODE (expr) == COND_EXPR)
1910 {
c628d1c3
MJ
1911 predicate p1
1912 = will_be_nonconstant_expr_predicate (fbi, summary,
40a777e8 1913 params_summary,
c628d1c3
MJ
1914 TREE_OPERAND (expr, 0),
1915 nonconstant_names);
27d020cf
JH
1916 if (p1 == true)
1917 return p1;
1918
c628d1c3
MJ
1919 predicate p2
1920 = will_be_nonconstant_expr_predicate (fbi, summary,
40a777e8 1921 params_summary,
c628d1c3
MJ
1922 TREE_OPERAND (expr, 1),
1923 nonconstant_names);
27d020cf
JH
1924 if (p2 == true)
1925 return p2;
1926 p1 = p1.or_with (summary->conds, p2);
c628d1c3 1927 p2 = will_be_nonconstant_expr_predicate (fbi, summary,
40a777e8 1928 params_summary,
27d020cf
JH
1929 TREE_OPERAND (expr, 2),
1930 nonconstant_names);
1931 return p2.or_with (summary->conds, p1);
1932 }
5126ae0c
KV
1933 else if (TREE_CODE (expr) == CALL_EXPR)
1934 return true;
27d020cf
JH
1935 else
1936 {
1937 debug_tree (expr);
1938 gcc_unreachable ();
1939 }
1940 return false;
1941}
1942
1943
1944/* Return predicate specifying when the STMT might have result that is not
1945 a compile time constant. */
1946
1947static predicate
1948will_be_nonconstant_predicate (struct ipa_func_body_info *fbi,
99b1c316 1949 class ipa_fn_summary *summary,
40a777e8 1950 class ipa_node_params *params_summary,
27d020cf
JH
1951 gimple *stmt,
1952 vec<predicate> nonconstant_names)
1953{
1954 predicate p = true;
1955 ssa_op_iter iter;
1956 tree use;
4307a485 1957 tree param_type = NULL_TREE;
27d020cf
JH
1958 predicate op_non_const;
1959 bool is_load;
1960 int base_index;
27d020cf
JH
1961 struct agg_position_info aggpos;
1962
956d615d 1963 /* What statements might be optimized away
27d020cf
JH
1964 when their arguments are constant. */
1965 if (gimple_code (stmt) != GIMPLE_ASSIGN
1966 && gimple_code (stmt) != GIMPLE_COND
1967 && gimple_code (stmt) != GIMPLE_SWITCH
1968 && (gimple_code (stmt) != GIMPLE_CALL
1969 || !(gimple_call_flags (stmt) & ECF_CONST)))
1970 return p;
1971
1972 /* Stores will stay anyway. */
1973 if (gimple_store_p (stmt))
1974 return p;
1975
1976 is_load = gimple_assign_load_p (stmt);
1977
1978 /* Loads can be optimized when the value is known. */
1979 if (is_load)
1980 {
4307a485
FX
1981 tree op = gimple_assign_rhs1 (stmt);
1982 if (!decompose_param_expr (fbi, stmt, op, &base_index, &param_type,
1983 &aggpos))
27d020cf
JH
1984 return p;
1985 }
1986 else
1987 base_index = -1;
1988
1989 /* See if we understand all operands before we start
1990 adding conditionals. */
1991 FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE)
1992 {
c628d1c3 1993 tree parm = unmodified_parm (fbi, stmt, use, NULL);
27d020cf
JH
1994 /* For arguments we can build a condition. */
1995 if (parm && ipa_get_param_decl_index (fbi->info, parm) >= 0)
1996 continue;
1997 if (TREE_CODE (use) != SSA_NAME)
1998 return p;
1999 /* If we know when operand is constant,
2000 we still can say something useful. */
2001 if (nonconstant_names[SSA_NAME_VERSION (use)] != true)
2002 continue;
2003 return p;
2004 }
2005
2006 if (is_load)
2007 op_non_const =
40a777e8
JH
2008 add_condition (summary, params_summary,
2009 base_index, param_type, &aggpos,
4307a485 2010 predicate::changed, NULL_TREE);
27d020cf
JH
2011 else
2012 op_non_const = false;
2013 FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE)
2014 {
4307a485 2015 tree parm = unmodified_parm (fbi, stmt, use, NULL);
27d020cf
JH
2016 int index;
2017
2018 if (parm && (index = ipa_get_param_decl_index (fbi->info, parm)) >= 0)
2019 {
2020 if (index != base_index)
40a777e8
JH
2021 p = add_condition (summary, params_summary, index,
2022 TREE_TYPE (parm), NULL,
4307a485 2023 predicate::changed, NULL_TREE);
27d020cf
JH
2024 else
2025 continue;
2026 }
2027 else
2028 p = nonconstant_names[SSA_NAME_VERSION (use)];
2029 op_non_const = p.or_with (summary->conds, op_non_const);
2030 }
2031 if ((gimple_code (stmt) == GIMPLE_ASSIGN || gimple_code (stmt) == GIMPLE_CALL)
2032 && gimple_op (stmt, 0)
2033 && TREE_CODE (gimple_op (stmt, 0)) == SSA_NAME)
2034 nonconstant_names[SSA_NAME_VERSION (gimple_op (stmt, 0))]
2035 = op_non_const;
2036 return op_non_const;
2037}
2038
2039struct record_modified_bb_info
2040{
3b2a6901 2041 tree op;
27d020cf
JH
2042 bitmap bb_set;
2043 gimple *stmt;
2044};
2045
956d615d 2046/* Value is initialized in INIT_BB and used in USE_BB. We want to compute
27d020cf 2047 probability how often it changes between USE_BB.
3b2a6901 2048 INIT_BB->count/USE_BB->count is an estimate, but if INIT_BB
27d020cf
JH
2049 is in different loop nest, we can do better.
2050 This is all just estimate. In theory we look for minimal cut separating
2051 INIT_BB and USE_BB, but we only want to anticipate loop invariant motion
2052 anyway. */
2053
2054static basic_block
2055get_minimal_bb (basic_block init_bb, basic_block use_bb)
2056{
99b1c316 2057 class loop *l = find_common_loop (init_bb->loop_father, use_bb->loop_father);
e7a74006 2058 if (l && l->header->count < init_bb->count)
27d020cf
JH
2059 return l->header;
2060 return init_bb;
2061}
2062
2063/* Callback of walk_aliased_vdefs. Records basic blocks where the value may be
2064 set except for info->stmt. */
2065
2066static bool
2067record_modified (ao_ref *ao ATTRIBUTE_UNUSED, tree vdef, void *data)
2068{
2069 struct record_modified_bb_info *info =
2070 (struct record_modified_bb_info *) data;
2071 if (SSA_NAME_DEF_STMT (vdef) == info->stmt)
2072 return false;
3b2a6901
JH
2073 if (gimple_clobber_p (SSA_NAME_DEF_STMT (vdef)))
2074 return false;
27d020cf
JH
2075 bitmap_set_bit (info->bb_set,
2076 SSA_NAME_IS_DEFAULT_DEF (vdef)
2077 ? ENTRY_BLOCK_PTR_FOR_FN (cfun)->index
2078 : get_minimal_bb
2079 (gimple_bb (SSA_NAME_DEF_STMT (vdef)),
2080 gimple_bb (info->stmt))->index);
3b2a6901
JH
2081 if (dump_file)
2082 {
2083 fprintf (dump_file, " Param ");
2084 print_generic_expr (dump_file, info->op, TDF_SLIM);
2085 fprintf (dump_file, " changed at bb %i, minimal: %i stmt: ",
2086 gimple_bb (SSA_NAME_DEF_STMT (vdef))->index,
2087 get_minimal_bb
2088 (gimple_bb (SSA_NAME_DEF_STMT (vdef)),
2089 gimple_bb (info->stmt))->index);
2090 print_gimple_stmt (dump_file, SSA_NAME_DEF_STMT (vdef), 0);
2091 }
27d020cf
JH
2092 return false;
2093}
2094
2095/* Return probability (based on REG_BR_PROB_BASE) that I-th parameter of STMT
2096 will change since last invocation of STMT.
2097
2098 Value 0 is reserved for compile time invariants.
2099 For common parameters it is REG_BR_PROB_BASE. For loop invariants it
2100 ought to be REG_BR_PROB_BASE / estimated_iters. */
2101
2102static int
c628d1c3 2103param_change_prob (ipa_func_body_info *fbi, gimple *stmt, int i)
27d020cf
JH
2104{
2105 tree op = gimple_call_arg (stmt, i);
2106 basic_block bb = gimple_bb (stmt);
2107
2108 if (TREE_CODE (op) == WITH_SIZE_EXPR)
2109 op = TREE_OPERAND (op, 0);
2110
2111 tree base = get_base_address (op);
2112
2113 /* Global invariants never change. */
2114 if (is_gimple_min_invariant (base))
2115 return 0;
2116
2117 /* We would have to do non-trivial analysis to really work out what
2118 is the probability of value to change (i.e. when init statement
2119 is in a sibling loop of the call).
2120
2121 We do an conservative estimate: when call is executed N times more often
2122 than the statement defining value, we take the frequency 1/N. */
2123 if (TREE_CODE (base) == SSA_NAME)
2124 {
3b2a6901 2125 profile_count init_count;
27d020cf 2126
3b2a6901 2127 if (!bb->count.nonzero_p ())
27d020cf
JH
2128 return REG_BR_PROB_BASE;
2129
2130 if (SSA_NAME_IS_DEFAULT_DEF (base))
3b2a6901 2131 init_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
27d020cf 2132 else
3b2a6901 2133 init_count = get_minimal_bb
27d020cf 2134 (gimple_bb (SSA_NAME_DEF_STMT (base)),
3b2a6901 2135 gimple_bb (stmt))->count;
27d020cf 2136
3b2a6901
JH
2137 if (init_count < bb->count)
2138 return MAX ((init_count.to_sreal_scale (bb->count)
2139 * REG_BR_PROB_BASE).to_int (), 1);
2140 return REG_BR_PROB_BASE;
27d020cf
JH
2141 }
2142 else
2143 {
2144 ao_ref refd;
3b2a6901 2145 profile_count max = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
27d020cf 2146 struct record_modified_bb_info info;
27d020cf
JH
2147 tree init = ctor_for_folding (base);
2148
2149 if (init != error_mark_node)
2150 return 0;
3b2a6901 2151 if (!bb->count.nonzero_p ())
27d020cf 2152 return REG_BR_PROB_BASE;
3b2a6901
JH
2153 if (dump_file)
2154 {
4307a485 2155 fprintf (dump_file, " Analyzing param change probability of ");
3b2a6901
JH
2156 print_generic_expr (dump_file, op, TDF_SLIM);
2157 fprintf (dump_file, "\n");
2158 }
27d020cf 2159 ao_ref_init (&refd, op);
3b2a6901 2160 info.op = op;
27d020cf
JH
2161 info.stmt = stmt;
2162 info.bb_set = BITMAP_ALLOC (NULL);
c628d1c3
MJ
2163 int walked
2164 = walk_aliased_vdefs (&refd, gimple_vuse (stmt), record_modified, &info,
2165 NULL, NULL, fbi->aa_walk_budget);
2166 if (walked < 0 || bitmap_bit_p (info.bb_set, bb->index))
27d020cf 2167 {
3b2a6901 2168 if (dump_file)
c628d1c3
MJ
2169 {
2170 if (walked < 0)
2171 fprintf (dump_file, " Ran out of AA walking budget.\n");
2172 else
2173 fprintf (dump_file, " Set in same BB as used.\n");
2174 }
27d020cf
JH
2175 BITMAP_FREE (info.bb_set);
2176 return REG_BR_PROB_BASE;
2177 }
2178
3b2a6901
JH
2179 bitmap_iterator bi;
2180 unsigned index;
2181 /* Lookup the most frequent update of the value and believe that
2182 it dominates all the other; precise analysis here is difficult. */
27d020cf 2183 EXECUTE_IF_SET_IN_BITMAP (info.bb_set, 0, index, bi)
3b2a6901
JH
2184 max = max.max (BASIC_BLOCK_FOR_FN (cfun, index)->count);
2185 if (dump_file)
2186 {
2187 fprintf (dump_file, " Set with count ");
2188 max.dump (dump_file);
2189 fprintf (dump_file, " and used with count ");
2190 bb->count.dump (dump_file);
2191 fprintf (dump_file, " freq %f\n",
2192 max.to_sreal_scale (bb->count).to_double ());
2193 }
27d020cf
JH
2194
2195 BITMAP_FREE (info.bb_set);
3b2a6901
JH
2196 if (max < bb->count)
2197 return MAX ((max.to_sreal_scale (bb->count)
2198 * REG_BR_PROB_BASE).to_int (), 1);
2199 return REG_BR_PROB_BASE;
27d020cf
JH
2200 }
2201}
2202
2203/* Find whether a basic block BB is the final block of a (half) diamond CFG
2204 sub-graph and if the predicate the condition depends on is known. If so,
2205 return true and store the pointer the predicate in *P. */
2206
2207static bool
c628d1c3 2208phi_result_unknown_predicate (ipa_func_body_info *fbi,
40a777e8
JH
2209 ipa_fn_summary *summary,
2210 class ipa_node_params *params_summary,
2211 basic_block bb,
27d020cf
JH
2212 predicate *p,
2213 vec<predicate> nonconstant_names)
2214{
2215 edge e;
2216 edge_iterator ei;
2217 basic_block first_bb = NULL;
2218 gimple *stmt;
2219
2220 if (single_pred_p (bb))
2221 {
2222 *p = false;
2223 return true;
2224 }
2225
2226 FOR_EACH_EDGE (e, ei, bb->preds)
2227 {
2228 if (single_succ_p (e->src))
2229 {
2230 if (!single_pred_p (e->src))
2231 return false;
2232 if (!first_bb)
2233 first_bb = single_pred (e->src);
2234 else if (single_pred (e->src) != first_bb)
2235 return false;
2236 }
2237 else
2238 {
2239 if (!first_bb)
2240 first_bb = e->src;
2241 else if (e->src != first_bb)
2242 return false;
2243 }
2244 }
2245
2246 if (!first_bb)
2247 return false;
2248
2249 stmt = last_stmt (first_bb);
2250 if (!stmt
2251 || gimple_code (stmt) != GIMPLE_COND
2252 || !is_gimple_ip_invariant (gimple_cond_rhs (stmt)))
2253 return false;
2254
40a777e8 2255 *p = will_be_nonconstant_expr_predicate (fbi, summary, params_summary,
27d020cf
JH
2256 gimple_cond_lhs (stmt),
2257 nonconstant_names);
2258 if (*p == true)
2259 return false;
2260 else
2261 return true;
2262}
2263
2264/* Given a PHI statement in a function described by inline properties SUMMARY
2265 and *P being the predicate describing whether the selected PHI argument is
2266 known, store a predicate for the result of the PHI statement into
2267 NONCONSTANT_NAMES, if possible. */
2268
2269static void
99b1c316 2270predicate_for_phi_result (class ipa_fn_summary *summary, gphi *phi,
27d020cf
JH
2271 predicate *p,
2272 vec<predicate> nonconstant_names)
2273{
2274 unsigned i;
2275
2276 for (i = 0; i < gimple_phi_num_args (phi); i++)
2277 {
2278 tree arg = gimple_phi_arg (phi, i)->def;
2279 if (!is_gimple_min_invariant (arg))
2280 {
2281 gcc_assert (TREE_CODE (arg) == SSA_NAME);
2282 *p = p->or_with (summary->conds,
2283 nonconstant_names[SSA_NAME_VERSION (arg)]);
2284 if (*p == true)
2285 return;
2286 }
2287 }
2288
2289 if (dump_file && (dump_flags & TDF_DETAILS))
2290 {
2291 fprintf (dump_file, "\t\tphi predicate: ");
2292 p->dump (dump_file, summary->conds);
2293 }
2294 nonconstant_names[SSA_NAME_VERSION (gimple_phi_result (phi))] = *p;
2295}
2296
27d020cf
JH
2297/* For a typical usage of __builtin_expect (a<b, 1), we
2298 may introduce an extra relation stmt:
2299 With the builtin, we have
2300 t1 = a <= b;
2301 t2 = (long int) t1;
2302 t3 = __builtin_expect (t2, 1);
2303 if (t3 != 0)
2304 goto ...
2305 Without the builtin, we have
2306 if (a<=b)
2307 goto...
2308 This affects the size/time estimation and may have
2309 an impact on the earlier inlining.
2310 Here find this pattern and fix it up later. */
2311
2312static gimple *
2313find_foldable_builtin_expect (basic_block bb)
2314{
2315 gimple_stmt_iterator bsi;
2316
2317 for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
2318 {
2319 gimple *stmt = gsi_stmt (bsi);
2320 if (gimple_call_builtin_p (stmt, BUILT_IN_EXPECT)
1e9168b2 2321 || gimple_call_builtin_p (stmt, BUILT_IN_EXPECT_WITH_PROBABILITY)
27d020cf
JH
2322 || gimple_call_internal_p (stmt, IFN_BUILTIN_EXPECT))
2323 {
2324 tree var = gimple_call_lhs (stmt);
2325 tree arg = gimple_call_arg (stmt, 0);
2326 use_operand_p use_p;
2327 gimple *use_stmt;
2328 bool match = false;
2329 bool done = false;
2330
2331 if (!var || !arg)
2332 continue;
2333 gcc_assert (TREE_CODE (var) == SSA_NAME);
2334
2335 while (TREE_CODE (arg) == SSA_NAME)
2336 {
2337 gimple *stmt_tmp = SSA_NAME_DEF_STMT (arg);
2338 if (!is_gimple_assign (stmt_tmp))
2339 break;
2340 switch (gimple_assign_rhs_code (stmt_tmp))
2341 {
2342 case LT_EXPR:
2343 case LE_EXPR:
2344 case GT_EXPR:
2345 case GE_EXPR:
2346 case EQ_EXPR:
2347 case NE_EXPR:
2348 match = true;
2349 done = true;
2350 break;
2351 CASE_CONVERT:
2352 break;
2353 default:
2354 done = true;
2355 break;
2356 }
2357 if (done)
2358 break;
2359 arg = gimple_assign_rhs1 (stmt_tmp);
2360 }
2361
2362 if (match && single_imm_use (var, &use_p, &use_stmt)
2363 && gimple_code (use_stmt) == GIMPLE_COND)
2364 return use_stmt;
2365 }
2366 }
2367 return NULL;
2368}
2369
2370/* Return true when the basic blocks contains only clobbers followed by RESX.
2371 Such BBs are kept around to make removal of dead stores possible with
2372 presence of EH and will be optimized out by optimize_clobbers later in the
2373 game.
2374
956d615d 2375 NEED_EH is used to recurse in case the clobber has non-EH predecessors
27d020cf
JH
2376 that can be clobber only, too.. When it is false, the RESX is not necessary
2377 on the end of basic block. */
2378
2379static bool
2380clobber_only_eh_bb_p (basic_block bb, bool need_eh = true)
2381{
2382 gimple_stmt_iterator gsi = gsi_last_bb (bb);
2383 edge_iterator ei;
2384 edge e;
2385
2386 if (need_eh)
2387 {
2388 if (gsi_end_p (gsi))
2389 return false;
2390 if (gimple_code (gsi_stmt (gsi)) != GIMPLE_RESX)
2391 return false;
2392 gsi_prev (&gsi);
2393 }
2394 else if (!single_succ_p (bb))
2395 return false;
2396
2397 for (; !gsi_end_p (gsi); gsi_prev (&gsi))
2398 {
2399 gimple *stmt = gsi_stmt (gsi);
2400 if (is_gimple_debug (stmt))
2401 continue;
2402 if (gimple_clobber_p (stmt))
2403 continue;
2404 if (gimple_code (stmt) == GIMPLE_LABEL)
2405 break;
2406 return false;
2407 }
2408
956d615d 2409 /* See if all predecessors are either throws or clobber only BBs. */
27d020cf
JH
2410 FOR_EACH_EDGE (e, ei, bb->preds)
2411 if (!(e->flags & EDGE_EH)
2412 && !clobber_only_eh_bb_p (e->src, false))
2413 return false;
2414
2415 return true;
2416}
2417
2418/* Return true if STMT compute a floating point expression that may be affected
2419 by -ffast-math and similar flags. */
2420
2421static bool
2422fp_expression_p (gimple *stmt)
2423{
2424 ssa_op_iter i;
2425 tree op;
2426
2427 FOR_EACH_SSA_TREE_OPERAND (op, stmt, i, SSA_OP_DEF|SSA_OP_USE)
2428 if (FLOAT_TYPE_P (TREE_TYPE (op)))
2429 return true;
2430 return false;
2431}
2432
0bceb671
JH
2433/* Analyze function body for NODE.
2434 EARLY indicates run from early optimization pipeline. */
27d020cf
JH
2435
2436static void
0bceb671 2437analyze_function_body (struct cgraph_node *node, bool early)
27d020cf 2438{
9340d345 2439 sreal time = opt_for_fn (node->decl, param_uninlined_function_time);
27d020cf 2440 /* Estimate static overhead for function prologue/epilogue and alignment. */
9340d345 2441 int size = opt_for_fn (node->decl, param_uninlined_function_insns);
27d020cf
JH
2442 /* Benefits are scaled by probability of elimination that is in range
2443 <0,2>. */
2444 basic_block bb;
2445 struct function *my_function = DECL_STRUCT_FUNCTION (node->decl);
b71289b1 2446 sreal freq;
99b1c316 2447 class ipa_fn_summary *info = ipa_fn_summaries->get_create (node);
40a777e8 2448 class ipa_node_params *params_summary = early ? NULL : IPA_NODE_REF (node);
27d020cf
JH
2449 predicate bb_predicate;
2450 struct ipa_func_body_info fbi;
2451 vec<predicate> nonconstant_names = vNULL;
2452 int nblocks, n;
2453 int *order;
27d020cf
JH
2454 gimple *fix_builtin_expect_stmt;
2455
2456 gcc_assert (my_function && my_function->cfg);
2457 gcc_assert (cfun == my_function);
2458
2459 memset(&fbi, 0, sizeof(fbi));
ddfb1317 2460 vec_free (info->conds);
27d020cf 2461 info->conds = NULL;
ddfb1317 2462 vec_free (info->size_time_table);
27d020cf
JH
2463 info->size_time_table = NULL;
2464
2465 /* When optimizing and analyzing for IPA inliner, initialize loop optimizer
2466 so we can produce proper inline hints.
2467
2468 When optimizing and analyzing for early inliner, initialize node params
2469 so we can produce correct BB predicates. */
2470
2471 if (opt_for_fn (node->decl, optimize))
2472 {
2473 calculate_dominance_info (CDI_DOMINATORS);
efe12656 2474 calculate_dominance_info (CDI_POST_DOMINATORS);
27d020cf
JH
2475 if (!early)
2476 loop_optimizer_init (LOOPS_NORMAL | LOOPS_HAVE_RECORDED_EXITS);
2477 else
2478 {
2479 ipa_check_create_node_params ();
2480 ipa_initialize_node_params (node);
2481 }
2482
2483 if (ipa_node_params_sum)
2484 {
2485 fbi.node = node;
2486 fbi.info = IPA_NODE_REF (node);
2487 fbi.bb_infos = vNULL;
cb3874dc 2488 fbi.bb_infos.safe_grow_cleared (last_basic_block_for_fn (cfun), true);
c628d1c3 2489 fbi.param_count = count_formal_params (node->decl);
fdfd7f53 2490 fbi.aa_walk_budget = opt_for_fn (node->decl, param_ipa_max_aa_steps);
c628d1c3 2491
27d020cf 2492 nonconstant_names.safe_grow_cleared
cb3874dc 2493 (SSANAMES (my_function)->length (), true);
27d020cf
JH
2494 }
2495 }
2496
2497 if (dump_file)
2498 fprintf (dump_file, "\nAnalyzing function body size: %s\n",
3629ff8a 2499 node->dump_name ());
27d020cf
JH
2500
2501 /* When we run into maximal number of entries, we assign everything to the
2502 constant truth case. Be sure to have it in list. */
2503 bb_predicate = true;
2504 info->account_size_time (0, 0, bb_predicate, bb_predicate);
2505
2506 bb_predicate = predicate::not_inlined ();
9340d345
JH
2507 info->account_size_time (opt_for_fn (node->decl,
2508 param_uninlined_function_insns)
d06f73a3 2509 * ipa_fn_summary::size_scale,
9340d345
JH
2510 opt_for_fn (node->decl,
2511 param_uninlined_function_time),
d06f73a3 2512 bb_predicate,
27d020cf
JH
2513 bb_predicate);
2514
2515 if (fbi.info)
40a777e8 2516 compute_bb_predicates (&fbi, node, info, params_summary);
27d020cf
JH
2517 order = XNEWVEC (int, n_basic_blocks_for_fn (cfun));
2518 nblocks = pre_and_rev_post_order_compute (NULL, order, false);
2519 for (n = 0; n < nblocks; n++)
2520 {
2521 bb = BASIC_BLOCK_FOR_FN (cfun, order[n]);
b71289b1 2522 freq = bb->count.to_sreal_scale (ENTRY_BLOCK_PTR_FOR_FN (cfun)->count);
27d020cf
JH
2523 if (clobber_only_eh_bb_p (bb))
2524 {
2525 if (dump_file && (dump_flags & TDF_DETAILS))
2526 fprintf (dump_file, "\n Ignoring BB %i;"
2527 " it will be optimized away by cleanup_clobbers\n",
2528 bb->index);
2529 continue;
2530 }
2531
2532 /* TODO: Obviously predicates can be propagated down across CFG. */
2533 if (fbi.info)
2534 {
2535 if (bb->aux)
2536 bb_predicate = *(predicate *) bb->aux;
2537 else
2538 bb_predicate = false;
2539 }
2540 else
2541 bb_predicate = true;
2542
2543 if (dump_file && (dump_flags & TDF_DETAILS))
2544 {
2545 fprintf (dump_file, "\n BB %i predicate:", bb->index);
2546 bb_predicate.dump (dump_file, info->conds);
2547 }
2548
2549 if (fbi.info && nonconstant_names.exists ())
2550 {
2551 predicate phi_predicate;
2552 bool first_phi = true;
2553
2554 for (gphi_iterator bsi = gsi_start_phis (bb); !gsi_end_p (bsi);
2555 gsi_next (&bsi))
2556 {
2557 if (first_phi
40a777e8
JH
2558 && !phi_result_unknown_predicate (&fbi, info,
2559 params_summary,
2560 bb,
27d020cf
JH
2561 &phi_predicate,
2562 nonconstant_names))
2563 break;
2564 first_phi = false;
2565 if (dump_file && (dump_flags & TDF_DETAILS))
2566 {
2567 fprintf (dump_file, " ");
2568 print_gimple_stmt (dump_file, gsi_stmt (bsi), 0);
2569 }
2570 predicate_for_phi_result (info, bsi.phi (), &phi_predicate,
2571 nonconstant_names);
2572 }
2573 }
2574
2575 fix_builtin_expect_stmt = find_foldable_builtin_expect (bb);
2576
d3ed5b56
JH
2577 for (gimple_stmt_iterator bsi = gsi_start_nondebug_bb (bb);
2578 !gsi_end_p (bsi); gsi_next_nondebug (&bsi))
27d020cf
JH
2579 {
2580 gimple *stmt = gsi_stmt (bsi);
2581 int this_size = estimate_num_insns (stmt, &eni_size_weights);
2582 int this_time = estimate_num_insns (stmt, &eni_time_weights);
2583 int prob;
2584 predicate will_be_nonconstant;
2585
2586 /* This relation stmt should be folded after we remove
956d615d 2587 __builtin_expect call. Adjust the cost here. */
27d020cf
JH
2588 if (stmt == fix_builtin_expect_stmt)
2589 {
2590 this_size--;
2591 this_time--;
2592 }
2593
2594 if (dump_file && (dump_flags & TDF_DETAILS))
2595 {
2596 fprintf (dump_file, " ");
2597 print_gimple_stmt (dump_file, stmt, 0);
2598 fprintf (dump_file, "\t\tfreq:%3.2f size:%3i time:%3i\n",
b71289b1 2599 freq.to_double (), this_size,
27d020cf
JH
2600 this_time);
2601 }
2602
27d020cf
JH
2603 if (is_gimple_call (stmt)
2604 && !gimple_call_internal_p (stmt))
2605 {
2606 struct cgraph_edge *edge = node->get_edge (stmt);
99353fcf 2607 ipa_call_summary *es = ipa_call_summaries->get_create (edge);
27d020cf
JH
2608
2609 /* Special case: results of BUILT_IN_CONSTANT_P will be always
2610 resolved as constant. We however don't want to optimize
2611 out the cgraph edges. */
2612 if (nonconstant_names.exists ()
2613 && gimple_call_builtin_p (stmt, BUILT_IN_CONSTANT_P)
2614 && gimple_call_lhs (stmt)
2615 && TREE_CODE (gimple_call_lhs (stmt)) == SSA_NAME)
2616 {
2617 predicate false_p = false;
2618 nonconstant_names[SSA_NAME_VERSION (gimple_call_lhs (stmt))]
2619 = false_p;
2620 }
2621 if (ipa_node_params_sum)
2622 {
2623 int count = gimple_call_num_args (stmt);
2624 int i;
2625
2626 if (count)
cb3874dc 2627 es->param.safe_grow_cleared (count, true);
27d020cf
JH
2628 for (i = 0; i < count; i++)
2629 {
c628d1c3 2630 int prob = param_change_prob (&fbi, stmt, i);
27d020cf
JH
2631 gcc_assert (prob >= 0 && prob <= REG_BR_PROB_BASE);
2632 es->param[i].change_prob = prob;
2633 }
2634 }
2635
2636 es->call_stmt_size = this_size;
2637 es->call_stmt_time = this_time;
2638 es->loop_depth = bb_loop_depth (bb);
2639 edge_set_predicate (edge, &bb_predicate);
959b8c82
JH
2640 if (edge->speculative)
2641 {
845bb366
JH
2642 cgraph_edge *indirect
2643 = edge->speculative_call_indirect_edge ();
959b8c82 2644 ipa_call_summary *es2
d380e329 2645 = ipa_call_summaries->get_create (indirect);
959b8c82
JH
2646 ipa_call_summaries->duplicate (edge, indirect,
2647 es, es2);
f1ba88b1 2648
845bb366
JH
2649 /* Edge is the first direct call.
2650 create and duplicate call summaries for multiple
f1ba88b1 2651 speculative call targets. */
845bb366
JH
2652 for (cgraph_edge *direct
2653 = edge->next_speculative_call_target ();
2654 direct;
2655 direct = direct->next_speculative_call_target ())
2656 {
2657 ipa_call_summary *es3
2658 = ipa_call_summaries->get_create (direct);
2659 ipa_call_summaries->duplicate (edge, direct,
2660 es, es3);
2661 }
959b8c82 2662 }
27d020cf
JH
2663 }
2664
956d615d 2665 /* TODO: When conditional jump or switch is known to be constant, but
27d020cf
JH
2666 we did not translate it into the predicates, we really can account
2667 just maximum of the possible paths. */
2668 if (fbi.info)
2669 will_be_nonconstant
40a777e8 2670 = will_be_nonconstant_predicate (&fbi, info, params_summary,
27d020cf
JH
2671 stmt, nonconstant_names);
2672 else
2673 will_be_nonconstant = true;
2674 if (this_time || this_size)
2675 {
b71289b1 2676 sreal final_time = (sreal)this_time * freq;
27d020cf 2677
c628d1c3 2678 prob = eliminated_by_inlining_prob (&fbi, stmt);
27d020cf
JH
2679 if (prob == 1 && dump_file && (dump_flags & TDF_DETAILS))
2680 fprintf (dump_file,
2681 "\t\t50%% will be eliminated by inlining\n");
2682 if (prob == 2 && dump_file && (dump_flags & TDF_DETAILS))
2683 fprintf (dump_file, "\t\tWill be eliminated by inlining\n");
2684
99b1c316 2685 class predicate p = bb_predicate & will_be_nonconstant;
27d020cf
JH
2686
2687 /* We can ignore statement when we proved it is never going
67914693 2688 to happen, but we cannot do that for call statements
27d020cf
JH
2689 because edges are accounted specially. */
2690
2691 if (*(is_gimple_call (stmt) ? &bb_predicate : &p) != false)
2692 {
b71289b1 2693 time += final_time;
27d020cf
JH
2694 size += this_size;
2695 }
2696
2697 /* We account everything but the calls. Calls have their own
2698 size/time info attached to cgraph edges. This is necessary
2699 in order to make the cost disappear after inlining. */
2700 if (!is_gimple_call (stmt))
2701 {
2702 if (prob)
2703 {
2704 predicate ip = bb_predicate & predicate::not_inlined ();
2705 info->account_size_time (this_size * prob,
121356b0 2706 (final_time * prob) / 2, ip,
27d020cf
JH
2707 p);
2708 }
2709 if (prob != 2)
2710 info->account_size_time (this_size * (2 - prob),
121356b0 2711 (final_time * (2 - prob) / 2),
27d020cf
JH
2712 bb_predicate,
2713 p);
2714 }
2715
2716 if (!info->fp_expressions && fp_expression_p (stmt))
2717 {
2718 info->fp_expressions = true;
2719 if (dump_file)
2720 fprintf (dump_file, " fp_expression set\n");
2721 }
a20f263b 2722 }
27d020cf 2723
a20f263b
JH
2724 /* Account cost of address calculations in the statements. */
2725 for (unsigned int i = 0; i < gimple_num_ops (stmt); i++)
2726 {
2727 for (tree op = gimple_op (stmt, i);
2728 op && handled_component_p (op);
2729 op = TREE_OPERAND (op, 0))
2730 if ((TREE_CODE (op) == ARRAY_REF
2731 || TREE_CODE (op) == ARRAY_RANGE_REF)
2732 && TREE_CODE (TREE_OPERAND (op, 1)) == SSA_NAME)
2733 {
2734 predicate p = bb_predicate;
2735 if (fbi.info)
2736 p = p & will_be_nonconstant_expr_predicate
40a777e8
JH
2737 (&fbi, info, params_summary,
2738 TREE_OPERAND (op, 1),
a20f263b
JH
2739 nonconstant_names);
2740 if (p != false)
2741 {
2742 time += freq;
2743 size += 1;
2744 if (dump_file)
2745 fprintf (dump_file,
2746 "\t\tAccounting address calculation.\n");
2747 info->account_size_time (ipa_fn_summary::size_scale,
2748 freq,
2749 bb_predicate,
2750 p);
2751 }
2752 }
27d020cf 2753 }
a20f263b 2754
27d020cf
JH
2755 }
2756 }
27d020cf
JH
2757 free (order);
2758
2759 if (nonconstant_names.exists () && !early)
2760 {
99b1c316 2761 class loop *loop;
27d020cf
JH
2762 predicate loop_iterations = true;
2763 predicate loop_stride = true;
2764
2765 if (dump_file && (dump_flags & TDF_DETAILS))
2766 flow_loops_dump (dump_file, NULL, 0);
2767 scev_initialize ();
2768 FOR_EACH_LOOP (loop, 0)
2769 {
2770 vec<edge> exits;
2771 edge ex;
2772 unsigned int j;
99b1c316 2773 class tree_niter_desc niter_desc;
776e48e0
JJ
2774 if (loop->header->aux)
2775 bb_predicate = *(predicate *) loop->header->aux;
2776 else
2777 bb_predicate = false;
27d020cf
JH
2778
2779 exits = get_loop_exit_edges (loop);
2780 FOR_EACH_VEC_ELT (exits, j, ex)
2781 if (number_of_iterations_exit (loop, ex, &niter_desc, false)
2782 && !is_gimple_min_invariant (niter_desc.niter))
2783 {
2784 predicate will_be_nonconstant
c628d1c3 2785 = will_be_nonconstant_expr_predicate (&fbi, info,
40a777e8 2786 params_summary,
27d020cf
JH
2787 niter_desc.niter,
2788 nonconstant_names);
2789 if (will_be_nonconstant != true)
2790 will_be_nonconstant = bb_predicate & will_be_nonconstant;
2791 if (will_be_nonconstant != true
2792 && will_be_nonconstant != false)
2793 /* This is slightly inprecise. We may want to represent each
2794 loop with independent predicate. */
2795 loop_iterations &= will_be_nonconstant;
2796 }
2797 exits.release ();
2798 }
2799
2800 /* To avoid quadratic behavior we analyze stride predicates only
2801 with respect to the containing loop. Thus we simply iterate
2802 over all defs in the outermost loop body. */
2803 for (loop = loops_for_fn (cfun)->tree_root->inner;
2804 loop != NULL; loop = loop->next)
2805 {
2806 basic_block *body = get_loop_body (loop);
2807 for (unsigned i = 0; i < loop->num_nodes; i++)
2808 {
2809 gimple_stmt_iterator gsi;
776e48e0
JJ
2810 if (body[i]->aux)
2811 bb_predicate = *(predicate *) body[i]->aux;
2812 else
2813 bb_predicate = false;
27d020cf
JH
2814 for (gsi = gsi_start_bb (body[i]); !gsi_end_p (gsi);
2815 gsi_next (&gsi))
2816 {
2817 gimple *stmt = gsi_stmt (gsi);
2818
2819 if (!is_gimple_assign (stmt))
2820 continue;
2821
2822 tree def = gimple_assign_lhs (stmt);
2823 if (TREE_CODE (def) != SSA_NAME)
2824 continue;
2825
2826 affine_iv iv;
2827 if (!simple_iv (loop_containing_stmt (stmt),
2828 loop_containing_stmt (stmt),
2829 def, &iv, true)
2830 || is_gimple_min_invariant (iv.step))
2831 continue;
2832
2833 predicate will_be_nonconstant
40a777e8
JH
2834 = will_be_nonconstant_expr_predicate (&fbi, info,
2835 params_summary,
2836 iv.step,
27d020cf
JH
2837 nonconstant_names);
2838 if (will_be_nonconstant != true)
2839 will_be_nonconstant = bb_predicate & will_be_nonconstant;
2840 if (will_be_nonconstant != true
2841 && will_be_nonconstant != false)
2842 /* This is slightly inprecise. We may want to represent
2843 each loop with independent predicate. */
2844 loop_stride = loop_stride & will_be_nonconstant;
2845 }
2846 }
2847 free (body);
2848 }
56f62793 2849 ipa_fn_summary *s = ipa_fn_summaries->get (node);
cf9b0b5f
ML
2850 set_hint_predicate (&s->loop_iterations, loop_iterations);
2851 set_hint_predicate (&s->loop_stride, loop_stride);
27d020cf
JH
2852 scev_finalize ();
2853 }
2854 FOR_ALL_BB_FN (bb, my_function)
2855 {
2856 edge e;
2857 edge_iterator ei;
2858
2859 if (bb->aux)
2860 edge_predicate_pool.remove ((predicate *)bb->aux);
2861 bb->aux = NULL;
2862 FOR_EACH_EDGE (e, ei, bb->succs)
2863 {
2864 if (e->aux)
2865 edge_predicate_pool.remove ((predicate *) e->aux);
2866 e->aux = NULL;
2867 }
2868 }
56f62793 2869 ipa_fn_summary *s = ipa_fn_summaries->get (node);
f658ad30 2870 ipa_size_summary *ss = ipa_size_summaries->get (node);
cf9b0b5f 2871 s->time = time;
f658ad30 2872 ss->self_size = size;
27d020cf
JH
2873 nonconstant_names.release ();
2874 ipa_release_body_info (&fbi);
2875 if (opt_for_fn (node->decl, optimize))
2876 {
2877 if (!early)
2878 loop_optimizer_finalize ();
2879 else if (!ipa_edge_args_sum)
2880 ipa_free_all_node_params ();
2881 free_dominance_info (CDI_DOMINATORS);
efe12656 2882 free_dominance_info (CDI_POST_DOMINATORS);
27d020cf
JH
2883 }
2884 if (dump_file)
2885 {
2886 fprintf (dump_file, "\n");
0bceb671 2887 ipa_dump_fn_summary (dump_file, node);
27d020cf
JH
2888 }
2889}
2890
2891
0bceb671
JH
2892/* Compute function summary.
2893 EARLY is true when we compute parameters during early opts. */
27d020cf
JH
2894
2895void
0bceb671 2896compute_fn_summary (struct cgraph_node *node, bool early)
27d020cf
JH
2897{
2898 HOST_WIDE_INT self_stack_size;
2899 struct cgraph_edge *e;
27d020cf 2900
a62bfab5 2901 gcc_assert (!node->inlined_to);
27d020cf 2902
0bceb671
JH
2903 if (!ipa_fn_summaries)
2904 ipa_fn_summary_alloc ();
27d020cf 2905
56f62793
ML
2906 /* Create a new ipa_fn_summary. */
2907 ((ipa_fn_summary_t *)ipa_fn_summaries)->remove_callees (node);
2908 ipa_fn_summaries->remove (node);
f658ad30
JH
2909 class ipa_fn_summary *info = ipa_fn_summaries->get_create (node);
2910 class ipa_size_summary *size_info = ipa_size_summaries->get_create (node);
27d020cf
JH
2911
2912 /* Estimate the stack size for the function if we're optimizing. */
2913 self_stack_size = optimize && !node->thunk.thunk_p
2914 ? estimated_stack_frame_size (node) : 0;
f658ad30 2915 size_info->estimated_self_stack_size = self_stack_size;
27d020cf 2916 info->estimated_stack_size = self_stack_size;
27d020cf
JH
2917
2918 if (node->thunk.thunk_p)
2919 {
99353fcf 2920 ipa_call_summary *es = ipa_call_summaries->get_create (node->callees);
27d020cf
JH
2921 predicate t = true;
2922
87f94429 2923 node->can_change_signature = false;
27d020cf
JH
2924 es->call_stmt_size = eni_size_weights.call_cost;
2925 es->call_stmt_time = eni_time_weights.call_cost;
d06f73a3 2926 info->account_size_time (ipa_fn_summary::size_scale
9340d345
JH
2927 * opt_for_fn (node->decl,
2928 param_uninlined_function_thunk_insns),
2929 opt_for_fn (node->decl,
2930 param_uninlined_function_thunk_time), t, t);
27d020cf 2931 t = predicate::not_inlined ();
0bceb671
JH
2932 info->account_size_time (2 * ipa_fn_summary::size_scale, 0, t, t);
2933 ipa_update_overall_fn_summary (node);
f658ad30 2934 size_info->self_size = size_info->size;
dbcdd561 2935 if (stdarg_p (TREE_TYPE (node->decl)))
ca04a532
ML
2936 {
2937 info->inlinable = false;
2938 node->callees->inline_failed = CIF_VARIADIC_THUNK;
2939 }
27d020cf
JH
2940 else
2941 info->inlinable = true;
2942 }
2943 else
2944 {
2945 /* Even is_gimple_min_invariant rely on current_function_decl. */
2946 push_cfun (DECL_STRUCT_FUNCTION (node->decl));
2947
ac0573de
JH
2948 /* During IPA profile merging we may be called w/o virtual SSA form
2949 built. */
2950 update_ssa (TODO_update_ssa_only_virtuals);
2951
27d020cf
JH
2952 /* Can this function be inlined at all? */
2953 if (!opt_for_fn (node->decl, optimize)
2954 && !lookup_attribute ("always_inline",
2955 DECL_ATTRIBUTES (node->decl)))
2956 info->inlinable = false;
2957 else
2958 info->inlinable = tree_inlinable_function_p (node->decl);
2959
27d020cf 2960 /* Type attributes can use parameter indices to describe them. */
3d8fb311
JJ
2961 if (TYPE_ATTRIBUTES (TREE_TYPE (node->decl))
2962 /* Likewise for #pragma omp declare simd functions or functions
2963 with simd attribute. */
2964 || lookup_attribute ("omp declare simd",
2965 DECL_ATTRIBUTES (node->decl)))
87f94429 2966 node->can_change_signature = false;
27d020cf
JH
2967 else
2968 {
2969 /* Otherwise, inlinable functions always can change signature. */
2970 if (info->inlinable)
87f94429 2971 node->can_change_signature = true;
27d020cf
JH
2972 else
2973 {
67914693 2974 /* Functions calling builtin_apply cannot change signature. */
27d020cf
JH
2975 for (e = node->callees; e; e = e->next_callee)
2976 {
2977 tree cdecl = e->callee->decl;
3d78e008
ML
2978 if (fndecl_built_in_p (cdecl, BUILT_IN_APPLY_ARGS)
2979 || fndecl_built_in_p (cdecl, BUILT_IN_VA_START))
27d020cf
JH
2980 break;
2981 }
87f94429 2982 node->can_change_signature = !e;
27d020cf
JH
2983 }
2984 }
0bceb671 2985 analyze_function_body (node, early);
27d020cf
JH
2986 pop_cfun ();
2987 }
27d020cf
JH
2988
2989 /* Inlining characteristics are maintained by the cgraph_mark_inline. */
f658ad30
JH
2990 size_info->size = size_info->self_size;
2991 info->estimated_stack_size = size_info->estimated_self_stack_size;
27d020cf
JH
2992
2993 /* Code above should compute exactly the same result as
0bceb671 2994 ipa_update_overall_fn_summary but because computation happens in
27d020cf 2995 different order the roundoff errors result in slight changes. */
0bceb671 2996 ipa_update_overall_fn_summary (node);
959b8c82 2997 /* In LTO mode we may have speculative edges set. */
f658ad30 2998 gcc_assert (in_lto_p || size_info->size == size_info->self_size);
27d020cf
JH
2999}
3000
3001
3002/* Compute parameters of functions used by inliner using
3003 current_function_decl. */
3004
3005static unsigned int
0bceb671 3006compute_fn_summary_for_current (void)
27d020cf 3007{
0bceb671 3008 compute_fn_summary (cgraph_node::get (current_function_decl), true);
27d020cf
JH
3009 return 0;
3010}
3011
27d020cf
JH
3012/* Estimate benefit devirtualizing indirect edge IE, provided KNOWN_VALS,
3013 KNOWN_CONTEXTS and KNOWN_AGGS. */
3014
3015static bool
3016estimate_edge_devirt_benefit (struct cgraph_edge *ie,
3017 int *size, int *time,
3018 vec<tree> known_vals,
3019 vec<ipa_polymorphic_call_context> known_contexts,
eb270950 3020 vec<ipa_agg_value_set> known_aggs)
27d020cf
JH
3021{
3022 tree target;
3023 struct cgraph_node *callee;
99b1c316 3024 class ipa_fn_summary *isummary;
27d020cf
JH
3025 enum availability avail;
3026 bool speculative;
3027
b0d55476 3028 if (!known_vals.length () && !known_contexts.length ())
27d020cf
JH
3029 return false;
3030 if (!opt_for_fn (ie->caller->decl, flag_indirect_inlining))
3031 return false;
3032
3033 target = ipa_get_indirect_edge_target (ie, known_vals, known_contexts,
3034 known_aggs, &speculative);
3035 if (!target || speculative)
3036 return false;
3037
3038 /* Account for difference in cost between indirect and direct calls. */
3039 *size -= (eni_size_weights.indirect_call_cost - eni_size_weights.call_cost);
3040 *time -= (eni_time_weights.indirect_call_cost - eni_time_weights.call_cost);
3041 gcc_checking_assert (*time >= 0);
3042 gcc_checking_assert (*size >= 0);
3043
3044 callee = cgraph_node::get (target);
3045 if (!callee || !callee->definition)
3046 return false;
3047 callee = callee->function_symbol (&avail);
3048 if (avail < AVAIL_AVAILABLE)
3049 return false;
56f62793 3050 isummary = ipa_fn_summaries->get (callee);
1d546c60
ML
3051 if (isummary == NULL)
3052 return false;
3053
27d020cf
JH
3054 return isummary->inlinable;
3055}
3056
3057/* Increase SIZE, MIN_SIZE (if non-NULL) and TIME for size and time needed to
3058 handle edge E with probability PROB.
3059 Set HINTS if edge may be devirtualized.
3060 KNOWN_VALS, KNOWN_AGGS and KNOWN_CONTEXTS describe context of the call
3061 site. */
3062
3063static inline void
3064estimate_edge_size_and_time (struct cgraph_edge *e, int *size, int *min_size,
3065 sreal *time,
27d020cf
JH
3066 vec<tree> known_vals,
3067 vec<ipa_polymorphic_call_context> known_contexts,
eb270950 3068 vec<ipa_agg_value_set> known_aggs,
0bceb671 3069 ipa_hints *hints)
27d020cf 3070{
99b1c316 3071 class ipa_call_summary *es = ipa_call_summaries->get (e);
27d020cf
JH
3072 int call_size = es->call_stmt_size;
3073 int call_time = es->call_stmt_time;
3074 int cur_size;
98450d19 3075
83263ef5 3076 if (!e->callee && hints && e->maybe_hot_p ()
27d020cf 3077 && estimate_edge_devirt_benefit (e, &call_size, &call_time,
83263ef5 3078 known_vals, known_contexts, known_aggs))
27d020cf 3079 *hints |= INLINE_HINT_indirect_call;
0bceb671 3080 cur_size = call_size * ipa_fn_summary::size_scale;
27d020cf
JH
3081 *size += cur_size;
3082 if (min_size)
3083 *min_size += cur_size;
98450d19 3084 if (time)
41f0e819 3085 *time += ((sreal)call_time) * e->sreal_frequency ();
27d020cf
JH
3086}
3087
3088
27d020cf
JH
3089/* Increase SIZE, MIN_SIZE and TIME for size and time needed to handle all
3090 calls in NODE. POSSIBLE_TRUTHS, KNOWN_VALS, KNOWN_AGGS and KNOWN_CONTEXTS
070e3489
JH
3091 describe context of the call site.
3092
3093 Helper for estimate_calls_size_and_time which does the same but
3094 (in most cases) faster. */
27d020cf
JH
3095
3096static void
070e3489
JH
3097estimate_calls_size_and_time_1 (struct cgraph_node *node, int *size,
3098 int *min_size, sreal *time,
3099 ipa_hints *hints,
3100 clause_t possible_truths,
3101 vec<tree> known_vals,
3102 vec<ipa_polymorphic_call_context> known_contexts,
3103 vec<ipa_agg_value_set> known_aggs)
27d020cf
JH
3104{
3105 struct cgraph_edge *e;
3106 for (e = node->callees; e; e = e->next_callee)
3107 {
7237f93e
JH
3108 if (!e->inline_failed)
3109 {
3110 gcc_checking_assert (!ipa_call_summaries->get (e));
070e3489
JH
3111 estimate_calls_size_and_time_1 (e->callee, size, min_size, time,
3112 hints,
3113 possible_truths,
3114 known_vals, known_contexts,
3115 known_aggs);
7237f93e
JH
3116 continue;
3117 }
3118 class ipa_call_summary *es = ipa_call_summaries->get (e);
27d020cf
JH
3119
3120 /* Do not care about zero sized builtins. */
7237f93e 3121 if (!es->call_stmt_size)
27d020cf
JH
3122 {
3123 gcc_checking_assert (!es->call_stmt_time);
3124 continue;
3125 }
3126 if (!es->predicate
3127 || es->predicate->evaluate (possible_truths))
3128 {
7237f93e 3129 /* Predicates of calls shall not use NOT_CHANGED codes,
956d615d 3130 so we do not need to compute probabilities. */
7237f93e
JH
3131 estimate_edge_size_and_time (e, size,
3132 es->predicate ? NULL : min_size,
98450d19 3133 time,
7237f93e
JH
3134 known_vals, known_contexts,
3135 known_aggs, hints);
27d020cf
JH
3136 }
3137 }
3138 for (e = node->indirect_calls; e; e = e->next_callee)
3139 {
7237f93e 3140 class ipa_call_summary *es = ipa_call_summaries->get (e);
27d020cf
JH
3141 if (!es->predicate
3142 || es->predicate->evaluate (possible_truths))
3143 estimate_edge_size_and_time (e, size,
3144 es->predicate ? NULL : min_size,
98450d19 3145 time,
27d020cf
JH
3146 known_vals, known_contexts, known_aggs,
3147 hints);
3148 }
3149}
3150
070e3489
JH
3151/* Populate sum->call_size_time_table for edges from NODE. */
3152
3153static void
3154summarize_calls_size_and_time (struct cgraph_node *node,
3155 ipa_fn_summary *sum)
3156{
3157 struct cgraph_edge *e;
3158 for (e = node->callees; e; e = e->next_callee)
3159 {
3160 if (!e->inline_failed)
3161 {
3162 gcc_checking_assert (!ipa_call_summaries->get (e));
3163 summarize_calls_size_and_time (e->callee, sum);
3164 continue;
3165 }
3166 int size = 0;
3167 sreal time = 0;
3168
3169 estimate_edge_size_and_time (e, &size, NULL, &time,
3170 vNULL, vNULL, vNULL, NULL);
3171
3172 struct predicate pred = true;
3173 class ipa_call_summary *es = ipa_call_summaries->get (e);
3174
3175 if (es->predicate)
3176 pred = *es->predicate;
3177 sum->account_size_time (size, time, pred, pred, true);
3178 }
3179 for (e = node->indirect_calls; e; e = e->next_callee)
3180 {
3181 int size = 0;
3182 sreal time = 0;
3183
3184 estimate_edge_size_and_time (e, &size, NULL, &time,
3185 vNULL, vNULL, vNULL, NULL);
3186 struct predicate pred = true;
3187 class ipa_call_summary *es = ipa_call_summaries->get (e);
3188
3189 if (es->predicate)
3190 pred = *es->predicate;
3191 sum->account_size_time (size, time, pred, pred, true);
3192 }
3193}
3194
3195/* Increase SIZE, MIN_SIZE and TIME for size and time needed to handle all
3196 calls in NODE. POSSIBLE_TRUTHS, KNOWN_VALS, KNOWN_AGGS and KNOWN_CONTEXTS
3197 describe context of the call site. */
3198
3199static void
3200estimate_calls_size_and_time (struct cgraph_node *node, int *size,
3201 int *min_size, sreal *time,
3202 ipa_hints *hints,
3203 clause_t possible_truths,
3204 vec<tree> known_vals,
3205 vec<ipa_polymorphic_call_context> known_contexts,
3206 vec<ipa_agg_value_set> known_aggs)
3207{
3208 class ipa_fn_summary *sum = ipa_fn_summaries->get (node);
3209 bool use_table = true;
3210
3211 gcc_assert (node->callees || node->indirect_calls);
3212
3213 /* During early inlining we do not calculate info for very
3214 large functions and thus there is no need for producing
3215 summaries. */
3216 if (!ipa_node_params_sum)
3217 use_table = false;
3218 /* Do not calculate summaries for simple wrappers; it is waste
3219 of memory. */
3220 else if (node->callees && node->indirect_calls
3221 && node->callees->inline_failed && !node->callees->next_callee)
3222 use_table = false;
3223 /* If there is an indirect edge that may be optimized, we need
3224 to go the slow way. */
3225 else if ((known_vals.length ()
3226 || known_contexts.length ()
3227 || known_aggs.length ()) && hints)
3228 {
3229 class ipa_node_params *params_summary = IPA_NODE_REF (node);
3230 unsigned int nargs = params_summary
3231 ? ipa_get_param_count (params_summary) : 0;
3232
3233 for (unsigned int i = 0; i < nargs && use_table; i++)
3234 {
3235 if (ipa_is_param_used_by_indirect_call (params_summary, i)
3236 && ((known_vals.length () > i && known_vals[i])
3237 || (known_aggs.length () > i
3238 && known_aggs[i].items.length ())))
3239 use_table = false;
3240 else if (ipa_is_param_used_by_polymorphic_call (params_summary, i)
3241 && (known_contexts.length () > i
3242 && !known_contexts[i].useless_p ()))
3243 use_table = false;
3244 }
3245 }
3246
3247 /* Fast path is via the call size time table. */
3248 if (use_table)
3249 {
3250 /* Build summary if it is absent. */
3251 if (!sum->call_size_time_table)
3252 {
3253 predicate true_pred = true;
3254 sum->account_size_time (0, 0, true_pred, true_pred, true);
3255 summarize_calls_size_and_time (node, sum);
3256 }
3257
3258 int old_size = *size;
3259 sreal old_time = time ? *time : 0;
3260
3261 if (min_size)
3262 *min_size += (*sum->call_size_time_table)[0].size;
3263
3264 unsigned int i;
3265 size_time_entry *e;
3266
3267 /* Walk the table and account sizes and times. */
3268 for (i = 0; vec_safe_iterate (sum->call_size_time_table, i, &e);
3269 i++)
3270 if (e->exec_predicate.evaluate (possible_truths))
3271 {
3272 *size += e->size;
3273 if (time)
3274 *time += e->time;
3275 }
3276
3277 /* Be careful and see if both methods agree. */
3278 if ((flag_checking || dump_file)
3279 /* Do not try to sanity check when we know we lost some
3280 precision. */
3281 && sum->call_size_time_table->length ()
3282 < ipa_fn_summary::max_size_time_table_size)
3283 {
3284 estimate_calls_size_and_time_1 (node, &old_size, NULL, &old_time, NULL,
3285 possible_truths, known_vals,
3286 known_contexts, known_aggs);
3287 gcc_assert (*size == old_size);
3288 if (time && (*time - old_time > 1 || *time - old_time < -1)
3289 && dump_file)
f5b25e15 3290 fprintf (dump_file, "Time mismatch in call summary %f!=%f\n",
070e3489
JH
3291 old_time.to_double (),
3292 time->to_double ());
3293 }
3294 }
3295 /* Slow path by walking all edges. */
3296 else
3297 estimate_calls_size_and_time_1 (node, size, min_size, time, hints,
3298 possible_truths, known_vals, known_contexts,
3299 known_aggs);
3300}
3301
1532500e 3302/* Default constructor for ipa call context.
956d615d 3303 Memory allocation of known_vals, known_contexts
1532500e
JH
3304 and known_aggs vectors is owned by the caller, but can
3305 be release by ipa_call_context::release.
3306
3307 inline_param_summary is owned by the caller. */
3308ipa_call_context::ipa_call_context (cgraph_node *node,
3309 clause_t possible_truths,
3310 clause_t nonspec_possible_truths,
3311 vec<tree> known_vals,
3312 vec<ipa_polymorphic_call_context>
3313 known_contexts,
eb270950 3314 vec<ipa_agg_value_set> known_aggs,
1532500e
JH
3315 vec<inline_param_summary>
3316 inline_param_summary)
3317: m_node (node), m_possible_truths (possible_truths),
3318 m_nonspec_possible_truths (nonspec_possible_truths),
3319 m_inline_param_summary (inline_param_summary),
3320 m_known_vals (known_vals),
3321 m_known_contexts (known_contexts),
3322 m_known_aggs (known_aggs)
3323{
3324}
3325
40a777e8
JH
3326/* Set THIS to be a duplicate of CTX. Copy all relevant info. */
3327
ac6f2e59
JH
3328void
3329ipa_call_context::duplicate_from (const ipa_call_context &ctx)
3330{
3331 m_node = ctx.m_node;
3332 m_possible_truths = ctx.m_possible_truths;
3333 m_nonspec_possible_truths = ctx.m_nonspec_possible_truths;
40a777e8 3334 class ipa_node_params *params_summary = IPA_NODE_REF (m_node);
6cf67b62
JH
3335 unsigned int nargs = params_summary
3336 ? ipa_get_param_count (params_summary) : 0;
ac6f2e59 3337
40a777e8
JH
3338 m_inline_param_summary = vNULL;
3339 /* Copy the info only if there is at least one useful entry. */
ac6f2e59 3340 if (ctx.m_inline_param_summary.exists ())
40a777e8
JH
3341 {
3342 unsigned int n = MIN (ctx.m_inline_param_summary.length (), nargs);
3343
3344 for (unsigned int i = 0; i < n; i++)
3345 if (ipa_is_param_used_by_ipa_predicates (params_summary, i)
3346 && !ctx.m_inline_param_summary[i].useless_p ())
3347 {
3348 m_inline_param_summary
3349 = ctx.m_inline_param_summary.copy ();
3350 break;
3351 }
3352 }
3353 m_known_vals = vNULL;
ac6f2e59 3354 if (ctx.m_known_vals.exists ())
40a777e8
JH
3355 {
3356 unsigned int n = MIN (ctx.m_known_vals.length (), nargs);
3357
3358 for (unsigned int i = 0; i < n; i++)
3359 if (ipa_is_param_used_by_indirect_call (params_summary, i)
3360 && ctx.m_known_vals[i])
3361 {
3362 m_known_vals = ctx.m_known_vals.copy ();
3363 break;
3364 }
3365 }
3366
3367 m_known_contexts = vNULL;
ac6f2e59 3368 if (ctx.m_known_contexts.exists ())
40a777e8
JH
3369 {
3370 unsigned int n = MIN (ctx.m_known_contexts.length (), nargs);
3371
3372 for (unsigned int i = 0; i < n; i++)
3373 if (ipa_is_param_used_by_polymorphic_call (params_summary, i)
3374 && !ctx.m_known_contexts[i].useless_p ())
3375 {
3376 m_known_contexts = ctx.m_known_contexts.copy ();
3377 break;
3378 }
3379 }
3380
3381 m_known_aggs = vNULL;
ac6f2e59 3382 if (ctx.m_known_aggs.exists ())
40a777e8
JH
3383 {
3384 unsigned int n = MIN (ctx.m_known_aggs.length (), nargs);
3385
3386 for (unsigned int i = 0; i < n; i++)
3387 if (ipa_is_param_used_by_indirect_call (params_summary, i)
eb270950 3388 && !ctx.m_known_aggs[i].is_empty ())
40a777e8 3389 {
eb270950 3390 m_known_aggs = ipa_copy_agg_values (ctx.m_known_aggs);
40a777e8
JH
3391 break;
3392 }
3393 }
ac6f2e59
JH
3394}
3395
3396/* Release memory used by known_vals/contexts/aggs vectors.
3397 If ALL is true release also inline_param_summary.
956d615d 3398 This happens when context was previously duplicated to be stored
ac6f2e59 3399 into cache. */
1532500e
JH
3400
3401void
ac6f2e59 3402ipa_call_context::release (bool all)
1532500e 3403{
ac6f2e59
JH
3404 /* See if context is initialized at first place. */
3405 if (!m_node)
3406 return;
b0d55476 3407 ipa_release_agg_values (m_known_aggs, all);
ac6f2e59 3408 if (all)
b0d55476
JH
3409 {
3410 m_known_vals.release ();
3411 m_known_contexts.release ();
3412 m_inline_param_summary.release ();
3413 }
ac6f2e59
JH
3414}
3415
3416/* Return true if CTX describes the same call context as THIS. */
3417
3418bool
3419ipa_call_context::equal_to (const ipa_call_context &ctx)
3420{
3421 if (m_node != ctx.m_node
3422 || m_possible_truths != ctx.m_possible_truths
3423 || m_nonspec_possible_truths != ctx.m_nonspec_possible_truths)
3424 return false;
40a777e8
JH
3425
3426 class ipa_node_params *params_summary = IPA_NODE_REF (m_node);
6cf67b62
JH
3427 unsigned int nargs = params_summary
3428 ? ipa_get_param_count (params_summary) : 0;
40a777e8
JH
3429
3430 if (m_inline_param_summary.exists () || ctx.m_inline_param_summary.exists ())
ac6f2e59 3431 {
40a777e8
JH
3432 for (unsigned int i = 0; i < nargs; i++)
3433 {
3434 if (!ipa_is_param_used_by_ipa_predicates (params_summary, i))
3435 continue;
3436 if (i >= m_inline_param_summary.length ()
3437 || m_inline_param_summary[i].useless_p ())
3438 {
3439 if (i < ctx.m_inline_param_summary.length ()
3440 && !ctx.m_inline_param_summary[i].useless_p ())
3441 return false;
3442 continue;
3443 }
3444 if (i >= ctx.m_inline_param_summary.length ()
3445 || ctx.m_inline_param_summary[i].useless_p ())
3446 {
3447 if (i < m_inline_param_summary.length ()
3448 && !m_inline_param_summary[i].useless_p ())
3449 return false;
3450 continue;
3451 }
3452 if (!m_inline_param_summary[i].equal_to
3453 (ctx.m_inline_param_summary[i]))
3454 return false;
3455 }
ac6f2e59 3456 }
40a777e8 3457 if (m_known_vals.exists () || ctx.m_known_vals.exists ())
ac6f2e59 3458 {
40a777e8 3459 for (unsigned int i = 0; i < nargs; i++)
ac6f2e59 3460 {
40a777e8
JH
3461 if (!ipa_is_param_used_by_indirect_call (params_summary, i))
3462 continue;
3463 if (i >= m_known_vals.length () || !m_known_vals[i])
3464 {
3465 if (i < ctx.m_known_vals.length () && ctx.m_known_vals[i])
3466 return false;
3467 continue;
3468 }
3469 if (i >= ctx.m_known_vals.length () || !ctx.m_known_vals[i])
3470 {
3471 if (i < m_known_vals.length () && m_known_vals[i])
3472 return false;
3473 continue;
3474 }
3475 if (m_known_vals[i] != ctx.m_known_vals[i])
ac6f2e59
JH
3476 return false;
3477 }
3478 }
40a777e8 3479 if (m_known_contexts.exists () || ctx.m_known_contexts.exists ())
ac6f2e59 3480 {
40a777e8
JH
3481 for (unsigned int i = 0; i < nargs; i++)
3482 {
3483 if (!ipa_is_param_used_by_polymorphic_call (params_summary, i))
3484 continue;
3485 if (i >= m_known_contexts.length ()
3486 || m_known_contexts[i].useless_p ())
3487 {
3488 if (i < ctx.m_known_contexts.length ()
3489 && !ctx.m_known_contexts[i].useless_p ())
3490 return false;
3491 continue;
3492 }
3493 if (i >= ctx.m_known_contexts.length ()
3494 || ctx.m_known_contexts[i].useless_p ())
3495 {
3496 if (i < m_known_contexts.length ()
3497 && !m_known_contexts[i].useless_p ())
3498 return false;
3499 continue;
3500 }
3501 if (!m_known_contexts[i].equal_to
3502 (ctx.m_known_contexts[i]))
3503 return false;
3504 }
ac6f2e59 3505 }
40a777e8 3506 if (m_known_aggs.exists () || ctx.m_known_aggs.exists ())
ac6f2e59 3507 {
40a777e8
JH
3508 for (unsigned int i = 0; i < nargs; i++)
3509 {
3510 if (!ipa_is_param_used_by_indirect_call (params_summary, i))
3511 continue;
eb270950 3512 if (i >= m_known_aggs.length () || m_known_aggs[i].is_empty ())
40a777e8 3513 {
eb270950
FX
3514 if (i < ctx.m_known_aggs.length ()
3515 && !ctx.m_known_aggs[i].is_empty ())
40a777e8
JH
3516 return false;
3517 continue;
3518 }
eb270950
FX
3519 if (i >= ctx.m_known_aggs.length ()
3520 || ctx.m_known_aggs[i].is_empty ())
40a777e8 3521 {
eb270950
FX
3522 if (i < m_known_aggs.length ()
3523 && !m_known_aggs[i].is_empty ())
40a777e8
JH
3524 return false;
3525 continue;
3526 }
eb270950 3527 if (!m_known_aggs[i].equal_to (ctx.m_known_aggs[i]))
40a777e8
JH
3528 return false;
3529 }
ac6f2e59
JH
3530 }
3531 return true;
1532500e 3532}
27d020cf 3533
1532500e 3534/* Estimate size and time needed to execute call in the given context.
956d615d 3535 Additionally determine hints determined by the context. Finally compute
27d020cf
JH
3536 minimal size needed for the call that is independent on the call context and
3537 can be used for fast estimates. Return the values in RET_SIZE,
3538 RET_MIN_SIZE, RET_TIME and RET_HINTS. */
3539
3540void
1532500e
JH
3541ipa_call_context::estimate_size_and_time (int *ret_size,
3542 int *ret_min_size,
3543 sreal *ret_time,
3544 sreal *ret_nonspecialized_time,
3545 ipa_hints *ret_hints)
27d020cf 3546{
7237f93e 3547 class ipa_fn_summary *info = ipa_fn_summaries->get (m_node);
27d020cf
JH
3548 size_time_entry *e;
3549 int size = 0;
3550 sreal time = 0;
3551 int min_size = 0;
0bceb671 3552 ipa_hints hints = 0;
27d020cf
JH
3553 int i;
3554
3555 if (dump_file && (dump_flags & TDF_DETAILS))
3556 {
3557 bool found = false;
d597b944
ML
3558 fprintf (dump_file, " Estimating body: %s\n"
3559 " Known to be false: ", m_node->dump_name ());
27d020cf
JH
3560
3561 for (i = predicate::not_inlined_condition;
3562 i < (predicate::first_dynamic_condition
3563 + (int) vec_safe_length (info->conds)); i++)
1532500e 3564 if (!(m_possible_truths & (1 << i)))
27d020cf
JH
3565 {
3566 if (found)
3567 fprintf (dump_file, ", ");
3568 found = true;
3569 dump_condition (dump_file, info->conds, i);
3570 }
3571 }
3572
070e3489
JH
3573 if (m_node->callees || m_node->indirect_calls)
3574 estimate_calls_size_and_time (m_node, &size, &min_size,
3575 ret_time ? &time : NULL,
3576 ret_hints ? &hints : NULL, m_possible_truths,
3577 m_known_vals, m_known_contexts, m_known_aggs);
83263ef5 3578
27d020cf
JH
3579 sreal nonspecialized_time = time;
3580
e3bd08dd 3581 min_size += (*info->size_time_table)[0].size;
27d020cf
JH
3582 for (i = 0; vec_safe_iterate (info->size_time_table, i, &e); i++)
3583 {
1532500e 3584 bool exec = e->exec_predicate.evaluate (m_nonspec_possible_truths);
3494e738
JH
3585
3586 /* Because predicates are conservative, it can happen that nonconst is 1
3587 but exec is 0. */
27d020cf
JH
3588 if (exec)
3589 {
1532500e 3590 bool nonconst = e->nonconst_predicate.evaluate (m_possible_truths);
3494e738 3591
27d020cf
JH
3592 gcc_checking_assert (e->time >= 0);
3593 gcc_checking_assert (time >= 0);
3594
3595 /* We compute specialized size only because size of nonspecialized
3596 copy is context independent.
3597
3598 The difference between nonspecialized execution and specialized is
3599 that nonspecialized is not going to have optimized out computations
3600 known to be constant in a specialized setting. */
3601 if (nonconst)
3602 size += e->size;
83263ef5
JH
3603 if (!ret_time)
3604 continue;
27d020cf
JH
3605 nonspecialized_time += e->time;
3606 if (!nonconst)
3607 ;
1532500e 3608 else if (!m_inline_param_summary.exists ())
27d020cf
JH
3609 {
3610 if (nonconst)
3611 time += e->time;
3612 }
3613 else
3614 {
3615 int prob = e->nonconst_predicate.probability
1532500e
JH
3616 (info->conds, m_possible_truths,
3617 m_inline_param_summary);
27d020cf
JH
3618 gcc_checking_assert (prob >= 0);
3619 gcc_checking_assert (prob <= REG_BR_PROB_BASE);
fd4656a2
JH
3620 if (prob == REG_BR_PROB_BASE)
3621 time += e->time;
3622 else
3623 time += e->time * prob / REG_BR_PROB_BASE;
27d020cf
JH
3624 }
3625 gcc_checking_assert (time >= 0);
3626 }
3627 }
3628 gcc_checking_assert ((*info->size_time_table)[0].exec_predicate == true);
3629 gcc_checking_assert ((*info->size_time_table)[0].nonconst_predicate == true);
e3bd08dd 3630 gcc_checking_assert (min_size >= 0);
27d020cf
JH
3631 gcc_checking_assert (size >= 0);
3632 gcc_checking_assert (time >= 0);
3633 /* nonspecialized_time should be always bigger than specialized time.
3634 Roundoff issues however may get into the way. */
59d27026 3635 gcc_checking_assert ((nonspecialized_time - time * 99 / 100) >= -1);
27d020cf
JH
3636
3637 /* Roundoff issues may make specialized time bigger than nonspecialized
956d615d 3638 time. We do not really want that to happen because some heuristics
27d020cf
JH
3639 may get confused by seeing negative speedups. */
3640 if (time > nonspecialized_time)
3641 time = nonspecialized_time;
3642
83263ef5
JH
3643 if (ret_hints)
3644 {
3645 if (info->loop_iterations
3646 && !info->loop_iterations->evaluate (m_possible_truths))
3647 hints |= INLINE_HINT_loop_iterations;
3648 if (info->loop_stride
3649 && !info->loop_stride->evaluate (m_possible_truths))
3650 hints |= INLINE_HINT_loop_stride;
3651 if (info->scc_no)
3652 hints |= INLINE_HINT_in_scc;
3653 if (DECL_DECLARED_INLINE_P (m_node->decl))
3654 hints |= INLINE_HINT_declared_inline;
3655 }
27d020cf 3656
0bceb671
JH
3657 size = RDIV (size, ipa_fn_summary::size_scale);
3658 min_size = RDIV (min_size, ipa_fn_summary::size_scale);
27d020cf
JH
3659
3660 if (dump_file && (dump_flags & TDF_DETAILS))
3661 fprintf (dump_file, "\n size:%i time:%f nonspec time:%f\n", (int) size,
3662 time.to_double (), nonspecialized_time.to_double ());
3663 if (ret_time)
3664 *ret_time = time;
3665 if (ret_nonspecialized_time)
3666 *ret_nonspecialized_time = nonspecialized_time;
3667 if (ret_size)
3668 *ret_size = size;
3669 if (ret_min_size)
3670 *ret_min_size = min_size;
3671 if (ret_hints)
3672 *ret_hints = hints;
3673 return;
3674}
3675
3676
3677/* Estimate size and time needed to execute callee of EDGE assuming that
3678 parameters known to be constant at caller of EDGE are propagated.
3679 KNOWN_VALS and KNOWN_CONTEXTS are vectors of assumed known constant values
3680 and types for parameters. */
3681
3682void
3683estimate_ipcp_clone_size_and_time (struct cgraph_node *node,
3684 vec<tree> known_vals,
3685 vec<ipa_polymorphic_call_context>
3686 known_contexts,
eb270950 3687 vec<ipa_agg_value_set> known_aggs,
27d020cf
JH
3688 int *ret_size, sreal *ret_time,
3689 sreal *ret_nonspec_time,
0bceb671 3690 ipa_hints *hints)
27d020cf
JH
3691{
3692 clause_t clause, nonspec_clause;
3693
68718e8e 3694 /* TODO: Also pass known value ranges. */
82c4b78d 3695 evaluate_conditions_for_known_args (node, false, known_vals, vNULL,
68718e8e 3696 known_aggs, &clause, &nonspec_clause);
1532500e
JH
3697 ipa_call_context ctx (node, clause, nonspec_clause,
3698 known_vals, known_contexts,
3699 known_aggs, vNULL);
3700 ctx.estimate_size_and_time (ret_size, NULL, ret_time,
3701 ret_nonspec_time, hints);
27d020cf
JH
3702}
3703
f658ad30
JH
3704/* Return stack frame offset where frame of NODE is supposed to start inside
3705 of the function it is inlined to.
3706 Return 0 for functions that are not inlined. */
3707
3708HOST_WIDE_INT
3709ipa_get_stack_frame_offset (struct cgraph_node *node)
3710{
3711 HOST_WIDE_INT offset = 0;
a62bfab5 3712 if (!node->inlined_to)
f658ad30
JH
3713 return 0;
3714 node = node->callers->caller;
3715 while (true)
3716 {
3717 offset += ipa_size_summaries->get (node)->estimated_self_stack_size;
a62bfab5 3718 if (!node->inlined_to)
f658ad30
JH
3719 return offset;
3720 node = node->callers->caller;
3721 }
3722}
3723
27d020cf
JH
3724
3725/* Update summary information of inline clones after inlining.
3726 Compute peak stack usage. */
3727
3728static void
3729inline_update_callee_summaries (struct cgraph_node *node, int depth)
3730{
3731 struct cgraph_edge *e;
f658ad30 3732
27d020cf
JH
3733 ipa_propagate_frequency (node);
3734 for (e = node->callees; e; e = e->next_callee)
3735 {
3736 if (!e->inline_failed)
3737 inline_update_callee_summaries (e->callee, depth);
7237f93e
JH
3738 else
3739 ipa_call_summaries->get (e)->loop_depth += depth;
27d020cf
JH
3740 }
3741 for (e = node->indirect_calls; e; e = e->next_callee)
56f62793 3742 ipa_call_summaries->get (e)->loop_depth += depth;
27d020cf
JH
3743}
3744
3745/* Update change_prob of EDGE after INLINED_EDGE has been inlined.
6cf67b62 3746 When function A is inlined in B and A calls C with parameter that
956d615d 3747 changes with probability PROB1 and C is known to be passthrough
27d020cf
JH
3748 of argument if B that change with probability PROB2, the probability
3749 of change is now PROB1*PROB2. */
3750
3751static void
3752remap_edge_change_prob (struct cgraph_edge *inlined_edge,
3753 struct cgraph_edge *edge)
3754{
3755 if (ipa_node_params_sum)
3756 {
3757 int i;
99b1c316 3758 class ipa_edge_args *args = IPA_EDGE_REF (edge);
a33c028e
JH
3759 if (!args)
3760 return;
99b1c316
MS
3761 class ipa_call_summary *es = ipa_call_summaries->get (edge);
3762 class ipa_call_summary *inlined_es
56f62793 3763 = ipa_call_summaries->get (inlined_edge);
27d020cf 3764
8c02e054
JH
3765 if (es->param.length () == 0)
3766 return;
3767
27d020cf
JH
3768 for (i = 0; i < ipa_get_cs_argument_count (args); i++)
3769 {
3770 struct ipa_jump_func *jfunc = ipa_get_ith_jump_func (args, i);
3771 if (jfunc->type == IPA_JF_PASS_THROUGH
3772 || jfunc->type == IPA_JF_ANCESTOR)
3773 {
3774 int id = jfunc->type == IPA_JF_PASS_THROUGH
3775 ? ipa_get_jf_pass_through_formal_id (jfunc)
3776 : ipa_get_jf_ancestor_formal_id (jfunc);
3777 if (id < (int) inlined_es->param.length ())
3778 {
3779 int prob1 = es->param[i].change_prob;
3780 int prob2 = inlined_es->param[id].change_prob;
3781 int prob = combine_probabilities (prob1, prob2);
3782
3783 if (prob1 && prob2 && !prob)
3784 prob = 1;
3785
3786 es->param[i].change_prob = prob;
3787 }
3788 }
3789 }
3790 }
3791}
3792
3793/* Update edge summaries of NODE after INLINED_EDGE has been inlined.
3794
3795 Remap predicates of callees of NODE. Rest of arguments match
3796 remap_predicate.
3797
3798 Also update change probabilities. */
3799
3800static void
3801remap_edge_summaries (struct cgraph_edge *inlined_edge,
3802 struct cgraph_node *node,
99b1c316 3803 class ipa_fn_summary *info,
40a777e8 3804 class ipa_node_params *params_summary,
99b1c316 3805 class ipa_fn_summary *callee_info,
27d020cf
JH
3806 vec<int> operand_map,
3807 vec<int> offset_map,
3808 clause_t possible_truths,
3809 predicate *toplev_predicate)
3810{
3811 struct cgraph_edge *e, *next;
3812 for (e = node->callees; e; e = next)
3813 {
27d020cf
JH
3814 predicate p;
3815 next = e->next_callee;
3816
3817 if (e->inline_failed)
3818 {
6cf67b62 3819 class ipa_call_summary *es = ipa_call_summaries->get (e);
27d020cf
JH
3820 remap_edge_change_prob (inlined_edge, e);
3821
3822 if (es->predicate)
3823 {
3824 p = es->predicate->remap_after_inlining
40a777e8
JH
3825 (info, params_summary,
3826 callee_info, operand_map,
27d020cf
JH
3827 offset_map, possible_truths,
3828 *toplev_predicate);
3829 edge_set_predicate (e, &p);
3830 }
3831 else
3832 edge_set_predicate (e, toplev_predicate);
3833 }
3834 else
40a777e8
JH
3835 remap_edge_summaries (inlined_edge, e->callee, info,
3836 params_summary, callee_info,
27d020cf
JH
3837 operand_map, offset_map, possible_truths,
3838 toplev_predicate);
3839 }
3840 for (e = node->indirect_calls; e; e = next)
3841 {
99b1c316 3842 class ipa_call_summary *es = ipa_call_summaries->get (e);
27d020cf
JH
3843 predicate p;
3844 next = e->next_callee;
3845
3846 remap_edge_change_prob (inlined_edge, e);
3847 if (es->predicate)
3848 {
3849 p = es->predicate->remap_after_inlining
40a777e8
JH
3850 (info, params_summary,
3851 callee_info, operand_map, offset_map,
27d020cf
JH
3852 possible_truths, *toplev_predicate);
3853 edge_set_predicate (e, &p);
3854 }
3855 else
3856 edge_set_predicate (e, toplev_predicate);
3857 }
3858}
3859
3860/* Same as remap_predicate, but set result into hint *HINT. */
3861
3862static void
99b1c316 3863remap_hint_predicate (class ipa_fn_summary *info,
40a777e8 3864 class ipa_node_params *params_summary,
99b1c316 3865 class ipa_fn_summary *callee_info,
27d020cf
JH
3866 predicate **hint,
3867 vec<int> operand_map,
3868 vec<int> offset_map,
3869 clause_t possible_truths,
3870 predicate *toplev_predicate)
3871{
3872 predicate p;
3873
3874 if (!*hint)
3875 return;
3876 p = (*hint)->remap_after_inlining
40a777e8 3877 (info, params_summary, callee_info,
27d020cf
JH
3878 operand_map, offset_map,
3879 possible_truths, *toplev_predicate);
3880 if (p != false && p != true)
3881 {
3882 if (!*hint)
3883 set_hint_predicate (hint, p);
3884 else
3885 **hint &= p;
3886 }
3887}
3888
3889/* We inlined EDGE. Update summary of the function we inlined into. */
3890
3891void
0bceb671 3892ipa_merge_fn_summary_after_inlining (struct cgraph_edge *edge)
27d020cf 3893{
56f62793 3894 ipa_fn_summary *callee_info = ipa_fn_summaries->get (edge->callee);
a62bfab5
ML
3895 struct cgraph_node *to = (edge->caller->inlined_to
3896 ? edge->caller->inlined_to : edge->caller);
99b1c316 3897 class ipa_fn_summary *info = ipa_fn_summaries->get (to);
27d020cf
JH
3898 clause_t clause = 0; /* not_inline is known to be false. */
3899 size_time_entry *e;
f658ad30
JH
3900 auto_vec<int, 8> operand_map;
3901 auto_vec<int, 8> offset_map;
27d020cf
JH
3902 int i;
3903 predicate toplev_predicate;
99b1c316 3904 class ipa_call_summary *es = ipa_call_summaries->get (edge);
40a777e8
JH
3905 class ipa_node_params *params_summary = (ipa_node_params_sum
3906 ? IPA_NODE_REF (to) : NULL);
27d020cf
JH
3907
3908 if (es->predicate)
3909 toplev_predicate = *es->predicate;
3910 else
3911 toplev_predicate = true;
3912
3913 info->fp_expressions |= callee_info->fp_expressions;
3914
3915 if (callee_info->conds)
b0d55476
JH
3916 {
3917 auto_vec<tree, 32> known_vals;
3918 auto_vec<ipa_agg_value_set, 32> known_aggs;
3919 evaluate_properties_for_edge (edge, true, &clause, NULL,
3920 &known_vals, NULL, &known_aggs);
3921 }
27d020cf
JH
3922 if (ipa_node_params_sum && callee_info->conds)
3923 {
99b1c316 3924 class ipa_edge_args *args = IPA_EDGE_REF (edge);
5a0236f8 3925 int count = args ? ipa_get_cs_argument_count (args) : 0;
27d020cf
JH
3926 int i;
3927
3928 if (count)
3929 {
cb3874dc
ML
3930 operand_map.safe_grow_cleared (count, true);
3931 offset_map.safe_grow_cleared (count, true);
27d020cf
JH
3932 }
3933 for (i = 0; i < count; i++)
3934 {
3935 struct ipa_jump_func *jfunc = ipa_get_ith_jump_func (args, i);
3936 int map = -1;
3937
3938 /* TODO: handle non-NOPs when merging. */
3939 if (jfunc->type == IPA_JF_PASS_THROUGH)
3940 {
3941 if (ipa_get_jf_pass_through_operation (jfunc) == NOP_EXPR)
3942 map = ipa_get_jf_pass_through_formal_id (jfunc);
3943 if (!ipa_get_jf_pass_through_agg_preserved (jfunc))
3944 offset_map[i] = -1;
3945 }
3946 else if (jfunc->type == IPA_JF_ANCESTOR)
3947 {
3948 HOST_WIDE_INT offset = ipa_get_jf_ancestor_offset (jfunc);
3949 if (offset >= 0 && offset < INT_MAX)
3950 {
3951 map = ipa_get_jf_ancestor_formal_id (jfunc);
3952 if (!ipa_get_jf_ancestor_agg_preserved (jfunc))
3953 offset = -1;
3954 offset_map[i] = offset;
3955 }
3956 }
3957 operand_map[i] = map;
40a777e8 3958 gcc_assert (map < ipa_get_param_count (params_summary));
27d020cf
JH
3959 }
3960 }
83263ef5 3961 sreal freq = edge->sreal_frequency ();
27d020cf
JH
3962 for (i = 0; vec_safe_iterate (callee_info->size_time_table, i, &e); i++)
3963 {
3964 predicate p;
3965 p = e->exec_predicate.remap_after_inlining
40a777e8
JH
3966 (info, params_summary,
3967 callee_info, operand_map,
27d020cf
JH
3968 offset_map, clause,
3969 toplev_predicate);
3970 predicate nonconstp;
3971 nonconstp = e->nonconst_predicate.remap_after_inlining
40a777e8
JH
3972 (info, params_summary,
3973 callee_info, operand_map,
27d020cf
JH
3974 offset_map, clause,
3975 toplev_predicate);
3976 if (p != false && nonconstp != false)
3977 {
83263ef5 3978 sreal add_time = ((sreal)e->time * freq);
27d020cf
JH
3979 int prob = e->nonconst_predicate.probability (callee_info->conds,
3980 clause, es->param);
fd4656a2
JH
3981 if (prob != REG_BR_PROB_BASE)
3982 add_time = add_time * prob / REG_BR_PROB_BASE;
27d020cf
JH
3983 if (prob != REG_BR_PROB_BASE
3984 && dump_file && (dump_flags & TDF_DETAILS))
3985 {
3986 fprintf (dump_file, "\t\tScaling time by probability:%f\n",
3987 (double) prob / REG_BR_PROB_BASE);
3988 }
3989 info->account_size_time (e->size, add_time, p, nonconstp);
3990 }
3991 }
40a777e8
JH
3992 remap_edge_summaries (edge, edge->callee, info, params_summary,
3993 callee_info, operand_map,
27d020cf 3994 offset_map, clause, &toplev_predicate);
40a777e8 3995 remap_hint_predicate (info, params_summary, callee_info,
27d020cf
JH
3996 &callee_info->loop_iterations,
3997 operand_map, offset_map, clause, &toplev_predicate);
40a777e8 3998 remap_hint_predicate (info, params_summary, callee_info,
27d020cf
JH
3999 &callee_info->loop_stride,
4000 operand_map, offset_map, clause, &toplev_predicate);
27d020cf 4001
f658ad30
JH
4002 HOST_WIDE_INT stack_frame_offset = ipa_get_stack_frame_offset (edge->callee);
4003 HOST_WIDE_INT peak = stack_frame_offset + callee_info->estimated_stack_size;
27d020cf 4004
f658ad30
JH
4005 if (info->estimated_stack_size < peak)
4006 info->estimated_stack_size = peak;
4007
4008 inline_update_callee_summaries (edge->callee, es->loop_depth);
d2bcf46c
JH
4009 if (info->call_size_time_table)
4010 {
4011 int edge_size = 0;
4012 sreal edge_time = 0;
4013
4014 estimate_edge_size_and_time (edge, &edge_size, NULL, &edge_time, vNULL,
4015 vNULL, vNULL, 0);
4016 /* Unaccount size and time of the optimized out call. */
4017 info->account_size_time (-edge_size, -edge_time,
4018 es->predicate ? *es->predicate : true,
4019 es->predicate ? *es->predicate : true,
4020 true);
4021 /* Account new calls. */
4022 summarize_calls_size_and_time (edge->callee, info);
4023 }
f658ad30
JH
4024
4025 /* Free summaries that are not maintained for inline clones/edges. */
4026 ipa_call_summaries->remove (edge);
4027 ipa_fn_summaries->remove (edge->callee);
7237f93e 4028 ipa_remove_from_growth_caches (edge);
27d020cf
JH
4029}
4030
f658ad30 4031/* For performance reasons ipa_merge_fn_summary_after_inlining is not updating
d2bcf46c
JH
4032 overall size and time. Recompute it.
4033 If RESET is true also recompute call_time_size_table. */
27d020cf
JH
4034
4035void
d2bcf46c 4036ipa_update_overall_fn_summary (struct cgraph_node *node, bool reset)
27d020cf 4037{
7237f93e
JH
4038 class ipa_fn_summary *info = ipa_fn_summaries->get (node);
4039 class ipa_size_summary *size_info = ipa_size_summaries->get (node);
27d020cf
JH
4040 size_time_entry *e;
4041 int i;
4042
f658ad30 4043 size_info->size = 0;
27d020cf
JH
4044 info->time = 0;
4045 for (i = 0; vec_safe_iterate (info->size_time_table, i, &e); i++)
4046 {
f658ad30 4047 size_info->size += e->size;
27d020cf
JH
4048 info->time += e->time;
4049 }
e3bd08dd 4050 info->min_size = (*info->size_time_table)[0].size;
d2bcf46c
JH
4051 if (reset)
4052 vec_free (info->call_size_time_table);
070e3489
JH
4053 if (node->callees || node->indirect_calls)
4054 estimate_calls_size_and_time (node, &size_info->size, &info->min_size,
4055 &info->time, NULL,
4056 ~(clause_t) (1 << predicate::false_condition),
4057 vNULL, vNULL, vNULL);
e3bd08dd
JH
4058 size_info->size = RDIV (size_info->size, ipa_fn_summary::size_scale);
4059 info->min_size = RDIV (info->min_size, ipa_fn_summary::size_scale);
27d020cf
JH
4060}
4061
4062
4063/* This function performs intraprocedural analysis in NODE that is required to
4064 inline indirect calls. */
4065
4066static void
4067inline_indirect_intraprocedural_analysis (struct cgraph_node *node)
4068{
4069 ipa_analyze_node (node);
4070 if (dump_file && (dump_flags & TDF_DETAILS))
4071 {
4072 ipa_print_node_params (dump_file, node);
4073 ipa_print_node_jump_functions (dump_file, node);
4074 }
4075}
4076
4077
4078/* Note function body size. */
4079
4080void
4081inline_analyze_function (struct cgraph_node *node)
4082{
4083 push_cfun (DECL_STRUCT_FUNCTION (node->decl));
4084
4085 if (dump_file)
d597b944 4086 fprintf (dump_file, "\nAnalyzing function: %s\n", node->dump_name ());
27d020cf
JH
4087 if (opt_for_fn (node->decl, optimize) && !node->thunk.thunk_p)
4088 inline_indirect_intraprocedural_analysis (node);
0bceb671 4089 compute_fn_summary (node, false);
27d020cf
JH
4090 if (!optimize)
4091 {
4092 struct cgraph_edge *e;
4093 for (e = node->callees; e; e = e->next_callee)
4094 e->inline_failed = CIF_FUNCTION_NOT_OPTIMIZED;
4095 for (e = node->indirect_calls; e; e = e->next_callee)
4096 e->inline_failed = CIF_FUNCTION_NOT_OPTIMIZED;
4097 }
4098
4099 pop_cfun ();
4100}
4101
4102
4103/* Called when new function is inserted to callgraph late. */
4104
4105void
0bceb671 4106ipa_fn_summary_t::insert (struct cgraph_node *node, ipa_fn_summary *)
27d020cf
JH
4107{
4108 inline_analyze_function (node);
4109}
4110
4111/* Note function body size. */
4112
d2db2e6b
JH
4113static void
4114ipa_fn_summary_generate (void)
27d020cf
JH
4115{
4116 struct cgraph_node *node;
4117
4118 FOR_EACH_DEFINED_FUNCTION (node)
4119 if (DECL_STRUCT_FUNCTION (node->decl))
87f94429 4120 node->versionable = tree_versionable_function_p (node->decl);
27d020cf 4121
0bceb671 4122 ipa_fn_summary_alloc ();
27d020cf 4123
0bceb671 4124 ipa_fn_summaries->enable_insertion_hook ();
27d020cf
JH
4125
4126 ipa_register_cgraph_hooks ();
27d020cf
JH
4127
4128 FOR_EACH_DEFINED_FUNCTION (node)
29f1e2b1
JH
4129 if (!node->alias
4130 && (flag_generate_lto || flag_generate_offload|| flag_wpa
4131 || opt_for_fn (node->decl, optimize)))
27d020cf
JH
4132 inline_analyze_function (node);
4133}
4134
4135
4136/* Write inline summary for edge E to OB. */
4137
4138static void
99b1c316 4139read_ipa_call_summary (class lto_input_block *ib, struct cgraph_edge *e,
ddfb1317 4140 bool prevails)
27d020cf 4141{
99b1c316 4142 class ipa_call_summary *es = prevails
ddfb1317 4143 ? ipa_call_summaries->get_create (e) : NULL;
27d020cf
JH
4144 predicate p;
4145 int length, i;
4146
ddfb1317
JH
4147 int size = streamer_read_uhwi (ib);
4148 int time = streamer_read_uhwi (ib);
4149 int depth = streamer_read_uhwi (ib);
4150
4151 if (es)
4152 {
4153 es->call_stmt_size = size;
4154 es->call_stmt_time = time;
4155 es->loop_depth = depth;
4156 }
0fab169b
PK
4157
4158 bitpack_d bp = streamer_read_bitpack (ib);
ddfb1317
JH
4159 if (es)
4160 es->is_return_callee_uncaptured = bp_unpack_value (&bp, 1);
4161 else
4162 bp_unpack_value (&bp, 1);
0fab169b 4163
27d020cf 4164 p.stream_in (ib);
ddfb1317
JH
4165 if (es)
4166 edge_set_predicate (e, &p);
27d020cf 4167 length = streamer_read_uhwi (ib);
ddfb1317 4168 if (length && es && e->possibly_call_in_translation_unit_p ())
27d020cf 4169 {
cb3874dc 4170 es->param.safe_grow_cleared (length, true);
27d020cf
JH
4171 for (i = 0; i < length; i++)
4172 es->param[i].change_prob = streamer_read_uhwi (ib);
4173 }
ddfb1317
JH
4174 else
4175 {
4176 for (i = 0; i < length; i++)
4177 streamer_read_uhwi (ib);
4178 }
27d020cf
JH
4179}
4180
4181
4182/* Stream in inline summaries from the section. */
4183
4184static void
4185inline_read_section (struct lto_file_decl_data *file_data, const char *data,
4186 size_t len)
4187{
4188 const struct lto_function_header *header =
4189 (const struct lto_function_header *) data;
4190 const int cfg_offset = sizeof (struct lto_function_header);
4191 const int main_offset = cfg_offset + header->cfg_size;
4192 const int string_offset = main_offset + header->main_size;
99b1c316 4193 class data_in *data_in;
27d020cf
JH
4194 unsigned int i, count2, j;
4195 unsigned int f_count;
4196
4197 lto_input_block ib ((const char *) data + main_offset, header->main_size,
4198 file_data->mode_table);
4199
4200 data_in =
4201 lto_data_in_create (file_data, (const char *) data + string_offset,
4202 header->string_size, vNULL);
4203 f_count = streamer_read_uhwi (&ib);
4204 for (i = 0; i < f_count; i++)
4205 {
4206 unsigned int index;
4207 struct cgraph_node *node;
99b1c316 4208 class ipa_fn_summary *info;
40a777e8 4209 class ipa_node_params *params_summary;
f658ad30 4210 class ipa_size_summary *size_info;
27d020cf
JH
4211 lto_symtab_encoder_t encoder;
4212 struct bitpack_d bp;
4213 struct cgraph_edge *e;
4214 predicate p;
4215
4216 index = streamer_read_uhwi (&ib);
4217 encoder = file_data->symtab_node_encoder;
4218 node = dyn_cast<cgraph_node *> (lto_symtab_encoder_deref (encoder,
4219 index));
ddfb1317 4220 info = node->prevailing_p () ? ipa_fn_summaries->get_create (node) : NULL;
40a777e8 4221 params_summary = node->prevailing_p () ? IPA_NODE_REF (node) : NULL;
f658ad30
JH
4222 size_info = node->prevailing_p ()
4223 ? ipa_size_summaries->get_create (node) : NULL;
27d020cf 4224
ddfb1317
JH
4225 int stack_size = streamer_read_uhwi (&ib);
4226 int size = streamer_read_uhwi (&ib);
4227 sreal time = sreal::stream_in (&ib);
4228
4229 if (info)
4230 {
4231 info->estimated_stack_size
f658ad30
JH
4232 = size_info->estimated_self_stack_size = stack_size;
4233 size_info->size = size_info->self_size = size;
ddfb1317
JH
4234 info->time = time;
4235 }
27d020cf
JH
4236
4237 bp = streamer_read_bitpack (&ib);
ddfb1317
JH
4238 if (info)
4239 {
4240 info->inlinable = bp_unpack_value (&bp, 1);
4241 info->fp_expressions = bp_unpack_value (&bp, 1);
4242 }
4243 else
4244 {
4245 bp_unpack_value (&bp, 1);
4246 bp_unpack_value (&bp, 1);
4247 }
27d020cf
JH
4248
4249 count2 = streamer_read_uhwi (&ib);
ddfb1317 4250 gcc_assert (!info || !info->conds);
360386c7
JH
4251 if (info)
4252 vec_safe_reserve_exact (info->conds, count2);
27d020cf
JH
4253 for (j = 0; j < count2; j++)
4254 {
4255 struct condition c;
4307a485 4256 unsigned int k, count3;
27d020cf 4257 c.operand_num = streamer_read_uhwi (&ib);
27d020cf 4258 c.code = (enum tree_code) streamer_read_uhwi (&ib);
4307a485 4259 c.type = stream_read_tree (&ib, data_in);
27d020cf
JH
4260 c.val = stream_read_tree (&ib, data_in);
4261 bp = streamer_read_bitpack (&ib);
4262 c.agg_contents = bp_unpack_value (&bp, 1);
4263 c.by_ref = bp_unpack_value (&bp, 1);
4264 if (c.agg_contents)
4265 c.offset = streamer_read_uhwi (&ib);
4307a485 4266 count3 = streamer_read_uhwi (&ib);
360386c7
JH
4267 c.param_ops = NULL;
4268 if (info)
4269 vec_safe_reserve_exact (c.param_ops, count3);
40a777e8
JH
4270 if (params_summary)
4271 ipa_set_param_used_by_ipa_predicates
4272 (params_summary, c.operand_num, true);
4307a485
FX
4273 for (k = 0; k < count3; k++)
4274 {
4275 struct expr_eval_op op;
4276 enum gimple_rhs_class rhs_class;
4277 op.code = (enum tree_code) streamer_read_uhwi (&ib);
4278 op.type = stream_read_tree (&ib, data_in);
4279 switch (rhs_class = get_gimple_rhs_class (op.code))
4280 {
4281 case GIMPLE_UNARY_RHS:
4282 op.index = 0;
4283 op.val[0] = NULL_TREE;
4284 op.val[1] = NULL_TREE;
4285 break;
4286
4287 case GIMPLE_BINARY_RHS:
4288 case GIMPLE_TERNARY_RHS:
4289 bp = streamer_read_bitpack (&ib);
4290 op.index = bp_unpack_value (&bp, 2);
4291 op.val[0] = stream_read_tree (&ib, data_in);
4292 if (rhs_class == GIMPLE_BINARY_RHS)
4293 op.val[1] = NULL_TREE;
4294 else
4295 op.val[1] = stream_read_tree (&ib, data_in);
4296 break;
4297
4298 default:
4299 fatal_error (UNKNOWN_LOCATION,
4300 "invalid fnsummary in LTO stream");
4301 }
360386c7
JH
4302 if (info)
4303 c.param_ops->quick_push (op);
4307a485 4304 }
ddfb1317 4305 if (info)
360386c7 4306 info->conds->quick_push (c);
27d020cf
JH
4307 }
4308 count2 = streamer_read_uhwi (&ib);
ddfb1317 4309 gcc_assert (!info || !info->size_time_table);
360386c7
JH
4310 if (info && count2)
4311 vec_safe_reserve_exact (info->size_time_table, count2);
27d020cf
JH
4312 for (j = 0; j < count2; j++)
4313 {
99b1c316 4314 class size_time_entry e;
27d020cf
JH
4315
4316 e.size = streamer_read_uhwi (&ib);
4317 e.time = sreal::stream_in (&ib);
4318 e.exec_predicate.stream_in (&ib);
4319 e.nonconst_predicate.stream_in (&ib);
4320
ddfb1317 4321 if (info)
360386c7 4322 info->size_time_table->quick_push (e);
27d020cf
JH
4323 }
4324
4325 p.stream_in (&ib);
ddfb1317
JH
4326 if (info)
4327 set_hint_predicate (&info->loop_iterations, p);
27d020cf 4328 p.stream_in (&ib);
ddfb1317
JH
4329 if (info)
4330 set_hint_predicate (&info->loop_stride, p);
27d020cf 4331 for (e = node->callees; e; e = e->next_callee)
ddfb1317 4332 read_ipa_call_summary (&ib, e, info != NULL);
27d020cf 4333 for (e = node->indirect_calls; e; e = e->next_callee)
ddfb1317 4334 read_ipa_call_summary (&ib, e, info != NULL);
27d020cf
JH
4335 }
4336
0bceb671 4337 lto_free_section_data (file_data, LTO_section_ipa_fn_summary, NULL, data,
27d020cf
JH
4338 len);
4339 lto_data_in_delete (data_in);
4340}
4341
4342
4343/* Read inline summary. Jump functions are shared among ipa-cp
4344 and inliner, so when ipa-cp is active, we don't need to write them
4345 twice. */
4346
d2db2e6b
JH
4347static void
4348ipa_fn_summary_read (void)
27d020cf
JH
4349{
4350 struct lto_file_decl_data **file_data_vec = lto_get_file_decl_data ();
4351 struct lto_file_decl_data *file_data;
4352 unsigned int j = 0;
4353
0bceb671 4354 ipa_fn_summary_alloc ();
27d020cf
JH
4355
4356 while ((file_data = file_data_vec[j++]))
4357 {
4358 size_t len;
3c56d8d8
ML
4359 const char *data
4360 = lto_get_summary_section_data (file_data, LTO_section_ipa_fn_summary,
4361 &len);
27d020cf
JH
4362 if (data)
4363 inline_read_section (file_data, data, len);
4364 else
4365 /* Fatal error here. We do not want to support compiling ltrans units
4366 with different version of compiler or different flags than the WPA
4367 unit, so this should never happen. */
4368 fatal_error (input_location,
4369 "ipa inline summary is missing in input file");
4370 }
29f1e2b1
JH
4371 ipa_register_cgraph_hooks ();
4372 if (!flag_ipa_cp)
4373 ipa_prop_read_jump_functions ();
27d020cf 4374
0bceb671
JH
4375 gcc_assert (ipa_fn_summaries);
4376 ipa_fn_summaries->enable_insertion_hook ();
27d020cf
JH
4377}
4378
4379
4380/* Write inline summary for edge E to OB. */
4381
4382static void
4383write_ipa_call_summary (struct output_block *ob, struct cgraph_edge *e)
4384{
99b1c316 4385 class ipa_call_summary *es = ipa_call_summaries->get (e);
27d020cf
JH
4386 int i;
4387
4388 streamer_write_uhwi (ob, es->call_stmt_size);
4389 streamer_write_uhwi (ob, es->call_stmt_time);
4390 streamer_write_uhwi (ob, es->loop_depth);
0fab169b
PK
4391
4392 bitpack_d bp = bitpack_create (ob->main_stream);
4393 bp_pack_value (&bp, es->is_return_callee_uncaptured, 1);
4394 streamer_write_bitpack (&bp);
4395
27d020cf
JH
4396 if (es->predicate)
4397 es->predicate->stream_out (ob);
4398 else
4399 streamer_write_uhwi (ob, 0);
4400 streamer_write_uhwi (ob, es->param.length ());
4401 for (i = 0; i < (int) es->param.length (); i++)
4402 streamer_write_uhwi (ob, es->param[i].change_prob);
4403}
4404
4405
4406/* Write inline summary for node in SET.
4407 Jump functions are shared among ipa-cp and inliner, so when ipa-cp is
4408 active, we don't need to write them twice. */
4409
d2db2e6b
JH
4410static void
4411ipa_fn_summary_write (void)
27d020cf 4412{
0bceb671 4413 struct output_block *ob = create_output_block (LTO_section_ipa_fn_summary);
16570c12 4414 lto_symtab_encoder_iterator lsei;
27d020cf
JH
4415 lto_symtab_encoder_t encoder = ob->decl_state->symtab_node_encoder;
4416 unsigned int count = 0;
27d020cf 4417
16570c12
JJ
4418 for (lsei = lsei_start_function_in_partition (encoder); !lsei_end_p (lsei);
4419 lsei_next_function_in_partition (&lsei))
27d020cf 4420 {
16570c12
JJ
4421 cgraph_node *cnode = lsei_cgraph_node (lsei);
4422 if (cnode->definition && !cnode->alias)
27d020cf
JH
4423 count++;
4424 }
4425 streamer_write_uhwi (ob, count);
4426
16570c12
JJ
4427 for (lsei = lsei_start_function_in_partition (encoder); !lsei_end_p (lsei);
4428 lsei_next_function_in_partition (&lsei))
27d020cf 4429 {
16570c12
JJ
4430 cgraph_node *cnode = lsei_cgraph_node (lsei);
4431 if (cnode->definition && !cnode->alias)
27d020cf 4432 {
99b1c316 4433 class ipa_fn_summary *info = ipa_fn_summaries->get (cnode);
f658ad30 4434 class ipa_size_summary *size_info = ipa_size_summaries->get (cnode);
27d020cf
JH
4435 struct bitpack_d bp;
4436 struct cgraph_edge *edge;
4437 int i;
4438 size_time_entry *e;
4439 struct condition *c;
4440
4441 streamer_write_uhwi (ob, lto_symtab_encoder_encode (encoder, cnode));
f658ad30
JH
4442 streamer_write_hwi (ob, size_info->estimated_self_stack_size);
4443 streamer_write_hwi (ob, size_info->self_size);
27d020cf
JH
4444 info->time.stream_out (ob);
4445 bp = bitpack_create (ob->main_stream);
4446 bp_pack_value (&bp, info->inlinable, 1);
5e9d6aa4 4447 bp_pack_value (&bp, false, 1);
27d020cf
JH
4448 bp_pack_value (&bp, info->fp_expressions, 1);
4449 streamer_write_bitpack (&bp);
4450 streamer_write_uhwi (ob, vec_safe_length (info->conds));
4451 for (i = 0; vec_safe_iterate (info->conds, i, &c); i++)
4452 {
4307a485
FX
4453 int j;
4454 struct expr_eval_op *op;
4455
27d020cf 4456 streamer_write_uhwi (ob, c->operand_num);
27d020cf 4457 streamer_write_uhwi (ob, c->code);
4307a485 4458 stream_write_tree (ob, c->type, true);
27d020cf
JH
4459 stream_write_tree (ob, c->val, true);
4460 bp = bitpack_create (ob->main_stream);
4461 bp_pack_value (&bp, c->agg_contents, 1);
4462 bp_pack_value (&bp, c->by_ref, 1);
4463 streamer_write_bitpack (&bp);
4464 if (c->agg_contents)
4465 streamer_write_uhwi (ob, c->offset);
4307a485
FX
4466 streamer_write_uhwi (ob, vec_safe_length (c->param_ops));
4467 for (j = 0; vec_safe_iterate (c->param_ops, j, &op); j++)
4468 {
4469 streamer_write_uhwi (ob, op->code);
4470 stream_write_tree (ob, op->type, true);
4471 if (op->val[0])
4472 {
4473 bp = bitpack_create (ob->main_stream);
4474 bp_pack_value (&bp, op->index, 2);
4475 streamer_write_bitpack (&bp);
4476 stream_write_tree (ob, op->val[0], true);
4477 if (op->val[1])
4478 stream_write_tree (ob, op->val[1], true);
4479 }
4480 }
27d020cf
JH
4481 }
4482 streamer_write_uhwi (ob, vec_safe_length (info->size_time_table));
4483 for (i = 0; vec_safe_iterate (info->size_time_table, i, &e); i++)
4484 {
4485 streamer_write_uhwi (ob, e->size);
4486 e->time.stream_out (ob);
4487 e->exec_predicate.stream_out (ob);
4488 e->nonconst_predicate.stream_out (ob);
4489 }
4490 if (info->loop_iterations)
4491 info->loop_iterations->stream_out (ob);
4492 else
4493 streamer_write_uhwi (ob, 0);
4494 if (info->loop_stride)
4495 info->loop_stride->stream_out (ob);
4496 else
4497 streamer_write_uhwi (ob, 0);
27d020cf
JH
4498 for (edge = cnode->callees; edge; edge = edge->next_callee)
4499 write_ipa_call_summary (ob, edge);
4500 for (edge = cnode->indirect_calls; edge; edge = edge->next_callee)
4501 write_ipa_call_summary (ob, edge);
4502 }
4503 }
4504 streamer_write_char_stream (ob->main_stream, 0);
4505 produce_asm (ob, NULL);
4506 destroy_output_block (ob);
4507
29f1e2b1 4508 if (!flag_ipa_cp)
27d020cf
JH
4509 ipa_prop_write_jump_functions ();
4510}
4511
4512
f658ad30 4513/* Release function summary. */
27d020cf
JH
4514
4515void
d2db2e6b 4516ipa_free_fn_summary (void)
27d020cf 4517{
27d020cf
JH
4518 if (!ipa_call_summaries)
4519 return;
ddf628e4 4520 ggc_delete (ipa_fn_summaries);
0bceb671 4521 ipa_fn_summaries = NULL;
27d020cf
JH
4522 delete ipa_call_summaries;
4523 ipa_call_summaries = NULL;
4524 edge_predicate_pool.release ();
f658ad30
JH
4525 /* During IPA this is one of largest datastructures to release. */
4526 if (flag_wpa)
4527 ggc_trim ();
4528}
4529
4530/* Release function summary. */
4531
4532void
4533ipa_free_size_summary (void)
4534{
4535 if (!ipa_size_summaries)
4536 return;
78cd68c0 4537 delete ipa_size_summaries;
f658ad30 4538 ipa_size_summaries = NULL;
27d020cf 4539}
d2db2e6b
JH
4540
4541namespace {
4542
4543const pass_data pass_data_local_fn_summary =
4544{
4545 GIMPLE_PASS, /* type */
4546 "local-fnsummary", /* name */
4547 OPTGROUP_INLINE, /* optinfo_flags */
4548 TV_INLINE_PARAMETERS, /* tv_id */
4549 0, /* properties_required */
4550 0, /* properties_provided */
4551 0, /* properties_destroyed */
4552 0, /* todo_flags_start */
4553 0, /* todo_flags_finish */
4554};
4555
4556class pass_local_fn_summary : public gimple_opt_pass
4557{
4558public:
4559 pass_local_fn_summary (gcc::context *ctxt)
4560 : gimple_opt_pass (pass_data_local_fn_summary, ctxt)
4561 {}
4562
4563 /* opt_pass methods: */
4564 opt_pass * clone () { return new pass_local_fn_summary (m_ctxt); }
4565 virtual unsigned int execute (function *)
4566 {
4567 return compute_fn_summary_for_current ();
4568 }
4569
4570}; // class pass_local_fn_summary
4571
4572} // anon namespace
4573
4574gimple_opt_pass *
4575make_pass_local_fn_summary (gcc::context *ctxt)
4576{
4577 return new pass_local_fn_summary (ctxt);
4578}
4579
4580
4581/* Free inline summary. */
4582
4583namespace {
4584
4585const pass_data pass_data_ipa_free_fn_summary =
4586{
4587 SIMPLE_IPA_PASS, /* type */
4588 "free-fnsummary", /* name */
4589 OPTGROUP_NONE, /* optinfo_flags */
4590 TV_IPA_FREE_INLINE_SUMMARY, /* tv_id */
4591 0, /* properties_required */
4592 0, /* properties_provided */
4593 0, /* properties_destroyed */
4594 0, /* todo_flags_start */
442db276 4595 0, /* todo_flags_finish */
d2db2e6b
JH
4596};
4597
4598class pass_ipa_free_fn_summary : public simple_ipa_opt_pass
4599{
4600public:
4601 pass_ipa_free_fn_summary (gcc::context *ctxt)
442db276
JJ
4602 : simple_ipa_opt_pass (pass_data_ipa_free_fn_summary, ctxt),
4603 small_p (false)
d2db2e6b
JH
4604 {}
4605
4606 /* opt_pass methods: */
442db276
JJ
4607 opt_pass *clone () { return new pass_ipa_free_fn_summary (m_ctxt); }
4608 void set_pass_param (unsigned int n, bool param)
4609 {
4610 gcc_assert (n == 0);
4611 small_p = param;
4612 }
f658ad30 4613 virtual bool gate (function *) { return true; }
d2db2e6b
JH
4614 virtual unsigned int execute (function *)
4615 {
4616 ipa_free_fn_summary ();
f658ad30
JH
4617 if (!flag_wpa)
4618 ipa_free_size_summary ();
12485662 4619 return 0;
d2db2e6b
JH
4620 }
4621
442db276
JJ
4622private:
4623 bool small_p;
d2db2e6b
JH
4624}; // class pass_ipa_free_fn_summary
4625
4626} // anon namespace
4627
4628simple_ipa_opt_pass *
4629make_pass_ipa_free_fn_summary (gcc::context *ctxt)
4630{
4631 return new pass_ipa_free_fn_summary (ctxt);
4632}
4633
4634namespace {
4635
4636const pass_data pass_data_ipa_fn_summary =
4637{
4638 IPA_PASS, /* type */
4639 "fnsummary", /* name */
4640 OPTGROUP_INLINE, /* optinfo_flags */
66447ef0 4641 TV_IPA_FNSUMMARY, /* tv_id */
d2db2e6b
JH
4642 0, /* properties_required */
4643 0, /* properties_provided */
4644 0, /* properties_destroyed */
4645 0, /* todo_flags_start */
4646 ( TODO_dump_symtab ), /* todo_flags_finish */
4647};
4648
4649class pass_ipa_fn_summary : public ipa_opt_pass_d
4650{
4651public:
4652 pass_ipa_fn_summary (gcc::context *ctxt)
4653 : ipa_opt_pass_d (pass_data_ipa_fn_summary, ctxt,
4654 ipa_fn_summary_generate, /* generate_summary */
4655 ipa_fn_summary_write, /* write_summary */
4656 ipa_fn_summary_read, /* read_summary */
4657 NULL, /* write_optimization_summary */
4658 NULL, /* read_optimization_summary */
4659 NULL, /* stmt_fixup */
4660 0, /* function_transform_todo_flags_start */
4661 NULL, /* function_transform */
4662 NULL) /* variable_transform */
4663 {}
4664
4665 /* opt_pass methods: */
4666 virtual unsigned int execute (function *) { return 0; }
4667
4668}; // class pass_ipa_fn_summary
4669
4670} // anon namespace
4671
4672ipa_opt_pass_d *
4673make_pass_ipa_fn_summary (gcc::context *ctxt)
4674{
4675 return new pass_ipa_fn_summary (ctxt);
4676}
de4381a4
DM
4677
4678/* Reset all state within ipa-fnsummary.c so that we can rerun the compiler
4679 within the same process. For use by toplev::finalize. */
4680
4681void
4682ipa_fnsummary_c_finalize (void)
4683{
4684 ipa_free_fn_summary ();
4685}