1 /* Read and annotate call graph profile from the auto profile data file.
2 Copyright (C) 2014-2024 Free Software Foundation, Inc.
3 Contributed by Dehao Chen (dehao@google.com)
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
25 #include "coretypes.h"
30 #include "alloc-pool.h"
31 #include "tree-pass.h"
35 #include "diagnostic-core.h"
37 #include "langhooks.h"
40 #include "tree-cfgcleanup.h"
41 #include "tree-into-ssa.h"
42 #include "gimple-iterator.h"
43 #include "value-prof.h"
44 #include "symbol-summary.h"
48 #include "ipa-fnsummary.h"
49 #include "ipa-inline.h"
50 #include "tree-inline.h"
51 #include "auto-profile.h"
52 #include "tree-pretty-print.h"
53 #include "gimple-pretty-print.h"
55 /* The following routines implements AutoFDO optimization.
57 This optimization uses sampling profiles to annotate basic block counts
58 and uses heuristics to estimate branch probabilities.
60 There are three phases in AutoFDO:
62 Phase 1: Read profile from the profile data file.
63 The following info is read from the profile datafile:
64 * string_table: a map between function name and its index.
65 * autofdo_source_profile: a map from function_instance name to
66 function_instance. This is represented as a forest of
68 * WorkingSet: a histogram of how many instructions are covered for a
69 given percentage of total cycles. This is describing the binary
70 level information (not source level). This info is used to help
71 decide if we want aggressive optimizations that could increase
72 code footprint (e.g. loop unroll etc.)
73 A function instance is an instance of function that could either be a
74 standalone symbol, or a clone of a function that is inlined into another
77 Phase 2: Early inline + value profile transformation.
78 Early inline uses autofdo_source_profile to find if a callsite is:
79 * inlined in the profiled binary.
80 * callee body is hot in the profiling run.
81 If both condition satisfies, early inline will inline the callsite
82 regardless of the code growth.
83 Phase 2 is an iterative process. During each iteration, we also check
84 if an indirect callsite is promoted and inlined in the profiling run.
85 If yes, vpt will happen to force promote it and in the next iteration,
86 einline will inline the promoted callsite in the next iteration.
88 Phase 3: Annotate control flow graph.
89 AutoFDO uses a separate pass to:
90 * Annotate basic block count
91 * Estimate branch probability
93 After the above 3 phases, all profile is readily annotated on the GCC IR.
94 AutoFDO tries to reuse all FDO infrastructure as much as possible to make
95 use of the profile. E.g. it uses existing mechanism to calculate the basic
96 block/edge frequency, as well as the cgraph node/edge count.
99 #define DEFAULT_AUTO_PROFILE_FILE "fbdata.afdo"
100 #define AUTO_PROFILE_VERSION 2
105 /* Intermediate edge info used when propagating AutoFDO profile information.
106 We can't edge->count() directly since it's computed from edge's probability
107 while probability is yet not decided during propagation. */
108 #define AFDO_EINFO(e) ((class edge_info *) e->aux)
112 edge_info () : count_ (profile_count::zero ().afdo ()), annotated_ (false) {}
113 bool is_annotated () const { return annotated_
; }
114 void set_annotated () { annotated_
= true; }
115 profile_count
get_count () const { return count_
; }
116 void set_count (profile_count count
) { count_
= count
; }
118 profile_count count_
;
122 /* Represent a source location: (function_decl, lineno). */
123 typedef std::pair
<tree
, unsigned> decl_lineno
;
125 /* Represent an inline stack. vector[0] is the leaf node. */
126 typedef auto_vec
<decl_lineno
> inline_stack
;
128 /* String array that stores function names. */
129 typedef auto_vec
<char *> string_vector
;
131 /* Map from function name's index in string_table to target's
133 typedef std::map
<unsigned, gcov_type
> icall_target_map
;
135 /* Set of gimple stmts. Used to track if the stmt has already been promoted
137 typedef std::set
<gimple
*> stmt_set
;
139 /* Represent count info of an inline stack. */
143 /* Sampled count of the inline stack. */
146 /* Map from indirect call target to its sample count. */
147 icall_target_map targets
;
149 /* Whether this inline stack is already used in annotation.
151 Each inline stack should only be used to annotate IR once.
152 This will be enforced when instruction-level discriminator
157 /* operator< for "const char *". */
158 struct string_compare
160 bool operator()(const char *a
, const char *b
) const
162 return strcmp (a
, b
) < 0;
166 /* Store a string array, indexed by string position in the array. */
175 /* For a given string, returns its index. */
176 int get_index (const char *name
) const;
178 /* For a given decl, returns the index of the decl name. */
179 int get_index_by_decl (tree decl
) const;
181 /* For a given index, returns the string. */
182 const char *get_name (int index
) const;
184 /* Read profile, return TRUE on success. */
188 typedef std::map
<const char *, unsigned, string_compare
> string_index_map
;
189 string_vector vector_
;
190 string_index_map map_
;
193 /* Profile of a function instance:
194 1. total_count of the function.
195 2. head_count (entry basic block count) of the function (only valid when
196 function is a top-level function_instance, i.e. it is the original copy
197 instead of the inlined copy).
198 3. map from source location (decl_lineno) to profile (count_info).
199 4. map from callsite to callee function_instance. */
200 class function_instance
203 typedef auto_vec
<function_instance
*> function_instance_stack
;
205 /* Read the profile and return a function_instance with head count as
206 HEAD_COUNT. Recursively read callsites to create nested function_instances
207 too. STACK is used to track the recursive creation process. */
208 static function_instance
*
209 read_function_instance (function_instance_stack
*stack
,
210 gcov_type head_count
);
212 /* Recursively deallocate all callsites (nested function_instances). */
213 ~function_instance ();
232 /* Traverse callsites of the current function_instance to find one at the
233 location of LINENO and callee name represented in DECL. */
234 function_instance
*get_function_instance_by_decl (unsigned lineno
,
237 /* Store the profile info for LOC in INFO. Return TRUE if profile info
239 bool get_count_info (location_t loc
, count_info
*info
) const;
241 /* Read the inlined indirect call target profile for STMT and store it in
242 MAP, return the total count for all inlined indirect calls. */
243 gcov_type
find_icall_target_map (gcall
*stmt
, icall_target_map
*map
) const;
245 /* Sum of counts that is used during annotation. */
246 gcov_type
total_annotated_count () const;
248 /* Mark LOC as annotated. */
249 void mark_annotated (location_t loc
);
252 /* Callsite, represented as (decl_lineno, callee_function_name_index). */
253 typedef std::pair
<unsigned, unsigned> callsite
;
255 /* Map from callsite to callee function_instance. */
256 typedef std::map
<callsite
, function_instance
*> callsite_map
;
258 function_instance (unsigned name
, gcov_type head_count
)
259 : name_ (name
), total_count_ (0), head_count_ (head_count
)
263 /* Map from source location (decl_lineno) to profile (count_info). */
264 typedef std::map
<unsigned, count_info
> position_count_map
;
266 /* function_instance name index in the string_table. */
269 /* Total sample count. */
270 gcov_type total_count_
;
272 /* Entry BB's sample count. */
273 gcov_type head_count_
;
275 /* Map from callsite location to callee function_instance. */
276 callsite_map callsites
;
278 /* Map from source location to count_info. */
279 position_count_map pos_counts
;
282 /* Profile for all functions. */
283 class autofdo_source_profile
286 static autofdo_source_profile
*
289 autofdo_source_profile
*map
= new autofdo_source_profile ();
297 ~autofdo_source_profile ();
299 /* For a given DECL, returns the top-level function_instance. */
300 function_instance
*get_function_instance_by_decl (tree decl
) const;
302 /* Find count_info for a given gimple STMT. If found, store the count_info
303 in INFO and return true; otherwise return false. */
304 bool get_count_info (gimple
*stmt
, count_info
*info
) const;
306 /* Find total count of the callee of EDGE. */
307 gcov_type
get_callsite_total_count (struct cgraph_edge
*edge
) const;
309 /* Update value profile INFO for STMT from the inlined indirect callsite.
310 Return true if INFO is updated. */
311 bool update_inlined_ind_target (gcall
*stmt
, count_info
*info
);
313 /* Mark LOC as annotated. */
314 void mark_annotated (location_t loc
);
317 /* Map from function_instance name index (in string_table) to
318 function_instance. */
319 typedef std::map
<unsigned, function_instance
*> name_function_instance_map
;
321 autofdo_source_profile () {}
323 /* Read AutoFDO profile and returns TRUE on success. */
326 /* Return the function_instance in the profile that correspond to the
329 get_function_instance_by_inline_stack (const inline_stack
&stack
) const;
331 name_function_instance_map map_
;
334 /* Store the strings read from the profile data file. */
335 static string_table
*afdo_string_table
;
337 /* Store the AutoFDO source profile. */
338 static autofdo_source_profile
*afdo_source_profile
;
340 /* gcov_summary structure to store the profile_info. */
341 static gcov_summary
*afdo_profile_info
;
343 /* Helper functions. */
345 /* Return the original name of NAME: strip the suffix that starts
346 with '.' Caller is responsible for freeing RET. */
349 get_original_name (const char *name
)
351 char *ret
= xstrdup (name
);
352 char *find
= strchr (ret
, '.');
358 /* Return the combined location, which is a 32bit integer in which
359 higher 16 bits stores the line offset of LOC to the start lineno
360 of DECL, The lower 16 bits stores the discriminator. */
363 get_combined_location (location_t loc
, tree decl
)
365 /* TODO: allow more bits for line and less bits for discriminator. */
366 if (LOCATION_LINE (loc
) - DECL_SOURCE_LINE (decl
) >= (1<<16))
367 warning_at (loc
, OPT_Woverflow
, "offset exceeds 16 bytes");
368 return ((LOCATION_LINE (loc
) - DECL_SOURCE_LINE (decl
)) << 16)
369 | get_discriminator_from_loc (loc
);
372 /* Return the function decl of a given lexical BLOCK. */
375 get_function_decl_from_block (tree block
)
377 if (!inlined_function_outer_scope_p (block
))
380 return BLOCK_ABSTRACT_ORIGIN (block
);
383 /* Store inline stack for STMT in STACK. */
386 get_inline_stack (location_t locus
, inline_stack
*stack
)
388 if (LOCATION_LOCUS (locus
) == UNKNOWN_LOCATION
)
391 tree block
= LOCATION_BLOCK (locus
);
392 if (block
&& TREE_CODE (block
) == BLOCK
)
394 for (block
= BLOCK_SUPERCONTEXT (block
);
395 block
&& (TREE_CODE (block
) == BLOCK
);
396 block
= BLOCK_SUPERCONTEXT (block
))
398 location_t tmp_locus
= BLOCK_SOURCE_LOCATION (block
);
399 if (LOCATION_LOCUS (tmp_locus
) == UNKNOWN_LOCATION
)
402 tree decl
= get_function_decl_from_block (block
);
404 std::make_pair (decl
, get_combined_location (locus
, decl
)));
409 std::make_pair (current_function_decl
,
410 get_combined_location (locus
, current_function_decl
)));
413 /* Return STMT's combined location, which is a 32bit integer in which
414 higher 16 bits stores the line offset of LOC to the start lineno
415 of DECL, The lower 16 bits stores the discriminator. */
418 get_relative_location_for_stmt (gimple
*stmt
)
420 location_t locus
= gimple_location (stmt
);
421 if (LOCATION_LOCUS (locus
) == UNKNOWN_LOCATION
)
422 return UNKNOWN_LOCATION
;
424 for (tree block
= gimple_block (stmt
); block
&& (TREE_CODE (block
) == BLOCK
);
425 block
= BLOCK_SUPERCONTEXT (block
))
426 if (LOCATION_LOCUS (BLOCK_SOURCE_LOCATION (block
)) != UNKNOWN_LOCATION
)
427 return get_combined_location (locus
,
428 get_function_decl_from_block (block
));
429 return get_combined_location (locus
, current_function_decl
);
432 /* Return true if BB contains indirect call. */
435 has_indirect_call (basic_block bb
)
437 gimple_stmt_iterator gsi
;
439 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
441 gimple
*stmt
= gsi_stmt (gsi
);
442 if (gimple_code (stmt
) == GIMPLE_CALL
&& !gimple_call_internal_p (stmt
)
443 && (gimple_call_fn (stmt
) == NULL
444 || TREE_CODE (gimple_call_fn (stmt
)) != FUNCTION_DECL
))
450 /* Member functions for string_table. */
454 string_table::~string_table ()
456 for (unsigned i
= 0; i
< vector_
.length (); i
++)
461 /* Return the index of a given function NAME. Return -1 if NAME is not
462 found in string table. */
465 string_table::get_index (const char *name
) const
469 string_index_map::const_iterator iter
= map_
.find (name
);
470 if (iter
== map_
.end ())
476 /* Return the index of a given function DECL. Return -1 if DECL is not
477 found in string table. */
480 string_table::get_index_by_decl (tree decl
) const
483 = get_original_name (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl
)));
484 int ret
= get_index (name
);
488 ret
= get_index (lang_hooks
.dwarf_name (decl
, 0));
491 if (DECL_FROM_INLINE (decl
))
492 return get_index_by_decl (DECL_ABSTRACT_ORIGIN (decl
));
497 /* Return the function name of a given INDEX. */
500 string_table::get_name (int index
) const
502 gcc_assert (index
> 0 && index
< (int)vector_
.length ());
503 return vector_
[index
];
506 /* Read the string table. Return TRUE if reading is successful. */
509 string_table::read ()
511 if (gcov_read_unsigned () != GCOV_TAG_AFDO_FILE_NAMES
)
513 /* Skip the length of the section. */
514 gcov_read_unsigned ();
515 /* Read in the file name table. */
516 unsigned string_num
= gcov_read_unsigned ();
517 for (unsigned i
= 0; i
< string_num
; i
++)
519 vector_
.safe_push (get_original_name (gcov_read_string ()));
520 map_
[vector_
.last ()] = i
;
525 /* Member functions for function_instance. */
527 function_instance::~function_instance ()
529 for (callsite_map::iterator iter
= callsites
.begin ();
530 iter
!= callsites
.end (); ++iter
)
534 /* Traverse callsites of the current function_instance to find one at the
535 location of LINENO and callee name represented in DECL. */
538 function_instance::get_function_instance_by_decl (unsigned lineno
,
541 int func_name_idx
= afdo_string_table
->get_index_by_decl (decl
);
542 if (func_name_idx
!= -1)
544 callsite_map::const_iterator ret
545 = callsites
.find (std::make_pair (lineno
, func_name_idx
));
546 if (ret
!= callsites
.end ())
550 = afdo_string_table
->get_index (lang_hooks
.dwarf_name (decl
, 0));
551 if (func_name_idx
!= -1)
553 callsite_map::const_iterator ret
554 = callsites
.find (std::make_pair (lineno
, func_name_idx
));
555 if (ret
!= callsites
.end ())
558 if (DECL_FROM_INLINE (decl
))
559 return get_function_instance_by_decl (lineno
, DECL_ABSTRACT_ORIGIN (decl
));
564 /* Store the profile info for LOC in INFO. Return TRUE if profile info
568 function_instance::get_count_info (location_t loc
, count_info
*info
) const
570 position_count_map::const_iterator iter
= pos_counts
.find (loc
);
571 if (iter
== pos_counts
.end ())
573 *info
= iter
->second
;
577 /* Mark LOC as annotated. */
580 function_instance::mark_annotated (location_t loc
)
582 position_count_map::iterator iter
= pos_counts
.find (loc
);
583 if (iter
== pos_counts
.end ())
585 iter
->second
.annotated
= true;
588 /* Read the inlined indirect call target profile for STMT and store it in
589 MAP, return the total count for all inlined indirect calls. */
592 function_instance::find_icall_target_map (gcall
*stmt
,
593 icall_target_map
*map
) const
596 unsigned stmt_offset
= get_relative_location_for_stmt (stmt
);
598 for (callsite_map::const_iterator iter
= callsites
.begin ();
599 iter
!= callsites
.end (); ++iter
)
601 unsigned callee
= iter
->second
->name ();
602 /* Check if callsite location match the stmt. */
603 if (iter
->first
.first
!= stmt_offset
)
605 struct cgraph_node
*node
= cgraph_node::get_for_asmname (
606 get_identifier (afdo_string_table
->get_name (callee
)));
609 (*map
)[callee
] = iter
->second
->total_count ();
610 ret
+= iter
->second
->total_count ();
615 /* Read the profile and create a function_instance with head count as
616 HEAD_COUNT. Recursively read callsites to create nested function_instances
617 too. STACK is used to track the recursive creation process. */
619 /* function instance profile format:
623 NUM_POS_COUNTS: 4 bytes
624 NUM_CALLSITES: 4 byte
626 POS_1_OFFSET: 4 bytes
630 VALUE_PROFILE_TYPE: 4 bytes
640 CALLSITE_1_OFFSET: 4 bytes
641 FUNCTION_INSTANCE_PROFILE (nested)
647 function_instance::read_function_instance (function_instance_stack
*stack
,
648 gcov_type head_count
)
650 unsigned name
= gcov_read_unsigned ();
651 unsigned num_pos_counts
= gcov_read_unsigned ();
652 unsigned num_callsites
= gcov_read_unsigned ();
653 function_instance
*s
= new function_instance (name
, head_count
);
654 stack
->safe_push (s
);
656 for (unsigned i
= 0; i
< num_pos_counts
; i
++)
658 unsigned offset
= gcov_read_unsigned ();
659 unsigned num_targets
= gcov_read_unsigned ();
660 gcov_type count
= gcov_read_counter ();
661 s
->pos_counts
[offset
].count
= count
;
662 for (unsigned j
= 0; j
< stack
->length (); j
++)
663 (*stack
)[j
]->total_count_
+= count
;
664 for (unsigned j
= 0; j
< num_targets
; j
++)
666 /* Only indirect call target histogram is supported now. */
667 gcov_read_unsigned ();
668 gcov_type target_idx
= gcov_read_counter ();
669 s
->pos_counts
[offset
].targets
[target_idx
] = gcov_read_counter ();
672 for (unsigned i
= 0; i
< num_callsites
; i
++)
674 unsigned offset
= gcov_read_unsigned ();
675 function_instance
*callee_function_instance
676 = read_function_instance (stack
, 0);
677 s
->callsites
[std::make_pair (offset
, callee_function_instance
->name ())]
678 = callee_function_instance
;
684 /* Sum of counts that is used during annotation. */
687 function_instance::total_annotated_count () const
690 for (callsite_map::const_iterator iter
= callsites
.begin ();
691 iter
!= callsites
.end (); ++iter
)
692 ret
+= iter
->second
->total_annotated_count ();
693 for (position_count_map::const_iterator iter
= pos_counts
.begin ();
694 iter
!= pos_counts
.end (); ++iter
)
695 if (iter
->second
.annotated
)
696 ret
+= iter
->second
.count
;
700 /* Member functions for autofdo_source_profile. */
702 autofdo_source_profile::~autofdo_source_profile ()
704 for (name_function_instance_map::const_iterator iter
= map_
.begin ();
705 iter
!= map_
.end (); ++iter
)
709 /* For a given DECL, returns the top-level function_instance. */
712 autofdo_source_profile::get_function_instance_by_decl (tree decl
) const
714 int index
= afdo_string_table
->get_index_by_decl (decl
);
717 name_function_instance_map::const_iterator ret
= map_
.find (index
);
718 return ret
== map_
.end () ? NULL
: ret
->second
;
721 /* Find count_info for a given gimple STMT. If found, store the count_info
722 in INFO and return true; otherwise return false. */
725 autofdo_source_profile::get_count_info (gimple
*stmt
, count_info
*info
) const
727 if (LOCATION_LOCUS (gimple_location (stmt
)) == cfun
->function_end_locus
)
731 get_inline_stack (gimple_location (stmt
), &stack
);
732 if (stack
.length () == 0)
734 function_instance
*s
= get_function_instance_by_inline_stack (stack
);
737 return s
->get_count_info (stack
[0].second
, info
);
740 /* Mark LOC as annotated. */
743 autofdo_source_profile::mark_annotated (location_t loc
)
746 get_inline_stack (loc
, &stack
);
747 if (stack
.length () == 0)
749 function_instance
*s
= get_function_instance_by_inline_stack (stack
);
752 s
->mark_annotated (stack
[0].second
);
755 /* Update value profile INFO for STMT from the inlined indirect callsite.
756 Return true if INFO is updated. */
759 autofdo_source_profile::update_inlined_ind_target (gcall
*stmt
,
764 fprintf (dump_file
, "Checking indirect call -> direct call ");
765 print_gimple_stmt (dump_file
, stmt
, 0, TDF_SLIM
);
768 if (LOCATION_LOCUS (gimple_location (stmt
)) == cfun
->function_end_locus
)
771 fprintf (dump_file
, " good locus\n");
776 get_count_info (stmt
, &old_info
);
778 for (icall_target_map::const_iterator iter
= old_info
.targets
.begin ();
779 iter
!= old_info
.targets
.end (); ++iter
)
780 total
+= iter
->second
;
782 /* Program behavior changed, original promoted (and inlined) target is not
783 hot any more. Will avoid promote the original target.
785 To check if original promoted target is still hot, we check the total
786 count of the unpromoted targets (stored in TOTAL). If a callsite count
787 (stored in INFO) is smaller than half of the total count, the original
788 promoted target is considered not hot any more. */
789 if (info
->count
< total
/ 2)
792 fprintf (dump_file
, " not hot anymore %ld < %ld",
799 get_inline_stack (gimple_location (stmt
), &stack
);
800 if (stack
.length () == 0)
803 fprintf (dump_file
, " no inline stack\n");
806 function_instance
*s
= get_function_instance_by_inline_stack (stack
);
810 fprintf (dump_file
, " function not found in inline stack\n");
813 icall_target_map map
;
814 if (s
->find_icall_target_map (stmt
, &map
) == 0)
817 fprintf (dump_file
, " no target map\n");
820 for (icall_target_map::const_iterator iter
= map
.begin ();
821 iter
!= map
.end (); ++iter
)
822 info
->targets
[iter
->first
] = iter
->second
;
824 fprintf (dump_file
, " looks good\n");
828 /* Find total count of the callee of EDGE. */
831 autofdo_source_profile::get_callsite_total_count (
832 struct cgraph_edge
*edge
) const
835 stack
.safe_push (std::make_pair (edge
->callee
->decl
, 0));
836 get_inline_stack (gimple_location (edge
->call_stmt
), &stack
);
838 function_instance
*s
= get_function_instance_by_inline_stack (stack
);
840 || afdo_string_table
->get_index (IDENTIFIER_POINTER (
841 DECL_ASSEMBLER_NAME (edge
->callee
->decl
))) != s
->name ())
844 return s
->total_count ();
847 /* Read AutoFDO profile and returns TRUE on success. */
849 /* source profile format:
851 GCOV_TAG_AFDO_FUNCTION: 4 bytes
853 NUM_FUNCTIONS: 4 bytes
857 FUNCTION_INSTANCE_N. */
860 autofdo_source_profile::read ()
862 if (gcov_read_unsigned () != GCOV_TAG_AFDO_FUNCTION
)
864 inform (UNKNOWN_LOCATION
, "Not expected TAG.");
868 /* Skip the length of the section. */
869 gcov_read_unsigned ();
871 /* Read in the function/callsite profile, and store it in local
873 unsigned function_num
= gcov_read_unsigned ();
874 for (unsigned i
= 0; i
< function_num
; i
++)
876 function_instance::function_instance_stack stack
;
877 function_instance
*s
= function_instance::read_function_instance (
878 &stack
, gcov_read_counter ());
879 map_
[s
->name ()] = s
;
884 /* Return the function_instance in the profile that correspond to the
888 autofdo_source_profile::get_function_instance_by_inline_stack (
889 const inline_stack
&stack
) const
891 name_function_instance_map::const_iterator iter
= map_
.find (
892 afdo_string_table
->get_index_by_decl (stack
[stack
.length () - 1].first
));
893 if (iter
== map_
.end())
895 function_instance
*s
= iter
->second
;
896 for (unsigned i
= stack
.length() - 1; i
> 0; i
--)
898 s
= s
->get_function_instance_by_decl (
899 stack
[i
].second
, stack
[i
- 1].first
);
906 /* Module profile is only used by LIPO. Here we simply ignore it. */
909 fake_read_autofdo_module_profile ()
911 /* Read in the module info. */
912 gcov_read_unsigned ();
914 /* Skip the length of the section. */
915 gcov_read_unsigned ();
917 /* Read in the file name table. */
918 unsigned total_module_num
= gcov_read_unsigned ();
919 gcc_assert (total_module_num
== 0);
922 /* Read data from profile data file. */
927 if (gcov_open (auto_profile_file
, 1) == 0)
929 error ("cannot open profile file %s", auto_profile_file
);
933 if (gcov_read_unsigned () != GCOV_DATA_MAGIC
)
935 error ("AutoFDO profile magic number does not match");
939 /* Skip the version number. */
940 unsigned version
= gcov_read_unsigned ();
941 if (version
!= AUTO_PROFILE_VERSION
)
943 error ("AutoFDO profile version %u does not match %u",
944 version
, AUTO_PROFILE_VERSION
);
948 /* Skip the empty integer. */
949 gcov_read_unsigned ();
952 afdo_string_table
= new string_table ();
953 if (!afdo_string_table
->read())
955 error ("cannot read string table from %s", auto_profile_file
);
959 /* autofdo_source_profile. */
960 afdo_source_profile
= autofdo_source_profile::create ();
961 if (afdo_source_profile
== NULL
)
963 error ("cannot read function profile from %s", auto_profile_file
);
967 /* autofdo_module_profile. */
968 fake_read_autofdo_module_profile ();
971 /* From AutoFDO profiles, find values inside STMT for that we want to measure
972 histograms for indirect-call optimization.
974 This function is actually served for 2 purposes:
975 * before annotation, we need to mark histogram, promote and inline
976 * after annotation, we just need to mark, and let follow-up logic to
977 decide if it needs to promote and inline. */
980 afdo_indirect_call (gimple_stmt_iterator
*gsi
, const icall_target_map
&map
,
983 gimple
*gs
= gsi_stmt (*gsi
);
986 if (map
.size () == 0)
988 gcall
*stmt
= dyn_cast
<gcall
*> (gs
);
990 || gimple_call_internal_p (stmt
)
991 || gimple_call_fndecl (stmt
) != NULL_TREE
)
995 icall_target_map::const_iterator max_iter
= map
.end ();
997 for (icall_target_map::const_iterator iter
= map
.begin ();
998 iter
!= map
.end (); ++iter
)
1000 total
+= iter
->second
;
1001 if (max_iter
== map
.end () || max_iter
->second
< iter
->second
)
1004 struct cgraph_node
*direct_call
= cgraph_node::get_for_asmname (
1005 get_identifier (afdo_string_table
->get_name (max_iter
->first
)));
1006 if (direct_call
== NULL
|| !direct_call
->profile_id
)
1009 callee
= gimple_call_fn (stmt
);
1011 histogram_value hist
= gimple_alloc_histogram_value (
1012 cfun
, HIST_TYPE_INDIR_CALL
, stmt
, callee
);
1013 hist
->n_counters
= 4;
1014 hist
->hvalue
.counters
= XNEWVEC (gcov_type
, hist
->n_counters
);
1015 gimple_add_histogram_value (cfun
, stmt
, hist
);
1018 hist
->hvalue
.counters
[0] = total
;
1019 /* Number of value/counter pairs */
1020 hist
->hvalue
.counters
[1] = 1;
1022 hist
->hvalue
.counters
[2] = direct_call
->profile_id
;
1024 hist
->hvalue
.counters
[3] = max_iter
->second
;
1029 cgraph_node
* current_function_node
= cgraph_node::get (current_function_decl
);
1031 /* If the direct call is a recursive call, don't promote it since
1032 we are not set up to inline recursive calls at this stage. */
1033 if (direct_call
== current_function_node
)
1036 struct cgraph_edge
*indirect_edge
1037 = current_function_node
->get_edge (stmt
);
1041 fprintf (dump_file
, "Indirect call -> direct call ");
1042 print_generic_expr (dump_file
, callee
, TDF_SLIM
);
1043 fprintf (dump_file
, " => ");
1044 print_generic_expr (dump_file
, direct_call
->decl
, TDF_SLIM
);
1047 if (direct_call
== NULL
)
1050 fprintf (dump_file
, " not transforming\n");
1053 if (DECL_STRUCT_FUNCTION (direct_call
->decl
) == NULL
)
1056 fprintf (dump_file
, " no declaration\n");
1062 fprintf (dump_file
, " transformation on insn ");
1063 print_gimple_stmt (dump_file
, stmt
, 0, TDF_SLIM
);
1064 fprintf (dump_file
, "\n");
1067 /* FIXME: Count should be initialized. */
1068 struct cgraph_edge
*new_edge
1069 = indirect_edge
->make_speculative (direct_call
,
1070 profile_count::uninitialized ());
1071 cgraph_edge::redirect_call_stmt_to_callee (new_edge
);
1072 gimple_remove_histogram_value (cfun
, stmt
, hist
);
1073 inline_call (new_edge
, true, NULL
, NULL
, false);
1077 /* From AutoFDO profiles, find values inside STMT for that we want to measure
1078 histograms and adds them to list VALUES. */
1081 afdo_vpt (gimple_stmt_iterator
*gsi
, const icall_target_map
&map
,
1084 return afdo_indirect_call (gsi
, map
, transform
);
1087 typedef std::set
<basic_block
> bb_set
;
1088 typedef std::set
<edge
> edge_set
;
1091 is_bb_annotated (const basic_block bb
, const bb_set
&annotated
)
1093 return annotated
.find (bb
) != annotated
.end ();
1097 set_bb_annotated (basic_block bb
, bb_set
*annotated
)
1099 annotated
->insert (bb
);
1102 /* For a given BB, set its execution count. Attach value profile if a stmt
1103 is not in PROMOTED, because we only want to promote an indirect call once.
1104 Return TRUE if BB is annotated. */
1107 afdo_set_bb_count (basic_block bb
, const stmt_set
&promoted
)
1109 gimple_stmt_iterator gsi
;
1112 gcov_type max_count
= 0;
1113 bool has_annotated
= false;
1115 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1118 gimple
*stmt
= gsi_stmt (gsi
);
1119 if (gimple_clobber_p (stmt
) || is_gimple_debug (stmt
))
1121 if (afdo_source_profile
->get_count_info (stmt
, &info
))
1123 if (info
.count
> max_count
)
1124 max_count
= info
.count
;
1125 has_annotated
= true;
1126 if (info
.targets
.size () > 0
1127 && promoted
.find (stmt
) == promoted
.end ())
1128 afdo_vpt (&gsi
, info
.targets
, false);
1135 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1136 afdo_source_profile
->mark_annotated (gimple_location (gsi_stmt (gsi
)));
1137 for (gphi_iterator gpi
= gsi_start_phis (bb
);
1141 gphi
*phi
= gpi
.phi ();
1143 for (i
= 0; i
< gimple_phi_num_args (phi
); i
++)
1144 afdo_source_profile
->mark_annotated (gimple_phi_arg_location (phi
, i
));
1146 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1147 afdo_source_profile
->mark_annotated (e
->goto_locus
);
1149 bb
->count
= profile_count::from_gcov_type (max_count
).afdo ();
1153 /* BB1 and BB2 are in an equivalent class iff:
1154 1. BB1 dominates BB2.
1155 2. BB2 post-dominates BB1.
1156 3. BB1 and BB2 are in the same loop nest.
1157 This function finds the equivalent class for each basic block, and
1158 stores a pointer to the first BB in its equivalent class. Meanwhile,
1159 set bb counts for the same equivalent class to be idenical. Update
1160 ANNOTATED_BB for the first BB in its equivalent class. */
1163 afdo_find_equiv_class (bb_set
*annotated_bb
)
1167 FOR_ALL_BB_FN (bb
, cfun
)
1170 FOR_ALL_BB_FN (bb
, cfun
)
1172 if (bb
->aux
!= NULL
)
1175 for (basic_block bb1
: get_dominated_by (CDI_DOMINATORS
, bb
))
1176 if (bb1
->aux
== NULL
&& dominated_by_p (CDI_POST_DOMINATORS
, bb
, bb1
)
1177 && bb1
->loop_father
== bb
->loop_father
)
1180 if (bb1
->count
> bb
->count
&& is_bb_annotated (bb1
, *annotated_bb
))
1182 bb
->count
= bb1
->count
;
1183 set_bb_annotated (bb
, annotated_bb
);
1187 for (basic_block bb1
: get_dominated_by (CDI_POST_DOMINATORS
, bb
))
1188 if (bb1
->aux
== NULL
&& dominated_by_p (CDI_DOMINATORS
, bb
, bb1
)
1189 && bb1
->loop_father
== bb
->loop_father
)
1192 if (bb1
->count
> bb
->count
&& is_bb_annotated (bb1
, *annotated_bb
))
1194 bb
->count
= bb1
->count
;
1195 set_bb_annotated (bb
, annotated_bb
);
1201 /* If a basic block's count is known, and only one of its in/out edges' count
1202 is unknown, its count can be calculated. Meanwhile, if all of the in/out
1203 edges' counts are known, then the basic block's unknown count can also be
1204 calculated. Also, if a block has a single predecessor or successor, the block's
1205 count can be propagated to that predecessor or successor.
1206 IS_SUCC is true if out edges of a basic blocks are examined.
1207 Update ANNOTATED_BB accordingly.
1208 Return TRUE if any basic block/edge count is changed. */
1211 afdo_propagate_edge (bool is_succ
, bb_set
*annotated_bb
)
1214 bool changed
= false;
1216 FOR_EACH_BB_FN (bb
, cfun
)
1218 edge e
, unknown_edge
= NULL
;
1220 int num_unknown_edge
= 0;
1222 profile_count total_known_count
= profile_count::zero ().afdo ();
1224 FOR_EACH_EDGE (e
, ei
, is_succ
? bb
->succs
: bb
->preds
)
1226 gcc_assert (AFDO_EINFO (e
) != NULL
);
1227 if (! AFDO_EINFO (e
)->is_annotated ())
1228 num_unknown_edge
++, unknown_edge
= e
;
1230 total_known_count
+= AFDO_EINFO (e
)->get_count ();
1234 /* Be careful not to annotate block with no successor in special cases. */
1235 if (num_unknown_edge
== 0 && total_known_count
> bb
->count
)
1237 bb
->count
= total_known_count
;
1238 if (!is_bb_annotated (bb
, *annotated_bb
))
1239 set_bb_annotated (bb
, annotated_bb
);
1242 else if (num_unknown_edge
== 1 && is_bb_annotated (bb
, *annotated_bb
))
1244 if (bb
->count
> total_known_count
)
1246 profile_count new_count
= bb
->count
- total_known_count
;
1247 AFDO_EINFO(unknown_edge
)->set_count(new_count
);
1250 basic_block succ_or_pred_bb
= is_succ
? unknown_edge
->dest
: unknown_edge
->src
;
1251 if (new_count
> succ_or_pred_bb
->count
)
1253 succ_or_pred_bb
->count
= new_count
;
1254 if (!is_bb_annotated (succ_or_pred_bb
, *annotated_bb
))
1255 set_bb_annotated (succ_or_pred_bb
, annotated_bb
);
1260 AFDO_EINFO (unknown_edge
)->set_count (profile_count::zero().afdo ());
1261 AFDO_EINFO (unknown_edge
)->set_annotated ();
1268 /* Special propagation for circuit expressions. Because GCC translates
1269 control flow into data flow for circuit expressions. E.g.
1276 will be translated into:
1291 tmp = PHI (0 (BB1), 0 (BB.t1), 1 (BB.t2)
1297 In this case, we need to propagate through PHI to determine the edge
1298 count of BB1->BB.t1, BB.t1->BB.t2. */
1301 afdo_propagate_circuit (const bb_set
&annotated_bb
)
1304 FOR_ALL_BB_FN (bb
, cfun
)
1307 tree cmp_rhs
, cmp_lhs
;
1308 gimple
*cmp_stmt
= last_nondebug_stmt (bb
);
1312 if (!cmp_stmt
|| gimple_code (cmp_stmt
) != GIMPLE_COND
)
1314 cmp_rhs
= gimple_cond_rhs (cmp_stmt
);
1315 cmp_lhs
= gimple_cond_lhs (cmp_stmt
);
1316 if (!TREE_CONSTANT (cmp_rhs
)
1317 || !(integer_zerop (cmp_rhs
) || integer_onep (cmp_rhs
)))
1319 if (TREE_CODE (cmp_lhs
) != SSA_NAME
)
1321 if (!is_bb_annotated (bb
, annotated_bb
))
1323 def_stmt
= SSA_NAME_DEF_STMT (cmp_lhs
);
1324 while (def_stmt
&& gimple_code (def_stmt
) == GIMPLE_ASSIGN
1325 && gimple_assign_single_p (def_stmt
)
1326 && TREE_CODE (gimple_assign_rhs1 (def_stmt
)) == SSA_NAME
)
1327 def_stmt
= SSA_NAME_DEF_STMT (gimple_assign_rhs1 (def_stmt
));
1330 gphi
*phi_stmt
= dyn_cast
<gphi
*> (def_stmt
);
1333 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1335 unsigned i
, total
= 0;
1337 bool check_value_one
= (((integer_onep (cmp_rhs
))
1338 ^ (gimple_cond_code (cmp_stmt
) == EQ_EXPR
))
1339 ^ ((e
->flags
& EDGE_TRUE_VALUE
) != 0));
1340 if (! AFDO_EINFO (e
)->is_annotated ())
1342 for (i
= 0; i
< gimple_phi_num_args (phi_stmt
); i
++)
1344 tree val
= gimple_phi_arg_def (phi_stmt
, i
);
1345 edge ep
= gimple_phi_arg_edge (phi_stmt
, i
);
1347 if (!TREE_CONSTANT (val
)
1348 || !(integer_zerop (val
) || integer_onep (val
)))
1350 if (check_value_one
^ integer_onep (val
))
1354 if (! (AFDO_EINFO (e
)->get_count ()).nonzero_p ()
1355 && ! AFDO_EINFO (ep
)->is_annotated ())
1357 AFDO_EINFO (ep
)->set_count (profile_count::zero ().afdo ());
1358 AFDO_EINFO (ep
)->set_annotated ();
1361 if (total
== 1 && ! AFDO_EINFO (only_one
)->is_annotated ())
1363 AFDO_EINFO (only_one
)->set_count (AFDO_EINFO (e
)->get_count ());
1364 AFDO_EINFO (only_one
)->set_annotated ();
1370 /* Propagate the basic block count and edge count on the control flow
1371 graph. We do the propagation iteratively until stablize. */
1374 afdo_propagate (bb_set
*annotated_bb
)
1377 bool changed
= true;
1380 FOR_ALL_BB_FN (bb
, cfun
)
1382 bb
->count
= ((basic_block
)bb
->aux
)->count
;
1383 if (is_bb_annotated ((basic_block
)bb
->aux
, *annotated_bb
))
1384 set_bb_annotated (bb
, annotated_bb
);
1387 while (changed
&& i
++ < 10)
1391 if (afdo_propagate_edge (true, annotated_bb
))
1393 if (afdo_propagate_edge (false, annotated_bb
))
1395 afdo_propagate_circuit (*annotated_bb
);
1399 /* Propagate counts on control flow graph and calculate branch
1403 afdo_calculate_branch_prob (bb_set
*annotated_bb
)
1409 calculate_dominance_info (CDI_POST_DOMINATORS
);
1410 calculate_dominance_info (CDI_DOMINATORS
);
1411 loop_optimizer_init (0);
1413 FOR_ALL_BB_FN (bb
, cfun
)
1415 gcc_assert (bb
->aux
== NULL
);
1416 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1418 gcc_assert (e
->aux
== NULL
);
1419 e
->aux
= new edge_info ();
1423 afdo_find_equiv_class (annotated_bb
);
1424 afdo_propagate (annotated_bb
);
1426 FOR_EACH_BB_FN (bb
, cfun
)
1428 int num_unknown_succ
= 0;
1429 profile_count total_count
= profile_count::zero ().afdo ();
1431 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1433 gcc_assert (AFDO_EINFO (e
) != NULL
);
1434 if (! AFDO_EINFO (e
)->is_annotated ())
1437 total_count
+= AFDO_EINFO (e
)->get_count ();
1439 if (num_unknown_succ
== 0 && total_count
.nonzero_p())
1441 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1443 = AFDO_EINFO (e
)->get_count ().probability_in (total_count
);
1446 FOR_ALL_BB_FN (bb
, cfun
)
1449 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1450 if (AFDO_EINFO (e
) != NULL
)
1452 delete AFDO_EINFO (e
);
1457 loop_optimizer_finalize ();
1458 free_dominance_info (CDI_DOMINATORS
);
1459 free_dominance_info (CDI_POST_DOMINATORS
);
1462 /* Perform value profile transformation using AutoFDO profile. Add the
1463 promoted stmts to PROMOTED_STMTS. Return TRUE if there is any
1464 indirect call promoted. */
1467 afdo_vpt_for_early_inline (stmt_set
*promoted_stmts
)
1470 if (afdo_source_profile
->get_function_instance_by_decl (
1471 current_function_decl
) == NULL
)
1474 compute_fn_summary (cgraph_node::get (current_function_decl
), true);
1476 bool has_vpt
= false;
1477 FOR_EACH_BB_FN (bb
, cfun
)
1479 if (!has_indirect_call (bb
))
1481 gimple_stmt_iterator gsi
;
1483 gcov_type bb_count
= 0;
1484 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1487 gimple
*stmt
= gsi_stmt (gsi
);
1488 if (afdo_source_profile
->get_count_info (stmt
, &info
))
1489 bb_count
= MAX (bb_count
, info
.count
);
1492 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1494 gcall
*stmt
= dyn_cast
<gcall
*> (gsi_stmt (gsi
));
1495 /* IC_promotion and early_inline_2 is done in multiple iterations.
1496 No need to promoted the stmt if its in promoted_stmts (means
1497 it is already been promoted in the previous iterations). */
1498 if ((!stmt
) || gimple_call_fn (stmt
) == NULL
1499 || TREE_CODE (gimple_call_fn (stmt
)) == FUNCTION_DECL
1500 || promoted_stmts
->find (stmt
) != promoted_stmts
->end ())
1504 afdo_source_profile
->get_count_info (stmt
, &info
);
1505 info
.count
= bb_count
;
1506 if (afdo_source_profile
->update_inlined_ind_target (stmt
, &info
))
1508 /* Promote the indirect call and update the promoted_stmts. */
1509 promoted_stmts
->insert (stmt
);
1510 if (afdo_vpt (&gsi
, info
.targets
, true))
1518 unsigned todo
= optimize_inline_calls (current_function_decl
);
1519 if (todo
& TODO_update_ssa_any
)
1520 update_ssa (TODO_update_ssa
);
1527 /* Annotate auto profile to the control flow graph. Do not annotate value
1528 profile for stmts in PROMOTED_STMTS. */
1531 afdo_annotate_cfg (const stmt_set
&promoted_stmts
)
1534 bb_set annotated_bb
;
1535 const function_instance
*s
1536 = afdo_source_profile
->get_function_instance_by_decl (
1537 current_function_decl
);
1541 cgraph_node::get (current_function_decl
)->count
1542 = profile_count::from_gcov_type (s
->head_count ()).afdo ();
1543 ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
1544 = profile_count::from_gcov_type (s
->head_count ()).afdo ();
1545 EXIT_BLOCK_PTR_FOR_FN (cfun
)->count
= profile_count::zero ().afdo ();
1546 profile_count max_count
= ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
;
1548 FOR_EACH_BB_FN (bb
, cfun
)
1550 /* As autoFDO uses sampling approach, we have to assume that all
1551 counters are zero when not seen by autoFDO. */
1552 bb
->count
= profile_count::zero ().afdo ();
1553 if (afdo_set_bb_count (bb
, promoted_stmts
))
1554 set_bb_annotated (bb
, &annotated_bb
);
1555 if (bb
->count
> max_count
)
1556 max_count
= bb
->count
;
1558 if (ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
1559 > ENTRY_BLOCK_PTR_FOR_FN (cfun
)->next_bb
->count
)
1561 ENTRY_BLOCK_PTR_FOR_FN (cfun
)->next_bb
->count
1562 = ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
;
1563 set_bb_annotated (ENTRY_BLOCK_PTR_FOR_FN (cfun
)->next_bb
, &annotated_bb
);
1565 if (ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
1566 > EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
->count
)
1568 EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
->count
1569 = ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
;
1570 set_bb_annotated (EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
, &annotated_bb
);
1572 afdo_source_profile
->mark_annotated (
1573 DECL_SOURCE_LOCATION (current_function_decl
));
1574 afdo_source_profile
->mark_annotated (cfun
->function_start_locus
);
1575 afdo_source_profile
->mark_annotated (cfun
->function_end_locus
);
1576 if (max_count
.nonzero_p())
1578 /* Calculate, propagate count and probability information on CFG. */
1579 afdo_calculate_branch_prob (&annotated_bb
);
1581 update_max_bb_count ();
1582 profile_status_for_fn (cfun
) = PROFILE_READ
;
1583 if (flag_value_profile_transformations
)
1585 gimple_value_profile_transformations ();
1586 free_dominance_info (CDI_DOMINATORS
);
1587 free_dominance_info (CDI_POST_DOMINATORS
);
1588 update_ssa (TODO_update_ssa
);
1592 /* Wrapper function to invoke early inliner. */
1597 compute_fn_summary (cgraph_node::get (current_function_decl
), true);
1598 unsigned int todo
= early_inliner (cfun
);
1599 if (todo
& TODO_update_ssa_any
)
1600 update_ssa (TODO_update_ssa
);
1604 /* Use AutoFDO profile to annoate the control flow graph.
1605 Return the todo flag. */
1610 struct cgraph_node
*node
;
1612 if (symtab
->state
== FINISHED
)
1615 init_node_map (true);
1616 profile_info
= autofdo::afdo_profile_info
;
1618 FOR_EACH_FUNCTION (node
)
1620 if (!gimple_has_body_p (node
->decl
))
1623 /* Don't profile functions produced for builtin stuff. */
1624 if (DECL_SOURCE_LOCATION (node
->decl
) == BUILTINS_LOCATION
)
1627 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
1629 /* First do indirect call promotion and early inline to make the
1630 IR match the profiled binary before actual annotation.
1632 This is needed because an indirect call might have been promoted
1633 and inlined in the profiled binary. If we do not promote and
1634 inline these indirect calls before annotation, the profile for
1635 these promoted functions will be lost.
1637 e.g. foo() --indirect_call--> bar()
1638 In profiled binary, the callsite is promoted and inlined, making
1639 the profile look like:
1649 Before AutoFDO pass, loc_foo_2 is not promoted thus not inlined.
1650 If we perform annotation on it, the profile inside bar@loc_foo2
1653 To avoid this, we promote loc_foo_2 and inline the promoted bar
1654 function before annotation, so the profile inside bar@loc_foo2
1656 autofdo::stmt_set promoted_stmts
;
1657 unsigned int todo
= 0;
1658 for (int i
= 0; i
< 10; i
++)
1660 if (!flag_value_profile_transformations
1661 || !autofdo::afdo_vpt_for_early_inline (&promoted_stmts
))
1663 todo
|= early_inline ();
1666 todo
|= early_inline ();
1667 autofdo::afdo_annotate_cfg (promoted_stmts
);
1668 compute_function_frequency ();
1670 /* Local pure-const may imply need to fixup the cfg. */
1671 todo
|= execute_fixup_cfg ();
1672 if (todo
& TODO_cleanup_cfg
)
1673 cleanup_tree_cfg ();
1675 free_dominance_info (CDI_DOMINATORS
);
1676 free_dominance_info (CDI_POST_DOMINATORS
);
1677 cgraph_edge::rebuild_edges ();
1678 compute_fn_summary (cgraph_node::get (current_function_decl
), true);
1684 } /* namespace autofdo. */
1686 /* Read the profile from the profile data file. */
1689 read_autofdo_file (void)
1691 if (auto_profile_file
== NULL
)
1692 auto_profile_file
= DEFAULT_AUTO_PROFILE_FILE
;
1694 autofdo::afdo_profile_info
= XNEW (gcov_summary
);
1695 autofdo::afdo_profile_info
->runs
= 1;
1696 autofdo::afdo_profile_info
->sum_max
= 0;
1698 /* Read the profile from the profile file. */
1699 autofdo::read_profile ();
1702 /* Free the resources. */
1705 end_auto_profile (void)
1707 delete autofdo::afdo_source_profile
;
1708 delete autofdo::afdo_string_table
;
1709 profile_info
= NULL
;
1712 /* Returns TRUE if EDGE is hot enough to be inlined early. */
1715 afdo_callsite_hot_enough_for_early_inline (struct cgraph_edge
*edge
)
1718 = autofdo::afdo_source_profile
->get_callsite_total_count (edge
);
1723 profile_count pcount
= profile_count::from_gcov_type (count
).afdo ();
1724 gcov_summary
*saved_profile_info
= profile_info
;
1725 /* At early inline stage, profile_info is not set yet. We need to
1726 temporarily set it to afdo_profile_info to calculate hotness. */
1727 profile_info
= autofdo::afdo_profile_info
;
1728 is_hot
= maybe_hot_count_p (NULL
, pcount
);
1729 profile_info
= saved_profile_info
;
1739 const pass_data pass_data_ipa_auto_profile
= {
1740 SIMPLE_IPA_PASS
, "afdo", /* name */
1741 OPTGROUP_NONE
, /* optinfo_flags */
1742 TV_IPA_AUTOFDO
, /* tv_id */
1743 0, /* properties_required */
1744 0, /* properties_provided */
1745 0, /* properties_destroyed */
1746 0, /* todo_flags_start */
1747 0, /* todo_flags_finish */
1750 class pass_ipa_auto_profile
: public simple_ipa_opt_pass
1753 pass_ipa_auto_profile (gcc::context
*ctxt
)
1754 : simple_ipa_opt_pass (pass_data_ipa_auto_profile
, ctxt
)
1758 /* opt_pass methods: */
1760 gate (function
*) final override
1762 return flag_auto_profile
;
1765 execute (function
*) final override
1767 return autofdo::auto_profile ();
1769 }; // class pass_ipa_auto_profile
1773 simple_ipa_opt_pass
*
1774 make_pass_ipa_auto_profile (gcc::context
*ctxt
)
1776 return new pass_ipa_auto_profile (ctxt
);