cgraphbuild.c (compute_call_stmt_bb_frequency): Use proper ENTRY_BLOCK_PTR.

	* cgraphbuild.c (compute_call_stmt_bb_frequency): Use proper ENTRY_BLOCK_PTR.
	* cgraph.c (cgraph_clone_edge): Avoid freq_scale 0 to completely zero out all
	callees.
	* cgraphunit.c (verify_cgraph_node): Verify cgraph nodes for frequency and count match.
	* ipa-inline.c (update_noncloned_frequencies): New function.
	(cgraph_clone_inlined_nodes): Use it.
	* tree-inline.c (copy_bb): Fix frequency scaling; output
	diagnostic on frequency mismatches to dump file.
	(initialize_cfun): Do not scale frequency; fix count scaling;
	initialize entry and exit block frequencies; copy profile
	info.
	(copy_cfg_body): Use frequency_scale as argument;
	fix count scaling.
	(copy_body): Use frequency_scale as argument.
	(expand_call_inline): Compute frequency scale and output diagnostic
	to dump file.
	(delete_unreachable_blocks_update_callgrah): Remove checking that
	has to be done after edge redirection.
	(tree_function_versioning): Update initialize_cfun and copy_body call.

From-SVN: r154205

diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index 9796fa09ce9c18f0da8416c77b2a4c2019523525..bee2ebc42d79748348bab85d94e633da39f13e76 100644 (file)
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -1,3 +1,25 @@
+2009-11-14  Jan Hubicka  <jh@suse.cz>
+
+       * cgraphbuild.c (compute_call_stmt_bb_frequency): Use proper ENTRY_BLOCK_PTR.
+       * cgraph.c (cgraph_clone_edge): Avoid freq_scale 0 to completely zero out all
+       callees.
+       * cgraphunit.c (verify_cgraph_node): Verify cgraph nodes for frequency and count match.
+       * ipa-inline.c (update_noncloned_frequencies): New function.
+       (cgraph_clone_inlined_nodes): Use it.
+       * tree-inline.c (copy_bb): Fix frequency scaling; output
+       diagnostic on frequency mismatches to dump file.
+       (initialize_cfun): Do not scale frequency; fix count scaling;
+       initialize entry and exit block frequencies; copy profile
+       info.
+       (copy_cfg_body): Use frequency_scale as argument;
+       fix count scaling.
+       (copy_body): Use frequency_scale as argument.
+       (expand_call_inline): Compute frequency scale and output diagnostic
+       to dump file.
+       (delete_unreachable_blocks_update_callgrah): Remove checking that
+       has to be done after edge redirection.
+       (tree_function_versioning): Update initialize_cfun and copy_body call.
+
 2009-11-14  Jan Hubicka  <jh@suse.cz>
 
        * cgraph.c (cgraph_release_function_body): Update use of

diff --git a/gcc/cgraph.c b/gcc/cgraph.c
index 840cf297bb5f167fb1f12a3c89a5451404dac160..f2d76091799cc4c1e8406f6206a9cf805825d1fc 100644 (file)
--- a/gcc/cgraph.c
+++ b/gcc/cgraph.c
@@ -1641,8 +1641,12 @@ cgraph_clone_edge (struct cgraph_edge *e, struct cgraph_node *n,
 {
   struct cgraph_edge *new_edge;
   gcov_type count = e->count * count_scale / REG_BR_PROB_BASE;
-  gcov_type freq = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE;
+  gcov_type freq;
 
+  /* We do not want to ignore loop nest after frequency drops to 0.  */
+  if (!freq_scale)
+    freq_scale = 1;
+  freq = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE;
   if (freq > CGRAPH_FREQ_MAX)
     freq = CGRAPH_FREQ_MAX;
   new_edge = cgraph_create_edge (n, e->callee, call_stmt, count, freq,

diff --git a/gcc/cgraphbuild.c b/gcc/cgraphbuild.c
index 8423c36b880a5624f3f0adcd7527dbbbb4c0d3b9..0c3bff2f90e875b2a4ee4a65ac64243099878088 100644 (file)
--- a/gcc/cgraphbuild.c
+++ b/gcc/cgraphbuild.c
@@ -109,7 +109,8 @@ reset_inline_failed (struct cgraph_node *node)
 int
 compute_call_stmt_bb_frequency (tree decl, basic_block bb)
 {
-  int entry_freq = ENTRY_BLOCK_PTR->frequency;
+  int entry_freq = ENTRY_BLOCK_PTR_FOR_FUNCTION
+                    (DECL_STRUCT_FUNCTION (decl))->frequency;
   int freq = bb->frequency;
 
   if (profile_status_for_function (DECL_STRUCT_FUNCTION (decl)) == PROFILE_ABSENT)

diff --git a/gcc/ipa-inline.c b/gcc/ipa-inline.c
index 08088dd3e7915a1a2bc2dfa51d13f19dcb99e2f3..e9d831153fe44e1c46d36e1520dc956fd43aa3f0 100644 (file)
--- a/gcc/ipa-inline.c
+++ b/gcc/ipa-inline.c
@@ -207,6 +207,29 @@ cgraph_estimate_size_after_inlining (int times, struct cgraph_node *to,
   return size;
 }
 
+/* Scale frequency of NODE edges by FREQ_SCALE and increase loop nest
+   by NEST.  */
+
+static void
+update_noncloned_frequencies (struct cgraph_node *node,
+                             int freq_scale, int nest)
+{
+  struct cgraph_edge *e;
+
+  /* We do not want to ignore high loop nest after freq drops to 0.  */
+  if (!freq_scale)
+    freq_scale = 1;
+  for (e = node->callees; e; e = e->next_callee)
+    {
+      e->loop_nest += nest;
+      e->frequency = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE;
+      if (e->frequency > CGRAPH_FREQ_MAX)
+        e->frequency = CGRAPH_FREQ_MAX;
+      if (!e->inline_failed)
+        update_noncloned_frequencies (e->callee, freq_scale, nest);
+    }
+}
+
 /* E is expected to be an edge being inlined.  Clone destination node of
    the edge and redirect it to the new clone.
    DUPLICATE is used for bookkeeping on whether we are actually creating new
@@ -234,6 +257,7 @@ cgraph_clone_inlined_nodes (struct cgraph_edge *e, bool duplicate,
            }
          duplicate = false;
          e->callee->local.externally_visible = false;
+          update_noncloned_frequencies (e->callee, e->frequency, e->loop_nest);
        }
       else
        {

diff --git a/gcc/tree-inline.c b/gcc/tree-inline.c
index 629ccfb524fb5b5c36b192c788af4582a067bb94..34a8e9d8d0743cbc19e701505ded3e17f65dfe29 100644 (file)
--- a/gcc/tree-inline.c
+++ b/gcc/tree-inline.c
@@ -1472,6 +1472,7 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale,
   gimple_stmt_iterator gsi, copy_gsi, seq_gsi;
   basic_block copy_basic_block;
   tree decl;
+  gcov_type freq;
 
   /* create_basic_block() will append every new block to
      basic_block_info automatically.  */
@@ -1481,11 +1482,12 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale,
 
   /* We are going to rebuild frequencies from scratch.  These values
      have just small importance to drive canonicalize_loop_headers.  */
-  copy_basic_block->frequency = ((gcov_type)bb->frequency
-                                * frequency_scale / REG_BR_PROB_BASE);
+  freq = ((gcov_type)bb->frequency * frequency_scale / REG_BR_PROB_BASE);
 
-  if (copy_basic_block->frequency > BB_FREQ_MAX)
-    copy_basic_block->frequency = BB_FREQ_MAX;
+  /* We recompute frequencies after inlining, so this is quite safe.  */
+  if (freq > BB_FREQ_MAX)
+    freq = BB_FREQ_MAX;
+  copy_basic_block->frequency = freq;
 
   copy_gsi = gsi_start_bb (copy_basic_block);
 
@@ -1631,10 +1633,34 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale,
                case CB_CGE_DUPLICATE:
                  edge = cgraph_edge (id->src_node, orig_stmt);
                  if (edge)
-                   edge = cgraph_clone_edge (edge, id->dst_node, stmt,
-                                             gimple_uid (stmt),
-                                             REG_BR_PROB_BASE, 1,
-                                             edge->frequency, true);
+                   {
+                     int edge_freq = edge->frequency;
+                     edge = cgraph_clone_edge (edge, id->dst_node, stmt,
+                                               gimple_uid (stmt),
+                                               REG_BR_PROB_BASE, CGRAPH_FREQ_BASE,
+                                               edge->frequency, true);
+                     /* We could also just rescale the frequency, but
+                        doing so would introduce roundoff errors and make
+                        verifier unhappy.  */
+                     edge->frequency 
+                       = compute_call_stmt_bb_frequency (id->dst_node->decl,
+                                                         copy_basic_block);
+                     if (dump_file
+                         && profile_status_for_function (cfun) != PROFILE_ABSENT
+                         && (edge_freq > edge->frequency + 10
+                             || edge_freq < edge->frequency - 10))
+                       {
+                         fprintf (dump_file, "Edge frequency estimated by "
+                                  "cgraph %i diverge from inliner's estimate %i\n",
+                                  edge_freq,
+                                  edge->frequency);
+                         fprintf (dump_file,
+                                  "Orig bb: %i, orig bb freq %i, new bb freq %i\n",
+                                  bb->index,
+                                  bb->frequency,
+                                  copy_basic_block->frequency);
+                       }
+                   }
                  break;
 
                case CB_CGE_MOVE_CLONES:
@@ -1674,7 +1700,8 @@ copy_bb (copy_body_data *id, basic_block bb, int frequency_scale,
                  if (id->transform_call_graph_edges == CB_CGE_MOVE_CLONES)
                    cgraph_create_edge_including_clones
                      (id->dst_node, dest, stmt, bb->count,
-                      compute_call_stmt_bb_frequency (id->dst_node->decl, bb),
+                      compute_call_stmt_bb_frequency (id->dst_node->decl, 
+                                                      copy_basic_block),
                       bb->loop_depth, CIF_ORIGINALLY_INDIRECT_CALL);
                  else
                    cgraph_create_edge (id->dst_node, dest, stmt,
@@ -1948,24 +1975,16 @@ remap_decl_1 (tree decl, void *data)
    NEW_FNDECL to be build.  CALLEE_FNDECL is the original */
 
 static void
-initialize_cfun (tree new_fndecl, tree callee_fndecl, gcov_type count,
-                int frequency)
+initialize_cfun (tree new_fndecl, tree callee_fndecl, gcov_type count)
 {
   struct function *src_cfun = DECL_STRUCT_FUNCTION (callee_fndecl);
-  gcov_type count_scale, frequency_scale;
+  gcov_type count_scale;
 
   if (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count)
     count_scale = (REG_BR_PROB_BASE * count
                   / ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count);
   else
-    count_scale = 1;
-
-  if (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency)
-    frequency_scale = (REG_BR_PROB_BASE * frequency
-                      /
-                      ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency);
-  else
-    frequency_scale = count_scale;
+    count_scale = REG_BR_PROB_BASE;
 
   /* Register specific tree functions.  */
   gimple_register_cfg_hooks ();
@@ -1998,18 +2017,17 @@ initialize_cfun (tree new_fndecl, tree callee_fndecl, gcov_type count,
 
   init_empty_tree_cfg ();
 
+  profile_status_for_function (cfun) = profile_status_for_function (src_cfun);
   ENTRY_BLOCK_PTR->count =
     (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count * count_scale /
      REG_BR_PROB_BASE);
-  ENTRY_BLOCK_PTR->frequency =
-    (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency *
-     frequency_scale / REG_BR_PROB_BASE);
+  ENTRY_BLOCK_PTR->frequency
+    = ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency;
   EXIT_BLOCK_PTR->count =
     (EXIT_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count * count_scale /
      REG_BR_PROB_BASE);
   EXIT_BLOCK_PTR->frequency =
-    (EXIT_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency *
-     frequency_scale / REG_BR_PROB_BASE);
+    EXIT_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency;
   if (src_cfun->eh)
     init_eh_for_function ();
 
@@ -2026,7 +2044,7 @@ initialize_cfun (tree new_fndecl, tree callee_fndecl, gcov_type count,
    another function.  Walks FN via CFG, returns new fndecl.  */
 
 static tree
-copy_cfg_body (copy_body_data * id, gcov_type count, int frequency,
+copy_cfg_body (copy_body_data * id, gcov_type count, int frequency_scale,
               basic_block entry_block_map, basic_block exit_block_map)
 {
   tree callee_fndecl = id->src_fn;
@@ -2035,21 +2053,14 @@ copy_cfg_body (copy_body_data * id, gcov_type count, int frequency,
   struct function *cfun_to_copy;
   basic_block bb;
   tree new_fndecl = NULL;
-  gcov_type count_scale, frequency_scale;
+  gcov_type count_scale;
   int last;
 
   if (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count)
     count_scale = (REG_BR_PROB_BASE * count
                   / ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->count);
   else
-    count_scale = 1;
-
-  if (ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency)
-    frequency_scale = (REG_BR_PROB_BASE * frequency
-                      /
-                      ENTRY_BLOCK_PTR_FOR_FUNCTION (src_cfun)->frequency);
-  else
-    frequency_scale = count_scale;
+    count_scale = REG_BR_PROB_BASE;
 
   /* Register specific tree functions.  */
   gimple_register_cfg_hooks ();
@@ -2204,7 +2215,7 @@ copy_tree_body (copy_body_data *id)
    another function.  */
 
 static tree
-copy_body (copy_body_data *id, gcov_type count, int frequency,
+copy_body (copy_body_data *id, gcov_type count, int frequency_scale,
           basic_block entry_block_map, basic_block exit_block_map)
 {
   tree fndecl = id->src_fn;
@@ -2212,7 +2223,7 @@ copy_body (copy_body_data *id, gcov_type count, int frequency,
 
   /* If this body has a CFG, walk CFG and copy.  */
   gcc_assert (ENTRY_BLOCK_PTR_FOR_FUNCTION (DECL_STRUCT_FUNCTION (fndecl)));
-  body = copy_cfg_body (id, count, frequency, entry_block_map, exit_block_map);
+  body = copy_cfg_body (id, count, frequency_scale, entry_block_map, exit_block_map);
   copy_debug_stmts (id);
 
   return body;
@@ -3732,12 +3743,23 @@ expand_call_inline (basic_block bb, gimple stmt, copy_body_data *id)
                                       cfun->local_decls);
     }
 
+  if (dump_file && (dump_flags & TDF_DETAILS))
+    {
+      fprintf (dump_file, "Inlining ");
+      print_generic_expr (dump_file, id->src_fn, 0); 
+      fprintf (dump_file, " to ");
+      print_generic_expr (dump_file, id->dst_fn, 0); 
+      fprintf (dump_file, " with frequency %i\n", cg_edge->frequency);
+    }
+
   /* This is it.  Duplicate the callee body.  Assume callee is
      pre-gimplified.  Note that we must not alter the caller
      function in any way before this point, as this CALL_EXPR may be
      a self-referential call; if we're calling ourselves, we need to
      duplicate our body before altering anything.  */
-  copy_body (id, bb->count, bb->frequency, bb, return_block);
+  copy_body (id, bb->count,
+            cg_edge->frequency * REG_BR_PROB_BASE / CGRAPH_FREQ_BASE,
+            bb, return_block);
 
   /* Reset the escaped and callused solutions.  */
   if (cfun->gimple_df)
@@ -4732,30 +4754,6 @@ delete_unreachable_blocks_update_callgraph (copy_body_data *id)
 
   if (changed)
     tidy_fallthru_edges ();
-#ifdef ENABLE_CHECKING0
-  verify_cgraph_node (id->dst_node);
-  if (id->transform_call_graph_edges == CB_CGE_MOVE_CLONES
-      && id->dst_node->clones)
-    {
-      struct cgraph_node *node;
-      for (node = id->dst_node->clones; node != id->dst_node;)
-       {
-         verify_cgraph_node (node);
-          
-         if (node->clones)
-           node = node->clones;
-         else if (node->next_sibling_clone)
-           node = node->next_sibling_clone;
-         else
-           {
-             while (node != id->dst_node && !node->next_sibling_clone)
-               node = node->clone_of;
-             if (node != id->dst_node)
-               node = node->next_sibling_clone;
-           }
-       }
-     }
-#endif
   return changed;
 }
 
@@ -4876,8 +4874,7 @@ tree_function_versioning (tree old_decl, tree new_decl,
   old_entry_block = ENTRY_BLOCK_PTR_FOR_FUNCTION
     (DECL_STRUCT_FUNCTION (old_decl));
   initialize_cfun (new_decl, old_decl,
-                  old_entry_block->count,
-                  old_entry_block->frequency);
+                  old_entry_block->count);
   push_cfun (DECL_STRUCT_FUNCTION (new_decl));
   
   /* Copy the function's static chain.  */
@@ -4947,7 +4944,7 @@ tree_function_versioning (tree old_decl, tree new_decl,
       }
   
   /* Copy the Function's body.  */
-  copy_body (&id, old_entry_block->count, old_entry_block->frequency,
+  copy_body (&id, old_entry_block->count, REG_BR_PROB_BASE,
             ENTRY_BLOCK_PTR, EXIT_BLOCK_PTR);
   
   if (DECL_RESULT (old_decl) != NULL_TREE)