Introduce -fprofile-reproducibility and support it with TOP N.

	PR ipa/92924
	* common.opt: Add -fprofile-reproducibility.
	* doc/invoke.texi: Document it.
	* value-prof.c (dump_histogram_value):
	Document and support behavior for counters[0]
	being a negative value.
	(get_nth_most_common_value): Handle negative
	counters[0] in respect to flag_profile_reproducible.
	PR ipa/92924
	* libgcov-merge.c (merge_topn_values_set): Record
	when a TOP N counter becomes invalid.  When merging
	remove a smallest value if the space is needed.

diff --git a/gcc/ChangeLog b/gcc/ChangeLog
index efc21166d6a1fb1e0c54e6354ebb41e41fda5a4d..db6740a500662488666bd942c3bb1338590fa22b 100644 (file)
--- a/gcc/ChangeLog
+++ b/gcc/ChangeLog
@@ -1,3 +1,14 @@
+2020-02-18  Martin Liska  <mliska@suse.cz>
+
+       PR ipa/92924
+       * common.opt: Add -fprofile-reproducibility.
+       * doc/invoke.texi: Document it.
+       * value-prof.c (dump_histogram_value):
+       Document and support behavior for counters[0]
+       being a negative value.
+       (get_nth_most_common_value): Handle negative
+       counters[0] in respect to flag_profile_reproducible.
+
 2020-02-18  Jakub Jelinek  <jakub@redhat.com>
 
        PR ipa/93797

diff --git a/gcc/common.opt b/gcc/common.opt
index 5692cd0437415079ea5d065e4c99a9c5d9facadc..fa9da505fc2766794e731312ef6394f75a940d82 100644 (file)
--- a/gcc/common.opt
+++ b/gcc/common.opt
@@ -2168,6 +2168,22 @@ fprofile-exclude-files=
 Common Joined RejectNegative Var(flag_profile_exclude_files)
 Instrument only functions from files where names do not match all the regular expressions (separated by a semi-colon).
 
+Enum
+Name(profile_reproducibility) Type(enum profile_reproducibility) UnknownError(unknown profile reproducibility method %qs)
+
+EnumValue
+Enum(profile_reproducibility) String(serial) Value(PROFILE_REPRODUCIBILITY_SERIAL)
+
+EnumValue
+Enum(profile_reproducibility) String(parallel-runs) Value(PROFILE_REPRODUCIBILITY_PARALLEL_RUNS)
+
+EnumValue
+Enum(profile_reproducibility) String(multithreaded) Value(PROFILE_REPRODUCIBILITY_MULTITHREADED)
+
+fprofile-reproducible
+Common Joined RejectNegative Var(flag_profile_reproducible) Enum(profile_reproducibility) Init(PROFILE_REPRODUCIBILITY_SERIAL)
+-fprofile-reproducible=[serial|parallel-runs|multithreaded] Control level of reproducibility of profile gathered by -fprofile-generate.
+
 Enum
 Name(profile_update) Type(enum profile_update) UnknownError(unknown profile update method %qs)
 

diff --git a/gcc/coretypes.h b/gcc/coretypes.h
index d8fd50d79a44b8841c79bb703487e32aa00ad908..cda22697cc39debc0a6be793a2bf6f2b104dd2d3 100644 (file)
--- a/gcc/coretypes.h
+++ b/gcc/coretypes.h
@@ -212,6 +212,13 @@ enum profile_update {
   PROFILE_UPDATE_PREFER_ATOMIC
 };
 
+/* Type of profile reproducibility methods.  */
+enum profile_reproducibility {
+    PROFILE_REPRODUCIBILITY_SERIAL,
+    PROFILE_REPRODUCIBILITY_PARALLEL_RUNS,
+    PROFILE_REPRODUCIBILITY_MULTITHREADED
+};
+
 /* Types of unwind/exception handling info that can be generated.  */
 
 enum unwind_info_type

diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
index 3e47d06f0d58c403d0e1a1c796d7b96923eaa360..ba2b7e42520f2cbbd5c534f393cfe2329ebbdb29 100644 (file)
--- a/gcc/doc/invoke.texi
+++ b/gcc/doc/invoke.texi
@@ -562,7 +562,7 @@ Objective-C and Objective-C++ Dialects}.
 -fprofile-abs-path @gol
 -fprofile-dir=@var{path}  -fprofile-generate  -fprofile-generate=@var{path} @gol
 -fprofile-note=@var{path}  -fprofile-update=@var{method} @gol
--fprofile-filter-files=@var{regex}  -fprofile-exclude-files=@var{regex} @gol
+-fprofile-filter-files=@var{regex}  -fprofile-exclude-files=@var{regex} -fprofile-reproducibility @gol
 -fsanitize=@var{style}  -fsanitize-recover  -fsanitize-recover=@var{style} @gol
 -fasan-shadow-offset=@var{number}  -fsanitize-sections=@var{s1},@var{s2},... @gol
 -fsanitize-undefined-trap-on-error  -fbounds-check @gol
@@ -13360,6 +13360,35 @@ all the regular expressions (separated by a semi-colon).
 For example, @option{-fprofile-exclude-files=/usr/*} will prevent instrumentation
 of all files that are located in @file{/usr/} folder.
 
+@item -fprofile-reproducible
+@opindex fprofile-reproducible
+Control level of reproducibility of profile gathered by
+@code{-fprofile-generate}.  This makes it possible to rebuild program
+with same outcome which is useful, for example, for distribution
+packages.
+
+With @option{-fprofile-reproducibility=serial} the profile gathered by
+@option{-fprofile-generate} is reproducible provided the trained program
+behaves the same at each invocation of the train run, it is not
+multi-threaded and profile data streaming is always done in the same
+order.  Note that profile streaming happens at the end of program run but
+also before @code{fork} function is invoked.
+
+Note that it is quite common that execution counts of some part of
+programs depends, for example, on length of temporary file names or
+memory space randomization (that may affect hash-table collision rate).
+Such non-reproducible part of programs may be annotated by
+@code{no_instrument_function} function attribute. @code{gcov-dump} with
+@option{-l} can be used to dump gathered data and verify that they are
+indeed reproducible.
+
+With @option{-fprofile-reproducibility=parallel-runs} collected profile
+stays reproducible regardless the order of streaming of the data into
+gcda files.  This setting makes it possible to run multiple instances of
+instrumented program in parallel (such as with @code{make -j}). This
+reduces quality of gathered data, in particular of indirect call
+profiling.
+
 @item -fsanitize=address
 @opindex fsanitize=address
 Enable AddressSanitizer, a fast memory error detector.

diff --git a/gcc/value-prof.c b/gcc/value-prof.c
index f0456c8e93d1013d4667cb5639d0d6acaea39e23..5f940f40399058c288a55c387632519c72f907ad 100644 (file)
--- a/gcc/value-prof.c
+++ b/gcc/value-prof.c
@@ -265,8 +265,10 @@ dump_histogram_value (FILE *dump_file, histogram_value hist)
                    ? "Top N value counter" : "Indirect call counter"));
          if (hist->hvalue.counters)
            {
-             fprintf (dump_file, " all: %" PRId64 ", values: ",
-                      (int64_t) hist->hvalue.counters[0]);
+             fprintf (dump_file, " all: %" PRId64 "%s, values: ",
+                      abs ((int64_t) hist->hvalue.counters[0]),
+                      hist->hvalue.counters[0] < 0
+                      ? " (values missing)": "");
              for (unsigned i = 0; i < GCOV_TOPN_VALUES; i++)
                {
                  fprintf (dump_file, "[%" PRId64 ":%" PRId64 "]",
@@ -719,26 +721,39 @@ gimple_divmod_fixed_value (gassign *stmt, tree value, profile_probability prob,
 
 /* Return the n-th value count of TOPN_VALUE histogram.  If
    there's a value, return true and set VALUE and COUNT
-   arguments.  */
+   arguments.
+
+   Counters have the following meaning.
+
+   abs (counters[0]) is the number of executions
+   for i in 0 ... TOPN-1
+     counters[2 * i + 1] is target
+     abs (counters[2 * i + 2]) is corresponding hitrate counter.
+
+   Value of counters[0] negative when counter became
+   full during merging and some values are lost.  */
 
 bool
 get_nth_most_common_value (gimple *stmt, const char *counter_type,
                           histogram_value hist, gcov_type *value,
                           gcov_type *count, gcov_type *all, unsigned n)
 {
-  if (hist->hvalue.counters[2] == -1)
-    return false;
-
   gcc_assert (n < GCOV_TOPN_VALUES);
 
   *count = 0;
   *value = 0;
 
-  gcov_type read_all = hist->hvalue.counters[0];
+  gcov_type read_all = abs (hist->hvalue.counters[0]);
 
   gcov_type v = hist->hvalue.counters[2 * n + 1];
   gcov_type c = hist->hvalue.counters[2 * n + 2];
 
+  if (hist->hvalue.counters[0] < 0
+      && (flag_profile_reproducible == PROFILE_REPRODUCIBILITY_PARALLEL_RUNS
+         || (flag_profile_reproducible
+             == PROFILE_REPRODUCIBILITY_MULTITHREADED)))
+    return false;
+
   /* Indirect calls can't be verified.  */
   if (stmt
       && check_counter (stmt, counter_type, &c, &read_all,

diff --git a/libgcc/ChangeLog b/libgcc/ChangeLog
index 7b46ccb40c953bb051910268c9bf253bc28caf30..dedc308b3bcfad159c154f50e7804108f216029d 100644 (file)
--- a/libgcc/ChangeLog
+++ b/libgcc/ChangeLog
@@ -1,3 +1,10 @@
+2020-02-18  Martin Liska  <mliska@suse.cz>
+
+       PR ipa/92924
+       * libgcov-merge.c (merge_topn_values_set): Record
+       when a TOP N counter becomes invalid.  When merging
+       remove a smallest value if the space is needed.
+
 2020-02-12  Sandra Loosemore  <sandra@codesourcery.com>
 
        PR libstdc++/79193

diff --git a/libgcc/libgcov-merge.c b/libgcc/libgcov-merge.c
index 19b8ee72ae99f9434fce806922558f17c0114df6..c0785b0bf10d55c587e78393479db3af480984d8 100644 (file)
--- a/libgcc/libgcov-merge.c
+++ b/libgcc/libgcov-merge.c
@@ -86,36 +86,47 @@ __gcov_merge_time_profile (gcov_type *counters, unsigned n_counters)
 
 #ifdef L_gcov_merge_topn
 
+/* To merging of TOPN profiles.
+   counters[0] is the number of executions
+   for i in 0 ... TOPN-1
+     counters[2 * i + 1] is target
+     counters[2 * i + 2] is corresponding hitrate counter.
+
+   Because we prune counters only those with probability >= 1/TOPN are
+   present now.
+
+   We use sign of counters[0] to track whether the number of different
+   targets exceeds TOPN.  */
+
 static void
 merge_topn_values_set (gcov_type *counters)
 {
   /* First value is number of total executions of the profiler.  */
-  gcov_type all = gcov_get_counter_ignore_scaling (-1);
-  counters[0] += all;
+  gcov_type all = gcov_get_counter ();
+  gcov_type *total = &counters[0];
   ++counters;
 
+  /* Negative value means that counter is missing some of values.  */
+  if (all < 0)
+    *total = -(*total);
+
+  *total += all;
+
   /* Read all part values.  */
   gcov_type read_counters[2 * GCOV_TOPN_VALUES];
-
   for (unsigned i = 0; i < GCOV_TOPN_VALUES; i++)
     {
       read_counters[2 * i] = gcov_get_counter_target ();
       read_counters[2 * i + 1] = gcov_get_counter_ignore_scaling (-1);
     }
 
-  if (read_counters[1] == -1)
-    {
-      counters[1] = -1;
-      return;
-    }
-
   for (unsigned i = 0; i < GCOV_TOPN_VALUES; i++)
     {
       if (read_counters[2 * i + 1] == 0)
        continue;
 
       unsigned j;
-      int slot = -1;
+      int slot = 0;
 
       for (j = 0; j < GCOV_TOPN_VALUES; j++)
        {
@@ -124,13 +135,15 @@ merge_topn_values_set (gcov_type *counters)
              counters[2 * j + 1] += read_counters[2 * i + 1];
              break;
            }
-         else if (counters[2 * j + 1] == 0)
+         else if (counters[2 * j + 1] < counters[2 * slot + 1])
            slot = j;
        }
 
       if (j == GCOV_TOPN_VALUES)
        {
-         if (slot > 0)
+         gcov_type slot_count = counters[2 * slot + 1];
+         /* We found an empty slot.  */
+         if (slot_count == 0)
            {
              /* If we found empty slot, add the value.  */
              counters[2 * slot] = read_counters[2 * i];
@@ -138,9 +151,16 @@ merge_topn_values_set (gcov_type *counters)
            }
          else
            {
-             /* We haven't found a slot, bail out.  */
-             counters[1] = -1;
-             return;
+             /* Here we are loosing some values.  */
+             if (*total >= 0)
+               *total = -(*total);
+             if (read_counters[2 * i + 1] > slot_count)
+               {
+                 counters[2 * slot] = read_counters[2 * i];
+                 counters[2 * slot + 1] = read_counters[2 * i + 1];
+               }
+             else
+               counters[2 * slot + 1] -= read_counters[2 * i + 1];
            }
        }
     }