Issue #24915: Make PGO builds support Clang and use the test suite for

profile data.

Thanks to Alecsandru Patrascu of Intel for the initial patch.

diff --git a/.gitignore b/.gitignore
index af56278bad1782cd11978a635c02d5cf6b15ff3a..2b87e9386d2443223aa7073bf273c056ae69167c 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,9 @@
 *.pyo
 *.rej
 *~
+*.gc??
+*.profclang?
+*.profraw
 Doc/build/
 Doc/tools/docutils/
 Doc/tools/jinja2/

diff --git a/.hgignore b/.hgignore
index 6fa534ff522c7a518eab0b99fdd850cabb955feb..007aa7b64ff368c0e3e9d6cea498601ba1b2a168 100644 (file)
--- a/.hgignore
+++ b/.hgignore
@@ -45,6 +45,9 @@ libpython*.so*
 *.pyd
 *.cover
 *~
+*.gc??
+*.profclang?
+*.profraw
 Lib/distutils/command/*.pdb
 Lib/lib2to3/*.pickle
 Lib/test/data/*

diff --git a/Makefile.pre.in b/Makefile.pre.in
index 59891430f62ad0f964e393a33183dee6512c987e..4082313bfd1ec54e2473647d678612f0f0f3eab4 100644 (file)
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -42,6 +42,11 @@ SVNVERSION=  @SVNVERSION@
 HGVERSION=     @HGVERSION@
 HGTAG=         @HGTAG@
 HGBRANCH=      @HGBRANCH@
+PGO_PROF_GEN_FLAG=@PGO_PROF_GEN_FLAG@
+PGO_PROF_USE_FLAG=@PGO_PROF_USE_FLAG@
+LLVM_PROF_MERGER=@LLVM_PROF_MERGER@
+LLVM_PROF_FILE=@LLVM_PROF_FILE@
+LLVM_PROF_ERR=@LLVM_PROF_ERR@
 
 GNULD=          @GNULD@
 
@@ -204,8 +209,7 @@ TCLTK_INCLUDES=     @TCLTK_INCLUDES@
 TCLTK_LIBS=    @TCLTK_LIBS@
 
 # The task to run while instrument when building the profile-opt target
-PROFILE_TASK=  $(srcdir)/Tools/pybench/pybench.py -n 2 --with-gc --with-syscheck
-#PROFILE_TASK= $(srcdir)/Lib/test/regrtest.py
+PROFILE_TASK=-m test.regrtest >/dev/null 2>&1
 
 # === Definitions added by makesetup ===
 
@@ -420,27 +424,38 @@ LIBRARY_OBJS=     \
 all:           build_all
 build_all:     $(BUILDPYTHON) oldsharedmods sharedmods gdbhooks
 
-# Compile a binary with gcc profile guided optimization.
+# Compile a binary with profile guided optimization.
 profile-opt:
+       @if [ $(LLVM_PROF_ERR) == yes ]; then \
+               echo "Error: Cannot perform PGO build because llvm-profdata was not found in PATH" ;\
+               echo "Please add it to PATH and run ./configure again" ;\
+               exit 1;\
+       fi
        @echo "Building with support for profile generation:"
        $(MAKE) clean
+       $(MAKE) profile-removal
        $(MAKE) build_all_generate_profile
-       @echo "Running benchmark to generate profile data:"
        $(MAKE) profile-removal
+       @echo "Running code to generate profile data (this can take a while):"
        $(MAKE) run_profile_task
+       $(MAKE) build_all_merge_profile
        @echo "Rebuilding with profile guided optimizations:"
        $(MAKE) clean
        $(MAKE) build_all_use_profile
+       $(MAKE) profile-removal
 
 build_all_generate_profile:
-       $(MAKE) all CFLAGS="$(CFLAGS) -fprofile-generate" LIBS="$(LIBS) -lgcov"
+       $(MAKE) all CFLAGS="$(CFLAGS) $(PGO_PROF_GEN_FLAG)" LDFLAGS="$(LDFLAGS) $(PGO_PROF_GEN_FLAG)" LIBS="$(LIBS)"
 
 run_profile_task:
        : # FIXME: can't run for a cross build
-       ./$(BUILDPYTHON) $(PROFILE_TASK)
+       $(LLVM_PROF_FILE) ./$(BUILDPYTHON) $(PROFILE_TASK) || true
+
+build_all_merge_profile:
+       $(LLVM_PROF_MERGER)
 
 build_all_use_profile:
-       $(MAKE) all CFLAGS="$(CFLAGS) -fprofile-use"
+       $(MAKE) all CFLAGS="$(CFLAGS) $(PGO_PROF_USE_FLAG)"
 
 coverage:
        @echo "Building with support for coverage checking:"
@@ -1330,9 +1345,11 @@ clean: pycremoval
        find build -name 'fficonfig.h' -exec rm -f {} ';' || true
        find build -name 'fficonfig.py' -exec rm -f {} ';' || true
        -rm -f Lib/lib2to3/*Grammar*.pickle
+       -rm -rf build
 
 profile-removal:
        find . -name '*.gc??' -exec rm -f {} ';'
+       find . -name '*.profclang?' -exec rm -f {} ';'
 
 clobber: clean profile-removal
        -rm -f $(BUILDPYTHON) $(PGEN) $(LIBRARY) $(LDLIBRARY) $(DLLLIBRARY) \

diff --git a/Misc/NEWS b/Misc/NEWS
index f1b5ff951b4f707c8bd33e7001e392108338c5a0..bc1532f9b56683612756356d05bf06e909a75ea4 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -150,6 +150,9 @@ Library
 Build
 -----
 
+- Issue #24915: When doing a PGO build, the test suite is now used instead of
+  pybench; Clang support was also added as part off this work.
+
 - Issue #24986: It is now possible to build Python on Windows without errors
   when external libraries are not available.
 

diff --git a/README b/README
index 252d90ea5cbc8889fdeb009077c92e0413373fd7..85c47bbc14f9d165b786c508eb37ede555ca412b 100644 (file)
--- a/README
+++ b/README
@@ -173,6 +173,11 @@ rebuilt.  In this case, you may have to run make again to correctly
 build your desired target.  The interpreter executable is built in the
 top level directory.
 
+If you need an optimized version of Python, you type "make profile-opt"
+in the top level directory. This will rebuild the interpreter executable
+using Profile Guided Optimization (PGO). For more details, see the
+section below.
+
 Once you have built a Python interpreter, see the subsections below on
 testing and installation.  If you run into trouble, see the next
 section.
@@ -185,6 +190,30 @@ guidance of the setup.py script, which is run by Make after the
 interpreter has been built.
 
 
+Profile Guided Optimization
+---------------------------
+
+PGO takes advantage of recent versions of the GCC or Clang compilers.
+If ran, the "profile-opt" rule will do several steps.
+
+First, the entire Python directory is cleaned of temporary files that
+may resulted in a previous compilation.
+
+Then, an instrumented version of the interpreter is built, using suitable
+compiler flags for each flavour. Note that this is just an intermediary
+step and the binary resulted after this step is not good for real life
+workloads, as it has profiling instructions embedded inside.
+
+After this instrumented version of the interpreter is built, the Makefile
+will automatically run a training workload. This is necessary in order to
+profile the interpreter execution. Note also that any output, both stdout
+and stderr, that may appear at this step is supressed.
+
+Finally, the last step is to rebuild the interpreter, using the information
+collected in the previous one. The end result will be a the Python binary
+that is optimized and suitable for distribution or production installation.
+
+
 Troubleshooting
 ---------------
 

diff --git a/configure b/configure
index 712787133fd1fbfee4d99a152d20644a8292e52e..7dab8975f1a56de969ebcfe0c29779a4b16762ad 100755 (executable)
--- a/configure
+++ b/configure
@@ -661,6 +661,12 @@ LDSHARED
 SO
 LIBTOOL_CRUFT
 OTHER_LIBTOOL_OPT
+LLVM_PROF_FOUND
+LLVM_PROF_ERR
+LLVM_PROF_FILE
+LLVM_PROF_MERGER
+PGO_PROF_USE_FLAG
+PGO_PROF_GEN_FLAG
 UNIVERSAL_ARCH_FLAGS
 BASECFLAGS
 OPT
@@ -6326,6 +6332,82 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
   CFLAGS=$save_CFLAGS
 fi
 
+
+# Enable PGO flags.
+# Extract the first word of "llvm-profdata", so it can be a program name with args.
+set dummy llvm-profdata; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_prog_LLVM_PROF_FOUND+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  if test -n "$LLVM_PROF_FOUND"; then
+  ac_cv_prog_LLVM_PROF_FOUND="$LLVM_PROF_FOUND" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+    for ac_exec_ext in '' $ac_executable_extensions; do
+  if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_LLVM_PROF_FOUND="found"
+    $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+  done
+IFS=$as_save_IFS
+
+  test -z "$ac_cv_prog_LLVM_PROF_FOUND" && ac_cv_prog_LLVM_PROF_FOUND="not-found"
+fi
+fi
+LLVM_PROF_FOUND=$ac_cv_prog_LLVM_PROF_FOUND
+if test -n "$LLVM_PROF_FOUND"; then
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: $LLVM_PROF_FOUND" >&5
+$as_echo "$LLVM_PROF_FOUND" >&6; }
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+LLVM_PROF_ERR=no
+case $CC in
+  *clang*)
+    # Any changes made here should be reflected in the GCC+Darwin case below
+    PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
+    PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
+    LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
+    LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
+    if test $LLVM_PROF_FOUND = not-found
+    then
+      LLVM_PROF_ERR=yes
+    fi
+    ;;
+  *gcc*)
+    case $ac_sys_system in
+      Darwin*)
+        PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
+        PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
+        LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
+        LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
+        if test $LLVM_PROF_FOUND = not-found
+        then
+          LLVM_PROF_ERR=yes
+        fi
+        ;;
+      *)
+        PGO_PROF_GEN_FLAG="-fprofile-generate"
+        PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction"
+        LLVM_PROF_MERGER="true"
+        LLVM_PROF_FILE=""
+        ;;
+    esac
+    ;;
+esac
+
+
 # On some compilers, pthreads are available without further options
 # (e.g. MacOS X). On some of these systems, the compiler will not
 # complain if unaccepted options are passed (e.g. gcc on Mac OS X).

diff --git a/configure.ac b/configure.ac
index 2c998b1ee98eab9b9ffd4b2f1f40b6a2657f2a00..decf0ec7424bc1fac0040e1d7e837a32823198a2 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -1352,6 +1352,51 @@ then
   CFLAGS=$save_CFLAGS
 fi
 
+
+# Enable PGO flags.
+AC_SUBST(PGO_PROF_GEN_FLAG)
+AC_SUBST(PGO_PROF_USE_FLAG)
+AC_SUBST(LLVM_PROF_MERGER)
+AC_SUBST(LLVM_PROF_FILE)
+AC_SUBST(LLVM_PROF_ERR)
+AC_SUBST(LLVM_PROF_FOUND)
+AC_CHECK_PROG(LLVM_PROF_FOUND, llvm-profdata, found, not-found)
+LLVM_PROF_ERR=no
+case $CC in
+  *clang*)
+    # Any changes made here should be reflected in the GCC+Darwin case below
+    PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
+    PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
+    LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
+    LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
+    if test $LLVM_PROF_FOUND = not-found
+    then
+      LLVM_PROF_ERR=yes
+    fi
+    ;;
+  *gcc*)
+    case $ac_sys_system in
+      Darwin*)
+        PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
+        PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
+        LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
+        LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
+        if test $LLVM_PROF_FOUND = not-found
+        then
+          LLVM_PROF_ERR=yes
+        fi
+        ;;
+      *)
+        PGO_PROF_GEN_FLAG="-fprofile-generate"
+        PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction"
+        LLVM_PROF_MERGER="true"
+        LLVM_PROF_FILE=""
+        ;;
+    esac
+    ;;
+esac
+
+
 # On some compilers, pthreads are available without further options
 # (e.g. MacOS X). On some of these systems, the compiler will not
 # complain if unaccepted options are passed (e.g. gcc on Mac OS X).