Issue #24915: Add Clang support to PGO builds and use the test suite

for profile data.

Thanks to Alecsandru Patrascu of Intel for the initial patch.

diff --git a/.gitignore b/.gitignore
index cc1cdcd50528b3a140f8fe50bed05dc14eb8d4c1..5be9ae80dc7ca3fcf73494b77af6ae9fe44dac06 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -11,6 +11,9 @@
 *.rej
 *.swp
 *~
+*.gc??
+*.profclang?
+*.profraw
 .gdb_history
 Doc/build/
 Doc/venv/

diff --git a/.hgignore b/.hgignore
index 806e15606ec104a49a29ac451f8bfa424d8760de..7f14dd63111a4ecf15225aa818ec71113086491e 100644 (file)
--- a/.hgignore
+++ b/.hgignore
@@ -50,6 +50,9 @@ libpython*.so*
 *.pyd
 *.cover
 *~
+*.gc??
+*.profclang?
+*.profraw
 Lib/distutils/command/*.pdb
 Lib/lib2to3/*.pickle
 Lib/test/data/*

diff --git a/Makefile.pre.in b/Makefile.pre.in
index ce2c0aa290064a3f85da813f3727f96a9d5913a7..9a8c1599b99fa36509e7f279e40213c3310f5aaf 100644 (file)
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -43,6 +43,11 @@ LDVERSION=   @LDVERSION@
 HGVERSION=     @HGVERSION@
 HGTAG=         @HGTAG@
 HGBRANCH=      @HGBRANCH@
+PGO_PROF_GEN_FLAG=@PGO_PROF_GEN_FLAG@
+PGO_PROF_USE_FLAG=@PGO_PROF_USE_FLAG@
+LLVM_PROF_MERGER=@LLVM_PROF_MERGER@
+LLVM_PROF_FILE=@LLVM_PROF_FILE@
+LLVM_PROF_ERR=@LLVM_PROF_ERR@
 
 GNULD=         @GNULD@
 
@@ -226,8 +231,7 @@ TCLTK_INCLUDES=     @TCLTK_INCLUDES@
 TCLTK_LIBS=    @TCLTK_LIBS@
 
 # The task to run while instrument when building the profile-opt target
-PROFILE_TASK=  $(srcdir)/Tools/pybench/pybench.py -n 2 --with-gc --with-syscheck
-#PROFILE_TASK= $(srcdir)/Lib/test/regrtest.py
+PROFILE_TASK=-m test.regrtest >/dev/null 2>&1
 
 # report files for gcov / lcov coverage report
 COVERAGE_INFO= $(abs_builddir)/coverage.info
@@ -477,27 +481,38 @@ LIBRARY_OBJS=     \
 all:           build_all
 build_all:     $(BUILDPYTHON) oldsharedmods sharedmods gdbhooks Programs/_testembed python-config
 
-# Compile a binary with gcc profile guided optimization.
+# Compile a binary with profile guided optimization.
 profile-opt:
+       @if [ $(LLVM_PROF_ERR) == yes ]; then \
+               echo "Error: Cannot perform PGO build because llvm-profdata was not found in PATH" ;\
+               echo "Please add it to PATH and run ./configure again" ;\
+               exit 1;\
+       fi
        @echo "Building with support for profile generation:"
        $(MAKE) clean
+       $(MAKE) profile-removal
        $(MAKE) build_all_generate_profile
-       @echo "Running benchmark to generate profile data:"
        $(MAKE) profile-removal
+       @echo "Running code to generate profile data (this can take a while):"
        $(MAKE) run_profile_task
+       $(MAKE) build_all_merge_profile
        @echo "Rebuilding with profile guided optimizations:"
        $(MAKE) clean
        $(MAKE) build_all_use_profile
+       $(MAKE) profile-removal
 
 build_all_generate_profile:
-       $(MAKE) all CFLAGS_NODIST="$(CFLAGS) -fprofile-generate" LDFLAGS="-fprofile-generate" LIBS="$(LIBS) -lgcov"
+       $(MAKE) all CFLAGS_NODIST="$(CFLAGS) $(PGO_PROF_GEN_FLAG)" LDFLAGS="$(LDFLAGS) $(PGO_PROF_GEN_FLAG)" LIBS="$(LIBS)"
 
 run_profile_task:
        : # FIXME: can't run for a cross build
-       $(RUNSHARED) ./$(BUILDPYTHON) $(PROFILE_TASK)
+       $(LLVM_PROF_FILE) $(RUNSHARED) ./$(BUILDPYTHON) $(PROFILE_TASK) || true
+
+build_all_merge_profile:
+       $(LLVM_PROF_MERGER)
 
 build_all_use_profile:
-       $(MAKE) all CFLAGS_NODIST="$(CFLAGS) -fprofile-use -fprofile-correction"
+       $(MAKE) all CFLAGS_NODIST="$(CFLAGS) $(PGO_PROF_USE_FLAG)"
 
 # Compile and run with gcov
 .PHONY=coverage coverage-lcov coverage-report
@@ -1568,9 +1583,11 @@ clean: pycremoval
        -rm -f pybuilddir.txt
        -rm -f Lib/lib2to3/*Grammar*.pickle
        -rm -f Programs/_testembed Programs/_freeze_importlib
+       -rm -rf build
 
 profile-removal:
        find . -name '*.gc??' -exec rm -f {} ';'
+       find . -name '*.profclang?' -exec rm -f {} ';'
        rm -f $(COVERAGE_INFO)
        rm -rf $(COVERAGE_REPORT)
 

diff --git a/Misc/ACKS b/Misc/ACKS
index 3704e650bfe104abe34d54badc8a0da42d86bfc7..ba144b703295de81e0d0e51bd1d9401b924f4452 100644 (file)
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -1079,6 +1079,7 @@ Heikki Partanen
 Harri Pasanen
 Gaël Pasgrimaud
 Ashish Nitin Patil
+Alecsandru Patrascu
 Randy Pausch
 Samuele Pedroni
 Justin Peel

diff --git a/Misc/NEWS b/Misc/NEWS
index c021452c5028c99ebc8848e5b719fe53078f790c..edd555feeba116358f8885872751e47b3dc3d527 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -115,6 +115,9 @@ Tests
 Build
 -----
 
+- Issue #24915: Add LLVM support for PGO builds and use the test suite to
+  generate the profile data. Initiial patch by Alecsandru Patrascu of Intel.
+
 - Issue #24910: Windows MSIs now have unique display names.
 
 - Issue #24986: It is now possible to build Python on Windows without errors

diff --git a/README b/README
index d37b96b316099c8e0d6351704b1b83f92df0643f..82d1697accf80aab278ce6b0afb613b5dff05443 100644 (file)
--- a/README
+++ b/README
@@ -46,6 +46,34 @@ For example:
 (This will fail if you *also* built at the top-level directory.
 You should do a "make clean" at the toplevel first.)
 
+If you need an optimized version of Python, you type "make profile-opt" in the
+top level directory. This will rebuild the interpreter executable using Profile
+Guided Optimization (PGO). For more details, see the section bellow.
+
+
+Profile Guided Optimization
+---------------------------
+
+PGO takes advantage of recent versions of the GCC or Clang compilers.
+If ran, the "profile-opt" rule will do several steps.
+
+First, the entire Python directory is cleaned of temporary files that
+may resulted in a previous compilation.
+
+Then, an instrumented version of the interpreter is built, using suitable
+compiler flags for each flavour. Note that this is just an intermediary
+step and the binary resulted after this step is not good for real life
+workloads, as it has profiling instructions embedded inside.
+
+After this instrumented version of the interpreter is built, the Makefile
+will automatically run a training workload. This is necessary in order to
+profile the interpreter execution. Note also that any output, both stdout
+and stderr, that may appear at this step is supressed.
+
+Finally, the last step is to rebuild the interpreter, using the information
+collected in the previous one. The end result will be a the Python binary
+that is optimized and suitable for distribution or production installation.
+
 
 What's New
 ----------

diff --git a/configure b/configure
index e823a083c67c415a0570189d019368dcfebb57d9..06b4d582e091194e88f6ce333fb5167560d1b502 100755 (executable)
--- a/configure
+++ b/configure
@@ -667,6 +667,12 @@ UNIVERSAL_ARCH_FLAGS
 CFLAGS_NODIST
 BASECFLAGS
 OPT
+LLVM_PROF_FOUND
+LLVM_PROF_ERR
+LLVM_PROF_FILE
+LLVM_PROF_MERGER
+PGO_PROF_USE_FLAG
+PGO_PROF_GEN_FLAG
 ABIFLAGS
 LN
 MKDIR_P
@@ -6431,6 +6437,80 @@ $as_echo "no" >&6; }
 fi
 
 
+# Enable PGO flags.
+# Extract the first word of "llvm-profdata", so it can be a program name with args.
+set dummy llvm-profdata; ac_word=$2
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
+$as_echo_n "checking for $ac_word... " >&6; }
+if ${ac_cv_prog_LLVM_PROF_FOUND+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  if test -n "$LLVM_PROF_FOUND"; then
+  ac_cv_prog_LLVM_PROF_FOUND="$LLVM_PROF_FOUND" # Let the user override the test.
+else
+as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
+for as_dir in $PATH
+do
+  IFS=$as_save_IFS
+  test -z "$as_dir" && as_dir=.
+    for ac_exec_ext in '' $ac_executable_extensions; do
+  if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
+    ac_cv_prog_LLVM_PROF_FOUND="found"
+    $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
+    break 2
+  fi
+done
+  done
+IFS=$as_save_IFS
+
+  test -z "$ac_cv_prog_LLVM_PROF_FOUND" && ac_cv_prog_LLVM_PROF_FOUND="not-found"
+fi
+fi
+LLVM_PROF_FOUND=$ac_cv_prog_LLVM_PROF_FOUND
+if test -n "$LLVM_PROF_FOUND"; then
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: $LLVM_PROF_FOUND" >&5
+$as_echo "$LLVM_PROF_FOUND" >&6; }
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+fi
+
+
+LLVM_PROF_ERR=no
+case $CC in
+  *clang*)
+    # Any changes made here should be reflected in the GCC+Darwin case below
+    PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
+    PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
+    LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
+    LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
+    if test $LLVM_PROF_FOUND = not-found
+    then
+      LLVM_PROF_ERR=yes
+    fi
+    ;;
+  *gcc*)
+    case $ac_sys_system in
+      Darwin*)
+        PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
+        PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
+        LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
+        LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
+        if test $LLVM_PROF_FOUND = not-found
+        then
+          LLVM_PROF_ERR=yes
+        fi
+        ;;
+      *)
+        PGO_PROF_GEN_FLAG="-fprofile-generate"
+        PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction"
+        LLVM_PROF_MERGER="true"
+        LLVM_PROF_FILE=""
+        ;;
+    esac
+    ;;
+esac
+
 # XXX Shouldn't the code above that fiddles with BASECFLAGS and OPT be
 # merged with this chunk of code?
 

diff --git a/configure.ac b/configure.ac
index 56a73df233dcb20a7aad6a803961efe9d3140e50..792f580c12ac9f9cf0f9e11f9cf656ef2d0a7e36 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -1218,6 +1218,49 @@ else AC_MSG_RESULT(no); Py_DEBUG='false'
 fi],
 [AC_MSG_RESULT(no)])
 
+# Enable PGO flags.
+AC_SUBST(PGO_PROF_GEN_FLAG)
+AC_SUBST(PGO_PROF_USE_FLAG)
+AC_SUBST(LLVM_PROF_MERGER)
+AC_SUBST(LLVM_PROF_FILE)
+AC_SUBST(LLVM_PROF_ERR)
+AC_SUBST(LLVM_PROF_FOUND)
+AC_CHECK_PROG(LLVM_PROF_FOUND, llvm-profdata, found, not-found)
+LLVM_PROF_ERR=no
+case $CC in
+  *clang*)
+    # Any changes made here should be reflected in the GCC+Darwin case below
+    PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
+    PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
+    LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
+    LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
+    if test $LLVM_PROF_FOUND = not-found
+    then
+      LLVM_PROF_ERR=yes
+    fi
+    ;;
+  *gcc*)
+    case $ac_sys_system in
+      Darwin*)
+        PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
+        PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
+        LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
+        LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
+        if test $LLVM_PROF_FOUND = not-found
+        then
+          LLVM_PROF_ERR=yes
+        fi
+        ;;
+      *)
+        PGO_PROF_GEN_FLAG="-fprofile-generate"
+        PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction"
+        LLVM_PROF_MERGER="true"
+        LLVM_PROF_FILE=""
+        ;;
+    esac
+    ;;
+esac
+
 # XXX Shouldn't the code above that fiddles with BASECFLAGS and OPT be
 # merged with this chunk of code?