bpo-28643: Record profile-opt build progress with stamp files (#4223)

* bpo-28643: Record profile-opt build progress with stamp files

The profile-opt makefile target is expensive to build. Since the
makefile does not contain complete dependency information for this
target, much extra work can get done if the build is interrupted and
re-started.  Even running "make" a second time will result in a huge
amount of redundant work.

As a minimal fix (rather than removing recursive "make" and adding a
proper dependency graph), split the profile-opt target into parts:

- ensure tree is clean (profile-clean-stamp)
- build with profile generation enabled (profile-gen-stamp)
- run task to generate profile information (profile-run-stamp)
- build optimized Python using above information (profile-opt)

We use "stamp" files to record completion of the steps.  Running
"make clean" will not remove the profile-run-stamp file.

Other minor changes:

- remove the "build_all_use_profile" target.  I don't expect callers
  of the makefile to use this target so that should be safe.

- remove execution of "profile-removal" at end of "profile-opt".  I
  don't see any reason to not to keep the profile information, given
  the cost to generate it.  Removing the "profile-run-stamp" file
  will force re-generation of it.

diff --git a/Makefile.pre.in b/Makefile.pre.in
index 6dacb872e0c243210846479c1aab34ecc553b027..e5d65bde26658e7136eb226635c8b67f57ec325e 100644 (file)
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -442,25 +442,37 @@ DTRACE_DEPS = \
 all:           @DEF_MAKE_ALL_RULE@
 build_all:     $(BUILDPYTHON) oldsharedmods sharedmods gdbhooks Programs/_testembed python-config
 
-# Compile a binary with profile guided optimization.
-profile-opt:
+# Profile generation build must start from a clean tree.
+profile-clean-stamp:
+       $(MAKE) clean profile-removal
+       touch $@
+
+# Compile with profile generation enabled.
+profile-gen-stamp: profile-clean-stamp
        @if [ $(LLVM_PROF_ERR) = yes ]; then \
                echo "Error: Cannot perform PGO build because llvm-profdata was not found in PATH" ;\
                echo "Please add it to PATH and run ./configure again" ;\
                exit 1;\
        fi
        @echo "Building with support for profile generation:"
-       $(MAKE) clean
-       $(MAKE) profile-removal
        $(MAKE) build_all_generate_profile
-       $(MAKE) profile-removal
+       touch $@
+
+# Run task with profile generation build to create profile information.
+profile-run-stamp:
        @echo "Running code to generate profile data (this can take a while):"
+       # First, we need to create a clean build with profile generation
+       # enabled.
+       $(MAKE) profile-gen-stamp
+       # Next, run the profile task to generate the profile information.
        $(MAKE) run_profile_task
        $(MAKE) build_all_merge_profile
-       @echo "Rebuilding with profile guided optimizations:"
+       # Remove profile generation binary since we are done with it.
        $(MAKE) clean
-       $(MAKE) build_all_use_profile
-       $(MAKE) profile-removal
+       # This is an expensive target to build and it does not have proper
+       # makefile dependancy information.  So, we create a "stamp" file
+       # to record its completion and avoid re-running it.
+       touch $@
 
 build_all_generate_profile:
        $(MAKE) @DEF_MAKE_RULE@ CFLAGS_NODIST="$(CFLAGS) $(PGO_PROF_GEN_FLAG)" LDFLAGS="$(LDFLAGS) $(PGO_PROF_GEN_FLAG)" LIBS="$(LIBS)"
@@ -472,7 +484,11 @@ run_profile_task:
 build_all_merge_profile:
        $(LLVM_PROF_MERGER)
 
-build_all_use_profile:
+# Compile Python binary with profile guided optimization.
+# To force re-running of the profile task, remove the profile-run-stamp file.
+profile-opt: profile-run-stamp
+       @echo "Rebuilding with profile guided optimizations:"
+       -rm -f profile-clean-stamp
        $(MAKE) @DEF_MAKE_RULE@ CFLAGS_NODIST="$(CFLAGS) $(PGO_PROF_USE_FLAG)" LDFLAGS="$(LDFLAGS)"
 
 # Compile and run with gcov
@@ -1621,6 +1637,7 @@ clean: pycremoval
        -rm -f Programs/_testembed Programs/_freeze_importlib
        -find build -type f -a ! -name '*.gc??' -exec rm -f {} ';'
        -rm -f Include/pydtrace_probes.h
+       -rm -f profile-gen-stamp
 
 profile-removal:
        find . -name '*.gc??' -exec rm -f {} ';'
@@ -1628,6 +1645,7 @@ profile-removal:
        find . -name '*.dyn' -exec rm -f {} ';'
        rm -f $(COVERAGE_INFO)
        rm -rf $(COVERAGE_REPORT)
+       rm -f profile-run-stamp
 
 clobber: clean profile-removal
        -rm -f $(BUILDPYTHON) $(PGEN) $(LIBRARY) $(LDLIBRARY) $(DLLLIBRARY) \
@@ -1636,6 +1654,7 @@ clobber: clean profile-removal
        -rm -rf build platform
        -rm -rf $(PYTHONFRAMEWORKDIR)
        -rm -f python-config.py python-config
+       -rm -f profile-gen-stamp profile-clean-stamp
 
 # Make things extra clean, before making a distribution:
 # remove all generated files, even Makefile[.pre]

diff --git a/Misc/NEWS.d/next/Build/2017-11-01-14-16-27.bpo-28643.9iPKJy.rst b/Misc/NEWS.d/next/Build/2017-11-01-14-16-27.bpo-28643.9iPKJy.rst
new file mode 100644 (file)
index 0000000..87f6d58
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2017-11-01-14-16-27.bpo-28643.9iPKJy.rst
@@ -0,0 +1 @@
+Record profile-opt build progress with stamp files.