[3.6] Improves the ability to build in CI (GH-5730)

diff --git a/Lib/test/support/script_helper.py b/Lib/test/support/script_helper.py
index ca5f9c20dd039f6964f07747db9b3674fc7cb6b2..507dc48848a5110a0c203c727584589987b8ec32 100644 (file)
--- a/Lib/test/support/script_helper.py
+++ b/Lib/test/support/script_helper.py
@@ -39,6 +39,11 @@ def interpreter_requires_environment():
     """
     global __cached_interp_requires_environment
     if __cached_interp_requires_environment is None:
+        # If PYTHONHOME is set, assume that we need it
+        if 'PYTHONHOME' in os.environ:
+            __cached_interp_requires_environment = True
+            return True
+
         # Try running an interpreter with -E to see if it works or not.
         try:
             subprocess.check_call([sys.executable, '-E',
@@ -165,7 +170,9 @@ def spawn_python(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw):
     kw is extra keyword args to pass to subprocess.Popen. Returns a Popen
     object.
     """
-    cmd_line = [sys.executable, '-E']
+    cmd_line = [sys.executable]
+    if not interpreter_requires_environment():
+        cmd_line.append('-E')
     cmd_line.extend(args)
     # Under Fedora (?), GNU readline can output junk on stderr when initialized,
     # depending on the TERM setting.  Setting TERM=vt100 is supposed to disable

diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py
index 98441b0f4a5d4d162f5955ef2de2648897bf1516..32d2df9d6c260203076d662c9fabb2e280f6de90 100644 (file)
--- a/Lib/test/test_cmd_line.py
+++ b/Lib/test/test_cmd_line.py
@@ -10,7 +10,7 @@ import subprocess
 import tempfile
 from test.support import script_helper, is_android
 from test.support.script_helper import (spawn_python, kill_python, assert_python_ok,
-    assert_python_failure)
+    assert_python_failure, interpreter_requires_environment)
 
 
 # XXX (ncoghlan): Move to script_helper and make consistent with run_python
@@ -57,6 +57,8 @@ class CmdLineTest(unittest.TestCase):
         rc, out, err = assert_python_ok('-vv')
         self.assertNotIn(b'stack overflow', err)
 
+    @unittest.skipIf(interpreter_requires_environment(),
+                     'Cannot run -E tests when PYTHON env vars are required.')
     def test_xoptions(self):
         def get_xoptions(*args):
             # use subprocess module directly because test.support.script_helper adds
@@ -272,11 +274,7 @@ class CmdLineTest(unittest.TestCase):
 
     def test_displayhook_unencodable(self):
         for encoding in ('ascii', 'latin-1', 'utf-8'):
-            # We are testing a PYTHON environment variable here, so we can't
-            # use -E, -I, or script_helper (which uses them).  So instead we do
-            # poor-man's isolation by deleting the PYTHON vars from env.
-            env = {key:value for (key,value) in os.environ.copy().items()
-                   if not key.startswith('PYTHON')}
+            env = os.environ.copy()
             env['PYTHONIOENCODING'] = encoding
             p = subprocess.Popen(
                 [sys.executable, '-i'],
@@ -491,6 +489,7 @@ class CmdLineTest(unittest.TestCase):
                                           cwd=tmpdir)
             self.assertEqual(out.strip(), b"ok")
 
+
 def test_main():
     test.support.run_unittest(CmdLineTest)
     test.support.reap_children()

diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat
index ba3a0f5a10daa38e05288469e1826663943cc1e4..17ce7ac609c1d14281aa295a2d105c1877ca98ae 100644 (file)
--- a/PCbuild/find_python.bat
+++ b/PCbuild/find_python.bat
@@ -31,10 +31,10 @@
 @if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found in externals directory) & goto :found
 
 @rem If HOST_PYTHON is recent enough, use that
-@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -c "import sys; assert sys.version_info[:2] >= (3, 6)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found
+@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -Ec "import sys; assert sys.version_info[:2] >= (3, 6)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found
 
 @rem If py.exe finds a recent enough version, use that one
-@py -3.6 -V >nul 2>&1 && (set PYTHON=py -3.6) && (set _Py_Python_Source=found with py.exe) && goto :found
+@py -3.6 -EV >nul 2>&1 && (set PYTHON=py -3.6) && (set _Py_Python_Source=found with py.exe) && goto :found
 
 @if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%"
 @set _Py_NUGET=%NUGET%
@@ -50,7 +50,7 @@
     @rem If it fails, retry with any available copy of Python
     @powershell.exe -Command Invoke-WebRequest %_Py_NUGET_URL% -OutFile '%_Py_NUGET%'
     @if errorlevel 1 (
-        @%_Py_HOST_PYTHON% "%~dp0\urlretrieve.py" "%_Py_NUGET_URL%" "%_Py_NUGET%"
+        @%_Py_HOST_PYTHON% -E "%~dp0\urlretrieve.py" "%_Py_NUGET_URL%" "%_Py_NUGET%"
     )
 )
 @echo Installing Python via nuget...

diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat
index 58e6c5de2baa122985c463dcc9f671be13a4f504..7c1ed2750a2fd1280b4316940bef10417ea072e8 100644 (file)
--- a/PCbuild/get_externals.bat
+++ b/PCbuild/get_externals.bat
@@ -56,7 +56,7 @@ for %%e in (%libraries%) do (
         git clone --depth 1 https://github.com/%ORG%/cpython-source-deps --branch %%e "%EXTERNALS_DIR%\%%e"
     ) else (
         echo.Fetching %%e...
-        %PYTHON% "%PCBUILD%\get_external.py" -O %ORG% %%e
+        %PYTHON% -E "%PCBUILD%\get_external.py" -O %ORG% -e "%EXTERNALS_DIR%" %%e
     )
 )
 
@@ -74,7 +74,7 @@ for %%b in (%binaries%) do (
         git clone --depth 1 https://github.com/%ORG%/cpython-bin-deps --branch %%b "%EXTERNALS_DIR%\%%b"
     ) else (
         echo.Fetching %%b...
-        %PYTHON% "%PCBUILD%\get_external.py" -b -O %ORG% %%b
+        %PYTHON% -E "%PCBUILD%\get_external.py" -b -O %ORG% -e "%EXTERNALS_DIR%" %%b
     )
 )
 

diff --git a/PCbuild/python.props b/PCbuild/python.props
index 16a7672315f246cbc1e8a6417093adcf6c15129a..6589a23c7ad7f4222918b8ed7e866c2195e0a2bb 100644 (file)
--- a/PCbuild/python.props
+++ b/PCbuild/python.props
@@ -43,7 +43,9 @@
     <BuildPath Condition="$(Configuration) == 'PGInstrument'">$(BuildPath)instrumented\</BuildPath>
     
     <!-- Directories of external projects. tcltk is handled in tcltk.props -->
-    <ExternalsDir>$([System.IO.Path]::GetFullPath(`$(PySourcePath)externals\`))</ExternalsDir>
+    <ExternalsDir>$(EXTERNALS_DIR)</ExternalsDir>
+    <ExternalsDir Condition="$(ExternalsDir) == ''">$([System.IO.Path]::GetFullPath(`$(PySourcePath)externals`))</ExternalsDir>
+    <ExternalsDir Condition="!HasTrailingSlash($(ExternalsDir))">$(ExternalsDir)\</ExternalsDir>
     <sqlite3Dir>$(ExternalsDir)sqlite-3.21.0.0\</sqlite3Dir>
     <bz2Dir>$(ExternalsDir)bzip2-1.0.6\</bz2Dir>
     <lzmaDir>$(ExternalsDir)xz-5.2.2\</lzmaDir>