bpo-29708: Setting SOURCE_DATE_EPOCH forces hash-based .pyc files (GH-5200)

To support reproducible builds, the setting of of SOURCE_DATE_EPOCH triggers the py_compile module -- and by extension, compileall -- to forcibly compile with hash-based .pyc files. This eliminates the possibility of timestamp-based .pyc files which vary between builds.

diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst
index a4f06de597b14e04d73a9ece8d656f9d7c8568d9..d720e01050577df465bd6522bee19436b0232e45 100644 (file)
--- a/Doc/library/py_compile.rst
+++ b/Doc/library/py_compile.rst
@@ -55,7 +55,9 @@ byte-code cache files in the directory containing the source code.
 
    *invalidation_mode* should be a member of the :class:`PycInvalidationMode`
    enum and controls how the generated ``.pyc`` files are invalidated at
-   runtime.
+   runtime. If the :envvar:`SOURCE_DATE_EPOCH` environment variable is set,
+   *invalidation_mode* will be forced to
+   :attr:`PycInvalidationMode.CHECKED_HASH`.
 
    .. versionchanged:: 3.2
       Changed default value of *cfile* to be :PEP:`3147`-compliant.  Previous
@@ -71,6 +73,9 @@ byte-code cache files in the directory containing the source code.
 
    .. versionchanged:: 3.7
       The *invalidation_mode* parameter was added as specified in :pep:`552`.
+      If the :envvar:`SOURCE_DATE_EPOCH` environment variable is set,
+      *invalidation_mode* will be forced to
+      :attr:`PycInvalidationMode.CHECKED_HASH`.
 
 
 .. class:: PycInvalidationMode

diff --git a/Lib/py_compile.py b/Lib/py_compile.py
index a0f4defdce68c9e719a3a8a64ee53305a20fa052..16dc0a011ffab2a8fe58bac8a8c956add95d76bb 100644 (file)
--- a/Lib/py_compile.py
+++ b/Lib/py_compile.py
@@ -112,6 +112,8 @@ def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1,
     the resulting file would be regular and thus not the same type of file as
     it was previously.
     """
+    if os.environ.get('SOURCE_DATE_EPOCH'):
+        invalidation_mode = PycInvalidationMode.CHECKED_HASH
     if cfile is None:
         if optimize >= 0:
             optimization = optimize if optimize >= 1 else ''

diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py
index bcb686c86b522968b26d8c9e051589988f4bef7c..8fc0b3308c91deeb17084560a1d665f302154fca 100644 (file)
--- a/Lib/test/test_py_compile.py
+++ b/Lib/test/test_py_compile.py
@@ -98,6 +98,18 @@ class PyCompileTests(unittest.TestCase):
         self.assertFalse(os.path.exists(
             importlib.util.cache_from_source(bad_coding)))
 
+    def test_source_date_epoch(self):
+        testtime = 123456789
+        with support.EnvironmentVarGuard() as env:
+            env["SOURCE_DATE_EPOCH"] = str(testtime)
+            py_compile.compile(self.source_path, self.pyc_path)
+        self.assertTrue(os.path.exists(self.pyc_path))
+        self.assertFalse(os.path.exists(self.cache_path))
+        with open(self.pyc_path, 'rb') as fp:
+            flags = importlib._bootstrap_external._classify_pyc(
+                fp.read(), 'test', {})
+        self.assertEqual(flags, 0b11)
+
     @unittest.skipIf(sys.flags.optimize > 0, 'test does not work with -O')
     def test_double_dot_no_clobber(self):
         # http://bugs.python.org/issue22966

diff --git a/Misc/NEWS.d/next/Build/2018-01-16-08-32-49.bpo-29708.YCaHEx.rst b/Misc/NEWS.d/next/Build/2018-01-16-08-32-49.bpo-29708.YCaHEx.rst
new file mode 100644 (file)
index 0000000..e52c001
--- /dev/null
+++ b/Misc/NEWS.d/next/Build/2018-01-16-08-32-49.bpo-29708.YCaHEx.rst
@@ -0,0 +1,2 @@
+If the :envvar:`SOURCE_DATE_EPOCH` environment variable is set,
+:mod:`py_compile` will always create hash-based ``.pyc`` files.