bpo-30822: Exclude tzdata from regrtest --all (#2775) (#2777)

When running the test suite using --use=all / -u all, exclude tzdata
since it makes test_datetime too slow (15-20 min on some buildbots)
which then times out on some buildbots.

-u tzdata must now be enabled explicitly, -u tzdata or -u all,tzdata,
to run all test_datetime tests.

Fix also regrtest command line parser to allow passing -u
extralargefile to run test_zipfile64.

Travis CI: remove -tzdata. Replace -u all,-tzdata,-cpu with -u all,-cpu since tzdata is now excluded from -u all.
(cherry picked from commit 5b392bbaeb9d9b1db961ecfc7315d8c8662c27f6)

diff --git a/.travis.yml b/.travis.yml
index e7b78db48bd829f4db232b7278d1f8b96bf8cae5..cf59d965a871d66357b497d534fb81868a7f147b 100644 (file)
--- a/.travis.yml
+++ b/.travis.yml
@@ -61,7 +61,7 @@ matrix:
             ./venv/bin/python -m pip install -U coverage
       script:
         # Skip tests that re-run the entire test suite.
-        - ./venv/bin/python -m coverage run --pylib -m test -uall,-cpu,-tzdata -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn
+        - ./venv/bin/python -m coverage run --pylib -m test -uall,-cpu -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn
       after_script:  # Probably should be after_success once test suite updated to run under coverage.py.
         # Make the `coverage` command available to Codecov w/ a version of Python that can parse all source files.
         - source ./venv/bin/activate
@@ -94,7 +94,7 @@ script:
   # Only run on Linux as the check only needs to be run once.
   - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./python Tools/scripts/patchcheck.py --travis $TRAVIS_PULL_REQUEST; fi
   # `-r -w` implicitly provided through `make buildbottest`.
-  - make buildbottest TESTOPTS="-j4 -uall,-cpu,-tzdata"
+  - make buildbottest TESTOPTS="-j4 -uall,-cpu"
 
 notifications:
   email: false

diff --git a/Lib/test/libregrtest/__init__.py b/Lib/test/libregrtest/__init__.py
index 7ba0e6e5356166134501a934efd6827b30e05272..3427b51b60af86ad83434e9cda49aaf0002758ee 100644 (file)
--- a/Lib/test/libregrtest/__init__.py
+++ b/Lib/test/libregrtest/__init__.py
@@ -1,5 +1,5 @@
 # We import importlib *ASAP* in order to test #15386
 import importlib
 
-from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES
+from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES, ALL_RESOURCES
 from test.libregrtest.main import main

diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py
index 2315cd59b4e6791f977b56184e1a7aee7cfca490..4999fa734114a13b7a379ee29e01d2e40c80e30a 100644 (file)
--- a/Lib/test/libregrtest/cmdline.py
+++ b/Lib/test/libregrtest/cmdline.py
@@ -127,8 +127,17 @@ Pattern examples:
 """
 
 
-RESOURCE_NAMES = ('audio', 'curses', 'largefile', 'network',
-                  'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui', 'tzdata')
+ALL_RESOURCES = ('audio', 'curses', 'largefile', 'network',
+                 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui')
+
+# Other resources excluded from --use=all:
+#
+# - extralagefile (ex: test_zipfile64): really too slow to be enabled
+#   "by default"
+# - tzdata: while needed to validate fully test_datetime, it makes
+#   test_datetime too slow (15-20 min on some buildbots) and so is disabled by
+#   default (see bpo-30822).
+RESOURCE_NAMES = ALL_RESOURCES + ('extralargefile', 'tzdata')
 
 class _ArgParser(argparse.ArgumentParser):
 
@@ -344,7 +353,7 @@ def _parse_args(args, **kwargs):
         for a in ns.use:
             for r in a:
                 if r == 'all':
-                    ns.use_resources[:] = RESOURCE_NAMES
+                    ns.use_resources[:] = ALL_RESOURCES
                     continue
                 if r == 'none':
                     del ns.use_resources[:]

diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
index a544f880d2be01b3f7f3bd6aaeefc5d2e4bab2bc..f63ed647f80e0b04aa2109d9a3cb616dedfe9b32 100644 (file)
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -191,15 +191,26 @@ class ParseArgsTestCase(unittest.TestCase):
             with self.subTest(opt=opt):
                 ns = libregrtest._parse_args([opt, 'gui,network'])
                 self.assertEqual(ns.use_resources, ['gui', 'network'])
+
                 ns = libregrtest._parse_args([opt, 'gui,none,network'])
                 self.assertEqual(ns.use_resources, ['network'])
-                expected = list(libregrtest.RESOURCE_NAMES)
+
+                expected = list(libregrtest.ALL_RESOURCES)
                 expected.remove('gui')
                 ns = libregrtest._parse_args([opt, 'all,-gui'])
                 self.assertEqual(ns.use_resources, expected)
                 self.checkError([opt], 'expected one argument')
                 self.checkError([opt, 'foo'], 'invalid resource')
 
+                # all + a resource not part of "all"
+                ns = libregrtest._parse_args([opt, 'all,tzdata'])
+                self.assertEqual(ns.use_resources,
+                                 list(libregrtest.ALL_RESOURCES) + ['tzdata'])
+
+                # test another resource which is not part of "all"
+                ns = libregrtest._parse_args([opt, 'extralargefile'])
+                self.assertEqual(ns.use_resources, ['extralargefile'])
+
     def test_memlimit(self):
         for opt in '-M', '--memlimit':
             with self.subTest(opt=opt):

diff --git a/Misc/NEWS.d/next/Tests/2017-07-20-14-29-54.bpo-30822.X0wREo.rst b/Misc/NEWS.d/next/Tests/2017-07-20-14-29-54.bpo-30822.X0wREo.rst
new file mode 100644 (file)
index 0000000..53557f6
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2017-07-20-14-29-54.bpo-30822.X0wREo.rst
@@ -0,0 +1,5 @@
+regrtest: Exclude tzdata from regrtest --all. When running the test suite
+using --use=all / -u all, exclude tzdata since it makes test_datetime too
+slow (15-20 min on some buildbots) which then times out on some buildbots.
+Fix also regrtest command line parser to allow passing -u extralargefile to
+run test_zipfile64.