Start fixing test_bigmem:

- bigmemtest is replaced by precisionbigmemtest
- add a poor man's watchdog thread to print memory consumption

diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py
index a22acc07114259afe5f95d217bf0be5e82526a53..49be72047d019f4ee30df25b81f0786fd5b75aa3 100644 (file)
--- a/Lib/test/pickletester.py
+++ b/Lib/test/pickletester.py
@@ -8,7 +8,7 @@ from http.cookies import SimpleCookie
 
 from test.support import (
     TestFailed, TESTFN, run_with_locale,
-    _2G, _4G, precisionbigmemtest,
+    _2G, _4G, bigmemtest,
     )
 
 from pickle import bytes_types
@@ -1159,7 +1159,7 @@ class BigmemPickleTests(unittest.TestCase):
 
     # Binary protocols can serialize longs of up to 2GB-1
 
-    @precisionbigmemtest(size=_2G, memuse=1 + 1, dry_run=False)
+    @bigmemtest(size=_2G, memuse=1 + 1, dry_run=False)
     def test_huge_long_32b(self, size):
         data = 1 << (8 * size)
         try:
@@ -1175,7 +1175,7 @@ class BigmemPickleTests(unittest.TestCase):
     # (older protocols don't have a dedicated opcode for bytes and are
     # too inefficient)
 
-    @precisionbigmemtest(size=_2G, memuse=1 + 1, dry_run=False)
+    @bigmemtest(size=_2G, memuse=1 + 1, dry_run=False)
     def test_huge_bytes_32b(self, size):
         data = b"abcd" * (size // 4)
         try:
@@ -1191,7 +1191,7 @@ class BigmemPickleTests(unittest.TestCase):
         finally:
             data = None
 
-    @precisionbigmemtest(size=_4G, memuse=1 + 1, dry_run=False)
+    @bigmemtest(size=_4G, memuse=1 + 1, dry_run=False)
     def test_huge_bytes_64b(self, size):
         data = b"a" * size
         try:
@@ -1206,7 +1206,7 @@ class BigmemPickleTests(unittest.TestCase):
     # All protocols use 1-byte per printable ASCII character; we add another
     # byte because the encoded form has to be copied into the internal buffer.
 
-    @precisionbigmemtest(size=_2G, memuse=2 + character_size, dry_run=False)
+    @bigmemtest(size=_2G, memuse=2 + character_size, dry_run=False)
     def test_huge_str_32b(self, size):
         data = "abcd" * (size // 4)
         try:
@@ -1223,7 +1223,7 @@ class BigmemPickleTests(unittest.TestCase):
     # BINUNICODE (protocols 1, 2 and 3) cannot carry more than
     # 2**32 - 1 bytes of utf-8 encoded unicode.
 
-    @precisionbigmemtest(size=_4G, memuse=1 + character_size, dry_run=False)
+    @bigmemtest(size=_4G, memuse=1 + character_size, dry_run=False)
     def test_huge_str_64b(self, size):
         data = "a" * size
         try:

diff --git a/Lib/test/support.py b/Lib/test/support.py
index 8e6ca2a5412cfcd4ed88ed54d85fe1848d7ba125..ab71d470126dd501c34fa56a6a0710a084e0756d 100644 (file)
--- a/Lib/test/support.py
+++ b/Lib/test/support.py
@@ -1053,43 +1053,52 @@ def set_memlimit(limit):
         raise ValueError('Memory limit %r too low to be useful' % (limit,))
     max_memuse = memlimit
 
-def bigmemtest(minsize, memuse):
+def _memory_watchdog(start_evt, finish_evt, period=10.0):
+    """A function which periodically watches the process' memory consumption
+    and prints it out.
+    """
+    # XXX: because of the GIL, and because the very long operations tested
+    # in most bigmem tests are uninterruptible, the loop below gets woken up
+    # much less often than expected.
+    # The polling code should be rewritten in raw C, without holding the GIL,
+    # and push results onto an anonymous pipe.
+    try:
+        page_size = os.sysconf('SC_PAGESIZE')
+    except (ValueError, AttributeError):
+        try:
+            page_size = os.sysconf('SC_PAGE_SIZE')
+        except (ValueError, AttributeError):
+            page_size = 4096
+    procfile = '/proc/{pid}/statm'.format(pid=os.getpid())
+    try:
+        f = open(procfile, 'rb')
+    except IOError as e:
+        warnings.warn('/proc not available for stats: {}'.format(e),
+                      RuntimeWarning)
+        sys.stderr.flush()
+        return
+    with f:
+        start_evt.set()
+        old_data = -1
+        while not finish_evt.wait(period):
+            f.seek(0)
+            statm = f.read().decode('ascii')
+            data = int(statm.split()[5])
+            if data != old_data:
+                old_data = data
+                print(" ... process data size: {data:.1f}G"
+                       .format(data=data * page_size / (1024 ** 3)))
+
+def bigmemtest(size, memuse, dry_run=True):
     """Decorator for bigmem tests.
 
     'minsize' is the minimum useful size for the test (in arbitrary,
     test-interpreted units.) 'memuse' is the number of 'bytes per size' for
     the test, or a good estimate of it.
 
-    The decorator tries to guess a good value for 'size' and passes it to
-    the decorated test function. If minsize * memuse is more than the
-    allowed memory use (as defined by max_memuse), the test is skipped.
-    Otherwise, minsize is adjusted upward to use up to max_memuse.
+    if 'dry_run' is False, it means the test doesn't support dummy runs
+    when -M is not specified.
     """
-    def decorator(f):
-        def wrapper(self):
-            # Retrieve values in case someone decided to adjust them
-            minsize = wrapper.minsize
-            memuse = wrapper.memuse
-            if not max_memuse:
-                # If max_memuse is 0 (the default),
-                # we still want to run the tests with size set to a few kb,
-                # to make sure they work. We still want to avoid using
-                # too much memory, though, but we do that noisily.
-                maxsize = 5147
-                self.assertFalse(maxsize * memuse > 20 * _1M)
-            else:
-                maxsize = int(max_memuse / memuse)
-                if maxsize < minsize:
-                    raise unittest.SkipTest(
-                        "not enough memory: %.1fG minimum needed"
-                        % (minsize * memuse / (1024 ** 3)))
-            return f(self, maxsize)
-        wrapper.minsize = minsize
-        wrapper.memuse = memuse
-        return wrapper
-    return decorator
-
-def precisionbigmemtest(size, memuse, dry_run=True):
     def decorator(f):
         def wrapper(self):
             size = wrapper.size
@@ -1105,7 +1114,28 @@ def precisionbigmemtest(size, memuse, dry_run=True):
                     "not enough memory: %.1fG minimum needed"
                     % (size * memuse / (1024 ** 3)))
 
-            return f(self, maxsize)
+            if real_max_memuse and verbose and threading:
+                print()
+                print(" ... expected peak memory use: {peak:.1f}G"
+                      .format(peak=size * memuse / (1024 ** 3)))
+                sys.stdout.flush()
+                start_evt = threading.Event()
+                finish_evt = threading.Event()
+                t = threading.Thread(target=_memory_watchdog,
+                                     args=(start_evt, finish_evt, 0.5))
+                t.daemon = True
+                t.start()
+                start_evt.set()
+            else:
+                t = None
+
+            try:
+                return f(self, maxsize)
+            finally:
+                if t:
+                    finish_evt.set()
+                    t.join()
+
         wrapper.size = size
         wrapper.memuse = memuse
         return wrapper

diff --git a/Lib/test/test_bigmem.py b/Lib/test/test_bigmem.py
index ac6b1096e1637c4f37ff421d8e305a756c7498b3..f3c6ebbf05fae5831519116cbd20a7fc6da636e8 100644 (file)
--- a/Lib/test/test_bigmem.py
+++ b/Lib/test/test_bigmem.py
@@ -1,5 +1,5 @@
 from test import support
-from test.support import bigmemtest, _1G, _2G, _4G, precisionbigmemtest
+from test.support import bigmemtest, _1G, _2G, _4G
 
 import unittest
 import operator
@@ -25,10 +25,10 @@ import functools
 #    a large object, make the subobject of a length that is not a power of
 #    2. That way, int-wrapping problems are more easily detected.
 #
-#  - While the bigmemtest decorator speaks of 'minsize', all tests will
-#    actually be called with a much smaller number too, in the normal
-#    test run (5Kb currently.) This is so the tests themselves get frequent
-#    testing. Consequently, always make all large allocations based on the
+#  - Despite the bigmemtest decorator, all tests will actually be called
+#    with a much smaller number too, in the normal test run (5Kb currently.)
+#    This is so the tests themselves get frequent testing.
+#    Consequently, always make all large allocations based on the
 #    passed-in 'size', and don't rely on the size being very large. Also,
 #    memuse-per-size should remain sane (less than a few thousand); if your
 #    test uses more, adjust 'size' upward, instead.
@@ -42,7 +42,7 @@ character_size = 4 if sys.maxunicode > 0xFFFF else 2
 
 class BaseStrTest:
 
-    @bigmemtest(minsize=_2G, memuse=2)
+    @bigmemtest(size=_2G, memuse=2)
     def test_capitalize(self, size):
         _ = self.from_latin1
         SUBSTR = self.from_latin1(' abc def ghi')
@@ -52,7 +52,7 @@ class BaseStrTest:
                          SUBSTR.capitalize())
         self.assertEqual(caps.lstrip(_('-')), SUBSTR)
 
-    @bigmemtest(minsize=_2G + 10, memuse=1)
+    @bigmemtest(size=_2G + 10, memuse=1)
     def test_center(self, size):
         SUBSTR = self.from_latin1(' abc def ghi')
         s = SUBSTR.center(size)
@@ -63,7 +63,7 @@ class BaseStrTest:
         self.assertEqual(s[lpadsize:-rpadsize], SUBSTR)
         self.assertEqual(s.strip(), SUBSTR.strip())
 
-    @bigmemtest(minsize=_2G, memuse=2)
+    @bigmemtest(size=_2G, memuse=2)
     def test_count(self, size):
         _ = self.from_latin1
         SUBSTR = _(' abc def ghi')
@@ -75,7 +75,7 @@ class BaseStrTest:
         self.assertEqual(s.count(_('i')), 1)
         self.assertEqual(s.count(_('j')), 0)
 
-    @bigmemtest(minsize=_2G, memuse=2)
+    @bigmemtest(size=_2G, memuse=2)
     def test_endswith(self, size):
         _ = self.from_latin1
         SUBSTR = _(' abc def ghi')
@@ -87,7 +87,7 @@ class BaseStrTest:
         self.assertFalse(s.endswith(_('a') + SUBSTR))
         self.assertFalse(SUBSTR.endswith(s))
 
-    @bigmemtest(minsize=_2G + 10, memuse=2)
+    @bigmemtest(size=_2G + 10, memuse=2)
     def test_expandtabs(self, size):
         _ = self.from_latin1
         s = _('-') * size
@@ -100,7 +100,7 @@ class BaseStrTest:
         self.assertEqual(len(s), size - remainder)
         self.assertEqual(len(s.strip(_(' '))), 0)
 
-    @bigmemtest(minsize=_2G, memuse=2)
+    @bigmemtest(size=_2G, memuse=2)
     def test_find(self, size):
         _ = self.from_latin1
         SUBSTR = _(' abc def ghi')
@@ -117,7 +117,7 @@ class BaseStrTest:
                          sublen + size + SUBSTR.find(_('i')))
         self.assertEqual(s.find(_('j')), -1)
 
-    @bigmemtest(minsize=_2G, memuse=2)
+    @bigmemtest(size=_2G, memuse=2)
     def test_index(self, size):
         _ = self.from_latin1
         SUBSTR = _(' abc def ghi')
@@ -134,7 +134,7 @@ class BaseStrTest:
                          sublen + size + SUBSTR.index(_('i')))
         self.assertRaises(ValueError, s.index, _('j'))
 
-    @bigmemtest(minsize=_2G, memuse=2)
+    @bigmemtest(size=_2G, memuse=2)
     def test_isalnum(self, size):
         _ = self.from_latin1
         SUBSTR = _('123456')
@@ -143,7 +143,7 @@ class BaseStrTest:
         s += _('.')
         self.assertFalse(s.isalnum())
 
-    @bigmemtest(minsize=_2G, memuse=2)
+    @bigmemtest(size=_2G, memuse=2)
     def test_isalpha(self, size):
         _ = self.from_latin1
         SUBSTR = _('zzzzzzz')
@@ -152,7 +152,7 @@ class BaseStrTest:
         s += _('.')
         self.assertFalse(s.isalpha())
 
-    @bigmemtest(minsize=_2G, memuse=2)
+    @bigmemtest(size=_2G, memuse=2)
     def test_isdigit(self, size):
         _ = self.from_latin1
         SUBSTR = _('123456')
@@ -161,7 +161,7 @@ class BaseStrTest:
         s += _('z')
         self.assertFalse(s.isdigit())
 
-    @bigmemtest(minsize=_2G, memuse=2)
+    @bigmemtest(size=_2G, memuse=2)
     def test_islower(self, size):
         _ = self.from_latin1
         chars = _(''.join(
@@ -172,7 +172,7 @@ class BaseStrTest:
         s += _('A')
         self.assertFalse(s.islower())
 
-    @bigmemtest(minsize=_2G, memuse=2)
+    @bigmemtest(size=_2G, memuse=2)
     def test_isspace(self, size):
         _ = self.from_latin1
         whitespace = _(' \f\n\r\t\v')
@@ -182,7 +182,7 @@ class BaseStrTest:
         s += _('j')
         self.assertFalse(s.isspace())
 
-    @bigmemtest(minsize=_2G, memuse=2)
+    @bigmemtest(size=_2G, memuse=2)
     def test_istitle(self, size):
         _ = self.from_latin1
         SUBSTR = _('123456')
@@ -193,7 +193,7 @@ class BaseStrTest:
         s += _('aA')
         self.assertFalse(s.istitle())
 
-    @bigmemtest(minsize=_2G, memuse=2)
+    @bigmemtest(size=_2G, memuse=2)
     def test_isupper(self, size):
         _ = self.from_latin1
         chars = _(''.join(
@@ -204,7 +204,7 @@ class BaseStrTest:
         s += _('a')
         self.assertFalse(s.isupper())
 
-    @bigmemtest(minsize=_2G, memuse=2)
+    @bigmemtest(size=_2G, memuse=2)
     def test_join(self, size):
         _ = self.from_latin1
         s = _('A') * size
@@ -214,7 +214,7 @@ class BaseStrTest:
         self.assertTrue(x.startswith(_('aaaaaA')))
         self.assertTrue(x.endswith(_('Abbbbb')))
 
-    @bigmemtest(minsize=_2G + 10, memuse=1)
+    @bigmemtest(size=_2G + 10, memuse=1)
     def test_ljust(self, size):
         _ = self.from_latin1
         SUBSTR = _(' abc def ghi')
@@ -223,7 +223,7 @@ class BaseStrTest:
         self.assertEqual(len(s), size)
         self.assertEqual(s.strip(), SUBSTR.strip())
 
-    @bigmemtest(minsize=_2G + 10, memuse=2)
+    @bigmemtest(size=_2G + 10, memuse=2)
     def test_lower(self, size):
         _ = self.from_latin1
         s = _('A') * size
@@ -231,7 +231,7 @@ class BaseStrTest:
         self.assertEqual(len(s), size)
         self.assertEqual(s.count(_('a')), size)
 
-    @bigmemtest(minsize=_2G + 10, memuse=1)
+    @bigmemtest(size=_2G + 10, memuse=1)
     def test_lstrip(self, size):
         _ = self.from_latin1
         SUBSTR = _('abc def ghi')
@@ -246,7 +246,7 @@ class BaseStrTest:
             stripped = s.lstrip()
             self.assertTrue(stripped is s)
 
-    @bigmemtest(minsize=_2G + 10, memuse=2)
+    @bigmemtest(size=_2G + 10, memuse=2)
     def test_replace(self, size):
         _ = self.from_latin1
         replacement = _('a')
@@ -259,7 +259,7 @@ class BaseStrTest:
         self.assertEqual(s.count(replacement), 4)
         self.assertEqual(s[-10:], _('      aaaa'))
 
-    @bigmemtest(minsize=_2G, memuse=2)
+    @bigmemtest(size=_2G, memuse=2)
     def test_rfind(self, size):
         _ = self.from_latin1
         SUBSTR = _(' abc def ghi')
@@ -275,7 +275,7 @@ class BaseStrTest:
                          SUBSTR.rfind(_('i')))
         self.assertEqual(s.rfind(_('j')), -1)
 
-    @bigmemtest(minsize=_2G, memuse=2)
+    @bigmemtest(size=_2G, memuse=2)
     def test_rindex(self, size):
         _ = self.from_latin1
         SUBSTR = _(' abc def ghi')
@@ -294,7 +294,7 @@ class BaseStrTest:
                          SUBSTR.rindex(_('i')))
         self.assertRaises(ValueError, s.rindex, _('j'))
 
-    @bigmemtest(minsize=_2G + 10, memuse=1)
+    @bigmemtest(size=_2G + 10, memuse=1)
     def test_rjust(self, size):
         _ = self.from_latin1
         SUBSTR = _(' abc def ghi')
@@ -303,7 +303,7 @@ class BaseStrTest:
         self.assertEqual(len(s), size)
         self.assertEqual(s.strip(), SUBSTR.strip())
 
-    @bigmemtest(minsize=_2G + 10, memuse=1)
+    @bigmemtest(size=_2G + 10, memuse=1)
     def test_rstrip(self, size):
         _ = self.from_latin1
         SUBSTR = _(' abc def ghi')
@@ -321,7 +321,7 @@ class BaseStrTest:
     # The test takes about size bytes to build a string, and then about
     # sqrt(size) substrings of sqrt(size) in size and a list to
     # hold sqrt(size) items. It's close but just over 2x size.
-    @bigmemtest(minsize=_2G, memuse=2.1)
+    @bigmemtest(size=_2G, memuse=2.1)
     def test_split_small(self, size):
         _ = self.from_latin1
         # Crudely calculate an estimate so that the result of s.split won't
@@ -347,7 +347,7 @@ class BaseStrTest:
     # suffer for the list size. (Otherwise, it'd cost another 48 times
     # size in bytes!) Nevertheless, a list of size takes
     # 8*size bytes.
-    @bigmemtest(minsize=_2G + 5, memuse=10)
+    @bigmemtest(size=_2G + 5, memuse=10)
     def test_split_large(self, size):
         _ = self.from_latin1
         s = _(' a') * size + _(' ')
@@ -359,7 +359,7 @@ class BaseStrTest:
         self.assertEqual(len(l), size + 1)
         self.assertEqual(set(l), set([_(' ')]))
 
-    @bigmemtest(minsize=_2G, memuse=2.1)
+    @bigmemtest(size=_2G, memuse=2.1)
     def test_splitlines(self, size):
         _ = self.from_latin1
         # Crudely calculate an estimate so that the result of s.split won't
@@ -373,7 +373,7 @@ class BaseStrTest:
         for item in l:
             self.assertEqual(item, expected)
 
-    @bigmemtest(minsize=_2G, memuse=2)
+    @bigmemtest(size=_2G, memuse=2)
     def test_startswith(self, size):
         _ = self.from_latin1
         SUBSTR = _(' abc def ghi')
@@ -382,7 +382,7 @@ class BaseStrTest:
         self.assertTrue(s.startswith(_('-') * size))
         self.assertFalse(s.startswith(SUBSTR))
 
-    @bigmemtest(minsize=_2G, memuse=1)
+    @bigmemtest(size=_2G, memuse=1)
     def test_strip(self, size):
         _ = self.from_latin1
         SUBSTR = _('   abc def ghi   ')
@@ -394,7 +394,7 @@ class BaseStrTest:
         self.assertEqual(len(s), size)
         self.assertEqual(s.strip(), SUBSTR.strip())
 
-    @bigmemtest(minsize=_2G, memuse=2)
+    @bigmemtest(size=_2G, memuse=2)
     def test_swapcase(self, size):
         _ = self.from_latin1
         SUBSTR = _("aBcDeFG12.'\xa9\x00")
@@ -406,7 +406,7 @@ class BaseStrTest:
         self.assertEqual(s[:sublen * 3], SUBSTR.swapcase() * 3)
         self.assertEqual(s[-sublen * 3:], SUBSTR.swapcase() * 3)
 
-    @bigmemtest(minsize=_2G, memuse=2)
+    @bigmemtest(size=_2G, memuse=2)
     def test_title(self, size):
         _ = self.from_latin1
         SUBSTR = _('SpaaHAaaAaham')
@@ -415,7 +415,7 @@ class BaseStrTest:
         self.assertTrue(s.startswith((SUBSTR * 3).title()))
         self.assertTrue(s.endswith(SUBSTR.lower() * 3))
 
-    @bigmemtest(minsize=_2G, memuse=2)
+    @bigmemtest(size=_2G, memuse=2)
     def test_translate(self, size):
         _ = self.from_latin1
         SUBSTR = _('aZz.z.Aaz.')
@@ -438,7 +438,7 @@ class BaseStrTest:
         self.assertEqual(s.count(_('!')), repeats * 2)
         self.assertEqual(s.count(_('z')), repeats * 3)
 
-    @bigmemtest(minsize=_2G + 5, memuse=2)
+    @bigmemtest(size=_2G + 5, memuse=2)
     def test_upper(self, size):
         _ = self.from_latin1
         s = _('a') * size
@@ -446,7 +446,7 @@ class BaseStrTest:
         self.assertEqual(len(s), size)
         self.assertEqual(s.count(_('A')), size)
 
-    @bigmemtest(minsize=_2G + 20, memuse=1)
+    @bigmemtest(size=_2G + 20, memuse=1)
     def test_zfill(self, size):
         _ = self.from_latin1
         SUBSTR = _('-568324723598234')
@@ -458,7 +458,7 @@ class BaseStrTest:
 
     # This test is meaningful even with size < 2G, as long as the
     # doubled string is > 2G (but it tests more if both are > 2G :)
-    @bigmemtest(minsize=_1G + 2, memuse=3)
+    @bigmemtest(size=_1G + 2, memuse=3)
     def test_concat(self, size):
         _ = self.from_latin1
         s = _('.') * size
@@ -469,7 +469,7 @@ class BaseStrTest:
 
     # This test is meaningful even with size < 2G, as long as the
     # repeated string is > 2G (but it tests more if both are > 2G :)
-    @bigmemtest(minsize=_1G + 2, memuse=3)
+    @bigmemtest(size=_1G + 2, memuse=3)
     def test_repeat(self, size):
         _ = self.from_latin1
         s = _('.') * size
@@ -478,7 +478,7 @@ class BaseStrTest:
         self.assertEqual(len(s), size * 2)
         self.assertEqual(s.count(_('.')), size * 2)
 
-    @bigmemtest(minsize=_2G + 20, memuse=2)
+    @bigmemtest(size=_2G + 20, memuse=2)
     def test_slice_and_getitem(self, size):
         _ = self.from_latin1
         SUBSTR = _('0123456789')
@@ -512,7 +512,7 @@ class BaseStrTest:
         self.assertRaises(IndexError, operator.getitem, s, len(s) + 1)
         self.assertRaises(IndexError, operator.getitem, s, len(s) + 1<<31)
 
-    @bigmemtest(minsize=_2G, memuse=2)
+    @bigmemtest(size=_2G, memuse=2)
     def test_contains(self, size):
         _ = self.from_latin1
         SUBSTR = _('0123456789')
@@ -526,7 +526,7 @@ class BaseStrTest:
         s += _('a')
         self.assertIn(_('a'), s)
 
-    @bigmemtest(minsize=_2G + 10, memuse=2)
+    @bigmemtest(size=_2G + 10, memuse=2)
     def test_compare(self, size):
         _ = self.from_latin1
         s1 = _('-') * size
@@ -539,7 +539,7 @@ class BaseStrTest:
         s2 = _('.') * size
         self.assertFalse(s1 == s2)
 
-    @bigmemtest(minsize=_2G + 10, memuse=1)
+    @bigmemtest(size=_2G + 10, memuse=1)
     def test_hash(self, size):
         # Not sure if we can do any meaningful tests here...  Even if we
         # start relying on the exact algorithm used, the result will be
@@ -590,46 +590,36 @@ class StrTest(unittest.TestCase, BaseStrTest):
             getattr(type(self), name).memuse = memuse
 
     # the utf8 encoder preallocates big time (4x the number of characters)
-    @bigmemtest(minsize=_2G + 2, memuse=character_size + 4)
+    @bigmemtest(size=_2G + 2, memuse=character_size + 4)
     def test_encode(self, size):
         return self.basic_encode_test(size, 'utf-8')
 
-    @precisionbigmemtest(size=_4G // 6 + 2, memuse=character_size + 1)
+    @bigmemtest(size=_4G // 6 + 2, memuse=character_size + 1)
     def test_encode_raw_unicode_escape(self, size):
         try:
             return self.basic_encode_test(size, 'raw_unicode_escape')
         except MemoryError:
             pass # acceptable on 32-bit
 
-    @precisionbigmemtest(size=_4G // 5 + 70, memuse=character_size + 1)
+    @bigmemtest(size=_4G // 5 + 70, memuse=character_size + 1)
     def test_encode_utf7(self, size):
         try:
             return self.basic_encode_test(size, 'utf7')
         except MemoryError:
             pass # acceptable on 32-bit
 
-    @precisionbigmemtest(size=_4G // 4 + 5, memuse=character_size + 4)
+    @bigmemtest(size=_4G // 4 + 5, memuse=character_size + 4)
     def test_encode_utf32(self, size):
         try:
             return self.basic_encode_test(size, 'utf32', expectedsize=4*size+4)
         except MemoryError:
             pass # acceptable on 32-bit
 
-    @precisionbigmemtest(size=_2G - 1, memuse=character_size + 1)
+    @bigmemtest(size=_2G - 1, memuse=character_size + 1)
     def test_encode_ascii(self, size):
         return self.basic_encode_test(size, 'ascii', c='A')
 
-    @precisionbigmemtest(size=_4G // 5, memuse=character_size * (6 + 1))
-    def test_unicode_repr_overflow(self, size):
-        try:
-            s = "\uDCBA"*size
-            r = repr(s)
-        except MemoryError:
-            pass # acceptable on 32-bit
-        else:
-            self.assertTrue(s == eval(r))
-
-    @bigmemtest(minsize=_2G + 10, memuse=character_size * 2)
+    @bigmemtest(size=_2G + 10, memuse=character_size * 2)
     def test_format(self, size):
         s = '-' * size
         sf = '%s' % (s,)
@@ -650,7 +640,7 @@ class StrTest(unittest.TestCase, BaseStrTest):
         self.assertEqual(s.count('.'), 3)
         self.assertEqual(s.count('-'), size * 2)
 
-    @bigmemtest(minsize=_2G + 10, memuse=character_size * 2)
+    @bigmemtest(size=_2G + 10, memuse=character_size * 2)
     def test_repr_small(self, size):
         s = '-' * size
         s = repr(s)
@@ -671,7 +661,7 @@ class StrTest(unittest.TestCase, BaseStrTest):
         self.assertEqual(s.count('\\'), size)
         self.assertEqual(s.count('0'), size * 2)
 
-    @bigmemtest(minsize=_2G + 10, memuse=character_size * 5)
+    @bigmemtest(size=_2G + 10, memuse=character_size * 5)
     def test_repr_large(self, size):
         s = '\x00' * size
         s = repr(s)
@@ -681,27 +671,46 @@ class StrTest(unittest.TestCase, BaseStrTest):
         self.assertEqual(s.count('\\'), size)
         self.assertEqual(s.count('0'), size * 2)
 
-    @bigmemtest(minsize=2**32 / 5, memuse=character_size * 7)
+    @bigmemtest(size=_2G // 5 + 1, memuse=character_size * 7)
     def test_unicode_repr(self, size):
         # Use an assigned, but not printable code point.
         # It is in the range of the low surrogates \uDC00-\uDFFF.
-        s = "\uDCBA" * size
-        for f in (repr, ascii):
-            r = f(s)
-            self.assertTrue(len(r) > size)
-            self.assertTrue(r.endswith(r"\udcba'"), r[-10:])
-            del r
+        char = "\uDCBA"
+        s = char * size
+        try:
+            for f in (repr, ascii):
+                r = f(s)
+                self.assertEqual(len(r), 2 + (len(f(char)) - 2) * size)
+                self.assertTrue(r.endswith(r"\udcba'"), r[-10:])
+                r = None
+        finally:
+            r = s = None
 
     # The character takes 4 bytes even in UCS-2 builds because it will
     # be decomposed into surrogates.
-    @bigmemtest(minsize=2**32 / 5, memuse=4 + character_size * 9)
+    @bigmemtest(size=_2G // 5 + 1, memuse=4 + character_size * 9)
     def test_unicode_repr_wide(self, size):
-        s = "\U0001DCBA" * size
-        for f in (repr, ascii):
-            r = f(s)
-            self.assertTrue(len(r) > size)
-            self.assertTrue(r.endswith(r"\U0001dcba'"), r[-12:])
-            del r
+        char = "\U0001DCBA"
+        s = char * size
+        try:
+            for f in (repr, ascii):
+                r = f(s)
+                self.assertEqual(len(r), 2 + (len(f(char)) - 2) * size)
+                self.assertTrue(r.endswith(r"\U0001dcba'"), r[-12:])
+                r = None
+        finally:
+            r = s = None
+
+    @bigmemtest(size=_4G // 5, memuse=character_size * (6 + 1))
+    def _test_unicode_repr_overflow(self, size):
+        # XXX not sure what this test is about
+        char = "\uDCBA"
+        s = char * size
+        try:
+            r = repr(s)
+            self.assertTrue(s == eval(r))
+        finally:
+            r = s = None
 
 
 class BytesTest(unittest.TestCase, BaseStrTest):
@@ -709,7 +718,7 @@ class BytesTest(unittest.TestCase, BaseStrTest):
     def from_latin1(self, s):
         return s.encode("latin1")
 
-    @bigmemtest(minsize=_2G + 2, memuse=1 + character_size)
+    @bigmemtest(size=_2G + 2, memuse=1 + character_size)
     def test_decode(self, size):
         s = self.from_latin1('.') * size
         self.assertEqual(len(s.decode('utf-8')), size)
@@ -720,7 +729,7 @@ class BytearrayTest(unittest.TestCase, BaseStrTest):
     def from_latin1(self, s):
         return bytearray(s.encode("latin1"))
 
-    @bigmemtest(minsize=_2G + 2, memuse=1 + character_size)
+    @bigmemtest(size=_2G + 2, memuse=1 + character_size)
     def test_decode(self, size):
         s = self.from_latin1('.') * size
         self.assertEqual(len(s.decode('utf-8')), size)
@@ -739,7 +748,7 @@ class TupleTest(unittest.TestCase):
     # having more than 2<<31 references to any given object. Hence the
     # use of different types of objects as contents in different tests.
 
-    @bigmemtest(minsize=_2G + 2, memuse=16)
+    @bigmemtest(size=_2G + 2, memuse=16)
     def test_compare(self, size):
         t1 = ('',) * size
         t2 = ('',) * size
@@ -762,15 +771,15 @@ class TupleTest(unittest.TestCase):
         t = t + t
         self.assertEqual(len(t), size * 2)
 
-    @bigmemtest(minsize=_2G // 2 + 2, memuse=24)
+    @bigmemtest(size=_2G // 2 + 2, memuse=24)
     def test_concat_small(self, size):
         return self.basic_concat_test(size)
 
-    @bigmemtest(minsize=_2G + 2, memuse=24)
+    @bigmemtest(size=_2G + 2, memuse=24)
     def test_concat_large(self, size):
         return self.basic_concat_test(size)
 
-    @bigmemtest(minsize=_2G // 5 + 10, memuse=8 * 5)
+    @bigmemtest(size=_2G // 5 + 10, memuse=8 * 5)
     def test_contains(self, size):
         t = (1, 2, 3, 4, 5) * size
         self.assertEqual(len(t), size * 5)
@@ -778,7 +787,7 @@ class TupleTest(unittest.TestCase):
         self.assertNotIn((1, 2, 3, 4, 5), t)
         self.assertNotIn(0, t)
 
-    @bigmemtest(minsize=_2G + 10, memuse=8)
+    @bigmemtest(size=_2G + 10, memuse=8)
     def test_hash(self, size):
         t1 = (0,) * size
         h1 = hash(t1)
@@ -786,7 +795,7 @@ class TupleTest(unittest.TestCase):
         t2 = (0,) * (size + 1)
         self.assertFalse(h1 == hash(t2))
 
-    @bigmemtest(minsize=_2G + 10, memuse=8)
+    @bigmemtest(size=_2G + 10, memuse=8)
     def test_index_and_slice(self, size):
         t = (None,) * size
         self.assertEqual(len(t), size)
@@ -811,19 +820,19 @@ class TupleTest(unittest.TestCase):
         t = t * 2
         self.assertEqual(len(t), size * 2)
 
-    @bigmemtest(minsize=_2G // 2 + 2, memuse=24)
+    @bigmemtest(size=_2G // 2 + 2, memuse=24)
     def test_repeat_small(self, size):
         return self.basic_test_repeat(size)
 
-    @bigmemtest(minsize=_2G + 2, memuse=24)
+    @bigmemtest(size=_2G + 2, memuse=24)
     def test_repeat_large(self, size):
         return self.basic_test_repeat(size)
 
-    @bigmemtest(minsize=_1G - 1, memuse=12)
+    @bigmemtest(size=_1G - 1, memuse=12)
     def test_repeat_large_2(self, size):
         return self.basic_test_repeat(size)
 
-    @precisionbigmemtest(size=_1G - 1, memuse=9)
+    @bigmemtest(size=_1G - 1, memuse=9)
     def test_from_2G_generator(self, size):
         self.skipTest("test needs much more memory than advertised, see issue5438")
         try:
@@ -837,7 +846,7 @@ class TupleTest(unittest.TestCase):
                 count += 1
             self.assertEqual(count, size)
 
-    @precisionbigmemtest(size=_1G - 25, memuse=9)
+    @bigmemtest(size=_1G - 25, memuse=9)
     def test_from_almost_2G_generator(self, size):
         self.skipTest("test needs much more memory than advertised, see issue5438")
         try:
@@ -860,11 +869,11 @@ class TupleTest(unittest.TestCase):
         self.assertEqual(s[-5:], '0, 0)')
         self.assertEqual(s.count('0'), size)
 
-    @bigmemtest(minsize=_2G // 3 + 2, memuse=8 + 3 * character_size)
+    @bigmemtest(size=_2G // 3 + 2, memuse=8 + 3 * character_size)
     def test_repr_small(self, size):
         return self.basic_test_repr(size)
 
-    @bigmemtest(minsize=_2G + 2, memuse=8 + 3 * character_size)
+    @bigmemtest(size=_2G + 2, memuse=8 + 3 * character_size)
     def test_repr_large(self, size):
         return self.basic_test_repr(size)
 
@@ -875,7 +884,7 @@ class ListTest(unittest.TestCase):
     # lists hold references to various objects to test their refcount
     # limits.
 
-    @bigmemtest(minsize=_2G + 2, memuse=16)
+    @bigmemtest(size=_2G + 2, memuse=16)
     def test_compare(self, size):
         l1 = [''] * size
         l2 = [''] * size
@@ -898,11 +907,11 @@ class ListTest(unittest.TestCase):
         l = l + l
         self.assertEqual(len(l), size * 2)
 
-    @bigmemtest(minsize=_2G // 2 + 2, memuse=24)
+    @bigmemtest(size=_2G // 2 + 2, memuse=24)
     def test_concat_small(self, size):
         return self.basic_test_concat(size)
 
-    @bigmemtest(minsize=_2G + 2, memuse=24)
+    @bigmemtest(size=_2G + 2, memuse=24)
     def test_concat_large(self, size):
         return self.basic_test_concat(size)
 
@@ -913,15 +922,15 @@ class ListTest(unittest.TestCase):
         self.assertTrue(l[0] is l[-1])
         self.assertTrue(l[size - 1] is l[size + 1])
 
-    @bigmemtest(minsize=_2G // 2 + 2, memuse=24)
+    @bigmemtest(size=_2G // 2 + 2, memuse=24)
     def test_inplace_concat_small(self, size):
         return self.basic_test_inplace_concat(size)
 
-    @bigmemtest(minsize=_2G + 2, memuse=24)
+    @bigmemtest(size=_2G + 2, memuse=24)
     def test_inplace_concat_large(self, size):
         return self.basic_test_inplace_concat(size)
 
-    @bigmemtest(minsize=_2G // 5 + 10, memuse=8 * 5)
+    @bigmemtest(size=_2G // 5 + 10, memuse=8 * 5)
     def test_contains(self, size):
         l = [1, 2, 3, 4, 5] * size
         self.assertEqual(len(l), size * 5)
@@ -929,12 +938,12 @@ class ListTest(unittest.TestCase):
         self.assertNotIn([1, 2, 3, 4, 5], l)
         self.assertNotIn(0, l)
 
-    @bigmemtest(minsize=_2G + 10, memuse=8)
+    @bigmemtest(size=_2G + 10, memuse=8)
     def test_hash(self, size):
         l = [0] * size
         self.assertRaises(TypeError, hash, l)
 
-    @bigmemtest(minsize=_2G + 10, memuse=8)
+    @bigmemtest(size=_2G + 10, memuse=8)
     def test_index_and_slice(self, size):
         l = [None] * size
         self.assertEqual(len(l), size)
@@ -998,11 +1007,11 @@ class ListTest(unittest.TestCase):
         l = l * 2
         self.assertEqual(len(l), size * 2)
 
-    @bigmemtest(minsize=_2G // 2 + 2, memuse=24)
+    @bigmemtest(size=_2G // 2 + 2, memuse=24)
     def test_repeat_small(self, size):
         return self.basic_test_repeat(size)
 
-    @bigmemtest(minsize=_2G + 2, memuse=24)
+    @bigmemtest(size=_2G + 2, memuse=24)
     def test_repeat_large(self, size):
         return self.basic_test_repeat(size)
 
@@ -1018,11 +1027,11 @@ class ListTest(unittest.TestCase):
         self.assertEqual(len(l), size * 2)
         self.assertTrue(l[size - 1] is l[-1])
 
-    @bigmemtest(minsize=_2G // 2 + 2, memuse=16)
+    @bigmemtest(size=_2G // 2 + 2, memuse=16)
     def test_inplace_repeat_small(self, size):
         return self.basic_test_inplace_repeat(size)
 
-    @bigmemtest(minsize=_2G + 2, memuse=16)
+    @bigmemtest(size=_2G + 2, memuse=16)
     def test_inplace_repeat_large(self, size):
         return self.basic_test_inplace_repeat(size)
 
@@ -1035,17 +1044,17 @@ class ListTest(unittest.TestCase):
         self.assertEqual(s[-5:], '0, 0]')
         self.assertEqual(s.count('0'), size)
 
-    @bigmemtest(minsize=_2G // 3 + 2, memuse=8 + 3 * character_size)
+    @bigmemtest(size=_2G // 3 + 2, memuse=8 + 3 * character_size)
     def test_repr_small(self, size):
         return self.basic_test_repr(size)
 
-    @bigmemtest(minsize=_2G + 2, memuse=8 + 3 * character_size)
+    @bigmemtest(size=_2G + 2, memuse=8 + 3 * character_size)
     def test_repr_large(self, size):
         return self.basic_test_repr(size)
 
     # list overallocates ~1/8th of the total size (on first expansion) so
     # the single list.append call puts memuse at 9 bytes per size.
-    @bigmemtest(minsize=_2G, memuse=9)
+    @bigmemtest(size=_2G, memuse=9)
     def test_append(self, size):
         l = [object()] * size
         l.append(object())
@@ -1053,7 +1062,7 @@ class ListTest(unittest.TestCase):
         self.assertTrue(l[-3] is l[-2])
         self.assertFalse(l[-2] is l[-1])
 
-    @bigmemtest(minsize=_2G // 5 + 2, memuse=8 * 5)
+    @bigmemtest(size=_2G // 5 + 2, memuse=8 * 5)
     def test_count(self, size):
         l = [1, 2, 3, 4, 5] * size
         self.assertEqual(l.count(1), size)
@@ -1066,15 +1075,15 @@ class ListTest(unittest.TestCase):
         self.assertTrue(l[0] is l[-1])
         self.assertTrue(l[size - 1] is l[size + 1])
 
-    @bigmemtest(minsize=_2G // 2 + 2, memuse=16)
+    @bigmemtest(size=_2G // 2 + 2, memuse=16)
     def test_extend_small(self, size):
         return self.basic_test_extend(size)
 
-    @bigmemtest(minsize=_2G + 2, memuse=16)
+    @bigmemtest(size=_2G + 2, memuse=16)
     def test_extend_large(self, size):
         return self.basic_test_extend(size)
 
-    @bigmemtest(minsize=_2G // 5 + 2, memuse=8 * 5)
+    @bigmemtest(size=_2G // 5 + 2, memuse=8 * 5)
     def test_index(self, size):
         l = [1, 2, 3, 4, 5] * size
         size *= 5
@@ -1085,7 +1094,7 @@ class ListTest(unittest.TestCase):
         self.assertRaises(ValueError, l.index, 6)
 
     # This tests suffers from overallocation, just like test_append.
-    @bigmemtest(minsize=_2G + 10, memuse=9)
+    @bigmemtest(size=_2G + 10, memuse=9)
     def test_insert(self, size):
         l = [1.0] * size
         l.insert(size - 1, "A")
@@ -1104,7 +1113,7 @@ class ListTest(unittest.TestCase):
         self.assertEqual(l[:3], [1.0, "C", 1.0])
         self.assertEqual(l[size - 3:], ["A", 1.0, "B"])
 
-    @bigmemtest(minsize=_2G // 5 + 4, memuse=8 * 5)
+    @bigmemtest(size=_2G // 5 + 4, memuse=8 * 5)
     def test_pop(self, size):
         l = ["a", "b", "c", "d", "e"] * size
         size *= 5
@@ -1128,7 +1137,7 @@ class ListTest(unittest.TestCase):
         self.assertEqual(item, "c")
         self.assertEqual(l[-2:], ["b", "d"])
 
-    @bigmemtest(minsize=_2G + 10, memuse=8)
+    @bigmemtest(size=_2G + 10, memuse=8)
     def test_remove(self, size):
         l = [10] * size
         self.assertEqual(len(l), size)
@@ -1148,7 +1157,7 @@ class ListTest(unittest.TestCase):
         self.assertEqual(len(l), size)
         self.assertEqual(l[-2:], [10, 10])
 
-    @bigmemtest(minsize=_2G // 5 + 2, memuse=8 * 5)
+    @bigmemtest(size=_2G // 5 + 2, memuse=8 * 5)
     def test_reverse(self, size):
         l = [1, 2, 3, 4, 5] * size
         l.reverse()
@@ -1156,7 +1165,7 @@ class ListTest(unittest.TestCase):
         self.assertEqual(l[-5:], [5, 4, 3, 2, 1])
         self.assertEqual(l[:5], [5, 4, 3, 2, 1])
 
-    @bigmemtest(minsize=_2G // 5 + 2, memuse=8 * 5)
+    @bigmemtest(size=_2G // 5 + 2, memuse=8 * 5)
     def test_sort(self, size):
         l = [1, 2, 3, 4, 5] * size
         l.sort()

diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py
index 17d752b893e47a01859cafd1e7996765b86879e1..97981dd20d55f7e3fa48049505f18f16cb88ad35 100644 (file)
--- a/Lib/test/test_hashlib.py
+++ b/Lib/test/test_hashlib.py
@@ -17,7 +17,7 @@ except ImportError:
 import unittest
 import warnings
 from test import support
-from test.support import _4G, precisionbigmemtest
+from test.support import _4G, bigmemtest
 
 # Were we compiled --with-pydebug or with #define Py_DEBUG?
 COMPILED_WITH_PYDEBUG = hasattr(sys, 'gettotalrefcount')
@@ -196,7 +196,7 @@ class HashLibTestCase(unittest.TestCase):
                    b'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789',
                    'd174ab98d277d9f5a5611c2c9f419d9f')
 
-    @precisionbigmemtest(size=_4G + 5, memuse=1)
+    @bigmemtest(size=_4G + 5, memuse=1)
     def test_case_md5_huge(self, size):
         if size == _4G + 5:
             try:
@@ -204,7 +204,7 @@ class HashLibTestCase(unittest.TestCase):
             except OverflowError:
                 pass # 32-bit arch
 
-    @precisionbigmemtest(size=_4G - 1, memuse=1)
+    @bigmemtest(size=_4G - 1, memuse=1)
     def test_case_md5_uintmax(self, size):
         if size == _4G - 1:
             try:

diff --git a/Lib/test/test_xml_etree_c.py b/Lib/test/test_xml_etree_c.py
index db69e5f2c5635fad2edee0fa2bd5449210e33dfc..2ff118fae88226a25608d7aa4c7d817afc353340 100644 (file)
--- a/Lib/test/test_xml_etree_c.py
+++ b/Lib/test/test_xml_etree_c.py
@@ -1,7 +1,7 @@
 # xml.etree test for cElementTree
 
 from test import support
-from test.support import precisionbigmemtest, _2G
+from test.support import bigmemtest, _2G
 import unittest
 
 cET = support.import_module('xml.etree.cElementTree')
@@ -35,7 +35,7 @@ def sanity():
 
 class MiscTests(unittest.TestCase):
     # Issue #8651.
-    @support.precisionbigmemtest(size=support._2G + 100, memuse=1)
+    @support.bigmemtest(size=support._2G + 100, memuse=1)
     def test_length_overflow(self, size):
         if size < support._2G + 100:
             self.skipTest("not enough free memory, need at least 2 GB")

diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py
index 68dd3eabe10a01f90c236c2dd9ede94d1c4b94fb..dffa2ca0da97b02ca6d5ec20539ec0eed793d4d7 100644 (file)
--- a/Lib/test/test_zlib.py
+++ b/Lib/test/test_zlib.py
@@ -3,7 +3,7 @@ from test import support
 import binascii
 import random
 import sys
-from test.support import precisionbigmemtest, _1G, _4G
+from test.support import bigmemtest, _1G, _4G
 
 zlib = support.import_module('zlib')
 
@@ -177,16 +177,16 @@ class CompressTestCase(BaseCompressTestCase, unittest.TestCase):
 
     # Memory use of the following functions takes into account overallocation
 
-    @precisionbigmemtest(size=_1G + 1024 * 1024, memuse=3)
+    @bigmemtest(size=_1G + 1024 * 1024, memuse=3)
     def test_big_compress_buffer(self, size):
         compress = lambda s: zlib.compress(s, 1)
         self.check_big_compress_buffer(size, compress)
 
-    @precisionbigmemtest(size=_1G + 1024 * 1024, memuse=2)
+    @bigmemtest(size=_1G + 1024 * 1024, memuse=2)
     def test_big_decompress_buffer(self, size):
         self.check_big_decompress_buffer(size, zlib.decompress)
 
-    @precisionbigmemtest(size=_4G + 100, memuse=1)
+    @bigmemtest(size=_4G + 100, memuse=1)
     def test_length_overflow(self, size):
         if size < _4G + 100:
             self.skipTest("not enough free memory, need at least 4 GB")
@@ -511,19 +511,19 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase):
 
     # Memory use of the following functions takes into account overallocation
 
-    @precisionbigmemtest(size=_1G + 1024 * 1024, memuse=3)
+    @bigmemtest(size=_1G + 1024 * 1024, memuse=3)
     def test_big_compress_buffer(self, size):
         c = zlib.compressobj(1)
         compress = lambda s: c.compress(s) + c.flush()
         self.check_big_compress_buffer(size, compress)
 
-    @precisionbigmemtest(size=_1G + 1024 * 1024, memuse=2)
+    @bigmemtest(size=_1G + 1024 * 1024, memuse=2)
     def test_big_decompress_buffer(self, size):
         d = zlib.decompressobj()
         decompress = lambda s: d.decompress(s) + d.flush()
         self.check_big_decompress_buffer(size, decompress)
 
-    @precisionbigmemtest(size=_4G + 100, memuse=1)
+    @bigmemtest(size=_4G + 100, memuse=1)
     def test_length_overflow(self, size):
         if size < _4G + 100:
             self.skipTest("not enough free memory, need at least 4 GB")