From: Serhiy Storchaka Date: Fri, 11 Nov 2016 09:46:44 +0000 (+0200) Subject: Issue #23839: Various caches now are cleared before running every test file. X-Git-Tag: v3.6.0b4~102 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=839102603cf81496b5b5c663fff6c06106db2a15;p=python Issue #23839: Various caches now are cleared before running every test file. --- 839102603cf81496b5b5c663fff6c06106db2a15 diff --cc Lib/test/libregrtest/refleak.py index 3bed59666f,0000000000..3b3d2458b1 mode 100644,000000..100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@@ -1,210 -1,0 +1,269 @@@ +import errno +import os +import re +import sys +import warnings +from inspect import isabstract +from test import support + + +try: + MAXFD = os.sysconf("SC_OPEN_MAX") +except Exception: + MAXFD = 256 + + +def fd_count(): + """Count the number of open file descriptors""" + if sys.platform.startswith(('linux', 'freebsd')): + try: + names = os.listdir("/proc/self/fd") + return len(names) + except FileNotFoundError: + pass + + count = 0 + for fd in range(MAXFD): + try: + # Prefer dup() over fstat(). fstat() can require input/output + # whereas dup() doesn't. + fd2 = os.dup(fd) + except OSError as e: + if e.errno != errno.EBADF: + raise + else: + os.close(fd2) + count += 1 + return count + + +def dash_R(the_module, test, indirect_test, huntrleaks): + """Run a test multiple times, looking for reference leaks. + + Returns: + False if the test didn't leak references; True if we detected refleaks. + """ + # This code is hackish and inelegant, but it seems to do the job. + import copyreg + import collections.abc + + if not hasattr(sys, 'gettotalrefcount'): + raise Exception("Tracking reference leaks requires a debug build " + "of Python") + + # Save current values for dash_R_cleanup() to restore. + fs = warnings.filters[:] + ps = copyreg.dispatch_table.copy() + pic = sys.path_importer_cache.copy() + try: + import zipimport + except ImportError: + zdc = None # Run unmodified on platforms without zipimport support + else: + zdc = zipimport._zip_directory_cache.copy() + abcs = {} + for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: + if not isabstract(abc): + continue + for obj in abc.__subclasses__() + [abc]: + abcs[obj] = obj._abc_registry.copy() + + nwarmup, ntracked, fname = huntrleaks + fname = os.path.join(support.SAVEDCWD, fname) + repcount = nwarmup + ntracked + rc_deltas = [0] * repcount + alloc_deltas = [0] * repcount + fd_deltas = [0] * repcount + + print("beginning", repcount, "repetitions", file=sys.stderr) + print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr, + flush=True) + # initialize variables to make pyflakes quiet + rc_before = alloc_before = fd_before = 0 + for i in range(repcount): + indirect_test() + alloc_after, rc_after, fd_after = dash_R_cleanup(fs, ps, pic, zdc, + abcs) + print('.', end='', flush=True) + if i >= nwarmup: + rc_deltas[i] = rc_after - rc_before + alloc_deltas[i] = alloc_after - alloc_before + fd_deltas[i] = fd_after - fd_before + alloc_before = alloc_after + rc_before = rc_after + fd_before = fd_after + print(file=sys.stderr) + # These checkers return False on success, True on failure + def check_rc_deltas(deltas): + return any(deltas) + def check_alloc_deltas(deltas): + # At least 1/3rd of 0s + if 3 * deltas.count(0) < len(deltas): + return True + # Nothing else than 1s, 0s and -1s + if not set(deltas) <= {1,0,-1}: + return True + return False + failed = False + for deltas, item_name, checker in [ + (rc_deltas, 'references', check_rc_deltas), + (alloc_deltas, 'memory blocks', check_alloc_deltas), + (fd_deltas, 'file descriptors', check_rc_deltas)]: + if checker(deltas): + msg = '%s leaked %s %s, sum=%s' % ( + test, deltas[nwarmup:], item_name, sum(deltas)) + print(msg, file=sys.stderr, flush=True) + with open(fname, "a") as refrep: + print(msg, file=refrep) + refrep.flush() + failed = True + return failed + + +def dash_R_cleanup(fs, ps, pic, zdc, abcs): + import gc, copyreg - import _strptime, linecache - import urllib.parse, urllib.request, mimetypes, doctest - import struct, filecmp, collections.abc - from distutils.dir_util import _path_created ++ import collections.abc + from weakref import WeakSet + - # Clear the warnings registry, so they can be displayed again - for mod in sys.modules.values(): - if hasattr(mod, '__warningregistry__'): - del mod.__warningregistry__ - + # Restore some original values. + warnings.filters[:] = fs + copyreg.dispatch_table.clear() + copyreg.dispatch_table.update(ps) + sys.path_importer_cache.clear() + sys.path_importer_cache.update(pic) + try: + import zipimport + except ImportError: + pass # Run unmodified on platforms without zipimport support + else: + zipimport._zip_directory_cache.clear() + zipimport._zip_directory_cache.update(zdc) + + # clear type cache + sys._clear_type_cache() + + # Clear ABC registries, restoring previously saved ABC registries. + for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: + if not isabstract(abc): + continue + for obj in abc.__subclasses__() + [abc]: + obj._abc_registry = abcs.get(obj, WeakSet()).copy() + obj._abc_cache.clear() + obj._abc_negative_cache.clear() + ++ clear_caches() ++ ++ # Collect cyclic trash and read memory statistics immediately after. ++ func1 = sys.getallocatedblocks ++ func2 = sys.gettotalrefcount ++ gc.collect() ++ return func1(), func2(), fd_count() ++ ++ ++def clear_caches(): ++ import gc ++ ++ # Clear the warnings registry, so they can be displayed again ++ for mod in sys.modules.values(): ++ if hasattr(mod, '__warningregistry__'): ++ del mod.__warningregistry__ ++ + # Flush standard output, so that buffered data is sent to the OS and + # associated Python objects are reclaimed. + for stream in (sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__): + if stream is not None: + stream.flush() + + # Clear assorted module caches. - _path_created.clear() ++ # Don't worry about resetting the cache if the module is not loaded ++ try: ++ distutils_dir_util = sys.modules['distutils.dir_util'] ++ except KeyError: ++ pass ++ else: ++ distutils_dir_util._path_created.clear() + re.purge() - _strptime._regex_cache.clear() - urllib.parse.clear_cache() - urllib.request.urlcleanup() - linecache.clearcache() - mimetypes._default_mime_types() - filecmp._cache.clear() - struct._clearcache() - doctest.master = None ++ + try: - import ctypes - except ImportError: - # Don't worry about resetting the cache if ctypes is not supported ++ _strptime = sys.modules['_strptime'] ++ except KeyError: ++ pass ++ else: ++ _strptime._regex_cache.clear() ++ ++ try: ++ urllib_parse = sys.modules['urllib.parse'] ++ except KeyError: ++ pass ++ else: ++ urllib_parse.clear_cache() ++ ++ try: ++ urllib_request = sys.modules['urllib.request'] ++ except KeyError: ++ pass ++ else: ++ urllib_request.urlcleanup() ++ ++ try: ++ linecache = sys.modules['linecache'] ++ except KeyError: ++ pass ++ else: ++ linecache.clearcache() ++ ++ try: ++ mimetypes = sys.modules['mimetypes'] ++ except KeyError: ++ pass ++ else: ++ mimetypes._default_mime_types() ++ ++ try: ++ filecmp = sys.modules['filecmp'] ++ except KeyError: ++ pass ++ else: ++ filecmp._cache.clear() ++ ++ try: ++ struct = sys.modules['struct'] ++ except KeyError: ++ pass ++ else: ++ struct._clearcache() ++ ++ try: ++ doctest = sys.modules['doctest'] ++ except KeyError: ++ pass ++ else: ++ doctest.master = None ++ ++ try: ++ ctypes = sys.modules['ctypes'] ++ except KeyError: + pass + else: + ctypes._reset_cache() + + try: + typing = sys.modules['typing'] + except KeyError: + pass + else: + for f in typing._cleanups: + f() + - # Collect cyclic trash and read memory statistics immediately after. - func1 = sys.getallocatedblocks - func2 = sys.gettotalrefcount + gc.collect() - return func1(), func2(), fd_count() + + +def warm_caches(): + # char cache + s = bytes(range(256)) + for i in range(256): + s[i:i+1] + # unicode cache + [chr(i) for i in range(256)] + # int cache + list(range(-5, 257)) diff --cc Lib/test/libregrtest/runtest.py index 6282543dc1,0000000000..ba0df0a317 mode 100644,000000..100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@@ -1,244 -1,0 +1,245 @@@ +import faulthandler +import importlib +import io +import os +import sys +import time +import traceback +import unittest +from test import support - from test.libregrtest.refleak import dash_R ++from test.libregrtest.refleak import dash_R, clear_caches +from test.libregrtest.save_env import saved_test_environment + + +# Test result constants. +PASSED = 1 +FAILED = 0 +ENV_CHANGED = -1 +SKIPPED = -2 +RESOURCE_DENIED = -3 +INTERRUPTED = -4 +CHILD_ERROR = -5 # error in a child process + +_FORMAT_TEST_RESULT = { + PASSED: '%s passed', + FAILED: '%s failed', + ENV_CHANGED: '%s failed (env changed)', + SKIPPED: '%s skipped', + RESOURCE_DENIED: '%s skipped (resource denied)', + INTERRUPTED: '%s interrupted', + CHILD_ERROR: '%s crashed', +} + +# Minimum duration of a test to display its duration or to mention that +# the test is running in background +PROGRESS_MIN_TIME = 30.0 # seconds + +# small set of tests to determine if we have a basically functioning interpreter +# (i.e. if any of these fail, then anything else is likely to follow) +STDTESTS = [ + 'test_grammar', + 'test_opcodes', + 'test_dict', + 'test_builtin', + 'test_exceptions', + 'test_types', + 'test_unittest', + 'test_doctest', + 'test_doctest2', + 'test_support' +] + +# set of tests that we don't want to be executed when using regrtest +NOTTESTS = set() + + +def format_test_result(test_name, result): + fmt = _FORMAT_TEST_RESULT.get(result, "%s") + return fmt % test_name + + +def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): + """Return a list of all applicable test modules.""" + testdir = findtestdir(testdir) + names = os.listdir(testdir) + tests = [] + others = set(stdtests) | nottests + for name in names: + mod, ext = os.path.splitext(name) + if mod[:5] == "test_" and ext in (".py", "") and mod not in others: + tests.append(mod) + return stdtests + sorted(tests) + + +def runtest(ns, test): + """Run a single test. + + ns -- regrtest namespace of options + test -- the name of the test + + Returns the tuple (result, test_time), where result is one of the + constants: + + INTERRUPTED KeyboardInterrupt when run under -j + RESOURCE_DENIED test skipped because resource denied + SKIPPED test skipped for some other reason + ENV_CHANGED test failed because it changed the execution environment + FAILED test failed + PASSED test passed + """ + + output_on_failure = ns.verbose3 + + use_timeout = (ns.timeout is not None) + if use_timeout: + faulthandler.dump_traceback_later(ns.timeout, exit=True) + try: + support.match_tests = ns.match_tests + if ns.failfast: + support.failfast = True + if output_on_failure: + support.verbose = True + + # Reuse the same instance to all calls to runtest(). Some + # tests keep a reference to sys.stdout or sys.stderr + # (eg. test_argparse). + if runtest.stringio is None: + stream = io.StringIO() + runtest.stringio = stream + else: + stream = runtest.stringio + stream.seek(0) + stream.truncate() + + orig_stdout = sys.stdout + orig_stderr = sys.stderr + try: + sys.stdout = stream + sys.stderr = stream + result = runtest_inner(ns, test, display_failure=False) + if result[0] != PASSED: + output = stream.getvalue() + orig_stderr.write(output) + orig_stderr.flush() + finally: + sys.stdout = orig_stdout + sys.stderr = orig_stderr + else: + support.verbose = ns.verbose # Tell tests to be moderately quiet + result = runtest_inner(ns, test, display_failure=not ns.verbose) + return result + finally: + if use_timeout: + faulthandler.cancel_dump_traceback_later() + cleanup_test_droppings(test, ns.verbose) +runtest.stringio = None + + +def runtest_inner(ns, test, display_failure=True): + support.unload(test) + + test_time = 0.0 + refleak = False # True if the test leaked references. + try: + if test.startswith('test.') or ns.testdir: + abstest = test + else: + # Always import it from the test package + abstest = 'test.' + test ++ clear_caches() + with saved_test_environment(test, ns.verbose, ns.quiet, pgo=ns.pgo) as environment: + start_time = time.time() + the_module = importlib.import_module(abstest) + # If the test has a test_main, that will run the appropriate + # tests. If not, use normal unittest test loading. + test_runner = getattr(the_module, "test_main", None) + if test_runner is None: + def test_runner(): + loader = unittest.TestLoader() + tests = loader.loadTestsFromModule(the_module) + for error in loader.errors: + print(error, file=sys.stderr) + if loader.errors: + raise Exception("errors while loading tests") + support.run_unittest(tests) + test_runner() + if ns.huntrleaks: + refleak = dash_R(the_module, test, test_runner, ns.huntrleaks) + test_time = time.time() - start_time + except support.ResourceDenied as msg: + if not ns.quiet and not ns.pgo: + print(test, "skipped --", msg, flush=True) + return RESOURCE_DENIED, test_time + except unittest.SkipTest as msg: + if not ns.quiet and not ns.pgo: + print(test, "skipped --", msg, flush=True) + return SKIPPED, test_time + except KeyboardInterrupt: + raise + except support.TestFailed as msg: + if not ns.pgo: + if display_failure: + print("test", test, "failed --", msg, file=sys.stderr, + flush=True) + else: + print("test", test, "failed", file=sys.stderr, flush=True) + return FAILED, test_time + except: + msg = traceback.format_exc() + if not ns.pgo: + print("test", test, "crashed --", msg, file=sys.stderr, + flush=True) + return FAILED, test_time + else: + if refleak: + return FAILED, test_time + if environment.changed: + return ENV_CHANGED, test_time + return PASSED, test_time + + +def cleanup_test_droppings(testname, verbose): + import shutil + import stat + import gc + + # First kill any dangling references to open files etc. + # This can also issue some ResourceWarnings which would otherwise get + # triggered during the following test run, and possibly produce failures. + gc.collect() + + # Try to clean up junk commonly left behind. While tests shouldn't leave + # any files or directories behind, when a test fails that can be tedious + # for it to arrange. The consequences can be especially nasty on Windows, + # since if a test leaves a file open, it cannot be deleted by name (while + # there's nothing we can do about that here either, we can display the + # name of the offending test, which is a real help). + for name in (support.TESTFN, + "db_home", + ): + if not os.path.exists(name): + continue + + if os.path.isdir(name): + kind, nuker = "directory", shutil.rmtree + elif os.path.isfile(name): + kind, nuker = "file", os.unlink + else: + raise SystemError("os.path says %r exists but is neither " + "directory nor file" % name) + + if verbose: + print("%r left behind %s %r" % (testname, kind, name)) + try: + # if we have chmod, fix possible permissions problems + # that might prevent cleanup + if (hasattr(os, 'chmod')): + os.chmod(name, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + nuker(name) + except Exception as msg: + print(("%r left behind %s %r and it couldn't be " + "removed: %s" % (testname, kind, name, msg)), file=sys.stderr) + + +def findtestdir(path=None): + return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir diff --cc Misc/NEWS index 4e6ee55346,c408e9912a..15f5725955 --- a/Misc/NEWS +++ b/Misc/NEWS @@@ -10,80 -10,107 +10,85 @@@ What's New in Python 3.6.0 beta Core and Builtins ----------------- -- Issue #28426: Fixed potential crash in PyUnicode_AsDecodedObject() in debug - build. - -- Issue #23782: Fixed possible memory leak in _PyTraceback_Add() and exception - loss in PyTraceBack_Here(). - -- Issue #28379: Added sanity checks and tests for PyUnicode_CopyCharacters(). +- Issue #28583: PyDict_SetDefault didn't combine split table when needed. Patch by Xiang Zhang. -- Issue #28376: The type of long range iterator is now registered as Iterator. - Patch by Oren Milman. - -- Issue #28376: The constructor of range_iterator now checks that step is not 0. - Patch by Oren Milman. - -- Issue #26906: Resolving special methods of uninitialized type now causes - implicit initialization of the type instead of a fail. - -- Issue #18287: PyType_Ready() now checks that tp_name is not NULL. - Original patch by Niklas Koep. +- Issue #27243: Change PendingDeprecationWarning -> DeprecationWarning. + As it was agreed in the issue, __aiter__ returning an awaitable + should result in PendingDeprecationWarning in 3.5 and in + DeprecationWarning in 3.6. -- Issue #24098: Fixed possible crash when AST is changed in process of - compiling it. +- Issue #26182: Fix a refleak in code that raises DeprecationWarning. -- Issue #28350: String constants with null character no longer interned. +Library +------- -- Issue #26617: Fix crash when GC runs during weakref callbacks. +- Issue #19717: Makes Path.resolve() succeed on paths that do not exist. + Patch by Vajrasky Kok -- Issue #27942: String constants now interned recursively in tuples and frozensets. +- Issue #28563: Fixed possible DoS and arbitrary code execution when handle + plural form selections in the gettext module. The expression parser now + supports exact syntax supported by GNU gettext. -- Issue #21578: Fixed misleading error message when ImportError called with - invalid keyword args. +- Issue #28387: Fixed possible crash in _io.TextIOWrapper deallocator when + the garbage collector is invoked in other thread. Based on patch by + Sebastian Cufre. -- Issue #28203: Fix incorrect type in error message from - ``complex(1.0, {2:3})``. Patch by Soumya Sharma. +- Issue #28600: Optimize loop.call_soon. -- Issue #27955: Fallback on reading /dev/urandom device when the getrandom() - syscall fails with EPERM, for example when blocked by SECCOMP. +- Issue #28613: Fix get_event_loop() return the current loop if + called from coroutines/callbacks. -- Issue #28131: Fix a regression in zipimport's compile_source(). zipimport - should use the same optimization level as the interpreter. +- Issue #28634: Fix asyncio.isfuture() to support unittest.Mock. -- Issue #25221: Fix corrupted result from PyLong_FromLong(0) when - Python is compiled with NSMALLPOSINTS = 0. +- Issue #26081: Fix refleak in _asyncio.Future.__iter__().throw. -- Issue #25758: Prevents zipimport from unnecessarily encoding a filename - (patch by Eryk Sun) +- Issue #28639: Fix inspect.isawaitable to always return bool + Patch by Justin Mayfield. -- Issue #28189: dictitems_contains no longer swallows compare errors. - (Patch by Xiang Zhang) +- Issue #28652: Make loop methods reject socket kinds they do not support. -- Issue #27812: Properly clear out a generator's frame's backreference to the - generator to prevent crashes in frame.clear(). +- Issue #28653: Fix a refleak in functools.lru_cache. -- Issue #27811: Fix a crash when a coroutine that has not been awaited is - finalized with warnings-as-errors enabled. +Documentation +------------- -- Issue #27587: Fix another issue found by PVS-Studio: Null pointer check - after use of 'def' in _PyState_AddModule(). - Initial patch by Christian Heimes. +- Issue #28513: Documented command-line interface of zipfile. -- Issue #26020: set literal evaluation order did not match documented behaviour. ++Tests ++----- + -- Issue #27782: Multi-phase extension module import now correctly allows the - ``m_methods`` field to be used to add module level functions to instances - of non-module types returned from ``Py_create_mod``. Patch by Xiang Zhang. ++- Issue #23839: Various caches now are cleared before running every test file. + -- Issue #27936: The round() function accepted a second None argument - for some types but not for others. Fixed the inconsistency by - accepting None for all numeric types. -- Issue #27487: Warn if a submodule argument to "python -m" or - runpy.run_module() is found in sys.modules after parent packages are - imported, but before the submodule is executed. +What's New in Python 3.6.0 beta 3 +================================= -- Issue #27558: Fix a SystemError in the implementation of "raise" statement. - In a brand new thread, raise a RuntimeError since there is no active - exception to reraise. Patch written by Xiang Zhang. +*Release date: 2016-10-31* -- Issue #27419: Standard __import__() no longer look up "__import__" in globals - or builtins for importing submodules or "from import". Fixed handling an - error of non-string package name. +Core and Builtins +----------------- -- Issue #27083: Respect the PYTHONCASEOK environment variable under Windows. +- Issue #28128: Deprecation warning for invalid str and byte escape + sequences now prints better information about where the error + occurs. Patch by Serhiy Storchaka and Eric Smith. -- Issue #27514: Make having too many statically nested blocks a SyntaxError - instead of SystemError. +- Issue #28509: dict.update() no longer allocate unnecessary large memory. -- Issue #27473: Fixed possible integer overflow in bytes and bytearray - concatenations. Patch by Xiang Zhang. +- Issue #28426: Fixed potential crash in PyUnicode_AsDecodedObject() in debug + build. -- Issue #27507: Add integer overflow check in bytearray.extend(). Patch by - Xiang Zhang. +- Issue #28517: Fixed of-by-one error in the peephole optimizer that caused + keeping unreachable code. -- Issue #27581: Don't rely on wrapping for overflow check in - PySequence_Tuple(). Patch by Xiang Zhang. +- Issue #28214: Improved exception reporting for problematic __set_name__ + attributes. -- Issue #27443: __length_hint__() of bytearray iterators no longer return a - negative integer for a resized bytearray. +- Issue #23782: Fixed possible memory leak in _PyTraceback_Add() and exception + loss in PyTraceBack_Here(). -- Issue #27942: Fix memory leak in codeobject.c +- Issue #28471: Fix "Python memory allocator called without holding the GIL" + crash in socket.setblocking. Library -------