--- /dev/null
- import _strptime, linecache
- import urllib.parse, urllib.request, mimetypes, doctest
- import struct, filecmp, collections.abc
- from distutils.dir_util import _path_created
+import errno
+import os
+import re
+import sys
+import warnings
+from inspect import isabstract
+from test import support
+
+
+try:
+ MAXFD = os.sysconf("SC_OPEN_MAX")
+except Exception:
+ MAXFD = 256
+
+
+def fd_count():
+ """Count the number of open file descriptors"""
+ if sys.platform.startswith(('linux', 'freebsd')):
+ try:
+ names = os.listdir("/proc/self/fd")
+ return len(names)
+ except FileNotFoundError:
+ pass
+
+ count = 0
+ for fd in range(MAXFD):
+ try:
+ # Prefer dup() over fstat(). fstat() can require input/output
+ # whereas dup() doesn't.
+ fd2 = os.dup(fd)
+ except OSError as e:
+ if e.errno != errno.EBADF:
+ raise
+ else:
+ os.close(fd2)
+ count += 1
+ return count
+
+
+def dash_R(the_module, test, indirect_test, huntrleaks):
+ """Run a test multiple times, looking for reference leaks.
+
+ Returns:
+ False if the test didn't leak references; True if we detected refleaks.
+ """
+ # This code is hackish and inelegant, but it seems to do the job.
+ import copyreg
+ import collections.abc
+
+ if not hasattr(sys, 'gettotalrefcount'):
+ raise Exception("Tracking reference leaks requires a debug build "
+ "of Python")
+
+ # Save current values for dash_R_cleanup() to restore.
+ fs = warnings.filters[:]
+ ps = copyreg.dispatch_table.copy()
+ pic = sys.path_importer_cache.copy()
+ try:
+ import zipimport
+ except ImportError:
+ zdc = None # Run unmodified on platforms without zipimport support
+ else:
+ zdc = zipimport._zip_directory_cache.copy()
+ abcs = {}
+ for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]:
+ if not isabstract(abc):
+ continue
+ for obj in abc.__subclasses__() + [abc]:
+ abcs[obj] = obj._abc_registry.copy()
+
+ nwarmup, ntracked, fname = huntrleaks
+ fname = os.path.join(support.SAVEDCWD, fname)
+ repcount = nwarmup + ntracked
+ rc_deltas = [0] * repcount
+ alloc_deltas = [0] * repcount
+ fd_deltas = [0] * repcount
+
+ print("beginning", repcount, "repetitions", file=sys.stderr)
+ print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr,
+ flush=True)
+ # initialize variables to make pyflakes quiet
+ rc_before = alloc_before = fd_before = 0
+ for i in range(repcount):
+ indirect_test()
+ alloc_after, rc_after, fd_after = dash_R_cleanup(fs, ps, pic, zdc,
+ abcs)
+ print('.', end='', flush=True)
+ if i >= nwarmup:
+ rc_deltas[i] = rc_after - rc_before
+ alloc_deltas[i] = alloc_after - alloc_before
+ fd_deltas[i] = fd_after - fd_before
+ alloc_before = alloc_after
+ rc_before = rc_after
+ fd_before = fd_after
+ print(file=sys.stderr)
+ # These checkers return False on success, True on failure
+ def check_rc_deltas(deltas):
+ return any(deltas)
+ def check_alloc_deltas(deltas):
+ # At least 1/3rd of 0s
+ if 3 * deltas.count(0) < len(deltas):
+ return True
+ # Nothing else than 1s, 0s and -1s
+ if not set(deltas) <= {1,0,-1}:
+ return True
+ return False
+ failed = False
+ for deltas, item_name, checker in [
+ (rc_deltas, 'references', check_rc_deltas),
+ (alloc_deltas, 'memory blocks', check_alloc_deltas),
+ (fd_deltas, 'file descriptors', check_rc_deltas)]:
+ if checker(deltas):
+ msg = '%s leaked %s %s, sum=%s' % (
+ test, deltas[nwarmup:], item_name, sum(deltas))
+ print(msg, file=sys.stderr, flush=True)
+ with open(fname, "a") as refrep:
+ print(msg, file=refrep)
+ refrep.flush()
+ failed = True
+ return failed
+
+
+def dash_R_cleanup(fs, ps, pic, zdc, abcs):
+ import gc, copyreg
- # Clear the warnings registry, so they can be displayed again
- for mod in sys.modules.values():
- if hasattr(mod, '__warningregistry__'):
- del mod.__warningregistry__
-
++ import collections.abc
+ from weakref import WeakSet
+
- _path_created.clear()
+ # Restore some original values.
+ warnings.filters[:] = fs
+ copyreg.dispatch_table.clear()
+ copyreg.dispatch_table.update(ps)
+ sys.path_importer_cache.clear()
+ sys.path_importer_cache.update(pic)
+ try:
+ import zipimport
+ except ImportError:
+ pass # Run unmodified on platforms without zipimport support
+ else:
+ zipimport._zip_directory_cache.clear()
+ zipimport._zip_directory_cache.update(zdc)
+
+ # clear type cache
+ sys._clear_type_cache()
+
+ # Clear ABC registries, restoring previously saved ABC registries.
+ for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]:
+ if not isabstract(abc):
+ continue
+ for obj in abc.__subclasses__() + [abc]:
+ obj._abc_registry = abcs.get(obj, WeakSet()).copy()
+ obj._abc_cache.clear()
+ obj._abc_negative_cache.clear()
+
++ clear_caches()
++
++ # Collect cyclic trash and read memory statistics immediately after.
++ func1 = sys.getallocatedblocks
++ func2 = sys.gettotalrefcount
++ gc.collect()
++ return func1(), func2(), fd_count()
++
++
++def clear_caches():
++ import gc
++
++ # Clear the warnings registry, so they can be displayed again
++ for mod in sys.modules.values():
++ if hasattr(mod, '__warningregistry__'):
++ del mod.__warningregistry__
++
+ # Flush standard output, so that buffered data is sent to the OS and
+ # associated Python objects are reclaimed.
+ for stream in (sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__):
+ if stream is not None:
+ stream.flush()
+
+ # Clear assorted module caches.
- _strptime._regex_cache.clear()
- urllib.parse.clear_cache()
- urllib.request.urlcleanup()
- linecache.clearcache()
- mimetypes._default_mime_types()
- filecmp._cache.clear()
- struct._clearcache()
- doctest.master = None
++ # Don't worry about resetting the cache if the module is not loaded
++ try:
++ distutils_dir_util = sys.modules['distutils.dir_util']
++ except KeyError:
++ pass
++ else:
++ distutils_dir_util._path_created.clear()
+ re.purge()
- import ctypes
- except ImportError:
- # Don't worry about resetting the cache if ctypes is not supported
++
+ try:
- # Collect cyclic trash and read memory statistics immediately after.
- func1 = sys.getallocatedblocks
- func2 = sys.gettotalrefcount
++ _strptime = sys.modules['_strptime']
++ except KeyError:
++ pass
++ else:
++ _strptime._regex_cache.clear()
++
++ try:
++ urllib_parse = sys.modules['urllib.parse']
++ except KeyError:
++ pass
++ else:
++ urllib_parse.clear_cache()
++
++ try:
++ urllib_request = sys.modules['urllib.request']
++ except KeyError:
++ pass
++ else:
++ urllib_request.urlcleanup()
++
++ try:
++ linecache = sys.modules['linecache']
++ except KeyError:
++ pass
++ else:
++ linecache.clearcache()
++
++ try:
++ mimetypes = sys.modules['mimetypes']
++ except KeyError:
++ pass
++ else:
++ mimetypes._default_mime_types()
++
++ try:
++ filecmp = sys.modules['filecmp']
++ except KeyError:
++ pass
++ else:
++ filecmp._cache.clear()
++
++ try:
++ struct = sys.modules['struct']
++ except KeyError:
++ pass
++ else:
++ struct._clearcache()
++
++ try:
++ doctest = sys.modules['doctest']
++ except KeyError:
++ pass
++ else:
++ doctest.master = None
++
++ try:
++ ctypes = sys.modules['ctypes']
++ except KeyError:
+ pass
+ else:
+ ctypes._reset_cache()
+
+ try:
+ typing = sys.modules['typing']
+ except KeyError:
+ pass
+ else:
+ for f in typing._cleanups:
+ f()
+
- return func1(), func2(), fd_count()
+ gc.collect()
+
+
+def warm_caches():
+ # char cache
+ s = bytes(range(256))
+ for i in range(256):
+ s[i:i+1]
+ # unicode cache
+ [chr(i) for i in range(256)]
+ # int cache
+ list(range(-5, 257))
--- /dev/null
- from test.libregrtest.refleak import dash_R
+import faulthandler
+import importlib
+import io
+import os
+import sys
+import time
+import traceback
+import unittest
+from test import support
++from test.libregrtest.refleak import dash_R, clear_caches
+from test.libregrtest.save_env import saved_test_environment
+
+
+# Test result constants.
+PASSED = 1
+FAILED = 0
+ENV_CHANGED = -1
+SKIPPED = -2
+RESOURCE_DENIED = -3
+INTERRUPTED = -4
+CHILD_ERROR = -5 # error in a child process
+
+_FORMAT_TEST_RESULT = {
+ PASSED: '%s passed',
+ FAILED: '%s failed',
+ ENV_CHANGED: '%s failed (env changed)',
+ SKIPPED: '%s skipped',
+ RESOURCE_DENIED: '%s skipped (resource denied)',
+ INTERRUPTED: '%s interrupted',
+ CHILD_ERROR: '%s crashed',
+}
+
+# Minimum duration of a test to display its duration or to mention that
+# the test is running in background
+PROGRESS_MIN_TIME = 30.0 # seconds
+
+# small set of tests to determine if we have a basically functioning interpreter
+# (i.e. if any of these fail, then anything else is likely to follow)
+STDTESTS = [
+ 'test_grammar',
+ 'test_opcodes',
+ 'test_dict',
+ 'test_builtin',
+ 'test_exceptions',
+ 'test_types',
+ 'test_unittest',
+ 'test_doctest',
+ 'test_doctest2',
+ 'test_support'
+]
+
+# set of tests that we don't want to be executed when using regrtest
+NOTTESTS = set()
+
+
+def format_test_result(test_name, result):
+ fmt = _FORMAT_TEST_RESULT.get(result, "%s")
+ return fmt % test_name
+
+
+def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS):
+ """Return a list of all applicable test modules."""
+ testdir = findtestdir(testdir)
+ names = os.listdir(testdir)
+ tests = []
+ others = set(stdtests) | nottests
+ for name in names:
+ mod, ext = os.path.splitext(name)
+ if mod[:5] == "test_" and ext in (".py", "") and mod not in others:
+ tests.append(mod)
+ return stdtests + sorted(tests)
+
+
+def runtest(ns, test):
+ """Run a single test.
+
+ ns -- regrtest namespace of options
+ test -- the name of the test
+
+ Returns the tuple (result, test_time), where result is one of the
+ constants:
+
+ INTERRUPTED KeyboardInterrupt when run under -j
+ RESOURCE_DENIED test skipped because resource denied
+ SKIPPED test skipped for some other reason
+ ENV_CHANGED test failed because it changed the execution environment
+ FAILED test failed
+ PASSED test passed
+ """
+
+ output_on_failure = ns.verbose3
+
+ use_timeout = (ns.timeout is not None)
+ if use_timeout:
+ faulthandler.dump_traceback_later(ns.timeout, exit=True)
+ try:
+ support.match_tests = ns.match_tests
+ if ns.failfast:
+ support.failfast = True
+ if output_on_failure:
+ support.verbose = True
+
+ # Reuse the same instance to all calls to runtest(). Some
+ # tests keep a reference to sys.stdout or sys.stderr
+ # (eg. test_argparse).
+ if runtest.stringio is None:
+ stream = io.StringIO()
+ runtest.stringio = stream
+ else:
+ stream = runtest.stringio
+ stream.seek(0)
+ stream.truncate()
+
+ orig_stdout = sys.stdout
+ orig_stderr = sys.stderr
+ try:
+ sys.stdout = stream
+ sys.stderr = stream
+ result = runtest_inner(ns, test, display_failure=False)
+ if result[0] != PASSED:
+ output = stream.getvalue()
+ orig_stderr.write(output)
+ orig_stderr.flush()
+ finally:
+ sys.stdout = orig_stdout
+ sys.stderr = orig_stderr
+ else:
+ support.verbose = ns.verbose # Tell tests to be moderately quiet
+ result = runtest_inner(ns, test, display_failure=not ns.verbose)
+ return result
+ finally:
+ if use_timeout:
+ faulthandler.cancel_dump_traceback_later()
+ cleanup_test_droppings(test, ns.verbose)
+runtest.stringio = None
+
+
+def runtest_inner(ns, test, display_failure=True):
+ support.unload(test)
+
+ test_time = 0.0
+ refleak = False # True if the test leaked references.
+ try:
+ if test.startswith('test.') or ns.testdir:
+ abstest = test
+ else:
+ # Always import it from the test package
+ abstest = 'test.' + test
++ clear_caches()
+ with saved_test_environment(test, ns.verbose, ns.quiet, pgo=ns.pgo) as environment:
+ start_time = time.time()
+ the_module = importlib.import_module(abstest)
+ # If the test has a test_main, that will run the appropriate
+ # tests. If not, use normal unittest test loading.
+ test_runner = getattr(the_module, "test_main", None)
+ if test_runner is None:
+ def test_runner():
+ loader = unittest.TestLoader()
+ tests = loader.loadTestsFromModule(the_module)
+ for error in loader.errors:
+ print(error, file=sys.stderr)
+ if loader.errors:
+ raise Exception("errors while loading tests")
+ support.run_unittest(tests)
+ test_runner()
+ if ns.huntrleaks:
+ refleak = dash_R(the_module, test, test_runner, ns.huntrleaks)
+ test_time = time.time() - start_time
+ except support.ResourceDenied as msg:
+ if not ns.quiet and not ns.pgo:
+ print(test, "skipped --", msg, flush=True)
+ return RESOURCE_DENIED, test_time
+ except unittest.SkipTest as msg:
+ if not ns.quiet and not ns.pgo:
+ print(test, "skipped --", msg, flush=True)
+ return SKIPPED, test_time
+ except KeyboardInterrupt:
+ raise
+ except support.TestFailed as msg:
+ if not ns.pgo:
+ if display_failure:
+ print("test", test, "failed --", msg, file=sys.stderr,
+ flush=True)
+ else:
+ print("test", test, "failed", file=sys.stderr, flush=True)
+ return FAILED, test_time
+ except:
+ msg = traceback.format_exc()
+ if not ns.pgo:
+ print("test", test, "crashed --", msg, file=sys.stderr,
+ flush=True)
+ return FAILED, test_time
+ else:
+ if refleak:
+ return FAILED, test_time
+ if environment.changed:
+ return ENV_CHANGED, test_time
+ return PASSED, test_time
+
+
+def cleanup_test_droppings(testname, verbose):
+ import shutil
+ import stat
+ import gc
+
+ # First kill any dangling references to open files etc.
+ # This can also issue some ResourceWarnings which would otherwise get
+ # triggered during the following test run, and possibly produce failures.
+ gc.collect()
+
+ # Try to clean up junk commonly left behind. While tests shouldn't leave
+ # any files or directories behind, when a test fails that can be tedious
+ # for it to arrange. The consequences can be especially nasty on Windows,
+ # since if a test leaves a file open, it cannot be deleted by name (while
+ # there's nothing we can do about that here either, we can display the
+ # name of the offending test, which is a real help).
+ for name in (support.TESTFN,
+ "db_home",
+ ):
+ if not os.path.exists(name):
+ continue
+
+ if os.path.isdir(name):
+ kind, nuker = "directory", shutil.rmtree
+ elif os.path.isfile(name):
+ kind, nuker = "file", os.unlink
+ else:
+ raise SystemError("os.path says %r exists but is neither "
+ "directory nor file" % name)
+
+ if verbose:
+ print("%r left behind %s %r" % (testname, kind, name))
+ try:
+ # if we have chmod, fix possible permissions problems
+ # that might prevent cleanup
+ if (hasattr(os, 'chmod')):
+ os.chmod(name, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
+ nuker(name)
+ except Exception as msg:
+ print(("%r left behind %s %r and it couldn't be "
+ "removed: %s" % (testname, kind, name, msg)), file=sys.stderr)
+
+
+def findtestdir(path=None):
+ return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir
Core and Builtins
-----------------
-- Issue #28426: Fixed potential crash in PyUnicode_AsDecodedObject() in debug
- build.
-
-- Issue #23782: Fixed possible memory leak in _PyTraceback_Add() and exception
- loss in PyTraceBack_Here().
-
-- Issue #28379: Added sanity checks and tests for PyUnicode_CopyCharacters().
+- Issue #28583: PyDict_SetDefault didn't combine split table when needed.
Patch by Xiang Zhang.
-- Issue #28376: The type of long range iterator is now registered as Iterator.
- Patch by Oren Milman.
-
-- Issue #28376: The constructor of range_iterator now checks that step is not 0.
- Patch by Oren Milman.
-
-- Issue #26906: Resolving special methods of uninitialized type now causes
- implicit initialization of the type instead of a fail.
-
-- Issue #18287: PyType_Ready() now checks that tp_name is not NULL.
- Original patch by Niklas Koep.
+- Issue #27243: Change PendingDeprecationWarning -> DeprecationWarning.
+ As it was agreed in the issue, __aiter__ returning an awaitable
+ should result in PendingDeprecationWarning in 3.5 and in
+ DeprecationWarning in 3.6.
-- Issue #24098: Fixed possible crash when AST is changed in process of
- compiling it.
+- Issue #26182: Fix a refleak in code that raises DeprecationWarning.
-- Issue #28350: String constants with null character no longer interned.
+Library
+-------
-- Issue #26617: Fix crash when GC runs during weakref callbacks.
+- Issue #19717: Makes Path.resolve() succeed on paths that do not exist.
+ Patch by Vajrasky Kok
-- Issue #27942: String constants now interned recursively in tuples and frozensets.
+- Issue #28563: Fixed possible DoS and arbitrary code execution when handle
+ plural form selections in the gettext module. The expression parser now
+ supports exact syntax supported by GNU gettext.
-- Issue #21578: Fixed misleading error message when ImportError called with
- invalid keyword args.
+- Issue #28387: Fixed possible crash in _io.TextIOWrapper deallocator when
+ the garbage collector is invoked in other thread. Based on patch by
+ Sebastian Cufre.
-- Issue #28203: Fix incorrect type in error message from
- ``complex(1.0, {2:3})``. Patch by Soumya Sharma.
+- Issue #28600: Optimize loop.call_soon.
-- Issue #27955: Fallback on reading /dev/urandom device when the getrandom()
- syscall fails with EPERM, for example when blocked by SECCOMP.
+- Issue #28613: Fix get_event_loop() return the current loop if
+ called from coroutines/callbacks.
-- Issue #28131: Fix a regression in zipimport's compile_source(). zipimport
- should use the same optimization level as the interpreter.
+- Issue #28634: Fix asyncio.isfuture() to support unittest.Mock.
-- Issue #25221: Fix corrupted result from PyLong_FromLong(0) when
- Python is compiled with NSMALLPOSINTS = 0.
+- Issue #26081: Fix refleak in _asyncio.Future.__iter__().throw.
-- Issue #25758: Prevents zipimport from unnecessarily encoding a filename
- (patch by Eryk Sun)
+- Issue #28639: Fix inspect.isawaitable to always return bool
+ Patch by Justin Mayfield.
-- Issue #28189: dictitems_contains no longer swallows compare errors.
- (Patch by Xiang Zhang)
+- Issue #28652: Make loop methods reject socket kinds they do not support.
-- Issue #27812: Properly clear out a generator's frame's backreference to the
- generator to prevent crashes in frame.clear().
+- Issue #28653: Fix a refleak in functools.lru_cache.
-- Issue #27811: Fix a crash when a coroutine that has not been awaited is
- finalized with warnings-as-errors enabled.
+Documentation
+-------------
-- Issue #27587: Fix another issue found by PVS-Studio: Null pointer check
- after use of 'def' in _PyState_AddModule().
- Initial patch by Christian Heimes.
+- Issue #28513: Documented command-line interface of zipfile.
-- Issue #26020: set literal evaluation order did not match documented behaviour.
++Tests
++-----
+
-- Issue #27782: Multi-phase extension module import now correctly allows the
- ``m_methods`` field to be used to add module level functions to instances
- of non-module types returned from ``Py_create_mod``. Patch by Xiang Zhang.
++- Issue #23839: Various caches now are cleared before running every test file.
+
-- Issue #27936: The round() function accepted a second None argument
- for some types but not for others. Fixed the inconsistency by
- accepting None for all numeric types.
-- Issue #27487: Warn if a submodule argument to "python -m" or
- runpy.run_module() is found in sys.modules after parent packages are
- imported, but before the submodule is executed.
+What's New in Python 3.6.0 beta 3
+=================================
-- Issue #27558: Fix a SystemError in the implementation of "raise" statement.
- In a brand new thread, raise a RuntimeError since there is no active
- exception to reraise. Patch written by Xiang Zhang.
+*Release date: 2016-10-31*
-- Issue #27419: Standard __import__() no longer look up "__import__" in globals
- or builtins for importing submodules or "from import". Fixed handling an
- error of non-string package name.
+Core and Builtins
+-----------------
-- Issue #27083: Respect the PYTHONCASEOK environment variable under Windows.
+- Issue #28128: Deprecation warning for invalid str and byte escape
+ sequences now prints better information about where the error
+ occurs. Patch by Serhiy Storchaka and Eric Smith.
-- Issue #27514: Make having too many statically nested blocks a SyntaxError
- instead of SystemError.
+- Issue #28509: dict.update() no longer allocate unnecessary large memory.
-- Issue #27473: Fixed possible integer overflow in bytes and bytearray
- concatenations. Patch by Xiang Zhang.
+- Issue #28426: Fixed potential crash in PyUnicode_AsDecodedObject() in debug
+ build.
-- Issue #27507: Add integer overflow check in bytearray.extend(). Patch by
- Xiang Zhang.
+- Issue #28517: Fixed of-by-one error in the peephole optimizer that caused
+ keeping unreachable code.
-- Issue #27581: Don't rely on wrapping for overflow check in
- PySequence_Tuple(). Patch by Xiang Zhang.
+- Issue #28214: Improved exception reporting for problematic __set_name__
+ attributes.
-- Issue #27443: __length_hint__() of bytearray iterators no longer return a
- negative integer for a resized bytearray.
+- Issue #23782: Fixed possible memory leak in _PyTraceback_Add() and exception
+ loss in PyTraceBack_Here().
-- Issue #27942: Fix memory leak in codeobject.c
+- Issue #28471: Fix "Python memory allocator called without holding the GIL"
+ crash in socket.setblocking.
Library
-------
projects / python / commitdiff