From: Benjamin Peterson Date: Sat, 6 Dec 2014 01:34:56 +0000 (-0500) Subject: merge 3.3 (#16043) X-Git-Tag: v3.4.3rc1~257 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=eca72d47f5a639a0ac66a98a2d63b30df2ce310f;p=python merge 3.3 (#16043) --- eca72d47f5a639a0ac66a98a2d63b30df2ce310f diff --cc Lib/test/test_xmlrpc.py index 9194740240,fe2bf035cb..71b590cc59 --- a/Lib/test/test_xmlrpc.py +++ b/Lib/test/test_xmlrpc.py @@@ -877,6 -859,26 +877,27 @@@ class GzipServerTestCase(BaseServerTest self.requestHandler.encode_threshold = old self.assertTrue(a>b) + ++@unittest.skipIf(gzip is None, 'requires gzip') + class GzipUtilTestCase(unittest.TestCase): + + def test_gzip_decode_limit(self): + max_gzip_decode = 20 * 1024 * 1024 + data = b'\0' * max_gzip_decode + encoded = xmlrpclib.gzip_encode(data) + decoded = xmlrpclib.gzip_decode(encoded) + self.assertEqual(len(decoded), max_gzip_decode) + + data = b'\0' * (max_gzip_decode + 1) + encoded = xmlrpclib.gzip_encode(data) + + with self.assertRaisesRegexp(ValueError, + "max gzipped payload length exceeded"): + xmlrpclib.gzip_decode(encoded) + + xmlrpclib.gzip_decode(encoded, max_decode=-1) + + #Test special attributes of the ServerProxy object class ServerProxyTestCase(unittest.TestCase): def setUp(self): @@@ -1102,13 -1104,24 +1123,13 @@@ class UseBuiltinTypesTestCase(unittest. @support.reap_threads def test_main(): - xmlrpc_tests = [XMLRPCTestCase, HelperTestCase, DateTimeTestCase, - BinaryTestCase, FaultTestCase] - xmlrpc_tests.append(UseBuiltinTypesTestCase) - xmlrpc_tests.append(SimpleServerTestCase) - xmlrpc_tests.append(KeepaliveServerTestCase1) - xmlrpc_tests.append(KeepaliveServerTestCase2) - try: - import gzip - xmlrpc_tests.append(GzipServerTestCase) - xmlrpc_tests.append(GzipUtilTestCase) - except ImportError: - pass #gzip not supported in this build - xmlrpc_tests.append(MultiPathServerTestCase) - xmlrpc_tests.append(ServerProxyTestCase) - xmlrpc_tests.append(FailingServerTestCase) - xmlrpc_tests.append(CGIHandlerTestCase) - - support.run_unittest(*xmlrpc_tests) + support.run_unittest(XMLRPCTestCase, HelperTestCase, DateTimeTestCase, + BinaryTestCase, FaultTestCase, UseBuiltinTypesTestCase, + SimpleServerTestCase, KeepaliveServerTestCase1, - KeepaliveServerTestCase2, GzipServerTestCase, ++ KeepaliveServerTestCase2, GzipServerTestCase, GzipUtilTestCase, + MultiPathServerTestCase, ServerProxyTestCase, FailingServerTestCase, + CGIHandlerTestCase) + if __name__ == "__main__": test_main() diff --cc Lib/xmlrpc/client.py index 50cedfc4bb,ca2ac9f094..e8c1944fdb --- a/Lib/xmlrpc/client.py +++ b/Lib/xmlrpc/client.py @@@ -1043,8 -1048,11 +1047,11 @@@ def gzip_decode(data, max_decode=209715 f = BytesIO(data) gzf = gzip.GzipFile(mode="rb", fileobj=f) try: - decoded = gzf.read() + if max_decode < 0: # no limit + decoded = gzf.read() + else: + decoded = gzf.read(max_decode + 1) - except IOError: + except OSError: raise ValueError("invalid data") f.close() gzf.close() diff --cc Misc/NEWS index b8ec14814b,c66da32f9e..7d6a99b51b --- a/Misc/NEWS +++ b/Misc/NEWS @@@ -39,273 -26,264 +39,276 @@@ Core and Builtin Library ------- + - Issue #16043: Add a default limit for the amount of data xmlrpclib.gzip_decode + will return. This resolves CVE-2013-1753. + -- Issue #22517: When a io.BufferedRWPair object is deallocated, clear its - weakrefs. +- Issue #14099: ZipFile.open() no longer reopen the underlying file. Objects + returned by ZipFile.open() can now operate independently of the ZipFile even + if the ZipFile was created by passing in a file-like object as the first + argument to the constructor. -- Issue #22419: Limit the length of incoming HTTP request in wsgiref server to - 65536 bytes and send a 414 error code for higher lengths. Patch contributed - by Devin Cook. +- Issue #22966: Fix __pycache__ pyc file name clobber when pyc_compile is + asked to compile a source file containing multiple dots in the source file + name. -- Lax cookie parsing in http.cookies could be a security issue when combined - with non-standard cookie handling in some Web browsers. Reported by - Sergey Bobrov. +- Issue #21971: Update turtledemo doc and add module to the index. -- Issue #21766: Prevent a security hole in CGIHTTPServer by URL unquoting paths - before checking for a CGI script at that path. +- Issue #21032. Fixed socket leak if HTTPConnection.getresponse() fails. + Original patch by Martin Panter. -- Fix arbitrary memory access in JSONDecoder.raw_decode with a negative second - parameter. Bug reported by Guido Vranken. +- Issue #22960: Add a context argument to xmlrpclib.ServerProxy constructor. -- Issue #20633: Replace relative import by absolute import. +- Issue #22915: SAX parser now supports files opened with file descriptor or + bytes path. -- Issue #21082: In os.makedirs, do not set the process-wide umask. Note this - changes behavior of makedirs when exist_ok=True. +- Issue #22609: Constructors and update methods of mapping classes in the + collections module now accept the self keyword argument. -- Issue #20875: Prevent possible gzip "'read' is not defined" NameError. - Patch by Claudiu Popa. +- Issue #22788: Add *context* parameter to logging.handlers.HTTPHandler. -- Issue #11599: When an external command (e.g. compiler) fails, distutils now - prints out the whole command line (instead of just the command name) if the - environment variable DISTUTILS_DEBUG is set. +- Issue #22921: Allow SSLContext to take the *hostname* parameter even if + OpenSSL doesn't support SNI. -- Issue #4931: distutils should not produce unhelpful "error: None" messages - anymore. distutils.util.grok_environment_error is kept but doc-deprecated. +- Issue #22894: TestCase.subTest() would cause the test suite to be stopped + when in failfast mode, even in the absence of failures. -- Issue #20283: RE pattern methods now accept the string keyword parameters - as documented. The pattern and source keyword parameters are left as - deprecated aliases. +- Issue #22638: SSLv3 is now disabled throughout the standard library. + It can still be enabled by instantiating a SSLContext manually. -- Issue #21323: Fix http.server to again handle scripts in CGI subdirectories, - broken by the fix for security issue #19435. Patch by Zach Byrne. +- Issue #22370: Windows detection in pathlib is now more robust. -Tests ------ +- Issue #22841: Reject coroutines in asyncio add_signal_handler(). + Patch by Ludovic.Gasc. -- Issue #17752: Fix distutils tests when run from the installed location. +- Issue #22849: Fix possible double free in the io.TextIOWrapper constructor. -- Issue #20946: Correct alignment assumptions of some ctypes tests. +- Issue #12728: Different Unicode characters having the same uppercase but + different lowercase are now matched in case-insensitive regular expressions. -- Issue #20939: Fix test_geturl failure in test_urllibnet due to - new redirect of http://www.python.org/ to https://www.python.org. +- Issue #22821: Fixed fcntl() with integer argument on 64-bit big-endian + platforms. +- Issue #22406: Fixed the uu_codec codec incorrectly ported to 3.x. + Based on patch by Martin Panter. -What's New in Python 3.3.5? -=========================== +- Issue #17293: uuid.getnode() now determines MAC address on AIX using netstat. + Based on patch by Aivars Kalvāns. -*Release date: 09-Mar-2014* +- Issue #22769: Fixed ttk.Treeview.tag_has() when called without arguments. -No changes from release candidate 2. +- Issue #22417: Verify certificates by default in httplib (PEP 476). +- Issue #22775: Fixed unpickling of http.cookies.SimpleCookie with protocol 2 + and above. Patch by Tim Graham. -What's New in Python 3.3.5 release candidate 2? -=============================================== +- Issue #22366: urllib.request.urlopen will accept a context object + (SSLContext) as an argument which will then used be for HTTPS connection. + Patch by Alex Gaynor. -*Release date: 02-Mar-2014* +- Issue #22776: Brought excluded code into the scope of a try block in + SysLogHandler.emit(). -Core and Builtins ------------------ +- Issue #22665: Add missing get_terminal_size and SameFileError to + shutil.__all__. -- Issue #20731: Properly position in source code files even if they - are opened in text mode. Patch by Serhiy Storchaka. +- Issue #17381: Fixed handling of case-insensitive ranges in regular + expressions. -- Issue #19619: str.encode, bytes.decode and bytearray.decode now use an - internal API to throw LookupError for known non-text encodings, rather - than attempting the encoding or decoding operation and then throwing a - TypeError for an unexpected output type. (The latter mechanism remains - in place for third party non-text encodings) +- Issue #22410: Module level functions in the re module now cache compiled + locale-dependent regular expressions taking into account the locale. -Library -------- +- Issue #22759: Query methods on pathlib.Path() (exists(), is_dir(), etc.) + now return False when the underlying stat call raises NotADirectoryError. -- Issue #20778: Fix modulefinder to work with bytecode-only modules. +- Issue #8876: distutils now falls back to copying files when hard linking + doesn't work. This allows use with special filesystems such as VirtualBox + shared folders. -- Issue #20791: copy.copy() now doesn't make a copy when the input is - a bytes object. Initial patch by Peter Otten. +- Issue #18853: Fixed ResourceWarning in shlex.__nain__. -- Issue #20621: Fixes a zipimport bug introduced in 3.3.4 that could cause - spurious crashes or SystemErrors when importing modules or packages from a - zip file. The change causing the problem was reverted. +- Issue #9351: Defaults set with set_defaults on an argparse subparser + are no longer ignored when also set on the parent parser. -- Issue #20404: io.TextIOWrapper (and hence the open() builtin) now uses the - internal codec marking system added for issue #19619 to throw LookupError - for known non-text encodings at stream construction time. The existing - output type checks remain in place to deal with unmarked third party - codecs. +- Issue #21991: Make email.headerregistry's header 'params' attributes + be read-only (MappingProxyType). Previously the dictionary was modifiable + but a new one was created on each access of the attribute. -Tests ------ +- Issue #22641: In asyncio, the default SSL context for client connections + is now created using ssl.create_default_context(), for stronger security. -- Issue #20743: Fix a reference leak in test_tcl. +- Issue #22435: Fix a file descriptor leak when SocketServer bind fails. -Tools/Demos ------------ +- Issue #13096: Fixed segfault in CTypes POINTER handling of large + values. -- Issue #20535: PYTHONWARNING no longer affects the run_tests.py script. - Patch by Arfrever Frehtes Taifersar Arahesis. +- Issue #11694: Raise ConversionError in xdrlib as documented. Patch + by Filip Gruszczyński and Claudiu Popa. +- Issue #22462: Fix pyexpat's creation of a dummy frame to make it + appear in exception tracebacks. -What's New in Python 3.3.5 release candidate 1? -=============================================== +- Issue #21173: Fix len() on a WeakKeyDictionary when .clear() was called + with an iterator alive. -*Release date: 23-Feb-2014* +- Issue #11866: Eliminated race condition in the computation of names + for new threads. -Core and Builtins ------------------ +- Issue #21905: Avoid RuntimeError in pickle.whichmodule() when sys.modules + is mutated while iterating. Patch by Olivier Grisel. -- Issue #20588: Make Python-ast.c C89 compliant. +- Issue #22219: The zipfile module CLI now adds entries for directories + (including empty directories) in ZIP file. -- Issue #20437: Fixed 21 potential bugs when deleting objects references. +- Issue #22449: In the ssl.SSLContext.load_default_certs, consult the + enviromental variables SSL_CERT_DIR and SSL_CERT_FILE on Windows. -- Issue #20538: UTF-7 incremental decoder produced inconsistant string when - input was truncated in BASE64 section. +- Issue #20076: Added non derived UTF-8 aliases to locale aliases table. -Library -------- +- Issue #20079: Added locales supported in glibc 2.18 to locale alias table. -- Issue #20635: Fixed grid_columnconfigure() and grid_rowconfigure() methods of - Tkinter widgets to work in wantobjects=True mode. +- Issue #22396: On 32-bit AIX platform, don't expose os.posix_fadvise() nor + os.posix_fallocate() because their prototypes in system headers are wrong. -- Issue #19612: On Windows, subprocess.Popen.communicate() now ignores - OSError(22, 'Invalid argument') when writing input data into stdin, whereas - the process already exited. +- Issue #22517: When a io.BufferedRWPair object is deallocated, clear its + weakrefs. -- Issue #6815: os.path.expandvars() now supports non-ASCII environment - variables names and values. +- Issue #22448: Improve canceled timer handles cleanup to prevent + unbound memory usage. Patch by Joshua Moore-Oliva. -- Issue #17671: Fixed a crash when use non-initialized io.BufferedRWPair. - Based on patch by Stephen Tu. +IDLE +---- -- Issue #8478: Untokenizer.compat processes first token from iterator input. - Patch based on lines from Georg Brandl, Eric Snow, and Gareth Rees. +- Issue #16893: Update Idle doc chapter to match current Idle and add new + information. -- Issue #20594: Avoid name clash with the libc function posix_close. +- Issue #3068: Add Idle extension configuration dialog to Options menu. + Changes are written to HOME/.idlerc/config-extensions.cfg. + Original patch by Tal Einat. -- Issue #19856: shutil.move() failed to move a directory to other directory - on Windows if source name ends with os.altsep. +- Issue #16233: A module browser (File : Class Browser, Alt+C) requires a + editor window with a filename. When Class Browser is requested otherwise, + from a shell, output window, or 'Untitled' editor, Idle no longer displays + an error box. It now pops up an Open Module box (Alt+M). If a valid name + is entered and a module is opened, a corresponding browser is also opened. -- Issue #14983: email.generator now always adds a line end after each MIME - boundary marker, instead of doing so only when there is an epilogue. This - fixes an RFC compliance bug and solves an issue with signed MIME parts. +- Issue #4832: Save As to type Python files automatically adds .py to the + name you enter (even if your system does not display it). Some systems + automatically add .txt when type is Text files. -- Issue #20540: Fix a performance regression (vs. Python 3.2) when layering - a multiprocessing Connection over a TCP socket. For small payloads, Nagle's - algorithm would introduce idle delays before the entire transmission of a - message. +- Issue #21986: Code objects are not normally pickled by the pickle module. + To match this, they are no longer pickled when running under Idle. -- Issue #16983: the new email header parsing code will now decode encoded words - that are (incorrectly) surrounded by quotes, and register a defect. +Tests +----- -- Issue #19772: email.generator no longer mutates the message object when - doing a down-transform from 8bit to 7bit CTEs. +- Issue #22838: All test_re tests now work with unittest test discovery. -- Issue #18805: the netmask/hostmask parsing in ipaddress now more reliably - filters out illegal values and correctly allows any valid prefix length. +- Issue #22173: Update lib2to3 tests to use unittest test discovery. -- Issue #17369: get_filename was raising an exception if the filename - parameter's RFC2231 encoding was broken in certain ways. This was - a regression relative to python2. +- Issue #16000: Convert test_curses to use unittest. -- Issue #20013: Some imap servers disconnect if the current mailbox is - deleted, and imaplib did not handle that case gracefully. Now it - handles the 'bye' correctly. +- Issue #21456: Skip two tests in test_urllib2net.py if _ssl module not + present. Patch by Remi Pointel. -- Issue #19920: TarFile.list() no longer fails when outputs a listing - containing non-encodable characters. Based on patch by Vajrasky Kok. +- Issue #22770: Prevent some Tk segfaults on OS X when running gui tests. -- Issue #20515: Fix NULL pointer dereference introduced by issue #20368. +Build +----- -- Issue #19186: Restore namespacing of expat symbols inside the pyexpat module. +- Issue #16537: Check whether self.extensions is empty in setup.py. Patch by + Jonathan Hosmer. -- Issue #20426: When passing the re.DEBUG flag, re.compile() displays the - debug output every time it is called, regardless of the compilation cache. +- Issue #18096: Fix library order returned by python-config. -- Issue #20368: The null character now correctly passed from Tcl to Python. - Improved error handling in variables-related commands. +- Issue #17219: Add library build dir for Python extension cross-builds. -- Issue #20435: Fix _pyio.StringIO.getvalue() to take into account newline - translation settings. +Documentation +------------- -- Issue #20288: fix handling of invalid numeric charrefs in HTMLParser. +- Issue #22914: Update the Python 2/3 porting HOWTO to describe a more automated + approach. -- Issue #20424: Python implementation of io.StringIO now supports lone surrogates. +- Issue #21514: The documentation of the json module now refers to new JSON RFC + 7159 instead of obsoleted RFC 4627. -- Issue #19456: ntpath.join() now joins relative paths correctly when a drive - is present. +Tools/Demos +----------- -- Issue #19077: tempfile.TemporaryDirectory cleanup is now most likely - successful when called during nulling out of modules during shutdown. - Misleading exception no longer raised when resource warning is emitted - during shutdown. +- Issue #22314: pydoc now works when the LINES environment variable is set. -- Issue #20367: Fix behavior of concurrent.futures.as_completed() for - duplicate arguments. Patch by Glenn Langford. +Windows +------- -- Issue #8260: The read(), readline() and readlines() methods of - codecs.StreamReader returned incomplete data when were called after - readline() or read(size). Based on patch by Amaury Forgeot d'Arc. +- Issue #17896: The Windows build scripts now expect external library sources + to be in ``PCbuild\..\externals`` rather than ``PCbuild\..\..``. -IDLE ----- +- Issue #17717: The Windows build scripts now use a copy of NASM pulled from + svn.python.org to build OpenSSL. -- Issue #20406: Use Python application icons for Idle window title bars. - Patch mostly by Serhiy Storchaka. +- Issue #22644: The bundled version of OpenSSL has been updated to 1.0.1j. -- Update the python.gif icon for the Idle classbrowser and pathbowser - from the old green snake to the new new blue and yellow snakes. +What's New in Python 3.4.2? +=========================== -- Issue #17721: Remove non-functional configuration dialog help button until we - make it actually gives some help when clicked. Patch by Guilherme Simões. +Release date: 2014-10-06 -Tests ------ +Core and Builtins +----------------- -- Issue #20510: Rewrote test_exit in test_sys to match existing comments, - use modern unittest features, and use helpers from test.script_helper - instead of using subprocess directly. Patch by Gareth Rees. +Library +------- -- Issue #20532: Tests which use _testcapi are now marked as CPython only. +- Issue #10510: distutils register and upload methods now use HTML standards + compliant CRLF line endings. -- Issue #19920: Added tests for TarFile.list(). Based on patch by Vajrasky Kok. +- Issue #9850: Fixed macpath.join() for empty first component. Patch by + Oleg Oshmyan. -- Issue #19990: Added tests for the imghdr module. Based on patch by - Claudiu Popa. +- Issue #22427: TemporaryDirectory no longer attempts to clean up twice when + used in the with statement in generator. -- Issue #20474: Fix test_socket "unexpected success" failures on OS X 10.7+. +- Issue #20912: Now directories added to ZIP file have correct Unix and MS-DOS + directory attributes. -- Issue #20605: Make test_socket getaddrinfo OS X segfault test more robust. +- Issue #21866: ZipFile.close() no longer writes ZIP64 central directory + records if allowZip64 is false. -Documentation -------------- +- Issue #22415: Fixed debugging output of the GROUPREF_EXISTS opcode in the re + module. Removed trailing spaces in debugging output. -- Issue #20488: Importlib is no longer *an* implementation of import, it's *the* - implementation. +- Issue #22423: Unhandled exception in thread no longer causes unhandled + AttributeError when sys.stderr is None. -Build ------ +- Issue #21332: Ensure that ``bufsize=1`` in subprocess.Popen() selects + line buffering, rather than block buffering. Patch by Akira Li. -- Issue #20221: Removed conflicting (or circular) hypot definition when - compiled with VS 2010 or above. Initial patch by Tabrez Mohammed. -- Issue #20609: Restored the ability to build 64-bit Windows binaries on - 32-bit Windows, which was broken by the change in issue #19788. +What's New in Python 3.4.2rc1? +============================== +Release date: 2014-09-22 -What's New in Python 3.3.4? -=========================== +Core and Builtins +----------------- -*Release date: 09-Feb-2014* +- Issue #22258: Fix the the internal function set_inheritable() on Illumos. + This platform exposes the function ``ioctl(FIOCLEX)``, but calling it fails + with errno is ENOTTY: "Inappropriate ioctl for device". set_inheritable() + now falls back to the slower ``fcntl()`` (``F_GETFD`` and then ``F_SETFD``). + +- Issue #21669: With the aid of heuristics in SyntaxError.__init__, the + parser now attempts to generate more meaningful (or at least more search + engine friendly) error messages when "exec" and "print" are used as + statements. + +- Issue #21642: If the conditional if-else expression, allow an integer written + with no space between itself and the ``else`` keyword (e.g. ``True if 42else + False``) to be valid syntax. + +- Issue #21523: Fix over-pessimistic computation of the stack effect of + some opcodes in the compiler. This also fixes a quadratic compilation + time issue noticeable when compiling code with a large number of "and" + and "or" operators. Library -------