From: Victor Stinner Date: Mon, 9 May 2011 22:48:41 +0000 (+0200) Subject: (Merge 3.1) Issue #12012: ssl.PROTOCOL_SSLv2 becomes optional X-Git-Tag: v3.2.1rc1~33 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=17ca323e7c0d6cc0495f4d1dbef628566a1f8f09;p=thirdparty%2FPython%2Fcpython.git (Merge 3.1) Issue #12012: ssl.PROTOCOL_SSLv2 becomes optional OpenSSL is now compiled with OPENSSL_NO_SSL2 defined (without the SSLv2 protocol) on Debian: fix the ssl module on Debian Testing and Debian Sid. Optimize also ssl.get_protocol_name(): speed does matter! --- 17ca323e7c0d6cc0495f4d1dbef628566a1f8f09 diff --cc Lib/ssl.py index e7c175f063a6,bbb81110e8d6..b12b9fd1e649 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@@ -59,12 -58,8 +59,10 @@@ import r import _ssl # if we can't import it, let the error propagate -from _ssl import SSLError +from _ssl import OPENSSL_VERSION_NUMBER, OPENSSL_VERSION_INFO, OPENSSL_VERSION +from _ssl import _SSLContext, SSLError from _ssl import CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED - from _ssl import (PROTOCOL_SSLv2, PROTOCOL_SSLv3, PROTOCOL_SSLv23, - PROTOCOL_TLSv1) +from _ssl import OP_ALL, OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_TLSv1 from _ssl import RAND_status, RAND_egd, RAND_add from _ssl import ( SSL_ERROR_ZERO_RETURN, @@@ -77,7 -72,18 +75,19 @@@ SSL_ERROR_EOF, SSL_ERROR_INVALID_ERROR_CODE, ) +from _ssl import HAS_SNI + from _ssl import PROTOCOL_SSLv3, PROTOCOL_SSLv23, PROTOCOL_TLSv1 + _PROTOCOL_NAMES = { + PROTOCOL_TLSv1: "TLSv1", + PROTOCOL_SSLv23: "SSLv23", + PROTOCOL_SSLv3: "SSLv3", + } + try: + from _ssl import PROTOCOL_SSLv2 + except ImportError: + pass + else: + _PROTOCOL_NAMES[PROTOCOL_SSLv2] = "SSLv2" from socket import getnameinfo as _getnameinfo from socket import error as socket_error diff --cc Lib/test/test_ssl.py index ba788e4c8444,f2f4f80fc69b..f65aceb057b9 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@@ -15,74 -17,30 +15,79 @@@ import urllib.reques import traceback import asyncore import weakref +import platform +import functools -from http.server import HTTPServer, SimpleHTTPRequestHandler +ssl = support.import_module("ssl") -# Optionally test SSL support, if we have it in the tested platform -skip_expected = False -try: - import ssl -except ImportError: - skip_expected = True +PROTOCOLS = [ - ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv3, ++ ssl.PROTOCOL_SSLv3, + ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1 +] ++if hasattr(ssl, 'PROTOCOL_SSLv2'): ++ PROTOCOLS.append(ssl.PROTOCOL_SSLv2) HOST = support.HOST -CERTFILE = None -SVN_PYTHON_ORG_ROOT_CERT = None + +data_file = lambda name: os.path.join(os.path.dirname(__file__), name) + +# The custom key and certificate files used in test_ssl are generated +# using Lib/test/make_ssl_certs.py. +# Other certificates are simply fetched from the Internet servers they +# are meant to authenticate. + +CERTFILE = data_file("keycert.pem") +BYTES_CERTFILE = os.fsencode(CERTFILE) +ONLYCERT = data_file("ssl_cert.pem") +ONLYKEY = data_file("ssl_key.pem") +BYTES_ONLYCERT = os.fsencode(ONLYCERT) +BYTES_ONLYKEY = os.fsencode(ONLYKEY) +CAPATH = data_file("capath") +BYTES_CAPATH = os.fsencode(CAPATH) + +SVN_PYTHON_ORG_ROOT_CERT = data_file("https_svn_python_org_root.pem") + +EMPTYCERT = data_file("nullcert.pem") +BADCERT = data_file("badcert.pem") +WRONGCERT = data_file("XXXnonexisting.pem") +BADKEY = data_file("badkey.pem") + def handle_error(prefix): exc_format = ' '.join(traceback.format_exception(*sys.exc_info())) if support.verbose: sys.stdout.write(prefix + exc_format) +def can_clear_options(): + # 0.9.8m or higher + return ssl.OPENSSL_VERSION_INFO >= (0, 9, 8, 13, 15) -class BasicTests(unittest.TestCase): +def no_sslv2_implies_sslv3_hello(): + # 0.9.7h or higher + return ssl.OPENSSL_VERSION_INFO >= (0, 9, 7, 8, 15) + + +# Issue #9415: Ubuntu hijacks their OpenSSL and forcefully disables SSLv2 +def skip_if_broken_ubuntu_ssl(func): - @functools.wraps(func) - def f(*args, **kwargs): - try: - ssl.SSLContext(ssl.PROTOCOL_SSLv2) - except ssl.SSLError: - if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and - platform.linux_distribution() == ('debian', 'squeeze/sid', '')): - raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour") - return func(*args, **kwargs) - return f ++ if hasattr(ssl, 'PROTOCOL_SSLv2'): ++ @functools.wraps(func) ++ def f(*args, **kwargs): ++ try: ++ ssl.SSLContext(ssl.PROTOCOL_SSLv2) ++ except ssl.SSLError: ++ if (ssl.OPENSSL_VERSION_INFO == (0, 9, 8, 15, 15) and ++ platform.linux_distribution() == ('debian', 'squeeze/sid', '')): ++ raise unittest.SkipTest("Patched Ubuntu OpenSSL breaks behaviour") ++ return func(*args, **kwargs) ++ return f ++ else: ++ return func + + +class BasicSocketTests(unittest.TestCase): def test_constants(self): - ssl.PROTOCOL_SSLv2 + #ssl.PROTOCOL_SSLv2 ssl.PROTOCOL_SSLv23 ssl.PROTOCOL_SSLv3 ssl.PROTOCOL_TLSv1 @@@ -183,262 -113,6 +188,263 @@@ ss = ssl.wrap_socket(s) self.assertEqual(timeout, ss.gettimeout()) + def test_errors(self): + sock = socket.socket() + self.assertRaisesRegex(ValueError, + "certfile must be specified", + ssl.wrap_socket, sock, keyfile=CERTFILE) + self.assertRaisesRegex(ValueError, + "certfile must be specified for server-side operations", + ssl.wrap_socket, sock, server_side=True) + self.assertRaisesRegex(ValueError, + "certfile must be specified for server-side operations", + ssl.wrap_socket, sock, server_side=True, certfile="") + s = ssl.wrap_socket(sock, server_side=True, certfile=CERTFILE) + self.assertRaisesRegex(ValueError, "can't connect in server-side mode", + s.connect, (HOST, 8080)) + with self.assertRaises(IOError) as cm: + with socket.socket() as sock: + ssl.wrap_socket(sock, certfile=WRONGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaises(IOError) as cm: + with socket.socket() as sock: + ssl.wrap_socket(sock, certfile=CERTFILE, keyfile=WRONGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaises(IOError) as cm: + with socket.socket() as sock: + ssl.wrap_socket(sock, certfile=WRONGCERT, keyfile=WRONGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + + def test_match_hostname(self): + def ok(cert, hostname): + ssl.match_hostname(cert, hostname) + def fail(cert, hostname): + self.assertRaises(ssl.CertificateError, + ssl.match_hostname, cert, hostname) + + cert = {'subject': ((('commonName', 'example.com'),),)} + ok(cert, 'example.com') + ok(cert, 'ExAmple.cOm') + fail(cert, 'www.example.com') + fail(cert, '.example.com') + fail(cert, 'example.org') + fail(cert, 'exampleXcom') + + cert = {'subject': ((('commonName', '*.a.com'),),)} + ok(cert, 'foo.a.com') + fail(cert, 'bar.foo.a.com') + fail(cert, 'a.com') + fail(cert, 'Xa.com') + fail(cert, '.a.com') + + cert = {'subject': ((('commonName', 'a.*.com'),),)} + ok(cert, 'a.foo.com') + fail(cert, 'a..com') + fail(cert, 'a.com') + + cert = {'subject': ((('commonName', 'f*.com'),),)} + ok(cert, 'foo.com') + ok(cert, 'f.com') + fail(cert, 'bar.com') + fail(cert, 'foo.a.com') + fail(cert, 'bar.foo.com') + + # Slightly fake real-world example + cert = {'notAfter': 'Jun 26 21:41:46 2011 GMT', + 'subject': ((('commonName', 'linuxfrz.org'),),), + 'subjectAltName': (('DNS', 'linuxfr.org'), + ('DNS', 'linuxfr.com'), + ('othername', ''))} + ok(cert, 'linuxfr.org') + ok(cert, 'linuxfr.com') + # Not a "DNS" entry + fail(cert, '') + # When there is a subjectAltName, commonName isn't used + fail(cert, 'linuxfrz.org') + + # A pristine real-world example + cert = {'notAfter': 'Dec 18 23:59:59 2011 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('organizationName', 'Google Inc'),), + (('commonName', 'mail.google.com'),))} + ok(cert, 'mail.google.com') + fail(cert, 'gmail.com') + # Only commonName is considered + fail(cert, 'California') + + # Neither commonName nor subjectAltName + cert = {'notAfter': 'Dec 18 23:59:59 2011 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('organizationName', 'Google Inc'),))} + fail(cert, 'mail.google.com') + + # No DNS entry in subjectAltName but a commonName + cert = {'notAfter': 'Dec 18 23:59:59 2099 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('commonName', 'mail.google.com'),)), + 'subjectAltName': (('othername', 'blabla'), )} + ok(cert, 'mail.google.com') + + # No DNS entry subjectAltName and no commonName + cert = {'notAfter': 'Dec 18 23:59:59 2099 GMT', + 'subject': ((('countryName', 'US'),), + (('stateOrProvinceName', 'California'),), + (('localityName', 'Mountain View'),), + (('organizationName', 'Google Inc'),)), + 'subjectAltName': (('othername', 'blabla'),)} + fail(cert, 'google.com') + + # Empty cert / no cert + self.assertRaises(ValueError, ssl.match_hostname, None, 'example.com') + self.assertRaises(ValueError, ssl.match_hostname, {}, 'example.com') + + def test_server_side(self): + # server_hostname doesn't work for server sockets + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + with socket.socket() as sock: + self.assertRaises(ValueError, ctx.wrap_socket, sock, True, + server_hostname="some.hostname") + +class ContextTests(unittest.TestCase): + + @skip_if_broken_ubuntu_ssl + def test_constructor(self): - ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv2) ++ if hasattr(ssl, 'PROTOCOL_SSLv2'): ++ ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv2) + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv3) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + self.assertRaises(TypeError, ssl.SSLContext) + self.assertRaises(ValueError, ssl.SSLContext, -1) + self.assertRaises(ValueError, ssl.SSLContext, 42) + + @skip_if_broken_ubuntu_ssl + def test_protocol(self): + for proto in PROTOCOLS: + ctx = ssl.SSLContext(proto) + self.assertEqual(ctx.protocol, proto) + + def test_ciphers(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.set_ciphers("ALL") + ctx.set_ciphers("DEFAULT") + with self.assertRaisesRegex(ssl.SSLError, "No cipher can be selected"): + ctx.set_ciphers("^$:,;?*'dorothyx") + + @skip_if_broken_ubuntu_ssl + def test_options(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + # OP_ALL is the default value + self.assertEqual(ssl.OP_ALL, ctx.options) + ctx.options |= ssl.OP_NO_SSLv2 + self.assertEqual(ssl.OP_ALL | ssl.OP_NO_SSLv2, + ctx.options) + ctx.options |= ssl.OP_NO_SSLv3 + self.assertEqual(ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3, + ctx.options) + if can_clear_options(): + ctx.options = (ctx.options & ~ssl.OP_NO_SSLv2) | ssl.OP_NO_TLSv1 + self.assertEqual(ssl.OP_ALL | ssl.OP_NO_TLSv1 | ssl.OP_NO_SSLv3, + ctx.options) + ctx.options = 0 + self.assertEqual(0, ctx.options) + else: + with self.assertRaises(ValueError): + ctx.options = 0 + + def test_verify(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + # Default value + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + ctx.verify_mode = ssl.CERT_OPTIONAL + self.assertEqual(ctx.verify_mode, ssl.CERT_OPTIONAL) + ctx.verify_mode = ssl.CERT_REQUIRED + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + ctx.verify_mode = ssl.CERT_NONE + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + with self.assertRaises(TypeError): + ctx.verify_mode = None + with self.assertRaises(ValueError): + ctx.verify_mode = 42 + + def test_load_cert_chain(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + # Combined key and cert in a single file + ctx.load_cert_chain(CERTFILE) + ctx.load_cert_chain(CERTFILE, keyfile=CERTFILE) + self.assertRaises(TypeError, ctx.load_cert_chain, keyfile=CERTFILE) + with self.assertRaises(IOError) as cm: + ctx.load_cert_chain(WRONGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(BADCERT) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(EMPTYCERT) + # Separate key and cert + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_cert_chain(ONLYCERT, ONLYKEY) + ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY) + ctx.load_cert_chain(certfile=BYTES_ONLYCERT, keyfile=BYTES_ONLYKEY) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(ONLYCERT) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(ONLYKEY) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_cert_chain(certfile=ONLYKEY, keyfile=ONLYCERT) + # Mismatching key and cert + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + with self.assertRaisesRegex(ssl.SSLError, "key values mismatch"): + ctx.load_cert_chain(SVN_PYTHON_ORG_ROOT_CERT, ONLYKEY) + + def test_load_verify_locations(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.load_verify_locations(CERTFILE) + ctx.load_verify_locations(cafile=CERTFILE, capath=None) + ctx.load_verify_locations(BYTES_CERTFILE) + ctx.load_verify_locations(cafile=BYTES_CERTFILE, capath=None) + self.assertRaises(TypeError, ctx.load_verify_locations) + self.assertRaises(TypeError, ctx.load_verify_locations, None, None) + with self.assertRaises(IOError) as cm: + ctx.load_verify_locations(WRONGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaisesRegex(ssl.SSLError, "PEM lib"): + ctx.load_verify_locations(BADCERT) + ctx.load_verify_locations(CERTFILE, CAPATH) + ctx.load_verify_locations(CERTFILE, capath=BYTES_CAPATH) + + # Issue #10989: crash if the second argument type is invalid + self.assertRaises(TypeError, ctx.load_verify_locations, None, True) + + @skip_if_broken_ubuntu_ssl + def test_session_stats(self): + for proto in PROTOCOLS: + ctx = ssl.SSLContext(proto) + self.assertEqual(ctx.session_stats(), { + 'number': 0, + 'connect': 0, + 'connect_good': 0, + 'connect_renegotiate': 0, + 'accept': 0, + 'accept_good': 0, + 'accept_renegotiate': 0, + 'hits': 0, + 'misses': 0, + 'timeouts': 0, + 'cache_full': 0, + }) + + def test_set_default_verify_paths(self): + # There's not much we can do to test that it acts as expected, + # so just check it doesn't crash or raise an exception. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + ctx.set_default_verify_paths() + class NetworkedTests(unittest.TestCase): @@@ -1198,7 -838,7 +1204,8 @@@ else finally: t.join() + @skip_if_broken_ubuntu_ssl + @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv2'), "need SSLv2") def test_protocol_sslv2(self): """Connecting to an SSLv2 server with various client options""" if support.verbose: @@@ -1223,15 -855,18 +1230,16 @@@ def test_protocol_sslv23(self): """Connecting to an SSLv23 server with various client options""" if support.verbose: - sys.stdout.write("\ntest_protocol_sslv23 disabled, " - "as it fails on OpenSSL 1.0.0+") - return + sys.stdout.write("\n") - try: - try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv2, True) - except (ssl.SSLError, socket.error) as x: - # this fails on some older versions of OpenSSL (0.9.7l, for instance) - if support.verbose: - sys.stdout.write( - " SSL2 client to SSL23 server test unexpectedly failed:\n %s\n" - % str(x)) + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try: + try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv2, True) + except (ssl.SSLError, socket.error) as x: + # this fails on some older versions of OpenSSL (0.9.7l, for instance) + if support.verbose: + sys.stdout.write( + " SSL2 client to SSL23 server test unexpectedly failed:\n %s\n" + % str(x)) try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv3, True) try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_SSLv23, True) try_protocol_combo(ssl.PROTOCOL_SSLv23, ssl.PROTOCOL_TLSv1, True) @@@ -1262,15 -888,11 +1270,16 @@@ try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True) try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_OPTIONAL) try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, True, ssl.CERT_REQUIRED) - try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False) + if hasattr(ssl, 'PROTOCOL_SSLv2'): + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False) try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, False) try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLSv1, False) + if no_sslv2_implies_sslv3_hello(): + # No SSLv2 => client will use an SSLv3 hello on recent OpenSSLs + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv23, True, + client_options=ssl.OP_NO_SSLv2) + @skip_if_broken_ubuntu_ssl def test_protocol_tlsv1(self): """Connecting to a TLSv1 server with various client options""" if support.verbose: diff --cc Misc/NEWS index 4a4767ebe62c,9f4d9a8b7ae0..4309858ec5d7 --- a/Misc/NEWS +++ b/Misc/NEWS @@@ -484,789 -398,65 +484,791 @@@ Core and Builtin Library ------- -- Issue #2236: distutils' mkpath ignored the mode parameter. +- Issue #9124: mailbox now accepts binary input and reads and writes mailbox + files in binary mode, using the email package's binary support to parse + arbitrary email messages. StringIO and text file input is deprecated, + and string input fails early if non-ASCII characters are used, where + previously it would fail when the email was processed in a later step. -- Fix typo in one sdist option (medata-check). +- Issue #10845: Mitigate the incompatibility between the multiprocessing + module on Windows and the use of package, zipfile or directory execution + by special casing main modules that actually *are* called __main__.py. -- Issue #10323: itertools.islice() now consumes the minimum number of - inputs before stopping. Formerly, the final state of the underlying - iterator was undefined. +- Issue #11045: Protect logging call against None argument. -- Issue #10565: The collections.Iterator ABC now checks for both - __iter__ and __next__. +- Issue #11052: Correct IDLE menu accelerators on Mac OS X for Save + commands. -- Issue #10561: In pdb, clear the breakpoints by the breakpoint number. +- Issue #11053: Fix IDLE "Syntax Error" windows to behave as in 2.x, + preventing a confusing hung appearance on OS X with the windows + obscured. -- Issue #10459: Update CJK character names to Unicode 5.1. +- Issue #10940: Workaround an IDLE hang on Mac OS X 10.6 when using the + menu accelerators for Open Module, Go to Line, and New Indent Width. + The accelerators still work but no longer appear in the menu items. -- Issue #10092: Properly reset locale in calendar.Locale*Calendar classes. +- Issue #10989: Fix a crash on SSLContext.load_verify_locations(None, True). -- Issue #6098: Don't claim DOM level 3 conformance in minidom. +- Issue #11020: Command-line pyclbr was broken because of missing 2-to-3 + conversion. -- Issue #5762: Fix AttributeError raised by ``xml.dom.minidom`` when an empty - XML namespace attribute is encountered. +- Issue #11019: Fixed BytesGenerator so that it correctly handles a Message + with a None body. -- Issue #1710703: Write structures for an empty ZIP archive when a ZipFile is - created in modes 'a' or 'w' and then closed without adding any files. Raise - BadZipfile (rather than IOError) when opening small non-ZIP files. +- Issue #11014: Make 'filter' argument in tarfile.Tarfile.add() into a + keyword-only argument. The preceding positional argument was deprecated, + so it made no sense to add filter as a positional argument. -- Issue #4493: urllib.request adds '/' in front of path components which does not - start with '/. Common behavior exhibited by browsers and other clients. +- Issue #11004: Repaired edge case in deque.count(). -- Issue #6378: idle.bat now runs with the appropriate Python version rather than - the system default. Patch by Sridhar Ratnakumar. +- Issue #10974: IDLE no longer crashes if its recent files list includes files + with non-ASCII characters in their path names. -- Issue #10407: Fix two NameErrors in distutils. +- Have hashlib.algorithms_available and hashlib.algorithms_guaranteed both + return sets instead of one returning a tuple and the other a frozenset. -- Issue #10198: fix duplicate header written to wave files when writeframes() - is called without data. +- Issue #10987: Fix the recursion limit handling in the _pickle module. -- Issue #10467: Fix BytesIO.readinto() after seeking into a position after the - end of the file. +- Issue #10983: Fix several bugs making tunnel requests in http.client. -- Issue #1682942: configparser supports alternative option/value delimiters. +- Issue #10955: zipimport uses ASCII encoding instead of cp437 to decode + filenames, at bootstrap, if the codec registry is not ready yet. It is still + possible to have non-ASCII filenames using the Unicode flag (UTF-8 encoding) + for all file entries in the ZIP file. -Build ------ +- Issue #10949: Improved robustness of rotating file handlers. -- Don't run pgen twice when using make -j. +- Issue #10955: Fix a potential crash when trying to mmap() a file past its + length. Initial patch by Ross Lagerwall. -- Backport r83399 to allow test_distutils to pass on installed versions. +- Issue #10898: Allow compiling the posix module when the C library defines + a symbol named FSTAT. -- Issue #1303434: Generate ZIP file containing all PDBs (already done for rc1). +- Issue #10980: the HTTP server now encodes headers with iso-8859-1 (latin1) + encoding. This is the preferred encoding of PEP 3333 and the base encoding + of HTTP 1.1. -- Stop packaging versioncheck tool (already done for rc1). +- To match the behaviour of HTTP server, the HTTP client library now also + encodes headers with iso-8859-1 (latin1) encoding. It was already doing + that for incoming headers which makes this behaviour now consistent in + both incoming and outgoing direction. -- Accept Oracle Berkeley DB 4.8, 5.0 and 5.1 as backend for the dbm extension. +- Issue #9509: argparse now properly handles IOErrors raised by + argparse.FileType. -Tests ------ +- Issue #10961: The new pydoc server now better handles exceptions raised + during request handling. -- Issue #9424: Replace deprecated assert* methods in the Python test suite. +- Issue #10680: Fix mutually exclusive arguments for argument groups in + argparse. + +Build +----- + +- Issue #11054: Allow Mac OS X installer builds to again work on 10.5 with + the system-provided Python. + + +What's New in Python 3.2 Release Candidate 1 +============================================ + +*Release date: 16-Jan-2011* + +Core and Builtins +----------------- + +- Issue #10889: range indexing and slicing now works correctly on ranges with + a length that exceeds sys.maxsize. + +- Issue #10892: Don't segfault when trying to delete __abstractmethods__ from a + class. + +- Issue #8020: Avoid a crash where the small objects allocator would read + non-Python managed memory while it is being modified by another thread. Patch + by Matt Bandy. + +- Issue #10841: On Windows, set the binary mode on stdin, stdout, stderr and all + io.FileIO objects (to not translate newlines, \r\n <=> \n). The Python parser + translates newlines (\r\n => \n). + +- Remove buffer API from stable ABI for now, see #10181. + +- Issue #8651: PyArg_Parse*() functions raise an OverflowError if the file + doesn't have PY_SSIZE_T_CLEAN define and the size doesn't fit in an int + (length bigger than 2^31-1 bytes). + +- Issue #9015, #9611: FileIO.readinto(), FileIO.write(), os.write() and + stdprinter.write() clamp the length to 2^31-1 on Windows. + +- Issue #8278: On Windows and with a NTFS filesystem, os.stat() and os.utime() + can now handle dates after 2038. + +- Issue #10780: PyErr_SetFromWindowsErrWithFilename() and + PyErr_SetExcFromWindowsErrWithFilename() decode the filename from the + filesystem encoding instead of UTF-8. + +- Issue #10779: PyErr_WarnExplicit() decodes the filename from the filesystem + encoding instead of UTF-8. + +- Add sys.flags attribute for the new -q command-line option. + +- Issue #11506: Trying to assign to a bytes literal should result in a + SyntaxError. + +Library +------- + ++- Issue #12012: ssl.PROTOCOL_SSLv2 becomes optional. ++ +- Issue #10916: mmap should not segfault when a file is mapped using 0 as length + and a non-zero offset, and an attempt to read past the end of file is made + (IndexError is raised instead). Patch by Ross Lagerwall. + +- Issue #10684: shutil.move used to delete a folder on case insensitive + filesystems when the source and destination name where the same except + for the case. + +- Issue #10907: Warn OS X 10.6 IDLE users to use ActiveState Tcl/Tk 8.5, rather + than the currently problematic Apple-supplied one, when running with the + 64-/32-bit installer variant. + +- Issue #4953: cgi.FieldStorage and cgi.parse() parse the request as bytes, not + as unicode, and accept binary files. Add encoding and errors attributes to + cgi.FieldStorage. Patch written by Pierre Quentel (with many inputs by Glenn + Linderman). + +- Add encoding and errors arguments to urllib.parse_qs() and urllib.parse_qsl(). + +- Issue #10899: No function type annotations in the standard library. Removed + function type annotations from _pyio.py. + +- Issue #10875: Update Regular Expression HOWTO; patch by 'SilentGhost'. + +- Issue #10872: The repr() of TextIOWrapper objects now includes the mode + if available. + +- Issue #10869: Fixed bug where ast.increment_lineno modified the root node + twice. + +- Issue #5871: email.header.Header.encode now raises an error if any + continuation line in the formatted value has no leading white space and looks + like a header. Since Generator uses Header to format all headers, this check + is made for all headers in any serialized message at serialization time. This + provides protection against header injection attacks. + +- Issue #10859: Make ``contextlib.GeneratorContextManager`` officially + private by renaming it to ``_GeneratorContextManager``. + +- Issue #10042: Fixed the total_ordering decorator to handle cross-type + comparisons that could lead to infinite recursion. + +- Issue #10686: the email package now :rfc:`2047`\ -encodes headers with + non-ASCII bytes (parsed by a Bytes Parser) when doing conversion to 7bit-clean + presentation, instead of replacing them with ?s. + +- email.header.Header was incorrectly encoding folding white space when + rfc2047-encoding header values with embedded newlines, leaving them without + folding whitespace. It now uses the continuation_ws, as it does for + continuation lines that it creates itself. + +- Issue #1777412, #10827: Changed the rules for 2-digit years. The + time.asctime(), time.ctime() and time.strftime() functions will now format + any year when ``time.accept2dyear`` is False and will accept years >= 1000 + otherwise. ``time.mktime`` and ``time.strftime`` now accept full range + supported by the OS. With Visual Studio or on Solaris, the year is limited to + the range [1; 9999]. Conversion of 2-digit years to 4-digit is deprecated. + +- Issue #7858: Raise an error properly when os.utime() fails under Windows + on an existing file. + +- Issue #3839: wsgiref should not override a Content-Length header set by + the application. Initial patch by Clovis Fabricio. + +- Issue #10492: bdb.Bdb.run() only traces the execution of the code, not the + compilation (if the input is a string). + +- Issue #7995: When calling accept() on a socket with a timeout, the returned + socket is now always blocking, regardless of the operating system. + +- Issue #10756: atexit normalizes the exception before displaying it. Patch by + Andreas Stührk. + +- Issue #10790: email.header.Header.append's charset logic now works correctly + for charsets whose output codec is different from its input codec. + +- Issue #10819: SocketIO.name property returns -1 when its closed, instead of + raising a ValueError, to fix repr(). + +- Issue #8650: zlib.compress() and zlib.decompress() raise an OverflowError if + the input buffer length doesn't fit into an unsigned int (length bigger than + 2^32-1 bytes). + +- Issue #6643: Reinitialize locks held within the threading module after fork to + avoid a potential rare deadlock or crash on some platforms. + +- Issue #10806, issue #9905: Fix subprocess pipes when some of the standard file + descriptors (0, 1, 2) are closed in the parent process. Initial patch by Ross + Lagerwall. + +- `unittest.TestCase` can be instantiated without a method name; for simpler + exploration from the interactive interpreter. + +- Issue #10798: Reject supporting concurrent.futures if the system has too + few POSIX semaphores. + +- Issue #10807: Remove base64, bz2, hex, quopri, rot13, uu and zlib codecs from + the codec aliases. They are still accessible via codecs.lookup(). + +- Issue #10801: In zipfile, support different encodings for the header and the + filenames. + +- Issue #6285: IDLE no longer crashes on missing help file; patch by Scott + David Daniels. + +- Fix collections.OrderedDict.setdefault() so that it works in subclasses that + define __missing__(). + +- Issue #10786: unittest.TextTestRunner default stream no longer bound at import + time. `sys.stderr` now looked up at instantiation time. Fix contributed by + Mark Roddy. + +- Issue #10753: Characters ';', '=' and ',' in the PATH_INFO environment variable + won't be quoted when the URI is constructed by the wsgiref.util's request_uri + method. According to RFC 3986, these characters can be a part of params in + PATH component of URI and need not be quoted. + +- Issue #10738: Fix webbrowser.Opera.raise_opts. + +- Issue #9824: SimpleCookie now encodes , and ; in values to cater to how + browsers actually parse cookies. + +- Issue #9333: os.symlink now available regardless of user privileges. The + function now raises OSError on Windows >=6.0 when the user is unable to create + symbolic links. XP and 2003 still raise NotImplementedError. + +- Issue #10783: struct.pack() no longer implicitly encodes unicode to UTF-8. + +- Issue #10730: Add SVG mime types to mimetypes module. + +- Issue #10768: Make the Tkinter ScrolledText widget work again. + +- Issue #10777: Fix "dictionary changed size during iteration" bug in + ElementTree register_namespace(). + +- Issue #10626: test_logging now preserves logger disabled states. + +- Issue #10774: test_logging now removes temp files created during tests. + +- Issue #5258/#10642: if site.py encounters a .pth file that generates an error, + it now prints the filename, line number, and traceback to stderr and skips + the rest of that individual file, instead of stopping processing entirely. + +- Issue #10763: subprocess.communicate() closes stdout and stderr if both are + pipes (bug specific to Windows). + +- Issue #1693546: fix email.message RFC 2231 parameter encoding to be in better + compliance (no "s around encoded values). + +- Improved the diff message in the unittest module's assertCountEqual(). + +- Issue #1155362: email.utils.parsedate_tz now handles a missing space before + the '-' of a timezone field as well as before a '+'. + +- Issue #4871: The zipfile module now gives a more useful error message if + an attempt is made to use a string to specify the archive password. + +- Issue #10750: The ``raw`` attribute of buffered IO objects is now read-only. + +- Deprecated assertDictContainsSubset() in the unittest module. + +C-API +----- + +- Issue #10913: Deprecate misleading functions PyEval_AcquireLock() and + PyEval_ReleaseLock(). The thread-state aware APIs should be used instead. + +- Issue #10333: Remove ancient GC API, which has been deprecated since Python + 2.2. + +Build +----- + +- Issue #10843: Update third-party library versions used in OS X 32-bit + installer builds: bzip2 1.0.6, readline 6.1.2, SQLite 3.7.4 (with FTS3/FTS4 + and RTREE enabled), and ncursesw 5.5 (wide-char support enabled). + +- Issue #10820: Fix OS X framework installs to support version-specific + scripts (#10679). + +- Issue #7716: Under Solaris, don't assume existence of /usr/xpg4/bin/grep in + the configure script but use $GREP instead. Patch by Fabian Groffen. + +- Issue #10475: Don't hardcode compilers for LDSHARED/LDCXXSHARED on NetBSD + and DragonFly BSD. Patch by Nicolas Joly. + +- Issue #10679: The "idle", "pydoc" and "2to3" scripts are now installed with + a version-specific suffix on "make altinstall". + +- Issue #10655: Fix the build on PowerPC on Linux with GCC when building with + timestamp profiling (--with-tsc): the preprocessor test for the PowerPC + support now looks for "__powerpc__" as well as "__ppc__": the latter seems to + only be present on OS X; the former is the correct one for Linux with GCC. + +- Issue #1099: Fix the build on MacOSX when building a framework with pydebug + using GCC 4.0. + +Tools/Demos +----------- + +- Issue #10843: Install the Tools directory on OS X in the applications Extras + (/Applications/Python 3.n/Extras/) where the Demo directory had previous been + installed. + +- Issue #7962: The Demo directory is gone. Most of the old and unmaintained + demos have been removed, others integrated in documentation or a new + Tools/demo subdirectory. + +- Issue #10502: Addition of the unittestgui tool. Originally by Steve Purcell. + Updated for test discovery by Mark Roddy and Python 3 compatibility by Brian + Curtin. + +Tests +----- + +- Issue #11910: Fix test_heapq to skip the C tests when _heapq is missing. + +- Fix test_startfile to wait for child process to terminate before finishing. + +- Issue #10822: Fix test_posix:test_getgroups failure under Solaris. Patch + by Ross Lagerwall. + +- Make the --coverage flag work for test.regrtest. + +- Issue #1677694: Refactor and improve test_timeout. Original patch by + Björn Lindqvist. + +- Issue #5485: Add tests for the UseForeignDTD method of expat parser objects. + Patch by Jean-Paul Calderone and Sandro Tosi. + +- Issue #6293: Have regrtest.py echo back sys.flags. This is done by default in + whole runs and enabled selectively using ``--header`` when running an explicit + list of tests. Original patch by Collin Winter. + + +What's New in Python 3.2 Beta 2? +================================ + +*Release date: 19-Dec-2010* + +Core and Builtins +----------------- + +- Issue #8844: Regular and recursive lock acquisitions can now be interrupted + by signals on platforms using pthreads. Patch by Reid Kleckner. + +- Issue #4236: PyModule_Create2 now checks the import machinery directly + rather than the Py_IsInitialized flag, avoiding a Fatal Python + error in certain circumstances when an import is done in __del__. + +- Issue #5587: add a repr to dict_proxy objects. Patch by David Stanek and + Daniel Urban. + +Library +------- + +- Issue #3243: Support iterable bodies in httplib. Patch Contributions by + Xuanji Li and Chris AtLee. + +- Issue #10611: SystemExit exception will no longer kill a unittest run. + +- Issue #9857: It is now possible to skip a test in a setUp, tearDown or clean + up function. + +- Issue #10573: use actual/expected consistently in unittest methods. + The order of the args of assertCountEqual is also changed. + +- Issue #9286: email.utils.parseaddr no longer concatenates blank-separated + words in the local part of email addresses, thereby preserving the input. + +- Issue #6791: Limit header line length (to 65535 bytes) in http.client + and http.server, to avoid denial of services from the other party. + +- Issue #10404: Use ctl-button-1 on OSX for the context menu in Idle. + +- Issue #9907: Fix tab handling on OSX when using editline by calling + rl_initialize first, then setting our custom defaults, then reading .editrc. + +- Issue #4188: Avoid creating dummy thread objects when logging operations + from the threading module (with the internal verbose flag activated). + +- Issue #10711: Remove HTTP 0.9 support from http.client. The ``strict`` + parameter to HTTPConnection and friends is deprecated. + +- Issue #9721: Fix the behavior of urljoin when the relative url starts with a + ';' character. Patch by Wes Chow. + +- Issue #10714: Limit length of incoming request in http.server to 65536 bytes + for security reasons. Initial patch by Ross Lagerwall. + +- Issue #9558: Fix distutils.command.build_ext with VS 8.0. + +- Issue #10667: Fast path for collections.Counter(). + +- Issue #10695: passing the port as a string value to telnetlib no longer + causes debug mode to fail. + +- Issue #1078919: add_header now automatically RFC2231 encodes parameters + that contain non-ascii values. + +- Issue #10188 (partial resolution): tempfile.TemporaryDirectory emits + a warning on sys.stderr rather than throwing a misleading exception + if cleanup fails due to nulling out of modules during shutdown. + Also avoids an AttributeError when mkdtemp call fails and issues + a ResourceWarning on implicit cleanup via __del__. + +- Issue #10107: Warn about unsaved files in IDLE on OSX. + +- Issue #7213: subprocess.Popen's default for close_fds has been changed. + It is now True in most cases other than on Windows when input, output or + error handles are provided. + +- Issue #6559: subprocess.Popen has a new pass_fds parameter (actually + added in 3.2beta1) to allow specifying a specific list of file descriptors + to keep open in the child process. + +- Issue #1731717: Fixed the problem where subprocess.wait() could cause an + OSError exception when The OS had been told to ignore SIGCLD in our process + or otherwise not wait for exiting child processes. + +Tests +----- + +- Issue #775964: test_grp now skips YP/NIS entries instead of failing when + encountering them. + +Tools/Demos +----------- + +- Issue #6075: IDLE on Mac OS X now works with both Carbon AquaTk and + Cocoa AquaTk. + +- Issue #10710: ``Misc/setuid-prog.c`` is removed from the source tree. + +- Issue #10706: Remove outdated script runtests.sh. Either ``make test`` + or ``python -m test`` should be used instead. + +Build +----- + +- The Windows build now uses Tcl/Tk 8.5.9 and sqlite3 3.7.4. + +- Issue #9234: argparse supports alias names for subparsers. + + +What's New in Python 3.2 Beta 1? +================================ + +*Release date: 05-Dec-2010* + +Core and Builtins +----------------- + +- Issue #10630: Return dict views from the dict proxy keys()/values()/items() + methods. + +- Issue #10596: Fix float.__mod__ to have the same behaviour as float.__divmod__ + with respect to signed zeros. -4.0 % 4.0 should be 0.0, not -0.0. + +- Issue #1772833: Add the -q command-line option to suppress copyright and + version output in interactive mode. + +- Provide an *optimize* parameter in the built-in compile() function. + +- Fixed several corner case issues on Windows in os.stat/os.lstat related to + reparse points. + +- PEP 384 (Defining a Stable ABI) is implemented. + +- Issue #2690: Range objects support negative indices and slicing. + +- Issue #9915: Speed up sorting with a key. + +- Issue #8685: Speed up set difference ``a - b`` when source set ``a`` is much + larger than operand ``b``. Patch by Andrew Bennetts. + +- Issue #10518: Bring back the callable() builtin. + +- Issue #7094: Added alternate formatting (specified by '#') to ``__format__`` + method of float, complex, and Decimal. This allows more precise control over + when decimal points are displayed. + +- Issue #10474: range.count() should return integers. + +- Issue #1574217: isinstance now catches only AttributeError, rather than + masking all errors. + +Library +------- + +- logging: added "handler of last resort". See http://bit.ly/last-resort-handler + +- test.support: Added TestHandler and Matcher classes for better support of + assertions about logging. + +- Issue #4391: Use proper plural forms in argparse. + +- Issue #10601: sys.displayhook uses 'backslashreplace' error handler on + UnicodeEncodeError. + +- Add the "display" and "undisplay" pdb commands. + +- Issue #7245: Add a SIGINT handler in pdb that allows to break a program again + after a "continue" command. + +- Add the "interact" pdb command. + +- Issue #7905: Actually respect the keyencoding parameter to shelve.Shelf. + +- Issue #1569291: Speed up array.repeat(). + +- Provide an interface to set the optimization level of compilation in + py_compile, compileall and zipfile.PyZipFile. + +- Issue #7904: Changes to urllib.parse.urlsplit to handle schemes as defined by + RFC3986. Anything before :// is considered a scheme and is followed by an + authority (or netloc) and by '/' led path, which is optional. + +- Issue #6045: dbm.gnu databases now support get() and setdefault() methods. + +- Issue #10620: `python -m unittest` can accept file paths instead of module + names for running specific tests. + +- Issue #9424: Deprecate the `unittest.TestCase` methods `assertEquals`, + `assertNotEquals`, `assertAlmostEquals`, `assertNotAlmostEquals` and `assert_` + and replace them with the correct methods in the Python test suite. + +- Issue #10272: The ssl module now raises socket.timeout instead of a generic + SSLError on socket timeouts. + +- Issue #10528: Allow translators to reorder placeholders in localizable + messages from argparse. + +- Issue #10497: Fix incorrect use of gettext in argparse. + +- Issue #10478: Reentrant calls inside buffered IO objects (for example by + way of a signal handler) now raise a RuntimeError instead of freezing the + current process. + +- logging: Added getLogRecordFactory/setLogRecordFactory with docs and tests. + +- Issue #10549: Fix pydoc traceback when text-documenting certain classes. + +- Issue #2001: New HTML server with enhanced Web page features. Patch by Ron + Adam. + +- Issue #10360: In WeakSet, do not raise TypeErrors when testing for membership + of non-weakrefable objects. + +- Issue #940286: pydoc.Helper.help() ignores input/output init parameters. + +- Issue #1745035: Add a command size and data size limit to smtpd.py, to prevent + DoS attacks. Patch by Savio Sena. + +- Issue #4925: Add filename to error message when executable can't be found in + subprocess. + +- Issue #10391: Don't dereference invalid memory in error messages in the ast + module. + +- Issue #10027: st_nlink was not being set on Windows calls to os.stat or + os.lstat. Patch by Hirokazu Yamamoto. + +- Issue #9333: Expose os.symlink only when the SeCreateSymbolicLinkPrivilege is + held by the user's account, i.e., when the function can actually be used. + +- Issue #8879: Add os.link support for Windows. + +- Issue #7911: ``unittest.TestCase.longMessage`` defaults to True for improved + failure messages by default. Patch by Mark Roddy. + +- Issue #1486713: HTMLParser now has an optional tolerant mode where it tries to + guess at the correct parsing of invalid html. + +- Issue #10554: Add context manager support to subprocess.Popen objects. + +- Issue #8989: email.utils.make_msgid now has a domain parameter that can + override the domain name used in the generated msgid. + +- Issue #9299: Add exist_ok parameter to os.makedirs to suppress the 'File + exists' exception when a target directory already exists with the specified + mode. Patch by Ray Allen. + +- Issue #9573: os.fork() now works correctly when triggered as a side effect of + a module import. + +- Issue #10464: netrc now correctly handles lines with embedded '#' characters. + +- Added itertools.accumulate(). + +- Issue #4113: Added custom ``__repr__`` method to ``functools.partial``. + Original patch by Daniel Urban. + +- Issue #10273: Rename `assertRegexpMatches` and `assertRaisesRegexp` to + `assertRegex` and `assertRaisesRegex`. + +- Issue #10535: Enable silenced warnings in unittest by default. + +- Issue #9873: The URL parsing functions in urllib.parse now accept ASCII byte + sequences as input in addition to character strings. + +- Issue #10586: The statistics API for the new functools.lru_cache has been + changed to a single cache_info() method returning a named tuple. + +- Issue #10323: itertools.islice() now consumes the minimum number of inputs + before stopping. Formerly, the final state of the underlying iterator was + undefined. + +- Issue #10565: The collections.Iterator ABC now checks for both __iter__ and + __next__. + +- Issue #10242: Fixed implementation of unittest.ItemsEqual and gave it a new + more informative name, unittest.CountEqual. + +- Issue #10561: In pdb, clear the breakpoints by the breakpoint number. + +- Issue #2986: difflib.SequenceMatcher gets a new parameter, autojunk, which can + be set to False to turn off the previously undocumented 'popularity' + heuristic. Patch by Terry Reedy and Eli Bendersky. + +- Issue #10534: in difflib, expose bjunk and bpopular sets; deprecate + undocumented and now redundant isbjunk and isbpopular methods. + +- Issue #9846: zipfile is now correctly closing underlying file objects. + +- Issue #10459: Update CJK character names to Unicode 6.0. + +- Issue #4493: urllib.request adds '/' in front of path components which does not + start with '/. Common behavior exhibited by browsers and other clients. + +- Issue #6378: idle.bat now runs with the appropriate Python version rather than + the system default. Patch by Sridhar Ratnakumar. + +- Issue #10470: 'python -m unittest' will now run test discovery by default, + when no extra arguments have been provided. + +- Issue #3709: BaseHTTPRequestHandler will buffer the headers and write to + output stream only when end_headers is invoked. This is a speedup and an + internal optimization. Patch by endian. + +- Issue #10220: Added inspect.getgeneratorstate. Initial patch by Rodolpho + Eckhardt. + +- Issue #10453: compileall now uses argparse instead of getopt, and thus + provides clean output when called with '-h'. + +- Issue #8078: Add constants for higher baud rates in the termios module. Patch + by Rodolpho Eckhardt. + +- Issue #10407: Fix two NameErrors in distutils. + +- Issue #10371: Deprecated undocumented functions in the trace module. + +- Issue #10467: Fix BytesIO.readinto() after seeking into a position after the + end of the file. + +- configparser: 100% test coverage. + +- Issue #10499: configparser supports pluggable interpolation handlers. The + default classic interpolation handler is called BasicInterpolation. Another + interpolation handler added (ExtendedInterpolation) which supports the syntax + used by zc.buildout (e.g. interpolation between sections). + +- configparser: the SafeConfigParser class has been renamed to ConfigParser. + The legacy ConfigParser class has been removed but its interpolation mechanism + is still available as LegacyInterpolation. + +- configparser: Usage of RawConfigParser is now discouraged for new projects + in favor of ConfigParser(interpolation=None). + +- Issue #1682942: configparser supports alternative option/value delimiters. + +- Issue #5412: configparser supports mapping protocol access. + +- Issue #9411: configparser supports specifying encoding for read operations. + +- Issue #9421: configparser's getint(), getfloat() and getboolean() methods + accept vars and default arguments just like get() does. + +- Issue #9452: configparser supports reading from strings and dictionaries + (thanks to the mapping protocol API, the latter can be used to copy data + between parsers). + +- configparser: accepted INI file structure is now customizable, including + comment prefixes, name of the DEFAULT section, empty lines in multiline + values, and indentation. + +- Issue #10326: unittest.TestCase instances can be pickled. + +- Issue #9926: Wrapped TestSuite subclass does not get __call__ executed. + +- Issue #9920: Skip tests for cmath.atan and cmath.atanh applied to complex + zeros on systems where the log1p function fails to respect the sign of zero. + This fixes a test failure on AIX. + +- Issue #9732: Addition of getattr_static to the inspect module. + +- Issue #10446: Module documentation generated by pydoc now links to a + version-specific online reference manual. + +- Make the 'No module named' exception message from importlib consistent. + +- Issue #10443: Add the SSLContext.set_default_verify_paths() method. + +- Issue #10440: Support RUSAGE_THREAD as a constant in the resource module. + Patch by Robert Collins. + +- Issue #10429: IMAP.starttls() stored the capabilities as bytes objects, rather + than strings. + +C-API +----- + +- Issue #10557: Added a new API function, PyUnicode_TransformDecimalToASCII(), + which transforms non-ASCII decimal digits in a Unicode string to their ASCII + equivalents. + +- Issue #9518: Extend the PyModuleDef_HEAD_INIT macro to explicitly + zero-initialize all fields, fixing compiler warnings seen when building + extension modules with gcc with "-Wmissing-field-initializers" (implied by + "-W"). + +- Issue #10255: Fix reference leak in Py_InitializeEx(). Patch by Neil + Schemenauer. + +- structseq.h is now included in Python.h. + +- Loosen PyArg_ValidateKeywordArguments to allow dict subclasses. + +Tests +----- + +- regrtest.py once again ensures the test directory is removed from sys.path + when it is invoked directly as the __main__ module. + +- `python -m test` can be used to run the test suite as well as `python -m + test.regrtest`. + +- Do not fail test_socket when the IP address of the local hostname cannot be + looked up. + +- Issue #8886: Use context managers throughout test_zipfile. Patch by Eric + Carstensen. + +Build +----- + +- Issue #10325: Fix two issues in the fallback definitions for PY_ULLONG_MAX and + PY_LLONG_MAX that made them unsuitable for use in preprocessor conditionals. Documentation ------------- diff --cc Modules/_ssl.c index e49426464e5c,8ebdc9b99db2..a813d5fef27c --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@@ -1427,441 -1550,6 +1429,443 @@@ static PyTypeObject PySSLSocket_Type = PySSLMethods, /*tp_methods*/ }; + +/* + * _SSLContext objects + */ + +static PyObject * +context_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + char *kwlist[] = {"protocol", NULL}; + PySSLContext *self; + int proto_version = PY_SSL_VERSION_SSL23; + SSL_CTX *ctx = NULL; + + if (!PyArg_ParseTupleAndKeywords( + args, kwds, "i:_SSLContext", kwlist, + &proto_version)) + return NULL; + + PySSL_BEGIN_ALLOW_THREADS + if (proto_version == PY_SSL_VERSION_TLS1) + ctx = SSL_CTX_new(TLSv1_method()); + else if (proto_version == PY_SSL_VERSION_SSL3) + ctx = SSL_CTX_new(SSLv3_method()); ++#ifndef OPENSSL_NO_SSL2 + else if (proto_version == PY_SSL_VERSION_SSL2) + ctx = SSL_CTX_new(SSLv2_method()); ++#endif + else if (proto_version == PY_SSL_VERSION_SSL23) + ctx = SSL_CTX_new(SSLv23_method()); + else + proto_version = -1; + PySSL_END_ALLOW_THREADS + + if (proto_version == -1) { + PyErr_SetString(PyExc_ValueError, + "invalid protocol version"); + return NULL; + } + if (ctx == NULL) { + PyErr_SetString(PySSLErrorObject, + "failed to allocate SSL context"); + return NULL; + } + + assert(type != NULL && type->tp_alloc != NULL); + self = (PySSLContext *) type->tp_alloc(type, 0); + if (self == NULL) { + SSL_CTX_free(ctx); + return NULL; + } + self->ctx = ctx; + /* Defaults */ + SSL_CTX_set_verify(self->ctx, SSL_VERIFY_NONE, NULL); + SSL_CTX_set_options(self->ctx, SSL_OP_ALL); + +#define SID_CTX "Python" + SSL_CTX_set_session_id_context(self->ctx, (const unsigned char *) SID_CTX, + sizeof(SID_CTX)); +#undef SID_CTX + + return (PyObject *)self; +} + +static void +context_dealloc(PySSLContext *self) +{ + SSL_CTX_free(self->ctx); + Py_TYPE(self)->tp_free(self); +} + +static PyObject * +set_ciphers(PySSLContext *self, PyObject *args) +{ + int ret; + const char *cipherlist; + + if (!PyArg_ParseTuple(args, "s:set_ciphers", &cipherlist)) + return NULL; + ret = SSL_CTX_set_cipher_list(self->ctx, cipherlist); + if (ret == 0) { + /* Clearing the error queue is necessary on some OpenSSL versions, + otherwise the error will be reported again when another SSL call + is done. */ + ERR_clear_error(); + PyErr_SetString(PySSLErrorObject, + "No cipher can be selected."); + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +get_verify_mode(PySSLContext *self, void *c) +{ + switch (SSL_CTX_get_verify_mode(self->ctx)) { + case SSL_VERIFY_NONE: + return PyLong_FromLong(PY_SSL_CERT_NONE); + case SSL_VERIFY_PEER: + return PyLong_FromLong(PY_SSL_CERT_OPTIONAL); + case SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT: + return PyLong_FromLong(PY_SSL_CERT_REQUIRED); + } + PyErr_SetString(PySSLErrorObject, + "invalid return value from SSL_CTX_get_verify_mode"); + return NULL; +} + +static int +set_verify_mode(PySSLContext *self, PyObject *arg, void *c) +{ + int n, mode; + if (!PyArg_Parse(arg, "i", &n)) + return -1; + if (n == PY_SSL_CERT_NONE) + mode = SSL_VERIFY_NONE; + else if (n == PY_SSL_CERT_OPTIONAL) + mode = SSL_VERIFY_PEER; + else if (n == PY_SSL_CERT_REQUIRED) + mode = SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT; + else { + PyErr_SetString(PyExc_ValueError, + "invalid value for verify_mode"); + return -1; + } + SSL_CTX_set_verify(self->ctx, mode, NULL); + return 0; +} + +static PyObject * +get_options(PySSLContext *self, void *c) +{ + return PyLong_FromLong(SSL_CTX_get_options(self->ctx)); +} + +static int +set_options(PySSLContext *self, PyObject *arg, void *c) +{ + long new_opts, opts, set, clear; + if (!PyArg_Parse(arg, "l", &new_opts)) + return -1; + opts = SSL_CTX_get_options(self->ctx); + clear = opts & ~new_opts; + set = ~opts & new_opts; + if (clear) { +#ifdef HAVE_SSL_CTX_CLEAR_OPTIONS + SSL_CTX_clear_options(self->ctx, clear); +#else + PyErr_SetString(PyExc_ValueError, + "can't clear options before OpenSSL 0.9.8m"); + return -1; +#endif + } + if (set) + SSL_CTX_set_options(self->ctx, set); + return 0; +} + +static PyObject * +load_cert_chain(PySSLContext *self, PyObject *args, PyObject *kwds) +{ + char *kwlist[] = {"certfile", "keyfile", NULL}; + PyObject *certfile, *keyfile = NULL; + PyObject *certfile_bytes = NULL, *keyfile_bytes = NULL; + int r; + + errno = 0; + ERR_clear_error(); + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O|O:load_cert_chain", kwlist, + &certfile, &keyfile)) + return NULL; + if (keyfile == Py_None) + keyfile = NULL; + if (!PyUnicode_FSConverter(certfile, &certfile_bytes)) { + PyErr_SetString(PyExc_TypeError, + "certfile should be a valid filesystem path"); + return NULL; + } + if (keyfile && !PyUnicode_FSConverter(keyfile, &keyfile_bytes)) { + PyErr_SetString(PyExc_TypeError, + "keyfile should be a valid filesystem path"); + goto error; + } + PySSL_BEGIN_ALLOW_THREADS + r = SSL_CTX_use_certificate_chain_file(self->ctx, + PyBytes_AS_STRING(certfile_bytes)); + PySSL_END_ALLOW_THREADS + if (r != 1) { + if (errno != 0) { + ERR_clear_error(); + PyErr_SetFromErrno(PyExc_IOError); + } + else { + _setSSLError(NULL, 0, __FILE__, __LINE__); + } + goto error; + } + PySSL_BEGIN_ALLOW_THREADS + r = SSL_CTX_use_PrivateKey_file(self->ctx, + PyBytes_AS_STRING(keyfile ? keyfile_bytes : certfile_bytes), + SSL_FILETYPE_PEM); + PySSL_END_ALLOW_THREADS + Py_XDECREF(keyfile_bytes); + Py_XDECREF(certfile_bytes); + if (r != 1) { + if (errno != 0) { + ERR_clear_error(); + PyErr_SetFromErrno(PyExc_IOError); + } + else { + _setSSLError(NULL, 0, __FILE__, __LINE__); + } + return NULL; + } + PySSL_BEGIN_ALLOW_THREADS + r = SSL_CTX_check_private_key(self->ctx); + PySSL_END_ALLOW_THREADS + if (r != 1) { + _setSSLError(NULL, 0, __FILE__, __LINE__); + return NULL; + } + Py_RETURN_NONE; + +error: + Py_XDECREF(keyfile_bytes); + Py_XDECREF(certfile_bytes); + return NULL; +} + +static PyObject * +load_verify_locations(PySSLContext *self, PyObject *args, PyObject *kwds) +{ + char *kwlist[] = {"cafile", "capath", NULL}; + PyObject *cafile = NULL, *capath = NULL; + PyObject *cafile_bytes = NULL, *capath_bytes = NULL; + const char *cafile_buf = NULL, *capath_buf = NULL; + int r; + + errno = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "|OO:load_verify_locations", kwlist, + &cafile, &capath)) + return NULL; + if (cafile == Py_None) + cafile = NULL; + if (capath == Py_None) + capath = NULL; + if (cafile == NULL && capath == NULL) { + PyErr_SetString(PyExc_TypeError, + "cafile and capath cannot be both omitted"); + return NULL; + } + if (cafile && !PyUnicode_FSConverter(cafile, &cafile_bytes)) { + PyErr_SetString(PyExc_TypeError, + "cafile should be a valid filesystem path"); + return NULL; + } + if (capath && !PyUnicode_FSConverter(capath, &capath_bytes)) { + Py_XDECREF(cafile_bytes); + PyErr_SetString(PyExc_TypeError, + "capath should be a valid filesystem path"); + return NULL; + } + if (cafile) + cafile_buf = PyBytes_AS_STRING(cafile_bytes); + if (capath) + capath_buf = PyBytes_AS_STRING(capath_bytes); + PySSL_BEGIN_ALLOW_THREADS + r = SSL_CTX_load_verify_locations(self->ctx, cafile_buf, capath_buf); + PySSL_END_ALLOW_THREADS + Py_XDECREF(cafile_bytes); + Py_XDECREF(capath_bytes); + if (r != 1) { + if (errno != 0) { + ERR_clear_error(); + PyErr_SetFromErrno(PyExc_IOError); + } + else { + _setSSLError(NULL, 0, __FILE__, __LINE__); + } + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +context_wrap_socket(PySSLContext *self, PyObject *args, PyObject *kwds) +{ + char *kwlist[] = {"sock", "server_side", "server_hostname", NULL}; + PySocketSockObject *sock; + int server_side = 0; + char *hostname = NULL; + PyObject *hostname_obj, *res; + + /* server_hostname is either None (or absent), or to be encoded + using the idna encoding. */ + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!i|O!:_wrap_socket", kwlist, + PySocketModule.Sock_Type, + &sock, &server_side, + Py_TYPE(Py_None), &hostname_obj)) { + PyErr_Clear(); + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!iet:_wrap_socket", kwlist, + PySocketModule.Sock_Type, + &sock, &server_side, + "idna", &hostname)) + return NULL; +#ifndef SSL_CTRL_SET_TLSEXT_HOSTNAME + PyMem_Free(hostname); + PyErr_SetString(PyExc_ValueError, "server_hostname is not supported " + "by your OpenSSL library"); + return NULL; +#endif + } + + res = (PyObject *) newPySSLSocket(self->ctx, sock, server_side, + hostname); + if (hostname != NULL) + PyMem_Free(hostname); + return res; +} + +static PyObject * +session_stats(PySSLContext *self, PyObject *unused) +{ + int r; + PyObject *value, *stats = PyDict_New(); + if (!stats) + return NULL; + +#define ADD_STATS(SSL_NAME, KEY_NAME) \ + value = PyLong_FromLong(SSL_CTX_sess_ ## SSL_NAME (self->ctx)); \ + if (value == NULL) \ + goto error; \ + r = PyDict_SetItemString(stats, KEY_NAME, value); \ + Py_DECREF(value); \ + if (r < 0) \ + goto error; + + ADD_STATS(number, "number"); + ADD_STATS(connect, "connect"); + ADD_STATS(connect_good, "connect_good"); + ADD_STATS(connect_renegotiate, "connect_renegotiate"); + ADD_STATS(accept, "accept"); + ADD_STATS(accept_good, "accept_good"); + ADD_STATS(accept_renegotiate, "accept_renegotiate"); + ADD_STATS(accept, "accept"); + ADD_STATS(hits, "hits"); + ADD_STATS(misses, "misses"); + ADD_STATS(timeouts, "timeouts"); + ADD_STATS(cache_full, "cache_full"); + +#undef ADD_STATS + + return stats; + +error: + Py_DECREF(stats); + return NULL; +} + +static PyObject * +set_default_verify_paths(PySSLContext *self, PyObject *unused) +{ + if (!SSL_CTX_set_default_verify_paths(self->ctx)) { + _setSSLError(NULL, 0, __FILE__, __LINE__); + return NULL; + } + Py_RETURN_NONE; +} + +static PyGetSetDef context_getsetlist[] = { + {"options", (getter) get_options, + (setter) set_options, NULL}, + {"verify_mode", (getter) get_verify_mode, + (setter) set_verify_mode, NULL}, + {NULL}, /* sentinel */ +}; + +static struct PyMethodDef context_methods[] = { + {"_wrap_socket", (PyCFunction) context_wrap_socket, + METH_VARARGS | METH_KEYWORDS, NULL}, + {"set_ciphers", (PyCFunction) set_ciphers, + METH_VARARGS, NULL}, + {"load_cert_chain", (PyCFunction) load_cert_chain, + METH_VARARGS | METH_KEYWORDS, NULL}, + {"load_verify_locations", (PyCFunction) load_verify_locations, + METH_VARARGS | METH_KEYWORDS, NULL}, + {"session_stats", (PyCFunction) session_stats, + METH_NOARGS, NULL}, + {"set_default_verify_paths", (PyCFunction) set_default_verify_paths, + METH_NOARGS, NULL}, + {NULL, NULL} /* sentinel */ +}; + +static PyTypeObject PySSLContext_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "_ssl._SSLContext", /*tp_name*/ + sizeof(PySSLContext), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)context_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_reserved*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + context_methods, /*tp_methods*/ + 0, /*tp_members*/ + context_getsetlist, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + context_new, /*tp_new*/ +}; + + + #ifdef HAVE_OPENSSL_RAND /* helper routines for seeding the SSL PRNG */