From: Benjamin Peterson Date: Thu, 6 Oct 2016 05:54:19 +0000 (-0700) Subject: merge 3.5 X-Git-Tag: v3.6.0b2~42 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=87845bcb4ded91fea604ff384090d1a13425599c;p=thirdparty%2FPython%2Fcpython.git merge 3.5 --- 87845bcb4ded91fea604ff384090d1a13425599c diff --cc Lib/test/test_mmap.py index bbb4070148e9,f3f70cc25119..56d85e75f87a --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@@ -713,13 -713,20 +713,27 @@@ class MmapTests(unittest.TestCase) gc_collect() self.assertIs(wr(), None) + def test_write_returning_the_number_of_bytes_written(self): + mm = mmap.mmap(-1, 16) + self.assertEqual(mm.write(b""), 0) + self.assertEqual(mm.write(b"x"), 1) + self.assertEqual(mm.write(b"yz"), 2) + self.assertEqual(mm.write(b"python"), 6) + + @unittest.skipIf(os.name == 'nt', 'cannot resize anonymous mmaps on Windows') + def test_resize_past_pos(self): + m = mmap.mmap(-1, 8192) + self.addCleanup(m.close) + m.read(5000) + try: + m.resize(4096) + except SystemError: + self.skipTest("resizing not supported") + self.assertEqual(m.read(14), b'') + self.assertRaises(ValueError, m.read_byte) + self.assertRaises(ValueError, m.write_byte, 42) + self.assertRaises(ValueError, m.write, b'abc') + class LargeMmapTests(unittest.TestCase): diff --cc Misc/NEWS index fef4f0e4debb,2da50588206c..d58d33dcfe74 --- a/Misc/NEWS +++ b/Misc/NEWS @@@ -141,271 -140,10 +141,274 @@@ Librar - Issue #25895: Enable WebSocket URL schemes in urllib.parse.urljoin. Patch by Gergely Imreh and Markus Holtermann. +- Issue #28114: Fix a crash in parse_envlist() when env contains byte strings. + Patch by Eryk Sun. + - Issue #27599: Fixed buffer overrun in binascii.b2a_qp() and binascii.a2b_qp(). -- Issue #19003:m email.generator now replaces only \r and/or \n line - endings, per the RFC, instead of all unicode line endings. +- Issue #27906: Fix socket accept exhaustion during high TCP traffic. + Patch by Kevin Conway. + +- Issue #28174: Handle when SO_REUSEPORT isn't properly supported. + Patch by Seth Michael Larson. + +- Issue #26654: Inspect functools.partial in asyncio.Handle.__repr__. + Patch by iceboy. + +- Issue #26909: Fix slow pipes IO in asyncio. + Patch by INADA Naoki. + +- Issue #28176: Fix callbacks race in asyncio.SelectorLoop.sock_connect. + +- Issue #27759: Fix selectors incorrectly retain invalid file descriptors. + Patch by Mark Williams. + +- Issue #28368: Refuse monitoring processes if the child watcher has no + loop attached. + Patch by Vincent Michel. + +- Issue #28369: Raise RuntimeError when transport's FD is used with + add_reader, add_writer, etc. + +- Issue #28370: Speedup asyncio.StreamReader.readexactly. + Patch by Коренберг Марк. + +- Issue #28371: Deprecate passing asyncio.Handles to run_in_executor. + +- Issue #28372: Fix asyncio to support formatting of non-python coroutines. + +Windows +------- + +- Issue #28251: Improvements to help manuals on Windows. + +- Issue #28110: launcher.msi has different product codes between 32-bit and + 64-bit + +- Issue #28161: Opening CON for write access fails + +- Issue #28162: WindowsConsoleIO readall() fails if first line starts with + Ctrl+Z + +- Issue #28163: WindowsConsoleIO fileno() passes wrong flags to + _open_osfhandle + +- Issue #28164: _PyIO_get_console_type fails for various paths + +- Issue #28137: Renames Windows path file to ._pth + +- Issue #28138: Windows ._pth file should allow import site + +Build +----- + +- Issue #28258: Fixed build with Estonian locale (python-config and distclean + targets in Makefile). Patch by Arfrever Frehtes Taifersar Arahesis. + +- Issue #26661: setup.py now detects system libffi with multiarch wrapper. + +- Issue #15819: Remove redundant include search directory option for building + outside the source tree. + +Tests +----- + +- Issue #28217: Adds _testconsole module to test console input. + +What's New in Python 3.6.0 beta 1 +================================= + +*Release date: 2016-09-12* + +Core and Builtins +----------------- + +- Issue #23722: The __class__ cell used by zero-argument super() is now + initialized from type.__new__ rather than __build_class__, so class methods + relying on that will now work correctly when called from metaclass methods + during class creation. Patch by Martin Teichmann. + +- Issue #25221: Fix corrupted result from PyLong_FromLong(0) when Python + is compiled with NSMALLPOSINTS = 0. + +- Issue #27080: Implement formatting support for PEP 515. Initial patch + by Chris Angelico. + +- Issue #27199: In tarfile, expose copyfileobj bufsize to improve throughput. + Patch by Jason Fried. + +- Issue #27948: In f-strings, only allow backslashes inside the braces + (where the expressions are). This is a breaking change from the 3.6 + alpha releases, where backslashes are allowed anywhere in an + f-string. Also, require that expressions inside f-strings be + enclosed within literal braces, and not escapes like + f'\x7b"hi"\x7d'. + +- Issue #28046: Remove platform-specific directories from sys.path. + +- Issue #28071: Add early-out for differencing from an empty set. + +- Issue #25758: Prevents zipimport from unnecessarily encoding a filename + (patch by Eryk Sun) + +- Issue #25856: The __module__ attribute of extension classes and functions + now is interned. This leads to more compact pickle data with protocol 4. + +- Issue #27213: Rework CALL_FUNCTION* opcodes to produce shorter and more + efficient bytecode. Patch by Demur Rumed, design by Serhiy Storchaka, + reviewed by Serhiy Storchaka and Victor Stinner. + +- Issue #26331: Implement tokenizing support for PEP 515. Patch by Georg Brandl. + +- Issue #27999: Make "global after use" a SyntaxError, and ditto for nonlocal. + Patch by Ivan Levkivskyi. + +- Issue #28003: Implement PEP 525 -- Asynchronous Generators. + +- Issue #27985: Implement PEP 526 -- Syntax for Variable Annotations. + Patch by Ivan Levkivskyi. + +- Issue #26058: Add a new private version to the builtin dict type, incremented + at each dictionary creation and at each dictionary change. Implementation of + the PEP 509. + +- Issue #27364: A backslash-character pair that is not a valid escape sequence + now generates a DeprecationWarning. Patch by Emanuel Barry. + +- Issue #27350: `dict` implementation is changed like PyPy. It is more compact + and preserves insertion order. + (Concept developed by Raymond Hettinger and patch by Inada Naoki.) + +- Issue #27911: Remove unnecessary error checks in + ``exec_builtin_or_dynamic()``. + +- Issue #27078: Added BUILD_STRING opcode. Optimized f-strings evaluation. + +- Issue #17884: Python now requires systems with inttypes.h and stdint.h + +- Issue #27961: Require platforms to support ``long long``. Python hasn't + compiled without ``long long`` for years, so this is basically a formality. + +- Issue #27355: Removed support for Windows CE. It was never finished, + and Windows CE is no longer a relevant platform for Python. + +- Implement PEP 523. + +- Issue #27870: A left shift of zero by a large integer no longer attempts + to allocate large amounts of memory. + +- Issue #25402: In int-to-decimal-string conversion, improve the estimate + of the intermediate memory required, and remove an unnecessarily strict + overflow check. Patch by Serhiy Storchaka. + +- Issue #27214: In long_invert, be more careful about modifying object + returned by long_add, and remove an unnecessary check for small longs. + Thanks Oren Milman for analysis and patch. + +- Issue #27506: Support passing the bytes/bytearray.translate() "delete" + argument by keyword. + +- Issue #27812: Properly clear out a generator's frame's backreference to the + generator to prevent crashes in frame.clear(). + +- Issue #27811: Fix a crash when a coroutine that has not been awaited is + finalized with warnings-as-errors enabled. + +- Issue #27587: Fix another issue found by PVS-Studio: Null pointer check + after use of 'def' in _PyState_AddModule(). + Initial patch by Christian Heimes. + +- Issue #27792: The modulo operation applied to ``bool`` and other + ``int`` subclasses now always returns an ``int``. Previously + the return type depended on the input values. Patch by Xiang Zhang. + +- Issue #26984: int() now always returns an instance of exact int. + +- Issue #25604: Fix a minor bug in integer true division; this bug could + potentially have caused off-by-one-ulp results on platforms with + unreliable ldexp implementations. + +- Issue #24254: Make class definition namespace ordered by default. + +- Issue #27662: Fix an overflow check in ``List_New``: the original code was + checking against ``Py_SIZE_MAX`` instead of the correct upper bound of + ``Py_SSIZE_T_MAX``. Patch by Xiang Zhang. + +- Issue #27782: Multi-phase extension module import now correctly allows the + ``m_methods`` field to be used to add module level functions to instances + of non-module types returned from ``Py_create_mod``. Patch by Xiang Zhang. + +- Issue #27936: The round() function accepted a second None argument + for some types but not for others. Fixed the inconsistency by + accepting None for all numeric types. + +- Issue #27487: Warn if a submodule argument to "python -m" or + runpy.run_module() is found in sys.modules after parent packages are + imported, but before the submodule is executed. + +- Issue #27157: Make only type() itself accept the one-argument form. + Patch by Eryk Sun and Emanuel Barry. + +- Issue #27558: Fix a SystemError in the implementation of "raise" statement. + In a brand new thread, raise a RuntimeError since there is no active + exception to reraise. Patch written by Xiang Zhang. + +- Issue #28008: Implement PEP 530 -- asynchronous comprehensions. + +Library +------- + +- Issue #28037: Use sqlite3_get_autocommit() instead of setting + Connection->inTransaction manually. + +- Issue #25283: Attributes tm_gmtoff and tm_zone are now available on + all platforms in the return values of time.localtime() and + time.gmtime(). + +- Issue #24454: Regular expression match object groups are now + accessible using __getitem__. "mo[x]" is equivalent to + "mo.group(x)". + +- Issue #10740: sqlite3 no longer implicitly commit an open transaction + before DDL statements. + +- Issue #17941: Add a *module* parameter to collections.namedtuple(). + +- Issue #22493: Inline flags now should be used only at the start of the + regular expression. Deprecation warning is emitted if uses them in the + middle of the regular expression. + +- Issue #26885: xmlrpc now supports unmarshalling additional data types used + by Apache XML-RPC implementation for numerics and None. + +- Issue #28070: Fixed parsing inline verbose flag in regular expressions. + +- Issue #19500: Add client-side SSL session resumption to the ssl module. + +- Issue #28022: Deprecate ssl-related arguments in favor of SSLContext. The + deprecation include manual creation of SSLSocket and certfile/keyfile + (or similar) in ftplib, httplib, imaplib, smtplib, poplib and urllib. + +- Issue #28043: SSLContext has improved default settings: OP_NO_SSLv2, + OP_NO_SSLv3, OP_NO_COMPRESSION, OP_CIPHER_SERVER_PREFERENCE, + OP_SINGLE_DH_USE, OP_SINGLE_ECDH_USE and HIGH ciphers without MD5. + +- Issue #24693: Changed some RuntimeError's in the zipfile module to more + appropriate types. Improved some error messages and debugging output. + +- Issue #17909: ``json.load`` and ``json.loads`` now support binary input + encoded as UTF-8, UTF-16 or UTF-32. Patch by Serhiy Storchaka. + +- Issue #27137: the pure Python fallback implementation of ``functools.partial`` + now matches the behaviour of its accelerated C counterpart for subclassing, + pickling and text representation purposes. Patch by Emanuel Barry and + Serhiy Storchaka. + ++- Fix possible integer overflows and crashes in the mmap module with unusual ++ usage patterns. ++ +- Issue #1703178: Fix the ability to pass the --link-objects option to the + distutils build_ext command. - Issue #28019: itertools.count() no longer rounds non-integer step in range between 1.0 and 2.0 to 1. diff --cc Modules/mmapmodule.c index 73c37d07121d,6e2db6134ccb..30084447c88f --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@@ -90,10 -90,10 +90,10 @@@ typedef enu typedef struct { PyObject_HEAD char * data; - size_t size; - size_t pos; /* relative to offset */ + Py_ssize_t size; + Py_ssize_t pos; /* relative to offset */ #ifdef MS_WINDOWS - PY_LONG_LONG offset; + long long offset; #else off_t offset; #endif @@@ -395,21 -383,20 +383,19 @@@ mmap_write_method(mmap_object *self if (!PyArg_ParseTuple(args, "y*:write", &data)) return(NULL); - if (!is_writable(self)) { - PyBuffer_Release(&data); + if (!is_writable(self)) return NULL; - } - if ((self->pos + data.len) > self->size) { - PyErr_SetString(PyExc_ValueError, "data out of range"); + if (self->pos > self->size || self->size - self->pos < data.len) { PyBuffer_Release(&data); + PyErr_SetString(PyExc_ValueError, "data out of range"); return NULL; } - memcpy(self->data + self->pos, data.buf, data.len); - self->pos = self->pos + data.len; - result = PyLong_FromSsize_t(data.len); + + memcpy(&self->data[self->pos], data.buf, data.len); + self->pos += data.len; PyBuffer_Release(&data); - return result; - Py_INCREF(Py_None); - return Py_None; ++ return PyLong_FromSsize_t(data.len); } static PyObject * @@@ -1265,9 -1232,8 +1231,8 @@@ static PyObject new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) { mmap_object *m_obj; - PyObject *map_size_obj = NULL; Py_ssize_t map_size; - PY_LONG_LONG offset = 0, size; + long long offset = 0, size; DWORD off_hi; /* upper 32 bits of offset */ DWORD off_lo; /* lower 32 bits of offset */ DWORD size_hi; /* upper 32 bits of size */