From: Andrew Svetlov Date: Sun, 19 Aug 2012 19:20:03 +0000 (+0300) Subject: Issue #15595: Fix subprocess.Popen(universal_newlines=True) X-Git-Tag: v3.3.0rc1~73 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=a19de803e4661af947938df2db72d0cd5538f4a4;p=thirdparty%2FPython%2Fcpython.git Issue #15595: Fix subprocess.Popen(universal_newlines=True) for certain locales (utf-16 and utf-32 family). Patch by Chris Jerdonek. --- a19de803e4661af947938df2db72d0cd5538f4a4 diff --cc Lib/test/test_subprocess.py index 74ce0912cb33,620cd8e5f35f..2420772c36f3 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@@ -646,34 -561,37 +647,65 @@@ class ProcessTestCase(BaseTestCase) p.communicate() self.assertEqual(p.returncode, 0) + def test_universal_newlines_communicate_stdin_stdout_stderr(self): + # universal newlines through communicate(), with stdin, stdout, stderr + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os;' + SETBINARY + textwrap.dedent(''' + s = sys.stdin.buffer.readline() + sys.stdout.buffer.write(s) + sys.stdout.buffer.write(b"line2\\r") + sys.stderr.buffer.write(b"eline2\\n") + s = sys.stdin.buffer.read() + sys.stdout.buffer.write(s) + sys.stdout.buffer.write(b"line4\\n") + sys.stdout.buffer.write(b"line5\\r\\n") + sys.stderr.buffer.write(b"eline6\\r") + sys.stderr.buffer.write(b"eline7\\r\\nz") + ''')], + stdin=subprocess.PIPE, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + universal_newlines=True) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stderr.close) + (stdout, stderr) = p.communicate("line1\nline3\n") + self.assertEqual(p.returncode, 0) + self.assertEqual("line1\nline2\nline3\nline4\nline5\n", stdout) + # Python debug build push something like "[42442 refs]\n" + # to stderr at exit of subprocess. + # Don't use assertStderrEqual because it strips CR and LF from output. + self.assertTrue(stderr.startswith("eline2\neline6\neline7\n")) ++ + def test_universal_newlines_communicate_encodings(self): + # Check that universal newlines mode works for various encodings, + # in particular for encodings in the UTF-16 and UTF-32 families. + # See issue #15595. + # + # UTF-16 and UTF-32-BE are sufficient to check both with BOM and + # without, and UTF-16 and UTF-32. + for encoding in ['utf-16', 'utf-32-be']: + old_getpreferredencoding = locale.getpreferredencoding + # Indirectly via io.TextIOWrapper, Popen() defaults to + # locale.getpreferredencoding(False) and earlier in Python 3.2 to + # locale.getpreferredencoding(). + def getpreferredencoding(do_setlocale=True): + return encoding + code = ("import sys; " + r"sys.stdout.buffer.write('1\r\n2\r3\n4'.encode('%s'))" % + encoding) + args = [sys.executable, '-c', code] + try: + locale.getpreferredencoding = getpreferredencoding + # We set stdin to be non-None because, as of this writing, + # a different code path is used when the number of pipes is + # zero or one. + popen = subprocess.Popen(args, universal_newlines=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + stdout, stderr = popen.communicate(input='') + finally: + locale.getpreferredencoding = old_getpreferredencoding - + self.assertEqual(stdout, '1\n2\n3\n4') def test_no_leaking(self): # Make sure we leak no resources diff --cc Misc/NEWS index 3b7576c4c895,52e297c7805b..9f16caf4f349 --- a/Misc/NEWS +++ b/Misc/NEWS @@@ -13,126 -13,93 +13,129 @@@ Core and Builtin - Issue #15604: Update uses of PyObject_IsTrue() to check for and handle errors correctly. Patch by Serhiy Storchaka. -- Issue #13119: sys.stdout and sys.stderr are now using "\r\n" newline on - Windows, as Python 2. +Library +------- -- Issue #14579: Fix CVE-2012-2135: vulnerability in the utf-16 decoder after - error handling. Patch by Serhiy Storchaka. ++- Issue #15595: Fix subprocess.Popen(universal_newlines=True) ++ for certain locales (utf-16 and utf-32 family). Patch by Chris Jerdonek. + -- Issue #15404: Refleak in PyMethodObject repr. +- Issue #15477: In cmath and math modules, add workaround for platforms whose + system-supplied log1p function doesn't respect signs of zeros. -- Issue #15394: An issue in PyModule_Create that caused references to - be leaked on some error paths has been fixed. Patch by Julia Lawall. +- Issue #15715: importlib.__import__() will silence an ImportError when the use + of fromlist leads to a failed import. -- Issue #15368: An issue that caused bytecode generation to be - non-deterministic when using randomized hashing (-R) has been fixed. +- Issue #14669: Fix pickling of connections and sockets on MacOSX + by sending/receiving an acknowledgment after file descriptor transfer. + TestPicklingConnection has been reenabled for MacOSX. -- Issue #15020: The program name used to search for Python's path is now - "python3" under Unix, not "python". +- Issue #11062: Fix adding a message from file to Babyl mailbox. -- Issue #15033: Fix the exit status bug when modules invoked using -m swith, - return the proper failure return value (1). Patch contributed by Jeff Knupp. +- Issue #15646: Prevent equivalent of a fork bomb when using + multiprocessing on Windows without the "if __name__ == '__main__'" + idiom. -- Issue #12268: File readline, readlines and read() or readall() methods - no longer lose data when an underlying read system call is interrupted. - IOError is no longer raised due to a read system call returning EINTR - from within these methods. +- Issue #15678: Fix IDLE menus when started from OS X command line + (3.3.0b2 regression). -- Issue #15142: Fix reference leak when deallocating instances of types - created using PyType_FromSpec(). +C API +----- -- Issue #10053: Don't close FDs when FileIO.__init__ fails. Loosely based on - the work by Hirokazu Yamamoto. +Extension Modules +----------------- -- Issue #14775: Fix a potential quadratic dict build-up due to the garbage - collector repeatedly trying to untrack dicts. +Tools/Demos +----------- -- Issue #14494: Fix __future__.py and its documentation to note that - absolute imports are the default behavior in 3.0 instead of 2.7. - Patch by Sven Marnach. +Documentation +------------- -- Issue #14761: Fix potential leak on an error case in the import machinery. +- Issue #15640: Document importlib.abc.Finder as deprecated. -- Issue #14699: Fix calling the classmethod descriptor directly. +- Issue #15630: Add an example for "continue" stmt in the tutorial. Patch by + Daniel Ellis. -- Issue #14433: Prevent msvcrt crash in interactive prompt when stdin - is closed. +Tests +----- -- Issue #11603 (again): Setting __repr__ to __str__ now raises a RuntimeError - when repr() or str() is called on such an object. +- Issue #15615: Add some tests for the json module's handling of invalid + input data. Patch by Kushal Das. -- Issue #14658: Fix binding a special method to a builtin implementation of a - special method with a different name. -- Issue #14630: Fix a memory access bug for instances of a subclass of int - with value 0. +What's New in Python 3.3.0 Beta 2? +================================== -- Issue #14612: Fix jumping around with blocks by setting f_lineno. +*Release date: 12-Aug-2012* -- Issue #14607: Fix keyword-only arguments which started with ``__``. +Core and Builtins +----------------- -- Issue #13889: Check and (if necessary) set FPU control word before calling - any of the dtoa.c string <-> float conversion functions, on MSVC builds of - Python. This fixes issues when embedding Python in a Delphi app. +- Issue #15568: Fix the return value of "yield from" when StopIteration is + raised by a custom iterator. -- Issue #14474: Save and restore exception state in thread.start_new_thread() - while writing error message if the thread leaves a unhandled exception. +- Issue #13119: sys.stdout and sys.stderr are now using "\r\n" newline on + Windows, as Python 2. -- Issue #13019: Fix potential reference leaks in bytearray.extend(). Patch - by Suman Saha. +- Issue #15534: Fix the fast-search function for non-ASCII Unicode strings. -- Issue #14378: Fix compiling ast.ImportFrom nodes with a "__future__" string as - the module name that was not interned. +- Issue #15508: Fix the docstring for __import__ to have the proper default + value of 0 for 'level' and to not mention negative levels since they are not + supported. -- Issue #14331: Use significantly less stack space when importing modules by - allocating path buffers on the heap instead of the stack. +- Issue #15425: Eliminated traceback noise from more situations involving + importlib. -- Issue #14334: Prevent in a segfault in type.__getattribute__ when it was not - passed strings. +- Issue #14578: Support modules registered in the Windows registry again. -- Issue #1469629: Allow cycles through an object's __dict__ slot to be - collected. (For example if ``x.__dict__ is x``). +- Issue #15466: Stop using TYPE_INT64 in marshal, to make importlib.h (and other + byte code files) equal between 32-bit and 64-bit systems. -- Issue #14172: Fix reference leak when marshalling a buffer-like object - (other than a bytes object). +- Issue #1692335: Move initial exception args assignment to + "BaseException.__new__" to help pickling of naive subclasses. -- Issue #13521: dict.setdefault() now does only one lookup for the given key, - making it "atomic" for many purposes. Patch by Filip Gruszczyński. +- Issue #12834: Fix PyBuffer_ToContiguous() for non-contiguous arrays. -- Issue #14471: Fix a possible buffer overrun in the winreg module. +- Issue #15456: Fix code __sizeof__ after #12399 change. Patch by Serhiy + Storchaka. + +- Issue #15404: Refleak in PyMethodObject repr. + +- Issue #15394: An issue in PyModule_Create that caused references to be leaked + on some error paths has been fixed. Patch by Julia Lawall. + +- Issue #15368: An issue that caused bytecode generation to be non-deterministic + has been fixed. + +- Issue #15202: Consistently use the name "follow_symlinks" for new parameters + in os and shutil functions. + +- Issue #15314: __main__.__loader__ is now set correctly during interpreter + startup. + +- Issue #15111: When a module imported using 'from import' has an ImportError + inside itself, don't mask that fact behind a generic ImportError for the + module itself. + +- Issue #15293: Add GC support to the AST base node type. + +- Issue #15291: Fix a memory leak where AST nodes where not properly + deallocated. + +- Issue #15110: Fix the tracebacks generated by "import xxx" to not show the + importlib stack frames. + +- Issue #15020: The program name used to search for Python's path is now + "python3" under Unix, not "python". + +- Issue #15033: Fix the exit status bug when modules invoked using -m swith, + return the proper failure return value (1). Patch contributed by Jeff Knupp. + +- Issue #15229: An OSError subclass whose __init__ doesn't call back + OSError.__init__ could produce incomplete instances, leading to crashes when + calling str() on them. + +- Issue 15307: Virtual environments now use symlinks with framework builds on + Mac OS X, like other POSIX builds. Library -------