From: Georg Brandl Date: Mon, 20 Feb 2012 20:31:46 +0000 (+0100) Subject: Merge from 3.1: Issue #13703: add a way to randomize the hash values of basic types... X-Git-Tag: v3.2.3rc1~34 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=09a7c72cad48f568e0781541167cf9ea6a3f0760;p=thirdparty%2FPython%2Fcpython.git Merge from 3.1: Issue #13703: add a way to randomize the hash values of basic types (str, bytes, datetime) in order to make algorithmic complexity attacks on (e.g.) web apps much more complicated. The environment variable PYTHONHASHSEED and the new command line flag -R control this behavior. --- 09a7c72cad48f568e0781541167cf9ea6a3f0760 diff --cc Doc/library/sys.rst index d611acf74ac3,95947560f21c..063e0a52c081 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@@ -252,12 -220,12 +252,16 @@@ always available :const:`ignore_environment` :option:`-E` :const:`verbose` :option:`-v` :const:`bytes_warning` :option:`-b` + :const:`quiet` :option:`-q` + :const:`hash_randomization` :option:`-R` ============================= ============================= - .. versionadded:: 3.1.5 + .. versionchanged:: 3.2 + Added ``quiet`` attribute for the new :option:`-q` flag. + ++ .. versionadded:: 3.2.3 + The ``hash_randomization`` attribute. + .. data:: float_info diff --cc Doc/using/cmdline.rst index c386fb41ee03,11e2d7d9c0c8..d0b330d9d58f --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@@ -24,7 -21,7 +24,7 @@@ Command lin When invoking Python, you may specify any of these options:: - python [-bBdEhiOsSuvVWx?] [-c command | -m module-name | script | - ] [args] - python [-bBdEhiORsSuvVWx?] [-c command | -m module-name | script | - ] [args] ++ python [-bBdEhiORqsSuvVWx?] [-c command | -m module-name | script | - ] [args] The most common use case is, of course, a simple invocation of a script:: @@@ -220,17 -215,32 +220,40 @@@ Miscellaneous option Discard docstrings in addition to the :option:`-O` optimizations. +.. cmdoption:: -q + + Don't display the copyright and version messages even in interactive mode. + + .. versionadded:: 3.2 + + + .. cmdoption:: -R + + Turn on hash randomization, so that the :meth:`__hash__` values of str, bytes + and datetime objects are "salted" with an unpredictable random value. + Although they remain constant within an individual Python process, they are + not predictable between repeated invocations of Python. + + This is intended to provide protection against a denial-of-service caused by + carefully-chosen inputs that exploit the worst case performance of a dict + insertion, O(n^2) complexity. See + http://www.ocert.org/advisories/ocert-2011-003.html for details. + + Changing hash values affects the order in which keys are retrieved from a + dict. Although Python has never made guarantees about this ordering (and it + typically varies between 32-bit and 64-bit builds), enough real-world code + implicitly relies on this non-guaranteed behavior that the randomization is + disabled by default. + + See also :envvar:`PYTHONHASHSEED`. + - .. versionadded:: 3.1.5 ++ .. versionadded:: 3.2.3 + + .. cmdoption:: -s - Don't add user site directory to sys.path + Don't add the :data:`user site-packages directory ` to + :data:`sys.path`. .. seealso:: @@@ -350,6 -347,12 +373,7 @@@ Options you shouldn't us .. _Jython: http://jython.org -.. cmdoption:: -X - - Reserved for alternative implementations of Python to use for their own - purposes. - + .. _using-on-envvars: Environment variables @@@ -454,16 -457,35 +478,37 @@@ These environment variables influence P .. envvar:: PYTHONDONTWRITEBYTECODE If this is set, Python won't try to write ``.pyc`` or ``.pyo`` files on the - import of source modules. + import of source modules. This is equivalent to specifying the :option:`-B` + option. + .. envvar:: PYTHONHASHSEED + + If this variable is set to ``random``, the effect is the same as specifying + the :option:`-R` option: a random value is used to seed the hashes of str, + bytes and datetime objects. + + If :envvar:`PYTHONHASHSEED` is set to an integer value, it is used as a fixed + seed for generating the hash() of the types covered by the hash + randomization. + + Its purpose is to allow repeatable hashing, such as for selftests for the + interpreter itself, or to allow a cluster of python processes to share hash + values. + + The integer must be a decimal number in the range [0,4294967295]. Specifying + the value 0 will lead to the same hash values as when hash randomization is + disabled. + - .. versionadded:: 3.1.5 ++ .. versionadded:: 3.2.3 + + .. envvar:: PYTHONIOENCODING - Overrides the encoding used for stdin/stdout/stderr, in the syntax - ``encodingname:errorhandler``. The ``:errorhandler`` part is optional and - has the same meaning as in :func:`str.encode`. + If this is set before running the interpreter, it overrides the encoding used + for stdin/stdout/stderr, in the syntax ``encodingname:errorhandler``. The + ``:errorhandler`` part is optional and has the same meaning as in + :func:`str.encode`. For stderr, the ``:errorhandler`` part is ignored; the handler will always be ``'backslashreplace'``. diff --cc Include/object.h index 2528841d520e,7848cf4c6ebd..a54c40077480 --- a/Include/object.h +++ b/Include/object.h @@@ -512,11 -470,15 +512,17 @@@ PyAPI_FUNC(int) Py_ReprEnter(PyObject * PyAPI_FUNC(void) Py_ReprLeave(PyObject *); /* Helpers for hash functions */ -PyAPI_FUNC(long) _Py_HashDouble(double); -PyAPI_FUNC(long) _Py_HashPointer(void*); +#ifndef Py_LIMITED_API +PyAPI_FUNC(Py_hash_t) _Py_HashDouble(double); +PyAPI_FUNC(Py_hash_t) _Py_HashPointer(void*); +#endif + typedef struct { - long prefix; - long suffix; ++ Py_hash_t prefix; ++ Py_hash_t suffix; + } _Py_HashSecret_t; + PyAPI_DATA(_Py_HashSecret_t) _Py_HashSecret; + /* Helper for passing objects to printf and the like */ #define PyObject_REPR(obj) _PyUnicode_AsString(PyObject_Repr(obj)) diff --cc Lib/test/regrtest.py index 135a90e7727a,d203600e2954..26ba9820d76d --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@@ -496,9 -428,14 +496,14 @@@ def main(tests=None, testdir=None, verb except ValueError: print("Couldn't find starting test (%s), using all tests" % start) if randomize: + hashseed = os.getenv('PYTHONHASHSEED') + if not hashseed: + os.environ['PYTHONHASHSEED'] = str(random_seed) + os.execv(sys.executable, [sys.executable] + sys.argv) + return random.seed(random_seed) print("Using random seed", random_seed) - random.shuffle(tests) + random.shuffle(selected) if trace: import trace, tempfile tracer = trace.Trace(ignoredirs=[sys.prefix, sys.exec_prefix, diff --cc Lib/test/test_cmd_line.py index 2fca25ea08cd,eacd7a6ae436..1a21281e6d2d --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@@ -208,128 -171,40 +208,144 @@@ class CmdLineTest(unittest.TestCase) self.assertTrue(data.startswith(b'x'), data) def test_large_PYTHONPATH(self): - with test.support.EnvironmentVarGuard() as env: - path1 = "ABCDE" * 100 - path2 = "FGHIJ" * 100 - env['PYTHONPATH'] = path1 + os.pathsep + path2 + path1 = "ABCDE" * 100 + path2 = "FGHIJ" * 100 + path = path1 + os.pathsep + path2 - code = """ -import sys -path = ":".join(sys.path) -path = path.encode("ascii", "backslashreplace") -sys.stdout.buffer.write(path)""" - code = code.strip().splitlines() - code = '; '.join(code) - p = _spawn_python('-S', '-c', code) - stdout, _ = p.communicate() - p.stdout.close() - self.assertTrue(path1.encode('ascii') in stdout) - self.assertTrue(path2.encode('ascii') in stdout) + code = """if 1: + import sys + path = ":".join(sys.path) + path = path.encode("ascii", "backslashreplace") + sys.stdout.buffer.write(path)""" + rc, out, err = assert_python_ok('-S', '-c', code, + PYTHONPATH=path) + self.assertIn(path1.encode('ascii'), out) + self.assertIn(path2.encode('ascii'), out) + + def test_displayhook_unencodable(self): + for encoding in ('ascii', 'latin1', 'utf8'): + env = os.environ.copy() + env['PYTHONIOENCODING'] = encoding + p = subprocess.Popen( + [sys.executable, '-i'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=env) + # non-ascii, surrogate, non-BMP printable, non-BMP unprintable + text = "a=\xe9 b=\uDC80 c=\U00010000 d=\U0010FFFF" + p.stdin.write(ascii(text).encode('ascii') + b"\n") + p.stdin.write(b'exit()\n') + data = kill_python(p) + escaped = repr(text).encode(encoding, 'backslashreplace') + self.assertIn(escaped, data) + + def check_input(self, code, expected): + with tempfile.NamedTemporaryFile("wb+") as stdin: + sep = os.linesep.encode('ASCII') + stdin.write(sep.join((b'abc', b'def'))) + stdin.flush() + stdin.seek(0) + with subprocess.Popen( + (sys.executable, "-c", code), + stdin=stdin, stdout=subprocess.PIPE) as proc: + stdout, stderr = proc.communicate() + self.assertEqual(stdout.rstrip(), expected) + + def test_stdin_readline(self): + # Issue #11272: check that sys.stdin.readline() replaces '\r\n' by '\n' + # on Windows (sys.stdin is opened in binary mode) + self.check_input( + "import sys; print(repr(sys.stdin.readline()))", + b"'abc\\n'") + + def test_builtin_input(self): + # Issue #11272: check that input() strips newlines ('\n' or '\r\n') + self.check_input( + "print(repr(input()))", + b"'abc'") + + def test_unmached_quote(self): + # Issue #10206: python program starting with unmatched quote + # spewed spaces to stdout + rc, out, err = assert_python_failure('-c', "'") + self.assertRegex(err.decode('ascii', 'ignore'), 'SyntaxError') + self.assertEqual(b'', out) + + def test_stdout_flush_at_shutdown(self): + # Issue #5319: if stdout.flush() fails at shutdown, an error should + # be printed out. + code = """if 1: + import os, sys + sys.stdout.write('x') + os.close(sys.stdout.fileno())""" + rc, out, err = assert_python_ok('-c', code) + self.assertEqual(b'', out) + self.assertRegex(err.decode('ascii', 'ignore'), + 'Exception IOError: .* ignored') + + def test_closed_stdout(self): + # Issue #13444: if stdout has been explicitly closed, we should + # not attempt to flush it at shutdown. + code = "import sys; sys.stdout.close()" + rc, out, err = assert_python_ok('-c', code) + self.assertEqual(b'', err) + + # Issue #7111: Python should work without standard streams + + @unittest.skipIf(os.name != 'posix', "test needs POSIX semantics") + def _test_no_stdio(self, streams): + code = """if 1: + import os, sys + for i, s in enumerate({streams}): + if getattr(sys, s) is not None: + os._exit(i + 1) + os._exit(42)""".format(streams=streams) + def preexec(): + if 'stdin' in streams: + os.close(0) + if 'stdout' in streams: + os.close(1) + if 'stderr' in streams: + os.close(2) + p = subprocess.Popen( + [sys.executable, "-E", "-c", code], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + preexec_fn=preexec) + out, err = p.communicate() + self.assertEqual(test.support.strip_python_stderr(err), b'') + self.assertEqual(p.returncode, 42) + + def test_no_stdin(self): + self._test_no_stdio(['stdin']) + + def test_no_stdout(self): + self._test_no_stdio(['stdout']) + + def test_no_stderr(self): + self._test_no_stdio(['stderr']) + + def test_no_std_streams(self): + self._test_no_stdio(['stdin', 'stdout', 'stderr']) + def test_hash_randomization(self): + # Verify that -R enables hash randomization: + self.verify_valid_flag('-R') + hashes = [] + for i in range(2): + code = 'print(hash("spam"))' - data, rc = self.start_python_and_exit_code('-R', '-c', code) ++ rc, out, err = assert_python_ok('-R', '-c', code) + self.assertEqual(rc, 0) - hashes.append(data) ++ hashes.append(out) + self.assertNotEqual(hashes[0], hashes[1]) + + # Verify that sys.flags contains hash_randomization + code = 'import sys; print("random is", sys.flags.hash_randomization)' - data, rc = self.start_python_and_exit_code('-R', '-c', code) ++ rc, out, err = assert_python_ok('-R', '-c', code) + self.assertEqual(rc, 0) - self.assertIn(b'random is 1', data) ++ self.assertIn(b'random is 1', out) def test_main(): test.support.run_unittest(CmdLineTest) diff --cc Lib/test/test_gdb.py index 651aaeccfcfb,000000000000..aea7c0c3eecc mode 100644,000000..100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@@ -1,718 -1,0 +1,724 @@@ +# Verify that gdb can pretty-print the various PyObject* types +# +# The code for testing gdb was adapted from similar work in Unladen Swallow's +# Lib/test/test_jit_gdb.py + +import os +import re +import subprocess +import sys +import unittest +import locale + +from test.support import run_unittest, findfile, python_is_optimized + +try: + gdb_version, _ = subprocess.Popen(["gdb", "--version"], + stdout=subprocess.PIPE).communicate() +except OSError: + # This is what "no gdb" looks like. There may, however, be other + # errors that manifest this way too. + raise unittest.SkipTest("Couldn't find gdb on the path") +gdb_version_number = re.search(b"^GNU gdb [^\d]*(\d+)\.", gdb_version) +if int(gdb_version_number.group(1)) < 7: + raise unittest.SkipTest("gdb versions before 7.0 didn't support python embedding" + " Saw:\n" + gdb_version.decode('ascii', 'replace')) + +# Verify that "gdb" was built with the embedded python support enabled: +cmd = "--eval-command=python import sys; print sys.version_info" +p = subprocess.Popen(["gdb", "--batch", cmd], + stdout=subprocess.PIPE) +gdbpy_version, _ = p.communicate() +if gdbpy_version == b'': + raise unittest.SkipTest("gdb not built with embedded python support") + +def gdb_has_frame_select(): + # Does this build of gdb have gdb.Frame.select ? + cmd = "--eval-command=python print(dir(gdb.Frame))" + p = subprocess.Popen(["gdb", "--batch", cmd], + stdout=subprocess.PIPE) + stdout, _ = p.communicate() + m = re.match(br'.*\[(.*)\].*', stdout) + if not m: + raise unittest.SkipTest("Unable to parse output from gdb.Frame.select test") + gdb_frame_dir = m.group(1).split(b', ') + return b"'select'" in gdb_frame_dir + +HAS_PYUP_PYDOWN = gdb_has_frame_select() + +BREAKPOINT_FN='builtin_id' + +class DebuggerTests(unittest.TestCase): + + """Test that the debugger can debug Python.""" + - def run_gdb(self, *args): ++ def run_gdb(self, *args, **env_vars): + """Runs gdb with the command line given by *args. + + Returns its stdout, stderr + """ ++ if env_vars: ++ env = os.environ.copy() ++ env.update(env_vars) ++ else: ++ env = None + out, err = subprocess.Popen( - args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ++ args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, + ).communicate() + return out.decode('utf-8', 'replace'), err.decode('utf-8', 'replace') + + def get_stack_trace(self, source=None, script=None, + breakpoint=BREAKPOINT_FN, + cmds_after_breakpoint=None, + import_site=False): + ''' + Run 'python -c SOURCE' under gdb with a breakpoint. + + Support injecting commands after the breakpoint is reached + + Returns the stdout from gdb + + cmds_after_breakpoint: if provided, a list of strings: gdb commands + ''' + # We use "set breakpoint pending yes" to avoid blocking with a: + # Function "foo" not defined. + # Make breakpoint pending on future shared library load? (y or [n]) + # error, which typically happens python is dynamically linked (the + # breakpoints of interest are to be found in the shared library) + # When this happens, we still get: + # Function "textiowrapper_write" not defined. + # emitted to stderr each time, alas. + + # Initially I had "--eval-command=continue" here, but removed it to + # avoid repeated print breakpoints when traversing hierarchical data + # structures + + # Generate a list of commands in gdb's language: + commands = ['set breakpoint pending yes', + 'break %s' % breakpoint, + 'run'] + if cmds_after_breakpoint: + commands += cmds_after_breakpoint + else: + commands += ['backtrace'] + + # print commands + + # Use "commands" to generate the arguments with which to invoke "gdb": + args = ["gdb", "--batch"] + args += ['--eval-command=%s' % cmd for cmd in commands] + args += ["--args", + sys.executable] + + if not import_site: + # -S suppresses the default 'import site' + args += ["-S"] + + if source: + args += ["-c", source] + elif script: + args += [script] + + # print args + # print ' '.join(args) + + # Use "args" to invoke gdb, capturing stdout, stderr: - out, err = self.run_gdb(*args) ++ out, err = self.run_gdb(*args, PYTHONHASHSEED='0') + + # Ignore some noise on stderr due to the pending breakpoint: + err = err.replace('Function "%s" not defined.\n' % breakpoint, '') + # Ignore some other noise on stderr (http://bugs.python.org/issue8600) + err = err.replace("warning: Unable to find libthread_db matching" + " inferior's thread library, thread debugging will" + " not be available.\n", + '') + err = err.replace("warning: Cannot initialize thread debugging" + " library: Debugger service failed\n", + '') + + # Ensure no unexpected error messages: + self.assertEqual(err, '') + + return out + + def get_gdb_repr(self, source, + cmds_after_breakpoint=None, + import_site=False): + # Given an input python source representation of data, + # run "python -c'id(DATA)'" under gdb with a breakpoint on + # builtin_id and scrape out gdb's representation of the "op" + # parameter, and verify that the gdb displays the same string + # + # Verify that the gdb displays the expected string + # + # For a nested structure, the first time we hit the breakpoint will + # give us the top-level structure + gdb_output = self.get_stack_trace(source, breakpoint=BREAKPOINT_FN, + cmds_after_breakpoint=cmds_after_breakpoint, + import_site=import_site) + # gdb can insert additional '\n' and space characters in various places + # in its output, depending on the width of the terminal it's connected + # to (using its "wrap_here" function) + m = re.match('.*#0\s+builtin_id\s+\(self\=.*,\s+v=\s*(.*?)\)\s+at\s+Python/bltinmodule.c.*', + gdb_output, re.DOTALL) + if not m: + self.fail('Unexpected gdb output: %r\n%s' % (gdb_output, gdb_output)) + return m.group(1), gdb_output + + def assertEndsWith(self, actual, exp_end): + '''Ensure that the given "actual" string ends with "exp_end"''' + self.assertTrue(actual.endswith(exp_end), + msg='%r did not end with %r' % (actual, exp_end)) + + def assertMultilineMatches(self, actual, pattern): + m = re.match(pattern, actual, re.DOTALL) + if not m: + self.fail(msg='%r did not match %r' % (actual, pattern)) + + def get_sample_script(self): + return findfile('gdb_sample.py') + +class PrettyPrintTests(DebuggerTests): + def test_getting_backtrace(self): + gdb_output = self.get_stack_trace('id(42)') + self.assertTrue(BREAKPOINT_FN in gdb_output) + + def assertGdbRepr(self, val, exp_repr=None, cmds_after_breakpoint=None): + # Ensure that gdb's rendering of the value in a debugged process + # matches repr(value) in this process: + gdb_repr, gdb_output = self.get_gdb_repr('id(' + ascii(val) + ')', + cmds_after_breakpoint) + if not exp_repr: + exp_repr = repr(val) + self.assertEqual(gdb_repr, exp_repr, + ('%r did not equal expected %r; full output was:\n%s' + % (gdb_repr, exp_repr, gdb_output))) + + def test_int(self): + 'Verify the pretty-printing of various "int"/long values' + self.assertGdbRepr(42) + self.assertGdbRepr(0) + self.assertGdbRepr(-7) + self.assertGdbRepr(1000000000000) + self.assertGdbRepr(-1000000000000000) + + def test_singletons(self): + 'Verify the pretty-printing of True, False and None' + self.assertGdbRepr(True) + self.assertGdbRepr(False) + self.assertGdbRepr(None) + + def test_dicts(self): + 'Verify the pretty-printing of dictionaries' + self.assertGdbRepr({}) + self.assertGdbRepr({'foo': 'bar'}) - self.assertGdbRepr({'foo': 'bar', 'douglas':42}) ++ self.assertGdbRepr({'foo': 'bar', 'douglas': 42}, ++ "{'foo': 'bar', 'douglas': 42}") + + def test_lists(self): + 'Verify the pretty-printing of lists' + self.assertGdbRepr([]) + self.assertGdbRepr(list(range(5))) + + def test_bytes(self): + 'Verify the pretty-printing of bytes' + self.assertGdbRepr(b'') + self.assertGdbRepr(b'And now for something hopefully the same') + self.assertGdbRepr(b'string with embedded NUL here \0 and then some more text') + self.assertGdbRepr(b'this is a tab:\t' + b' this is a slash-N:\n' + b' this is a slash-R:\r' + ) + + self.assertGdbRepr(b'this is byte 255:\xff and byte 128:\x80') + + self.assertGdbRepr(bytes([b for b in range(255)])) + + def test_strings(self): + 'Verify the pretty-printing of unicode strings' + encoding = locale.getpreferredencoding() + def check_repr(text): + try: + text.encode(encoding) + printable = True + except UnicodeEncodeError: + self.assertGdbRepr(text, ascii(text)) + else: + self.assertGdbRepr(text) + + self.assertGdbRepr('') + self.assertGdbRepr('And now for something hopefully the same') + self.assertGdbRepr('string with embedded NUL here \0 and then some more text') + + # Test printing a single character: + # U+2620 SKULL AND CROSSBONES + check_repr('\u2620') + + # Test printing a Japanese unicode string + # (I believe this reads "mojibake", using 3 characters from the CJK + # Unified Ideographs area, followed by U+3051 HIRAGANA LETTER KE) + check_repr('\u6587\u5b57\u5316\u3051') + + # Test a character outside the BMP: + # U+1D121 MUSICAL SYMBOL C CLEF + # This is: + # UTF-8: 0xF0 0x9D 0x84 0xA1 + # UTF-16: 0xD834 0xDD21 + check_repr(chr(0x1D121)) + + def test_tuples(self): + 'Verify the pretty-printing of tuples' + self.assertGdbRepr(tuple()) + self.assertGdbRepr((1,), '(1,)') + self.assertGdbRepr(('foo', 'bar', 'baz')) + + def test_sets(self): + 'Verify the pretty-printing of sets' + self.assertGdbRepr(set()) - self.assertGdbRepr(set(['a', 'b'])) - self.assertGdbRepr(set([4, 5, 6])) ++ self.assertGdbRepr(set(['a', 'b']), "{'a', 'b'}") ++ self.assertGdbRepr(set([4, 5, 6]), "{4, 5, 6}") + + # Ensure that we handle sets containing the "dummy" key value, + # which happens on deletion: + gdb_repr, gdb_output = self.get_gdb_repr('''s = set(['a','b']) +s.pop() +id(s)''') + self.assertEqual(gdb_repr, "{'b'}") + + def test_frozensets(self): + 'Verify the pretty-printing of frozensets' + self.assertGdbRepr(frozenset()) - self.assertGdbRepr(frozenset(['a', 'b'])) - self.assertGdbRepr(frozenset([4, 5, 6])) ++ self.assertGdbRepr(frozenset(['a', 'b']), "frozenset({'a', 'b'})") ++ self.assertGdbRepr(frozenset([4, 5, 6]), "frozenset({4, 5, 6})") + + def test_exceptions(self): + # Test a RuntimeError + gdb_repr, gdb_output = self.get_gdb_repr(''' +try: + raise RuntimeError("I am an error") +except RuntimeError as e: + id(e) +''') + self.assertEqual(gdb_repr, + "RuntimeError('I am an error',)") + + + # Test division by zero: + gdb_repr, gdb_output = self.get_gdb_repr(''' +try: + a = 1 / 0 +except ZeroDivisionError as e: + id(e) +''') + self.assertEqual(gdb_repr, + "ZeroDivisionError('division by zero',)") + + def test_modern_class(self): + 'Verify the pretty-printing of new-style class instances' + gdb_repr, gdb_output = self.get_gdb_repr(''' +class Foo: + pass +foo = Foo() +foo.an_int = 42 +id(foo)''') + m = re.match(r'', gdb_repr) + self.assertTrue(m, + msg='Unexpected new-style class rendering %r' % gdb_repr) + + def test_subclassing_list(self): + 'Verify the pretty-printing of an instance of a list subclass' + gdb_repr, gdb_output = self.get_gdb_repr(''' +class Foo(list): + pass +foo = Foo() +foo += [1, 2, 3] +foo.an_int = 42 +id(foo)''') + m = re.match(r'', gdb_repr) + + self.assertTrue(m, + msg='Unexpected new-style class rendering %r' % gdb_repr) + + def test_subclassing_tuple(self): + 'Verify the pretty-printing of an instance of a tuple subclass' + # This should exercise the negative tp_dictoffset code in the + # new-style class support + gdb_repr, gdb_output = self.get_gdb_repr(''' +class Foo(tuple): + pass +foo = Foo((1, 2, 3)) +foo.an_int = 42 +id(foo)''') + m = re.match(r'', gdb_repr) + + self.assertTrue(m, + msg='Unexpected new-style class rendering %r' % gdb_repr) + + def assertSane(self, source, corruption, exprepr=None): + '''Run Python under gdb, corrupting variables in the inferior process + immediately before taking a backtrace. + + Verify that the variable's representation is the expected failsafe + representation''' + if corruption: + cmds_after_breakpoint=[corruption, 'backtrace'] + else: + cmds_after_breakpoint=['backtrace'] + + gdb_repr, gdb_output = \ + self.get_gdb_repr(source, + cmds_after_breakpoint=cmds_after_breakpoint) + if exprepr: + if gdb_repr == exprepr: + # gdb managed to print the value in spite of the corruption; + # this is good (see http://bugs.python.org/issue8330) + return + + # Match anything for the type name; 0xDEADBEEF could point to + # something arbitrary (see http://bugs.python.org/issue8330) + pattern = '<.* at remote 0x[0-9a-f]+>' + + m = re.match(pattern, gdb_repr) + if not m: + self.fail('Unexpected gdb representation: %r\n%s' % \ + (gdb_repr, gdb_output)) + + def test_NULL_ptr(self): + 'Ensure that a NULL PyObject* is handled gracefully' + gdb_repr, gdb_output = ( + self.get_gdb_repr('id(42)', + cmds_after_breakpoint=['set variable v=0', + 'backtrace']) + ) + + self.assertEqual(gdb_repr, '0x0') + + def test_NULL_ob_type(self): + 'Ensure that a PyObject* with NULL ob_type is handled gracefully' + self.assertSane('id(42)', + 'set v->ob_type=0') + + def test_corrupt_ob_type(self): + 'Ensure that a PyObject* with a corrupt ob_type is handled gracefully' + self.assertSane('id(42)', + 'set v->ob_type=0xDEADBEEF', + exprepr='42') + + def test_corrupt_tp_flags(self): + 'Ensure that a PyObject* with a type with corrupt tp_flags is handled' + self.assertSane('id(42)', + 'set v->ob_type->tp_flags=0x0', + exprepr='42') + + def test_corrupt_tp_name(self): + 'Ensure that a PyObject* with a type with corrupt tp_name is handled' + self.assertSane('id(42)', + 'set v->ob_type->tp_name=0xDEADBEEF', + exprepr='42') + + def test_builtins_help(self): + 'Ensure that the new-style class _Helper in site.py can be handled' + # (this was the issue causing tracebacks in + # http://bugs.python.org/issue8032#msg100537 ) + gdb_repr, gdb_output = self.get_gdb_repr('id(__builtins__.help)', import_site=True) + + m = re.match(r'<_Helper at remote 0x[0-9a-f]+>', gdb_repr) + self.assertTrue(m, + msg='Unexpected rendering %r' % gdb_repr) + + def test_selfreferential_list(self): + '''Ensure that a reference loop involving a list doesn't lead proxyval + into an infinite loop:''' + gdb_repr, gdb_output = \ + self.get_gdb_repr("a = [3, 4, 5] ; a.append(a) ; id(a)") + self.assertEqual(gdb_repr, '[3, 4, 5, [...]]') + + gdb_repr, gdb_output = \ + self.get_gdb_repr("a = [3, 4, 5] ; b = [a] ; a.append(b) ; id(a)") + self.assertEqual(gdb_repr, '[3, 4, 5, [[...]]]') + + def test_selfreferential_dict(self): + '''Ensure that a reference loop involving a dict doesn't lead proxyval + into an infinite loop:''' + gdb_repr, gdb_output = \ + self.get_gdb_repr("a = {} ; b = {'bar':a} ; a['foo'] = b ; id(a)") + + self.assertEqual(gdb_repr, "{'foo': {'bar': {...}}}") + + def test_selfreferential_old_style_instance(self): + gdb_repr, gdb_output = \ + self.get_gdb_repr(''' +class Foo: + pass +foo = Foo() +foo.an_attr = foo +id(foo)''') + self.assertTrue(re.match('\) at remote 0x[0-9a-f]+>', + gdb_repr), + 'Unexpected gdb representation: %r\n%s' % \ + (gdb_repr, gdb_output)) + + def test_selfreferential_new_style_instance(self): + gdb_repr, gdb_output = \ + self.get_gdb_repr(''' +class Foo(object): + pass +foo = Foo() +foo.an_attr = foo +id(foo)''') + self.assertTrue(re.match('\) at remote 0x[0-9a-f]+>', + gdb_repr), + 'Unexpected gdb representation: %r\n%s' % \ + (gdb_repr, gdb_output)) + + gdb_repr, gdb_output = \ + self.get_gdb_repr(''' +class Foo(object): + pass +a = Foo() +b = Foo() +a.an_attr = b +b.an_attr = a +id(a)''') + self.assertTrue(re.match('\) at remote 0x[0-9a-f]+>\) at remote 0x[0-9a-f]+>', + gdb_repr), + 'Unexpected gdb representation: %r\n%s' % \ + (gdb_repr, gdb_output)) + + def test_truncation(self): + 'Verify that very long output is truncated' + gdb_repr, gdb_output = self.get_gdb_repr('id(list(range(1000)))') + self.assertEqual(gdb_repr, + "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, " + "14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, " + "27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, " + "40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, " + "53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, " + "66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, " + "79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, " + "92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, " + "104, 105, 106, 107, 108, 109, 110, 111, 112, 113, " + "114, 115, 116, 117, 118, 119, 120, 121, 122, 123, " + "124, 125, 126, 127, 128, 129, 130, 131, 132, 133, " + "134, 135, 136, 137, 138, 139, 140, 141, 142, 143, " + "144, 145, 146, 147, 148, 149, 150, 151, 152, 153, " + "154, 155, 156, 157, 158, 159, 160, 161, 162, 163, " + "164, 165, 166, 167, 168, 169, 170, 171, 172, 173, " + "174, 175, 176, 177, 178, 179, 180, 181, 182, 183, " + "184, 185, 186, 187, 188, 189, 190, 191, 192, 193, " + "194, 195, 196, 197, 198, 199, 200, 201, 202, 203, " + "204, 205, 206, 207, 208, 209, 210, 211, 212, 213, " + "214, 215, 216, 217, 218, 219, 220, 221, 222, 223, " + "224, 225, 226...(truncated)") + self.assertEqual(len(gdb_repr), + 1024 + len('...(truncated)')) + + def test_builtin_method(self): + gdb_repr, gdb_output = self.get_gdb_repr('import sys; id(sys.stdout.readlines)') + self.assertTrue(re.match('', + gdb_repr), + 'Unexpected gdb representation: %r\n%s' % \ + (gdb_repr, gdb_output)) + + def test_frames(self): + gdb_output = self.get_stack_trace(''' +def foo(a, b, c): + pass + +foo(3, 4, 5) +id(foo.__code__)''', + breakpoint='builtin_id', + cmds_after_breakpoint=['print (PyFrameObject*)(((PyCodeObject*)v)->co_zombieframe)'] + ) + self.assertTrue(re.match('.*\s+\$1 =\s+Frame 0x[0-9a-f]+, for file , line 3, in foo \(\)\s+.*', + gdb_output, + re.DOTALL), + 'Unexpected gdb representation: %r\n%s' % (gdb_output, gdb_output)) + +@unittest.skipIf(python_is_optimized(), + "Python was compiled with optimizations") +class PyListTests(DebuggerTests): + def assertListing(self, expected, actual): + self.assertEndsWith(actual, expected) + + def test_basic_command(self): + 'Verify that the "py-list" command works' + bt = self.get_stack_trace(script=self.get_sample_script(), + cmds_after_breakpoint=['py-list']) + + self.assertListing(' 5 \n' + ' 6 def bar(a, b, c):\n' + ' 7 baz(a, b, c)\n' + ' 8 \n' + ' 9 def baz(*args):\n' + ' >10 id(42)\n' + ' 11 \n' + ' 12 foo(1, 2, 3)\n', + bt) + + def test_one_abs_arg(self): + 'Verify the "py-list" command with one absolute argument' + bt = self.get_stack_trace(script=self.get_sample_script(), + cmds_after_breakpoint=['py-list 9']) + + self.assertListing(' 9 def baz(*args):\n' + ' >10 id(42)\n' + ' 11 \n' + ' 12 foo(1, 2, 3)\n', + bt) + + def test_two_abs_args(self): + 'Verify the "py-list" command with two absolute arguments' + bt = self.get_stack_trace(script=self.get_sample_script(), + cmds_after_breakpoint=['py-list 1,3']) + + self.assertListing(' 1 # Sample script for use by test_gdb.py\n' + ' 2 \n' + ' 3 def foo(a, b, c):\n', + bt) + +class StackNavigationTests(DebuggerTests): + @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands") + @unittest.skipIf(python_is_optimized(), + "Python was compiled with optimizations") + def test_pyup_command(self): + 'Verify that the "py-up" command works' + bt = self.get_stack_trace(script=self.get_sample_script(), + cmds_after_breakpoint=['py-up']) + self.assertMultilineMatches(bt, + r'''^.* +#[0-9]+ Frame 0x[0-9a-f]+, for file .*gdb_sample.py, line 7, in bar \(a=1, b=2, c=3\) + baz\(a, b, c\) +$''') + + @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands") + def test_down_at_bottom(self): + 'Verify handling of "py-down" at the bottom of the stack' + bt = self.get_stack_trace(script=self.get_sample_script(), + cmds_after_breakpoint=['py-down']) + self.assertEndsWith(bt, + 'Unable to find a newer python frame\n') + + @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands") + def test_up_at_top(self): + 'Verify handling of "py-up" at the top of the stack' + bt = self.get_stack_trace(script=self.get_sample_script(), + cmds_after_breakpoint=['py-up'] * 4) + self.assertEndsWith(bt, + 'Unable to find an older python frame\n') + + @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands") + @unittest.skipIf(python_is_optimized(), + "Python was compiled with optimizations") + def test_up_then_down(self): + 'Verify "py-up" followed by "py-down"' + bt = self.get_stack_trace(script=self.get_sample_script(), + cmds_after_breakpoint=['py-up', 'py-down']) + self.assertMultilineMatches(bt, + r'''^.* +#[0-9]+ Frame 0x[0-9a-f]+, for file .*gdb_sample.py, line 7, in bar \(a=1, b=2, c=3\) + baz\(a, b, c\) +#[0-9]+ Frame 0x[0-9a-f]+, for file .*gdb_sample.py, line 10, in baz \(args=\(1, 2, 3\)\) + id\(42\) +$''') + +class PyBtTests(DebuggerTests): + @unittest.skipIf(python_is_optimized(), + "Python was compiled with optimizations") + def test_bt(self): + 'Verify that the "py-bt" command works' + bt = self.get_stack_trace(script=self.get_sample_script(), + cmds_after_breakpoint=['py-bt']) + self.assertMultilineMatches(bt, + r'''^.* +Traceback \(most recent call first\): + File ".*gdb_sample.py", line 10, in baz + id\(42\) + File ".*gdb_sample.py", line 7, in bar + baz\(a, b, c\) + File ".*gdb_sample.py", line 4, in foo + bar\(a, b, c\) + File ".*gdb_sample.py", line 12, in + foo\(1, 2, 3\) +''') + + @unittest.skipIf(python_is_optimized(), + "Python was compiled with optimizations") + def test_bt_full(self): + 'Verify that the "py-bt-full" command works' + bt = self.get_stack_trace(script=self.get_sample_script(), + cmds_after_breakpoint=['py-bt-full']) + self.assertMultilineMatches(bt, + r'''^.* +#[0-9]+ Frame 0x[0-9a-f]+, for file .*gdb_sample.py, line 7, in bar \(a=1, b=2, c=3\) + baz\(a, b, c\) +#[0-9]+ Frame 0x[0-9a-f]+, for file .*gdb_sample.py, line 4, in foo \(a=1, b=2, c=3\) + bar\(a, b, c\) +#[0-9]+ Frame 0x[0-9a-f]+, for file .*gdb_sample.py, line 12, in \(\) + foo\(1, 2, 3\) +''') + +class PyPrintTests(DebuggerTests): + @unittest.skipIf(python_is_optimized(), + "Python was compiled with optimizations") + def test_basic_command(self): + 'Verify that the "py-print" command works' + bt = self.get_stack_trace(script=self.get_sample_script(), + cmds_after_breakpoint=['py-print args']) + self.assertMultilineMatches(bt, + r".*\nlocal 'args' = \(1, 2, 3\)\n.*") + + @unittest.skipIf(python_is_optimized(), + "Python was compiled with optimizations") + @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands") + def test_print_after_up(self): + bt = self.get_stack_trace(script=self.get_sample_script(), + cmds_after_breakpoint=['py-up', 'py-print c', 'py-print b', 'py-print a']) + self.assertMultilineMatches(bt, + r".*\nlocal 'c' = 3\nlocal 'b' = 2\nlocal 'a' = 1\n.*") + + @unittest.skipIf(python_is_optimized(), + "Python was compiled with optimizations") + def test_printing_global(self): + bt = self.get_stack_trace(script=self.get_sample_script(), + cmds_after_breakpoint=['py-print __name__']) + self.assertMultilineMatches(bt, + r".*\nglobal '__name__' = '__main__'\n.*") + + @unittest.skipIf(python_is_optimized(), + "Python was compiled with optimizations") + def test_printing_builtin(self): + bt = self.get_stack_trace(script=self.get_sample_script(), + cmds_after_breakpoint=['py-print len']) + self.assertMultilineMatches(bt, + r".*\nbuiltin 'len' = \n.*") + +class PyLocalsTests(DebuggerTests): + @unittest.skipIf(python_is_optimized(), + "Python was compiled with optimizations") + def test_basic_command(self): + bt = self.get_stack_trace(script=self.get_sample_script(), + cmds_after_breakpoint=['py-locals']) + self.assertMultilineMatches(bt, + r".*\nargs = \(1, 2, 3\)\n.*") + + @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands") + @unittest.skipIf(python_is_optimized(), + "Python was compiled with optimizations") + def test_locals_after_up(self): + bt = self.get_stack_trace(script=self.get_sample_script(), + cmds_after_breakpoint=['py-up', 'py-locals']) + self.assertMultilineMatches(bt, + r".*\na = 1\nb = 2\nc = 3\n.*") + +def test_main(): + run_unittest(PrettyPrintTests, + PyListTests, + StackNavigationTests, + PyBtTests, + PyPrintTests, + PyLocalsTests + ) + +if __name__ == "__main__": + test_main() diff --cc Lib/test/test_hash.py index fea1025e9105,f5736b2de7fa..8253610c8715 --- a/Lib/test/test_hash.py +++ b/Lib/test/test_hash.py @@@ -3,10 -3,16 +3,16 @@@ # # Also test that hash implementations are inherited as expected + import datetime + import os -import struct ++import sys import unittest from test import support + from test.script_helper import assert_python_ok from collections import Hashable -IS_64BIT = (struct.calcsize('l') == 8) ++IS_64BIT = sys.maxsize > 2**32 + class HashEqualityTestCase(unittest.TestCase): diff --cc Lib/test/test_os.py index e573bd2a5602,bff4f0bdea09..8bc8ba9fa625 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@@ -7,14 -7,9 +7,15 @@@ import errn import unittest import warnings import sys +import signal +import subprocess +import time import shutil from test import support +import contextlib +import mmap +import uuid + from test.script_helper import assert_python_ok # Detect whether we're on a Linux system that uses the (now outdated # and unmaintained) linuxthreads threading library. There's an issue @@@ -604,55 -567,42 +605,74 @@@ class MakedirTests(unittest.TestCase) class DevNullTests(unittest.TestCase): def test_devnull(self): - f = open(os.devnull, 'w') - f.write('hello') - f.close() - f = open(os.devnull, 'r') - self.assertEqual(f.read(), '') - f.close() + with open(os.devnull, 'wb') as f: + f.write(b'hello') + f.close() + with open(os.devnull, 'rb') as f: + self.assertEqual(f.read(), b'') class URandomTests(unittest.TestCase): - def test_urandom(self): - try: - self.assertEqual(len(os.urandom(1)), 1) - self.assertEqual(len(os.urandom(10)), 10) - self.assertEqual(len(os.urandom(100)), 100) - self.assertEqual(len(os.urandom(1000)), 1000) - except NotImplementedError: - pass + def test_urandom_length(self): + self.assertEqual(len(os.urandom(0)), 0) + self.assertEqual(len(os.urandom(1)), 1) + self.assertEqual(len(os.urandom(10)), 10) + self.assertEqual(len(os.urandom(100)), 100) + self.assertEqual(len(os.urandom(1000)), 1000) + + def test_urandom_value(self): + data1 = os.urandom(16) + data2 = os.urandom(16) + self.assertNotEqual(data1, data2) + + def get_urandom_subprocess(self, count): + code = '\n'.join(( + 'import os, sys', + 'data = os.urandom(%s)' % count, + 'sys.stdout.buffer.write(data)', + 'sys.stdout.buffer.flush()')) + out = assert_python_ok('-c', code) + stdout = out[1] + self.assertEqual(len(stdout), 16) + return stdout + + def test_urandom_subprocess(self): + data1 = self.get_urandom_subprocess(16) + data2 = self.get_urandom_subprocess(16) + self.assertNotEqual(data1, data2) +@contextlib.contextmanager +def _execvpe_mockup(defpath=None): + """ + Stubs out execv and execve functions when used as context manager. + Records exec calls. The mock execv and execve functions always raise an + exception as they would normally never return. + """ + # A list of tuples containing (function name, first arg, args) + # of calls to execv or execve that have been made. + calls = [] + + def mock_execv(name, *args): + calls.append(('execv', name, args)) + raise RuntimeError("execv called") + + def mock_execve(name, *args): + calls.append(('execve', name, args)) + raise OSError(errno.ENOTDIR, "execve called") + + try: + orig_execv = os.execv + orig_execve = os.execve + orig_defpath = os.defpath + os.execv = mock_execv + os.execve = mock_execve + if defpath is not None: + os.defpath = defpath + yield calls + finally: + os.execv = orig_execv + os.execve = orig_execve + os.defpath = orig_defpath + class ExecTests(unittest.TestCase): @unittest.skipIf(USING_LINUXTHREADS, "avoid triggering a linuxthreads bug: see issue #4970") diff --cc Lib/test/test_set.py index 07bfe0657ec0,5d5e2324e9df..6642440deae8 --- a/Lib/test/test_set.py +++ b/Lib/test/test_set.py @@@ -916,11 -931,13 +931,13 @@@ class TestBasicOpsMixedStringBytes(Test self.set = set(self.values) self.dup = set(self.values) self.length = 4 - self.repr = "{'a', b'a', 'b', b'b'}" def tearDown(self): - warnings.filters = self.warning_filters + self._warning_filters.__exit__(None, None, None) + def test_repr(self): + self.check_repr_against_values() + #============================================================================== def baditer(): diff --cc Lib/test/test_sys.py index 5d3404f75e08,7732c4c325f3..3268b1a141c8 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@@ -503,7 -446,7 +503,7 @@@ class SysModuleTest(unittest.TestCase) attrs = ("debug", "division_warning", "inspect", "interactive", "optimize", "dont_write_bytecode", "no_user_site", "no_site", "ignore_environment", "verbose", - "bytes_warning", "quiet") - "bytes_warning", "hash_randomization") ++ "bytes_warning", "quiet", "hash_randomization") for attr in attrs: self.assertTrue(hasattr(sys.flags, attr), attr) self.assertEqual(type(getattr(sys.flags, attr)), int, attr) diff --cc Lib/test/test_urllib.py index f6b48cba234d,482acc1c0f22..c6f6f6121f37 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@@ -11,8 -11,8 +11,9 @@@ from test import suppor import os import sys import tempfile -import warnings + +from base64 import b64encode + import collections def hexescape(char): """Escape char as RFC 2396 specifies""" diff --cc Lib/test/test_urlparse.py index a6e7ee8e1ca2,252eb138d06f..ada0ca87887a mode 100644,100644..100755 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@@ -636,166 -463,6 +636,167 @@@ class UrlParseTestCase(unittest.TestCas ('s3', 'foo.com', '/stuff', '', '', '')) self.assertEqual(urllib.parse.urlparse("x-newscheme://foo.com/stuff"), ('x-newscheme', 'foo.com', '/stuff', '', '', '')) + # And for bytes... + self.assertEqual(urllib.parse.urlparse(b"s3://foo.com/stuff"), + (b's3', b'foo.com', b'/stuff', b'', b'', b'')) + self.assertEqual(urllib.parse.urlparse(b"x-newscheme://foo.com/stuff"), + (b'x-newscheme', b'foo.com', b'/stuff', b'', b'', b'')) + + def test_mixed_types_rejected(self): + # Several functions that process either strings or ASCII encoded bytes + # accept multiple arguments. Check they reject mixed type input + with self.assertRaisesRegex(TypeError, "Cannot mix str"): + urllib.parse.urlparse("www.python.org", b"http") + with self.assertRaisesRegex(TypeError, "Cannot mix str"): + urllib.parse.urlparse(b"www.python.org", "http") + with self.assertRaisesRegex(TypeError, "Cannot mix str"): + urllib.parse.urlsplit("www.python.org", b"http") + with self.assertRaisesRegex(TypeError, "Cannot mix str"): + urllib.parse.urlsplit(b"www.python.org", "http") + with self.assertRaisesRegex(TypeError, "Cannot mix str"): + urllib.parse.urlunparse(( b"http", "www.python.org","","","","")) + with self.assertRaisesRegex(TypeError, "Cannot mix str"): + urllib.parse.urlunparse(("http", b"www.python.org","","","","")) + with self.assertRaisesRegex(TypeError, "Cannot mix str"): + urllib.parse.urlunsplit((b"http", "www.python.org","","","")) + with self.assertRaisesRegex(TypeError, "Cannot mix str"): + urllib.parse.urlunsplit(("http", b"www.python.org","","","")) + with self.assertRaisesRegex(TypeError, "Cannot mix str"): + urllib.parse.urljoin("http://python.org", b"http://python.org") + with self.assertRaisesRegex(TypeError, "Cannot mix str"): + urllib.parse.urljoin(b"http://python.org", "http://python.org") + + def _check_result_type(self, str_type): + num_args = len(str_type._fields) + bytes_type = str_type._encoded_counterpart + self.assertIs(bytes_type._decoded_counterpart, str_type) + str_args = ('',) * num_args + bytes_args = (b'',) * num_args + str_result = str_type(*str_args) + bytes_result = bytes_type(*bytes_args) + encoding = 'ascii' + errors = 'strict' + self.assertEqual(str_result, str_args) + self.assertEqual(bytes_result.decode(), str_args) + self.assertEqual(bytes_result.decode(), str_result) + self.assertEqual(bytes_result.decode(encoding), str_args) + self.assertEqual(bytes_result.decode(encoding), str_result) + self.assertEqual(bytes_result.decode(encoding, errors), str_args) + self.assertEqual(bytes_result.decode(encoding, errors), str_result) + self.assertEqual(bytes_result, bytes_args) + self.assertEqual(str_result.encode(), bytes_args) + self.assertEqual(str_result.encode(), bytes_result) + self.assertEqual(str_result.encode(encoding), bytes_args) + self.assertEqual(str_result.encode(encoding), bytes_result) + self.assertEqual(str_result.encode(encoding, errors), bytes_args) + self.assertEqual(str_result.encode(encoding, errors), bytes_result) + + def test_result_pairs(self): + # Check encoding and decoding between result pairs + result_types = [ + urllib.parse.DefragResult, + urllib.parse.SplitResult, + urllib.parse.ParseResult, + ] + for result_type in result_types: + self._check_result_type(result_type) + + def test_parse_qs_encoding(self): + result = urllib.parse.parse_qs("key=\u0141%E9", encoding="latin-1") + self.assertEqual(result, {'key': ['\u0141\xE9']}) + result = urllib.parse.parse_qs("key=\u0141%C3%A9", encoding="utf-8") + self.assertEqual(result, {'key': ['\u0141\xE9']}) + result = urllib.parse.parse_qs("key=\u0141%C3%A9", encoding="ascii") + self.assertEqual(result, {'key': ['\u0141\ufffd\ufffd']}) + result = urllib.parse.parse_qs("key=\u0141%E9-", encoding="ascii") + self.assertEqual(result, {'key': ['\u0141\ufffd-']}) + result = urllib.parse.parse_qs("key=\u0141%E9-", encoding="ascii", + errors="ignore") + self.assertEqual(result, {'key': ['\u0141-']}) + + def test_parse_qsl_encoding(self): + result = urllib.parse.parse_qsl("key=\u0141%E9", encoding="latin-1") + self.assertEqual(result, [('key', '\u0141\xE9')]) + result = urllib.parse.parse_qsl("key=\u0141%C3%A9", encoding="utf-8") + self.assertEqual(result, [('key', '\u0141\xE9')]) + result = urllib.parse.parse_qsl("key=\u0141%C3%A9", encoding="ascii") + self.assertEqual(result, [('key', '\u0141\ufffd\ufffd')]) + result = urllib.parse.parse_qsl("key=\u0141%E9-", encoding="ascii") + self.assertEqual(result, [('key', '\u0141\ufffd-')]) + result = urllib.parse.parse_qsl("key=\u0141%E9-", encoding="ascii", + errors="ignore") + self.assertEqual(result, [('key', '\u0141-')]) + + def test_splitnport(self): + # Normal cases are exercised by other tests; ensure that we also + # catch cases with no port specified. (testcase ensuring coverage) + result = urllib.parse.splitnport('parrot:88') + self.assertEqual(result, ('parrot', 88)) + result = urllib.parse.splitnport('parrot') + self.assertEqual(result, ('parrot', -1)) + result = urllib.parse.splitnport('parrot', 55) + self.assertEqual(result, ('parrot', 55)) + result = urllib.parse.splitnport('parrot:') + self.assertEqual(result, ('parrot', None)) + + def test_splitquery(self): + # Normal cases are exercised by other tests; ensure that we also + # catch cases with no port specified (testcase ensuring coverage) + result = urllib.parse.splitquery('http://python.org/fake?foo=bar') + self.assertEqual(result, ('http://python.org/fake', 'foo=bar')) + result = urllib.parse.splitquery('http://python.org/fake?foo=bar?') + self.assertEqual(result, ('http://python.org/fake?foo=bar', '')) + result = urllib.parse.splitquery('http://python.org/fake') + self.assertEqual(result, ('http://python.org/fake', None)) + + def test_splitvalue(self): + # Normal cases are exercised by other tests; test pathological cases + # with no key/value pairs. (testcase ensuring coverage) + result = urllib.parse.splitvalue('foo=bar') + self.assertEqual(result, ('foo', 'bar')) + result = urllib.parse.splitvalue('foo=') + self.assertEqual(result, ('foo', '')) + result = urllib.parse.splitvalue('foobar') + self.assertEqual(result, ('foobar', None)) + + def test_to_bytes(self): + result = urllib.parse.to_bytes('http://www.python.org') + self.assertEqual(result, 'http://www.python.org') + self.assertRaises(UnicodeError, urllib.parse.to_bytes, + 'http://www.python.org/medi\u00e6val') + + def test_urlencode_sequences(self): + # Other tests incidentally urlencode things; test non-covered cases: + # Sequence and object values. + result = urllib.parse.urlencode({'a': [1, 2], 'b': (3, 4, 5)}, True) - self.assertEqual(result, 'a=1&a=2&b=3&b=4&b=5') ++ # we cannot rely on ordering here ++ assert set(result.split('&')) == {'a=1', 'a=2', 'b=3', 'b=4', 'b=5'} + + class Trivial: + def __str__(self): + return 'trivial' + + result = urllib.parse.urlencode({'a': Trivial()}, True) + self.assertEqual(result, 'a=trivial') + + def test_quote_from_bytes(self): + self.assertRaises(TypeError, urllib.parse.quote_from_bytes, 'foo') + result = urllib.parse.quote_from_bytes(b'archaeological arcana') + self.assertEqual(result, 'archaeological%20arcana') + result = urllib.parse.quote_from_bytes(b'') + self.assertEqual(result, '') + + def test_unquote_to_bytes(self): + result = urllib.parse.unquote_to_bytes('abc%20def') + self.assertEqual(result, b'abc def') + result = urllib.parse.unquote_to_bytes('') + self.assertEqual(result, b'') + + def test_quote_errors(self): + self.assertRaises(TypeError, urllib.parse.quote, b'foo', + encoding='utf-8') + self.assertRaises(TypeError, urllib.parse.quote, b'foo', errors='strict') + def test_main(): support.run_unittest(UrlParseTestCase) diff --cc Makefile.pre.in index c49d777a7e0d,e4470bdb88da..7ffc3ecab3b6 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@@ -321,7 -305,7 +321,8 @@@ PYTHON_OBJS= Python/pymath.o \ Python/pystate.o \ Python/pythonrun.o \ + Python/pytime.o \ + Python/random.o \ Python/structmember.o \ Python/symtable.o \ Python/sysmodule.o \ diff --cc Misc/NEWS index b9cd7644aa96,486da13674d9..63281cf88c48 --- a/Misc/NEWS +++ b/Misc/NEWS @@@ -10,109 -10,11 +10,114 @@@ What's New in Python 3.2.3 Core and Builtins ----------------- + - Issue #13703: oCERT-2011-003: add -R command-line option and PYTHONHASHSEED + environment variables, to provide an opt-in way to protect against denial of + service attacks due to hash collisions within the dict and set types. Patch + by David Malcolm, based on work by Victor Stinner. + +- Issue #13020: Fix a reference leak when allocating a structsequence object + fails. Patch by Suman Saha. + +- Issue #13908: Ready types returned from PyType_FromSpec. + +- Issue #11235: Fix OverflowError when trying to import a source file whose + modification time doesn't fit in a 32-bit timestamp. + +- Fix the builtin module initialization code to store the init function for + future reinitialization. + +- Issue #8052: The posix subprocess module would take a long time closing + all possible file descriptors in the child process rather than just open + file descriptors. It now closes only the open fds if possible for the + default close_fds=True behavior. + +- Issue #13629: Renumber the tokens in token.h so that they match the indexes + into _PyParser_TokenNames. + +- Fix the fix for issue #12149: it was incorrect, although it had the side + effect of appearing to resolve the issue. Thanks to Mark Shannon for + noticing. + +- Issue #13505: Pickle bytes objects in a way that is compatible with + Python 2 when using protocols <= 2. + +- Issue #11147: Fix an unused argument in _Py_ANNOTATE_MEMORY_ORDER. (Fix + given by Campbell Barton). + +- Issue #7111: Python can now be run without a stdin, stdout or stderr + stream. It was already the case with Python 2. However, the corresponding + sys module entries are now set to None (instead of an unusable file object). + +- Issue #13436: Fix a bogus error message when an AST object was passed + an invalid integer value. + +- Issue #13338: Handle all enumerations in _Py_ANNOTATE_MEMORY_ORDER + to allow compiling extension modules with -Wswitch-enum on gcc. + Initial patch by Floris Bruynooghe. + +- Issue #13333: The UTF-7 decoder now accepts lone surrogates (the encoder + already accepts them). + +- Issue #13342: input() used to ignore sys.stdin's and sys.stdout's unicode + error handler in interactive mode (when calling into PyOS_Readline()). + +- Issue #13343: Fix a SystemError when a lambda expression uses a global + variable in the default value of a keyword-only argument: + (lambda *, arg=GLOBAL_NAME: None) + +- Issue #10519: Avoid unnecessary recursive function calls in + setobject.c. + +- Issue #10363: Deallocate global locks in Py_Finalize(). + +- Issue #13018: Fix reference leaks in error paths in dictobject.c. + Patch by Suman Saha. + +- Issue #1294232: In a few cases involving metaclass inheritance, the + interpreter would sometimes invoke the wrong metaclass when building a new + class object. These cases now behave correctly. Patch by Daniel Urban. + +- Issue #12604: VTRACE macro expanded to no-op in _sre.c to avoid compiler + warnings. Patch by Josh Triplett and Petri Lehtinen. + +- Issue #13188: When called without an explicit traceback argument, + generator.throw() now gets the traceback from the passed exception's + ``__traceback__`` attribute. Patch by Petri Lehtinen. + +- Issue #7833: Extension modules built using distutils on Windows will no + longer include a "manifest" to prevent them failing at import time in some + embedded situations. + +- Issue #13063: the Windows error ERROR_NO_DATA (numbered 232 and described + as "The pipe is being closed") is now mapped to POSIX errno EPIPE + (previously EINVAL). + +- Issue #12911: Fix memory consumption when calculating the repr() of huge + tuples or lists. + +- Issue #7732: Don't open a directory as a file anymore while importing a + module. Ignore the direcotry if its name matchs the module name (e.g. + "__init__.py") and raise a ImportError instead. + +- Issue #13021: Missing decref on an error path. Thanks to Suman Saha for + finding the bug and providing a patch. + +- Issue #12973: Fix overflow checks that relied on undefined behaviour in + list_repeat (listobject.c) and islice_next (itertoolsmodule.c). These bugs + caused test failures with recent versions of Clang. + +- Issue #12802: the Windows error ERROR_DIRECTORY (numbered 267) is now + mapped to POSIX errno ENOTDIR (previously EINVAL). + +- Issue #9200: The str.is* methods now work with strings that contain non-BMP + characters even in narrow Unicode builds. + +- Issue #12791: Break reference cycles early when a generator exits with + an exception. + +- Issue #12266: Fix str.capitalize() to correctly uppercase/lowercase + titlecased and cased non-letter characters. + Library ------- diff --cc Misc/python.man index 3dca6045d60e,5b4eeef6546e..fc3566fcacce --- a/Misc/python.man +++ b/Misc/python.man @@@ -148,10 -148,18 +151,22 @@@ to \fI.pyo\fP. Given twice, causes doc .B \-OO Discard docstrings in addition to the \fB-O\fP optimizations. .TP +.B \-q +Do not print the version and copyright messages. These messages are +also suppressed in non-interactive mode. +.TP + .B \-R + Turn on "hash randomization", so that the hash() values of str, bytes and + datetime objects are "salted" with an unpredictable pseudo-random value. + Although they remain constant within an individual Python process, they are + not predictable between repeated invocations of Python. + .IP + This is intended to provide protection against a denial of service + caused by carefully-chosen inputs that exploit the worst case performance + of a dict insertion, O(n^2) complexity. See + http://www.ocert.org/advisories/ocert-2011-003.html + for details. + .TP .BI "\-Q " argument Division control; see PEP 238. The argument must be one of "old" (the default, int/int and long/long return an int or long), "new" (new @@@ -410,9 -418,20 +425,23 @@@ the \fB\-u\fP option If this is set to a non-empty string it is equivalent to specifying the \fB\-v\fP option. If set to an integer, it is equivalent to specifying \fB\-v\fP multiple times. +.IP PYTHONWARNINGS +If this is set to a comma-separated string it is equivalent to +specifying the \fB\-W\fP option for each separate value. + .IP PYTHONHASHSEED + If this variable is set to "random", the effect is the same as specifying + the \fB-R\fP option: a random value is used to seed the hashes of str, + bytes and datetime objects. + + If PYTHONHASHSEED is set to an integer value, it is used as a fixed seed for + generating the hash() of the types covered by the hash randomization. Its + purpose is to allow repeatable hashing, such as for selftests for the + interpreter itself, or to allow a cluster of python processes to share hash + values. + + The integer must be a decimal number in the range [0,4294967295]. Specifying + the value 0 will lead to the same hash values as when hash randomization is + disabled. .SH AUTHOR The Python Software Foundation: http://www.python.org/psf .SH INTERNET RESOURCES diff --cc Modules/_datetimemodule.c index 6ee531768a55,f3103eaf6995..85c5c4de1980 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@@ -2782,10 -2563,11 +2782,11 @@@ static Py_hash_ generic_hash(unsigned char *data, int len) { register unsigned char *p; - register long x; + register Py_hash_t x; p = (unsigned char *) data; - x = *p << 7; + x = _Py_HashSecret.prefix; + x ^= *p << 7; while (--len >= 0) x = (1000003*x) ^ *p++; x ^= len; diff --cc Modules/main.c index 5a985a5c16f8,9607cb334db2..ed84aa028604 --- a/Modules/main.c +++ b/Modules/main.c @@@ -46,7 -47,7 +46,7 @@@ static wchar_t **orig_argv static int orig_argc; /* command line options */ - #define BASE_OPTS L"bBc:dEhiJm:OqsStuvVW:xX:?" -#define BASE_OPTS L"bBc:dEhiJm:ORsStuvVW:xX?" ++#define BASE_OPTS L"bBc:dEhiJm:OqRsStuvVW:xX:?" #define PROGRAM_OPTS BASE_OPTS @@@ -71,7 -72,9 +71,10 @@@ static char *usage_2 = " -m mod : run library module as a script (terminates option list)\n\ -O : optimize generated bytecode slightly; also PYTHONOPTIMIZE=x\n\ -OO : remove doc-strings in addition to the -O optimizations\n\ +-q : don't print version and copyright messages on interactive startup\n\ + -R : use a pseudo-random salt to make hash() values of various types be\n\ + unpredictable between separate invocations of the interpreter, as\n\ + a defence against denial-of-service attacks\n\ -s : don't add user site directory to sys.path; also PYTHONNOUSERSITE\n\ -S : don't imply 'import site' on initialization\n\ "; @@@ -95,13 -96,41 +98,19 @@@ PYTHONSTARTUP: file executed on interac PYTHONPATH : '%c'-separated list of directories prefixed to the\n\ default module search path. The result is sys.path.\n\ "; -static char *usage_5 = "\ -PYTHONHOME : alternate directory (or %c).\n\ - The default module search path uses %s.\n\ -PYTHONCASEOK : ignore case in 'import' statements (Windows).\n\ -PYTHONIOENCODING: Encoding[:errors] used for stdin/stdout/stderr.\n\ +static char *usage_5 = +"PYTHONHOME : alternate directory (or %c).\n" +" The default module search path uses %s.\n" +"PYTHONCASEOK : ignore case in 'import' statements (Windows).\n" - "PYTHONIOENCODING: Encoding[:errors] used for stdin/stdout/stderr.\n" - ; ++"PYTHONIOENCODING: Encoding[:errors] used for stdin/stdout/stderr.\n\ + "; + static char *usage_6 = "\ + PYTHONHASHSEED: if this variable is set to ``random``, the effect is the same \n\ + as specifying the :option:`-R` option: a random value is used to seed the\n\ + hashes of str, bytes and datetime objects. It can also be set to an integer\n\ + in the range [0,4294967295] to get hash values with a predictable seed.\n\ + "; -#ifndef MS_WINDOWS -static FILE* -_wfopen(const wchar_t *path, const wchar_t *mode) -{ - char cpath[PATH_MAX]; - char cmode[10]; - size_t r; - r = wcstombs(cpath, path, PATH_MAX); - if (r == (size_t)-1 || r >= PATH_MAX) { - errno = EINVAL; - return NULL; - } - r = wcstombs(cmode, mode, 10); - if (r == (size_t)-1 || r >= 10) { - errno = EINVAL; - return NULL; - } - return fopen(cpath, cmode); -} -#endif - - static int usage(int exitcode, wchar_t* program) { @@@ -421,14 -383,10 +431,18 @@@ Py_Main(int argc, wchar_t **argv PySys_AddWarnOption(_PyOS_optarg); break; + case 'X': + PySys_AddXOption(_PyOS_optarg); + break; + + case 'q': + Py_QuietFlag++; + break; + + case 'R': + Py_HashRandomizationFlag++; + break; + /* This space reserved for other options */ default: diff --cc Modules/posixmodule.c index 0afab3cd93c3,dbbc29f95efd..9f57673fbe42 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@@ -7761,84 -7008,7 +7680,83 @@@ posix_urandom(PyObject *self, PyObject } return result; } - #endif +#ifdef HAVE_SETRESUID +PyDoc_STRVAR(posix_setresuid__doc__, +"setresuid(ruid, euid, suid)\n\n\ +Set the current process's real, effective, and saved user ids."); + +static PyObject* +posix_setresuid (PyObject *self, PyObject *args) +{ + /* We assume uid_t is no larger than a long. */ + long ruid, euid, suid; + if (!PyArg_ParseTuple(args, "lll", &ruid, &euid, &suid)) + return NULL; + if (setresuid(ruid, euid, suid) < 0) + return posix_error(); + Py_RETURN_NONE; +} +#endif + +#ifdef HAVE_SETRESGID +PyDoc_STRVAR(posix_setresgid__doc__, +"setresgid(rgid, egid, sgid)\n\n\ +Set the current process's real, effective, and saved group ids."); + +static PyObject* +posix_setresgid (PyObject *self, PyObject *args) +{ + /* We assume uid_t is no larger than a long. */ + long rgid, egid, sgid; + if (!PyArg_ParseTuple(args, "lll", &rgid, &egid, &sgid)) + return NULL; + if (setresgid(rgid, egid, sgid) < 0) + return posix_error(); + Py_RETURN_NONE; +} +#endif + +#ifdef HAVE_GETRESUID +PyDoc_STRVAR(posix_getresuid__doc__, +"getresuid() -> (ruid, euid, suid)\n\n\ +Get tuple of the current process's real, effective, and saved user ids."); + +static PyObject* +posix_getresuid (PyObject *self, PyObject *noargs) +{ + uid_t ruid, euid, suid; + long l_ruid, l_euid, l_suid; + if (getresuid(&ruid, &euid, &suid) < 0) + return posix_error(); + /* Force the values into long's as we don't know the size of uid_t. */ + l_ruid = ruid; + l_euid = euid; + l_suid = suid; + return Py_BuildValue("(lll)", l_ruid, l_euid, l_suid); +} +#endif + +#ifdef HAVE_GETRESGID +PyDoc_STRVAR(posix_getresgid__doc__, +"getresgid() -> (rgid, egid, sgid)\n\n\ +Get tuple of the current process's real, effective, and saved group ids."); + +static PyObject* +posix_getresgid (PyObject *self, PyObject *noargs) +{ + uid_t rgid, egid, sgid; + long l_rgid, l_egid, l_sgid; + if (getresgid(&rgid, &egid, &sgid) < 0) + return posix_error(); + /* Force the values into long's as we don't know the size of uid_t. */ + l_rgid = rgid; + l_egid = egid; + l_sgid = sgid; + return Py_BuildValue("(lll)", l_rgid, l_egid, l_sgid); +} +#endif + static PyMethodDef posix_methods[] = { {"access", posix_access, METH_VARARGS, posix_access__doc__}, #ifdef HAVE_TTYNAME @@@ -8137,25 -7292,7 +8055,19 @@@ #ifdef HAVE_GETLOADAVG {"getloadavg", posix_getloadavg, METH_NOARGS, posix_getloadavg__doc__}, #endif - #ifdef MS_WINDOWS - {"urandom", win32_urandom, METH_VARARGS, win32_urandom__doc__}, - #endif - #ifdef __VMS - {"urandom", vms_urandom, METH_VARARGS, vms_urandom__doc__}, - #endif + {"urandom", posix_urandom, METH_VARARGS, posix_urandom__doc__}, +#ifdef HAVE_SETRESUID + {"setresuid", posix_setresuid, METH_VARARGS, posix_setresuid__doc__}, +#endif +#ifdef HAVE_SETRESGID + {"setresgid", posix_setresgid, METH_VARARGS, posix_setresgid__doc__}, +#endif +#ifdef HAVE_GETRESUID + {"getresuid", posix_getresuid, METH_NOARGS, posix_getresuid__doc__}, +#endif +#ifdef HAVE_GETRESGID + {"getresgid", posix_getresgid, METH_NOARGS, posix_getresgid__doc__}, +#endif - {NULL, NULL} /* Sentinel */ }; diff --cc Objects/bytesobject.c index a0d4cbd9ae86,e6ab440caa95..d63fabcc9ebb --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@@ -878,11 -899,21 +878,21 @@@ bytes_hash(PyBytesObject *a if (a->ob_shash != -1) return a->ob_shash; len = Py_SIZE(a); + /* + We make the hash of the empty string be 0, rather than using + (prefix ^ suffix), since this slightly obfuscates the hash secret + */ + if (len == 0) { + a->ob_shash = 0; + return 0; + } p = (unsigned char *) a->ob_sval; - x = *p << 7; + x = _Py_HashSecret.prefix; + x ^= *p << 7; while (--len >= 0) - x = (1000003*x) ^ *p++; + x = (_PyHASH_MULTIPLIER*x) ^ *p++; x ^= Py_SIZE(a); + x ^= _Py_HashSecret.suffix; if (x == -1) x = -2; a->ob_shash = x; diff --cc Objects/object.c index 694e7e719e94,0b1c656cc600..84ec2f34f582 --- a/Objects/object.c +++ b/Objects/object.c @@@ -754,7 -712,9 +754,9 @@@ PyObject_HashNotImplemented(PyObject *v return -1; } + _Py_HashSecret_t _Py_HashSecret; + -long +Py_hash_t PyObject_Hash(PyObject *v) { PyTypeObject *tp = Py_TYPE(v); diff --cc Objects/unicodeobject.c index 70856f54552c,5986fb8ea072..467f95c444f5 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@@ -7676,11 -7344,21 +7676,21 @@@ unicode_hash(PyUnicodeObject *self if (self->hash != -1) return self->hash; len = Py_SIZE(self); + /* + We make the hash of the empty string be 0, rather than using + (prefix ^ suffix), since this slightly obfuscates the hash secret + */ + if (len == 0) { + self->hash = 0; + return 0; + } p = self->str; - x = *p << 7; + x = _Py_HashSecret.prefix; + x ^= *p << 7; while (--len >= 0) - x = (1000003*x) ^ *p++; + x = (_PyHASH_MULTIPLIER*x) ^ *p++; x ^= Py_SIZE(self); + x ^= _Py_HashSecret.suffix; if (x == -1) x = -2; self->hash = x; diff --cc Python/pythonrun.c index ec69bcba8f45,4474e79b0f4e..718362d47922 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@@ -89,9 -92,8 +90,10 @@@ int Py_FrozenFlag; /* Needed by getpath int Py_IgnoreEnvironmentFlag; /* e.g. PYTHONPATH, PYTHONHOME */ int Py_NoUserSiteDirectory = 0; /* for -s and site.py */ int Py_UnbufferedStdioFlag = 0; /* Unbuffered binary std{in,out,err} */ + int Py_HashRandomizationFlag = 0; /* for -R and PYTHONHASHSEED */ +PyThreadState *_Py_Finalizing = NULL; + /* PyModule_GetWarningsModule is no longer necessary as of 2.6 since _warnings is builtin. This API should not be used. */ PyObject * diff --cc Python/sysmodule.c index 8a659c5e460a,6a7e91432c0b..c4f27d086877 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@@ -1366,8 -1125,8 +1366,9 @@@ static PyStructSequence_Field flags_fie #endif /* {"unbuffered", "-u"}, */ /* {"skip_first", "-x"}, */ - {"bytes_warning", "-b"}, - {"hash_randomization", "-R"}, + {"bytes_warning", "-b"}, + {"quiet", "-q"}, ++ {"hash_randomization", "-R"}, {0} }; @@@ -1376,9 -1135,9 +1377,9 @@@ static PyStructSequence_Desc flags_des flags__doc__, /* doc */ flags_fields, /* fields */ #ifdef RISCOS -- 13 ++ 14 #else -- 12 ++ 13 #endif }; @@@ -1411,7 -1170,7 +1412,8 @@@ make_flags(void /* SetFlag(saw_unbuffered_flag); */ /* SetFlag(skipfirstline); */ SetFlag(Py_BytesWarningFlag); + SetFlag(Py_QuietFlag); + SetFlag(Py_HashRandomizationFlag); #undef SetFlag if (PyErr_Occurred()) {