Two more patches by Tony Lownds (SF# 1607548).

(1)
Combines the code paths for MAKE_FUNCTION and MAKE_CLOSURE.
Fixes a crash where functions with closures and either annotations or
keyword-only arguments result in MAKE_CLOSURE, but only
MAKE_FUNCTION has the code to handle annotations or keyword-only
arguments.
Includes enough tests to trigger the bug.

(2)
Change peepholer to not bail in the presence of EXTENDED_ARG +
MAKE_FUNCTION.
Enforce the natural 16-bit limit of annotations in compile.c.

Also update Misc/NEWS with the "input = raw_input" change.

diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py
index 022f7c0d929fc33dcd66988a5e2ed6d699947056..d5fda13be92524b3ecd4a754f60ef5be1743b3aa 100644 (file)
--- a/Lib/test/test_compile.py
+++ b/Lib/test/test_compile.py
@@ -393,6 +393,19 @@ if 1:
         del d[..., ...]
         self.assertEqual((Ellipsis, Ellipsis) in d, False)
 
+    def test_annotation_limit(self):
+        # 16 bits are available for # of annotations, and the
+        # tuple of annotations names is counted, hence 65534
+        # is the max. Ensure the result of too many annotations is a
+        # SyntaxError.
+        s = "def f((%s)): pass"
+        s %= ', '.join('a%d:%d' % (i,i) for i in xrange(65535))        
+        self.assertRaises(SyntaxError, compile, s, '?', 'exec')
+        # Test that the max # of annotations compiles.
+        s = "def f((%s)): pass"
+        s %= ', '.join('a%d:%d' % (i,i) for i in xrange(65534))
+        compile(s, '?', 'exec')
+        
 def test_main():
     test_support.run_unittest(TestSpecifics)
 

diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py
index cb3702189118b2ae6a949bf6c7f8fed58643c48f..1a1475628e5db3c67a0c5bb34a27af7cce5108ed 100644 (file)
--- a/Lib/test/test_grammar.py
+++ b/Lib/test/test_grammar.py
@@ -321,6 +321,13 @@ class GrammarTests(unittest.TestCase):
         self.assertEquals(f.__annotations__,
                           {'b': 1, 'c': 2, 'e': 3, 'g': 6, 'h': 7, 'j': 9,
                            'k': 11, 'return': 12})
+                           
+        # test MAKE_CLOSURE with a variety of oparg's
+        closure = 1
+        def f(): return closure
+        def f(x=1): return closure
+        def f(*, k=1): return closure
+        def f() -> int: return closure
 
     def testLambdef(self):
         ### lambdef: 'lambda' [varargslist] ':' test

diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py
index 213edd2ccb0da40e6cefbe71abac274383a08073..9ed814a6530ed99322b4c21695aec53753986e53 100644 (file)
--- a/Lib/test/test_peepholer.py
+++ b/Lib/test/test_peepholer.py
@@ -195,6 +195,14 @@ class TestTranforms(unittest.TestCase):
         # There should be one jump for the while loop.
         self.assertEqual(asm.split().count('JUMP_ABSOLUTE'), 1)
         self.assertEqual(asm.split().count('RETURN_VALUE'), 2)
+        
+    def test_make_function_doesnt_bail(self):
+        def f():
+            def g()->1+1: 
+                pass
+            return g
+        asm = disassemble(f)
+        self.assert_('BINARY_ADD' not in asm)              
 
 
 def test_main(verbose=None):

diff --git a/Misc/NEWS b/Misc/NEWS
index 91e1f0256bdcb21f2ca3d35ecf8a9aa96b4f02c0..4eb2f47b9053210decacde9078de4551a05a4f3c 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -28,6 +28,10 @@ TO DO
 Core and Builtins
 -----------------
 
+- input() becomes raw_input(): the name input() now implements the
+  functionality formerly known as raw_input(); the name raw_input()
+  is no longer defined.
+
 - Objects listed in an 'except' clause must inherit from BaseException.
 
 - PEP 3106: dict.iterkeys(), .iteritems(), .itervalues() are now gone;
@@ -82,7 +86,7 @@ Core and Builtins
   backticks (`x`), <>
 
 - Removed these Python builtins:
-  apply(), coerce(), input(), raw_input()
+  apply(), coerce()
 
 - Removed these Python methods:
   {}.has_key

diff --git a/Python/ceval.c b/Python/ceval.c
index 86dcea28c1d338d0c7df9206e7397dbcf5bc527a..fe5de0375aa6ac24abf2170fafa23ca300b2e211 100644 (file)
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -2236,6 +2236,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
                    break;
                }
 
+               case MAKE_CLOSURE:              
                case MAKE_FUNCTION:
                {
                    int posdefaults = oparg & 0xff;
@@ -2245,6 +2246,12 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
                        v = POP(); /* code object */
                        x = PyFunction_New(v, f->f_globals);
                        Py_DECREF(v);
+                       
+                       if (x != NULL && opcode == MAKE_CLOSURE) {
+                               v = POP();
+                               err = PyFunction_SetClosure(x, v);
+                               Py_DECREF(v);
+                       }
 
                        if (x != NULL && num_annotations > 0) {
                                Py_ssize_t name_ix;
@@ -2308,34 +2315,6 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
                        break;
                }
 
-               case MAKE_CLOSURE:
-               {
-                       v = POP(); /* code object */
-                       x = PyFunction_New(v, f->f_globals);
-                       Py_DECREF(v);
-                       if (x != NULL) {
-                               v = POP();
-                               err = PyFunction_SetClosure(x, v);
-                               Py_DECREF(v);
-                       }
-                       if (x != NULL && oparg > 0) {
-                               v = PyTuple_New(oparg);
-                               if (v == NULL) {
-                                       Py_DECREF(x);
-                                       x = NULL;
-                                       break;
-                               }
-                               while (--oparg >= 0) {
-                                       w = POP();
-                                       PyTuple_SET_ITEM(v, oparg, w);
-                               }
-                               err = PyFunction_SetDefaults(x, v);
-                               Py_DECREF(v);
-                       }
-                       PUSH(x);
-                       break;
-               }
-
                case BUILD_SLICE:
                        if (oparg == 3)
                                w = POP();

diff --git a/Python/compile.c b/Python/compile.c
index ed0bdcf92a57df2d450d8f037bb98fcba4add4b5..7f0fc503bb47bb75f47dd5e758aff1bfb9128eb5 100644 (file)
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -836,6 +836,8 @@ opcode_stack_effect(int opcode, int oparg)
                        return -NARGS(oparg)-2;
                case MAKE_FUNCTION:
                        return -NARGS(oparg) - ((oparg >> 16) & 0xffff);
+               case MAKE_CLOSURE:
+                       return -1 - NARGS(oparg) - ((oparg >> 16) & 0xffff);
 #undef NARGS
                case BUILD_SLICE:
                        if (oparg == 3)
@@ -843,8 +845,6 @@ opcode_stack_effect(int opcode, int oparg)
                        else
                                return -1;
 
-               case MAKE_CLOSURE:
-                       return -oparg;
                case LOAD_CLOSURE:
                        return 1;
                case LOAD_DEREF:
@@ -1367,8 +1367,12 @@ static int
 compiler_visit_annotations(struct compiler *c, arguments_ty args,
                            expr_ty returns)
 {
-       /* push arg annotations and a list of the argument names. return the #
-          of items pushed. this is out-of-order wrt the source code. */
+       /* Push arg annotations and a list of the argument names. Return the #
+          of items pushed. The expressions are evaluated out-of-order wrt the 
+          source code. 
+          
+          More than 2^16-1 annotations is a SyntaxError. Returns -1 on error.
+          */
        static identifier return_str;
        PyObject *names;
        int len;
@@ -1399,6 +1403,12 @@ compiler_visit_annotations(struct compiler *c, arguments_ty args,
        }
 
        len = PyList_GET_SIZE(names);
+       if (len > 65534) {
+               /* len must fit in 16 bits, and len is incremented below */
+               PyErr_SetString(PyExc_SyntaxError,
+                               "too many annotations");
+               goto error;
+       }       
        if (len) {
                /* convert names to a tuple and place on stack */
                PyObject *elt;
@@ -1449,6 +1459,9 @@ compiler_function(struct compiler *c, stmt_ty s)
        if (args->defaults)
                VISIT_SEQ(c, expr, args->defaults);
        num_annotations = compiler_visit_annotations(c, args, returns);
+       if (num_annotations < 0)
+               return 0;
+       assert((num_annotations & 0xFFFF) == num_annotations);
 
        if (!compiler_enter_scope(c, s->v.FunctionDef.name, (void *)s,
                                  s->lineno))

diff --git a/Python/peephole.c b/Python/peephole.c
index 28e4c4c32f1213694b7f9a60a0b5500c8a00096e..f2e0c0b0886e50f790138ff863790e046ba06ead 100644 (file)
--- a/Python/peephole.c
+++ b/Python/peephole.c
@@ -261,10 +261,12 @@ markblocks(unsigned char *code, int len)
    The consts object should still be in list form to allow new constants 
    to be appended.
 
-   To keep the optimizer simple, it bails out (does nothing) for code
-   containing extended arguments or that has a length over 32,700.  That 
-   allows us to avoid overflow and sign issues.         Likewise, it bails when
-   the lineno table has complex encoding for gaps >= 255.
+   To keep the optimizer simple, it bails out (does nothing) for code that
+   has a length over 32,700, and does not calculate extended arguments. 
+   That allows us to avoid overflow and sign issues. Likewise, it bails when
+   the lineno table has complex encoding for gaps >= 255. EXTENDED_ARG can
+   appear before MAKE_FUNCTION; in this case both opcodes are skipped.
+   EXTENDED_ARG preceding any other opcode causes the optimizer to bail.
 
    Optimizations are restricted to simple transformations occuring within a
    single basic block. All transformations keep the code size the same or 
@@ -535,7 +537,11 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names,
                                break;
 
                        case EXTENDED_ARG:
-                               goto exitUnchanged;
+                               if (codestr[i+3] != MAKE_FUNCTION)
+                                       goto exitUnchanged;
+                               /* don't visit MAKE_FUNCTION as GETARG will be wrong */
+                               i += 3;
+                               break;
 
                                /* Replace RETURN LOAD_CONST None RETURN with just RETURN */
                                /* Remove unreachable JUMPs after RETURN */