Ugly. A pile of new xxxFlags() functions, to communicate to the parser

that 'yield' is a keyword.  This doesn't help test_generators at all!  I
don't know why not.  These things do work now (and didn't before this
patch):

1. "from __future__ import generators" now works in a native shell.

2. Similarly "python -i xxx.py" now has generators enabled in the
   shell if xxx.py had them enabled.

3. This program (which was my doctest proxy) works fine:

from __future__ import generators

source = """\
def f():
    yield 1
"""

exec compile(source, "", "single") in globals()
print type(f())

diff --git a/Include/parsetok.h b/Include/parsetok.h
index aebc83cdd856020b443508cab0030e21fce99faa..66359d42521673ae118551eb0ab6662d6a8d3867 100644 (file)
--- a/Include/parsetok.h
+++ b/Include/parsetok.h
@@ -17,11 +17,19 @@ typedef struct {
     int expected;
 } perrdetail;
 
+#define PyPARSE_YIELD_IS_KEYWORD       0x0001
+
 extern DL_IMPORT(node *) PyParser_ParseString(char *, grammar *, int,
                                               perrdetail *);
 extern DL_IMPORT(node *) PyParser_ParseFile (FILE *, char *, grammar *, int,
                                              char *, char *, perrdetail *);
 
+extern DL_IMPORT(node *) PyParser_ParseStringFlags(char *, grammar *, int,
+                                              perrdetail *, int);
+extern DL_IMPORT(node *) PyParser_ParseFileFlags(FILE *, char *, grammar *,
+                                                int, char *, char *,
+                                                perrdetail *, int);
+
 #ifdef __cplusplus
 }
 #endif

diff --git a/Include/pythonrun.h b/Include/pythonrun.h
index 3877a8237dde27b57ad80ee32571375ad50e1233..6e9821bf7fcdf8da701a94d7319ace1ea06d9d30 100644 (file)
--- a/Include/pythonrun.h
+++ b/Include/pythonrun.h
@@ -45,6 +45,9 @@ DL_IMPORT(int) PyRun_InteractiveLoopFlags(FILE *, char *, PyCompilerFlags *);
 
 DL_IMPORT(struct _node *) PyParser_SimpleParseString(char *, int);
 DL_IMPORT(struct _node *) PyParser_SimpleParseFile(FILE *, char *, int);
+DL_IMPORT(struct _node *) PyParser_SimpleParseStringFlags(char *, int, int);
+DL_IMPORT(struct _node *) PyParser_SimpleParseFileFlags(FILE *, char *,
+                                                       int, int);
 
 DL_IMPORT(PyObject *) PyRun_String(char *, int, PyObject *, PyObject *);
 DL_IMPORT(PyObject *) PyRun_File(FILE *, char *, int, PyObject *, PyObject *);

diff --git a/Lib/doctest.py b/Lib/doctest.py
index b59a68a2ff7c5b84d25bbf197523eba56c9cd9f5..fc826df50dda653d90698c97ab1e289ca8df2d3a 100644 (file)
--- a/Lib/doctest.py
+++ b/Lib/doctest.py
@@ -348,6 +348,8 @@ Test passed.
 # 0,9,7    9-Feb-2001
 #    string method conversion
 
+from __future__ import generators
+
 __version__ = 0, 9, 7
 
 import types

diff --git a/Parser/parsetok.c b/Parser/parsetok.c
index b68fbd701a1ca0870b08b49e8559d95c3281fe26..6017e5f7f8621a689a0d853a75cee4fdf6f91a11 100644 (file)
--- a/Parser/parsetok.c
+++ b/Parser/parsetok.c
@@ -13,12 +13,18 @@ int Py_TabcheckFlag;
 
 
 /* Forward */
-static node *parsetok(struct tok_state *, grammar *, int, perrdetail *);
+static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int);
 
 /* Parse input coming from a string.  Return error code, print some errors. */
-
 node *
 PyParser_ParseString(char *s, grammar *g, int start, perrdetail *err_ret)
+{
+       return PyParser_ParseStringFlags(s, g, start, err_ret, 0);
+}
+
+node *
+PyParser_ParseStringFlags(char *s, grammar *g, int start,
+                         perrdetail *err_ret, int flags)
 {
        struct tok_state *tok;
 
@@ -42,7 +48,7 @@ PyParser_ParseString(char *s, grammar *g, int start, perrdetail *err_ret)
                        tok->alterror++;
        }
 
-       return parsetok(tok, g, start, err_ret);
+       return parsetok(tok, g, start, err_ret, flags);
 }
 
 
@@ -51,6 +57,14 @@ PyParser_ParseString(char *s, grammar *g, int start, perrdetail *err_ret)
 node *
 PyParser_ParseFile(FILE *fp, char *filename, grammar *g, int start,
                   char *ps1, char *ps2, perrdetail *err_ret)
+{
+       return PyParser_ParseFileFlags(fp, filename, g, start, ps1, ps2,
+                                      err_ret, 0);
+}
+
+node *
+PyParser_ParseFileFlags(FILE *fp, char *filename, grammar *g, int start,
+                       char *ps1, char *ps2, perrdetail *err_ret, int flags)
 {
        struct tok_state *tok;
 
@@ -72,14 +86,15 @@ PyParser_ParseFile(FILE *fp, char *filename, grammar *g, int start,
        }
 
 
-       return parsetok(tok, g, start, err_ret);
+       return parsetok(tok, g, start, err_ret, flags);
 }
 
 /* Parse input coming from the given tokenizer structure.
    Return error code. */
 
 static node *
-parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret)
+parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
+        int flags)
 {
        parser_state *ps;
        node *n;
@@ -90,6 +105,8 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret)
                err_ret->error = E_NOMEM;
                return NULL;
        }
+       if (flags & PyPARSE_YIELD_IS_KEYWORD)
+               ps->p_generators = 1;
 
        for (;;) {
                char *a, *b;

diff --git a/Python/pythonrun.c b/Python/pythonrun.c
index 0412b7e70d052f22dd7e16e44efa939f55a5956c..1d20deb174aad0eb0103352071c08f4e62df61cb 100644 (file)
--- a/Python/pythonrun.c
+++ b/Python/pythonrun.c
@@ -533,6 +533,7 @@ PyRun_InteractiveOneFlags(FILE *fp, char *filename, PyCompilerFlags *flags)
        node *n;
        perrdetail err;
        char *ps1 = "", *ps2 = "";
+
        v = PySys_GetObject("ps1");
        if (v != NULL) {
                v = PyObject_Str(v);
@@ -549,8 +550,11 @@ PyRun_InteractiveOneFlags(FILE *fp, char *filename, PyCompilerFlags *flags)
                else if (PyString_Check(w))
                        ps2 = PyString_AsString(w);
        }
-       n = PyParser_ParseFile(fp, filename, &_PyParser_Grammar,
-                              Py_single_input, ps1, ps2, &err);
+       n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar,
+                                   Py_single_input, ps1, ps2, &err,
+                                   (flags &&
+                                    flags->cf_flags & PyCF_GENERATORS) ?
+                                       PyPARSE_YIELD_IS_KEYWORD : 0);
        Py_XDECREF(v);
        Py_XDECREF(w);
        if (n == NULL) {
@@ -1017,7 +1021,9 @@ PyObject *
 PyRun_FileExFlags(FILE *fp, char *filename, int start, PyObject *globals,
                  PyObject *locals, int closeit, PyCompilerFlags *flags)
 {
-       node *n = PyParser_SimpleParseFile(fp, filename, start);
+       node *n = PyParser_SimpleParseFileFlags(fp, filename, start,
+                       (flags && flags->cf_flags & PyCF_GENERATORS) ?
+                               PyPARSE_YIELD_IS_KEYWORD : 0);
        if (closeit)
                fclose(fp);
        return run_err_node(n, filename, globals, locals, flags);
@@ -1101,7 +1107,9 @@ Py_CompileStringFlags(char *str, char *filename, int start,
 {
        node *n;
        PyCodeObject *co;
-       n = PyParser_SimpleParseString(str, start);
+       n = PyParser_SimpleParseStringFlags(str, start,
+               (flags && flags->cf_flags & PyCF_GENERATORS) ?
+                       PyPARSE_YIELD_IS_KEYWORD : 0);
        if (n == NULL)
                return NULL;
        co = PyNode_CompileFlags(n, filename, flags);
@@ -1125,30 +1133,43 @@ Py_SymtableString(char *str, char *filename, int start)
 /* Simplified interface to parsefile -- return node or set exception */
 
 node *
-PyParser_SimpleParseFile(FILE *fp, char *filename, int start)
+PyParser_SimpleParseFileFlags(FILE *fp, char *filename, int start, int flags)
 {
        node *n;
        perrdetail err;
-       n = PyParser_ParseFile(fp, filename, &_PyParser_Grammar, start,
-                               (char *)0, (char *)0, &err);
+       n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar, start,
+                                       (char *)0, (char *)0, &err, flags);
        if (n == NULL)
                err_input(&err);
        return n;
 }
 
+node *
+PyParser_SimpleParseFile(FILE *fp, char *filename, int start)
+{
+       return PyParser_SimpleParseFileFlags(fp, filename, start, 0);
+}
+
 /* Simplified interface to parsestring -- return node or set exception */
 
 node *
-PyParser_SimpleParseString(char *str, int start)
+PyParser_SimpleParseStringFlags(char *str, int start, int flags)
 {
        node *n;
        perrdetail err;
-       n = PyParser_ParseString(str, &_PyParser_Grammar, start, &err);
+       n = PyParser_ParseStringFlags(str, &_PyParser_Grammar, start, &err,
+                                     flags);
        if (n == NULL)
                err_input(&err);
        return n;
 }
 
+node *
+PyParser_SimpleParseString(char *str, int start)
+{
+       return PyParser_SimpleParseStringFlags(str, start, 0);
+}
+
 /* Set the error appropriate to the given input error code (see errcode.h) */
 
 static void