Part way to allowing "from __future__ import generators" to communicate

that info to code dynamically compiled *by* code compiled with generators
enabled.  Doesn't yet work because there's still no way to tell the parser
that "yield" is OK (unlike nested_scopes, the parser has its fingers in
this too).
Replaced PyEval_GetNestedScopes by a more-general
PyEval_MergeCompilerFlags.  Perhaps I should not have?  I doubted it was
*intended* to be part of the public API, so just did.

diff --git a/Include/Python.h b/Include/Python.h
index 5d9112592d68a1adc26ff3e3b6e3c12ad91923ae..24d33b266dc061c2aab0ad73527aa32caae84b24 100644 (file)
--- a/Include/Python.h
+++ b/Include/Python.h
@@ -96,8 +96,8 @@
 #include "pystate.h"
 
 #include "modsupport.h"
-#include "ceval.h"
 #include "pythonrun.h"
+#include "ceval.h"
 #include "sysmodule.h"
 #include "intrcheck.h"
 #include "import.h"

diff --git a/Include/ceval.h b/Include/ceval.h
index 19ac064fdc07868237994bc00e4cbae071c6edc3..0fc5cbabfe0b5275c9fcfd188ab903a674aa3a30 100644 (file)
--- a/Include/ceval.h
+++ b/Include/ceval.h
@@ -31,7 +31,11 @@ DL_IMPORT(PyObject *) PyEval_GetLocals(void);
 DL_IMPORT(PyObject *) PyEval_GetOwner(void);
 DL_IMPORT(PyObject *) PyEval_GetFrame(void);
 DL_IMPORT(int) PyEval_GetRestricted(void);
-DL_IMPORT(int) PyEval_GetNestedScopes(void);
+
+/* Look at the current frame's (if any) code's co_flags, and turn on
+   the corresponding compiler flags in cf->cf_flags.  Return 1 if any
+   flag was set, else return 0. */
+DL_IMPORT(int) PyEval_MergeCompilerFlags(PyCompilerFlags *cf);
 
 DL_IMPORT(int) Py_FlushLine(void);
 

diff --git a/Include/compile.h b/Include/compile.h
index 5e6e572ab45dee6de956e18d443d22b312c44567..89c5ba46e06203d0dfe1a98df4f5b56c13d1fe78 100644 (file)
--- a/Include/compile.h
+++ b/Include/compile.h
@@ -34,6 +34,13 @@ typedef struct {
 #define CO_VARKEYWORDS 0x0008
 #define CO_NESTED       0x0010
 #define CO_GENERATOR    0x0020
+/* XXX Temporary hack.  Until generators are a permanent part of the
+   language, we need a way for a code object to record that generators
+   were *possible* when it was compiled.  This is so code dynamically
+   compiled *by* a code object knows whether to allow yield stmts.  In
+   effect, this passes on the "from __future__ import generators" state
+   in effect when the code block was compiled. */
+#define CO_GENERATOR_ALLOWED    0x1000
 
 extern DL_IMPORT(PyTypeObject) PyCode_Type;
 
@@ -56,6 +63,7 @@ typedef struct {
     int ff_found_docstring;
     int ff_last_lineno;
     int ff_nested_scopes;
+    int ff_generators;
 } PyFutureFeatures;
 
 DL_IMPORT(PyFutureFeatures *) PyNode_Future(struct _node *, char *);

diff --git a/Include/pythonrun.h b/Include/pythonrun.h
index e7274ec489f384565820f25bbac674abdc606892..3877a8237dde27b57ad80ee32571375ad50e1233 100644 (file)
--- a/Include/pythonrun.h
+++ b/Include/pythonrun.h
@@ -7,8 +7,13 @@
 extern "C" {
 #endif
 
+/* These flags are named after the __future__ statements that introduced
+   them.  May not remain true for later additions, so fiddle this comment
+   accordingly then. */
+#define PyCF_NESTED_SCOPES     (0x00000001UL)
+#define PyCF_GENERATORS                (0x00000002UL)
 typedef struct {
-       int cf_nested_scopes;
+       unsigned long cf_flags;  /* bitmask of PyCF_xxx flags */
 } PyCompilerFlags;
 
 DL_IMPORT(void) Py_SetProgramName(char *);

diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py
index 9aa462fbbbc7bc0fe3d10a7711bd6c8f90381e03..2a174c3d811db34a2e7e5fa3b734f0fc95fbaf0b 100644 (file)
--- a/Lib/test/test_generators.py
+++ b/Lib/test/test_generators.py
@@ -1,3 +1,5 @@
+from __future__ import generators
+
 tutorial_tests = """
 Let's try a simple generator:
 

diff --git a/Modules/main.c b/Modules/main.c
index 64abaf3eb27eab6898f421a15797a0e305d775f6..3ce83c3538c14625e6dc6479ffac2cdb2fa86e41 100644 (file)
--- a/Modules/main.c
+++ b/Modules/main.c
@@ -298,7 +298,7 @@ Py_Main(int argc, char **argv)
                        Py_DECREF(v);
        }
 
-       cf.cf_nested_scopes = 0;
+       cf.cf_flags = 0;
 
        if (command) {
                sts = PyRun_SimpleString(command) != 0;

diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
index 11d6f4c41cb7234ba5dce0250a6b0ae38b8f74c1..fcc7d170b93fe5a85b61c2cc194d83ddd62753a3 100644 (file)
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -393,6 +393,7 @@ builtin_compile(PyObject *self, PyObject *args)
        char *filename;
        char *startstr;
        int start;
+       PyCompilerFlags cf;
 
        if (!PyArg_ParseTuple(args, "sss:compile", &str, &filename, &startstr))
                return NULL;
@@ -407,11 +408,10 @@ builtin_compile(PyObject *self, PyObject *args)
                   "compile() arg 3 must be 'exec' or 'eval' or 'single'");
                return NULL;
        }
-       if (PyEval_GetNestedScopes()) {
-               PyCompilerFlags cf;
-               cf.cf_nested_scopes = 1;
+       cf.cf_flags = 0;
+       if (PyEval_MergeCompilerFlags(&cf))
                return Py_CompileStringFlags(str, filename, start, &cf);
-       } else
+       else
                return Py_CompileString(str, filename, start);
 }
 
@@ -822,6 +822,7 @@ builtin_execfile(PyObject *self, PyObject *args)
        PyObject *globals = Py_None, *locals = Py_None;
        PyObject *res;
        FILE* fp;
+       PyCompilerFlags cf;
 
        if (!PyArg_ParseTuple(args, "s|O!O!:execfile",
                        &filename,
@@ -847,12 +848,11 @@ builtin_execfile(PyObject *self, PyObject *args)
                PyErr_SetFromErrno(PyExc_IOError);
                return NULL;
        }
-       if (PyEval_GetNestedScopes()) {
-               PyCompilerFlags cf;
-               cf.cf_nested_scopes = 1;
+       cf.cf_flags = 0;
+       if (PyEval_MergeCompilerFlags(&cf))
                res = PyRun_FileExFlags(fp, filename, Py_file_input, globals,
                                   locals, 1, &cf);
-       } else
+       else
                res = PyRun_FileEx(fp, filename, Py_file_input, globals,
                                   locals, 1);
        return res;

diff --git a/Python/ceval.c b/Python/ceval.c
index f42165ed385f74188ec710c6eace5f935a1c9e2b..e614e17fc16eb5c8879b7a386802a3f7fe33e414 100644 (file)
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -2524,7 +2524,7 @@ eval_code2(PyCodeObject *co, PyObject *globals, PyObject *locals,
        if (co->co_flags & CO_GENERATOR) {
                /* Don't need to keep the reference to f_back, it will be set
                 * when the generator is resumed. */
-               Py_DECREF(f->f_back);
+               Py_XDECREF(f->f_back);
                f->f_back = NULL;
 
                /* Create a new generator that owns the ready to run frame
@@ -2906,11 +2906,23 @@ PyEval_GetRestricted(void)
 }
 
 int
-PyEval_GetNestedScopes(void)
+PyEval_MergeCompilerFlags(PyCompilerFlags *cf)
 {
        PyFrameObject *current_frame = PyThreadState_Get()->frame;
-       return current_frame == NULL ? 0 : 
-           current_frame->f_code->co_flags & CO_NESTED;
+       int result = 0;
+
+       if (current_frame != NULL) {
+               const int codeflags = current_frame->f_code->co_flags;
+               if (codeflags & CO_NESTED) {
+                       result = 1;
+                       cf->cf_flags |= PyCF_NESTED_SCOPES;
+               }
+               if (codeflags & CO_GENERATOR_ALLOWED) {
+                       result = 1;
+                       cf->cf_flags |= PyCF_GENERATORS;
+               }
+       }
+       return result;
 }
 
 int
@@ -3730,26 +3742,25 @@ exec_statement(PyFrameObject *f, PyObject *prog, PyObject *globals,
        else if (PyFile_Check(prog)) {
                FILE *fp = PyFile_AsFile(prog);
                char *name = PyString_AsString(PyFile_Name(prog));
-               if (PyEval_GetNestedScopes()) {
-                       PyCompilerFlags cf;
-                       cf.cf_nested_scopes = 1;
+               PyCompilerFlags cf;
+               cf.cf_flags = 0;
+               if (PyEval_MergeCompilerFlags(&cf))
                        v = PyRun_FileFlags(fp, name, Py_file_input, globals,
                                            locals, &cf); 
-               } else {
+               else
                        v = PyRun_File(fp, name, Py_file_input, globals,
                                       locals); 
-               }
        }
        else {
                char *str;
+               PyCompilerFlags cf;
                if (PyString_AsStringAndSize(prog, &str, NULL))
                        return -1;
-               if (PyEval_GetNestedScopes()) {
-                       PyCompilerFlags cf;
-                       cf.cf_nested_scopes = 1;
+               cf.cf_flags = 0;
+               if (PyEval_MergeCompilerFlags(&cf))
                        v = PyRun_StringFlags(str, Py_file_input, globals, 
                                              locals, &cf);
-               } else
+               else
                        v = PyRun_String(str, Py_file_input, globals, locals);
        }
        if (plain)

diff --git a/Python/compile.c b/Python/compile.c
index 92322fcbd5a8396eb6c88b643362907418701124..e950f0fb088a6d8101321256b3bedd8d8aadebf5 100644 (file)
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -3953,10 +3953,15 @@ jcompile(node *n, char *filename, struct compiling *base,
                        return NULL;
                }
                if (flags) {
-                       if (flags->cf_nested_scopes)
+                       if (flags->cf_flags & PyCF_NESTED_SCOPES)
                                sc.c_future->ff_nested_scopes = 1;
                        else if (sc.c_future->ff_nested_scopes)
-                               flags->cf_nested_scopes = 1;
+                               flags->cf_flags |= PyCF_NESTED_SCOPES;
+
+                       if (flags->cf_flags & PyCF_GENERATORS)
+                               sc.c_future->ff_generators = 1;
+                       else if (sc.c_future->ff_generators)
+                               flags->cf_flags |= PyCF_GENERATORS;
                }
                if (symtable_build(&sc, n) < 0) {
                        com_free(&sc);
@@ -4426,8 +4431,12 @@ static int
 symtable_update_flags(struct compiling *c, PySymtableEntryObject *ste,
                      struct symbol_info *si)
 {
-       if (c->c_future && c->c_future->ff_nested_scopes)
-               c->c_flags |= CO_NESTED;
+       if (c->c_future) {
+               if (c->c_future->ff_nested_scopes)
+                       c->c_flags |= CO_NESTED;
+               if (c->c_future->ff_generators)
+                       c->c_flags |= CO_GENERATOR_ALLOWED;
+       }
        if (ste->ste_generator)
                c->c_flags |= CO_GENERATOR;
        if (ste->ste_type != TYPE_MODULE)

diff --git a/Python/future.c b/Python/future.c
index 70be26b548523ef506603fc4c7e76993619c7f1a..f0960ae861c0b6b8154f407c0bf2b0db6e3de0f8 100644 (file)
--- a/Python/future.c
+++ b/Python/future.c
@@ -32,7 +32,7 @@ future_check_features(PyFutureFeatures *ff, node *n, char *filename)
                if (strcmp(feature, FUTURE_NESTED_SCOPES) == 0) {
                        ff->ff_nested_scopes = 1;
                } else if (strcmp(feature, FUTURE_GENERATORS) == 0) {
-                       /* OK; this is processed by the parser */
+                       ff->ff_generators= 1;
                } else if (strcmp(feature, "braces") == 0) {
                        PyErr_SetString(PyExc_SyntaxError,
                                        "not a chance");
@@ -233,6 +233,7 @@ PyNode_Future(node *n, char *filename)
        ff->ff_found_docstring = 0;
        ff->ff_last_lineno = -1;
        ff->ff_nested_scopes = 0;
+       ff->ff_generators = 0;
 
        if (future_parse(ff, n, filename) < 0) {
                PyMem_Free((void *)ff);

diff --git a/Python/pythonrun.c b/Python/pythonrun.c
index c67f50e1e6ea5fac9dbb8cf5145c70b3607152d6..d5705b97bae4e9034df63b061fe3985f880319aa 100644 (file)
--- a/Python/pythonrun.c
+++ b/Python/pythonrun.c
@@ -494,7 +494,7 @@ PyRun_InteractiveLoopFlags(FILE *fp, char *filename, PyCompilerFlags *flags)
 
        if (flags == NULL) {
                flags = &local_flags;
-               local_flags.cf_nested_scopes = 0;
+               local_flags.cf_flags = 0;
        }
        v = PySys_GetObject("ps1");
        if (v == NULL) {
@@ -1075,10 +1075,14 @@ run_pyc_file(FILE *fp, char *filename, PyObject *globals, PyObject *locals,
        v = PyEval_EvalCode(co, globals, locals);
        if (v && flags) {
                if (co->co_flags & CO_NESTED)
-                       flags->cf_nested_scopes = 1;
+                       flags->cf_flags |= PyCF_NESTED_SCOPES;
+               if (co->co_flags & CO_GENERATOR)
+                       flags->cf_flags |= PyCF_GENERATORS;
 #if 0
                fprintf(stderr, "run_pyc_file: nested_scopes: %d\n",
-                       flags->cf_nested_scopes);
+                       flags->cf_flags & PyCF_NESTED_SCOPES);
+               fprintf(stderr, "run_pyc_file: generators: %d\n",
+                       flags->cf_flags & PyCF_GENERATORS);
 #endif
        }
        Py_DECREF(co);