Convert code from sys.stdin.encoding to UTF-8 in

interactive mode. Fixes #1100.

diff --git a/Include/parsetok.h b/Include/parsetok.h
index 2b4ce1ea4ba62ab49f0e16790bbe3b8706d1c56a..71033dc70478f29e570d8670b2a03b64745150e8 100644 (file)
--- a/Include/parsetok.h
+++ b/Include/parsetok.h
@@ -34,7 +34,8 @@ PyAPI_FUNC(node *) PyParser_ParseFile (FILE *, const char *, grammar *, int,
 
 PyAPI_FUNC(node *) PyParser_ParseStringFlags(const char *, grammar *, int,
                                               perrdetail *, int);
-PyAPI_FUNC(node *) PyParser_ParseFileFlags(FILE *, const char *, grammar *,
+PyAPI_FUNC(node *) PyParser_ParseFileFlags(FILE *, const char *, 
+                                          const char*, grammar *,
                                                 int, char *, char *,
                                                 perrdetail *, int);
 

diff --git a/Include/pythonrun.h b/Include/pythonrun.h
index 08278cf8fb55380d28101fbf9cf16d6886b12435..607826ad199fa72b11a331c5568a3fa548f998ab 100644 (file)
--- a/Include/pythonrun.h
+++ b/Include/pythonrun.h
@@ -40,7 +40,8 @@ PyAPI_FUNC(int) PyRun_InteractiveLoopFlags(FILE *, const char *, PyCompilerFlags
 PyAPI_FUNC(struct _mod *) PyParser_ASTFromString(const char *, const char *, 
                                                 int, PyCompilerFlags *flags,
                                                  PyArena *);
-PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile(FILE *, const char *, int, 
+PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile(FILE *, const char *, 
+                                              const char*, int, 
                                               char *, char *,
                                                PyCompilerFlags *, int *,
                                                PyArena *);

diff --git a/Parser/parsetok.c b/Parser/parsetok.c
index 71bed2991976b50505e9e66b3109e35638dad7a8..b9664ea3ea221004e1016ec2db527ede91249cda 100644 (file)
--- a/Parser/parsetok.c
+++ b/Parser/parsetok.c
@@ -59,19 +59,20 @@ node *
 PyParser_ParseFile(FILE *fp, const char *filename, grammar *g, int start,
                   char *ps1, char *ps2, perrdetail *err_ret)
 {
-       return PyParser_ParseFileFlags(fp, filename, g, start, ps1, ps2,
-                                      err_ret, 0);
+       return PyParser_ParseFileFlags(fp, filename, NULL, 
+                                      g, start, ps1, ps2, err_ret, 0);
 }
 
 node *
-PyParser_ParseFileFlags(FILE *fp, const char *filename, grammar *g, int start,
+PyParser_ParseFileFlags(FILE *fp, const char *filename, const char* enc,
+                       grammar *g, int start,
                        char *ps1, char *ps2, perrdetail *err_ret, int flags)
 {
        struct tok_state *tok;
 
        initerr(err_ret, filename);
 
-       if ((tok = PyTokenizer_FromFile(fp, ps1, ps2)) == NULL) {
+       if ((tok = PyTokenizer_FromFile(fp, enc, ps1, ps2)) == NULL) {
                err_ret->error = E_NOMEM;
                return NULL;
        }

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 776183d80582d7d7d92e548d44f4cc8e35a47e9d..7f51e143fa504bbb45a2f15dfaf5adaf063551fc 100644 (file)
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -677,7 +677,7 @@ PyTokenizer_FromString(const char *str)
 /* Set up tokenizer for file */
 
 struct tok_state *
-PyTokenizer_FromFile(FILE *fp, char *ps1, char *ps2)
+PyTokenizer_FromFile(FILE *fp, char* enc, char *ps1, char *ps2)
 {
        struct tok_state *tok = tok_new();
        if (tok == NULL)
@@ -691,6 +691,17 @@ PyTokenizer_FromFile(FILE *fp, char *ps1, char *ps2)
        tok->fp = fp;
        tok->prompt = ps1;
        tok->nextprompt = ps2;
+       if (enc != NULL) {
+               /* Must copy encoding declaration since it
+                  gets copied into the parse tree. */
+               tok->encoding = PyMem_MALLOC(strlen(enc)+1);
+               if (!tok->encoding) {
+                       PyTokenizer_Free(tok);
+                       return NULL;
+               }
+               strcpy(tok->encoding, enc);
+               tok->decoding_state = -1;
+       }
        return tok;
 }
 
@@ -742,6 +753,29 @@ tok_nextc(register struct tok_state *tok)
                }
                if (tok->prompt != NULL) {
                        char *newtok = PyOS_Readline(stdin, stdout, tok->prompt);
+#ifndef PGEN
+                       if (tok->encoding && newtok && *newtok) {
+                               /* Recode to UTF-8 */
+                               Py_ssize_t buflen;
+                               const char* buf;
+                               PyObject *u = translate_into_utf8(newtok, tok->encoding);
+                               PyMem_FREE(newtok);
+                               if (!u) {
+                                       tok->done = E_DECODE;
+                                       return EOF;
+                               }
+                               buflen = PyBytes_Size(u);
+                               buf = PyBytes_AsString(u);
+                               if (!buf) {
+                                       Py_DECREF(u);
+                                       tok->done = E_DECODE;
+                                       return EOF;
+                               }
+                               newtok = PyMem_MALLOC(buflen+1);
+                               strcpy(newtok, buf);
+                               Py_DECREF(u);
+                       }
+#endif
                        if (tok->nextprompt != NULL)
                                tok->prompt = tok->nextprompt;
                        if (newtok == NULL)

diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
index 5e7ebf74f11551890713df5e903dd76b6b1d5ccb..ba90a5f63c23dcc02be777245a7571731507d6de 100644 (file)
--- a/Parser/tokenizer.h
+++ b/Parser/tokenizer.h
@@ -55,7 +55,8 @@ struct tok_state {
 };
 
 extern struct tok_state *PyTokenizer_FromString(const char *);
-extern struct tok_state *PyTokenizer_FromFile(FILE *, char *, char *);
+extern struct tok_state *PyTokenizer_FromFile(FILE *, char*,
+                                             char *, char *);
 extern void PyTokenizer_Free(struct tok_state *);
 extern int PyTokenizer_Get(struct tok_state *, char **, char **);
 

diff --git a/Python/import.c b/Python/import.c
index 72837836bc035ff64a351f4ebe09f60bcb7cfb9d..c2f42e995b885913f5ce23bc010af5600661d8f9 100644 (file)
--- a/Python/import.c
+++ b/Python/import.c
@@ -809,7 +809,8 @@ parse_source_module(const char *pathname, FILE *fp)
        if (arena == NULL)
                return NULL;
 
-       mod = PyParser_ASTFromFile(fp, pathname, Py_file_input, 0, 0, 0, 
+       mod = PyParser_ASTFromFile(fp, pathname, NULL,
+                                  Py_file_input, 0, 0, 0, 
                                   NULL, arena);
        if (mod) {
                co = PyAST_Compile(mod, pathname, NULL, arena);

diff --git a/Python/pythonrun.c b/Python/pythonrun.c
index eeed820faaa028ce2bed15e8e02d721a82f51662..d89f5f95812bcd400fe73dc479d7c098e5026d56 100644 (file)
--- a/Python/pythonrun.c
+++ b/Python/pythonrun.c
@@ -744,12 +744,22 @@ PyRun_InteractiveLoopFlags(FILE *fp, const char *filename, PyCompilerFlags *flag
 int
 PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags)
 {
-       PyObject *m, *d, *v, *w;
+       PyObject *m, *d, *v, *w, *oenc = NULL;
        mod_ty mod;
        PyArena *arena;
-       char *ps1 = "", *ps2 = "";
+       char *ps1 = "", *ps2 = "", *enc = NULL;
        int errcode = 0;
 
+       if (fp == stdin) {
+               /* Fetch encoding from sys.stdin */
+               v = PySys_GetObject("stdin");
+               if (!v)
+                       return -1;
+               oenc = PyObject_GetAttrString(v, "encoding");
+               if (!oenc)
+                       return -1;
+               enc = PyUnicode_AsString(oenc);
+       }
        v = PySys_GetObject("ps1");
        if (v != NULL) {
                v = PyObject_Str(v);
@@ -770,13 +780,15 @@ PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags
        if (arena == NULL) {
                Py_XDECREF(v);
                Py_XDECREF(w);
+               Py_XDECREF(oenc);
                return -1;
        }
-       mod = PyParser_ASTFromFile(fp, filename,
+       mod = PyParser_ASTFromFile(fp, filename, enc,
                                   Py_single_input, ps1, ps2,
                                   flags, &errcode, arena);
        Py_XDECREF(v);
        Py_XDECREF(w);
+       Py_XDECREF(oenc);
        if (mod == NULL) {
                PyArena_Free(arena);
                if (errcode == E_EOF) {
@@ -1254,7 +1266,7 @@ PyRun_FileExFlags(FILE *fp, const char *filename, int start, PyObject *globals,
        if (arena == NULL)
                return NULL;
        
-       mod = PyParser_ASTFromFile(fp, filename, start, 0, 0,
+       mod = PyParser_ASTFromFile(fp, filename, NULL, start, 0, 0,
                                   flags, NULL, arena);
        if (closeit)
                fclose(fp);
@@ -1379,13 +1391,15 @@ PyParser_ASTFromString(const char *s, const char *filename, int start,
 }
 
 mod_ty
-PyParser_ASTFromFile(FILE *fp, const char *filename, int start, char *ps1,
+PyParser_ASTFromFile(FILE *fp, const char *filename, const char* enc,
+                    int start, char *ps1,
                     char *ps2, PyCompilerFlags *flags, int *errcode,
                     PyArena *arena)
 {
        mod_ty mod;
        perrdetail err;
-       node *n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar,
+       node *n = PyParser_ParseFileFlags(fp, filename, enc,
+                                         &_PyParser_Grammar,
                                start, ps1, ps2, &err, PARSER_FLAGS(flags));
        if (n) {
                mod = PyAST_FromNode(n, flags, filename, arena);
@@ -1406,7 +1420,8 @@ node *
 PyParser_SimpleParseFileFlags(FILE *fp, const char *filename, int start, int flags)
 {
        perrdetail err;
-       node *n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar,
+       node *n = PyParser_ParseFileFlags(fp, filename, NULL,
+                                         &_PyParser_Grammar,
                                          start, NULL, NULL, &err, flags);
        if (n == NULL)
                err_input(&err);