Problem found while converting from PyBytes to PyString:

Re-enable (and correct) a test for the BOM at the beginning of a code unit.
And properly "unget" characters when the BOM is incomplete.

diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py
index 4f843289acbc19e54ccad57d22e285af9da3ff41..d54375118de10f3a148ad7ff0700c9bd2507ee23 100644 (file)
--- a/Lib/test/test_builtin.py
+++ b/Lib/test/test_builtin.py
@@ -200,8 +200,8 @@ class BuiltinTest(unittest.TestCase):
 
     def test_compile(self):
         compile('print(1)\n', '', 'exec')
-##         bom = b'\xef\xbb\xbf'
-##         compile(bom + b'print(1)\n', '', 'exec')
+        bom = b'\xef\xbb\xbf'
+        compile(bom + b'print(1)\n', '', 'exec')
         compile(source='pass', filename='?', mode='exec')
         compile(dont_inherit=0, filename='tmp', source='0', mode='eval')
         compile('pass', '?', dont_inherit=1, mode='exec')
@@ -327,11 +327,12 @@ class BuiltinTest(unittest.TestCase):
         self.assertEqual(eval('c', globals, locals), 300)
         globals = {'a': 1, 'b': 2}
         locals = {'b': 200, 'c': 300}
-##         bom = b'\xef\xbb\xbf'
-##         self.assertEqual(eval(bom + b'a', globals, locals), 1)
+        bom = b'\xef\xbb\xbf'
+        self.assertEqual(eval(bom + b'a', globals, locals), 1)
         self.assertEqual(eval('"\xe5"', globals), "\xe5")
         self.assertRaises(TypeError, eval)
         self.assertRaises(TypeError, eval, ())
+        self.assertRaises(SyntaxError, eval, bom[:2] + b'a')
 
     def test_general_eval(self):
         # Tests that general mappings can be used for the locals argument

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 1c2b8e8e107e4c8f1bfb5cff68d0f19c095191a2..2c7da7c257143aa8942cac54a0d136b37b50443c 100644 (file)
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -323,8 +323,21 @@ check_bom(int get_char(struct tok_state *),
        if (ch == EOF) {
                return 1;
        } else if (ch == 0xEF) {
-               ch = get_char(tok); if (ch != 0xBB) goto NON_BOM;
-               ch = get_char(tok); if (ch != 0xBF) goto NON_BOM;
+               ch = get_char(tok); 
+               if (ch != 0xBB) {
+                       unget_char(ch, tok);
+                       unget_char(0xEF, tok);
+                       /* any token beginning with '\xEF' is a bad token */
+                       return 1;
+               }
+               ch = get_char(tok); 
+               if (ch != 0xBF) {
+                       unget_char(ch, tok);
+                       unget_char(0xBB, tok);
+                       unget_char(0xEF, tok);
+                       /* any token beginning with '\xEF' is a bad token */
+                       return 1;
+               }
 #if 0
        /* Disable support for UTF-16 BOMs until a decision
           is made whether this needs to be supported.  */
@@ -344,10 +357,7 @@ check_bom(int get_char(struct tok_state *),
        if (tok->encoding != NULL)
                PyMem_FREE(tok->encoding);
        tok->encoding = new_string("utf-8", 5); /* resulting is in utf-8 */
-       return 1;
-  NON_BOM:
-       /* any token beginning with '\xEF', '\xFE', '\xFF' is a bad token */
-       unget_char(0xFF, tok);  /* XXX this will cause a syntax error */
+       /* No need to set_readline: input is already utf-8 */
        return 1;
 }
 
@@ -641,7 +651,7 @@ decode_str(const char *str, struct tok_state *tok)
                utf8 = translate_into_utf8(str, tok->enc);
                if (utf8 == NULL)
                        return error_ret(tok);
-               str = PyBytes_AsString(utf8);
+               str = PyString_AsString(utf8);
        }
        for (s = str;; s++) {
                if (*s == '\0') break;