From: Benjamin Peterson <benjamin@python.org>
Date: Thu, 18 Mar 2010 22:34:15 +0000 (+0000)
Subject: fix for files with coding cookies and BOMs
X-Git-Tag: v3.2a1~1426
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=1613ed810801df8327ae6f55b7785fec3a9dc6bb;p=thirdparty%2FPython%2Fcpython.git

fix for files with coding cookies and BOMs
---

diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 89721371b998..519dfa578492 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -333,9 +333,11 @@ def detect_encoding(readline):
             # This behaviour mimics the Python interpreter
             raise SyntaxError("unknown encoding: " + encoding)
 
-        if bom_found and codec.name != 'utf-8':
-            # This behaviour mimics the Python interpreter
-            raise SyntaxError('encoding problem: utf-8')
+        if bom_found:
+            if codec.name != 'utf-8':
+                # This behaviour mimics the Python interpreter
+                raise SyntaxError('encoding problem: utf-8')
+            encoding += '-sig'
         return encoding
 
     first = read_or_stop()