Fix UnicodeEncodeError on file encoding detection

If the first line of a python file is not a valid latin-1 string,
parse_encoding dies with "UnicodeDecodeError". These strings nonetheless can be
valid in some scenarios (for example, Mako extractor uses
babel.messages.extract.extract_python), and it makes more sense to ignore this
exception and return None.

diff --git a/babel/util.py b/babel/util.py
index 1849e8a04be490c617e03acb06fcd4df8685a6b8..54f7d2db0d6c1370de054cb5d12d1cccc4b0d28e 100644 (file)
--- a/babel/util.py
+++ b/babel/util.py
@@ -65,7 +65,7 @@ def parse_encoding(fp):
             try:
                 import parser
                 parser.suite(line1.decode('latin-1'))
-            except (ImportError, SyntaxError):
+            except (ImportError, SyntaxError, UnicodeEncodeError):
                 # Either it's a real syntax error, in which case the source is
                 # not valid python source, or line2 is a continuation of line1,
                 # in which case we don't want to scan line2 for a magic

diff --git a/tests/test_util.py b/tests/test_util.py
index bb2bfdb4ba742d60fdb99a1dd35eeeab8fcd3ce5..d4b4be523a2f0e3343e3f66d12de92de20e8fa3a 100644 (file)
--- a/tests/test_util.py
+++ b/tests/test_util.py
@@ -14,6 +14,7 @@
 import unittest
 
 from babel import util
+from babel._compat import BytesIO
 
 
 def test_distinct():
@@ -52,3 +53,17 @@ class FixedOffsetTimezoneTestCase(unittest.TestCase):
     def test_zone_positive_offset(self):
         self.assertEqual('Etc/GMT+330', util.FixedOffsetTimezone(330).zone)
 
+
+parse_encoding = lambda s: util.parse_encoding(BytesIO(s.encode('utf-8')))
+
+
+def test_parse_encoding_defined():
+    assert parse_encoding(u'# coding: utf-8') == 'utf-8'
+
+
+def test_parse_encoding_undefined():
+    assert parse_encoding(u'') is None
+
+
+def test_parse_encoding_non_ascii():
+    assert parse_encoding(u'K\xf6ln') is None