Open template files in binary mode (to be decoded as utf8 later)

Text mode in python 3 uses an environment-dependent encoding, so
add a test and run it in both C and utf-8 locales.

diff --git a/MANIFEST.in b/MANIFEST.in
index dcd3459ffac706c0ff54858d77e1c70e5e385cee..3614dac5c85860ebe7e96299d5bb85f58383fdc9 100644 (file)
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -5,4 +5,5 @@ include tornado/test/README
 include tornado/test/test.crt
 include tornado/test/test.key
 include tornado/test/static/robots.txt
+include tornado/test/templates/utf8.html
 global-exclude _auto2to3*
\ No newline at end of file

diff --git a/setup.py b/setup.py
index 872c737c4c285ff94413aca065929bcd670157ea..dca062d5c67f42eda64bf4f612c99be11199ec7c 100644 (file)
--- a/setup.py
+++ b/setup.py
@@ -45,7 +45,8 @@ distutils.core.setup(
     packages = ["tornado", "tornado.test", "tornado.platform"],
     package_data = {
         "tornado": ["ca-certificates.crt"],
-        "tornado.test": ["README", "test.crt", "test.key", "static/robots.txt"],
+        "tornado.test": ["README", "test.crt", "test.key", "static/robots.txt",
+                         "templates/utf8.html"],
         },
     ext_modules = extensions,
     author="Facebook",

diff --git a/tornado/template.py b/tornado/template.py
index a211f396029ec48d9d14fdaec1da01719490ead1..86a668facd9294b2c8d571bdaa7ff2ebe0a12abc 100644 (file)
--- a/tornado/template.py
+++ b/tornado/template.py
@@ -352,7 +352,7 @@ class Loader(BaseLoader):
 
     def _create_template(self, name):
         path = os.path.join(self.root, name)
-        f = open(path, "r")
+        f = open(path, "rb")
         template = Template(f.read(), name=name, loader=self)
         f.close()
         return template

diff --git a/tornado/test/template_test.py b/tornado/test/template_test.py
index c0175deea3f1dbbdabf108a516d4fcac6f098202..02f07be70c7e3dec3f36a82ca2b60b461919e345 100644 (file)
--- a/tornado/test/template_test.py
+++ b/tornado/test/template_test.py
@@ -1,9 +1,10 @@
 from __future__ import absolute_import, division, with_statement
 
+import os
 import traceback
 
-from tornado.escape import utf8, native_str
-from tornado.template import Template, DictLoader, ParseError
+from tornado.escape import utf8, native_str, to_unicode
+from tornado.template import Template, DictLoader, ParseError, Loader
 from tornado.testing import LogTrapTestCase
 from tornado.util import b, bytes_type, ObjectDict
 
@@ -310,3 +311,12 @@ raw: {% raw name %}""",
                          b("""s = "';sys.exit()"\n"""))
         self.assertEqual(render("foo.py", ["not a string"]),
                          b("""s = "['not a string']"\n"""))
+
+class TemplateLoaderTest(LogTrapTestCase):
+    def setUp(self):
+        self.loader = Loader(os.path.join(os.path.dirname(__file__), "templates"))
+
+    def test_utf8_in_file(self):
+        tmpl = self.loader.load("utf8.html")
+        result = tmpl.generate()
+        self.assertEqual(to_unicode(result).strip(), u"H\u00e9llo")

diff --git a/tornado/test/templates/utf8.html b/tornado/test/templates/utf8.html
new file mode 100644 (file)
index 0000000..c5253df
--- /dev/null
+++ b/tornado/test/templates/utf8.html
@@ -0,0 +1 @@
+Héllo

diff --git a/tox.ini b/tox.ini
index e4f8e456162317fc344a23c17c5a06ddfc34273c..0ab3581ca6526a80170868d6c9b892bb48fc02b9 100644 (file)
--- a/tox.ini
+++ b/tox.ini
@@ -13,7 +13,7 @@
 [tox]
 # "-full" variants include optional dependencies, to ensure
 # that things work both in a bare install and with all the extras.
-envlist = py27-full, py27-curl, py25-full, py32, pypy, py25, py26, py26-full, py27, py33
+envlist = py27-full, py27-curl, py25-full, py32, pypy, py25, py26, py26-full, py27, py32-utf8, py33
 [testenv]
 commands = python -m tornado.test.runtests {posargs:}
 
@@ -69,6 +69,17 @@ commands = python -m tornado.test.runtests --httpclient=tornado.curl_httpclient.
 # twisted under pypy takes a *very* long time.  MySQL-python builds with
 # pypy, but doesn't work.
 
+# In python 3, opening files in text mode uses a system-dependent encoding by
+# default.  Run the tests with "C" (ascii) and "utf-8" locales to ensure
+# we don't have hidden dependencies on this setting.
+[testenv:py32]
+basepython = python3.2
+setenv = LANG=C
+
+[testenv:py32-utf8]
+basepython = python3.2
+setenv = LANG=en_US.utf-8
+
 # No py32-full yet: none of our dependencies currently work on python3.
 
 [testenv:py33]