From: Ben Darnell <ben@bendarnell.com>
Date: Thu, 19 Apr 2012 00:45:09 +0000 (-0700)
Subject: Open template files in binary mode (to be decoded as utf8 later)
X-Git-Tag: v2.3.0~54
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=983fb8bae82268d07ddeaa7938c200513d5f2a67;p=thirdparty%2Ftornado.git

Open template files in binary mode (to be decoded as utf8 later)

Text mode in python 3 uses an environment-dependent encoding, so
add a test and run it in both C and utf-8 locales.
---

diff --git a/MANIFEST.in b/MANIFEST.in
index dcd3459ff..3614dac5c 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -5,4 +5,5 @@ include tornado/test/README
 include tornado/test/test.crt
 include tornado/test/test.key
 include tornado/test/static/robots.txt
+include tornado/test/templates/utf8.html
 global-exclude _auto2to3*
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 872c737c4..dca062d5c 100644
--- a/setup.py
+++ b/setup.py
@@ -45,7 +45,8 @@ distutils.core.setup(
     packages = ["tornado", "tornado.test", "tornado.platform"],
     package_data = {
         "tornado": ["ca-certificates.crt"],
-        "tornado.test": ["README", "test.crt", "test.key", "static/robots.txt"],
+        "tornado.test": ["README", "test.crt", "test.key", "static/robots.txt",
+                         "templates/utf8.html"],
         },
     ext_modules = extensions,
     author="Facebook",
diff --git a/tornado/template.py b/tornado/template.py
index a211f3960..86a668fac 100644
--- a/tornado/template.py
+++ b/tornado/template.py
@@ -352,7 +352,7 @@ class Loader(BaseLoader):
 
     def _create_template(self, name):
         path = os.path.join(self.root, name)
-        f = open(path, "r")
+        f = open(path, "rb")
         template = Template(f.read(), name=name, loader=self)
         f.close()
         return template
diff --git a/tornado/test/template_test.py b/tornado/test/template_test.py
index c0175deea..02f07be70 100644
--- a/tornado/test/template_test.py
+++ b/tornado/test/template_test.py
@@ -1,9 +1,10 @@
 from __future__ import absolute_import, division, with_statement
 
+import os
 import traceback
 
-from tornado.escape import utf8, native_str
-from tornado.template import Template, DictLoader, ParseError
+from tornado.escape import utf8, native_str, to_unicode
+from tornado.template import Template, DictLoader, ParseError, Loader
 from tornado.testing import LogTrapTestCase
 from tornado.util import b, bytes_type, ObjectDict
 
@@ -310,3 +311,12 @@ raw: {% raw name %}""",
                          b("""s = "';sys.exit()"\n"""))
         self.assertEqual(render("foo.py", ["not a string"]),
                          b("""s = "['not a string']"\n"""))
+
+class TemplateLoaderTest(LogTrapTestCase):
+    def setUp(self):
+        self.loader = Loader(os.path.join(os.path.dirname(__file__), "templates"))
+
+    def test_utf8_in_file(self):
+        tmpl = self.loader.load("utf8.html")
+        result = tmpl.generate()
+        self.assertEqual(to_unicode(result).strip(), u"H\u00e9llo")
diff --git a/tornado/test/templates/utf8.html b/tornado/test/templates/utf8.html
new file mode 100644
index 000000000..c5253dfa8
--- /dev/null
+++ b/tornado/test/templates/utf8.html
@@ -0,0 +1 @@
+Héllo
diff --git a/tox.ini b/tox.ini
index e4f8e4561..0ab3581ca 100644
--- a/tox.ini
+++ b/tox.ini
@@ -13,7 +13,7 @@
 [tox]
 # "-full" variants include optional dependencies, to ensure
 # that things work both in a bare install and with all the extras.
-envlist = py27-full, py27-curl, py25-full, py32, pypy, py25, py26, py26-full, py27, py33
+envlist = py27-full, py27-curl, py25-full, py32, pypy, py25, py26, py26-full, py27, py32-utf8, py33
 [testenv]
 commands = python -m tornado.test.runtests {posargs:}
 
@@ -69,6 +69,17 @@ commands = python -m tornado.test.runtests --httpclient=tornado.curl_httpclient.
 # twisted under pypy takes a *very* long time.  MySQL-python builds with
 # pypy, but doesn't work.
 
+# In python 3, opening files in text mode uses a system-dependent encoding by
+# default.  Run the tests with "C" (ascii) and "utf-8" locales to ensure
+# we don't have hidden dependencies on this setting.
+[testenv:py32]
+basepython = python3.2
+setenv = LANG=C
+
+[testenv:py32-utf8]
+basepython = python3.2
+setenv = LANG=en_US.utf-8
+
 # No py32-full yet: none of our dependencies currently work on python3.
 
 [testenv:py33]