Better ($LANG-independent) fix for csv translation loading on py3.

diff --git a/MANIFEST.in b/MANIFEST.in
index 3614dac5c85860ebe7e96299d5bb85f58383fdc9..409978e9e6794cb72e169aeeef4e41516b6ac222 100644 (file)
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -4,6 +4,7 @@ include tornado/ca-certificates.crt
 include tornado/test/README
 include tornado/test/test.crt
 include tornado/test/test.key
+include tornado/test/csv_translations/fr_FR.csv
 include tornado/test/static/robots.txt
 include tornado/test/templates/utf8.html
 global-exclude _auto2to3*
\ No newline at end of file

diff --git a/setup.py b/setup.py
index d02e75f92eed04c9864e87ad4fdc8efc242fe0f9..180306cc72294381d2e0f6b2392f18126e7619ad 100644 (file)
--- a/setup.py
+++ b/setup.py
@@ -45,8 +45,9 @@ distutils.core.setup(
     packages = ["tornado", "tornado.test", "tornado.platform"],
     package_data = {
         "tornado": ["ca-certificates.crt"],
+        # data files need to be listed both here and in MANIFEST.in
         "tornado.test": ["README", "test.crt", "test.key", "static/robots.txt",
-                         "templates/utf8.html"],
+                         "templates/utf8.html", "csv_translations/fr_FR.csv"],
         },
     ext_modules = extensions,
     author="Facebook",

diff --git a/tornado/locale.py b/tornado/locale.py
index 1e361d46147df364961491711113e2cea1f404ff..c883a4e91a7ae29372663aee8dabfb6ec7cb11dc 100644 (file)
--- a/tornado/locale.py
+++ b/tornado/locale.py
@@ -121,7 +121,16 @@ def load_translations(directory):
             logging.error("Unrecognized locale %r (path: %s)", locale,
                           os.path.join(directory, path))
             continue
-        f = open(os.path.join(directory, path), "r")
+        full_path = os.path.join(directory, path)
+        try:
+            # python 3: csv.reader requires a file open in text mode.
+            # Force utf8 to avoid dependence on $LANG environment variable.
+            f = open(full_path, "r", encoding="utf-8")
+        except TypeError:
+            # python 2: files return byte strings, which are decoded below.
+            # Once we drop python 2.5, this could use io.open instead
+            # on both 2 and 3.
+            f = open(full_path, "r")
         _translations[locale] = {}
         for i, row in enumerate(csv.reader(f)):
             if not row or len(row) < 2:
@@ -139,7 +148,7 @@ def load_translations(directory):
             _translations[locale].setdefault(plural, {})[english] = translation
         f.close()
     _supported_locales = frozenset(_translations.keys() + [_default_locale])
-    logging.info("Supported locales: %s", sorted(_supported_locales))
+    logging.debug("Supported locales: %s", sorted(_supported_locales))
 
 
 def load_gettext_translations(directory, domain):

diff --git a/tornado/test/csv_translations/fr_FR.csv b/tornado/test/csv_translations/fr_FR.csv
new file mode 100644 (file)
index 0000000..6321b6e
--- /dev/null
+++ b/tornado/test/csv_translations/fr_FR.csv
@@ -0,0 +1 @@
+"school","école"

diff --git a/tornado/test/locale_test.py b/tornado/test/locale_test.py
new file mode 100644 (file)
index 0000000..eabc8ff
--- /dev/null
+++ b/tornado/test/locale_test.py
@@ -0,0 +1,24 @@
+from __future__ import absolute_import, division, with_statement
+
+import os
+import tornado.locale
+import unittest
+
+class TranslationLoaderTest(unittest.TestCase):
+    # TODO: less hacky way to get isolated tests
+    SAVE_VARS = ['_translations', '_supported_locales', '_use_gettext']
+
+    def setUp(self):
+        self.saved = {}
+        for var in TranslationLoaderTest.SAVE_VARS:
+            self.saved[var] = getattr(tornado.locale, var)
+
+    def tearDown(self):
+        for k, v in self.saved.items():
+            setattr(tornado.locale, k, v)
+
+    def test_csv(self):
+        tornado.locale.load_translations(
+            os.path.join(os.path.dirname(__file__), 'csv_translations'))
+        locale = tornado.locale.get("fr_FR")
+        self.assertEqual(locale.translate("school"), u"\u00e9cole")

diff --git a/tornado/test/runtests.py b/tornado/test/runtests.py
index 66e90f2bdddf50c643ba584e8448bd07ef1a7214..7a86366152d49006c84c6825d2ef20dbde3890a0 100755 (executable)
--- a/tornado/test/runtests.py
+++ b/tornado/test/runtests.py
@@ -17,6 +17,7 @@ TEST_MODULES = [
     'tornado.test.import_test',
     'tornado.test.ioloop_test',
     'tornado.test.iostream_test',
+    'tornado.test.locale_test',
     'tornado.test.options_test',
     'tornado.test.process_test',
     'tornado.test.simple_httpclient_test',