Add a policy for the ascii literal behavior. Fixes #392

diff --git a/CHANGES b/CHANGES
index 65bf9691662c5e2902292326cc551e075592824a..da6774f6e1afa5677e45518e994fc7212606a668 100644 (file)
--- a/CHANGES
+++ b/CHANGES
@@ -28,6 +28,8 @@ Version 2.9
 - Ported a modified version of the `tojson` filter from Flask to Jinja2
   and hooked it up with the new policy framework.
 - Block sets are now marked `safe` by default.
+- On Python 2 the asciification of ASCII strings can now be disabled with
+  the `compiler.ascii_str` policy.
 
 Version 2.8.2
 -------------

diff --git a/docs/api.rst b/docs/api.rst
index 8bf0fdfe0986e1fe73814c2aa6e9540bc3f65ec9..3b5bccde42943a49b8dbcd273367a9b75a29919c 100644 (file)
--- a/docs/api.rst
+++ b/docs/api.rst
@@ -556,6 +556,16 @@ Example::
 
     env.policies['urlize.rel'] = 'nofollow noopener'
 
+``compiler.ascii_str``:
+    This boolean controls on Python 2 if Jinja2 should store ASCII only
+    literals as bytestring instead of unicode strings.  This used to be
+    always enabled for Jinja versions below 2.9 and now can be changed.
+    Traditionally it was done this way since some APIs in Python 2 failed
+    badly for unicode strings (for instance the datetime strftime API).
+    Now however sometimes the inverse is true (for instance str.format).
+    If this is set to False then all strings are stored as unicode
+    internally.
+
 ``urlize.rel``:
     A string that defines the items for the `rel` attribute of generated
     links with the `urlize` filter.  These items are always added.  The

diff --git a/jinja2/compiler.py b/jinja2/compiler.py
index 65bffe6c0e6e4deec3c74ea5dbf67e987223fbe0..6595d63214c37364f25d9cc51fa95e0645762da2 100644 (file)
--- a/jinja2/compiler.py
+++ b/jinja2/compiler.py
@@ -1340,7 +1340,7 @@ class CodeGenerator(NodeVisitor):
         self.write(ref)
 
     def visit_Const(self, node, frame):
-        val = node.value
+        val = node.as_const(frame.eval_ctx)
         if isinstance(val, float):
             self.write(str(val))
         else:

diff --git a/jinja2/defaults.py b/jinja2/defaults.py
index 90ccb65f851429b18ba9a45a653117631b0d3be4..38c53764ae41978b81d3e7258df89b921d6152bd 100644 (file)
--- a/jinja2/defaults.py
+++ b/jinja2/defaults.py
@@ -41,6 +41,7 @@ DEFAULT_NAMESPACE = {
 
 # default policies
 DEFAULT_POLICIES = {
+    'compiler.ascii_str':   True,
     'urlize.rel':           'noopener',
     'urlize.target':        None,
     'json.dumps_function':  None,

diff --git a/jinja2/lexer.py b/jinja2/lexer.py
index c8dac214eddd80556621f63981373e91e875953c..d2ca32fcbb4c0fbb6643908640ee7bb03b96682f 100644 (file)
--- a/jinja2/lexer.py
+++ b/jinja2/lexer.py
@@ -574,15 +574,6 @@ class Lexer(object):
                 except Exception as e:
                     msg = str(e).split(':')[-1].strip()
                     raise TemplateSyntaxError(msg, lineno, name, filename)
-                # if we can express it as bytestring (ascii only)
-                # we do that for support of semi broken APIs
-                # as datetime.datetime.strftime.  On python 3 this
-                # call becomes a noop thanks to 2to3
-                if PY2:
-                    try:
-                        value = value.encode('ascii')
-                    except UnicodeError:
-                        pass
             elif token == 'integer':
                 value = int(value)
             elif token == 'float':

diff --git a/jinja2/nodes.py b/jinja2/nodes.py
index 4d62cccb865c8ee8792c759abb3f7128a56e1836..5e0726a365df8b4b4c8a71bc255cdb6206e7d0aa 100644 (file)
--- a/jinja2/nodes.py
+++ b/jinja2/nodes.py
@@ -17,7 +17,7 @@ import operator
 
 from collections import deque
 from jinja2.utils import Markup
-from jinja2._compat import izip, with_metaclass, text_type
+from jinja2._compat import izip, with_metaclass, text_type, PY2
 
 
 #: the types we support for context functions
@@ -470,7 +470,14 @@ class Const(Literal):
     fields = ('value',)
 
     def as_const(self, eval_ctx=None):
-        return self.value
+        rv = self.value
+        if PY2 and type(rv) is text_type and \
+           self.environment.policies['compiler.ascii_str']:
+            try:
+                rv = rv.encode('ascii')
+            except UnicodeError:
+                pass
+        return rv
 
     @classmethod
     def from_untrusted(cls, value, lineno=None, environment=None):

diff --git a/tests/test_features.py b/tests/test_features.py
index 25d58e4c8834b6ccb61f2732db9084ca749c87dc..3187890e853781f61aa6497741f01cf59477764d 100644 (file)
--- a/tests/test_features.py
+++ b/tests/test_features.py
@@ -1,7 +1,7 @@
 import sys
 import pytest
 
-from jinja2 import Template
+from jinja2 import Template, Environment, contextfilter
 
 
 @pytest.mark.skipif(sys.version_info < (3, 5),
@@ -14,3 +14,27 @@ def test_generator_stop():
     t = Template('a{{ bad.bar() }}b')
     with pytest.raises(RuntimeError):
         t.render(bad=X())
+
+
+@pytest.mark.skipif(sys.version_info[0] > 2,
+                    reason='Feature only supported on 2.x')
+def test_ascii_str():
+    @contextfilter
+    def assert_func(context, value):
+        assert type(value) is context['expected_type']
+
+    env = Environment()
+    env.filters['assert'] = assert_func
+
+    env.policies['compiler.ascii_str'] = False
+    t = env.from_string('{{ "foo"|assert }}')
+    t.render(expected_type=unicode)
+
+    env.policies['compiler.ascii_str'] = True
+    t = env.from_string('{{ "foo"|assert }}')
+    t.render(expected_type=str)
+
+    for val in True, False:
+        env.policies['compiler.ascii_str'] = val
+        t = env.from_string(u'{{ "\N{SNOWMAN}"|assert }}')
+        t.render(expected_type=unicode)