From 6c733602ac1b2d68d9b39acb7b1d96faae6a7717 Mon Sep 17 00:00:00 2001 From: David Lord Date: Sun, 4 Apr 2021 17:20:23 -0700 Subject: [PATCH] add 'is filter' and 'is test' tests This required allowing tests to be decorated with '@environmentfilter'. Tests are essentially the same as filters now, the node, compiler, and environment have been refactored to extract common behavior. --- CHANGES.rst | 5 ++ docs/api.rst | 167 ++++++++++++++++++++++---------------- src/jinja2/compiler.py | 95 +++++++++++++++------- src/jinja2/environment.py | 95 ++++++++++++++-------- src/jinja2/nodes.py | 75 +++++++++-------- src/jinja2/tests.py | 43 ++++++++++ tests/test_filters.py | 12 +-- tests/test_tests.py | 25 ++++++ 8 files changed, 342 insertions(+), 175 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 4516d422..aa8f4a8b 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -41,6 +41,11 @@ Unreleased already loaded. :issue:`295` - Do not raise an error for undefined filters in unexecuted if-statements and conditional expressions. :issue:`842` +- Add ``is filter`` and ``is test`` tests to test if a name is a + registered filter or test. This allows checking if a filter is + available in a template before using it. Test functions can be + decorated with ``@environmentfunction``, ``@evalcontextfunction``, + or ``@contextfunction``. :issue:`842`, :pr:`1248` Version 2.11.3 diff --git a/docs/api.rst b/docs/api.rst index 9ae47ef2..a0123937 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -666,56 +666,119 @@ Exceptions Custom Filters -------------- -Custom filters are just regular Python functions that take the left side of -the filter as first argument and the arguments passed to the filter as -extra arguments or keyword arguments. +Filters are Python functions that take the value to the left of the +filter as the first argument and produce a new value. Arguments passed +to the filter are passed after the value. -For example in the filter ``{{ 42|myfilter(23) }}`` the function would be -called with ``myfilter(42, 23)``. Here for example a simple filter that can -be applied to datetime objects to format them:: +For example, the filter ``{{ 42|myfilter(23) }}`` is called behind the +scenes as ``myfilter(42, 23)``. - def datetimeformat(value, format='%H:%M / %d-%m-%Y'): - return value.strftime(format) +Jinja comes with some :ref:`built-in filters `. To use +a custom filter, write a function that takes at least a ``value`` +argument, then register it in :attr:`Environment.filters`. + +Here's a filter that formats datetime objects: -You can register it on the template environment by updating the -:attr:`~Environment.filters` dict on the environment:: +.. code-block:: python + + def datetime_format(value, format="%H:%M %d-%m-%y"): + return value.strftime(format) - environment.filters['datetimeformat'] = datetimeformat + environment.filters["datetime_format"] = datetime_format -Inside the template it can then be used as follows: +Now it can be used in templates: .. sourcecode:: jinja - written on: {{ article.pub_date|datetimeformat }} - publication date: {{ article.pub_date|datetimeformat('%d-%m-%Y') }} + {{ article.pub_date|datetimeformat }} + {{ article.pub_date|datetimeformat("%B %Y") }} -Filters can also be passed the current template context or environment. This -is useful if a filter wants to return an undefined value or check the current -:attr:`~Environment.autoescape` setting. For this purpose three decorators -exist: :func:`environmentfilter`, :func:`contextfilter` and -:func:`evalcontextfilter`. +Some decorators are available to tell Jinja to pass extra information to +the filter. The object is passed as the first argument, making the value +being filtered the second argument. -Here a small example filter that breaks a text into HTML line breaks and -paragraphs and marks the return value as safe HTML string if autoescaping is -enabled:: +- :func:`environmentfilter` passes the :class:`Environment`. +- :func:`evalcontextfilter` passes the :ref:`eval-context`. +- :func:`contextfilter` passes the current + :class:`~jinja2.runtime.Context`. - import re - from jinja2 import evalcontextfilter, Markup, escape +Here's a filter that converts line breaks into HTML ``
`` and ``

`` +tags. It uses the eval context to check if autoescape is currently +enabled before escaping the input and marking the output safe. - _paragraph_re = re.compile(r"(?:\r\n|\r(?!\n)|\n){2,}") +.. code-block:: python + + import re + from jinja2 import evalcontextfilter + from markupsafe import Markup, escape @evalcontextfilter def nl2br(eval_ctx, value): + br = "
\n" + + if eval_ctx.autoescape: + value = escape(value) + br = Markup(br) + result = "\n\n".join( - f"

{p.replace('\n', Markup('
\n'))}

" - for p in _paragraph_re.split(escape(value)) + f"

{br.join(p.splitlines())}<\p>" + for p in re.split(r"(?:\r\n|\r(?!\n)|\n){2,}", value) ) - if eval_ctx.autoescape: - result = Markup(result) - return result + return Markup(result) if autoescape else result + + +.. _writing-tests: + +Custom Tests +------------ + +Test are Python functions that take the value to the left of the test as +the first argument, and return ``True`` or ``False``. Arguments passed +to the test are passed after the value. + +For example, the test ``{{ 42 is even }}`` is called behind the scenes +as ``is_even(42)``. + +Jinja comes with some :ref:`built-in tests `. To use a +custom tests, write a function that takes at least a ``value`` argument, +then register it in :attr:`Environment.tests`. + +Here's a test that checks if a value is a prime number: + +.. code-block:: python + + import math + + def is_prime(n): + if n == 2: + return True + + for i in range(2, int(math.ceil(math.sqrt(n))) + 1): + if n % i == 0: + return False -Context filters work the same just that the first argument is the current -active :class:`Context` rather than the environment. + return True + + environment.tests["prime"] = is_prime + +Now it can be used in templates: + +.. sourcecode:: jinja + + {% if value is prime %} + {{ value }} is a prime number + {% else %} + {{ value }} is not a prime number + {% endif %} + +Some decorators are available to tell Jinja to pass extra information to +the filter. The object is passed as the first argument, making the value +being filtered the second argument. + +- :func:`environmentfunction` passes the :class:`Environment`. +- :func:`evalcontextfunction` passes the :ref:`eval-context`. +- :func:`contextfunction` passes the current + :class:`~jinja2.runtime.Context`. .. _eval-context: @@ -780,46 +843,6 @@ eval context object itself. time. At runtime this should always be `False`. -.. _writing-tests: - -Custom Tests ------------- - -Tests work like filters just that there is no way for a test to get access -to the environment or context and that they can't be chained. The return -value of a test should be `True` or `False`. The purpose of a test is to -give the template designers the possibility to perform type and conformability -checks. - -Here a simple test that checks if a variable is a prime number:: - - import math - - def is_prime(n): - if n == 2: - return True - for i in range(2, int(math.ceil(math.sqrt(n))) + 1): - if n % i == 0: - return False - return True - - -You can register it on the template environment by updating the -:attr:`~Environment.tests` dict on the environment:: - - environment.tests['prime'] = is_prime - -A template designer can then use the test like this: - -.. sourcecode:: jinja - - {% if 42 is prime %} - 42 is a prime number - {% else %} - 42 is not a prime number - {% endif %} - - .. _global-namespace: The Global Namespace diff --git a/src/jinja2/compiler.py b/src/jinja2/compiler.py index e3bde168..46b05e89 100644 --- a/src/jinja2/compiler.py +++ b/src/jinja2/compiler.py @@ -1,6 +1,7 @@ """Compiles nodes from the parser into Python code.""" import typing as t from collections import namedtuple +from contextlib import contextmanager from functools import update_wrapper from io import StringIO from itertools import chain @@ -440,15 +441,28 @@ class CodeGenerator(NodeVisitor): self.visit(node.dyn_kwargs, frame) def pull_dependencies(self, nodes): - """Pull all the dependencies.""" + """Find all filter and test names used in the template and + assign them to variables in the compiled namespace. Checking + that the names are registered with the environment is done when + compiling the Filter and Test nodes. If the node is in an If or + CondExpr node, the check is done at runtime instead. + + .. versionchanged:: 3.0 + Filters and tests in If and CondExpr nodes are checked at + runtime instead of compile time. + """ visitor = DependencyFinderVisitor() + for node in nodes: visitor.visit(node) + for dependency in "filters", "tests": mapping = getattr(self, dependency) + for name in getattr(visitor, dependency): if name not in mapping: mapping[name] = self.temporary_identifier() + # add check during runtime that dependencies used inside of executed # blocks are defined, as this step may be skipped during compile time self.writeline("try:") @@ -461,7 +475,8 @@ class CodeGenerator(NodeVisitor): self.writeline(f"def {mapping[name]}(*unused):") self.indent() self.writeline( - f'raise TemplateRuntimeError("no filter named {name!r} found")' + f'raise TemplateRuntimeError("No {dependency[:-1]}' + f' named {name!r} found.")' ) self.outdent() self.outdent() @@ -1657,47 +1672,69 @@ class CodeGenerator(NodeVisitor): self.write(":") self.visit(node.step, frame) - @optimizeconst - def visit_Filter(self, node, frame): + @contextmanager + def _filter_test_common(self, node, frame, is_filter): + if is_filter: + compiler_map = self.filters + env_map = self.environment.filters + type_name = mark_name = "filter" + else: + compiler_map = self.tests + env_map = self.environment.tests + type_name = "test" + # Filters use "contextfilter", tests and calls use "contextfunction". + mark_name = "function" + if self.environment.is_async: self.write("await auto_await(") - self.write(self.filters[node.name] + "(") - func = self.environment.filters.get(node.name) + + self.write(compiler_map[node.name] + "(") + func = env_map.get(node.name) + + # When inside an If or CondExpr frame, allow the filter to be + # undefined at compile time and only raise an error if it's + # actually called at runtime. See pull_dependencies. if func is None and not frame.soft_frame: - self.fail(f"no filter named {node.name!r}", node.lineno) - if getattr(func, "contextfilter", False) is True: + self.fail(f"No {type_name} named {node.name!r}.", node.lineno) + + if getattr(func, f"context{mark_name}", False) is True: self.write("context, ") - elif getattr(func, "evalcontextfilter", False) is True: + elif getattr(func, f"evalcontext{mark_name}", False) is True: self.write("context.eval_ctx, ") - elif getattr(func, "environmentfilter", False) is True: + elif getattr(func, f"environment{mark_name}", False) is True: self.write("environment, ") - # if the filter node is None we are inside a filter block - # and want to write to the current buffer - if node.node is not None: - self.visit(node.node, frame) - elif frame.eval_ctx.volatile: - self.write( - f"(Markup(concat({frame.buffer}))" - f" if context.eval_ctx.autoescape else concat({frame.buffer}))" - ) - elif frame.eval_ctx.autoescape: - self.write(f"Markup(concat({frame.buffer}))") - else: - self.write(f"concat({frame.buffer})") + # Back to the visitor function to handle visiting the target of + # the filter or test. + yield + self.signature(node, frame) self.write(")") + if self.environment.is_async: self.write(")") + @optimizeconst + def visit_Filter(self, node, frame): + with self._filter_test_common(node, frame, True): + # if the filter node is None we are inside a filter block + # and want to write to the current buffer + if node.node is not None: + self.visit(node.node, frame) + elif frame.eval_ctx.volatile: + self.write( + f"(Markup(concat({frame.buffer}))" + f" if context.eval_ctx.autoescape else concat({frame.buffer}))" + ) + elif frame.eval_ctx.autoescape: + self.write(f"Markup(concat({frame.buffer}))") + else: + self.write(f"concat({frame.buffer})") + @optimizeconst def visit_Test(self, node, frame): - self.write(self.tests[node.name] + "(") - if node.name not in self.environment.tests: - self.fail(f"no test named {node.name!r}", node.lineno) - self.visit(node.node, frame) - self.signature(node, frame) - self.write(")") + with self._filter_test_common(node, frame, False): + self.visit(node.node, frame) @optimizeconst def visit_CondExpr(self, node, frame): diff --git a/src/jinja2/environment.py b/src/jinja2/environment.py index 2bbdcb45..ad7190f5 100644 --- a/src/jinja2/environment.py +++ b/src/jinja2/environment.py @@ -102,17 +102,6 @@ def load_extensions(environment, extensions): return result -def fail_for_missing_callable(thing, name): - msg = f"no {thing} named {name!r}" - - if isinstance(name, Undefined): - try: - name._fail_with_undefined_error() - except Exception as e: - msg = f"{msg} ({e}; did you forget to quote the callable name?)" - raise TemplateRuntimeError(msg) - - def _environment_sanity_check(environment): """Perform a sanity check on the environment.""" assert issubclass( @@ -470,47 +459,87 @@ class Environment: except (TypeError, LookupError, AttributeError): return self.undefined(obj=obj, name=attribute) - def call_filter( - self, name, value, args=None, kwargs=None, context=None, eval_ctx=None + def _filter_test_common( + self, name, value, args, kwargs, context, eval_ctx, is_filter ): - """Invokes a filter on a value the same way the compiler does. + if is_filter: + env_map = self.filters + type_name = mark_name = "filter" + else: + env_map = self.tests + type_name = "test" + # Filters use "contextfilter", tests and calls use "contextfunction". + mark_name = "function" - This might return a coroutine if the filter is running from an - environment in async mode and the filter supports async - execution. It's your responsibility to await this if needed. + func = env_map.get(name) - .. versionadded:: 2.7 - """ - func = self.filters.get(name) if func is None: - fail_for_missing_callable("filter", name) - args = [value] + list(args or ()) - if getattr(func, "contextfilter", False) is True: + msg = f"No {type_name} named {name!r}." + + if isinstance(name, Undefined): + try: + name._fail_with_undefined_error() + except Exception as e: + msg = f"{msg} ({e}; did you forget to quote the callable name?)" + + raise TemplateRuntimeError(msg) + + args = [value, *(args if args is not None else ())] + kwargs = kwargs if kwargs is not None else {} + + if getattr(func, f"context{mark_name}", False) is True: if context is None: raise TemplateRuntimeError( - "Attempted to invoke context filter without context" + f"Attempted to invoke a context {type_name} without context." ) + args.insert(0, context) - elif getattr(func, "evalcontextfilter", False) is True: + elif getattr(func, f"evalcontext{mark_name}", False) is True: if eval_ctx is None: if context is not None: eval_ctx = context.eval_ctx else: eval_ctx = EvalContext(self) + args.insert(0, eval_ctx) - elif getattr(func, "environmentfilter", False) is True: + elif getattr(func, f"environment{mark_name}", False) is True: args.insert(0, self) - return func(*args, **(kwargs or {})) - def call_test(self, name, value, args=None, kwargs=None): - """Invokes a test on a value the same way the compiler does it. + return func(*args, **kwargs) + + def call_filter( + self, name, value, args=None, kwargs=None, context=None, eval_ctx=None + ): + """Invoke a filter on a value the same way the compiler does. + + This might return a coroutine if the filter is running from an + environment in async mode and the filter supports async + execution. It's your responsibility to await this if needed. .. versionadded:: 2.7 """ - func = self.tests.get(name) - if func is None: - fail_for_missing_callable("test", name) - return func(value, *(args or ()), **(kwargs or {})) + return self._filter_test_common( + name, value, args, kwargs, context, eval_ctx, True + ) + + def call_test( + self, name, value, args=None, kwargs=None, context=None, eval_ctx=None + ): + """Invoke a test on a value the same way the compiler does. + + This might return a coroutine if the test is running from an + environment in async mode and the test supports async execution. + It's your responsibility to await this if needed. + + .. versionchanged:: 3.0 + Tests support ``@contextfunction``, etc. decorators. Added + the ``context`` and ``eval_ctx`` parameters. + + .. versionadded:: 2.7 + """ + return self._filter_test_common( + name, value, args, kwargs, context, eval_ctx, False + ) @internalcode def parse(self, source, name=None, filename=None): diff --git a/src/jinja2/nodes.py b/src/jinja2/nodes.py index a0d719de..49e56d29 100644 --- a/src/jinja2/nodes.py +++ b/src/jinja2/nodes.py @@ -633,71 +633,76 @@ def args_as_const(node, eval_ctx): return args, kwargs -class Filter(Expr): - """This node applies a filter on an expression. `name` is the name of - the filter, the rest of the fields are the same as for :class:`Call`. - - If the `node` of a filter is `None` the contents of the last buffer are - filtered. Buffers are created by macros and filter blocks. - """ - +class _FilterTestCommon(Expr): fields = ("node", "name", "args", "kwargs", "dyn_args", "dyn_kwargs") + abstract = True + _is_filter = True def as_const(self, eval_ctx=None): eval_ctx = get_eval_context(self, eval_ctx) - if eval_ctx.volatile or self.node is None: + if eval_ctx.volatile: raise Impossible() - filter_ = self.environment.filters.get(self.name) + if self._is_filter: + env_map = eval_ctx.environment.filters + mark_name = "filter" + else: + env_map = eval_ctx.environment.tests + # Filters use "contextfilter", tests and calls use "contextfunction". + mark_name = "function" - if filter_ is None or getattr(filter_, "contextfilter", False) is True: + func = env_map.get(self.name) + + if func is None or getattr(func, f"context{mark_name}", False) is True: raise Impossible() - # We cannot constant handle async filters, so we need to make - # sure to not go down this path. Account for both sync/async and - # pure-async filters. if eval_ctx.environment.is_async and ( - getattr(filter_, "asyncfiltervariant", False) - or inspect.iscoroutinefunction(filter_) + getattr(func, f"async{mark_name}variant", False) + or inspect.iscoroutinefunction(func) ): raise Impossible() args, kwargs = args_as_const(self, eval_ctx) args.insert(0, self.node.as_const(eval_ctx)) - if getattr(filter_, "evalcontextfilter", False) is True: + if getattr(func, f"evalcontext{mark_name}", False) is True: args.insert(0, eval_ctx) - elif getattr(filter_, "environmentfilter", False) is True: - args.insert(0, self.environment) + elif getattr(func, f"environment{mark_name}", False) is True: + args.insert(0, eval_ctx.environment) try: - return filter_(*args, **kwargs) + return func(*args, **kwargs) except Exception: raise Impossible() -class Test(Expr): - """Applies a test on an expression. `name` is the name of the test, the - rest of the fields are the same as for :class:`Call`. - """ +class Filter(_FilterTestCommon): + """Apply a filter to an expression. ``name`` is the name of the + filter, the other fields are the same as :class:`Call`. - fields = ("node", "name", "args", "kwargs", "dyn_args", "dyn_kwargs") + If ``node`` is ``None``, the filter is being used in a filter block + and is applied to the content of the block. + """ def as_const(self, eval_ctx=None): - test = self.environment.tests.get(self.name) - - if test is None: + if self.node is None: raise Impossible() - eval_ctx = get_eval_context(self, eval_ctx) - args, kwargs = args_as_const(self, eval_ctx) - args.insert(0, self.node.as_const(eval_ctx)) + return super().as_const(eval_ctx=eval_ctx) - try: - return test(*args, **kwargs) - except Exception: - raise Impossible() + +class Test(_FilterTestCommon): + """Apply a test to an expression. ``name`` is the name of the test, + the other field are the same as :class:`Call`. + + .. versionchanged:: 3.0 + ``as_const`` shares the same logic for filters and tests. Tests + check for volatile, async, and ``@contextfunction`` etc. + decorators. + """ + + _is_filter = False class Call(Expr): diff --git a/src/jinja2/tests.py b/src/jinja2/tests.py index bc763268..62b83220 100644 --- a/src/jinja2/tests.py +++ b/src/jinja2/tests.py @@ -5,6 +5,7 @@ from collections import abc from numbers import Number from .runtime import Undefined +from .utils import environmentfunction number_re = re.compile(r"^-?\d+(\.\d+)?$") regex_type = type(number_re) @@ -48,6 +49,46 @@ def test_undefined(value): return isinstance(value, Undefined) +@environmentfunction +def test_filter(env, value): + """Check if a filter exists by name. Useful if a filter may be + optionally available. + + .. code-block:: jinja + + {% if 'markdown' is filter %} + {{ value | markdown }} + {% else %} + {{ value }} + {% endif %} + + .. versionadded:: 3.0 + """ + return value in env.filters + + +@environmentfunction +def test_test(env, value): + """Check if a test exists by name. Useful if a test may be + optionally available. + + .. code-block:: jinja + + {% if 'loud' is test %} + {% if value is loud %} + {{ value|upper }} + {% else %} + {{ value|lower }} + {% endif %} + {% else %} + {{ value }} + {% endif %} + + .. versionadded:: 3.0 + """ + return value in env.tests + + def test_none(value): """Return true if the variable is none.""" return value is None @@ -176,6 +217,8 @@ TESTS = { "divisibleby": test_divisibleby, "defined": test_defined, "undefined": test_undefined, + "filter": test_filter, + "test": test_test, "none": test_none, "boolean": test_boolean, "false": test_false, diff --git a/tests/test_filters.py b/tests/test_filters.py index 44be6ad2..5a11d3f9 100644 --- a/tests/test_filters.py +++ b/tests/test_filters.py @@ -778,13 +778,13 @@ class TestFilter: assert result == "Hello!\nThis is Jinja saying\nsomething." def test_filter_undefined(self, env): - with pytest.raises(TemplateAssertionError, match="no filter named 'f'"): + with pytest.raises(TemplateAssertionError, match="No filter named 'f'"): env.from_string("{{ var|f }}") def test_filter_undefined_in_if(self, env): t = env.from_string("{%- if x is defined -%}{{ x|f }}{%- else -%}x{% endif %}") assert t.render() == "x" - with pytest.raises(TemplateRuntimeError, match="no filter named 'f'"): + with pytest.raises(TemplateRuntimeError, match="No filter named 'f'"): t.render(x=42) def test_filter_undefined_in_elif(self, env): @@ -793,7 +793,7 @@ class TestFilter: "{{ y|f }}{%- else -%}foo{%- endif -%}" ) assert t.render() == "foo" - with pytest.raises(TemplateRuntimeError, match="no filter named 'f'"): + with pytest.raises(TemplateRuntimeError, match="No filter named 'f'"): t.render(y=42) def test_filter_undefined_in_else(self, env): @@ -801,7 +801,7 @@ class TestFilter: "{%- if x is not defined -%}foo{%- else -%}{{ x|f }}{%- endif -%}" ) assert t.render() == "foo" - with pytest.raises(TemplateRuntimeError, match="no filter named 'f'"): + with pytest.raises(TemplateRuntimeError, match="No filter named 'f'"): t.render(x=42) def test_filter_undefined_in_nested_if(self, env): @@ -811,7 +811,7 @@ class TestFilter: ) assert t.render() == "foo" assert t.render(x=42) == "42" - with pytest.raises(TemplateRuntimeError, match="no filter named 'f'"): + with pytest.raises(TemplateRuntimeError, match="No filter named 'f'"): t.render(x=24, y=42) def test_filter_undefined_in_condexpr(self, env): @@ -819,6 +819,6 @@ class TestFilter: t2 = env.from_string("{{ 'foo' if x is not defined else x|f }}") assert t1.render() == t2.render() == "foo" - with pytest.raises(TemplateRuntimeError, match="no filter named 'f'"): + with pytest.raises(TemplateRuntimeError, match="No filter named 'f'"): t1.render(x=42) t2.render(x=42) diff --git a/tests/test_tests.py b/tests/test_tests.py index d363653d..4d56a15d 100644 --- a/tests/test_tests.py +++ b/tests/test_tests.py @@ -2,6 +2,8 @@ import pytest from jinja2 import Environment from jinja2 import Markup +from jinja2 import TemplateAssertionError +from jinja2 import TemplateRuntimeError class MyDict(dict): @@ -206,3 +208,26 @@ class TestTestsCase: '{{ "baz" is in {"bar": 1}}}' ) assert tmpl.render() == "True|True|False|True|False|True|False|True|False" + + +def test_name_undefined(env): + with pytest.raises(TemplateAssertionError, match="No test named 'f'"): + env.from_string("{{ x is f }}") + + +def test_name_undefined_in_if(env): + t = env.from_string("{% if x is defined %}{{ x is f }}{% endif %}") + assert t.render() == "" + + with pytest.raises(TemplateRuntimeError, match="No test named 'f'"): + t.render(x=1) + + +def test_is_filter(env): + assert env.call_test("filter", "title") + assert not env.call_test("filter", "bad-name") + + +def test_is_test(env): + assert env.call_test("test", "number") + assert not env.call_test("test", "bad-name") -- 2.47.2