Allow symbols to be overwritten by the environment

This introduces a change to both the grammar and the parsing
environment that allows people to override start/end symbols in the
grammar through the environment. This finally brings the parser on
the same level as the old parser and lexer when it comes to handling
those customizations.

This means that the grammar must be compiled dynamically to account
for these customizations per environment. A module-level LRU cache
has been implemented to handle this fact, so grammars can be cached
instead of compiled every time. This should handle most cases other
than the unit tests, since most people aren't frequently changing up
their environment within their applications.

This also adds proper handling to the closing line block statement
so it waits for the end of a line or the end of the expression.

diff --git a/grammar.ebnf b/grammar.ebnf
index 11d26e6ec68df0eb96b41abb0d3cc726f68636a6..0f8a6ada1f700c6317f4750d4923c8159ac20c2d 100644 (file)
--- a/grammar.ebnf
+++ b/grammar.ebnf
@@ -67,14 +67,24 @@ block_end
 \r
 block_open\r
     =\r
-    | ( {SP}* @:"{%-" {SP}* )\r
-    | @:"{%" {SP}*\r
+    | ( {SP}* block_open_symbol "-" {SP}* )\r
+    | block_open_symbol {SP}*\r
+    ;\r
+\r
+block_open_symbol\r
+    =\r
+    "{%"\r
     ;\r
 \r
 block_close\r
     =\r
-    | ( @:"-%}" {SP}* )\r
-    | @:"%}"\r
+    | ( "-" block_close_symbol {SP}* )\r
+    | block_close_symbol\r
+    ;\r
+\r
+block_close_symbol\r
+    =\r
+    "%}"\r
     ;\r
 \r
 line_block_expression\r
@@ -90,12 +100,17 @@ line_block_start
 \r
 line_block_end\r
     =\r
-    line_block_open "end" name:IDENTIFIER "\n"\r
+    line_block_open "end" name:IDENTIFIER ("\n" | $)\r
     ;\r
 \r
 line_block_open\r
     =\r
-    "\n" {SP}* @:"# " {SP}*\r
+    "\n" {SP}* line_block_open_symbol {SP}*\r
+    ;\r
+\r
+line_block_open_symbol\r
+    =\r
+    "#"\r
     ;\r
 \r
 line_block_parameters\r
@@ -148,15 +163,27 @@ variable_expression
     =\r
     variable_open type:`variable` name:conditional_expression variable_close\r
     ;\r
+\r
 variable_open\r
     =\r
-    | ( {SP}* @:"{{-" {SP}* )\r
-    | ( @:"{{" {SP}* )\r
+    | ( {SP}* variable_open_symbol "-" {SP}* )\r
+    | ( variable_open_symbol {SP}* )\r
+    ;\r
+\r
+variable_open_symbol\r
+    =\r
+    "{{"\r
     ;\r
+\r
 variable_close\r
     =\r
-    | ( {SP}* @:"-}}" {SP}* )\r
-    | ( {SP}* @:"}}" )\r
+    | ( {SP}* "-" variable_close_symbol {SP}* )\r
+    | ( {SP}* variable_close_symbol )\r
+    ;\r
+\r
+variable_close_symbol\r
+    =\r
+    "}}"\r
     ;\r
 \r
 variable_identifier\r
@@ -406,12 +433,21 @@ comment_expression
     comment_open comment:comment_content comment_close\r
     ;\r
 \r
-comment_open\r
+comment_open =\r
+    comment_open_symbol\r
+    ;\r
+\r
+comment_open_symbol\r
     =\r
     "{#"\r
     ;\r
 \r
 comment_close\r
+    =\r
+    comment_close_symbol\r
+    ;\r
+\r
+comment_close_symbol\r
     =\r
     "#}"\r
     ;\r
@@ -428,7 +464,12 @@ line_comment_expression
 \r
 line_comment_open\r
     =\r
-    {SP}* "## "\r
+    {SP}* line_comment_open_symbol\r
+    ;\r
+\r
+line_comment_open_symbol\r
+    =\r
+    '##'\r
     ;\r
 \r
 line_comment_content\r

diff --git a/src/jinja2/environment.py b/src/jinja2/environment.py
index 3c93c4843f867f3155943a91f8b34356c8a62d31..94ace5349ef8067de457d998cd2c1ef0109ff182 100644 (file)
--- a/src/jinja2/environment.py
+++ b/src/jinja2/environment.py
@@ -50,6 +50,7 @@ from .utils import missing
 
 # for direct template usage we have up to ten living environments
 _spontaneous_environments = LRUCache(10)
+_grammar_cache = LRUCache(10)
 
 
 def get_spontaneous_environment(cls, *args):
@@ -527,6 +528,69 @@ class Environment:
         """Internal parsing function used by `parse` and `compile`."""
         return Parser(self, source, name, filename).parse()
 
+    def get_grammar(self):
+        import tatsu
+
+        grammar_extensions = ''
+
+        with open('grammar.ebnf', 'r') as grammar_file:
+            base_grammar = grammar_file.read()
+
+        if self.block_start_string:
+            grammar_extensions += '''
+            @override
+            block_open_symbol = %r;
+            ''' % (self.block_start_string)
+
+        if self.block_end_string:
+            grammar_extensions += '''
+            @override
+            block_close_symbol = %r;
+            ''' % (self.block_end_string)
+
+        if self.variable_start_string:
+            grammar_extensions += '''
+            @override
+            variable_open_symbol = %r;
+            ''' % (self.variable_start_string)
+
+        if self.variable_end_string:
+            grammar_extensions += '''
+            @override
+            variable_close_symbol = %r;
+            ''' % (self.variable_end_string)
+
+        if self.comment_start_string:
+            grammar_extensions += '''
+            @override
+            comment_open_symbol = %r;
+            ''' % (self.comment_start_string)
+
+        if self.comment_end_string:
+            grammar_extensions += '''
+            @override
+            comment_close_symbol = %r;
+            ''' % (self.comment_end_string)
+
+        if self.line_statement_prefix:
+            grammar_extensions += '''
+            @override
+            line_block_open_symbol = %r;
+            ''' % (self.line_statement_prefix)
+
+        if self.line_comment_prefix:
+            grammar_extensions += '''
+            @override
+            line_comment_open_symbol = %r;
+            ''' % (self.line_comment_prefix)
+
+        final_grammar = base_grammar + grammar_extensions
+
+        if final_grammar not in _grammar_cache:
+            _grammar_cache[final_grammar] = tatsu.compile(final_grammar)
+
+        return _grammar_cache[final_grammar]
+
     def lex(self, source, name=None, filename=None):
         """Lex the given sourcecode and return a generator that yields
         tokens as tuples in the form ``(lineno, token_type, value)``.

diff --git a/src/jinja2/parser.py b/src/jinja2/parser.py
index 721988cd9b04b21afb5d0925aa9b94880f6ba945..47a11b02464cc9f1166e51acf41a738f8825dc7b 100644 (file)
--- a/src/jinja2/parser.py
+++ b/src/jinja2/parser.py
@@ -5,11 +5,6 @@ from .exceptions import TemplateSyntaxError
 from .lexer import describe_token
 from .lexer import describe_token_expr
 
-import tatsu
-
-with open('grammar.ebnf', 'r') as grammar_file:
-    grammar = tatsu.compile(grammar_file.read())
-
 
 _statement_keywords = frozenset(
     [
@@ -47,6 +42,7 @@ class Parser:
     def __init__(self, environment, source, name=None, filename=None, state=None):
         self.environment = environment
         self.source = source
+        self.grammar = environment.get_grammar()
         self.stream = environment._tokenize(source, name, filename, state)
         self.name = name
         self.filename = filename
@@ -944,7 +940,7 @@ class Parser:
         from .new_parser import JinjaSemantics, parse_template
 
         result = parse_template(
-            grammar.parse(
+            self.grammar.parse(
                 self.source.rstrip('\n'),
                 whitespace='',
                 parseinfo=True,