Start working on new AST parser

This parser will take the Tatsu-generated AST and generate a compatible
Jinja AST from it. This should allow us to refine the grammar to
generate a better AST and also verify that it is producing comparable
Jinja ASTs that can be used by the current compiler.

diff --git a/.gitignore b/.gitignore
index dacb502c82abd2b1a0ce4a3683845f47985a6a70..914d146d118454862e65331868e9bc9a31db1ad9 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -18,4 +18,6 @@ venv-*/
 htmlcov
 .pytest_cache/
 /.vscode/
-tatsu_test.json
\ No newline at end of file
+tatsu_jinja.json
+tatsu_jinja.py
+parsed_jinja.py
\ No newline at end of file

diff --git a/new_parser.py b/new_parser.py
new file mode 100644 (file)
index 0000000..8eec89c
--- /dev/null
+++ b/new_parser.py
@@ -0,0 +1,127 @@
+from jinja2 import nodes\r
+\r
+\r
+def lineno_from_parseinfo(parseinfo):\r
+    return parseinfo.line + 1\r
+\r
+def parse(ast):\r
+    if isinstance(ast, list):\r
+        return [parse(item) for item in ast]\r
+\r
+    if isinstance(ast, str):\r
+        return parse_output(ast)\r
+\r
+    if 'type' in ast and ast['type'] == 'variable':\r
+        return parse_print(ast)\r
+\r
+    if 'block' in ast:\r
+        return parse_block(ast)\r
+\r
+    if 'start' in ast and 'end' in ast:\r
+        return parse_block_pair(ast)\r
+\r
+    return None\r
+\r
+def parse_block(ast):\r
+    if ast['block']['name'] == 'from':\r
+        return parse_block_from(ast)\r
+\r
+    return None\r
+\r
+def parse_block_pair(ast):\r
+    if ast['start']['name'] == 'with':\r
+        return parse_block_with(ast)\r
+\r
+    return None\r
+\r
+def parse_block_from(ast):\r
+    names = []\r
+    parameters = ast['block']['parameters']\r
+\r
+    if len(parameters) > 2:\r
+        names = []#parameters[2:]\r
+\r
+    from_import = nodes.FromImport(\r
+        lineno=lineno_from_parseinfo(ast['parseinfo'])\r
+    )\r
+    from_import.template = parse_variable(parameters[0]['value'])\r
+    from_import.names = names\r
+\r
+    return from_import\r
+\r
+def parse_block_with(ast):\r
+    with_node = nodes.With(\r
+        lineno=lineno_from_parseinfo(ast['parseinfo'])\r
+    )\r
+\r
+    targets = []\r
+    values = []\r
+\r
+    for parameter in ast['start']['parameters']:\r
+        if 'key' not in parameter:\r
+            raise\r
+\r
+        targets.append(parameter['key'])\r
+        values.append(parameter['value'])\r
+\r
+    with_node.targets = targets\r
+    with_node.values = values\r
+\r
+    return with_node\r
+\r
+def parse_literal(ast):\r
+    if 'literal_type' not in ast:\r
+        raise\r
+\r
+    literal_type = ast['literal_type']\r
+\r
+    if literal_type == 'string':\r
+        return nodes.Const(\r
+            ''.join(ast['value']),\r
+            lineno=lineno_from_parseinfo(ast['parseinfo'])\r
+        )\r
+\r
+def parse_output(ast):\r
+    return nodes.Output(\r
+        [nodes.TemplateData(ast)]\r
+    )\r
+\r
+def parse_print(ast):\r
+    variable = ast['name']\r
+\r
+    node = parse_variable(variable)\r
+\r
+    for accessor_ast in variable['accessors']:\r
+        node = parse_variable_accessor(node, accessor_ast)\r
+\r
+    return nodes.Output(node)\r
+\r
+def parse_template(ast):\r
+    return nodes.Template(parse(ast), lineno=1)\r
+\r
+def parse_variable(ast):\r
+    ast = ast['variable']\r
+\r
+    if 'literal_type' in ast:\r
+        return parse_literal(ast)\r
+\r
+    return nodes.Name(\r
+        ast,\r
+        'load'\r
+    )\r
+\r
+def parse_variable_accessor(node, ast):\r
+    accessor_type = ast['accessor_type']\r
+\r
+    if accessor_type == 'brackets':\r
+        accessor_node = nodes.Getitem()\r
+        accessor_node.arg = parse_variable(ast['parameter'])\r
+    elif accessor_type == 'dot':\r
+        accessor_node = nodes.Getattr()\r
+        accessor_node.attr = ast['parameter']['variable']\r
+\r
+    accessor_node.node = node\r
+    accessor_node.ctx = "load"\r
+    accessor_node.lineno = lineno_from_parseinfo(ast['parseinfo'])\r
+\r
+    return accessor_node
\ No newline at end of file

diff --git a/test_tatsu.py b/test_tatsu.py
index 8ff2e45b1685cea4d9f9d6276a7e58e9883e84cc..24a34ebe491f40c1336dd9d4aa40c4c1dc9b37e8 100644 (file)
--- a/test_tatsu.py
+++ b/test_tatsu.py
@@ -1,12 +1,17 @@
 from datetime import datetime\r
 from tatsu.util import asjson\r
 import json\r
+import pprint\r
 import tatsu\r
 import sys\r
+from new_parser import parse_template\r
+from jinja2.environment import Environment\r
 \r
 \r
 with open('grammar.ebnf', 'r') as tatsu_grammar:\r
     with open('test_template.jinja', 'r') as test_template:\r
+        template_string = test_template.read()\r
+\r
         grammar_start = datetime.now()\r
 \r
         grammar = tatsu.compile(tatsu_grammar.read())\r
@@ -15,11 +20,34 @@ with open('grammar.ebnf', 'r') as tatsu_grammar:
 \r
         parse_start = datetime.now()\r
 \r
-        ast = grammar.parse(test_template.read(), whitespace='')\r
+        ast = grammar.parse(template_string, whitespace='', parseinfo=True)\r
 \r
         parse_end = datetime.now()\r
 \r
-        print(json.dumps(asjson(ast), indent=2))\r
+        with open('tatsu_jinja.json', 'w') as tatsu_ast_file:\r
+            json.dump(asjson(ast), tatsu_ast_file, indent=2)\r
+\r
+        new_parse_start = datetime.now()\r
+\r
+        new_ast = parse_template(ast)\r
+\r
+        new_parse_end = datetime.now()\r
+\r
+        with open('tatsu_jinja.py', 'w') as new_ast_file:\r
+            pprint.pprint(new_ast, indent=2, stream=new_ast_file)\r
+\r
+        env = Environment()\r
+\r
+        jinja_parse_start = datetime.now()\r
+\r
+        jinja_ast = env.parse(template_string)\r
+\r
+        jinja_parse_end = datetime.now()\r
+\r
+        with open('parsed_jinja.py', 'w') as jinja_ast_file:\r
+            pprint.pprint(jinja_ast, indent=2, stream=jinja_ast_file)\r
 \r
         print("Grammar", grammar_end - grammar_start, file=sys.stderr)\r
-        print("Parser", parse_end - parse_start, file=sys.stderr)
\ No newline at end of file
+        print("Parser", parse_end - parse_start, file=sys.stderr)\r
+        print("New Parser", new_parse_end - new_parse_start, file=sys.stderr)\r
+        print("Jinja Parser", jinja_parse_end - jinja_parse_start, file=sys.stderr)\r

diff --git a/test_template.jinja b/test_template.jinja
index af50f6503d2951e4a56a5cb04d85aa5889029a02..b67be129f1b0a6c6a638355d65fca1155e9c3257 100644 (file)
--- a/test_template.jinja
+++ b/test_template.jinja
@@ -1,9 +1,8 @@
 {% from 'forms.html' import input as input_field, textarea %}\r
-{% with key=val.test %}\r
 {{ dict_var['single']["double"].dot  |test("first" ,2_000, named=3.14)|filter | lastFilter}}\r
-{%block single key=val param=value %}\r
+{%with key=val %}\r
 test {{var}}\r
-{%endblock%}\r
+{%endwith%}\r
 {% for item in dict_var.values() %}\r
     <li>{% block loop_item %}{{ item }}{% endblock %}</li>\r
 {% endfor %}\r
@@ -19,7 +18,6 @@ across lines #}
                          ('downloads.html', 'Downloads')] %}\r
     <li><a href="{{ href }}">{{ caption }}</a></li>\r
 {% endfor %}\r
-{% trans user=user.username %}Hello, {{ user }}!{% endtrans %}\r
 {% set navigation = [('index.html', 'Index'), ('about.html', 'About')] %}\r
 {% set key, value = call_something() %}\r
 {% for value in values %}\r