From: Aarni Koskela <akx@iki.fi>
Date: Mon, 18 Jan 2016 15:35:05 +0000 (+0200)
Subject: JavaScript: add basic support for template strings
X-Git-Tag: 2.3.1~5^2~1
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=f061cc16415dc0399f7fcea62012617885ab495b;p=thirdparty%2Fbabel.git

JavaScript: add basic support for template strings

Refs #329
---

diff --git a/babel/messages/extract.py b/babel/messages/extract.py
index ee90afb7..2e505375 100644
--- a/babel/messages/extract.py
+++ b/babel/messages/extract.py
@@ -508,6 +508,8 @@ def extract_javascript(fileobj, keywords, comment_tags, options):
     :param options: a dictionary of additional options (optional)
                     Supported options are:
                     * `jsx` -- set to false to disable JSX/E4X support.
+                    * `template_string` -- set to false to disable ES6
+                                           template string support.
     """
     from babel.messages.jslexer import tokenize, unquote_string
     funcname = message_lineno = None
@@ -523,6 +525,7 @@ def extract_javascript(fileobj, keywords, comment_tags, options):
     for token in tokenize(
         fileobj.read().decode(encoding),
         jsx=options.get("jsx", True),
+        template_string=options.get("template_string", True),
         dotted=dotted
     ):
         if token.type == 'operator' and token.value == '(':
@@ -584,7 +587,7 @@ def extract_javascript(fileobj, keywords, comment_tags, options):
                 messages = []
                 call_stack = -1
 
-            elif token.type == 'string':
+            elif token.type in ('string', 'template_string'):
                 new_value = unquote_string(token.value)
                 if concatenate_next:
                     last_argument = (last_argument or '') + new_value
diff --git a/babel/messages/jslexer.py b/babel/messages/jslexer.py
index e6310cf3..aed39f33 100644
--- a/babel/messages/jslexer.py
+++ b/babel/messages/jslexer.py
@@ -47,6 +47,7 @@ _rules = [
     )''')),
     ('jsx_tag', re.compile(r'<(?:/?)\w+.+?>', re.I)),  # May be mangled in `get_rules`
     ('operator', re.compile(r'(%s)' % '|'.join(map(re.escape, operators)))),
+    ('template_string', re.compile(r'''`(?:[^`\\]*(?:\\.[^`\\]*)*)`''', re.UNICODE)),
     ('string', re.compile(r'''(?xs)(
         '(?:[^'\\]*(?:\\.[^'\\]*)*)'  |
         "(?:[^"\\]*(?:\\.[^"\\]*)*)"
@@ -54,7 +55,7 @@ _rules = [
 ]
 
 
-def get_rules(jsx, dotted):
+def get_rules(jsx, dotted, template_string):
     """
     Get a tokenization rule list given the passed syntax options.
 
@@ -64,6 +65,8 @@ def get_rules(jsx, dotted):
     for token_type, rule in _rules:
         if not jsx and token_type and 'jsx' in token_type:
             continue
+        if not template_string and token_type == 'template_string':
+            continue
         if token_type == 'dotted_name':
             if not dotted:
                 continue
@@ -83,9 +86,9 @@ def indicates_division(token):
 
 def unquote_string(string):
     """Unquote a string with JavaScript rules.  The string has to start with
-    string delimiters (``'`` or ``"``.)
+    string delimiters (``'``, ``"`` or the back-tick/grave accent (for template strings).)
     """
-    assert string and string[0] == string[-1] and string[0] in '"\'', \
+    assert string and string[0] == string[-1] and string[0] in '"\'`', \
         'string provided is not properly delimited'
     string = line_join_re.sub('\\1', string[1:-1])
     result = []
@@ -137,18 +140,19 @@ def unquote_string(string):
     return u''.join(result)
 
 
-def tokenize(source, jsx=True, dotted=True):
+def tokenize(source, jsx=True, dotted=True, template_string=True):
     """
     Tokenize JavaScript/JSX source.  Returns a generator of tokens.
 
     :param jsx: Enable (limited) JSX parsing.
     :param dotted: Read dotted names as single name token.
+    :param template_string: Support ES6 template strings
     """
     may_divide = False
     pos = 0
     lineno = 1
     end = len(source)
-    rules = get_rules(jsx=jsx, dotted=dotted)
+    rules = get_rules(jsx=jsx, dotted=dotted, template_string=template_string)
 
     while pos < end:
         # handle regular rules first
diff --git a/tests/messages/test_js_extract.py b/tests/messages/test_js_extract.py
index 203bf4cb..f8193190 100644
--- a/tests/messages/test_js_extract.py
+++ b/tests/messages/test_js_extract.py
@@ -131,3 +131,12 @@ def test_dotted_keyword_extract():
     )
 
     assert messages == [(1, 'Insert coin to continue', [], None)]
+
+
+def test_template_string_standard_usage():
+    buf = BytesIO(b"msg1 = gettext(`Very template, wow`)")
+    messages = list(
+        extract.extract('javascript', buf, {"gettext": None}, [], {})
+    )
+
+    assert messages == [(1, 'Very template, wow', [], None)]
diff --git a/tests/messages/test_jslexer.py b/tests/messages/test_jslexer.py
index 7277104f..84221a1d 100644
--- a/tests/messages/test_jslexer.py
+++ b/tests/messages/test_jslexer.py
@@ -25,3 +25,10 @@ def test_dotted_name_end():
     assert list(jslexer.tokenize("foo.bar", dotted=True)) == [
         ('name', 'foo.bar', 1),
     ]
+
+
+def test_template_string():
+    assert list(jslexer.tokenize("gettext `foo\"bar\"p`", template_string=True)) == [
+        ('name', 'gettext', 1),
+        ('template_string', '`foo"bar"p`', 1)
+    ]