From: Aarni Koskela <akx@iki.fi>
Date: Mon, 18 Jan 2016 15:09:57 +0000 (+0200)
Subject: JavaScript: Support dotted names as keywords
X-Git-Tag: 2.3.1~5^2~2
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=d0c3853728cf242cfd1b637d17a685a4f1864d73;p=thirdparty%2Fbabel.git

JavaScript: Support dotted names as keywords

Fixes #148
---

diff --git a/babel/messages/extract.py b/babel/messages/extract.py
index 153fe511..ee90afb7 100644
--- a/babel/messages/extract.py
+++ b/babel/messages/extract.py
@@ -518,8 +518,13 @@ def extract_javascript(fileobj, keywords, comment_tags, options):
     encoding = options.get('encoding', 'utf-8')
     last_token = None
     call_stack = -1
+    dotted = any('.' in kw for kw in keywords)
 
-    for token in tokenize(fileobj.read().decode(encoding), jsx=options.get("jsx", True)):
+    for token in tokenize(
+        fileobj.read().decode(encoding),
+        jsx=options.get("jsx", True),
+        dotted=dotted
+    ):
         if token.type == 'operator' and token.value == '(':
             if funcname:
                 message_lineno = token.lineno
diff --git a/babel/messages/jslexer.py b/babel/messages/jslexer.py
index 32500676..e6310cf3 100644
--- a/babel/messages/jslexer.py
+++ b/babel/messages/jslexer.py
@@ -22,12 +22,13 @@ operators = sorted([
 
 escapes = {'b': '\b', 'f': '\f', 'n': '\n', 'r': '\r', 't': '\t'}
 
+name_re = re.compile(r'[\w$_][\w\d$_]*', re.UNICODE)
+dotted_name_re = re.compile(r'[\w$_][\w\d$_.]*[\w\d$_.]', re.UNICODE)
 division_re = re.compile(r'/=?')
 regex_re = re.compile(r'/(?:[^/\\]*(?:\\.[^/\\]*)*)/[a-zA-Z]*(?s)')
 line_re = re.compile(r'(\r\n|\n|\r)')
 line_join_re = re.compile(r'\\' + line_re.pattern)
 uni_escape_re = re.compile(r'[a-fA-F0-9]{1,4}')
-name_re = re.compile(r'(\$+\w*|[^\W\d]\w*)(?u)')
 
 Token = namedtuple('Token', 'type value lineno')
 
@@ -36,6 +37,7 @@ _rules = [
     (None, re.compile(r'<!--.*')),
     ('linecomment', re.compile(r'//.*')),
     ('multilinecomment', re.compile(r'/\*.*?\*/(?us)')),
+    ('dotted_name', dotted_name_re),
     ('name', name_re),
     ('number', re.compile(r'''(?x)(
         (?:0|[1-9]\d*)
@@ -52,7 +54,7 @@ _rules = [
 ]
 
 
-def get_rules(jsx):
+def get_rules(jsx, dotted):
     """
     Get a tokenization rule list given the passed syntax options.
 
@@ -62,6 +64,10 @@ def get_rules(jsx):
     for token_type, rule in _rules:
         if not jsx and token_type and 'jsx' in token_type:
             continue
+        if token_type == 'dotted_name':
+            if not dotted:
+                continue
+            token_type = 'name'
         rules.append((token_type, rule))
     return rules
 
@@ -131,17 +137,18 @@ def unquote_string(string):
     return u''.join(result)
 
 
-def tokenize(source, jsx=True):
+def tokenize(source, jsx=True, dotted=True):
     """
     Tokenize JavaScript/JSX source.  Returns a generator of tokens.
 
     :param jsx: Enable (limited) JSX parsing.
+    :param dotted: Read dotted names as single name token.
     """
     may_divide = False
     pos = 0
     lineno = 1
     end = len(source)
-    rules = get_rules(jsx=jsx)
+    rules = get_rules(jsx=jsx, dotted=dotted)
 
     while pos < end:
         # handle regular rules first
diff --git a/tests/messages/test_js_extract.py b/tests/messages/test_js_extract.py
index ae6d277b..203bf4cb 100644
--- a/tests/messages/test_js_extract.py
+++ b/tests/messages/test_js_extract.py
@@ -122,3 +122,12 @@ def test_jsx_extraction(jsx_enabled):
         assert messages == EXPECTED_JSX_MESSAGES
     else:
         assert messages != EXPECTED_JSX_MESSAGES
+
+
+def test_dotted_keyword_extract():
+    buf = BytesIO(b"msg1 = com.corporate.i18n.formatMessage('Insert coin to continue')")
+    messages = list(
+        extract.extract('javascript', buf, {"com.corporate.i18n.formatMessage": None}, [], {})
+    )
+
+    assert messages == [(1, 'Insert coin to continue', [], None)]
diff --git a/tests/messages/test_jslexer.py b/tests/messages/test_jslexer.py
index 4ffb4a9e..7277104f 100644
--- a/tests/messages/test_jslexer.py
+++ b/tests/messages/test_jslexer.py
@@ -6,3 +6,22 @@ from babel.messages import jslexer
 def test_unquote():
     assert jslexer.unquote_string('""') == ''
     assert jslexer.unquote_string(r'"h\u00ebllo"') == u"hëllo"
+
+
+def test_dollar_in_identifier():
+    assert list(jslexer.tokenize('dollar$dollar')) == [('name', 'dollar$dollar', 1)]
+
+
+def test_dotted_name():
+    assert list(jslexer.tokenize("foo.bar(quux)", dotted=True)) == [
+        ('name', 'foo.bar', 1),
+        ('operator', '(', 1),
+        ('name', 'quux', 1),
+        ('operator', ')', 1)
+    ]
+
+
+def test_dotted_name_end():
+    assert list(jslexer.tokenize("foo.bar", dotted=True)) == [
+        ('name', 'foo.bar', 1),
+    ]