From: Armin Ronacher <armin.ronacher@active-4.com>
Date: Sat, 14 Jun 2008 22:07:41 +0000 (+0000)
Subject: JavaScript lexer falls back silently now on syntax errors and tries to recover.
X-Git-Tag: 1.0~334
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=ac564c76ee8b3fcad7d62314ff42e208473f0cb8;p=thirdparty%2Fbabel.git

JavaScript lexer falls back silently now on syntax errors and tries to recover.
---

diff --git a/babel/messages/jslexer.py b/babel/messages/jslexer.py
index 9c666f0b..7b5c9a43 100644
--- a/babel/messages/jslexer.py
+++ b/babel/messages/jslexer.py
@@ -23,7 +23,7 @@ operators = [
     '+', '-', '*', '%', '!=', '==', '<', '>', '<=', '>=', '=',
     '+=', '-=', '*=', '%=', '<<', '>>', '>>>', '<<=', '>>=',
     '>>>=', '&', '&=', '|', '|=', '&&', '||', '^', '^=', '(', ')',
-    '[', ']', '{', '}', '!', '--', '++', '~', ',', ';', '.'
+    '[', ']', '{', '}', '!', '--', '++', '~', ',', ';', '.', ':'
 ]
 operators.sort(lambda a, b: cmp(-len(a), -len(b)))
 
@@ -55,10 +55,6 @@ line_join_re = re.compile(r'\\' + line_re.pattern)
 uni_escape_re = re.compile(r'[a-fA-F0-9]{1,4}')
 
 
-class TokenError(ValueError):
-    """Raised if the tokenizer stumbled upon invalid tokens."""
-
-
 class Token(tuple):
     """Represents a token as returned by `tokenize`."""
     __slots__ = ()
@@ -166,7 +162,9 @@ def tokenize(source):
                 match = regex_re.match(source, pos)
                 token_type = 'regexp'
             if match is None:
-                raise TokenError('invalid syntax around line %d' % lineno)
+                # woops. invalid syntax. jump one char ahead and try again.
+                pos += 1
+                continue
 
         token_value = match.group()
         if token_type is not None: