JavaScript lexer falls back silently now on syntax errors and tries to recover.

diff --git a/babel/messages/jslexer.py b/babel/messages/jslexer.py
index 9c666f0bc40c250f612ef45c03783f3972d9a2e3..7b5c9a43b64f732930466baba9b7cccf77250f34 100644 (file)
--- a/babel/messages/jslexer.py
+++ b/babel/messages/jslexer.py
@@ -23,7 +23,7 @@ operators = [
     '+', '-', '*', '%', '!=', '==', '<', '>', '<=', '>=', '=',
     '+=', '-=', '*=', '%=', '<<', '>>', '>>>', '<<=', '>>=',
     '>>>=', '&', '&=', '|', '|=', '&&', '||', '^', '^=', '(', ')',
-    '[', ']', '{', '}', '!', '--', '++', '~', ',', ';', '.'
+    '[', ']', '{', '}', '!', '--', '++', '~', ',', ';', '.', ':'
 ]
 operators.sort(lambda a, b: cmp(-len(a), -len(b)))
 
@@ -55,10 +55,6 @@ line_join_re = re.compile(r'\\' + line_re.pattern)
 uni_escape_re = re.compile(r'[a-fA-F0-9]{1,4}')
 
 
-class TokenError(ValueError):
-    """Raised if the tokenizer stumbled upon invalid tokens."""
-
-
 class Token(tuple):
     """Represents a token as returned by `tokenize`."""
     __slots__ = ()
@@ -166,7 +162,9 @@ def tokenize(source):
                 match = regex_re.match(source, pos)
                 token_type = 'regexp'
             if match is None:
-                raise TokenError('invalid syntax around line %d' % lineno)
+                # woops. invalid syntax. jump one char ahead and try again.
+                pos += 1
+                continue
 
         token_value = match.group()
         if token_type is not None: