bpo-40334: Always show the caret on SyntaxErrors (GH-20050)

This commit fixes SyntaxError locations when the caret is not displayed,
by doing the following:

- `col_number` always gets set to the location of the offending
  node/expr. When no caret is to be displayed, this gets achieved
  by setting the object holding the error line to None.

- Introduce a new function `_PyPegen_raise_error_known_location`,
  which can be called, when an arbitrary `lineno`/`col_offset`
  needs to be passed. This function then gets used in the grammar
  (through some new macros and inline functions) so that SyntaxError
  locations of the new parser match that of the old.

diff --git a/Grammar/python.gram b/Grammar/python.gram
index 0542107cac3e6bb8c7160b3c3631b68386383047..84c89330e3ee9dd207b9fbf6d08aa3e34426c892 100644 (file)
--- a/Grammar/python.gram
+++ b/Grammar/python.gram
@@ -623,26 +623,31 @@ t_atom[expr_ty]:
 # From here on, there are rules for invalid syntax with specialised error messages
 incorrect_arguments:
     | args ',' '*' { RAISE_SYNTAX_ERROR("iterable argument unpacking follows keyword argument unpacking") }
-    | expression for_if_clauses ',' [args | expression for_if_clauses] {
-        RAISE_SYNTAX_ERROR("Generator expression must be parenthesized") }
+    | a=expression for_if_clauses ',' [args | expression for_if_clauses] {
+        RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "Generator expression must be parenthesized") }
     | a=args ',' args { _PyPegen_arguments_parsing_error(p, a) }
 invalid_kwarg:
-    | expression '=' { RAISE_SYNTAX_ERROR("expression cannot contain assignment, perhaps you meant \"==\"?") }
+    | a=expression '=' {
+        RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
+            a, "expression cannot contain assignment, perhaps you meant \"==\"?") }
 invalid_named_expression:
     | a=expression ':=' expression {
-        RAISE_SYNTAX_ERROR("cannot use assignment expressions with %s", _PyPegen_get_expr_name(a)) }
+        RAISE_SYNTAX_ERROR_KNOWN_LOCATION(
+            a, "cannot use assignment expressions with %s", _PyPegen_get_expr_name(a)) }
 invalid_assignment:
-    | list ':' { RAISE_SYNTAX_ERROR("only single target (not list) can be annotated") }
-    | tuple ':' { RAISE_SYNTAX_ERROR("only single target (not tuple) can be annotated") }
-    | expression ':' expression ['=' annotated_rhs] {
-        RAISE_SYNTAX_ERROR("illegal target for annotation") }
+    | a=list ':' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "only single target (not list) can be annotated") }
+    | a=tuple ':' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "only single target (not tuple) can be annotated") }
+    | a=star_named_expression ',' star_named_expressions* ':' {
+        RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "only single target (not tuple) can be annotated") }
+    | a=expression ':' expression ['=' annotated_rhs] {
+        RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "illegal target for annotation") }
     | a=expression ('=' | augassign) (yield_expr | star_expressions) {
-        RAISE_SYNTAX_ERROR_NO_COL_OFFSET("cannot assign to %s", _PyPegen_get_expr_name(a)) }
+        RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot assign to %s", _PyPegen_get_expr_name(a)) }
 invalid_block:
     | NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block") }
 invalid_comprehension:
-    | ('[' | '(' | '{') '*' expression for_if_clauses {
-        RAISE_SYNTAX_ERROR("iterable unpacking cannot be used in comprehension") }
+    | ('[' | '(' | '{') a=starred_expression for_if_clauses {
+        RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "iterable unpacking cannot be used in comprehension") }
 invalid_parameters:
     | param_no_default* (slash_with_default | param_with_default+) param_no_default {
         RAISE_SYNTAX_ERROR("non-default argument follows default argument") }
@@ -655,4 +660,4 @@ invalid_double_type_comments:
         RAISE_SYNTAX_ERROR("Cannot have two type comments on def") }
 invalid_del_target:
     | a=star_expression &del_target_end {
-        RAISE_SYNTAX_ERROR("cannot delete %s", _PyPegen_get_expr_name(a)) }
+        RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot delete %s", _PyPegen_get_expr_name(a)) }

diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py
index dbd7fa6bdd938531b5ba79a2a5d53d67ff7c2c88..b689ec7aed18de0a6315c0c06de8ce0a3bdfe0e2 100644 (file)
--- a/Lib/test/test_exceptions.py
+++ b/Lib/test/test_exceptions.py
@@ -242,16 +242,13 @@ class ExceptionTests(unittest.TestCase):
         check('from __future__ import doesnt_exist', 1, 1)
         check('from __future__ import braces', 1, 1)
         check('x=1\nfrom __future__ import division', 2, 1)
-        check('(yield i) = 2', 1, 1)
+        check('foo(1=2)', 1, 5)
+        check('def f():\n  x, y: int', 2, 3)
+        check('[*x for x in xs]', 1, 2)
+        check('foo(x for x in range(10), 100)', 1, 5)
+        check('(yield i) = 2', 1, 1 if support.use_old_parser() else 2)
         check('def f(*):\n  pass', 1, 7 if support.use_old_parser() else 8)
-        check('foo(1=2)', 1, 5 if support.use_old_parser() else 6)
-
-    @support.skip_if_new_parser("Pegen column offsets might be different")
-    def testSyntaxErrorOffsetCustom(self):
-        self.check('for 1 in []: pass', 1, 5)
-        self.check('[*x for x in xs]', 1, 2)
-        self.check('def f():\n  x, y: int', 2, 3)
-        self.check('foo(x for x in range(10), 100)', 1, 5)
+        check('for 1 in []: pass', 1, 5 if support.use_old_parser() else 7)
 
     @cpython_only
     def testSettingException(self):

diff --git a/Parser/pegen/parse.c b/Parser/pegen/parse.c
index a1a6f4c06bf63d1e35579a89b089e290b29018db..b1b248187ea3ed988b2adc03399e92a2ac5464ad 100644 (file)
--- a/Parser/pegen/parse.c
+++ b/Parser/pegen/parse.c
@@ -346,12 +346,12 @@ static KeywordToken *reserved_keywords[] = {
 #define _loop0_124_type 1275
 #define _gather_123_type 1276
 #define _tmp_125_type 1277
-#define _tmp_126_type 1278
+#define _loop0_126_type 1278
 #define _tmp_127_type 1279
 #define _tmp_128_type 1280
 #define _tmp_129_type 1281
-#define _loop0_130_type 1282
-#define _tmp_131_type 1283
+#define _tmp_130_type 1282
+#define _loop0_131_type 1283
 #define _tmp_132_type 1284
 #define _tmp_133_type 1285
 #define _tmp_134_type 1286
@@ -365,9 +365,10 @@ static KeywordToken *reserved_keywords[] = {
 #define _tmp_142_type 1294
 #define _tmp_143_type 1295
 #define _tmp_144_type 1296
-#define _loop1_145_type 1297
-#define _tmp_146_type 1298
+#define _tmp_145_type 1297
+#define _loop1_146_type 1298
 #define _tmp_147_type 1299
+#define _tmp_148_type 1300
 
 static mod_ty file_rule(Parser *p);
 static mod_ty interactive_rule(Parser *p);
@@ -647,12 +648,12 @@ static asdl_seq *_gather_121_rule(Parser *p);
 static asdl_seq *_loop0_124_rule(Parser *p);
 static asdl_seq *_gather_123_rule(Parser *p);
 static void *_tmp_125_rule(Parser *p);
-static void *_tmp_126_rule(Parser *p);
+static asdl_seq *_loop0_126_rule(Parser *p);
 static void *_tmp_127_rule(Parser *p);
 static void *_tmp_128_rule(Parser *p);
 static void *_tmp_129_rule(Parser *p);
-static asdl_seq *_loop0_130_rule(Parser *p);
-static void *_tmp_131_rule(Parser *p);
+static void *_tmp_130_rule(Parser *p);
+static asdl_seq *_loop0_131_rule(Parser *p);
 static void *_tmp_132_rule(Parser *p);
 static void *_tmp_133_rule(Parser *p);
 static void *_tmp_134_rule(Parser *p);
@@ -666,9 +667,10 @@ static void *_tmp_141_rule(Parser *p);
 static void *_tmp_142_rule(Parser *p);
 static void *_tmp_143_rule(Parser *p);
 static void *_tmp_144_rule(Parser *p);
-static asdl_seq *_loop1_145_rule(Parser *p);
-static void *_tmp_146_rule(Parser *p);
+static void *_tmp_145_rule(Parser *p);
+static asdl_seq *_loop1_146_rule(Parser *p);
 static void *_tmp_147_rule(Parser *p);
+static void *_tmp_148_rule(Parser *p);
 
 
 // file: statements? $
@@ -10629,10 +10631,10 @@ incorrect_arguments_rule(Parser *p)
         Token * _literal;
         void *_opt_var;
         UNUSED(_opt_var); // Silence compiler warnings
-        expr_ty expression_var;
+        expr_ty a;
         asdl_seq* for_if_clauses_var;
         if (
-            (expression_var = expression_rule(p))  // expression
+            (a = expression_rule(p))  // expression
             &&
             (for_if_clauses_var = for_if_clauses_rule(p))  // for_if_clauses
             &&
@@ -10641,7 +10643,7 @@ incorrect_arguments_rule(Parser *p)
             (_opt_var = _tmp_125_rule(p), 1)  // [args | expression for_if_clauses]
         )
         {
-            _res = RAISE_SYNTAX_ERROR ( "Generator expression must be parenthesized" );
+            _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "Generator expression must be parenthesized" );
             if (_res == NULL && PyErr_Occurred()) {
                 p->error_indicator = 1;
                 return NULL;
@@ -10687,14 +10689,14 @@ invalid_kwarg_rule(Parser *p)
     int _mark = p->mark;
     { // expression '='
         Token * _literal;
-        expr_ty expression_var;
+        expr_ty a;
         if (
-            (expression_var = expression_rule(p))  // expression
+            (a = expression_rule(p))  // expression
             &&
             (_literal = _PyPegen_expect_token(p, 22))  // token='='
         )
         {
-            _res = RAISE_SYNTAX_ERROR ( "expression cannot contain assignment, perhaps you meant \"==\"?" );
+            _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "expression cannot contain assignment, perhaps you meant \"==\"?" );
             if (_res == NULL && PyErr_Occurred()) {
                 p->error_indicator = 1;
                 return NULL;
@@ -10729,7 +10731,7 @@ invalid_named_expression_rule(Parser *p)
             (expression_var = expression_rule(p))  // expression
         )
         {
-            _res = RAISE_SYNTAX_ERROR ( "cannot use assignment expressions with %s" , _PyPegen_get_expr_name ( a ) );
+            _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "cannot use assignment expressions with %s" , _PyPegen_get_expr_name ( a ) );
             if (_res == NULL && PyErr_Occurred()) {
                 p->error_indicator = 1;
                 return NULL;
@@ -10746,6 +10748,7 @@ invalid_named_expression_rule(Parser *p)
 // invalid_assignment:
 //     | list ':'
 //     | tuple ':'
+//     | star_named_expression ',' star_named_expressions* ':'
 //     | expression ':' expression ['=' annotated_rhs]
 //     | expression ('=' | augassign) (yield_expr | star_expressions)
 static void *
@@ -10758,14 +10761,14 @@ invalid_assignment_rule(Parser *p)
     int _mark = p->mark;
     { // list ':'
         Token * _literal;
-        expr_ty list_var;
+        expr_ty a;
         if (
-            (list_var = list_rule(p))  // list
+            (a = list_rule(p))  // list
             &&
             (_literal = _PyPegen_expect_token(p, 11))  // token=':'
         )
         {
-            _res = RAISE_SYNTAX_ERROR ( "only single target (not list) can be annotated" );
+            _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "only single target (not list) can be annotated" );
             if (_res == NULL && PyErr_Occurred()) {
                 p->error_indicator = 1;
                 return NULL;
@@ -10776,14 +10779,38 @@ invalid_assignment_rule(Parser *p)
     }
     { // tuple ':'
         Token * _literal;
-        expr_ty tuple_var;
+        expr_ty a;
         if (
-            (tuple_var = tuple_rule(p))  // tuple
+            (a = tuple_rule(p))  // tuple
             &&
             (_literal = _PyPegen_expect_token(p, 11))  // token=':'
         )
         {
-            _res = RAISE_SYNTAX_ERROR ( "only single target (not tuple) can be annotated" );
+            _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "only single target (not tuple) can be annotated" );
+            if (_res == NULL && PyErr_Occurred()) {
+                p->error_indicator = 1;
+                return NULL;
+            }
+            goto done;
+        }
+        p->mark = _mark;
+    }
+    { // star_named_expression ',' star_named_expressions* ':'
+        Token * _literal;
+        Token * _literal_1;
+        asdl_seq * _loop0_126_var;
+        expr_ty a;
+        if (
+            (a = star_named_expression_rule(p))  // star_named_expression
+            &&
+            (_literal = _PyPegen_expect_token(p, 12))  // token=','
+            &&
+            (_loop0_126_var = _loop0_126_rule(p))  // star_named_expressions*
+            &&
+            (_literal_1 = _PyPegen_expect_token(p, 11))  // token=':'
+        )
+        {
+            _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "only single target (not tuple) can be annotated" );
             if (_res == NULL && PyErr_Occurred()) {
                 p->error_indicator = 1;
                 return NULL;
@@ -10796,19 +10823,19 @@ invalid_assignment_rule(Parser *p)
         Token * _literal;
         void *_opt_var;
         UNUSED(_opt_var); // Silence compiler warnings
+        expr_ty a;
         expr_ty expression_var;
-        expr_ty expression_var_1;
         if (
-            (expression_var = expression_rule(p))  // expression
+            (a = expression_rule(p))  // expression
             &&
             (_literal = _PyPegen_expect_token(p, 11))  // token=':'
             &&
-            (expression_var_1 = expression_rule(p))  // expression
+            (expression_var = expression_rule(p))  // expression
             &&
-            (_opt_var = _tmp_126_rule(p), 1)  // ['=' annotated_rhs]
+            (_opt_var = _tmp_127_rule(p), 1)  // ['=' annotated_rhs]
         )
         {
-            _res = RAISE_SYNTAX_ERROR ( "illegal target for annotation" );
+            _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "illegal target for annotation" );
             if (_res == NULL && PyErr_Occurred()) {
                 p->error_indicator = 1;
                 return NULL;
@@ -10818,18 +10845,18 @@ invalid_assignment_rule(Parser *p)
         p->mark = _mark;
     }
     { // expression ('=' | augassign) (yield_expr | star_expressions)
-        void *_tmp_127_var;
         void *_tmp_128_var;
+        void *_tmp_129_var;
         expr_ty a;
         if (
             (a = expression_rule(p))  // expression
             &&
-            (_tmp_127_var = _tmp_127_rule(p))  // '=' | augassign
+            (_tmp_128_var = _tmp_128_rule(p))  // '=' | augassign
             &&
-            (_tmp_128_var = _tmp_128_rule(p))  // yield_expr | star_expressions
+            (_tmp_129_var = _tmp_129_rule(p))  // yield_expr | star_expressions
         )
         {
-            _res = RAISE_SYNTAX_ERROR_NO_COL_OFFSET ( "cannot assign to %s" , _PyPegen_get_expr_name ( a ) );
+            _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "cannot assign to %s" , _PyPegen_get_expr_name ( a ) );
             if (_res == NULL && PyErr_Occurred()) {
                 p->error_indicator = 1;
                 return NULL;
@@ -10874,7 +10901,7 @@ invalid_block_rule(Parser *p)
     return _res;
 }
 
-// invalid_comprehension: ('[' | '(' | '{') '*' expression for_if_clauses
+// invalid_comprehension: ('[' | '(' | '{') starred_expression for_if_clauses
 static void *
 invalid_comprehension_rule(Parser *p)
 {
@@ -10883,22 +10910,19 @@ invalid_comprehension_rule(Parser *p)
     }
     void * _res = NULL;
     int _mark = p->mark;
-    { // ('[' | '(' | '{') '*' expression for_if_clauses
-        Token * _literal;
-        void *_tmp_129_var;
-        expr_ty expression_var;
+    { // ('[' | '(' | '{') starred_expression for_if_clauses
+        void *_tmp_130_var;
+        expr_ty a;
         asdl_seq* for_if_clauses_var;
         if (
-            (_tmp_129_var = _tmp_129_rule(p))  // '[' | '(' | '{'
-            &&
-            (_literal = _PyPegen_expect_token(p, 16))  // token='*'
+            (_tmp_130_var = _tmp_130_rule(p))  // '[' | '(' | '{'
             &&
-            (expression_var = expression_rule(p))  // expression
+            (a = starred_expression_rule(p))  // starred_expression
             &&
             (for_if_clauses_var = for_if_clauses_rule(p))  // for_if_clauses
         )
         {
-            _res = RAISE_SYNTAX_ERROR ( "iterable unpacking cannot be used in comprehension" );
+            _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "iterable unpacking cannot be used in comprehension" );
             if (_res == NULL && PyErr_Occurred()) {
                 p->error_indicator = 1;
                 return NULL;
@@ -10923,13 +10947,13 @@ invalid_parameters_rule(Parser *p)
     void * _res = NULL;
     int _mark = p->mark;
     { // param_no_default* (slash_with_default | param_with_default+) param_no_default
-        asdl_seq * _loop0_130_var;
-        void *_tmp_131_var;
+        asdl_seq * _loop0_131_var;
+        void *_tmp_132_var;
         arg_ty param_no_default_var;
         if (
-            (_loop0_130_var = _loop0_130_rule(p))  // param_no_default*
+            (_loop0_131_var = _loop0_131_rule(p))  // param_no_default*
             &&
-            (_tmp_131_var = _tmp_131_rule(p))  // slash_with_default | param_with_default+
+            (_tmp_132_var = _tmp_132_rule(p))  // slash_with_default | param_with_default+
             &&
             (param_no_default_var = param_no_default_rule(p))  // param_no_default
         )
@@ -10959,11 +10983,11 @@ invalid_star_etc_rule(Parser *p)
     int _mark = p->mark;
     { // '*' (')' | ',' (')' | '**'))
         Token * _literal;
-        void *_tmp_132_var;
+        void *_tmp_133_var;
         if (
             (_literal = _PyPegen_expect_token(p, 16))  // token='*'
             &&
-            (_tmp_132_var = _tmp_132_rule(p))  // ')' | ',' (')' | '**')
+            (_tmp_133_var = _tmp_133_rule(p))  // ')' | ',' (')' | '**')
         )
         {
             _res = RAISE_SYNTAX_ERROR ( "named arguments must follow bare *" );
@@ -10991,11 +11015,11 @@ invalid_lambda_star_etc_rule(Parser *p)
     int _mark = p->mark;
     { // '*' (':' | ',' (':' | '**'))
         Token * _literal;
-        void *_tmp_133_var;
+        void *_tmp_134_var;
         if (
             (_literal = _PyPegen_expect_token(p, 16))  // token='*'
             &&
-            (_tmp_133_var = _tmp_133_rule(p))  // ':' | ',' (':' | '**')
+            (_tmp_134_var = _tmp_134_rule(p))  // ':' | ',' (':' | '**')
         )
         {
             _res = RAISE_SYNTAX_ERROR ( "named arguments must follow bare *" );
@@ -11070,7 +11094,7 @@ invalid_del_target_rule(Parser *p)
             _PyPegen_lookahead(1, del_target_end_rule, p)
         )
         {
-            _res = RAISE_SYNTAX_ERROR ( "cannot delete %s" , _PyPegen_get_expr_name ( a ) );
+            _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "cannot delete %s" , _PyPegen_get_expr_name ( a ) );
             if (_res == NULL && PyErr_Occurred()) {
                 p->error_indicator = 1;
                 return NULL;
@@ -11979,12 +12003,12 @@ _loop1_22_rule(Parser *p)
     ssize_t _children_capacity = 1;
     ssize_t _n = 0;
     { // (star_targets '=')
-        void *_tmp_134_var;
+        void *_tmp_135_var;
         while (
-            (_tmp_134_var = _tmp_134_rule(p))  // star_targets '='
+            (_tmp_135_var = _tmp_135_rule(p))  // star_targets '='
         )
         {
-            _res = _tmp_134_var;
+            _res = _tmp_135_var;
             if (_n == _children_capacity) {
                 _children_capacity *= 2;
                 _children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));
@@ -12306,12 +12330,12 @@ _loop0_30_rule(Parser *p)
     ssize_t _children_capacity = 1;
     ssize_t _n = 0;
     { // ('.' | '...')
-        void *_tmp_135_var;
+        void *_tmp_136_var;
         while (
-            (_tmp_135_var = _tmp_135_rule(p))  // '.' | '...'
+            (_tmp_136_var = _tmp_136_rule(p))  // '.' | '...'
         )
         {
-            _res = _tmp_135_var;
+            _res = _tmp_136_var;
             if (_n == _children_capacity) {
                 _children_capacity *= 2;
                 _children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));
@@ -12355,12 +12379,12 @@ _loop1_31_rule(Parser *p)
     ssize_t _children_capacity = 1;
     ssize_t _n = 0;
     { // ('.' | '...')
-        void *_tmp_136_var;
+        void *_tmp_137_var;
         while (
-            (_tmp_136_var = _tmp_136_rule(p))  // '.' | '...'
+            (_tmp_137_var = _tmp_137_rule(p))  // '.' | '...'
         )
         {
-            _res = _tmp_136_var;
+            _res = _tmp_137_var;
             if (_n == _children_capacity) {
                 _children_capacity *= 2;
                 _children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));
@@ -13937,12 +13961,12 @@ _loop1_67_rule(Parser *p)
     ssize_t _children_capacity = 1;
     ssize_t _n = 0;
     { // ('@' named_expression NEWLINE)
-        void *_tmp_137_var;
+        void *_tmp_138_var;
         while (
-            (_tmp_137_var = _tmp_137_rule(p))  // '@' named_expression NEWLINE
+            (_tmp_138_var = _tmp_138_rule(p))  // '@' named_expression NEWLINE
         )
         {
-            _res = _tmp_137_var;
+            _res = _tmp_138_var;
             if (_n == _children_capacity) {
                 _children_capacity *= 2;
                 _children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));
@@ -14110,12 +14134,12 @@ _loop1_71_rule(Parser *p)
     ssize_t _children_capacity = 1;
     ssize_t _n = 0;
     { // (',' star_expression)
-        void *_tmp_138_var;
+        void *_tmp_139_var;
         while (
-            (_tmp_138_var = _tmp_138_rule(p))  // ',' star_expression
+            (_tmp_139_var = _tmp_139_rule(p))  // ',' star_expression
         )
         {
-            _res = _tmp_138_var;
+            _res = _tmp_139_var;
             if (_n == _children_capacity) {
                 _children_capacity *= 2;
                 _children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));
@@ -14248,12 +14272,12 @@ _loop1_74_rule(Parser *p)
     ssize_t _children_capacity = 1;
     ssize_t _n = 0;
     { // (',' expression)
-        void *_tmp_139_var;
+        void *_tmp_140_var;
         while (
-            (_tmp_139_var = _tmp_139_rule(p))  // ',' expression
+            (_tmp_140_var = _tmp_140_rule(p))  // ',' expression
         )
         {
-            _res = _tmp_139_var;
+            _res = _tmp_140_var;
             if (_n == _children_capacity) {
                 _children_capacity *= 2;
                 _children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));
@@ -15015,12 +15039,12 @@ _loop1_89_rule(Parser *p)
     ssize_t _children_capacity = 1;
     ssize_t _n = 0;
     { // ('or' conjunction)
-        void *_tmp_140_var;
+        void *_tmp_141_var;
         while (
-            (_tmp_140_var = _tmp_140_rule(p))  // 'or' conjunction
+            (_tmp_141_var = _tmp_141_rule(p))  // 'or' conjunction
         )
         {
-            _res = _tmp_140_var;
+            _res = _tmp_141_var;
             if (_n == _children_capacity) {
                 _children_capacity *= 2;
                 _children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));
@@ -15068,12 +15092,12 @@ _loop1_90_rule(Parser *p)
     ssize_t _children_capacity = 1;
     ssize_t _n = 0;
     { // ('and' inversion)
-        void *_tmp_141_var;
+        void *_tmp_142_var;
         while (
-            (_tmp_141_var = _tmp_141_rule(p))  // 'and' inversion
+            (_tmp_142_var = _tmp_142_rule(p))  // 'and' inversion
         )
         {
-            _res = _tmp_141_var;
+            _res = _tmp_142_var;
             if (_n == _children_capacity) {
                 _children_capacity *= 2;
                 _children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));
@@ -15723,12 +15747,12 @@ _loop0_105_rule(Parser *p)
     ssize_t _children_capacity = 1;
     ssize_t _n = 0;
     { // ('if' disjunction)
-        void *_tmp_142_var;
+        void *_tmp_143_var;
         while (
-            (_tmp_142_var = _tmp_142_rule(p))  // 'if' disjunction
+            (_tmp_143_var = _tmp_143_rule(p))  // 'if' disjunction
         )
         {
-            _res = _tmp_142_var;
+            _res = _tmp_143_var;
             if (_n == _children_capacity) {
                 _children_capacity *= 2;
                 _children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));
@@ -15772,12 +15796,12 @@ _loop0_106_rule(Parser *p)
     ssize_t _children_capacity = 1;
     ssize_t _n = 0;
     { // ('if' disjunction)
-        void *_tmp_143_var;
+        void *_tmp_144_var;
         while (
-            (_tmp_143_var = _tmp_143_rule(p))  // 'if' disjunction
+            (_tmp_144_var = _tmp_144_rule(p))  // 'if' disjunction
         )
         {
-            _res = _tmp_143_var;
+            _res = _tmp_144_var;
             if (_n == _children_capacity) {
                 _children_capacity *= 2;
                 _children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));
@@ -16225,12 +16249,12 @@ _loop0_117_rule(Parser *p)
     ssize_t _children_capacity = 1;
     ssize_t _n = 0;
     { // (',' star_target)
-        void *_tmp_144_var;
+        void *_tmp_145_var;
         while (
-            (_tmp_144_var = _tmp_144_rule(p))  // ',' star_target
+            (_tmp_145_var = _tmp_145_rule(p))  // ',' star_target
         )
         {
-            _res = _tmp_144_var;
+            _res = _tmp_145_var;
             if (_n == _children_capacity) {
                 _children_capacity *= 2;
                 _children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));
@@ -16577,9 +16601,58 @@ _tmp_125_rule(Parser *p)
     return _res;
 }
 
-// _tmp_126: '=' annotated_rhs
+// _loop0_126: star_named_expressions
+static asdl_seq *
+_loop0_126_rule(Parser *p)
+{
+    if (p->error_indicator) {
+        return NULL;
+    }
+    void *_res = NULL;
+    int _mark = p->mark;
+    int _start_mark = p->mark;
+    void **_children = PyMem_Malloc(sizeof(void *));
+    if (!_children) {
+        PyErr_Format(PyExc_MemoryError, "Parser out of memory");
+        return NULL;
+    }
+    ssize_t _children_capacity = 1;
+    ssize_t _n = 0;
+    { // star_named_expressions
+        asdl_seq* star_named_expressions_var;
+        while (
+            (star_named_expressions_var = star_named_expressions_rule(p))  // star_named_expressions
+        )
+        {
+            _res = star_named_expressions_var;
+            if (_n == _children_capacity) {
+                _children_capacity *= 2;
+                _children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));
+                if (!_children) {
+                    PyErr_Format(PyExc_MemoryError, "realloc None");
+                    return NULL;
+                }
+            }
+            _children[_n++] = _res;
+            _mark = p->mark;
+        }
+        p->mark = _mark;
+    }
+    asdl_seq *_seq = _Py_asdl_seq_new(_n, p->arena);
+    if (!_seq) {
+        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_126");
+        PyMem_Free(_children);
+        return NULL;
+    }
+    for (int i = 0; i < _n; i++) asdl_seq_SET(_seq, i, _children[i]);
+    PyMem_Free(_children);
+    _PyPegen_insert_memo(p, _start_mark, _loop0_126_type, _seq);
+    return _seq;
+}
+
+// _tmp_127: '=' annotated_rhs
 static void *
-_tmp_126_rule(Parser *p)
+_tmp_127_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -16605,9 +16678,9 @@ _tmp_126_rule(Parser *p)
     return _res;
 }
 
-// _tmp_127: '=' | augassign
+// _tmp_128: '=' | augassign
 static void *
-_tmp_127_rule(Parser *p)
+_tmp_128_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -16641,9 +16714,9 @@ _tmp_127_rule(Parser *p)
     return _res;
 }
 
-// _tmp_128: yield_expr | star_expressions
+// _tmp_129: yield_expr | star_expressions
 static void *
-_tmp_128_rule(Parser *p)
+_tmp_129_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -16677,9 +16750,9 @@ _tmp_128_rule(Parser *p)
     return _res;
 }
 
-// _tmp_129: '[' | '(' | '{'
+// _tmp_130: '[' | '(' | '{'
 static void *
-_tmp_129_rule(Parser *p)
+_tmp_130_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -16724,9 +16797,9 @@ _tmp_129_rule(Parser *p)
     return _res;
 }
 
-// _loop0_130: param_no_default
+// _loop0_131: param_no_default
 static asdl_seq *
-_loop0_130_rule(Parser *p)
+_loop0_131_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -16763,19 +16836,19 @@ _loop0_130_rule(Parser *p)
     }
     asdl_seq *_seq = _Py_asdl_seq_new(_n, p->arena);
     if (!_seq) {
-        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_130");
+        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_131");
         PyMem_Free(_children);
         return NULL;
     }
     for (int i = 0; i < _n; i++) asdl_seq_SET(_seq, i, _children[i]);
     PyMem_Free(_children);
-    _PyPegen_insert_memo(p, _start_mark, _loop0_130_type, _seq);
+    _PyPegen_insert_memo(p, _start_mark, _loop0_131_type, _seq);
     return _seq;
 }
 
-// _tmp_131: slash_with_default | param_with_default+
+// _tmp_132: slash_with_default | param_with_default+
 static void *
-_tmp_131_rule(Parser *p)
+_tmp_132_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -16794,12 +16867,12 @@ _tmp_131_rule(Parser *p)
         p->mark = _mark;
     }
     { // param_with_default+
-        asdl_seq * _loop1_145_var;
+        asdl_seq * _loop1_146_var;
         if (
-            (_loop1_145_var = _loop1_145_rule(p))  // param_with_default+
+            (_loop1_146_var = _loop1_146_rule(p))  // param_with_default+
         )
         {
-            _res = _loop1_145_var;
+            _res = _loop1_146_var;
             goto done;
         }
         p->mark = _mark;
@@ -16809,9 +16882,9 @@ _tmp_131_rule(Parser *p)
     return _res;
 }
 
-// _tmp_132: ')' | ',' (')' | '**')
+// _tmp_133: ')' | ',' (')' | '**')
 static void *
-_tmp_132_rule(Parser *p)
+_tmp_133_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -16831,14 +16904,14 @@ _tmp_132_rule(Parser *p)
     }
     { // ',' (')' | '**')
         Token * _literal;
-        void *_tmp_146_var;
+        void *_tmp_147_var;
         if (
             (_literal = _PyPegen_expect_token(p, 12))  // token=','
             &&
-            (_tmp_146_var = _tmp_146_rule(p))  // ')' | '**'
+            (_tmp_147_var = _tmp_147_rule(p))  // ')' | '**'
         )
         {
-            _res = _PyPegen_dummy_name(p, _literal, _tmp_146_var);
+            _res = _PyPegen_dummy_name(p, _literal, _tmp_147_var);
             goto done;
         }
         p->mark = _mark;
@@ -16848,9 +16921,9 @@ _tmp_132_rule(Parser *p)
     return _res;
 }
 
-// _tmp_133: ':' | ',' (':' | '**')
+// _tmp_134: ':' | ',' (':' | '**')
 static void *
-_tmp_133_rule(Parser *p)
+_tmp_134_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -16870,14 +16943,14 @@ _tmp_133_rule(Parser *p)
     }
     { // ',' (':' | '**')
         Token * _literal;
-        void *_tmp_147_var;
+        void *_tmp_148_var;
         if (
             (_literal = _PyPegen_expect_token(p, 12))  // token=','
             &&
-            (_tmp_147_var = _tmp_147_rule(p))  // ':' | '**'
+            (_tmp_148_var = _tmp_148_rule(p))  // ':' | '**'
         )
         {
-            _res = _PyPegen_dummy_name(p, _literal, _tmp_147_var);
+            _res = _PyPegen_dummy_name(p, _literal, _tmp_148_var);
             goto done;
         }
         p->mark = _mark;
@@ -16887,9 +16960,9 @@ _tmp_133_rule(Parser *p)
     return _res;
 }
 
-// _tmp_134: star_targets '='
+// _tmp_135: star_targets '='
 static void *
-_tmp_134_rule(Parser *p)
+_tmp_135_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -16919,9 +16992,9 @@ _tmp_134_rule(Parser *p)
     return _res;
 }
 
-// _tmp_135: '.' | '...'
+// _tmp_136: '.' | '...'
 static void *
-_tmp_135_rule(Parser *p)
+_tmp_136_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -16955,9 +17028,9 @@ _tmp_135_rule(Parser *p)
     return _res;
 }
 
-// _tmp_136: '.' | '...'
+// _tmp_137: '.' | '...'
 static void *
-_tmp_136_rule(Parser *p)
+_tmp_137_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -16991,9 +17064,9 @@ _tmp_136_rule(Parser *p)
     return _res;
 }
 
-// _tmp_137: '@' named_expression NEWLINE
+// _tmp_138: '@' named_expression NEWLINE
 static void *
-_tmp_137_rule(Parser *p)
+_tmp_138_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -17026,9 +17099,9 @@ _tmp_137_rule(Parser *p)
     return _res;
 }
 
-// _tmp_138: ',' star_expression
+// _tmp_139: ',' star_expression
 static void *
-_tmp_138_rule(Parser *p)
+_tmp_139_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -17058,9 +17131,9 @@ _tmp_138_rule(Parser *p)
     return _res;
 }
 
-// _tmp_139: ',' expression
+// _tmp_140: ',' expression
 static void *
-_tmp_139_rule(Parser *p)
+_tmp_140_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -17090,9 +17163,9 @@ _tmp_139_rule(Parser *p)
     return _res;
 }
 
-// _tmp_140: 'or' conjunction
+// _tmp_141: 'or' conjunction
 static void *
-_tmp_140_rule(Parser *p)
+_tmp_141_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -17122,9 +17195,9 @@ _tmp_140_rule(Parser *p)
     return _res;
 }
 
-// _tmp_141: 'and' inversion
+// _tmp_142: 'and' inversion
 static void *
-_tmp_141_rule(Parser *p)
+_tmp_142_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -17154,9 +17227,9 @@ _tmp_141_rule(Parser *p)
     return _res;
 }
 
-// _tmp_142: 'if' disjunction
+// _tmp_143: 'if' disjunction
 static void *
-_tmp_142_rule(Parser *p)
+_tmp_143_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -17186,9 +17259,9 @@ _tmp_142_rule(Parser *p)
     return _res;
 }
 
-// _tmp_143: 'if' disjunction
+// _tmp_144: 'if' disjunction
 static void *
-_tmp_143_rule(Parser *p)
+_tmp_144_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -17218,9 +17291,9 @@ _tmp_143_rule(Parser *p)
     return _res;
 }
 
-// _tmp_144: ',' star_target
+// _tmp_145: ',' star_target
 static void *
-_tmp_144_rule(Parser *p)
+_tmp_145_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -17250,9 +17323,9 @@ _tmp_144_rule(Parser *p)
     return _res;
 }
 
-// _loop1_145: param_with_default
+// _loop1_146: param_with_default
 static asdl_seq *
-_loop1_145_rule(Parser *p)
+_loop1_146_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -17293,19 +17366,19 @@ _loop1_145_rule(Parser *p)
     }
     asdl_seq *_seq = _Py_asdl_seq_new(_n, p->arena);
     if (!_seq) {
-        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop1_145");
+        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop1_146");
         PyMem_Free(_children);
         return NULL;
     }
     for (int i = 0; i < _n; i++) asdl_seq_SET(_seq, i, _children[i]);
     PyMem_Free(_children);
-    _PyPegen_insert_memo(p, _start_mark, _loop1_145_type, _seq);
+    _PyPegen_insert_memo(p, _start_mark, _loop1_146_type, _seq);
     return _seq;
 }
 
-// _tmp_146: ')' | '**'
+// _tmp_147: ')' | '**'
 static void *
-_tmp_146_rule(Parser *p)
+_tmp_147_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -17339,9 +17412,9 @@ _tmp_146_rule(Parser *p)
     return _res;
 }
 
-// _tmp_147: ':' | '**'
+// _tmp_148: ':' | '**'
 static void *
-_tmp_147_rule(Parser *p)
+_tmp_148_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;

diff --git a/Parser/pegen/pegen.c b/Parser/pegen/pegen.c
index 5f8c862c1f88bea534f6fec2b2653815814ac56d..083088bd9657bd547f99c4869a73cb1fb0496a31 100644 (file)
--- a/Parser/pegen/pegen.c
+++ b/Parser/pegen/pegen.c
@@ -380,48 +380,57 @@ tokenizer_error(Parser *p)
 }
 
 void *
-_PyPegen_raise_error(Parser *p, PyObject *errtype, int with_col_number, const char *errmsg, ...)
+_PyPegen_raise_error(Parser *p, PyObject *errtype, const char *errmsg, ...)
+{
+    Token *t = p->known_err_token != NULL ? p->known_err_token : p->tokens[p->fill - 1];
+    int col_offset;
+    if (t->col_offset == -1) {
+        col_offset = Py_SAFE_DOWNCAST(p->tok->cur - p->tok->buf,
+                                      intptr_t, int);
+    } else {
+        col_offset = t->col_offset + 1;
+    }
+
+    va_list va;
+    va_start(va, errmsg);
+    _PyPegen_raise_error_known_location(p, errtype, t->lineno,
+                                        col_offset, errmsg, va);
+    va_end(va);
+
+    return NULL;
+}
+
+
+void *
+_PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
+                                    int lineno, int col_offset,
+                                    const char *errmsg, va_list va)
 {
     PyObject *value = NULL;
     PyObject *errstr = NULL;
-    PyObject *loc = NULL;
+    PyObject *error_line = NULL;
     PyObject *tmp = NULL;
-    Token *t = p->known_err_token != NULL ? p->known_err_token : p->tokens[p->fill - 1];
-    Py_ssize_t col_number = !with_col_number;
-    va_list va;
     p->error_indicator = 1;
 
-    va_start(va, errmsg);
     errstr = PyUnicode_FromFormatV(errmsg, va);
-    va_end(va);
     if (!errstr) {
         goto error;
     }
 
     if (p->start_rule == Py_file_input) {
-        loc = PyErr_ProgramTextObject(p->tok->filename, t->lineno);
+        error_line = PyErr_ProgramTextObject(p->tok->filename, lineno);
     }
 
-    if (!loc) {
-        loc = get_error_line(p->tok->buf, p->start_rule == Py_file_input);
-    }
-
-    if (loc && with_col_number) {
-        int col_offset;
-        if (t->col_offset == -1) {
-            col_offset = Py_SAFE_DOWNCAST(p->tok->cur - p->tok->buf,
-                                          intptr_t, int);
-        } else {
-            col_offset = t->col_offset + 1;
+    if (!error_line) {
+        error_line = get_error_line(p->tok->buf, p->start_rule == Py_file_input);
+        if (!error_line) {
+            goto error;
         }
-        col_number = byte_offset_to_character_offset(loc, col_offset);
-    }
-    else if (!loc) {
-        Py_INCREF(Py_None);
-        loc = Py_None;
     }
 
-    tmp = Py_BuildValue("(OiiN)", p->tok->filename, t->lineno, col_number, loc);
+    int col_number = byte_offset_to_character_offset(error_line, col_offset);
+
+    tmp = Py_BuildValue("(OiiN)", p->tok->filename, lineno, col_number, error_line);
     if (!tmp) {
         goto error;
     }
@@ -438,7 +447,7 @@ _PyPegen_raise_error(Parser *p, PyObject *errtype, int with_col_number, const ch
 
 error:
     Py_XDECREF(errstr);
-    Py_XDECREF(loc);
+    Py_XDECREF(error_line);
     return NULL;
 }
 

diff --git a/Parser/pegen/pegen.h b/Parser/pegen/pegen.h
index b55a652ac8060d97b2572b45f88341d64a90a396..e5b1b757bd894bf63da25032e1c285767e377f5a 100644 (file)
--- a/Parser/pegen/pegen.h
+++ b/Parser/pegen/pegen.h
@@ -127,15 +127,32 @@ expr_ty _PyPegen_name_token(Parser *p);
 expr_ty _PyPegen_number_token(Parser *p);
 void *_PyPegen_string_token(Parser *p);
 const char *_PyPegen_get_expr_name(expr_ty);
-void *_PyPegen_raise_error(Parser *p, PyObject *errtype, int with_col_number, const char *errmsg, ...);
+void *_PyPegen_raise_error(Parser *p, PyObject *errtype, const char *errmsg, ...);
+void *_PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
+                                          int lineno, int col_offset,
+                                          const char *errmsg, va_list va);
 void *_PyPegen_dummy_name(Parser *p, ...);
 
+Py_LOCAL_INLINE(void *)
+RAISE_ERROR_KNOWN_LOCATION(Parser *p, PyObject *errtype, int lineno,
+                           int col_offset, const char *errmsg, ...)
+{
+    va_list va;
+    va_start(va, errmsg);
+    _PyPegen_raise_error_known_location(p, errtype, lineno, col_offset + 1,
+                                        errmsg, va);
+    va_end(va);
+    return NULL;
+}
+
+
 #define UNUSED(expr) do { (void)(expr); } while (0)
 #define EXTRA_EXPR(head, tail) head->lineno, head->col_offset, tail->end_lineno, tail->end_col_offset, p->arena
 #define EXTRA _start_lineno, _start_col_offset, _end_lineno, _end_col_offset, p->arena
-#define RAISE_SYNTAX_ERROR(msg, ...) _PyPegen_raise_error(p, PyExc_SyntaxError, 1, msg, ##__VA_ARGS__)
-#define RAISE_INDENTATION_ERROR(msg, ...) _PyPegen_raise_error(p, PyExc_IndentationError, 1, msg, ##__VA_ARGS__)
-#define RAISE_SYNTAX_ERROR_NO_COL_OFFSET(msg, ...) _PyPegen_raise_error(p, PyExc_SyntaxError, 0, msg, ##__VA_ARGS__)
+#define RAISE_SYNTAX_ERROR(msg, ...) _PyPegen_raise_error(p, PyExc_SyntaxError, msg, ##__VA_ARGS__)
+#define RAISE_INDENTATION_ERROR(msg, ...) _PyPegen_raise_error(p, PyExc_IndentationError, msg, ##__VA_ARGS__)
+#define RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, msg, ...) \
+    RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, a->lineno, a->col_offset, msg, ##__VA_ARGS__)
 
 Py_LOCAL_INLINE(void *)
 CHECK_CALL(Parser *p, void *result)