bpo-40334: Support CO_FUTURE_BARRY_AS_BDFL in the new parser (GH-19721)

This commit also allows to pass flags to the new parser in all interfaces and fixes a bug in the parser generator that was causing to inline rules with actions, making them disappear.

diff --git a/Grammar/python.gram b/Grammar/python.gram
index 40ca3dc8d12a5da23612824324abf8fdf9242afd..0ff2dcca884f1c1f480eb3695b261aaf8e965463 100644 (file)
--- a/Grammar/python.gram
+++ b/Grammar/python.gram
@@ -323,7 +323,8 @@ compare_op_bitwise_or_pair[CmpopExprPair*]:
     | isnot_bitwise_or
     | is_bitwise_or
 eq_bitwise_or[CmpopExprPair*]: '==' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, Eq, a) }
-noteq_bitwise_or[CmpopExprPair*]: '!=' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, NotEq, a) }
+noteq_bitwise_or[CmpopExprPair*]:
+    | (tok='!=' {_PyPegen_check_barry_as_flufl(p) ? NULL : tok}) a=bitwise_or {_PyPegen_cmpop_expr_pair(p, NotEq, a) }
 lte_bitwise_or[CmpopExprPair*]: '<=' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, LtE, a) }
 lt_bitwise_or[CmpopExprPair*]: '<' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, Lt, a) }
 gte_bitwise_or[CmpopExprPair*]: '>=' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, GtE, a) }

diff --git a/Include/internal/pegen_interface.h b/Include/internal/pegen_interface.h
index d8621c1a8892743e98107db7c3d398e543720610..adff7315681e369f7046c7572614def7b94a748e 100644 (file)
--- a/Include/internal/pegen_interface.h
+++ b/Include/internal/pegen_interface.h
@@ -11,21 +11,24 @@ extern "C" {
 #include "Python.h"
 #include "Python-ast.h"
 
-PyAPI_FUNC(mod_ty) PyPegen_ASTFromFile(const char *filename, int mode, PyArena *arena);
+PyAPI_FUNC(mod_ty) PyPegen_ASTFromFile(const char *filename, int mode, PyCompilerFlags*, PyArena *arena);
 PyAPI_FUNC(mod_ty) PyPegen_ASTFromString(const char *str, int mode, PyCompilerFlags *flags,
                                          PyArena *arena);
 PyAPI_FUNC(mod_ty) PyPegen_ASTFromStringObject(const char *str, PyObject* filename, int mode,
                                                PyCompilerFlags *flags, PyArena *arena);
 PyAPI_FUNC(mod_ty) PyPegen_ASTFromFileObject(FILE *fp, PyObject *filename_ob,
                                              int mode, const char *enc, const char *ps1,
-                                             const char *ps2, int *errcode, PyArena *arena);
-PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromFile(const char *filename, int mode);
+                                             const char *ps2, PyCompilerFlags *flags,
+                                             int *errcode, PyArena *arena);
+PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromFile(const char *filename, int mode, PyCompilerFlags *flags);
 PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromString(const char *str, int mode,
                                                         PyCompilerFlags *flags);
 PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromFileObject(FILE *, PyObject *filename_ob,
-                                                            int mode, const char *enc,
+                                                            int mode,
                                                             const char *ps1,
                                                             const char *ps2,
+                                                            PyCompilerFlags *flags,
+                                                            const char *enc,
                                                             int *errcode);
 
 #ifdef __cplusplus

diff --git a/Lib/test/test_flufl.py b/Lib/test/test_flufl.py
index 35ab934ab373d202e666b3147b9046d416b4d6b3..b71442804c72baa5f96de3317427e9e0fc2ff4b2 100644 (file)
--- a/Lib/test/test_flufl.py
+++ b/Lib/test/test_flufl.py
@@ -4,7 +4,6 @@ import sys
 from test import support
 
 
-@support.skip_if_new_parser("Not supported by pegen yet")
 class FLUFLTests(unittest.TestCase):
 
     def test_barry_as_bdfl(self):
@@ -16,10 +15,13 @@ class FLUFLTests(unittest.TestCase):
                     __future__.CO_FUTURE_BARRY_AS_BDFL)
         self.assertRegex(str(cm.exception),
                          "with Barry as BDFL, use '<>' instead of '!='")
-        self.assertEqual(cm.exception.text, '2 != 3\n')
+        self.assertIn('2 != 3', cm.exception.text)
         self.assertEqual(cm.exception.filename, '<FLUFL test>')
-        self.assertEqual(cm.exception.lineno, 2)
-        self.assertEqual(cm.exception.offset, 4)
+
+        self.assertTrue(cm.exception.lineno, 2)
+        # The old parser reports the end of the token and the new
+        # parser reports the start of the token
+        self.assertEqual(cm.exception.offset, 4 if support.use_old_parser() else 3)
 
     def test_guido_as_bdfl(self):
         code = '2 {0} 3'
@@ -27,10 +29,12 @@ class FLUFLTests(unittest.TestCase):
         with self.assertRaises(SyntaxError) as cm:
             compile(code.format('<>'), '<FLUFL test>', 'exec')
         self.assertRegex(str(cm.exception), "invalid syntax")
-        self.assertEqual(cm.exception.text, '2 <> 3\n')
+        self.assertIn('2 <> 3', cm.exception.text)
         self.assertEqual(cm.exception.filename, '<FLUFL test>')
         self.assertEqual(cm.exception.lineno, 1)
-        self.assertEqual(cm.exception.offset, 4)
+        # The old parser reports the end of the token and the new
+        # parser reports the start of the token
+        self.assertEqual(cm.exception.offset, 4 if support.use_old_parser() else 3)
 
 
 if __name__ == '__main__':

diff --git a/Modules/_peg_parser.c b/Modules/_peg_parser.c
index cb5f9aa63aea39f7d0bfa972ec4b0c532e8afe98..e1ec36e07bd57fd9cc7c62f9a99474828270de69 100644 (file)
--- a/Modules/_peg_parser.c
+++ b/Modules/_peg_parser.c
@@ -28,9 +28,10 @@ _Py_parse_file(PyObject *self, PyObject *args, PyObject *kwds)
         return NULL;
     }
 
+    PyCompilerFlags flags = _PyCompilerFlags_INIT;
     PyObject *result = NULL;
 
-    mod_ty res = PyPegen_ASTFromFile(filename, mode, arena);
+    mod_ty res = PyPegen_ASTFromFile(filename, mode, &flags, arena);
     if (res == NULL) {
         goto error;
     }

diff --git a/Parser/pegen/parse.c b/Parser/pegen/parse.c
index 25607eaf73cdcb63600ee19ea5ce7cac186d8e95..b26f7327bd273bfecfece4515bb4e80045142917 100644 (file)
--- a/Parser/pegen/parse.c
+++ b/Parser/pegen/parse.c
@@ -297,37 +297,37 @@ static KeywordToken *reserved_keywords[] = {
 #define _loop1_83_type 1226
 #define _loop1_84_type 1227
 #define _loop1_85_type 1228
-#define _loop0_87_type 1229
-#define _gather_86_type 1230
-#define _tmp_88_type 1231
+#define _tmp_86_type 1229
+#define _loop0_88_type 1230
+#define _gather_87_type 1231
 #define _tmp_89_type 1232
 #define _tmp_90_type 1233
 #define _tmp_91_type 1234
-#define _loop1_92_type 1235
-#define _tmp_93_type 1236
+#define _tmp_92_type 1235
+#define _loop1_93_type 1236
 #define _tmp_94_type 1237
-#define _loop0_96_type 1238
-#define _gather_95_type 1239
-#define _loop1_97_type 1240
-#define _tmp_98_type 1241
+#define _tmp_95_type 1238
+#define _loop0_97_type 1239
+#define _gather_96_type 1240
+#define _loop1_98_type 1241
 #define _tmp_99_type 1242
-#define _loop0_101_type 1243
-#define _gather_100_type 1244
-#define _loop0_103_type 1245
-#define _gather_102_type 1246
-#define _loop0_105_type 1247
-#define _gather_104_type 1248
-#define _loop0_107_type 1249
-#define _gather_106_type 1250
-#define _loop0_108_type 1251
-#define _loop0_110_type 1252
-#define _gather_109_type 1253
-#define _tmp_111_type 1254
-#define _loop0_113_type 1255
-#define _gather_112_type 1256
-#define _loop0_115_type 1257
-#define _gather_114_type 1258
-#define _tmp_116_type 1259
+#define _tmp_100_type 1243
+#define _loop0_102_type 1244
+#define _gather_101_type 1245
+#define _loop0_104_type 1246
+#define _gather_103_type 1247
+#define _loop0_106_type 1248
+#define _gather_105_type 1249
+#define _loop0_108_type 1250
+#define _gather_107_type 1251
+#define _loop0_109_type 1252
+#define _loop0_111_type 1253
+#define _gather_110_type 1254
+#define _tmp_112_type 1255
+#define _loop0_114_type 1256
+#define _gather_113_type 1257
+#define _loop0_116_type 1258
+#define _gather_115_type 1259
 #define _tmp_117_type 1260
 #define _tmp_118_type 1261
 #define _tmp_119_type 1262
@@ -346,8 +346,9 @@ static KeywordToken *reserved_keywords[] = {
 #define _tmp_132_type 1275
 #define _tmp_133_type 1276
 #define _tmp_134_type 1277
-#define _loop0_135_type 1278
-#define _tmp_136_type 1279
+#define _tmp_135_type 1278
+#define _loop0_136_type 1279
+#define _tmp_137_type 1280
 
 static mod_ty file_rule(Parser *p);
 static mod_ty interactive_rule(Parser *p);
@@ -578,37 +579,37 @@ static asdl_seq *_gather_81_rule(Parser *p);
 static asdl_seq *_loop1_83_rule(Parser *p);
 static asdl_seq *_loop1_84_rule(Parser *p);
 static asdl_seq *_loop1_85_rule(Parser *p);
-static asdl_seq *_loop0_87_rule(Parser *p);
-static asdl_seq *_gather_86_rule(Parser *p);
-static void *_tmp_88_rule(Parser *p);
+static void *_tmp_86_rule(Parser *p);
+static asdl_seq *_loop0_88_rule(Parser *p);
+static asdl_seq *_gather_87_rule(Parser *p);
 static void *_tmp_89_rule(Parser *p);
 static void *_tmp_90_rule(Parser *p);
 static void *_tmp_91_rule(Parser *p);
-static asdl_seq *_loop1_92_rule(Parser *p);
-static void *_tmp_93_rule(Parser *p);
+static void *_tmp_92_rule(Parser *p);
+static asdl_seq *_loop1_93_rule(Parser *p);
 static void *_tmp_94_rule(Parser *p);
-static asdl_seq *_loop0_96_rule(Parser *p);
-static asdl_seq *_gather_95_rule(Parser *p);
-static asdl_seq *_loop1_97_rule(Parser *p);
-static void *_tmp_98_rule(Parser *p);
+static void *_tmp_95_rule(Parser *p);
+static asdl_seq *_loop0_97_rule(Parser *p);
+static asdl_seq *_gather_96_rule(Parser *p);
+static asdl_seq *_loop1_98_rule(Parser *p);
 static void *_tmp_99_rule(Parser *p);
-static asdl_seq *_loop0_101_rule(Parser *p);
-static asdl_seq *_gather_100_rule(Parser *p);
-static asdl_seq *_loop0_103_rule(Parser *p);
-static asdl_seq *_gather_102_rule(Parser *p);
-static asdl_seq *_loop0_105_rule(Parser *p);
-static asdl_seq *_gather_104_rule(Parser *p);
-static asdl_seq *_loop0_107_rule(Parser *p);
-static asdl_seq *_gather_106_rule(Parser *p);
+static void *_tmp_100_rule(Parser *p);
+static asdl_seq *_loop0_102_rule(Parser *p);
+static asdl_seq *_gather_101_rule(Parser *p);
+static asdl_seq *_loop0_104_rule(Parser *p);
+static asdl_seq *_gather_103_rule(Parser *p);
+static asdl_seq *_loop0_106_rule(Parser *p);
+static asdl_seq *_gather_105_rule(Parser *p);
 static asdl_seq *_loop0_108_rule(Parser *p);
-static asdl_seq *_loop0_110_rule(Parser *p);
-static asdl_seq *_gather_109_rule(Parser *p);
-static void *_tmp_111_rule(Parser *p);
-static asdl_seq *_loop0_113_rule(Parser *p);
-static asdl_seq *_gather_112_rule(Parser *p);
-static asdl_seq *_loop0_115_rule(Parser *p);
-static asdl_seq *_gather_114_rule(Parser *p);
-static void *_tmp_116_rule(Parser *p);
+static asdl_seq *_gather_107_rule(Parser *p);
+static asdl_seq *_loop0_109_rule(Parser *p);
+static asdl_seq *_loop0_111_rule(Parser *p);
+static asdl_seq *_gather_110_rule(Parser *p);
+static void *_tmp_112_rule(Parser *p);
+static asdl_seq *_loop0_114_rule(Parser *p);
+static asdl_seq *_gather_113_rule(Parser *p);
+static asdl_seq *_loop0_116_rule(Parser *p);
+static asdl_seq *_gather_115_rule(Parser *p);
 static void *_tmp_117_rule(Parser *p);
 static void *_tmp_118_rule(Parser *p);
 static void *_tmp_119_rule(Parser *p);
@@ -627,8 +628,9 @@ static void *_tmp_131_rule(Parser *p);
 static void *_tmp_132_rule(Parser *p);
 static void *_tmp_133_rule(Parser *p);
 static void *_tmp_134_rule(Parser *p);
-static asdl_seq *_loop0_135_rule(Parser *p);
-static void *_tmp_136_rule(Parser *p);
+static void *_tmp_135_rule(Parser *p);
+static asdl_seq *_loop0_136_rule(Parser *p);
+static void *_tmp_137_rule(Parser *p);
 
 
 // file: statements? $
@@ -5557,7 +5559,7 @@ eq_bitwise_or_rule(Parser *p)
     return res;
 }
 
-// noteq_bitwise_or: '!=' bitwise_or
+// noteq_bitwise_or: ('!=') bitwise_or
 static CmpopExprPair*
 noteq_bitwise_or_rule(Parser *p)
 {
@@ -5566,11 +5568,11 @@ noteq_bitwise_or_rule(Parser *p)
     }
     CmpopExprPair* res = NULL;
     int mark = p->mark;
-    { // '!=' bitwise_or
+    { // ('!=') bitwise_or
+        void *_tmp_86_var;
         expr_ty a;
-        void *literal;
         if (
-            (literal = _PyPegen_expect_token(p, 28))
+            (_tmp_86_var = _tmp_86_rule(p))
             &&
             (a = bitwise_or_rule(p))
         )
@@ -7012,7 +7014,7 @@ slices_rule(Parser *p)
         void *opt_var;
         UNUSED(opt_var); // Silence compiler warnings
         if (
-            (a = _gather_86_rule(p))
+            (a = _gather_87_rule(p))
             &&
             (opt_var = _PyPegen_expect_token(p, 12), 1)
         )
@@ -7068,7 +7070,7 @@ slice_rule(Parser *p)
             &&
             (b = expression_rule(p), 1)
             &&
-            (c = _tmp_88_rule(p), 1)
+            (c = _tmp_89_rule(p), 1)
         )
         {
             Token *token = _PyPegen_get_last_nonnwhitespace_token(p);
@@ -7256,40 +7258,40 @@ atom_rule(Parser *p)
         p->mark = mark;
     }
     { // &'(' (tuple | group | genexp)
-        void *_tmp_89_var;
+        void *_tmp_90_var;
         if (
             _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 7)
             &&
-            (_tmp_89_var = _tmp_89_rule(p))
+            (_tmp_90_var = _tmp_90_rule(p))
         )
         {
-            res = _tmp_89_var;
+            res = _tmp_90_var;
             goto done;
         }
         p->mark = mark;
     }
     { // &'[' (list | listcomp)
-        void *_tmp_90_var;
+        void *_tmp_91_var;
         if (
             _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 9)
             &&
-            (_tmp_90_var = _tmp_90_rule(p))
+            (_tmp_91_var = _tmp_91_rule(p))
         )
         {
-            res = _tmp_90_var;
+            res = _tmp_91_var;
             goto done;
         }
         p->mark = mark;
     }
     { // &'{' (dict | set | dictcomp | setcomp)
-        void *_tmp_91_var;
+        void *_tmp_92_var;
         if (
             _PyPegen_lookahead_with_int(1, _PyPegen_expect_token, p, 25)
             &&
-            (_tmp_91_var = _tmp_91_rule(p))
+            (_tmp_92_var = _tmp_92_rule(p))
         )
         {
-            res = _tmp_91_var;
+            res = _tmp_92_var;
             goto done;
         }
         p->mark = mark;
@@ -7336,7 +7338,7 @@ strings_rule(Parser *p)
     { // STRING+
         asdl_seq * a;
         if (
-            (a = _loop1_92_rule(p))
+            (a = _loop1_93_rule(p))
         )
         {
             res = _PyPegen_concatenate_strings ( p , a );
@@ -7494,7 +7496,7 @@ tuple_rule(Parser *p)
         if (
             (literal = _PyPegen_expect_token(p, 7))
             &&
-            (a = _tmp_93_rule(p), 1)
+            (a = _tmp_94_rule(p), 1)
             &&
             (literal_1 = _PyPegen_expect_token(p, 8))
         )
@@ -7537,7 +7539,7 @@ group_rule(Parser *p)
         if (
             (literal = _PyPegen_expect_token(p, 7))
             &&
-            (a = _tmp_94_rule(p))
+            (a = _tmp_95_rule(p))
             &&
             (literal_1 = _PyPegen_expect_token(p, 8))
         )
@@ -7856,7 +7858,7 @@ kvpairs_rule(Parser *p)
         void *opt_var;
         UNUSED(opt_var); // Silence compiler warnings
         if (
-            (a = _gather_95_rule(p))
+            (a = _gather_96_rule(p))
             &&
             (opt_var = _PyPegen_expect_token(p, 12), 1)
         )
@@ -7940,7 +7942,7 @@ for_if_clauses_rule(Parser *p)
     { // ((ASYNC? 'for' star_targets 'in' disjunction (('if' disjunction))*))+
         asdl_seq * a;
         if (
-            (a = _loop1_97_rule(p))
+            (a = _loop1_98_rule(p))
         )
         {
             res = a;
@@ -8106,7 +8108,7 @@ args_rule(Parser *p)
         if (
             (a = starred_expression_rule(p))
             &&
-            (b = _tmp_98_rule(p), 1)
+            (b = _tmp_99_rule(p), 1)
         )
         {
             Token *token = _PyPegen_get_last_nonnwhitespace_token(p);
@@ -8155,7 +8157,7 @@ args_rule(Parser *p)
         if (
             (a = named_expression_rule(p))
             &&
-            (b = _tmp_99_rule(p), 1)
+            (b = _tmp_100_rule(p), 1)
         )
         {
             Token *token = _PyPegen_get_last_nonnwhitespace_token(p);
@@ -8197,11 +8199,11 @@ kwargs_rule(Parser *p)
         asdl_seq * b;
         void *literal;
         if (
-            (a = _gather_100_rule(p))
+            (a = _gather_101_rule(p))
             &&
             (literal = _PyPegen_expect_token(p, 12))
             &&
-            (b = _gather_102_rule(p))
+            (b = _gather_103_rule(p))
         )
         {
             res = _PyPegen_join_sequences ( p , a , b );
@@ -8214,23 +8216,23 @@ kwargs_rule(Parser *p)
         p->mark = mark;
     }
     { // ','.kwarg_or_starred+
-        asdl_seq * _gather_104_var;
+        asdl_seq * _gather_105_var;
         if (
-            (_gather_104_var = _gather_104_rule(p))
+            (_gather_105_var = _gather_105_rule(p))
         )
         {
-            res = _gather_104_var;
+            res = _gather_105_var;
             goto done;
         }
         p->mark = mark;
     }
     { // ','.kwarg_or_double_starred+
-        asdl_seq * _gather_106_var;
+        asdl_seq * _gather_107_var;
         if (
-            (_gather_106_var = _gather_106_rule(p))
+            (_gather_107_var = _gather_107_rule(p))
         )
         {
-            res = _gather_106_var;
+            res = _gather_107_var;
             goto done;
         }
         p->mark = mark;
@@ -8473,7 +8475,7 @@ star_targets_rule(Parser *p)
         if (
             (a = star_target_rule(p))
             &&
-            (b = _loop0_108_rule(p))
+            (b = _loop0_109_rule(p))
             &&
             (opt_var = _PyPegen_expect_token(p, 12), 1)
         )
@@ -8514,7 +8516,7 @@ star_targets_seq_rule(Parser *p)
         void *opt_var;
         UNUSED(opt_var); // Silence compiler warnings
         if (
-            (a = _gather_109_rule(p))
+            (a = _gather_110_rule(p))
             &&
             (opt_var = _PyPegen_expect_token(p, 12), 1)
         )
@@ -8562,7 +8564,7 @@ star_target_rule(Parser *p)
         if (
             (literal = _PyPegen_expect_token(p, 16))
             &&
-            (a = _tmp_111_rule(p))
+            (a = _tmp_112_rule(p))
         )
         {
             Token *token = _PyPegen_get_last_nonnwhitespace_token(p);
@@ -8951,7 +8953,7 @@ del_targets_rule(Parser *p)
         void *opt_var;
         UNUSED(opt_var); // Silence compiler warnings
         if (
-            (a = _gather_112_rule(p))
+            (a = _gather_113_rule(p))
             &&
             (opt_var = _PyPegen_expect_token(p, 12), 1)
         )
@@ -9204,7 +9206,7 @@ targets_rule(Parser *p)
         void *opt_var;
         UNUSED(opt_var); // Silence compiler warnings
         if (
-            (a = _gather_114_rule(p))
+            (a = _gather_115_rule(p))
             &&
             (opt_var = _PyPegen_expect_token(p, 12), 1)
         )
@@ -9732,7 +9734,7 @@ incorrect_arguments_rule(Parser *p)
             &&
             (literal = _PyPegen_expect_token(p, 12))
             &&
-            (opt_var = _tmp_116_rule(p), 1)
+            (opt_var = _tmp_117_rule(p), 1)
         )
         {
             res = RAISE_SYNTAX_ERROR ( "Generator expression must be parenthesized" );
@@ -9867,7 +9869,7 @@ invalid_assignment_rule(Parser *p)
             &&
             (expression_var_1 = expression_rule(p))
             &&
-            (opt_var = _tmp_117_rule(p), 1)
+            (opt_var = _tmp_118_rule(p), 1)
         )
         {
             res = RAISE_SYNTAX_ERROR ( "illegal target for annotation" );
@@ -9880,15 +9882,15 @@ invalid_assignment_rule(Parser *p)
         p->mark = mark;
     }
     { // expression ('=' | augassign) (yield_expr | star_expressions)
-        void *_tmp_118_var;
         void *_tmp_119_var;
+        void *_tmp_120_var;
         expr_ty a;
         if (
             (a = expression_rule(p))
             &&
-            (_tmp_118_var = _tmp_118_rule(p))
-            &&
             (_tmp_119_var = _tmp_119_rule(p))
+            &&
+            (_tmp_120_var = _tmp_120_rule(p))
         )
         {
             res = RAISE_SYNTAX_ERROR ( "cannot assign to %s" , _PyPegen_get_expr_name ( a ) );
@@ -9946,12 +9948,12 @@ invalid_comprehension_rule(Parser *p)
     void * res = NULL;
     int mark = p->mark;
     { // ('[' | '(' | '{') '*' expression for_if_clauses
-        void *_tmp_120_var;
+        void *_tmp_121_var;
         expr_ty expression_var;
         asdl_seq* for_if_clauses_var;
         void *literal;
         if (
-            (_tmp_120_var = _tmp_120_rule(p))
+            (_tmp_121_var = _tmp_121_rule(p))
             &&
             (literal = _PyPegen_expect_token(p, 16))
             &&
@@ -9985,15 +9987,15 @@ invalid_parameters_rule(Parser *p)
     void * res = NULL;
     int mark = p->mark;
     { // [plain_names ','] (slash_with_default | names_with_default) ',' plain_names
-        void *_tmp_122_var;
+        void *_tmp_123_var;
         void *literal;
         void *opt_var;
         UNUSED(opt_var); // Silence compiler warnings
         asdl_seq* plain_names_var;
         if (
-            (opt_var = _tmp_121_rule(p), 1)
+            (opt_var = _tmp_122_rule(p), 1)
             &&
-            (_tmp_122_var = _tmp_122_rule(p))
+            (_tmp_123_var = _tmp_123_rule(p))
             &&
             (literal = _PyPegen_expect_token(p, 12))
             &&
@@ -10520,12 +10522,12 @@ _loop1_13_rule(Parser *p)
     ssize_t children_capacity = 1;
     ssize_t n = 0;
     { // (star_targets '=')
-        void *_tmp_123_var;
+        void *_tmp_124_var;
         while (
-            (_tmp_123_var = _tmp_123_rule(p))
+            (_tmp_124_var = _tmp_124_rule(p))
         )
         {
-            res = _tmp_123_var;
+            res = _tmp_124_var;
             if (n == children_capacity) {
                 children_capacity *= 2;
                 children = PyMem_Realloc(children, children_capacity*sizeof(void *));
@@ -10847,12 +10849,12 @@ _loop0_21_rule(Parser *p)
     ssize_t children_capacity = 1;
     ssize_t n = 0;
     { // ('.' | '...')
-        void *_tmp_124_var;
+        void *_tmp_125_var;
         while (
-            (_tmp_124_var = _tmp_124_rule(p))
+            (_tmp_125_var = _tmp_125_rule(p))
         )
         {
-            res = _tmp_124_var;
+            res = _tmp_125_var;
             if (n == children_capacity) {
                 children_capacity *= 2;
                 children = PyMem_Realloc(children, children_capacity*sizeof(void *));
@@ -10896,12 +10898,12 @@ _loop1_22_rule(Parser *p)
     ssize_t children_capacity = 1;
     ssize_t n = 0;
     { // ('.' | '...')
-        void *_tmp_125_var;
+        void *_tmp_126_var;
         while (
-            (_tmp_125_var = _tmp_125_rule(p))
+            (_tmp_126_var = _tmp_126_rule(p))
         )
         {
-            res = _tmp_125_var;
+            res = _tmp_126_var;
             if (n == children_capacity) {
                 children_capacity *= 2;
                 children = PyMem_Realloc(children, children_capacity*sizeof(void *));
@@ -12110,7 +12112,7 @@ _loop0_55_rule(Parser *p)
         while (
             (literal = _PyPegen_expect_token(p, 12))
             &&
-            (elem = _tmp_126_rule(p))
+            (elem = _tmp_127_rule(p))
         )
         {
             res = elem;
@@ -12157,7 +12159,7 @@ _gather_54_rule(Parser *p)
         void *elem;
         asdl_seq * seq;
         if (
-            (elem = _tmp_126_rule(p))
+            (elem = _tmp_127_rule(p))
             &&
             (seq = _loop0_55_rule(p))
         )
@@ -12222,12 +12224,12 @@ _loop1_57_rule(Parser *p)
     ssize_t children_capacity = 1;
     ssize_t n = 0;
     { // ('@' named_expression NEWLINE)
-        void *_tmp_127_var;
+        void *_tmp_128_var;
         while (
-            (_tmp_127_var = _tmp_127_rule(p))
+            (_tmp_128_var = _tmp_128_rule(p))
         )
         {
-            res = _tmp_127_var;
+            res = _tmp_128_var;
             if (n == children_capacity) {
                 children_capacity *= 2;
                 children = PyMem_Realloc(children, children_capacity*sizeof(void *));
@@ -12395,12 +12397,12 @@ _loop1_61_rule(Parser *p)
     ssize_t children_capacity = 1;
     ssize_t n = 0;
     { // (',' star_expression)
-        void *_tmp_128_var;
+        void *_tmp_129_var;
         while (
-            (_tmp_128_var = _tmp_128_rule(p))
+            (_tmp_129_var = _tmp_129_rule(p))
         )
         {
-            res = _tmp_128_var;
+            res = _tmp_129_var;
             if (n == children_capacity) {
                 children_capacity *= 2;
                 children = PyMem_Realloc(children, children_capacity*sizeof(void *));
@@ -12533,12 +12535,12 @@ _loop1_64_rule(Parser *p)
     ssize_t children_capacity = 1;
     ssize_t n = 0;
     { // (',' expression)
-        void *_tmp_129_var;
+        void *_tmp_130_var;
         while (
-            (_tmp_129_var = _tmp_129_rule(p))
+            (_tmp_130_var = _tmp_130_rule(p))
         )
         {
-            res = _tmp_129_var;
+            res = _tmp_130_var;
             if (n == children_capacity) {
                 children_capacity *= 2;
                 children = PyMem_Realloc(children, children_capacity*sizeof(void *));
@@ -13162,7 +13164,7 @@ _loop0_82_rule(Parser *p)
         while (
             (literal = _PyPegen_expect_token(p, 12))
             &&
-            (elem = _tmp_130_rule(p))
+            (elem = _tmp_131_rule(p))
         )
         {
             res = elem;
@@ -13209,7 +13211,7 @@ _gather_81_rule(Parser *p)
         void *elem;
         asdl_seq * seq;
         if (
-            (elem = _tmp_130_rule(p))
+            (elem = _tmp_131_rule(p))
             &&
             (seq = _loop0_82_rule(p))
         )
@@ -13242,12 +13244,12 @@ _loop1_83_rule(Parser *p)
     ssize_t children_capacity = 1;
     ssize_t n = 0;
     { // ('or' conjunction)
-        void *_tmp_131_var;
+        void *_tmp_132_var;
         while (
-            (_tmp_131_var = _tmp_131_rule(p))
+            (_tmp_132_var = _tmp_132_rule(p))
         )
         {
-            res = _tmp_131_var;
+            res = _tmp_132_var;
             if (n == children_capacity) {
                 children_capacity *= 2;
                 children = PyMem_Realloc(children, children_capacity*sizeof(void *));
@@ -13295,12 +13297,12 @@ _loop1_84_rule(Parser *p)
     ssize_t children_capacity = 1;
     ssize_t n = 0;
     { // ('and' inversion)
-        void *_tmp_132_var;
+        void *_tmp_133_var;
         while (
-            (_tmp_132_var = _tmp_132_rule(p))
+            (_tmp_133_var = _tmp_133_rule(p))
         )
         {
-            res = _tmp_132_var;
+            res = _tmp_133_var;
             if (n == children_capacity) {
                 children_capacity *= 2;
                 children = PyMem_Realloc(children, children_capacity*sizeof(void *));
@@ -13383,9 +13385,38 @@ _loop1_85_rule(Parser *p)
     return seq;
 }
 
-// _loop0_87: ',' slice
+// _tmp_86: '!='
+static void *
+_tmp_86_rule(Parser *p)
+{
+    if (p->error_indicator) {
+        return NULL;
+    }
+    void * res = NULL;
+    int mark = p->mark;
+    { // '!='
+        void *tok;
+        if (
+            (tok = _PyPegen_expect_token(p, 28))
+        )
+        {
+            res = _PyPegen_check_barry_as_flufl ( p ) ? NULL : tok;
+            if (res == NULL && PyErr_Occurred()) {
+                p->error_indicator = 1;
+                return NULL;
+            }
+            goto done;
+        }
+        p->mark = mark;
+    }
+    res = NULL;
+  done:
+    return res;
+}
+
+// _loop0_88: ',' slice
 static asdl_seq *
-_loop0_87_rule(Parser *p)
+_loop0_88_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -13430,32 +13461,32 @@ _loop0_87_rule(Parser *p)
     }
     asdl_seq *seq = _Py_asdl_seq_new(n, p->arena);
     if (!seq) {
-        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_87");
+        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_88");
         PyMem_Free(children);
         return NULL;
     }
     for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]);
     PyMem_Free(children);
-    _PyPegen_insert_memo(p, start_mark, _loop0_87_type, seq);
+    _PyPegen_insert_memo(p, start_mark, _loop0_88_type, seq);
     return seq;
 }
 
-// _gather_86: slice _loop0_87
+// _gather_87: slice _loop0_88
 static asdl_seq *
-_gather_86_rule(Parser *p)
+_gather_87_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
     }
     asdl_seq * res = NULL;
     int mark = p->mark;
-    { // slice _loop0_87
+    { // slice _loop0_88
         expr_ty elem;
         asdl_seq * seq;
         if (
             (elem = slice_rule(p))
             &&
-            (seq = _loop0_87_rule(p))
+            (seq = _loop0_88_rule(p))
         )
         {
             res = _PyPegen_seq_insert_in_front(p, elem, seq);
@@ -13468,9 +13499,9 @@ _gather_86_rule(Parser *p)
     return res;
 }
 
-// _tmp_88: ':' expression?
+// _tmp_89: ':' expression?
 static void *
-_tmp_88_rule(Parser *p)
+_tmp_89_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -13500,9 +13531,9 @@ _tmp_88_rule(Parser *p)
     return res;
 }
 
-// _tmp_89: tuple | group | genexp
+// _tmp_90: tuple | group | genexp
 static void *
-_tmp_89_rule(Parser *p)
+_tmp_90_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -13547,9 +13578,9 @@ _tmp_89_rule(Parser *p)
     return res;
 }
 
-// _tmp_90: list | listcomp
+// _tmp_91: list | listcomp
 static void *
-_tmp_90_rule(Parser *p)
+_tmp_91_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -13583,9 +13614,9 @@ _tmp_90_rule(Parser *p)
     return res;
 }
 
-// _tmp_91: dict | set | dictcomp | setcomp
+// _tmp_92: dict | set | dictcomp | setcomp
 static void *
-_tmp_91_rule(Parser *p)
+_tmp_92_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -13641,9 +13672,9 @@ _tmp_91_rule(Parser *p)
     return res;
 }
 
-// _loop1_92: STRING
+// _loop1_93: STRING
 static asdl_seq *
-_loop1_92_rule(Parser *p)
+_loop1_93_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -13684,19 +13715,19 @@ _loop1_92_rule(Parser *p)
     }
     asdl_seq *seq = _Py_asdl_seq_new(n, p->arena);
     if (!seq) {
-        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop1_92");
+        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop1_93");
         PyMem_Free(children);
         return NULL;
     }
     for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]);
     PyMem_Free(children);
-    _PyPegen_insert_memo(p, start_mark, _loop1_92_type, seq);
+    _PyPegen_insert_memo(p, start_mark, _loop1_93_type, seq);
     return seq;
 }
 
-// _tmp_93: star_named_expression ',' star_named_expressions?
+// _tmp_94: star_named_expression ',' star_named_expressions?
 static void *
-_tmp_93_rule(Parser *p)
+_tmp_94_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -13729,9 +13760,9 @@ _tmp_93_rule(Parser *p)
     return res;
 }
 
-// _tmp_94: yield_expr | named_expression
+// _tmp_95: yield_expr | named_expression
 static void *
-_tmp_94_rule(Parser *p)
+_tmp_95_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -13765,9 +13796,9 @@ _tmp_94_rule(Parser *p)
     return res;
 }
 
-// _loop0_96: ',' kvpair
+// _loop0_97: ',' kvpair
 static asdl_seq *
-_loop0_96_rule(Parser *p)
+_loop0_97_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -13812,32 +13843,32 @@ _loop0_96_rule(Parser *p)
     }
     asdl_seq *seq = _Py_asdl_seq_new(n, p->arena);
     if (!seq) {
-        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_96");
+        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_97");
         PyMem_Free(children);
         return NULL;
     }
     for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]);
     PyMem_Free(children);
-    _PyPegen_insert_memo(p, start_mark, _loop0_96_type, seq);
+    _PyPegen_insert_memo(p, start_mark, _loop0_97_type, seq);
     return seq;
 }
 
-// _gather_95: kvpair _loop0_96
+// _gather_96: kvpair _loop0_97
 static asdl_seq *
-_gather_95_rule(Parser *p)
+_gather_96_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
     }
     asdl_seq * res = NULL;
     int mark = p->mark;
-    { // kvpair _loop0_96
+    { // kvpair _loop0_97
         KeyValuePair* elem;
         asdl_seq * seq;
         if (
             (elem = kvpair_rule(p))
             &&
-            (seq = _loop0_96_rule(p))
+            (seq = _loop0_97_rule(p))
         )
         {
             res = _PyPegen_seq_insert_in_front(p, elem, seq);
@@ -13850,9 +13881,9 @@ _gather_95_rule(Parser *p)
     return res;
 }
 
-// _loop1_97: (ASYNC? 'for' star_targets 'in' disjunction (('if' disjunction))*)
+// _loop1_98: (ASYNC? 'for' star_targets 'in' disjunction (('if' disjunction))*)
 static asdl_seq *
-_loop1_97_rule(Parser *p)
+_loop1_98_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -13868,12 +13899,12 @@ _loop1_97_rule(Parser *p)
     ssize_t children_capacity = 1;
     ssize_t n = 0;
     { // (ASYNC? 'for' star_targets 'in' disjunction (('if' disjunction))*)
-        void *_tmp_133_var;
+        void *_tmp_134_var;
         while (
-            (_tmp_133_var = _tmp_133_rule(p))
+            (_tmp_134_var = _tmp_134_rule(p))
         )
         {
-            res = _tmp_133_var;
+            res = _tmp_134_var;
             if (n == children_capacity) {
                 children_capacity *= 2;
                 children = PyMem_Realloc(children, children_capacity*sizeof(void *));
@@ -13893,19 +13924,19 @@ _loop1_97_rule(Parser *p)
     }
     asdl_seq *seq = _Py_asdl_seq_new(n, p->arena);
     if (!seq) {
-        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop1_97");
+        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop1_98");
         PyMem_Free(children);
         return NULL;
     }
     for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]);
     PyMem_Free(children);
-    _PyPegen_insert_memo(p, start_mark, _loop1_97_type, seq);
+    _PyPegen_insert_memo(p, start_mark, _loop1_98_type, seq);
     return seq;
 }
 
-// _tmp_98: ',' args
+// _tmp_99: ',' args
 static void *
-_tmp_98_rule(Parser *p)
+_tmp_99_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -13935,9 +13966,9 @@ _tmp_98_rule(Parser *p)
     return res;
 }
 
-// _tmp_99: ',' args
+// _tmp_100: ',' args
 static void *
-_tmp_99_rule(Parser *p)
+_tmp_100_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -13967,9 +13998,9 @@ _tmp_99_rule(Parser *p)
     return res;
 }
 
-// _loop0_101: ',' kwarg_or_starred
+// _loop0_102: ',' kwarg_or_starred
 static asdl_seq *
-_loop0_101_rule(Parser *p)
+_loop0_102_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -14014,32 +14045,32 @@ _loop0_101_rule(Parser *p)
     }
     asdl_seq *seq = _Py_asdl_seq_new(n, p->arena);
     if (!seq) {
-        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_101");
+        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_102");
         PyMem_Free(children);
         return NULL;
     }
     for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]);
     PyMem_Free(children);
-    _PyPegen_insert_memo(p, start_mark, _loop0_101_type, seq);
+    _PyPegen_insert_memo(p, start_mark, _loop0_102_type, seq);
     return seq;
 }
 
-// _gather_100: kwarg_or_starred _loop0_101
+// _gather_101: kwarg_or_starred _loop0_102
 static asdl_seq *
-_gather_100_rule(Parser *p)
+_gather_101_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
     }
     asdl_seq * res = NULL;
     int mark = p->mark;
-    { // kwarg_or_starred _loop0_101
+    { // kwarg_or_starred _loop0_102
         KeywordOrStarred* elem;
         asdl_seq * seq;
         if (
             (elem = kwarg_or_starred_rule(p))
             &&
-            (seq = _loop0_101_rule(p))
+            (seq = _loop0_102_rule(p))
         )
         {
             res = _PyPegen_seq_insert_in_front(p, elem, seq);
@@ -14052,9 +14083,9 @@ _gather_100_rule(Parser *p)
     return res;
 }
 
-// _loop0_103: ',' kwarg_or_double_starred
+// _loop0_104: ',' kwarg_or_double_starred
 static asdl_seq *
-_loop0_103_rule(Parser *p)
+_loop0_104_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -14099,32 +14130,32 @@ _loop0_103_rule(Parser *p)
     }
     asdl_seq *seq = _Py_asdl_seq_new(n, p->arena);
     if (!seq) {
-        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_103");
+        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_104");
         PyMem_Free(children);
         return NULL;
     }
     for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]);
     PyMem_Free(children);
-    _PyPegen_insert_memo(p, start_mark, _loop0_103_type, seq);
+    _PyPegen_insert_memo(p, start_mark, _loop0_104_type, seq);
     return seq;
 }
 
-// _gather_102: kwarg_or_double_starred _loop0_103
+// _gather_103: kwarg_or_double_starred _loop0_104
 static asdl_seq *
-_gather_102_rule(Parser *p)
+_gather_103_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
     }
     asdl_seq * res = NULL;
     int mark = p->mark;
-    { // kwarg_or_double_starred _loop0_103
+    { // kwarg_or_double_starred _loop0_104
         KeywordOrStarred* elem;
         asdl_seq * seq;
         if (
             (elem = kwarg_or_double_starred_rule(p))
             &&
-            (seq = _loop0_103_rule(p))
+            (seq = _loop0_104_rule(p))
         )
         {
             res = _PyPegen_seq_insert_in_front(p, elem, seq);
@@ -14137,9 +14168,9 @@ _gather_102_rule(Parser *p)
     return res;
 }
 
-// _loop0_105: ',' kwarg_or_starred
+// _loop0_106: ',' kwarg_or_starred
 static asdl_seq *
-_loop0_105_rule(Parser *p)
+_loop0_106_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -14184,32 +14215,32 @@ _loop0_105_rule(Parser *p)
     }
     asdl_seq *seq = _Py_asdl_seq_new(n, p->arena);
     if (!seq) {
-        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_105");
+        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_106");
         PyMem_Free(children);
         return NULL;
     }
     for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]);
     PyMem_Free(children);
-    _PyPegen_insert_memo(p, start_mark, _loop0_105_type, seq);
+    _PyPegen_insert_memo(p, start_mark, _loop0_106_type, seq);
     return seq;
 }
 
-// _gather_104: kwarg_or_starred _loop0_105
+// _gather_105: kwarg_or_starred _loop0_106
 static asdl_seq *
-_gather_104_rule(Parser *p)
+_gather_105_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
     }
     asdl_seq * res = NULL;
     int mark = p->mark;
-    { // kwarg_or_starred _loop0_105
+    { // kwarg_or_starred _loop0_106
         KeywordOrStarred* elem;
         asdl_seq * seq;
         if (
             (elem = kwarg_or_starred_rule(p))
             &&
-            (seq = _loop0_105_rule(p))
+            (seq = _loop0_106_rule(p))
         )
         {
             res = _PyPegen_seq_insert_in_front(p, elem, seq);
@@ -14222,9 +14253,9 @@ _gather_104_rule(Parser *p)
     return res;
 }
 
-// _loop0_107: ',' kwarg_or_double_starred
+// _loop0_108: ',' kwarg_or_double_starred
 static asdl_seq *
-_loop0_107_rule(Parser *p)
+_loop0_108_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -14269,32 +14300,32 @@ _loop0_107_rule(Parser *p)
     }
     asdl_seq *seq = _Py_asdl_seq_new(n, p->arena);
     if (!seq) {
-        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_107");
+        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_108");
         PyMem_Free(children);
         return NULL;
     }
     for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]);
     PyMem_Free(children);
-    _PyPegen_insert_memo(p, start_mark, _loop0_107_type, seq);
+    _PyPegen_insert_memo(p, start_mark, _loop0_108_type, seq);
     return seq;
 }
 
-// _gather_106: kwarg_or_double_starred _loop0_107
+// _gather_107: kwarg_or_double_starred _loop0_108
 static asdl_seq *
-_gather_106_rule(Parser *p)
+_gather_107_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
     }
     asdl_seq * res = NULL;
     int mark = p->mark;
-    { // kwarg_or_double_starred _loop0_107
+    { // kwarg_or_double_starred _loop0_108
         KeywordOrStarred* elem;
         asdl_seq * seq;
         if (
             (elem = kwarg_or_double_starred_rule(p))
             &&
-            (seq = _loop0_107_rule(p))
+            (seq = _loop0_108_rule(p))
         )
         {
             res = _PyPegen_seq_insert_in_front(p, elem, seq);
@@ -14307,9 +14338,9 @@ _gather_106_rule(Parser *p)
     return res;
 }
 
-// _loop0_108: (',' star_target)
+// _loop0_109: (',' star_target)
 static asdl_seq *
-_loop0_108_rule(Parser *p)
+_loop0_109_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -14325,12 +14356,12 @@ _loop0_108_rule(Parser *p)
     ssize_t children_capacity = 1;
     ssize_t n = 0;
     { // (',' star_target)
-        void *_tmp_134_var;
+        void *_tmp_135_var;
         while (
-            (_tmp_134_var = _tmp_134_rule(p))
+            (_tmp_135_var = _tmp_135_rule(p))
         )
         {
-            res = _tmp_134_var;
+            res = _tmp_135_var;
             if (n == children_capacity) {
                 children_capacity *= 2;
                 children = PyMem_Realloc(children, children_capacity*sizeof(void *));
@@ -14346,19 +14377,19 @@ _loop0_108_rule(Parser *p)
     }
     asdl_seq *seq = _Py_asdl_seq_new(n, p->arena);
     if (!seq) {
-        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_108");
+        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_109");
         PyMem_Free(children);
         return NULL;
     }
     for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]);
     PyMem_Free(children);
-    _PyPegen_insert_memo(p, start_mark, _loop0_108_type, seq);
+    _PyPegen_insert_memo(p, start_mark, _loop0_109_type, seq);
     return seq;
 }
 
-// _loop0_110: ',' star_target
+// _loop0_111: ',' star_target
 static asdl_seq *
-_loop0_110_rule(Parser *p)
+_loop0_111_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -14403,32 +14434,32 @@ _loop0_110_rule(Parser *p)
     }
     asdl_seq *seq = _Py_asdl_seq_new(n, p->arena);
     if (!seq) {
-        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_110");
+        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_111");
         PyMem_Free(children);
         return NULL;
     }
     for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]);
     PyMem_Free(children);
-    _PyPegen_insert_memo(p, start_mark, _loop0_110_type, seq);
+    _PyPegen_insert_memo(p, start_mark, _loop0_111_type, seq);
     return seq;
 }
 
-// _gather_109: star_target _loop0_110
+// _gather_110: star_target _loop0_111
 static asdl_seq *
-_gather_109_rule(Parser *p)
+_gather_110_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
     }
     asdl_seq * res = NULL;
     int mark = p->mark;
-    { // star_target _loop0_110
+    { // star_target _loop0_111
         expr_ty elem;
         asdl_seq * seq;
         if (
             (elem = star_target_rule(p))
             &&
-            (seq = _loop0_110_rule(p))
+            (seq = _loop0_111_rule(p))
         )
         {
             res = _PyPegen_seq_insert_in_front(p, elem, seq);
@@ -14441,9 +14472,9 @@ _gather_109_rule(Parser *p)
     return res;
 }
 
-// _tmp_111: !'*' star_target
+// _tmp_112: !'*' star_target
 static void *
-_tmp_111_rule(Parser *p)
+_tmp_112_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -14468,9 +14499,9 @@ _tmp_111_rule(Parser *p)
     return res;
 }
 
-// _loop0_113: ',' del_target
+// _loop0_114: ',' del_target
 static asdl_seq *
-_loop0_113_rule(Parser *p)
+_loop0_114_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -14515,32 +14546,32 @@ _loop0_113_rule(Parser *p)
     }
     asdl_seq *seq = _Py_asdl_seq_new(n, p->arena);
     if (!seq) {
-        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_113");
+        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_114");
         PyMem_Free(children);
         return NULL;
     }
     for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]);
     PyMem_Free(children);
-    _PyPegen_insert_memo(p, start_mark, _loop0_113_type, seq);
+    _PyPegen_insert_memo(p, start_mark, _loop0_114_type, seq);
     return seq;
 }
 
-// _gather_112: del_target _loop0_113
+// _gather_113: del_target _loop0_114
 static asdl_seq *
-_gather_112_rule(Parser *p)
+_gather_113_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
     }
     asdl_seq * res = NULL;
     int mark = p->mark;
-    { // del_target _loop0_113
+    { // del_target _loop0_114
         expr_ty elem;
         asdl_seq * seq;
         if (
             (elem = del_target_rule(p))
             &&
-            (seq = _loop0_113_rule(p))
+            (seq = _loop0_114_rule(p))
         )
         {
             res = _PyPegen_seq_insert_in_front(p, elem, seq);
@@ -14553,9 +14584,9 @@ _gather_112_rule(Parser *p)
     return res;
 }
 
-// _loop0_115: ',' target
+// _loop0_116: ',' target
 static asdl_seq *
-_loop0_115_rule(Parser *p)
+_loop0_116_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -14600,32 +14631,32 @@ _loop0_115_rule(Parser *p)
     }
     asdl_seq *seq = _Py_asdl_seq_new(n, p->arena);
     if (!seq) {
-        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_115");
+        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_116");
         PyMem_Free(children);
         return NULL;
     }
     for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]);
     PyMem_Free(children);
-    _PyPegen_insert_memo(p, start_mark, _loop0_115_type, seq);
+    _PyPegen_insert_memo(p, start_mark, _loop0_116_type, seq);
     return seq;
 }
 
-// _gather_114: target _loop0_115
+// _gather_115: target _loop0_116
 static asdl_seq *
-_gather_114_rule(Parser *p)
+_gather_115_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
     }
     asdl_seq * res = NULL;
     int mark = p->mark;
-    { // target _loop0_115
+    { // target _loop0_116
         expr_ty elem;
         asdl_seq * seq;
         if (
             (elem = target_rule(p))
             &&
-            (seq = _loop0_115_rule(p))
+            (seq = _loop0_116_rule(p))
         )
         {
             res = _PyPegen_seq_insert_in_front(p, elem, seq);
@@ -14638,9 +14669,9 @@ _gather_114_rule(Parser *p)
     return res;
 }
 
-// _tmp_116: args | expression for_if_clauses
+// _tmp_117: args | expression for_if_clauses
 static void *
-_tmp_116_rule(Parser *p)
+_tmp_117_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -14677,9 +14708,9 @@ _tmp_116_rule(Parser *p)
     return res;
 }
 
-// _tmp_117: '=' annotated_rhs
+// _tmp_118: '=' annotated_rhs
 static void *
-_tmp_117_rule(Parser *p)
+_tmp_118_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -14705,9 +14736,9 @@ _tmp_117_rule(Parser *p)
     return res;
 }
 
-// _tmp_118: '=' | augassign
+// _tmp_119: '=' | augassign
 static void *
-_tmp_118_rule(Parser *p)
+_tmp_119_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -14741,9 +14772,9 @@ _tmp_118_rule(Parser *p)
     return res;
 }
 
-// _tmp_119: yield_expr | star_expressions
+// _tmp_120: yield_expr | star_expressions
 static void *
-_tmp_119_rule(Parser *p)
+_tmp_120_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -14777,9 +14808,9 @@ _tmp_119_rule(Parser *p)
     return res;
 }
 
-// _tmp_120: '[' | '(' | '{'
+// _tmp_121: '[' | '(' | '{'
 static void *
-_tmp_120_rule(Parser *p)
+_tmp_121_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -14824,9 +14855,9 @@ _tmp_120_rule(Parser *p)
     return res;
 }
 
-// _tmp_121: plain_names ','
+// _tmp_122: plain_names ','
 static void *
-_tmp_121_rule(Parser *p)
+_tmp_122_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -14852,9 +14883,9 @@ _tmp_121_rule(Parser *p)
     return res;
 }
 
-// _tmp_122: slash_with_default | names_with_default
+// _tmp_123: slash_with_default | names_with_default
 static void *
-_tmp_122_rule(Parser *p)
+_tmp_123_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -14888,9 +14919,9 @@ _tmp_122_rule(Parser *p)
     return res;
 }
 
-// _tmp_123: star_targets '='
+// _tmp_124: star_targets '='
 static void *
-_tmp_123_rule(Parser *p)
+_tmp_124_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -14920,9 +14951,9 @@ _tmp_123_rule(Parser *p)
     return res;
 }
 
-// _tmp_124: '.' | '...'
+// _tmp_125: '.' | '...'
 static void *
-_tmp_124_rule(Parser *p)
+_tmp_125_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -14956,9 +14987,9 @@ _tmp_124_rule(Parser *p)
     return res;
 }
 
-// _tmp_125: '.' | '...'
+// _tmp_126: '.' | '...'
 static void *
-_tmp_125_rule(Parser *p)
+_tmp_126_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -14992,9 +15023,9 @@ _tmp_125_rule(Parser *p)
     return res;
 }
 
-// _tmp_126: plain_name !'='
+// _tmp_127: plain_name !'='
 static void *
-_tmp_126_rule(Parser *p)
+_tmp_127_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -15019,9 +15050,9 @@ _tmp_126_rule(Parser *p)
     return res;
 }
 
-// _tmp_127: '@' named_expression NEWLINE
+// _tmp_128: '@' named_expression NEWLINE
 static void *
-_tmp_127_rule(Parser *p)
+_tmp_128_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -15054,9 +15085,9 @@ _tmp_127_rule(Parser *p)
     return res;
 }
 
-// _tmp_128: ',' star_expression
+// _tmp_129: ',' star_expression
 static void *
-_tmp_128_rule(Parser *p)
+_tmp_129_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -15086,9 +15117,9 @@ _tmp_128_rule(Parser *p)
     return res;
 }
 
-// _tmp_129: ',' expression
+// _tmp_130: ',' expression
 static void *
-_tmp_129_rule(Parser *p)
+_tmp_130_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -15118,9 +15149,9 @@ _tmp_129_rule(Parser *p)
     return res;
 }
 
-// _tmp_130: lambda_plain_name !'='
+// _tmp_131: lambda_plain_name !'='
 static void *
-_tmp_130_rule(Parser *p)
+_tmp_131_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -15145,9 +15176,9 @@ _tmp_130_rule(Parser *p)
     return res;
 }
 
-// _tmp_131: 'or' conjunction
+// _tmp_132: 'or' conjunction
 static void *
-_tmp_131_rule(Parser *p)
+_tmp_132_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -15177,9 +15208,9 @@ _tmp_131_rule(Parser *p)
     return res;
 }
 
-// _tmp_132: 'and' inversion
+// _tmp_133: 'and' inversion
 static void *
-_tmp_132_rule(Parser *p)
+_tmp_133_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -15209,9 +15240,9 @@ _tmp_132_rule(Parser *p)
     return res;
 }
 
-// _tmp_133: ASYNC? 'for' star_targets 'in' disjunction (('if' disjunction))*
+// _tmp_134: ASYNC? 'for' star_targets 'in' disjunction (('if' disjunction))*
 static void *
-_tmp_133_rule(Parser *p)
+_tmp_134_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -15236,7 +15267,7 @@ _tmp_133_rule(Parser *p)
             &&
             (b = disjunction_rule(p))
             &&
-            (c = _loop0_135_rule(p))
+            (c = _loop0_136_rule(p))
         )
         {
             res = _Py_comprehension ( a , b , c , y != NULL , p -> arena );
@@ -15253,9 +15284,9 @@ _tmp_133_rule(Parser *p)
     return res;
 }
 
-// _tmp_134: ',' star_target
+// _tmp_135: ',' star_target
 static void *
-_tmp_134_rule(Parser *p)
+_tmp_135_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -15285,9 +15316,9 @@ _tmp_134_rule(Parser *p)
     return res;
 }
 
-// _loop0_135: ('if' disjunction)
+// _loop0_136: ('if' disjunction)
 static asdl_seq *
-_loop0_135_rule(Parser *p)
+_loop0_136_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;
@@ -15303,12 +15334,12 @@ _loop0_135_rule(Parser *p)
     ssize_t children_capacity = 1;
     ssize_t n = 0;
     { // ('if' disjunction)
-        void *_tmp_136_var;
+        void *_tmp_137_var;
         while (
-            (_tmp_136_var = _tmp_136_rule(p))
+            (_tmp_137_var = _tmp_137_rule(p))
         )
         {
-            res = _tmp_136_var;
+            res = _tmp_137_var;
             if (n == children_capacity) {
                 children_capacity *= 2;
                 children = PyMem_Realloc(children, children_capacity*sizeof(void *));
@@ -15324,19 +15355,19 @@ _loop0_135_rule(Parser *p)
     }
     asdl_seq *seq = _Py_asdl_seq_new(n, p->arena);
     if (!seq) {
-        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_135");
+        PyErr_Format(PyExc_MemoryError, "asdl_seq_new _loop0_136");
         PyMem_Free(children);
         return NULL;
     }
     for (int i = 0; i < n; i++) asdl_seq_SET(seq, i, children[i]);
     PyMem_Free(children);
-    _PyPegen_insert_memo(p, start_mark, _loop0_135_type, seq);
+    _PyPegen_insert_memo(p, start_mark, _loop0_136_type, seq);
     return seq;
 }
 
-// _tmp_136: 'if' disjunction
+// _tmp_137: 'if' disjunction
 static void *
-_tmp_136_rule(Parser *p)
+_tmp_137_rule(Parser *p)
 {
     if (p->error_indicator) {
         return NULL;

diff --git a/Parser/pegen/parse_string.c b/Parser/pegen/parse_string.c
index 41485a9669d6876b7265398ed104ee7f837d5480..9a78a28d241965b3e72e8c5eeb7a7f345101c53c 100644 (file)
--- a/Parser/pegen/parse_string.c
+++ b/Parser/pegen/parse_string.c
@@ -586,7 +586,7 @@ fstring_compile_expr(Parser *p, const char *expr_start, const char *expr_end,
         return NULL;
     }
 
-    Parser *p2 = _PyPegen_Parser_New(tok, Py_fstring_input, NULL, p->arena);
+    Parser *p2 = _PyPegen_Parser_New(tok, Py_fstring_input, p->flags, NULL, p->arena);
     p2->starting_lineno = p->starting_lineno + p->tok->first_lineno - 1;
     p2->starting_col_offset = p->tok->first_lineno == p->tok->lineno
                               ? p->starting_col_offset + t->col_offset : 0;

diff --git a/Parser/pegen/peg_api.c b/Parser/pegen/peg_api.c
index c42aa680c8602df43a334966d782401cfd7ce05b..31ac2e1399265d6d3044d6670ff26de96311ed3e 100644 (file)
--- a/Parser/pegen/peg_api.c
+++ b/Parser/pegen/peg_api.c
@@ -22,20 +22,19 @@ PyPegen_ASTFromStringObject(const char *str, PyObject* filename, int mode, PyCom
         return NULL;
     }
 
-    int iflags = flags != NULL ? flags->cf_flags : PyCF_IGNORE_COOKIE;
-    mod_ty result = _PyPegen_run_parser_from_string(str, mode, filename, iflags, arena);
+    mod_ty result = _PyPegen_run_parser_from_string(str, mode, filename, flags, arena);
     return result;
 }
 
 mod_ty
-PyPegen_ASTFromFile(const char *filename, int mode, PyArena *arena)
+PyPegen_ASTFromFile(const char *filename, int mode, PyCompilerFlags *flags, PyArena *arena)
 {
     PyObject *filename_ob = PyUnicode_FromString(filename);
     if (filename_ob == NULL) {
         return NULL;
     }
 
-    mod_ty result = _PyPegen_run_parser_from_file(filename, mode, filename_ob, arena);
+    mod_ty result = _PyPegen_run_parser_from_file(filename, mode, filename_ob, flags, arena);
     Py_XDECREF(filename_ob);
     return result;
 }
@@ -43,13 +42,13 @@ PyPegen_ASTFromFile(const char *filename, int mode, PyArena *arena)
 mod_ty
 PyPegen_ASTFromFileObject(FILE *fp, PyObject *filename_ob, int mode,
                           const char *enc, const char *ps1, const char* ps2,
-                          int *errcode, PyArena *arena)
+                          PyCompilerFlags *flags, int *errcode, PyArena *arena)
 {
     if (PySys_Audit("compile", "OO", Py_None, filename_ob) < 0) {
         return NULL;
     }
     return _PyPegen_run_parser_from_file_pointer(fp, mode, filename_ob, enc, ps1, ps2,
-                                        errcode, arena);
+                                        flags, errcode, arena);
 }
 
 PyCodeObject *
@@ -81,7 +80,7 @@ error:
 }
 
 PyCodeObject *
-PyPegen_CodeObjectFromFile(const char *filename, int mode)
+PyPegen_CodeObjectFromFile(const char *filename, int mode, PyCompilerFlags* flags)
 {
     PyArena *arena = PyArena_New();
     if (arena == NULL) {
@@ -95,7 +94,7 @@ PyPegen_CodeObjectFromFile(const char *filename, int mode)
         goto error;
     }
 
-    mod_ty res = PyPegen_ASTFromFile(filename, mode, arena);
+    mod_ty res = PyPegen_ASTFromFile(filename, mode, flags, arena);
     if (res == NULL) {
         goto error;
     }
@@ -110,8 +109,8 @@ error:
 
 PyCodeObject *
 PyPegen_CodeObjectFromFileObject(FILE *fp, PyObject *filename_ob, int mode,
-                                 const char *ps1, const char *ps2, const char *enc,
-                                 int *errcode)
+                                 const char *ps1, const char *ps2, 
+                                 PyCompilerFlags *flags, const char *enc, int *errcode)
 {
     PyArena *arena = PyArena_New();
     if (arena == NULL) {
@@ -121,7 +120,7 @@ PyPegen_CodeObjectFromFileObject(FILE *fp, PyObject *filename_ob, int mode,
     PyCodeObject *result = NULL;
 
     mod_ty res = PyPegen_ASTFromFileObject(fp, filename_ob, mode, enc, ps1, ps2,
-                                           errcode, arena);
+                                           flags, errcode, arena);
     if (res == NULL) {
         goto error;
     }

diff --git a/Parser/pegen/pegen.c b/Parser/pegen/pegen.c
index c8f5c95b473e29b4c42b89845f2b2fd22dec329b..081da8238c35ca6bebcacffc02a2f1021f366c82 100644 (file)
--- a/Parser/pegen/pegen.c
+++ b/Parser/pegen/pegen.c
@@ -25,6 +25,24 @@ init_normalization(Parser *p)
     return 1;
 }
 
+/* Checks if the NOTEQUAL token is valid given the current parser flags
+0 indicates success and nonzero indicates failure (an exception may be set) */
+int
+_PyPegen_check_barry_as_flufl(Parser *p) {
+    Token *t = p->tokens[p->fill - 1];
+    assert(t->bytes != NULL);
+    assert(t->type == NOTEQUAL);
+
+    char* tok_str = PyBytes_AS_STRING(t->bytes);
+    if (p->flags & PyPARSE_BARRY_AS_BDFL && strcmp(tok_str, "<>")){
+        RAISE_SYNTAX_ERROR("with Barry as BDFL, use '<>' instead of '!='");
+        return -1;
+    } else if (!(p->flags & PyPARSE_BARRY_AS_BDFL)) {
+        return strcmp(tok_str, "!=");
+    }
+    return 0;
+}
+
 PyObject *
 _PyPegen_new_identifier(Parser *p, char *n)
 {
@@ -401,7 +419,6 @@ _PyPegen_raise_error(Parser *p, PyObject *errtype, const char *errmsg, ...)
         loc = Py_None;
     }
 
-
     tmp = Py_BuildValue("(OiiN)", p->tok->filename, t->lineno, col_number, loc);
     if (!tmp) {
         goto error;
@@ -902,8 +919,31 @@ _PyPegen_Parser_Free(Parser *p)
     PyMem_Free(p);
 }
 
+static int
+compute_parser_flags(PyCompilerFlags *flags)
+{
+    int parser_flags = 0;
+    if (!flags) {
+        return 0;
+    }
+    if (flags->cf_flags & PyCF_DONT_IMPLY_DEDENT) {
+        parser_flags |= PyPARSE_DONT_IMPLY_DEDENT;
+    }
+    if (flags->cf_flags & PyCF_IGNORE_COOKIE) {
+        parser_flags |= PyPARSE_IGNORE_COOKIE;
+    }
+    if (flags->cf_flags & CO_FUTURE_BARRY_AS_BDFL) {
+        parser_flags |= PyPARSE_BARRY_AS_BDFL;
+    }
+    if (flags->cf_flags & PyCF_TYPE_COMMENTS) {
+        parser_flags |= PyPARSE_TYPE_COMMENTS;
+    }
+    return parser_flags;
+}
+
 Parser *
-_PyPegen_Parser_New(struct tok_state *tok, int start_rule, int *errcode, PyArena *arena)
+_PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags,
+                    int *errcode, PyArena *arena)
 {
     Parser *p = PyMem_Malloc(sizeof(Parser));
     if (p == NULL) {
@@ -938,6 +978,7 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int *errcode, PyArena
 
     p->starting_lineno = 0;
     p->starting_col_offset = 0;
+    p->flags = flags;
 
     return p;
 }
@@ -976,7 +1017,7 @@ _PyPegen_run_parser(Parser *p)
 mod_ty
 _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filename_ob,
                              const char *enc, const char *ps1, const char *ps2,
-                             int *errcode, PyArena *arena)
+                             PyCompilerFlags *flags, int *errcode, PyArena *arena)
 {
     struct tok_state *tok = PyTokenizer_FromFile(fp, enc, ps1, ps2);
     if (tok == NULL) {
@@ -993,7 +1034,8 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena
     // From here on we need to clean up even if there's an error
     mod_ty result = NULL;
 
-    Parser *p = _PyPegen_Parser_New(tok, start_rule, errcode, arena);
+    int parser_flags = compute_parser_flags(flags);
+    Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, errcode, arena);
     if (p == NULL) {
         goto error;
     }
@@ -1008,7 +1050,7 @@ error:
 
 mod_ty
 _PyPegen_run_parser_from_file(const char *filename, int start_rule,
-                     PyObject *filename_ob, PyArena *arena)
+                     PyObject *filename_ob, PyCompilerFlags *flags, PyArena *arena)
 {
     FILE *fp = fopen(filename, "rb");
     if (fp == NULL) {
@@ -1017,7 +1059,7 @@ _PyPegen_run_parser_from_file(const char *filename, int start_rule,
     }
 
     mod_ty result = _PyPegen_run_parser_from_file_pointer(fp, start_rule, filename_ob,
-                                                 NULL, NULL, NULL, NULL, arena);
+                                                 NULL, NULL, NULL, flags, NULL, arena);
 
     fclose(fp);
     return result;
@@ -1025,12 +1067,12 @@ _PyPegen_run_parser_from_file(const char *filename, int start_rule,
 
 mod_ty
 _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filename_ob,
-                       int iflags, PyArena *arena)
+                       PyCompilerFlags *flags, PyArena *arena)
 {
     int exec_input = start_rule == Py_file_input;
 
     struct tok_state *tok;
-    if (iflags & PyCF_IGNORE_COOKIE) {
+    if (flags == NULL || flags->cf_flags & PyCF_IGNORE_COOKIE) {
         tok = PyTokenizer_FromUTF8(str, exec_input);
     } else {
         tok = PyTokenizer_FromString(str, exec_input);
@@ -1048,7 +1090,8 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen
     // We need to clear up from here on
     mod_ty result = NULL;
 
-    Parser *p = _PyPegen_Parser_New(tok, start_rule, NULL, arena);
+    int parser_flags = compute_parser_flags(flags);
+    Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, NULL, arena);
     if (p == NULL) {
         goto error;
     }

diff --git a/Parser/pegen/pegen.h b/Parser/pegen/pegen.h
index a20ec4a0e42746e808f7bc5ec41bd4eec7287dc3..0ac9b317efe598dc9df8f494e95bdf3efcfda983 100644 (file)
--- a/Parser/pegen/pegen.h
+++ b/Parser/pegen/pegen.h
@@ -7,6 +7,23 @@
 #include <Python-ast.h>
 #include <pyarena.h>
 
+#if 0
+#define PyPARSE_YIELD_IS_KEYWORD        0x0001
+#endif
+
+#define PyPARSE_DONT_IMPLY_DEDENT       0x0002
+
+#if 0
+#define PyPARSE_WITH_IS_KEYWORD         0x0003
+#define PyPARSE_PRINT_IS_FUNCTION       0x0004
+#define PyPARSE_UNICODE_LITERALS        0x0008
+#endif
+
+#define PyPARSE_IGNORE_COOKIE 0x0010
+#define PyPARSE_BARRY_AS_BDFL 0x0020
+#define PyPARSE_TYPE_COMMENTS 0x0040
+#define PyPARSE_ASYNC_HACKS   0x0080
+
 typedef struct _memo {
     int type;
     void *node;
@@ -41,6 +58,7 @@ typedef struct {
     int starting_lineno;
     int starting_col_offset;
     int error_indicator;
+    int flags;
 } Parser;
 
 typedef struct {
@@ -137,13 +155,13 @@ CHECK_CALL_NULL_ALLOWED(Parser *p, void *result)
 #define CHECK_NULL_ALLOWED(result) CHECK_CALL_NULL_ALLOWED(p, result)
 
 PyObject *_PyPegen_new_identifier(Parser *, char *);
-Parser *_PyPegen_Parser_New(struct tok_state *, int, int *, PyArena *);
+Parser *_PyPegen_Parser_New(struct tok_state *, int, int, int *, PyArena *);
 void _PyPegen_Parser_Free(Parser *);
 mod_ty _PyPegen_run_parser_from_file_pointer(FILE *, int, PyObject *, const char *,
-                                    const char *, const char *, int *, PyArena *);
+                                    const char *, const char *, PyCompilerFlags *, int *, PyArena *);
 void *_PyPegen_run_parser(Parser *);
-mod_ty _PyPegen_run_parser_from_file(const char *, int, PyObject *, PyArena *);
-mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, int, PyArena *);
+mod_ty _PyPegen_run_parser_from_file(const char *, int, PyObject *, PyCompilerFlags *, PyArena *);
+mod_ty _PyPegen_run_parser_from_string(const char *, int, PyObject *, PyCompilerFlags *, PyArena *);
 void *_PyPegen_interactive_exit(Parser *);
 asdl_seq *_PyPegen_singleton_seq(Parser *, void *);
 asdl_seq *_PyPegen_seq_insert_in_front(Parser *, void *, asdl_seq *);
@@ -174,6 +192,7 @@ asdl_seq *_PyPegen_seq_delete_starred_exprs(Parser *, asdl_seq *);
 expr_ty _PyPegen_concatenate_strings(Parser *p, asdl_seq *);
 asdl_seq *_PyPegen_join_sequences(Parser *, asdl_seq *, asdl_seq *);
 void *_PyPegen_arguments_parsing_error(Parser *, expr_ty);
+int _PyPegen_check_barry_as_flufl(Parser *);
 
 void *_PyPegen_parse(Parser *);
 

diff --git a/Python/pythonrun.c b/Python/pythonrun.c
index 3a2fe966c08ba1041de2f9a264178a9bb06030cb..79147e430a1ad74e1b1c5d1358481fa446bff308 100644 (file)
--- a/Python/pythonrun.c
+++ b/Python/pythonrun.c
@@ -241,7 +241,7 @@ PyRun_InteractiveOneObjectEx(FILE *fp, PyObject *filename,
 
     if (use_peg) {
         mod = PyPegen_ASTFromFileObject(fp, filename, Py_single_input,
-                                        enc, ps1, ps2, &errcode, arena);
+                                        enc, ps1, ps2, flags, &errcode, arena);
     }
     else {
         mod = PyParser_ASTFromFileObject(fp, filename, enc,
@@ -1073,7 +1073,7 @@ PyRun_FileExFlags(FILE *fp, const char *filename_str, int start, PyObject *globa
 
     if (use_peg) {
         mod = PyPegen_ASTFromFileObject(fp, filename, start, NULL, NULL, NULL,
-                                        NULL, arena);
+                                        flags, NULL, arena);
     }
     else {
         mod = PyParser_ASTFromFileObject(fp, filename, NULL, start, 0, 0,

diff --git a/Tools/peg_generator/peg_extension/peg_extension.c b/Tools/peg_generator/peg_extension/peg_extension.c
index d8d36a0a1a5b0c45f9f69b38a0c05a5abe2e53ee..fb552eed3ba01a8fe27ad4b258f47ecff5cd3c74 100644 (file)
--- a/Tools/peg_generator/peg_extension/peg_extension.c
+++ b/Tools/peg_generator/peg_extension/peg_extension.c
@@ -12,7 +12,6 @@ _build_return_object(mod_ty module, int mode, PyObject *filename_ob, PyArena *ar
     } else {
         result = Py_None;
         Py_INCREF(result);
-        
     }
 
     return result;
@@ -43,7 +42,8 @@ parse_file(PyObject *self, PyObject *args, PyObject *kwds)
         goto error;
     }
 
-    mod_ty res = _PyPegen_run_parser_from_file(filename, Py_file_input, filename_ob, arena);
+    PyCompilerFlags flags = _PyCompilerFlags_INIT;
+    mod_ty res = _PyPegen_run_parser_from_file(filename, Py_file_input, filename_ob, &flags, arena);
     if (res == NULL) {
         goto error;
     }
@@ -81,8 +81,9 @@ parse_string(PyObject *self, PyObject *args, PyObject *kwds)
         goto error;
     }
 
+    PyCompilerFlags flags = _PyCompilerFlags_INIT;
     mod_ty res = _PyPegen_run_parser_from_string(the_string, Py_file_input, filename_ob,
-                                        PyCF_IGNORE_COOKIE, arena);
+                                        &flags, arena);
     if (res == NULL) {
         goto error;
     }

diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py
index 5b9d80453ca6b33653b4a73afcef4c454c53e33e..6c4b8f1e7df85356850200f50aeadf382fb48b10 100644 (file)
--- a/Tools/peg_generator/pegen/c_generator.py
+++ b/Tools/peg_generator/pegen/c_generator.py
@@ -73,9 +73,17 @@ class CCallMakerVisitor(GrammarVisitor):
             return "literal", f"_PyPegen_expect_token(p, {type})"
 
     def visit_Rhs(self, node: Rhs) -> Tuple[Optional[str], str]:
+        def can_we_inline(node):
+            if len(node.alts) != 1 or len(node.alts[0].items) != 1:
+                return False
+            # If the alternative has an action we cannot inline
+            if getattr(node.alts[0], "action", None) is not None:
+                return False
+            return True
+
         if node in self.cache:
             return self.cache[node]
-        if len(node.alts) == 1 and len(node.alts[0].items) == 1:
+        if can_we_inline(node):
             self.cache[node] = self.visit(node.alts[0].items[0])
         else:
             name = self.gen.name_node(node)