Regenerated files (make regen-token)

diff --git a/Doc/library/token-list.inc b/Doc/library/token-list.inc
index cd6e0f26968eeaf68b3c43dcd6697261776432ba..290d8f2cd89a75528f17affb41fb91ca61a01fed 100644 (file)
--- a/Doc/library/token-list.inc
+++ b/Doc/library/token-list.inc
@@ -199,6 +199,10 @@
 
 .. data:: OP
 
+.. data:: TYPE_IGNORE
+
+.. data:: TYPE_COMMENT
+
 .. data:: ERRORTOKEN
 
 .. data:: N_TOKENS

diff --git a/Include/token.h b/Include/token.h
index 2d491e6927d1a912ada438fd5b45f915e3e6db1e..6a1fa2cb9b824330d5042f139b0313d70395ec03 100644 (file)
--- a/Include/token.h
+++ b/Include/token.h
@@ -64,8 +64,10 @@ extern "C" {
 #define RARROW          51
 #define ELLIPSIS        52
 #define OP              53
-#define ERRORTOKEN      54
-#define N_TOKENS        58
+#define TYPE_IGNORE     54
+#define TYPE_COMMENT    55
+#define ERRORTOKEN      56
+#define N_TOKENS        60
 #define NT_OFFSET       256
 
 /* Special definitions for cooperation with parser */

diff --git a/Lib/token.py b/Lib/token.py
index 5af7e6b91eaca3eb00a91e472f87b1fd1f223dad..43eb246c3f04123f523257279b36db790af9ab15 100644 (file)
--- a/Lib/token.py
+++ b/Lib/token.py
@@ -57,12 +57,14 @@ ATEQUAL = 50
 RARROW = 51
 ELLIPSIS = 52
 OP = 53
+TYPE_IGNORE = 54
+TYPE_COMMENT = 55
 # These aren't used by the C tokenizer but are needed for tokenize.py
-ERRORTOKEN = 54
-COMMENT = 55
-NL = 56
-ENCODING = 57
-N_TOKENS = 58
+ERRORTOKEN = 56
+COMMENT = 57
+NL = 58
+ENCODING = 59
+N_TOKENS = 60
 # Special definitions for cooperation with parser
 NT_OFFSET = 256
 

diff --git a/Parser/token.c b/Parser/token.c
index 35519aa4b61161e8bed0a29e9c89ae3f6910f246..9404fe63d611c2ea45516487c5490041327213be 100644 (file)
--- a/Parser/token.c
+++ b/Parser/token.c
@@ -60,6 +60,8 @@ const char * const _PyParser_TokenNames[] = {
     "RARROW",
     "ELLIPSIS",
     "OP",
+    "TYPE_IGNORE",
+    "TYPE_COMMENT",
     "<ERRORTOKEN>",
     "<COMMENT>",
     "<NL>",