From 8e0fd8a5681b717de68052b8822813721748b65e Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sat, 19 Jan 2019 09:29:14 -0800 Subject: [PATCH] Regenerated files (make regen-token) --- Doc/library/token-list.inc | 4 ++++ Include/token.h | 6 ++++-- Lib/token.py | 12 +++++++----- Parser/token.c | 2 ++ 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/Doc/library/token-list.inc b/Doc/library/token-list.inc index cd6e0f26968e..290d8f2cd89a 100644 --- a/Doc/library/token-list.inc +++ b/Doc/library/token-list.inc @@ -199,6 +199,10 @@ .. data:: OP +.. data:: TYPE_IGNORE + +.. data:: TYPE_COMMENT + .. data:: ERRORTOKEN .. data:: N_TOKENS diff --git a/Include/token.h b/Include/token.h index 2d491e6927d1..6a1fa2cb9b82 100644 --- a/Include/token.h +++ b/Include/token.h @@ -64,8 +64,10 @@ extern "C" { #define RARROW 51 #define ELLIPSIS 52 #define OP 53 -#define ERRORTOKEN 54 -#define N_TOKENS 58 +#define TYPE_IGNORE 54 +#define TYPE_COMMENT 55 +#define ERRORTOKEN 56 +#define N_TOKENS 60 #define NT_OFFSET 256 /* Special definitions for cooperation with parser */ diff --git a/Lib/token.py b/Lib/token.py index 5af7e6b91eac..43eb246c3f04 100644 --- a/Lib/token.py +++ b/Lib/token.py @@ -57,12 +57,14 @@ ATEQUAL = 50 RARROW = 51 ELLIPSIS = 52 OP = 53 +TYPE_IGNORE = 54 +TYPE_COMMENT = 55 # These aren't used by the C tokenizer but are needed for tokenize.py -ERRORTOKEN = 54 -COMMENT = 55 -NL = 56 -ENCODING = 57 -N_TOKENS = 58 +ERRORTOKEN = 56 +COMMENT = 57 +NL = 58 +ENCODING = 59 +N_TOKENS = 60 # Special definitions for cooperation with parser NT_OFFSET = 256 diff --git a/Parser/token.c b/Parser/token.c index 35519aa4b611..9404fe63d611 100644 --- a/Parser/token.c +++ b/Parser/token.c @@ -60,6 +60,8 @@ const char * const _PyParser_TokenNames[] = { "RARROW", "ELLIPSIS", "OP", + "TYPE_IGNORE", + "TYPE_COMMENT", "", "", "", -- 2.47.3