Make recognition of type comments conditional on a flag (off by default)

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 17af219178c451eb2dfaa9752fae48b62aec7090..d9d7f999466ab80238e89ede6d53c78c82fcbafd 100644 (file)
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -86,6 +86,7 @@ tok_new(void)
     tok->decoding_readline = NULL;
     tok->decoding_buffer = NULL;
 #endif
+    tok->type_comments = 0;
 
     return tok;
 }
@@ -1253,51 +1254,54 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
     if (c == '#') {
         const char *prefix, *p, *type_start;
 
-        while (c != EOF && c != '\n')
+        while (c != EOF && c != '\n') {
             c = tok_nextc(tok);
+        }
 
-        p = tok->start;
-        prefix = type_comment_prefix;
-        while (*prefix && p < tok->cur) {
-            if (*prefix == ' ') {
-                while (*p == ' ' || *p == '\t')
+        if (tok->type_comments) {
+            p = tok->start;
+            prefix = type_comment_prefix;
+            while (*prefix && p < tok->cur) {
+                if (*prefix == ' ') {
+                    while (*p == ' ' || *p == '\t')
+                        p++;
+                } else if (*prefix == *p) {
                     p++;
-            } else if (*prefix == *p) {
-                p++;
-            } else {
-                break;
-            }
+                } else {
+                    break;
+                }
 
-            prefix++;
-        }
+                prefix++;
+            }
 
-        /* This is a type comment if we matched all of type_comment_prefix. */
-        if (!*prefix) {
-            int is_type_ignore = 1;
-            tok_backup(tok, c);  /* don't eat the newline or EOF */
+            /* This is a type comment if we matched all of type_comment_prefix. */
+            if (!*prefix) {
+                int is_type_ignore = 1;
+                tok_backup(tok, c);  /* don't eat the newline or EOF */
 
-            type_start = p;
+                type_start = p;
 
-            is_type_ignore = tok->cur >= p + 6 && memcmp(p, "ignore", 6) == 0;
-            p += 6;
-            while (is_type_ignore && p < tok->cur) {
-              if (*p == '#')
-                  break;
-              is_type_ignore = is_type_ignore && (*p == ' ' || *p == '\t');
-              p++;
-            }
+                is_type_ignore = tok->cur >= p + 6 && memcmp(p, "ignore", 6) == 0;
+                p += 6;
+                while (is_type_ignore && p < tok->cur) {
+                  if (*p == '#')
+                      break;
+                  is_type_ignore = is_type_ignore && (*p == ' ' || *p == '\t');
+                  p++;
+                }
 
-            if (is_type_ignore) {
-                /* If this type ignore is the only thing on the line, consume the newline also. */
-                if (blankline) {
-                    tok_nextc(tok);
-                    tok->atbol = 1;
+                if (is_type_ignore) {
+                    /* If this type ignore is the only thing on the line, consume the newline also. */
+                    if (blankline) {
+                        tok_nextc(tok);
+                        tok->atbol = 1;
+                    }
+                    return TYPE_IGNORE;
+                } else {
+                    *p_start = (char *) type_start;  /* after type_comment_prefix */
+                    *p_end = tok->cur;
+                    return TYPE_COMMENT;
                 }
-                return TYPE_IGNORE;
-            } else {
-                *p_start = (char *) type_start;  /* after type_comment_prefix */
-                *p_end = tok->cur;
-                return TYPE_COMMENT;
             }
         }
     }

diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
index 096ce687ec54e4f8bf5cfa92a09656453ac47fe0..9639c658b1c2b34a89af5c5edcf1c6dc1b758cee 100644 (file)
--- a/Parser/tokenizer.h
+++ b/Parser/tokenizer.h
@@ -70,6 +70,8 @@ struct tok_state {
     const char* enc;        /* Encoding for the current str. */
     const char* str;
     const char* input; /* Tokenizer's newline translated copy of the string. */
+
+    int type_comments;      /* Whether to look for type comments */
 };
 
 extern struct tok_state *PyTokenizer_FromString(const char *, int);