From: Guido van Rossum Date: Tue, 22 Jan 2019 02:15:39 +0000 (-0800) Subject: Support TYPE_COMMENT in ast.c for all nodes that need it X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=d273d6746aec19a666008afd5c1b4c2cd35807a4;p=thirdparty%2FPython%2Fcpython.git Support TYPE_COMMENT in ast.c for all nodes that need it This works, it just needs tests --- diff --git a/Python/ast.c b/Python/ast.c index f8ca99e39da4..9a8c9a6b4215 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -696,6 +696,13 @@ ast_error(struct compiling *c, const node *n, const char *errmsg, ...) small_stmt elements is returned. */ +static string +new_type_comment(const char *s, struct compiling *c) +{ + return PyUnicode_DecodeUTF8(s, strlen(s), NULL); +} +#define NEW_TYPE_COMMENT(n) new_type_comment(STR(n), c) + static int num_stmts(const node *n) { @@ -723,11 +730,15 @@ num_stmts(const node *n) case simple_stmt: return NCH(n) / 2; /* Divide by 2 to remove count of semi-colons */ case suite: + /* suite: simple_stmt | NEWLINE [TYPE_COMMENT NEWLINE] INDENT stmt+ DEDENT */ if (NCH(n) == 1) return num_stmts(CHILD(n, 0)); else { + i = 2; l = 0; - for (i = 2; i < (NCH(n) - 1); i++) + if (TYPE(CHILD(n, 1)) == TYPE_COMMENT) + i += 2; + for (; i < (NCH(n) - 1); i++) l += num_stmts(CHILD(n, i)); return l; } @@ -756,6 +767,8 @@ PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags, node *ch; struct compiling c; mod_ty res = NULL; + asdl_seq *argtypes = NULL; + expr_ty ret, arg; c.c_arena = arena; /* borrowed reference */ @@ -862,6 +875,40 @@ PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags, res = Interactive(stmts, arena); } break; + case func_type_input: + n = CHILD(n, 0); + REQ(n, func_type); + + if (TYPE(CHILD(n, 1)) == typelist) { + ch = CHILD(n, 1); + /* this is overly permissive -- we don't pay any attention to + * stars on the args -- just parse them into an ordered list */ + num = 0; + for (i = 0; i < NCH(ch); i++) { + if (TYPE(CHILD(ch, i)) == test) + num++; + } + + argtypes = _Py_asdl_seq_new(num, arena); + + j = 0; + for (i = 0; i < NCH(ch); i++) { + if (TYPE(CHILD(ch, i)) == test) { + arg = ast_for_expr(&c, CHILD(ch, i)); + if (!arg) + goto out; + asdl_seq_SET(argtypes, j++, arg); + } + } + } + else + argtypes = _Py_asdl_seq_new(0, arena); + + ret = ast_for_expr(&c, CHILD(n, NCH(n) - 1)); + if (!ret) + goto out; + res = FunctionType(argtypes, ret, arena); + break; default: PyErr_Format(PyExc_SystemError, "invalid node %d for PyAST_FromNode", TYPE(n)); @@ -1314,7 +1361,14 @@ handle_keywordonly_args(struct compiling *c, const node *n, int start, if (!arg) goto error; asdl_seq_SET(kwonlyargs, j++, arg); - i += 2; /* the name and the comma */ + i += 1; /* the name */ + if (TYPE(CHILD(n, i)) == COMMA) + i += 1; /* the comma, if present */ + break; + case TYPE_COMMENT: + /* arg will be equal to the last argument processed */ + arg->type_comment = NEW_TYPE_COMMENT(ch); + i += 1; break; case DOUBLESTAR: return i; @@ -1444,11 +1498,14 @@ ast_for_arguments(struct compiling *c, const node *n) if (!arg) return NULL; asdl_seq_SET(posargs, k++, arg); - i += 2; /* the name and the comma */ + i += 1; /* the name */ + if (TYPE(CHILD(n, i)) == COMMA) + i += 1; /* the comma, if present */ break; case STAR: if (i+1 >= NCH(n) || - (i+2 == NCH(n) && TYPE(CHILD(n, i+1)) == COMMA)) { + (i+2 == NCH(n) && (TYPE(CHILD(n, i+1)) == COMMA + || TYPE(CHILD(n, i+1)) == TYPE_COMMENT))) { ast_error(c, CHILD(n, i), "named arguments must follow bare *"); return NULL; @@ -1457,6 +1514,13 @@ ast_for_arguments(struct compiling *c, const node *n) if (TYPE(ch) == COMMA) { int res = 0; i += 2; /* now follows keyword only arguments */ + + if (TYPE(CHILD(n, i)) == TYPE_COMMENT) { + ast_error(c, CHILD(n, i), + "bare * has associated type comment"); + return NULL; + } + res = handle_keywordonly_args(c, n, i, kwonlyargs, kwdefaults); if (res == -1) return NULL; @@ -1467,7 +1531,15 @@ ast_for_arguments(struct compiling *c, const node *n) if (!vararg) return NULL; - i += 3; + i += 2; /* the star and the name */ + if (TYPE(CHILD(n, i)) == COMMA) + i += 1; /* the comma, if present */ + + if (TYPE(CHILD(n, i)) == TYPE_COMMENT) { + vararg->type_comment = NEW_TYPE_COMMENT(CHILD(n, i)); + i += 1; + } + if (i < NCH(n) && (TYPE(CHILD(n, i)) == tfpdef || TYPE(CHILD(n, i)) == vfpdef)) { int res = 0; @@ -1484,7 +1556,19 @@ ast_for_arguments(struct compiling *c, const node *n) kwarg = ast_for_arg(c, ch); if (!kwarg) return NULL; - i += 3; + i += 2; /* the double star and the name */ + if (TYPE(CHILD(n, i)) == COMMA) + i += 1; /* the comma, if present */ + break; + case TYPE_COMMENT: + assert(i); + + if (kwarg) + arg = kwarg; + + /* arg will be equal to the last argument processed */ + arg->type_comment = NEW_TYPE_COMMENT(ch); + i += 1; break; default: PyErr_Format(PyExc_SystemError, @@ -1593,7 +1677,7 @@ static stmt_ty ast_for_funcdef_impl(struct compiling *c, const node *n0, asdl_seq *decorator_seq, bool is_async) { - /* funcdef: 'def' NAME parameters ['->' test] ':' suite */ + /* funcdef: 'def' NAME parameters ['->' test] ':' [TYPE_COMMENT] suite */ const node * const n = is_async ? CHILD(n0, 1) : n0; identifier name; arguments_ty args; @@ -1601,6 +1685,8 @@ ast_for_funcdef_impl(struct compiling *c, const node *n0, expr_ty returns = NULL; int name_i = 1; int end_lineno, end_col_offset; + node *tc; + string type_comment = NULL; REQ(n, funcdef); @@ -1618,16 +1704,29 @@ ast_for_funcdef_impl(struct compiling *c, const node *n0, return NULL; name_i += 2; } + if (TYPE(CHILD(n, name_i + 3)) == TYPE_COMMENT) { + type_comment = NEW_TYPE_COMMENT(CHILD(n, name_i + 3)); + name_i += 1; + } body = ast_for_suite(c, CHILD(n, name_i + 3)); if (!body) return NULL; get_last_end_pos(body, &end_lineno, &end_col_offset); + if (!type_comment && NCH(CHILD(n, name_i + 3)) > 1) { + /* If the function doesn't have a type comment on the same line, check + * if the suite has a type comment in it. */ + tc = CHILD(CHILD(n, name_i + 3), 1); + + if (TYPE(tc) == TYPE_COMMENT) + type_comment = NEW_TYPE_COMMENT(tc); + } + if (is_async) - return AsyncFunctionDef(name, args, body, decorator_seq, returns, NULL, + return AsyncFunctionDef(name, args, body, decorator_seq, returns, type_comment, LINENO(n0), n0->n_col_offset, end_lineno, end_col_offset, c->c_arena); else - return FunctionDef(name, args, body, decorator_seq, returns, NULL, + return FunctionDef(name, args, body, decorator_seq, returns, type_comment, LINENO(n), n->n_col_offset, end_lineno, end_col_offset, c->c_arena); } @@ -2968,15 +3067,16 @@ ast_for_expr_stmt(struct compiling *c, const node *n) { REQ(n, expr_stmt); /* expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) | - ('=' (yield_expr|testlist_star_expr))*) + ('=' (yield_expr|testlist_star_expr))* [TYPE_COMMENT]) annassign: ':' test ['=' test] testlist_star_expr: (test|star_expr) (',' test|star_expr)* [','] augassign: '+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=' | '**=' | '//=' test: ... here starts the operator precedence dance */ + int num = NCH(n); - if (NCH(n) == 1) { + if (num == 1 || (num == 2 && TYPE(CHILD(n, 1)) == TYPE_COMMENT)) { expr_ty e = ast_for_testlist(c, CHILD(n, 0)); if (!e) return NULL; @@ -3096,17 +3196,22 @@ ast_for_expr_stmt(struct compiling *c, const node *n) } } else { - int i; + int i, nch_minus_type, has_type_comment; asdl_seq *targets; node *value; expr_ty expression; + string type_comment; /* a normal assignment */ REQ(CHILD(n, 1), EQUAL); - targets = _Py_asdl_seq_new(NCH(n) / 2, c->c_arena); + + has_type_comment = TYPE(CHILD(n, num - 1)) == TYPE_COMMENT; + nch_minus_type = num - has_type_comment; + + targets = _Py_asdl_seq_new(nch_minus_type / 2, c->c_arena); if (!targets) return NULL; - for (i = 0; i < NCH(n) - 2; i += 2) { + for (i = 0; i < nch_minus_type - 2; i += 2) { expr_ty e; node *ch = CHILD(n, i); if (TYPE(ch) == yield_expr) { @@ -3123,14 +3228,18 @@ ast_for_expr_stmt(struct compiling *c, const node *n) asdl_seq_SET(targets, i / 2, e); } - value = CHILD(n, NCH(n) - 1); + value = CHILD(n, nch_minus_type - 1); if (TYPE(value) == testlist_star_expr) expression = ast_for_testlist(c, value); else expression = ast_for_expr(c, value); if (!expression) return NULL; - return Assign(targets, expression, NULL, LINENO(n), n->n_col_offset, + if (has_type_comment) + type_comment = NEW_TYPE_COMMENT(CHILD(n, nch_minus_type)); + else + type_comment = NULL; + return Assign(targets, expression, type_comment, LINENO(n), n->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } } @@ -3557,7 +3666,7 @@ ast_for_assert_stmt(struct compiling *c, const node *n) static asdl_seq * ast_for_suite(struct compiling *c, const node *n) { - /* suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT */ + /* suite: simple_stmt | NEWLINE [TYPE_COMMENT NEWLINE] INDENT stmt+ DEDENT */ asdl_seq *seq; stmt_ty s; int i, total, num, end, pos = 0; @@ -3587,7 +3696,11 @@ ast_for_suite(struct compiling *c, const node *n) } } else { - for (i = 2; i < (NCH(n) - 1); i++) { + i = 2; + if (TYPE(CHILD(n, 1)) == TYPE_COMMENT) + i += 2; + + for (; i < (NCH(n) - 1); i++) { ch = CHILD(n, i); REQ(ch, stmt); num = num_stmts(ch); @@ -3821,11 +3934,15 @@ ast_for_for_stmt(struct compiling *c, const node *n0, bool is_async) expr_ty target, first; const node *node_target; int end_lineno, end_col_offset; - /* for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] */ + int has_type_comment; + string type_comment; + /* for_stmt: 'for' exprlist 'in' testlist ':' [TYPE_COMMENT] suite ['else' ':' suite] */ REQ(n, for_stmt); - if (NCH(n) == 9) { - seq = ast_for_suite(c, CHILD(n, 8)); + has_type_comment = TYPE(CHILD(n, 5)) == TYPE_COMMENT; + + if (NCH(n) == 9 + has_type_comment) { + seq = ast_for_suite(c, CHILD(n, 8 + has_type_comment)); if (!seq) return NULL; } @@ -3847,7 +3964,7 @@ ast_for_for_stmt(struct compiling *c, const node *n0, bool is_async) expression = ast_for_testlist(c, CHILD(n, 3)); if (!expression) return NULL; - suite_seq = ast_for_suite(c, CHILD(n, 5)); + suite_seq = ast_for_suite(c, CHILD(n, 5 + has_type_comment)); if (!suite_seq) return NULL; @@ -3856,12 +3973,18 @@ ast_for_for_stmt(struct compiling *c, const node *n0, bool is_async) } else { get_last_end_pos(suite_seq, &end_lineno, &end_col_offset); } + + if (has_type_comment) + type_comment = NEW_TYPE_COMMENT(CHILD(n, 5)); + else + type_comment = NULL; + if (is_async) - return AsyncFor(target, expression, suite_seq, seq, NULL, + return AsyncFor(target, expression, suite_seq, seq, type_comment, LINENO(n0), n0->n_col_offset, end_lineno, end_col_offset, c->c_arena); else - return For(target, expression, suite_seq, seq, NULL, + return For(target, expression, suite_seq, seq, type_comment, LINENO(n), n->n_col_offset, end_lineno, end_col_offset, c->c_arena); } @@ -4029,21 +4152,25 @@ ast_for_with_item(struct compiling *c, const node *n) return withitem(context_expr, optional_vars, c->c_arena); } -/* with_stmt: 'with' with_item (',' with_item)* ':' suite */ +/* with_stmt: 'with' with_item (',' with_item)* ':' [TYPE_COMMENT] suite */ static stmt_ty ast_for_with_stmt(struct compiling *c, const node *n0, bool is_async) { const node * const n = is_async ? CHILD(n0, 1) : n0; - int i, n_items, end_lineno, end_col_offset; + int i, n_items, nch_minus_type, has_type_comment, end_lineno, end_col_offset; asdl_seq *items, *body; + string type_comment; REQ(n, with_stmt); - n_items = (NCH(n) - 2) / 2; + has_type_comment = TYPE(CHILD(n, NCH(n) - 2)) == TYPE_COMMENT; + nch_minus_type = NCH(n) - has_type_comment; + + n_items = (nch_minus_type - 2) / 2; items = _Py_asdl_seq_new(n_items, c->c_arena); if (!items) return NULL; - for (i = 1; i < NCH(n) - 2; i += 2) { + for (i = 1; i < nch_minus_type - 2; i += 2) { withitem_ty item = ast_for_with_item(c, CHILD(n, i)); if (!item) return NULL; @@ -4055,11 +4182,16 @@ ast_for_with_stmt(struct compiling *c, const node *n0, bool is_async) return NULL; get_last_end_pos(body, &end_lineno, &end_col_offset); + if (has_type_comment) + type_comment = NEW_TYPE_COMMENT(CHILD(n, NCH(n) - 2)); + else + type_comment = NULL; + if (is_async) - return AsyncWith(items, body, NULL, LINENO(n0), n0->n_col_offset, + return AsyncWith(items, body, type_comment, LINENO(n0), n0->n_col_offset, end_lineno, end_col_offset, c->c_arena); else - return With(items, body, NULL, LINENO(n), n->n_col_offset, + return With(items, body, type_comment, LINENO(n), n->n_col_offset, end_lineno, end_col_offset, c->c_arena); }