| fstring_replacement_field
| t=FSTRING_MIDDLE { _PyPegen_constant_from_token(p, t) }
fstring_replacement_field[expr_ty]:
- | '{' a=(yield_expr | star_expressions) debug_expr="="? conversion=[fstring_conversion] format=[fstring_full_format_spec] '}' {
- _PyPegen_formatted_value(p, a, debug_expr, conversion, format, EXTRA)
- }
+ | '{' a=(yield_expr | star_expressions) debug_expr="="? conversion=[fstring_conversion] format=[fstring_full_format_spec] rbrace='}' {
+ _PyPegen_formatted_value(p, a, debug_expr, conversion, format, rbrace, EXTRA) }
| invalid_replacement_field
-fstring_conversion[expr_ty]:
+fstring_conversion[ResultTokenWithMetadata*]:
| conv_token="!" conv=NAME { _PyPegen_check_fstring_conversion(p, conv_token, conv) }
-fstring_full_format_spec[expr_ty]:
- | ':' spec=fstring_format_spec* { spec ? _PyAST_JoinedStr((asdl_expr_seq*)spec, EXTRA) : NULL }
+fstring_full_format_spec[ResultTokenWithMetadata*]:
+ | colon=':' spec=fstring_format_spec* { _PyPegen_setup_full_format_spec(p, colon, (asdl_expr_seq *) spec, EXTRA) }
fstring_format_spec[expr_ty]:
| t=FSTRING_MIDDLE { _PyPegen_constant_from_token(p, t) }
| fstring_replacement_field
self.assertAllRaise(SyntaxError, "unterminated triple-quoted f-string literal",
['f"""', "f'''"])
+ def test_syntax_error_after_debug(self):
+ self.assertAllRaise(SyntaxError, "f-string: expecting a valid expression after '{'",
+ [
+ "f'{1=}{;'",
+ "f'{1=}{+;'",
+ "f'{1=}{2}{;'",
+ "f'{1=}{3}{;'",
+ ])
+ self.assertAllRaise(SyntaxError, "f-string: expecting '=', or '!', or ':', or '}'",
+ [
+ "f'{1=}{1;'",
+ "f'{1=}{1;}'",
+ ])
+
if __name__ == '__main__':
unittest.main()
return 0;
}
-expr_ty
-_PyPegen_check_fstring_conversion(Parser *p, Token* symbol, expr_ty conv) {
- if (symbol->lineno != conv->lineno || symbol->end_col_offset != conv->col_offset) {
+static ResultTokenWithMetadata *
+result_token_with_metadata(Parser *p, void *result, PyObject *metadata)
+{
+ ResultTokenWithMetadata *res = _PyArena_Malloc(p->arena, sizeof(ResultTokenWithMetadata));
+ if (res == NULL) {
+ return NULL;
+ }
+ res->metadata = metadata;
+ res->result = result;
+ return res;
+}
+
+ResultTokenWithMetadata *
+_PyPegen_check_fstring_conversion(Parser *p, Token* conv_token, expr_ty conv)
+{
+ if (conv_token->lineno != conv->lineno || conv_token->end_col_offset != conv->col_offset) {
return RAISE_SYNTAX_ERROR_KNOWN_RANGE(
- symbol, conv,
+ conv_token, conv,
"f-string: conversion type must come right after the exclamanation mark"
);
}
- return conv;
+ return result_token_with_metadata(p, conv, conv_token->metadata);
}
+ResultTokenWithMetadata *
+_PyPegen_setup_full_format_spec(Parser *p, Token *colon, asdl_expr_seq *spec, int lineno, int col_offset,
+ int end_lineno, int end_col_offset, PyArena *arena)
+{
+ if (!spec) {
+ return NULL;
+ }
+ expr_ty res = _PyAST_JoinedStr(spec, lineno, col_offset, end_lineno, end_col_offset, p->arena);
+ if (!res) {
+ return NULL;
+ }
+ return result_token_with_metadata(p, res, colon->metadata);
+}
const char *
_PyPegen_get_expr_name(expr_ty e)
// Fstring stuff
-static expr_ty
-decode_fstring_buffer(Parser *p, int lineno, int col_offset, int end_lineno,
- int end_col_offset)
-{
- tokenizer_mode *tok_mode = &(p->tok->tok_mode_stack[p->tok->tok_mode_stack_index]);
- assert(tok_mode->last_expr_buffer != NULL);
- assert(tok_mode->last_expr_size >= 0 && tok_mode->last_expr_end >= 0);
-
- PyObject *res = PyUnicode_DecodeUTF8(
- tok_mode->last_expr_buffer,
- tok_mode->last_expr_size - tok_mode->last_expr_end,
- NULL
- );
- if (!res || _PyArena_AddPyObject(p->arena, res) < 0) {
- Py_XDECREF(res);
- return NULL;
- }
-
- return _PyAST_Constant(res, NULL, lineno, col_offset, end_lineno, end_col_offset, p->arena);
-}
-
static expr_ty
_PyPegen_decode_fstring_part(Parser* p, int is_raw, expr_ty constant) {
assert(PyUnicode_CheckExact(constant->v.Constant.value));
return _PyAST_Constant(s, kind, tok->lineno, tok->col_offset, tok->end_lineno, tok->end_col_offset, p->arena);
}
-expr_ty _PyPegen_formatted_value(Parser *p, expr_ty expression, Token *debug, expr_ty conversion,
- expr_ty format, int lineno, int col_offset, int end_lineno, int end_col_offset,
- PyArena *arena) {
+expr_ty _PyPegen_formatted_value(Parser *p, expr_ty expression, Token *debug, ResultTokenWithMetadata *conversion,
+ ResultTokenWithMetadata *format, Token *closing_brace, int lineno, int col_offset,
+ int end_lineno, int end_col_offset, PyArena *arena) {
int conversion_val = -1;
if (conversion != NULL) {
- assert(conversion->kind == Name_kind);
- Py_UCS4 first = PyUnicode_READ_CHAR(conversion->v.Name.id, 0);
+ expr_ty conversion_expr = (expr_ty) conversion->result;
+ assert(conversion_expr->kind == Name_kind);
+ Py_UCS4 first = PyUnicode_READ_CHAR(conversion_expr->v.Name.id, 0);
- if (PyUnicode_GET_LENGTH(conversion->v.Name.id) > 1 ||
+ if (PyUnicode_GET_LENGTH(conversion_expr->v.Name.id) > 1 ||
!(first == 's' || first == 'r' || first == 'a')) {
- RAISE_SYNTAX_ERROR_KNOWN_LOCATION(conversion,
+ RAISE_SYNTAX_ERROR_KNOWN_LOCATION(conversion_expr,
"f-string: invalid conversion character %R: expected 's', 'r', or 'a'",
- conversion->v.Name.id);
+ conversion_expr->v.Name.id);
return NULL;
}
}
expr_ty formatted_value = _PyAST_FormattedValue(
- expression, conversion_val, format,
+ expression, conversion_val, format ? (expr_ty) format->result : NULL,
lineno, col_offset, end_lineno,
end_col_offset, arena
);
if (debug) {
/* Find the non whitespace token after the "=" */
int debug_end_line, debug_end_offset;
+ PyObject *debug_metadata;
if (conversion) {
- debug_end_line = conversion->lineno;
- debug_end_offset = conversion->col_offset;
+ debug_end_line = ((expr_ty) conversion->result)->lineno;
+ debug_end_offset = ((expr_ty) conversion->result)->col_offset;
+ debug_metadata = conversion->metadata;
}
else if (format) {
- debug_end_line = format->lineno;
- debug_end_offset = format->col_offset + 1; // HACK: ??
+ debug_end_line = ((expr_ty) format->result)->lineno;
+ debug_end_offset = ((expr_ty) format->result)->col_offset + 1;
+ debug_metadata = format->metadata;
}
else {
debug_end_line = end_lineno;
debug_end_offset = end_col_offset;
+ debug_metadata = closing_brace->metadata;
}
- expr_ty debug_text = decode_fstring_buffer(p, lineno, col_offset + 1,
- debug_end_line, debug_end_offset - 1);
+ expr_ty debug_text = _PyAST_Constant(debug_metadata, NULL, lineno, col_offset + 1, debug_end_line,
+ debug_end_offset - 1, p->arena);
if (!debug_text) {
return NULL;
}
static arg_ty lambda_param_rule(Parser *p);
static expr_ty fstring_middle_rule(Parser *p);
static expr_ty fstring_replacement_field_rule(Parser *p);
-static expr_ty fstring_conversion_rule(Parser *p);
-static expr_ty fstring_full_format_spec_rule(Parser *p);
+static ResultTokenWithMetadata* fstring_conversion_rule(Parser *p);
+static ResultTokenWithMetadata* fstring_full_format_spec_rule(Parser *p);
static expr_ty fstring_format_spec_rule(Parser *p);
static expr_ty string_rule(Parser *p);
static expr_ty strings_rule(Parser *p);
}
D(fprintf(stderr, "%*c> fstring_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) \"=\"? fstring_conversion? fstring_full_format_spec? '}'"));
Token * _literal;
- Token * _literal_1;
void *a;
void *conversion;
void *debug_expr;
void *format;
+ Token * rbrace;
if (
(_literal = _PyPegen_expect_token(p, 25)) // token='{'
&&
&&
(format = fstring_full_format_spec_rule(p), !p->error_indicator) // fstring_full_format_spec?
&&
- (_literal_1 = _PyPegen_expect_token(p, 26)) // token='}'
+ (rbrace = _PyPegen_expect_token(p, 26)) // token='}'
)
{
D(fprintf(stderr, "%*c+ fstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' (yield_expr | star_expressions) \"=\"? fstring_conversion? fstring_full_format_spec? '}'"));
UNUSED(_end_lineno); // Only used by EXTRA macro
int _end_col_offset = _token->end_col_offset;
UNUSED(_end_col_offset); // Only used by EXTRA macro
- _res = _PyPegen_formatted_value ( p , a , debug_expr , conversion , format , EXTRA );
+ _res = _PyPegen_formatted_value ( p , a , debug_expr , conversion , format , rbrace , EXTRA );
if (_res == NULL && PyErr_Occurred()) {
p->error_indicator = 1;
p->level--;
}
// fstring_conversion: "!" NAME
-static expr_ty
+static ResultTokenWithMetadata*
fstring_conversion_rule(Parser *p)
{
if (p->level++ == MAXSTACK) {
p->level--;
return NULL;
}
- expr_ty _res = NULL;
+ ResultTokenWithMetadata* _res = NULL;
int _mark = p->mark;
{ // "!" NAME
if (p->error_indicator) {
}
// fstring_full_format_spec: ':' fstring_format_spec*
-static expr_ty
+static ResultTokenWithMetadata*
fstring_full_format_spec_rule(Parser *p)
{
if (p->level++ == MAXSTACK) {
p->level--;
return NULL;
}
- expr_ty _res = NULL;
+ ResultTokenWithMetadata* _res = NULL;
int _mark = p->mark;
if (p->mark == p->fill && _PyPegen_fill_token(p) < 0) {
p->error_indicator = 1;
return NULL;
}
D(fprintf(stderr, "%*c> fstring_full_format_spec[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':' fstring_format_spec*"));
- Token * _literal;
+ Token * colon;
asdl_seq * spec;
if (
- (_literal = _PyPegen_expect_token(p, 11)) // token=':'
+ (colon = _PyPegen_expect_token(p, 11)) // token=':'
&&
(spec = _loop0_112_rule(p)) // fstring_format_spec*
)
UNUSED(_end_lineno); // Only used by EXTRA macro
int _end_col_offset = _token->end_col_offset;
UNUSED(_end_col_offset); // Only used by EXTRA macro
- _res = spec ? _PyAST_JoinedStr ( ( asdl_expr_seq* ) spec , EXTRA ) : NULL;
+ _res = _PyPegen_setup_full_format_spec ( p , colon , ( asdl_expr_seq* ) spec , EXTRA );
if (_res == NULL && PyErr_Occurred()) {
p->error_indicator = 1;
p->level--;
return -1;
}
+ parser_token->metadata = NULL;
+ if (new_token->metadata != NULL) {
+ if (_PyArena_AddPyObject(p->arena, new_token->metadata) < 0) {
+ Py_DECREF(parser_token->metadata);
+ return -1;
+ }
+ parser_token->metadata = new_token->metadata;
+ new_token->metadata = NULL;
+ }
+
parser_token->level = new_token->level;
parser_token->lineno = new_token->lineno;
parser_token->col_offset = p->tok->lineno == p->starting_lineno ? p->starting_col_offset + new_token->col_offset
_PyPegen_fill_token(Parser *p)
{
struct token new_token;
+ new_token.metadata = NULL;
int type = _PyTokenizer_Get(p->tok, &new_token);
// Record and skip '# type: ignore' comments
char *tag = PyMem_Malloc(len + 1);
if (tag == NULL) {
PyErr_NoMemory();
- return -1;
+ goto error;
}
strncpy(tag, new_token.start, len);
tag[len] = '\0';
// Ownership of tag passes to the growable array
if (!growable_comment_array_add(&p->type_ignore_comments, p->tok->lineno, tag)) {
PyErr_NoMemory();
- return -1;
+ goto error;
}
type = _PyTokenizer_Get(p->tok, &new_token);
}
// Check if we are at the limit of the token array capacity and resize if needed
if ((p->fill == p->size) && (_resize_tokens_array(p) != 0)) {
- return -1;
+ goto error;
}
Token *t = p->tokens[p->fill];
return initialize_token(p, t, &new_token, type);
+error:
+ Py_XDECREF(new_token.metadata);
+ return -1;
}
#if defined(Py_DEBUG)
int level;
int lineno, col_offset, end_lineno, end_col_offset;
Memo *memo;
+ PyObject *metadata;
} Token;
typedef struct {
int is_keyword;
} KeywordOrStarred;
+typedef struct {
+ void *result;
+ PyObject *metadata;
+} ResultTokenWithMetadata;
+
// Internal parser functions
#if defined(Py_DEBUG)
void _PyPegen_clear_memo_statistics(void);
arguments_ty _PyPegen_make_arguments(Parser *, asdl_arg_seq *, SlashWithDefault *,
asdl_arg_seq *, asdl_seq *, StarEtc *);
arguments_ty _PyPegen_empty_arguments(Parser *);
-expr_ty _PyPegen_formatted_value(Parser *, expr_ty, Token *, expr_ty, expr_ty, int, int, int, int, PyArena *);
+expr_ty _PyPegen_formatted_value(Parser *, expr_ty, Token *, ResultTokenWithMetadata *, ResultTokenWithMetadata *, Token *,
+ int, int, int, int, PyArena *);
AugOperator *_PyPegen_augoperator(Parser*, operator_ty type);
stmt_ty _PyPegen_function_def_decorators(Parser *, asdl_expr_seq *, stmt_ty);
stmt_ty _PyPegen_class_def_decorators(Parser *, asdl_expr_seq *, stmt_ty);
asdl_seq *_PyPegen_join_sequences(Parser *, asdl_seq *, asdl_seq *);
int _PyPegen_check_barry_as_flufl(Parser *, Token *);
int _PyPegen_check_legacy_stmt(Parser *p, expr_ty t);
-expr_ty _PyPegen_check_fstring_conversion(Parser *p, Token *, expr_ty t);
+ResultTokenWithMetadata *_PyPegen_check_fstring_conversion(Parser *p, Token *, expr_ty t);
+ResultTokenWithMetadata *_PyPegen_setup_full_format_spec(Parser *, Token *, asdl_expr_seq *, int, int,
+ int, int, PyArena *);
mod_ty _PyPegen_make_module(Parser *, asdl_stmt_seq *);
void *_PyPegen_arguments_parsing_error(Parser *, expr_ty);
expr_ty _PyPegen_get_last_comprehension_item(comprehension_ty comprehension);
int ret = 0;
struct token new_token;
+ new_token.metadata = NULL;
for (;;) {
switch (_PyTokenizer_Get(p->tok, &new_token)) {
exit:
+ Py_XDECREF(new_token.metadata);
// If we're in an f-string, we want the syntax error in the expression part
// to propagate, so that tokenizer errors (like expecting '}') that happen afterwards
// do not swallow it.
tok->interactive_underflow = IUNDERFLOW_NORMAL;
tok->str = NULL;
tok->report_warnings = 1;
- tok->tok_mode_stack[0] = (tokenizer_mode){.kind =TOK_REGULAR_MODE, .f_string_quote='\0', .f_string_quote_size = 0};
+ tok->tok_mode_stack[0] = (tokenizer_mode){.kind =TOK_REGULAR_MODE, .f_string_quote='\0', .f_string_quote_size = 0, .f_string_debug=0};
tok->tok_mode_stack_index = 0;
tok->tok_report_warnings = 1;
#ifdef Py_DEBUG
}
}
+static int
+set_fstring_expr(struct tok_state* tok, struct token *token, char c) {
+ assert(token != NULL);
+ assert(c == '}' || c == ':' || c == '!');
+ tokenizer_mode *tok_mode = TOK_GET_MODE(tok);
+
+ if (!tok_mode->f_string_debug || token->metadata) {
+ return 0;
+ }
+
+ PyObject *res = PyUnicode_DecodeUTF8(
+ tok_mode->last_expr_buffer,
+ tok_mode->last_expr_size - tok_mode->last_expr_end,
+ NULL
+ );
+ if (!res) {
+ return -1;
+ }
+ token->metadata = res;
+ return 0;
+}
+
static int
update_fstring_expr(struct tok_state *tok, char cur)
{
the_current_tok->last_expr_buffer = NULL;
the_current_tok->last_expr_size = 0;
the_current_tok->last_expr_end = -1;
+ the_current_tok->f_string_debug = 0;
switch (*tok->start) {
case 'F':
* by the `{` case, so for ensuring that we are on the 0th level, we need
* to adjust it manually */
int cursor = current_tok->curly_bracket_depth - (c != '{');
-
if (cursor == 0 && !update_fstring_expr(tok, c)) {
return MAKE_TOKEN(ENDMARKER);
}
+ if (cursor == 0 && c != '{' && set_fstring_expr(tok, token, c)) {
+ return MAKE_TOKEN(ERRORTOKEN);
+ }
if (c == ':' && cursor == current_tok->curly_bracket_expr_start_depth) {
current_tok->kind = TOK_FSTRING_MODE;
if (c == '}' && current_tok->curly_bracket_depth == current_tok->curly_bracket_expr_start_depth) {
current_tok->curly_bracket_expr_start_depth--;
current_tok->kind = TOK_FSTRING_MODE;
+ current_tok->f_string_debug = 0;
}
}
break;
return MAKE_TOKEN(syntaxerror(tok, "invalid non-printable character U+%s", hex));
}
+ if( c == '=' && INSIDE_FSTRING_EXPR(current_tok)) {
+ current_tok->f_string_debug = 1;
+ }
+
/* Punctuation character */
p_start = tok->start;
p_end = tok->cur;
int level;
int lineno, col_offset, end_lineno, end_col_offset;
const char *start, *end;
+ PyObject *metadata;
};
enum tokenizer_mode_kind_t {
Py_ssize_t last_expr_size;
Py_ssize_t last_expr_end;
char* last_expr_buffer;
+ int f_string_debug;
} tokenizer_mode;
/* Tokenizer state */
projects / thirdparty / Python / cpython.git / commitdiff
