["f'{ {{}} }'", # dict in a set
])
+ def test_double_brace_ast_location_covers_both_source_braces(self):
+ value = ast.parse('f"a{{"').body[0].value.values[0]
+ self.assertIsInstance(value, ast.Constant)
+ self.assertEqual(value.value, "a{")
+ self.assertEqual(
+ (value.lineno, value.col_offset, value.end_lineno,
+ value.end_col_offset),
+ (1, 2, 1, 5),
+ )
+
def test_compile_time_concat(self):
x = 'def'
self.assertEqual('abc' f'## {x}ghi', 'abc## defghi')
self.assertEqual(tokens, expected_tokens)
class CTokenizeTest(TestCase):
+ @staticmethod
+ def _get_tokens(source, *, extra_tokens=False):
+ return list(tokenize._generate_tokens_from_c_tokenizer(
+ StringIO(source).readline,
+ extra_tokens=extra_tokens,
+ ))
+
def check_tokenize(self, s, expected):
# Format the tokens in s in a table format.
# The ENDMARKER and final NEWLINE are omitted.
))
self.assertEqual(tokens, expected)
+ def test_extra_tokens_relaxes_lexer_errors(self):
+ cases = [
+ (
+ "2sin(x)",
+ ("invalid decimal literal", (1, 1)),
+ [
+ (token.NUMBER, "2", (1, 0), (1, 1)),
+ (token.NAME, "sin", (1, 1), (1, 4)),
+ (token.OP, "(", (1, 4), (1, 5)),
+ (token.NAME, "x", (1, 5), (1, 6)),
+ (token.OP, ")", (1, 6), (1, 7)),
+ ],
+ ),
+ (
+ "01234",
+ (
+ "leading zeros in decimal integer literals are not permitted; "
+ "use an 0o prefix for octal integers",
+ (1, 1),
+ ),
+ [(token.NUMBER, "01234", (1, 0), (1, 5))],
+ ),
+ (
+ ")",
+ ("unmatched ')'", (1, 1)),
+ [(token.OP, ")", (1, 0), (1, 1))],
+ ),
+ (
+ "(]",
+ (
+ "closing parenthesis ']' does not match opening parenthesis '('",
+ (1, 2),
+ ),
+ [
+ (token.OP, "(", (1, 0), (1, 1)),
+ (token.OP, "]", (1, 1), (1, 2)),
+ ],
+ ),
+ (
+ "a☃b",
+ ("invalid character '☃' (U+2603)", (1, 2)),
+ [(token.NAME, "a☃b", (1, 0), (1, 3))],
+ ),
+ ]
+
+ for source, error, expected in cases:
+ with self.subTest(source=source):
+ with self.assertRaises(tokenize.TokenError) as caught:
+ self._get_tokens(source)
+ self.assertEqual(caught.exception.args, error)
+
+ tokens = self._get_tokens(source, extra_tokens=True)
+ self.assertEqual(
+ [(tok.type, tok.string, tok.start, tok.end)
+ for tok in tokens[:-2]],
+ expected,
+ )
+
+ def test_extra_tokens_emits_comments_and_real_indent_locations(self):
+ source = "if x:\n # c\n y\nz\n"
+ kinds = {token.COMMENT, token.NL, token.INDENT, token.DEDENT}
+
+ parser_tokens = self._get_tokens(source)
+ self.assertEqual(
+ [(tok.type, tok.string, tok.start, tok.end)
+ for tok in parser_tokens if tok.type in kinds],
+ [
+ (token.INDENT, "", (3, -1), (3, -1)),
+ (token.DEDENT, "", (4, -1), (4, -1)),
+ ],
+ )
+
+ tolerant_tokens = self._get_tokens(source, extra_tokens=True)
+ self.assertEqual(
+ [(tok.type, tok.string, tok.start, tok.end)
+ for tok in tolerant_tokens if tok.type in kinds],
+ [
+ (token.COMMENT, "# c", (2, 2), (2, 5)),
+ (token.NL, "\n", (2, 5), (2, 6)),
+ (token.INDENT, " ", (3, 0), (3, 2)),
+ (token.DEDENT, "", (4, 0), (4, 0)),
+ ],
+ )
+
+ def test_printable_invalid_operator_streams_in_both_modes(self):
+ for extra_tokens in (False, True):
+ with self.subTest(extra_tokens=extra_tokens):
+ first = self._get_tokens(
+ "1 $ 2",
+ extra_tokens=extra_tokens,
+ )[1]
+ self.assertEqual(
+ (first.type, first.string, first.start, first.end),
+ (token.OP, "$", (1, 2), (1, 3)),
+ )
+
+ def test_degraded_fstring_format_spec(self):
+ tokens = self._get_tokens('f"{1:{2}{{3}}}"')
+ self.assertEqual(
+ [(tok.string, tok.start, tok.end)
+ for tok in tokens if tok.type == token.FSTRING_MIDDLE],
+ [
+ ("{", (1, 8), (1, 9)),
+ ("3", (1, 10), (1, 11)),
+ ("}", (1, 12), (1, 13)),
+ ],
+ )
+
+ tokens = self._get_tokens('f"{x:{y}}"')
+ self.assertEqual(
+ [(tok.string, tok.start, tok.end)
+ for tok in tokens if tok.type == token.FSTRING_MIDDLE],
+ [("", (1, 8), (1, 8))],
+ )
+
+ with self.assertRaises(tokenize.TokenError) as caught:
+ self._get_tokens('f"{1:{2}x}}y"')
+ self.assertEqual(
+ caught.exception.args,
+ ("f-string: single '}' is not allowed", (1, 11)),
+ )
+
+ def test_escaped_fstring_brace_has_a_position_gap(self):
+ tokens = self._get_tokens('f"a{{"', extra_tokens=True)
+ self.assertEqual(
+ [(tok.type, tok.string, tok.start, tok.end)
+ for tok in tokens
+ if tok.type in {token.FSTRING_MIDDLE, token.FSTRING_END}],
+ [
+ (token.FSTRING_MIDDLE, "a{", (1, 2), (1, 4)),
+ (token.FSTRING_END, '"', (1, 5), (1, 6)),
+ ],
+ )
+
+ def test_tolerant_incompatible_prefix_position_after_non_ascii(self):
+ with self.assertRaises(tokenize.TokenError) as caught:
+ self._get_tokens('bé )tf"2 ', extra_tokens=True)
+ self.assertEqual(
+ caught.exception.args,
+ ("'f' and 't' prefixes are incompatible", (1, 6)),
+ )
+
+ def test_tolerant_fstring_closer_at_expression_entry_depth(self):
+ source = "f'1:{]]{}}r}''"
+ for extra_tokens, position in ((False, (1, 6)), (True, (1, 7))):
+ with self.subTest(extra_tokens=extra_tokens):
+ with self.assertRaises(tokenize.TokenError) as caught:
+ self._get_tokens(source, extra_tokens=extra_tokens)
+ self.assertEqual(
+ caught.exception.args,
+ ("f-string: unmatched ']'", position),
+ )
+
def test_int(self):
self.check_tokenize('0xff <= 255', """\
TOKENIZER_OBJS= \
Parser/lexer/buffer.o \
Parser/lexer/lexer.o \
+ Parser/lexer/number.o \
Parser/lexer/state.o \
+ Parser/lexer/string.o \
Parser/tokenizer/file_tokenizer.o \
Parser/tokenizer/readline_tokenizer.o \
Parser/tokenizer/string_tokenizer.o \
TOKENIZER_HEADERS= \
Parser/lexer/buffer.h \
Parser/lexer/lexer.h \
+ Parser/lexer/lexer_internal.h \
Parser/lexer/state.h \
Parser/tokenizer/tokenizer.h \
Parser/tokenizer/helpers.h
<ClCompile Include="..\Parser\lexer\buffer.c" />
<ClCompile Include="..\Parser\lexer\state.c" />
<ClCompile Include="..\Parser\lexer\lexer.c" />
+ <ClCompile Include="..\Parser\lexer\number.c" />
+ <ClCompile Include="..\Parser\lexer\string.c" />
<ClCompile Include="..\Parser\tokenizer\string_tokenizer.c" />
<ClCompile Include="..\Parser\tokenizer\file_tokenizer.c" />
<ClCompile Include="..\Parser\tokenizer\utf8_tokenizer.c" />
<ClCompile Include="..\Parser\lexer\lexer.c">
<Filter>Source Files</Filter>
</ClCompile>
+ <ClCompile Include="..\Parser\lexer\number.c">
+ <Filter>Source Files</Filter>
+ </ClCompile>
+ <ClCompile Include="..\Parser\lexer\string.c">
+ <Filter>Source Files</Filter>
+ </ClCompile>
<ClCompile Include="..\Parser\lexer\buffer.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClInclude Include="..\Objects\unicodetype_db.h" />
<ClInclude Include="..\Parser\lexer\state.h" />
<ClInclude Include="..\Parser\lexer\lexer.h" />
+ <ClInclude Include="..\Parser\lexer\lexer_internal.h" />
<ClInclude Include="..\Parser\lexer\buffer.h" />
<ClInclude Include="..\Parser\tokenizer\helpers.h" />
<ClInclude Include="..\Parser\tokenizer\tokenizer.h" />
<ClCompile Include="..\Parser\myreadline.c" />
<ClCompile Include="..\Parser\lexer\state.c" />
<ClCompile Include="..\Parser\lexer\lexer.c" />
+ <ClCompile Include="..\Parser\lexer\number.c" />
+ <ClCompile Include="..\Parser\lexer\string.c" />
<ClCompile Include="..\Parser\lexer\buffer.c" />
<ClCompile Include="..\Parser\tokenizer\string_tokenizer.c" />
<ClCompile Include="..\Parser\tokenizer\file_tokenizer.c" />
<ClInclude Include="..\Parser\lexer\state.h">
<Filter>Parser</Filter>
</ClInclude>
+ <ClInclude Include="..\Parser\lexer\lexer_internal.h">
+ <Filter>Parser</Filter>
+ </ClInclude>
<ClInclude Include="..\Parser\lexer\buffer.h">
<Filter>Parser</Filter>
</ClInclude>
<ClCompile Include="..\Parser\lexer\lexer.c">
<Filter>Parser</Filter>
</ClCompile>
+ <ClCompile Include="..\Parser\lexer\number.c">
+ <Filter>Parser</Filter>
+ </ClCompile>
+ <ClCompile Include="..\Parser\lexer\string.c">
+ <Filter>Parser</Filter>
+ </ClCompile>
<ClCompile Include="..\Parser\lexer\state.c">
<Filter>Parser</Filter>
</ClCompile>
#include "pycore_unicodeobject.h"
#include "errcode.h"
-#include "state.h"
+#include "lexer_internal.h"
#include "../tokenizer/helpers.h"
/* Alternate tab spacing */
#define ALTTABSIZE 1
-#define is_potential_identifier_start(c) (\
- (c >= 'a' && c <= 'z')\
- || (c >= 'A' && c <= 'Z')\
- || c == '_'\
- || (c >= 128))
-#define is_potential_identifier_char(c) (\
- (c >= 'a' && c <= 'z')\
- || (c >= 'A' && c <= 'Z')\
- || (c >= '0' && c <= '9')\
- || c == '_'\
- || (c >= 128))
-
-#ifdef Py_DEBUG
-static inline tokenizer_mode* TOK_GET_MODE(struct tok_state* tok) {
- assert(tok->tok_mode_stack_index >= 0);
- assert(tok->tok_mode_stack_index < MAXFSTRINGLEVEL);
- return &(tok->tok_mode_stack[tok->tok_mode_stack_index]);
-}
-static inline tokenizer_mode* TOK_NEXT_MODE(struct tok_state* tok) {
- assert(tok->tok_mode_stack_index >= 0);
- assert(tok->tok_mode_stack_index + 1 < MAXFSTRINGLEVEL);
- return &(tok->tok_mode_stack[++tok->tok_mode_stack_index]);
-}
-#else
-#define TOK_GET_MODE(tok) (&(tok->tok_mode_stack[tok->tok_mode_stack_index]))
-#define TOK_NEXT_MODE(tok) (&(tok->tok_mode_stack[++tok->tok_mode_stack_index]))
-#endif
-
-#define FTSTRING_MIDDLE(tok_mode) (tok_mode->string_kind == TSTRING ? TSTRING_MIDDLE : FSTRING_MIDDLE)
-#define FTSTRING_END(tok_mode) (tok_mode->string_kind == TSTRING ? TSTRING_END : FSTRING_END)
-#define TOK_GET_STRING_PREFIX(tok) (TOK_GET_MODE(tok)->string_kind == TSTRING ? 't' : 'f')
#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
#define MAKE_TYPE_COMMENT_TOKEN(token_type, col_offset, end_col_offset) (\
_PyLexer_type_comment_token_setup(tok, token, token_type, col_offset, end_col_offset, p_start, p_end))
}
/* Get next char, updating state; error code goes into tok->done */
-static int
-tok_nextc(struct tok_state *tok)
+int
+_PyLexer_nextc(struct tok_state *tok)
{
int rc;
for (;;) {
}
/* Back-up one character */
-static void
-tok_backup(struct tok_state *tok, int c)
+void
+_PyLexer_backup(struct tok_state *tok, int c)
{
if (c != EOF) {
if (--tok->cur < tok->buf) {
}
}
-static int
-set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
- assert(token != NULL);
- assert(c == '}' || c == ':' || c == '!');
- tokenizer_mode *tok_mode = TOK_GET_MODE(tok);
-
- if (!(tok_mode->in_debug || tok_mode->string_kind == TSTRING) || token->metadata) {
- return 0;
- }
- PyObject *res = NULL;
-
- // Look for a # character outside of string literals
- int hash_detected = 0;
- int in_string = 0;
- char quote_char = 0;
-
- for (Py_ssize_t i = 0; i < tok_mode->last_expr_size - tok_mode->last_expr_end; i++) {
- char ch = tok_mode->last_expr_buffer[i];
-
- // Skip escaped characters
- if (ch == '\\') {
- i++;
- continue;
- }
-
- // Handle quotes
- if (ch == '"' || ch == '\'') {
- // The following if/else block works becase there is an off number
- // of quotes in STRING tokens and the lexer only ever reaches this
- // function with valid STRING tokens.
- // For example: """hello"""
- // First quote: in_string = 1
- // Second quote: in_string = 0
- // Third quote: in_string = 1
- if (!in_string) {
- in_string = 1;
- quote_char = ch;
- }
- else if (ch == quote_char) {
- in_string = 0;
- }
- continue;
- }
-
- // Check for # outside strings
- if (ch == '#' && !in_string) {
- hash_detected = 1;
- break;
- }
- }
- // If we found a # character in the expression, we need to handle comments
- if (hash_detected) {
- // Allocate buffer for processed result
- char *result = (char *)PyMem_Malloc((tok_mode->last_expr_size - tok_mode->last_expr_end + 1) * sizeof(char));
- if (!result) {
- return -1;
- }
-
- Py_ssize_t i = 0; // Input position
- Py_ssize_t j = 0; // Output position
- in_string = 0; // Whether we're in a string
- quote_char = 0; // Current string quote char
-
- // Process each character
- while (i < tok_mode->last_expr_size - tok_mode->last_expr_end) {
- char ch = tok_mode->last_expr_buffer[i];
-
- // Handle string quotes
- if (ch == '"' || ch == '\'') {
- // See comment above to understand this part
- if (!in_string) {
- in_string = 1;
- quote_char = ch;
- } else if (ch == quote_char) {
- in_string = 0;
- }
- result[j++] = ch;
- }
- // Skip comments
- else if (ch == '#' && !in_string) {
- while (i < tok_mode->last_expr_size - tok_mode->last_expr_end &&
- tok_mode->last_expr_buffer[i] != '\n') {
- i++;
- }
- if (i < tok_mode->last_expr_size - tok_mode->last_expr_end) {
- result[j++] = '\n';
- }
- }
- // Copy other chars
- else {
- result[j++] = ch;
- }
- i++;
- }
-
- result[j] = '\0'; // Null-terminate the result string
- res = PyUnicode_DecodeUTF8(result, j, NULL);
- PyMem_Free(result);
- } else {
- res = PyUnicode_DecodeUTF8(
- tok_mode->last_expr_buffer,
- tok_mode->last_expr_size - tok_mode->last_expr_end,
- NULL
- );
- }
-
- if (!res) {
- return -1;
- }
- token->metadata = res;
- return 0;
-}
-
-int
-_PyLexer_update_ftstring_expr(struct tok_state *tok, char cur)
-{
- assert(tok->cur != NULL);
-
- Py_ssize_t size = strlen(tok->cur);
- tokenizer_mode *tok_mode = TOK_GET_MODE(tok);
-
- switch (cur) {
- case 0:
- if (!tok_mode->last_expr_buffer || tok_mode->last_expr_end >= 0) {
- return 1;
- }
- char *new_buffer = PyMem_Realloc(
- tok_mode->last_expr_buffer,
- tok_mode->last_expr_size + size
- );
- if (new_buffer == NULL) {
- PyMem_Free(tok_mode->last_expr_buffer);
- goto error;
- }
- tok_mode->last_expr_buffer = new_buffer;
- strncpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size, tok->cur, size);
- tok_mode->last_expr_size += size;
- break;
- case '{':
- if (tok_mode->last_expr_buffer != NULL) {
- PyMem_Free(tok_mode->last_expr_buffer);
- }
- tok_mode->last_expr_buffer = PyMem_Malloc(size);
- if (tok_mode->last_expr_buffer == NULL) {
- goto error;
- }
- tok_mode->last_expr_size = size;
- tok_mode->last_expr_end = -1;
- strncpy(tok_mode->last_expr_buffer, tok->cur, size);
- break;
- case '}':
- case '!':
- tok_mode->last_expr_end = strlen(tok->start);
- break;
- case ':':
- if (tok_mode->last_expr_end == -1) {
- tok_mode->last_expr_end = strlen(tok->start);
- }
- break;
- default:
- Py_UNREACHABLE();
- }
- return 1;
-error:
- tok->done = E_NOMEM;
- return 0;
-}
-
-static int
-lookahead(struct tok_state *tok, const char *test)
-{
- const char *s = test;
- int res = 0;
- while (1) {
- int c = tok_nextc(tok);
- if (*s == 0) {
- res = !is_potential_identifier_char(c);
- }
- else if (c == *s) {
- s++;
- continue;
- }
-
- tok_backup(tok, c);
- while (s != test) {
- tok_backup(tok, *--s);
- }
- return res;
- }
-}
-static int
-verify_end_of_number(struct tok_state *tok, int c, const char *kind) {
- if (tok->tok_extra_tokens) {
- // When we are parsing extra tokens, we don't want to emit warnings
- // about invalid literals, because we want to be a bit more liberal.
- return 1;
- }
- /* Emit a deprecation warning only if the numeric literal is immediately
- * followed by one of keywords which can occur after a numeric literal
- * in valid code: "and", "else", "for", "if", "in", "is" and "or".
- * It allows to gradually deprecate existing valid code without adding
- * warning before error in most cases of invalid numeric literal (which
- * would be confusing and break existing tests).
- * Raise a syntax error with slightly better message than plain
- * "invalid syntax" if the numeric literal is immediately followed by
- * other keyword or identifier.
- */
- int r = 0;
- if (c == 'a') {
- r = lookahead(tok, "nd");
- }
- else if (c == 'e') {
- r = lookahead(tok, "lse");
- }
- else if (c == 'f') {
- r = lookahead(tok, "or");
- }
- else if (c == 'i') {
- int c2 = tok_nextc(tok);
- if (c2 == 'f' || c2 == 'n' || c2 == 's') {
- r = 1;
- }
- tok_backup(tok, c2);
- }
- else if (c == 'o') {
- r = lookahead(tok, "r");
- }
- else if (c == 'n') {
- r = lookahead(tok, "ot");
- }
- if (r) {
- tok_backup(tok, c);
- if (_PyTokenizer_parser_warn(tok, PyExc_SyntaxWarning,
- "invalid %s literal", kind))
- {
- return 0;
- }
- tok_nextc(tok);
- }
- else /* In future releases, only error will remain. */
- if (c < 128 && is_potential_identifier_char(c)) {
- tok_backup(tok, c);
- _PyTokenizer_syntaxerror(tok, "invalid %s literal", kind);
- return 0;
- }
- return 1;
-}
/* Verify that the identifier follows PEP 3131. */
static int
return 1;
}
-static int
-tok_decimal_tail(struct tok_state *tok)
-{
- int c;
- while (1) {
- do {
- c = tok_nextc(tok);
- } while (Py_ISDIGIT(c));
- if (c != '_') {
- break;
- }
- c = tok_nextc(tok);
- if (!Py_ISDIGIT(c)) {
- tok_backup(tok, c);
- _PyTokenizer_syntaxerror(tok, "invalid decimal literal");
- return 0;
- }
- }
- return c;
-}
static inline int
tok_continuation_line(struct tok_state *tok) {
return c;
}
-static int
-maybe_raise_syntax_error_for_string_prefixes(struct tok_state *tok,
- int saw_b, int saw_r, int saw_u,
- int saw_f, int saw_t) {
- // Supported: rb, rf, rt (in any order)
- // Unsupported: ub, ur, uf, ut, bf, bt, ft (in any order)
-
-#define RETURN_SYNTAX_ERROR(PREFIX1, PREFIX2) \
- do { \
- (void)_PyTokenizer_syntaxerror_known_range( \
- tok, (int)(tok->start + 1 - tok->line_start), \
- (int)(tok->cur - tok->line_start), \
- "'" PREFIX1 "' and '" PREFIX2 "' prefixes are incompatible"); \
- return -1; \
- } while (0)
-
- if (saw_u && saw_b) {
- RETURN_SYNTAX_ERROR("u", "b");
- }
- if (saw_u && saw_r) {
- RETURN_SYNTAX_ERROR("u", "r");
- }
- if (saw_u && saw_f) {
- RETURN_SYNTAX_ERROR("u", "f");
- }
- if (saw_u && saw_t) {
- RETURN_SYNTAX_ERROR("u", "t");
- }
- if (saw_b && saw_f) {
- RETURN_SYNTAX_ERROR("b", "f");
- }
- if (saw_b && saw_t) {
- RETURN_SYNTAX_ERROR("b", "t");
- }
- if (saw_f && saw_t) {
- RETURN_SYNTAX_ERROR("f", "t");
- }
-
-#undef RETURN_SYNTAX_ERROR
-
- return 0;
-}
-
-static int
-tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct token *token)
+int
+_PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct token *token)
{
int c;
int blankline, nonascii;
c = tok_nextc(tok);
if (c == '"' || c == '\'') {
// Raise error on incompatible string prefixes:
- int status = maybe_raise_syntax_error_for_string_prefixes(
+ int status = _PyLexer_check_string_prefixes(
tok, saw_b, saw_r, saw_u, saw_f, saw_t);
if (status < 0) {
return MAKE_TOKEN(ERRORTOKEN);
// Handle valid f or t string creation:
if (saw_f || saw_t) {
- goto f_string_quote;
+ return _PyLexer_scan_fstring_start(tok, token, c);
}
- goto letter_quote;
+ return _PyLexer_scan_string(tok, token, c);
}
}
while (is_potential_identifier_char(c)) {
if (c == '.') {
c = tok_nextc(tok);
if (Py_ISDIGIT(c)) {
- goto fraction;
+ return _PyLexer_scan_number(tok, token, c, 1);
} else if (c == '.') {
c = tok_nextc(tok);
if (c == '.') {
return MAKE_TOKEN(DOT);
}
+
/* Number */
if (Py_ISDIGIT(c)) {
- if (c == '0') {
- /* Hex, octal or binary -- maybe. */
- c = tok_nextc(tok);
- if (c == 'x' || c == 'X') {
- /* Hex */
- c = tok_nextc(tok);
- do {
- if (c == '_') {
- c = tok_nextc(tok);
- }
- if (!Py_ISXDIGIT(c)) {
- tok_backup(tok, c);
- return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid hexadecimal literal"));
- }
- do {
- c = tok_nextc(tok);
- } while (Py_ISXDIGIT(c));
- } while (c == '_');
- if (!verify_end_of_number(tok, c, "hexadecimal")) {
- return MAKE_TOKEN(ERRORTOKEN);
- }
- }
- else if (c == 'o' || c == 'O') {
- /* Octal */
- c = tok_nextc(tok);
- do {
- if (c == '_') {
- c = tok_nextc(tok);
- }
- if (c < '0' || c >= '8') {
- if (Py_ISDIGIT(c)) {
- return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
- "invalid digit '%c' in octal literal", c));
- }
- else {
- tok_backup(tok, c);
- return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid octal literal"));
- }
- }
- do {
- c = tok_nextc(tok);
- } while ('0' <= c && c < '8');
- } while (c == '_');
- if (Py_ISDIGIT(c)) {
- return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
- "invalid digit '%c' in octal literal", c));
- }
- if (!verify_end_of_number(tok, c, "octal")) {
- return MAKE_TOKEN(ERRORTOKEN);
- }
- }
- else if (c == 'b' || c == 'B') {
- /* Binary */
- c = tok_nextc(tok);
- do {
- if (c == '_') {
- c = tok_nextc(tok);
- }
- if (c != '0' && c != '1') {
- if (Py_ISDIGIT(c)) {
- return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c));
- }
- else {
- tok_backup(tok, c);
- return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid binary literal"));
- }
- }
- do {
- c = tok_nextc(tok);
- } while (c == '0' || c == '1');
- } while (c == '_');
- if (Py_ISDIGIT(c)) {
- return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c));
- }
- if (!verify_end_of_number(tok, c, "binary")) {
- return MAKE_TOKEN(ERRORTOKEN);
- }
- }
- else {
- int nonzero = 0;
- /* maybe old-style octal; c is first char of it */
- /* in any case, allow '0' as a literal */
- while (1) {
- if (c == '_') {
- c = tok_nextc(tok);
- if (!Py_ISDIGIT(c)) {
- tok_backup(tok, c);
- return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal"));
- }
- }
- if (c != '0') {
- break;
- }
- c = tok_nextc(tok);
- }
- char* zeros_end = tok->cur;
- if (Py_ISDIGIT(c)) {
- nonzero = 1;
- c = tok_decimal_tail(tok);
- if (c == 0) {
- return MAKE_TOKEN(ERRORTOKEN);
- }
- }
- if (c == '.') {
- c = tok_nextc(tok);
- goto fraction;
- }
- else if (c == 'e' || c == 'E') {
- goto exponent;
- }
- else if (c == 'j' || c == 'J') {
- goto imaginary;
- }
- else if (nonzero && !tok->tok_extra_tokens) {
- /* Old-style octal: now disallowed. */
- tok_backup(tok, c);
- return MAKE_TOKEN(_PyTokenizer_syntaxerror_known_range(
- tok, (int)(tok->start + 1 - tok->line_start),
- (int)(zeros_end - tok->line_start),
- "leading zeros in decimal integer "
- "literals are not permitted; "
- "use an 0o prefix for octal integers"));
- }
- if (!verify_end_of_number(tok, c, "decimal")) {
- return MAKE_TOKEN(ERRORTOKEN);
- }
- }
- }
- else {
- /* Decimal */
- c = tok_decimal_tail(tok);
- if (c == 0) {
- return MAKE_TOKEN(ERRORTOKEN);
- }
- {
- /* Accept floating-point numbers. */
- if (c == '.') {
- c = tok_nextc(tok);
- fraction:
- /* Fraction */
- if (Py_ISDIGIT(c)) {
- c = tok_decimal_tail(tok);
- if (c == 0) {
- return MAKE_TOKEN(ERRORTOKEN);
- }
- }
- }
- if (c == 'e' || c == 'E') {
- int e;
- exponent:
- e = c;
- /* Exponent part */
- c = tok_nextc(tok);
- if (c == '+' || c == '-') {
- c = tok_nextc(tok);
- if (!Py_ISDIGIT(c)) {
- tok_backup(tok, c);
- return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal"));
- }
- } else if (!Py_ISDIGIT(c)) {
- tok_backup(tok, c);
- if (!verify_end_of_number(tok, e, "decimal")) {
- return MAKE_TOKEN(ERRORTOKEN);
- }
- tok_backup(tok, e);
- p_start = tok->start;
- p_end = tok->cur;
- return MAKE_TOKEN(NUMBER);
- }
- c = tok_decimal_tail(tok);
- if (c == 0) {
- return MAKE_TOKEN(ERRORTOKEN);
- }
- }
- if (c == 'j' || c == 'J') {
- /* Imaginary part */
- imaginary:
- c = tok_nextc(tok);
- if (!verify_end_of_number(tok, c, "imaginary")) {
- return MAKE_TOKEN(ERRORTOKEN);
- }
- }
- else if (!verify_end_of_number(tok, c, "decimal")) {
- return MAKE_TOKEN(ERRORTOKEN);
- }
- }
- }
- tok_backup(tok, c);
- p_start = tok->start;
- p_end = tok->cur;
- return MAKE_TOKEN(NUMBER);
- }
-
- f_string_quote:
- if (((Py_TOLOWER(*tok->start) == 'f' || Py_TOLOWER(*tok->start) == 'r' || Py_TOLOWER(*tok->start) == 't')
- && (c == '\'' || c == '"'))) {
-
- int quote = c;
- int quote_size = 1; /* 1 or 3 */
-
- /* Nodes of type STRING, especially multi line strings
- must be handled differently in order to get both
- the starting line number and the column offset right.
- (cf. issue 16806) */
- tok->first_lineno = tok->lineno;
- tok->multi_line_start = tok->line_start;
-
- /* Find the quote size and start of string */
- int after_quote = tok_nextc(tok);
- if (after_quote == quote) {
- int after_after_quote = tok_nextc(tok);
- if (after_after_quote == quote) {
- quote_size = 3;
- }
- else {
- // TODO: Check this
- tok_backup(tok, after_after_quote);
- tok_backup(tok, after_quote);
- }
- }
- if (after_quote != quote) {
- tok_backup(tok, after_quote);
- }
-
-
- p_start = tok->start;
- p_end = tok->cur;
- if (tok->tok_mode_stack_index + 1 >= MAXFSTRINGLEVEL) {
- return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "too many nested f-strings or t-strings"));
- }
- tokenizer_mode *the_current_tok = TOK_NEXT_MODE(tok);
- the_current_tok->kind = TOK_FSTRING_MODE;
- the_current_tok->quote = quote;
- the_current_tok->quote_size = quote_size;
- the_current_tok->start = tok->start;
- the_current_tok->multi_line_start = tok->line_start;
- the_current_tok->first_line = tok->lineno;
- the_current_tok->start_offset = -1;
- the_current_tok->multi_line_start_offset = -1;
- the_current_tok->last_expr_buffer = NULL;
- the_current_tok->last_expr_size = 0;
- the_current_tok->last_expr_end = -1;
- the_current_tok->in_format_spec = 0;
- the_current_tok->in_debug = 0;
-
- enum string_kind_t string_kind = FSTRING;
- switch (*tok->start) {
- case 'T':
- case 't':
- the_current_tok->raw = Py_TOLOWER(*(tok->start + 1)) == 'r';
- string_kind = TSTRING;
- break;
- case 'F':
- case 'f':
- the_current_tok->raw = Py_TOLOWER(*(tok->start + 1)) == 'r';
- break;
- case 'R':
- case 'r':
- the_current_tok->raw = 1;
- if (Py_TOLOWER(*(tok->start + 1)) == 't') {
- string_kind = TSTRING;
- }
- break;
- default:
- Py_UNREACHABLE();
- }
-
- the_current_tok->string_kind = string_kind;
- the_current_tok->curly_bracket_depth = 0;
- the_current_tok->curly_bracket_expr_start_depth = -1;
- return string_kind == TSTRING ? MAKE_TOKEN(TSTRING_START) : MAKE_TOKEN(FSTRING_START);
+ return _PyLexer_scan_number(tok, token, c, 0);
}
- letter_quote:
/* String */
if (c == '\'' || c == '"') {
- int quote = c;
- int quote_size = 1; /* 1 or 3 */
- int end_quote_size = 0;
- int has_escaped_quote = 0;
-
- /* Nodes of type STRING, especially multi line strings
- must be handled differently in order to get both
- the starting line number and the column offset right.
- (cf. issue 16806) */
- tok->first_lineno = tok->lineno;
- tok->multi_line_start = tok->line_start;
-
- /* Find the quote size and start of string */
- c = tok_nextc(tok);
- if (c == quote) {
- c = tok_nextc(tok);
- if (c == quote) {
- quote_size = 3;
- }
- else {
- end_quote_size = 1; /* empty string found */
- }
- }
- if (c != quote) {
- tok_backup(tok, c);
- }
-
- /* Get rest of string */
- while (end_quote_size != quote_size) {
- c = tok_nextc(tok);
- if (tok->done == E_ERROR) {
- return MAKE_TOKEN(ERRORTOKEN);
- }
- if (tok->done == E_DECODE) {
- break;
- }
- if (c == EOF || (quote_size == 1 && c == '\n')) {
- assert(tok->multi_line_start != NULL);
- // shift the tok_state's location into
- // the start of string, and report the error
- // from the initial quote character
- tok->cur = (char *)tok->start;
- tok->cur++;
- tok->line_start = tok->multi_line_start;
- int start = tok->lineno;
- tok->lineno = tok->first_lineno;
-
- if (INSIDE_FSTRING(tok)) {
- /* When we are in an f-string, before raising the
- * unterminated string literal error, check whether
- * does the initial quote matches with f-strings quotes
- * and if it is, then this must be a missing '}' token
- * so raise the proper error */
- tokenizer_mode *the_current_tok = TOK_GET_MODE(tok);
- if (the_current_tok->quote == quote &&
- the_current_tok->quote_size == quote_size) {
- return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
- "%c-string: expecting '}'", TOK_GET_STRING_PREFIX(tok)));
- }
- }
-
- if (quote_size == 3) {
- _PyTokenizer_syntaxerror(tok, "unterminated triple-quoted string literal"
- " (detected at line %d)", start);
- if (c != '\n') {
- tok->done = E_EOFS;
- }
- return MAKE_TOKEN(ERRORTOKEN);
- }
- else {
- if (has_escaped_quote) {
- _PyTokenizer_syntaxerror(
- tok,
- "unterminated string literal (detected at line %d); "
- "perhaps you escaped the end quote?",
- start
- );
- } else {
- _PyTokenizer_syntaxerror(
- tok, "unterminated string literal (detected at line %d)", start
- );
- }
- if (c != '\n') {
- tok->done = E_EOLS;
- }
- return MAKE_TOKEN(ERRORTOKEN);
- }
- }
- if (c == quote) {
- end_quote_size += 1;
- }
- else {
- end_quote_size = 0;
- if (c == '\\') {
- c = tok_nextc(tok); /* skip escaped char */
- if (c == quote) { /* but record whether the escaped char was a quote */
- has_escaped_quote = 1;
- }
- if (c == '\r') {
- c = tok_nextc(tok);
- }
- }
- }
- }
-
- p_start = tok->start;
- p_end = tok->cur;
- return MAKE_TOKEN(STRING);
+ return _PyLexer_scan_string(tok, token, c);
}
/* Line continuation */
if ((cursor_valid) && !_PyLexer_update_ftstring_expr(tok, c)) {
return MAKE_TOKEN(ENDMARKER);
}
- if ((cursor_valid) && c != '{' && set_ftstring_expr(tok, token, c)) {
+ if ((cursor_valid) && c != '{' && _PyLexer_set_ftstring_expr(tok, token, c)) {
return MAKE_TOKEN(ERRORTOKEN);
}
return MAKE_TOKEN(_PyToken_OneChar(c));
}
-static int
-tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct token *token)
-{
- const char *p_start = NULL;
- const char *p_end = NULL;
- int end_quote_size = 0;
- int unicode_escape = 0;
-
- tok->start = tok->cur;
- tok->first_lineno = tok->lineno;
- tok->starting_col_offset = tok->col_offset;
-
- // If we start with a bracket, we defer to the normal mode as there is nothing for us to tokenize
- // before it.
- int start_char = tok_nextc(tok);
- if (start_char == '{') {
- int peek1 = tok_nextc(tok);
- tok_backup(tok, peek1);
- tok_backup(tok, start_char);
- if (peek1 != '{') {
- current_tok->curly_bracket_expr_start_depth++;
- if (current_tok->curly_bracket_expr_start_depth >= MAX_EXPR_NESTING) {
- return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
- "%c-string: expressions nested too deeply", TOK_GET_STRING_PREFIX(tok)));
- }
- TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE;
- return tok_get_normal_mode(tok, current_tok, token);
- }
- }
- else {
- tok_backup(tok, start_char);
- }
-
- // Check if we are at the end of the string
- for (int i = 0; i < current_tok->quote_size; i++) {
- int quote = tok_nextc(tok);
- if (quote != current_tok->quote) {
- tok_backup(tok, quote);
- goto f_string_middle;
- }
- }
-
- if (current_tok->last_expr_buffer != NULL) {
- PyMem_Free(current_tok->last_expr_buffer);
- current_tok->last_expr_buffer = NULL;
- current_tok->last_expr_size = 0;
- current_tok->last_expr_end = -1;
- }
-
- p_start = tok->start;
- p_end = tok->cur;
- tok->tok_mode_stack_index--;
- return MAKE_TOKEN(FTSTRING_END(current_tok));
-
-f_string_middle:
-
- // TODO: This is a bit of a hack, but it works for now. We need to find a better way to handle
- // this.
- tok->multi_line_start = tok->line_start;
- while (end_quote_size != current_tok->quote_size) {
- int c = tok_nextc(tok);
- if (tok->done == E_ERROR || tok->done == E_DECODE) {
- return MAKE_TOKEN(ERRORTOKEN);
- }
- int in_format_spec = (
- current_tok->in_format_spec
- &&
- INSIDE_FSTRING_EXPR(current_tok)
- );
-
- if (c == EOF || (current_tok->quote_size == 1 && c == '\n')) {
- if (tok->decoding_erred) {
- return MAKE_TOKEN(ERRORTOKEN);
- }
-
- // If we are in a format spec and we found a newline,
- // it means that the format spec ends here and we should
- // return to the regular mode.
- if (in_format_spec && c == '\n') {
- if (current_tok->quote_size == 1) {
- return MAKE_TOKEN(
- _PyTokenizer_syntaxerror(
- tok,
- "%c-string: newlines are not allowed in format specifiers for single quoted %c-strings",
- TOK_GET_STRING_PREFIX(tok), TOK_GET_STRING_PREFIX(tok)
- )
- );
- }
- tok_backup(tok, c);
- TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE;
- current_tok->in_format_spec = 0;
- p_start = tok->start;
- p_end = tok->cur;
- return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
- }
-
- assert(tok->multi_line_start != NULL);
- // shift the tok_state's location into
- // the start of string, and report the error
- // from the initial quote character
- tok->cur = (char *)current_tok->start;
- tok->cur++;
- tok->line_start = current_tok->multi_line_start;
- int start = tok->lineno;
-
- tokenizer_mode *the_current_tok = TOK_GET_MODE(tok);
- tok->lineno = the_current_tok->first_line;
-
- if (current_tok->quote_size == 3) {
- _PyTokenizer_syntaxerror(tok,
- "unterminated triple-quoted %c-string literal"
- " (detected at line %d)",
- TOK_GET_STRING_PREFIX(tok), start);
- if (c != '\n') {
- tok->done = E_EOFS;
- }
- return MAKE_TOKEN(ERRORTOKEN);
- }
- else {
- return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
- "unterminated %c-string literal (detected at"
- " line %d)", TOK_GET_STRING_PREFIX(tok), start));
- }
- }
-
- if (c == current_tok->quote) {
- end_quote_size += 1;
- continue;
- } else {
- end_quote_size = 0;
- }
-
- if (c == '{') {
- if (!_PyLexer_update_ftstring_expr(tok, c)) {
- return MAKE_TOKEN(ENDMARKER);
- }
- int peek = tok_nextc(tok);
- if (peek != '{' || in_format_spec) {
- tok_backup(tok, peek);
- tok_backup(tok, c);
- current_tok->curly_bracket_expr_start_depth++;
- if (current_tok->curly_bracket_expr_start_depth >= MAX_EXPR_NESTING) {
- return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
- "%c-string: expressions nested too deeply", TOK_GET_STRING_PREFIX(tok)));
- }
- TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE;
- current_tok->in_format_spec = 0;
- p_start = tok->start;
- p_end = tok->cur;
- } else {
- p_start = tok->start;
- p_end = tok->cur - 1;
- }
- return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
- } else if (c == '}') {
- if (unicode_escape) {
- p_start = tok->start;
- p_end = tok->cur;
- return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
- }
- int peek = tok_nextc(tok);
-
- // The tokenizer can only be in the format spec if we have already completed the expression
- // scanning (indicated by the end of the expression being set) and we are not at the top level
- // of the bracket stack (-1 is the top level). Since format specifiers can't legally use double
- // brackets, we can bypass it here.
- int cursor = current_tok->curly_bracket_depth;
- if (peek == '}' && !in_format_spec && cursor == 0) {
- p_start = tok->start;
- p_end = tok->cur - 1;
- } else {
- tok_backup(tok, peek);
- tok_backup(tok, c);
- TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE;
- current_tok->in_format_spec = 0;
- p_start = tok->start;
- p_end = tok->cur;
- }
- return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
- } else if (c == '\\') {
- int peek = tok_nextc(tok);
- if (peek == '\r') {
- peek = tok_nextc(tok);
- }
- // Special case when the backslash is right before a curly
- // brace. We have to restore and return the control back
- // to the loop for the next iteration.
- if (peek == '{' || peek == '}') {
- if (!current_tok->raw) {
- if (_PyTokenizer_warn_invalid_escape_sequence(tok, peek)) {
- return MAKE_TOKEN(ERRORTOKEN);
- }
- }
- tok_backup(tok, peek);
- continue;
- }
-
- if (!current_tok->raw) {
- if (peek == 'N') {
- /* Handle named unicode escapes (\N{BULLET}) */
- peek = tok_nextc(tok);
- if (peek == '{') {
- unicode_escape = 1;
- } else {
- tok_backup(tok, peek);
- }
- }
- } /* else {
- skip the escaped character
- }*/
- }
- }
-
- // Backup the f-string quotes to emit a final FSTRING_MIDDLE and
- // add the quotes to the FSTRING_END in the next tokenizer iteration.
- for (int i = 0; i < current_tok->quote_size; i++) {
- tok_backup(tok, current_tok->quote);
- }
- p_start = tok->start;
- p_end = tok->cur;
- return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
-}
static int
tok_get(struct tok_state *tok, struct token *token)
{
tokenizer_mode *current_tok = TOK_GET_MODE(tok);
if (current_tok->kind == TOK_REGULAR_MODE) {
- return tok_get_normal_mode(tok, current_tok, token);
+ return _PyLexer_get_normal_mode(tok, current_tok, token);
} else {
- return tok_get_fstring_mode(tok, current_tok, token);
+ return _PyLexer_get_fstring_mode(tok, current_tok, token);
}
}
--- /dev/null
+#ifndef _PY_LEXER_INTERNAL_H_
+#define _PY_LEXER_INTERNAL_H_
+
+#include "lexer.h"
+
+#define is_potential_identifier_start(c) (\
+ (c >= 'a' && c <= 'z')\
+ || (c >= 'A' && c <= 'Z')\
+ || c == '_'\
+ || (c >= 128))
+
+#define is_potential_identifier_char(c) (\
+ (c >= 'a' && c <= 'z')\
+ || (c >= 'A' && c <= 'Z')\
+ || (c >= '0' && c <= '9')\
+ || c == '_'\
+ || (c >= 128))
+
+#ifdef Py_DEBUG
+static inline tokenizer_mode *
+TOK_GET_MODE(struct tok_state *tok)
+{
+ assert(tok->tok_mode_stack_index >= 0);
+ assert(tok->tok_mode_stack_index < MAXFSTRINGLEVEL);
+ return &tok->tok_mode_stack[tok->tok_mode_stack_index];
+}
+
+static inline tokenizer_mode *
+TOK_NEXT_MODE(struct tok_state *tok)
+{
+ assert(tok->tok_mode_stack_index >= 0);
+ assert(tok->tok_mode_stack_index + 1 < MAXFSTRINGLEVEL);
+ return &tok->tok_mode_stack[++tok->tok_mode_stack_index];
+}
+#else
+#define TOK_GET_MODE(tok) (&(tok)->tok_mode_stack[(tok)->tok_mode_stack_index])
+#define TOK_NEXT_MODE(tok) (&(tok)->tok_mode_stack[++(tok)->tok_mode_stack_index])
+#endif
+
+#define FTSTRING_MIDDLE(tok_mode) ((tok_mode)->string_kind == TSTRING ? TSTRING_MIDDLE : FSTRING_MIDDLE)
+#define FTSTRING_END(tok_mode) ((tok_mode)->string_kind == TSTRING ? TSTRING_END : FSTRING_END)
+#define TOK_GET_STRING_PREFIX(tok) (TOK_GET_MODE(tok)->string_kind == TSTRING ? 't' : 'f')
+
+#define tok_nextc _PyLexer_nextc
+#define tok_backup _PyLexer_backup
+
+int _PyLexer_nextc(struct tok_state *);
+void _PyLexer_backup(struct tok_state *, int);
+int _PyLexer_set_ftstring_expr(struct tok_state *, struct token *, char);
+int _PyLexer_check_string_prefixes(struct tok_state *, int, int, int, int, int);
+int _PyLexer_scan_number(struct tok_state *, struct token *, int, int);
+int _PyLexer_scan_fstring_start(struct tok_state *, struct token *, int);
+int _PyLexer_scan_string(struct tok_state *, struct token *, int);
+int _PyLexer_get_normal_mode(struct tok_state *, tokenizer_mode *, struct token *);
+int _PyLexer_get_fstring_mode(struct tok_state *, tokenizer_mode *, struct token *);
+
+#endif
--- /dev/null
+#include "Python.h"
+#include "pycore_token.h"
+
+#include "lexer_internal.h"
+#include "../tokenizer/helpers.h"
+
+#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
+
+static int
+lookahead(struct tok_state *tok, const char *test)
+{
+ const char *s = test;
+ int res = 0;
+ while (1) {
+ int c = tok_nextc(tok);
+ if (*s == 0) {
+ res = !is_potential_identifier_char(c);
+ }
+ else if (c == *s) {
+ s++;
+ continue;
+ }
+
+ tok_backup(tok, c);
+ while (s != test) {
+ tok_backup(tok, *--s);
+ }
+ return res;
+ }
+}
+
+static int
+verify_end_of_number(struct tok_state *tok, int c, const char *kind) {
+ if (tok->tok_extra_tokens) {
+ // When we are parsing extra tokens, we don't want to emit warnings
+ // about invalid literals, because we want to be a bit more liberal.
+ return 1;
+ }
+ /* Emit a deprecation warning only if the numeric literal is immediately
+ * followed by one of keywords which can occur after a numeric literal
+ * in valid code: "and", "else", "for", "if", "in", "is" and "or".
+ * It allows to gradually deprecate existing valid code without adding
+ * warning before error in most cases of invalid numeric literal (which
+ * would be confusing and break existing tests).
+ * Raise a syntax error with slightly better message than plain
+ * "invalid syntax" if the numeric literal is immediately followed by
+ * other keyword or identifier.
+ */
+ int r = 0;
+ if (c == 'a') {
+ r = lookahead(tok, "nd");
+ }
+ else if (c == 'e') {
+ r = lookahead(tok, "lse");
+ }
+ else if (c == 'f') {
+ r = lookahead(tok, "or");
+ }
+ else if (c == 'i') {
+ int c2 = tok_nextc(tok);
+ if (c2 == 'f' || c2 == 'n' || c2 == 's') {
+ r = 1;
+ }
+ tok_backup(tok, c2);
+ }
+ else if (c == 'o') {
+ r = lookahead(tok, "r");
+ }
+ else if (c == 'n') {
+ r = lookahead(tok, "ot");
+ }
+ if (r) {
+ tok_backup(tok, c);
+ if (_PyTokenizer_parser_warn(tok, PyExc_SyntaxWarning,
+ "invalid %s literal", kind))
+ {
+ return 0;
+ }
+ tok_nextc(tok);
+ }
+ else /* In future releases, only error will remain. */
+ if (c < 128 && is_potential_identifier_char(c)) {
+ tok_backup(tok, c);
+ _PyTokenizer_syntaxerror(tok, "invalid %s literal", kind);
+ return 0;
+ }
+ return 1;
+}
+
+static int
+tok_decimal_tail(struct tok_state *tok)
+{
+ int c;
+
+ while (1) {
+ do {
+ c = tok_nextc(tok);
+ } while (Py_ISDIGIT(c));
+ if (c != '_') {
+ break;
+ }
+ c = tok_nextc(tok);
+ if (!Py_ISDIGIT(c)) {
+ tok_backup(tok, c);
+ _PyTokenizer_syntaxerror(tok, "invalid decimal literal");
+ return 0;
+ }
+ }
+ return c;
+}
+
+int
+_PyLexer_scan_number(struct tok_state *tok, struct token *token, int c,
+ int leading_dot)
+{
+ const char *p_start = NULL;
+ const char *p_end = NULL;
+
+ if (leading_dot) {
+ goto fraction;
+ }
+ if (c == '0') {
+ /* Hex, octal or binary -- maybe. */
+ c = tok_nextc(tok);
+ if (c == 'x' || c == 'X') {
+ /* Hex */
+ c = tok_nextc(tok);
+ do {
+ if (c == '_') {
+ c = tok_nextc(tok);
+ }
+ if (!Py_ISXDIGIT(c)) {
+ tok_backup(tok, c);
+ return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid hexadecimal literal"));
+ }
+ do {
+ c = tok_nextc(tok);
+ } while (Py_ISXDIGIT(c));
+ } while (c == '_');
+ if (!verify_end_of_number(tok, c, "hexadecimal")) {
+ return MAKE_TOKEN(ERRORTOKEN);
+ }
+ }
+ else if (c == 'o' || c == 'O') {
+ /* Octal */
+ c = tok_nextc(tok);
+ do {
+ if (c == '_') {
+ c = tok_nextc(tok);
+ }
+ if (c < '0' || c >= '8') {
+ if (Py_ISDIGIT(c)) {
+ return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
+ "invalid digit '%c' in octal literal", c));
+ }
+ else {
+ tok_backup(tok, c);
+ return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid octal literal"));
+ }
+ }
+ do {
+ c = tok_nextc(tok);
+ } while ('0' <= c && c < '8');
+ } while (c == '_');
+ if (Py_ISDIGIT(c)) {
+ return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
+ "invalid digit '%c' in octal literal", c));
+ }
+ if (!verify_end_of_number(tok, c, "octal")) {
+ return MAKE_TOKEN(ERRORTOKEN);
+ }
+ }
+ else if (c == 'b' || c == 'B') {
+ /* Binary */
+ c = tok_nextc(tok);
+ do {
+ if (c == '_') {
+ c = tok_nextc(tok);
+ }
+ if (c != '0' && c != '1') {
+ if (Py_ISDIGIT(c)) {
+ return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c));
+ }
+ else {
+ tok_backup(tok, c);
+ return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid binary literal"));
+ }
+ }
+ do {
+ c = tok_nextc(tok);
+ } while (c == '0' || c == '1');
+ } while (c == '_');
+ if (Py_ISDIGIT(c)) {
+ return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c));
+ }
+ if (!verify_end_of_number(tok, c, "binary")) {
+ return MAKE_TOKEN(ERRORTOKEN);
+ }
+ }
+ else {
+ int nonzero = 0;
+ /* maybe old-style octal; c is first char of it */
+ /* in any case, allow '0' as a literal */
+ while (1) {
+ if (c == '_') {
+ c = tok_nextc(tok);
+ if (!Py_ISDIGIT(c)) {
+ tok_backup(tok, c);
+ return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal"));
+ }
+ }
+ if (c != '0') {
+ break;
+ }
+ c = tok_nextc(tok);
+ }
+ char* zeros_end = tok->cur;
+ if (Py_ISDIGIT(c)) {
+ nonzero = 1;
+ c = tok_decimal_tail(tok);
+ if (c == 0) {
+ return MAKE_TOKEN(ERRORTOKEN);
+ }
+ }
+ if (c == '.') {
+ c = tok_nextc(tok);
+ goto fraction;
+ }
+ else if (c == 'e' || c == 'E') {
+ goto exponent;
+ }
+ else if (c == 'j' || c == 'J') {
+ goto imaginary;
+ }
+ else if (nonzero && !tok->tok_extra_tokens) {
+ /* Old-style octal: now disallowed. */
+ tok_backup(tok, c);
+ return MAKE_TOKEN(_PyTokenizer_syntaxerror_known_range(
+ tok, (int)(tok->start + 1 - tok->line_start),
+ (int)(zeros_end - tok->line_start),
+ "leading zeros in decimal integer "
+ "literals are not permitted; "
+ "use an 0o prefix for octal integers"));
+ }
+ if (!verify_end_of_number(tok, c, "decimal")) {
+ return MAKE_TOKEN(ERRORTOKEN);
+ }
+ }
+ }
+ else {
+ /* Decimal */
+ c = tok_decimal_tail(tok);
+ if (c == 0) {
+ return MAKE_TOKEN(ERRORTOKEN);
+ }
+ {
+ /* Accept floating-point numbers. */
+ if (c == '.') {
+ c = tok_nextc(tok);
+ fraction:
+ /* Fraction */
+ if (Py_ISDIGIT(c)) {
+ c = tok_decimal_tail(tok);
+ if (c == 0) {
+ return MAKE_TOKEN(ERRORTOKEN);
+ }
+ }
+ }
+ if (c == 'e' || c == 'E') {
+ int e;
+ exponent:
+ e = c;
+ /* Exponent part */
+ c = tok_nextc(tok);
+ if (c == '+' || c == '-') {
+ c = tok_nextc(tok);
+ if (!Py_ISDIGIT(c)) {
+ tok_backup(tok, c);
+ return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal"));
+ }
+ } else if (!Py_ISDIGIT(c)) {
+ tok_backup(tok, c);
+ if (!verify_end_of_number(tok, e, "decimal")) {
+ return MAKE_TOKEN(ERRORTOKEN);
+ }
+ tok_backup(tok, e);
+ p_start = tok->start;
+ p_end = tok->cur;
+ return MAKE_TOKEN(NUMBER);
+ }
+ c = tok_decimal_tail(tok);
+ if (c == 0) {
+ return MAKE_TOKEN(ERRORTOKEN);
+ }
+ }
+ if (c == 'j' || c == 'J') {
+ /* Imaginary part */
+ imaginary:
+ c = tok_nextc(tok);
+ if (!verify_end_of_number(tok, c, "imaginary")) {
+ return MAKE_TOKEN(ERRORTOKEN);
+ }
+ }
+ else if (!verify_end_of_number(tok, c, "decimal")) {
+ return MAKE_TOKEN(ERRORTOKEN);
+ }
+ }
+ }
+ tok_backup(tok, c);
+ p_start = tok->start;
+ p_end = tok->cur;
+ return MAKE_TOKEN(NUMBER);
+}
--- /dev/null
+#include "Python.h"
+#include "pycore_token.h"
+#include "errcode.h"
+
+#include "lexer_internal.h"
+#include "../tokenizer/helpers.h"
+
+#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
+
+int
+_PyLexer_set_ftstring_expr(struct tok_state* tok, struct token *token, char c) {
+ assert(token != NULL);
+ assert(c == '}' || c == ':' || c == '!');
+ tokenizer_mode *tok_mode = TOK_GET_MODE(tok);
+
+ if (!(tok_mode->in_debug || tok_mode->string_kind == TSTRING) || token->metadata) {
+ return 0;
+ }
+ PyObject *res = NULL;
+
+ // Look for a # character outside of string literals
+ int hash_detected = 0;
+ int in_string = 0;
+ char quote_char = 0;
+
+ for (Py_ssize_t i = 0; i < tok_mode->last_expr_size - tok_mode->last_expr_end; i++) {
+ char ch = tok_mode->last_expr_buffer[i];
+
+ // Skip escaped characters
+ if (ch == '\\') {
+ i++;
+ continue;
+ }
+
+ // Handle quotes
+ if (ch == '"' || ch == '\'') {
+ // The following if/else block works becase there is an off number
+ // of quotes in STRING tokens and the lexer only ever reaches this
+ // function with valid STRING tokens.
+ // For example: """hello"""
+ // First quote: in_string = 1
+ // Second quote: in_string = 0
+ // Third quote: in_string = 1
+ if (!in_string) {
+ in_string = 1;
+ quote_char = ch;
+ }
+ else if (ch == quote_char) {
+ in_string = 0;
+ }
+ continue;
+ }
+
+ // Check for # outside strings
+ if (ch == '#' && !in_string) {
+ hash_detected = 1;
+ break;
+ }
+ }
+ // If we found a # character in the expression, we need to handle comments
+ if (hash_detected) {
+ // Allocate buffer for processed result
+ char *result = (char *)PyMem_Malloc((tok_mode->last_expr_size - tok_mode->last_expr_end + 1) * sizeof(char));
+ if (!result) {
+ return -1;
+ }
+
+ Py_ssize_t i = 0; // Input position
+ Py_ssize_t j = 0; // Output position
+ in_string = 0; // Whether we're in a string
+ quote_char = 0; // Current string quote char
+
+ // Process each character
+ while (i < tok_mode->last_expr_size - tok_mode->last_expr_end) {
+ char ch = tok_mode->last_expr_buffer[i];
+
+ // Handle string quotes
+ if (ch == '"' || ch == '\'') {
+ // See comment above to understand this part
+ if (!in_string) {
+ in_string = 1;
+ quote_char = ch;
+ } else if (ch == quote_char) {
+ in_string = 0;
+ }
+ result[j++] = ch;
+ }
+ // Skip comments
+ else if (ch == '#' && !in_string) {
+ while (i < tok_mode->last_expr_size - tok_mode->last_expr_end &&
+ tok_mode->last_expr_buffer[i] != '\n') {
+ i++;
+ }
+ if (i < tok_mode->last_expr_size - tok_mode->last_expr_end) {
+ result[j++] = '\n';
+ }
+ }
+ // Copy other chars
+ else {
+ result[j++] = ch;
+ }
+ i++;
+ }
+
+ result[j] = '\0'; // Null-terminate the result string
+ res = PyUnicode_DecodeUTF8(result, j, NULL);
+ PyMem_Free(result);
+ } else {
+ res = PyUnicode_DecodeUTF8(
+ tok_mode->last_expr_buffer,
+ tok_mode->last_expr_size - tok_mode->last_expr_end,
+ NULL
+ );
+ }
+
+ if (!res) {
+ return -1;
+ }
+ token->metadata = res;
+ return 0;
+}
+
+int
+_PyLexer_update_ftstring_expr(struct tok_state *tok, char cur)
+{
+ assert(tok->cur != NULL);
+
+ Py_ssize_t size = strlen(tok->cur);
+ tokenizer_mode *tok_mode = TOK_GET_MODE(tok);
+
+ switch (cur) {
+ case 0:
+ if (!tok_mode->last_expr_buffer || tok_mode->last_expr_end >= 0) {
+ return 1;
+ }
+ char *new_buffer = PyMem_Realloc(
+ tok_mode->last_expr_buffer,
+ tok_mode->last_expr_size + size
+ );
+ if (new_buffer == NULL) {
+ PyMem_Free(tok_mode->last_expr_buffer);
+ goto error;
+ }
+ tok_mode->last_expr_buffer = new_buffer;
+ strncpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size, tok->cur, size);
+ tok_mode->last_expr_size += size;
+ break;
+ case '{':
+ if (tok_mode->last_expr_buffer != NULL) {
+ PyMem_Free(tok_mode->last_expr_buffer);
+ }
+ tok_mode->last_expr_buffer = PyMem_Malloc(size);
+ if (tok_mode->last_expr_buffer == NULL) {
+ goto error;
+ }
+ tok_mode->last_expr_size = size;
+ tok_mode->last_expr_end = -1;
+ strncpy(tok_mode->last_expr_buffer, tok->cur, size);
+ break;
+ case '}':
+ case '!':
+ tok_mode->last_expr_end = strlen(tok->start);
+ break;
+ case ':':
+ if (tok_mode->last_expr_end == -1) {
+ tok_mode->last_expr_end = strlen(tok->start);
+ }
+ break;
+ default:
+ Py_UNREACHABLE();
+ }
+ return 1;
+error:
+ tok->done = E_NOMEM;
+ return 0;
+}
+
+int
+_PyLexer_check_string_prefixes(struct tok_state *tok,
+ int saw_b, int saw_r, int saw_u,
+ int saw_f, int saw_t) {
+ // Supported: rb, rf, rt (in any order)
+ // Unsupported: ub, ur, uf, ut, bf, bt, ft (in any order)
+
+#define RETURN_SYNTAX_ERROR(PREFIX1, PREFIX2) \
+ do { \
+ (void)_PyTokenizer_syntaxerror_known_range( \
+ tok, (int)(tok->start + 1 - tok->line_start), \
+ (int)(tok->cur - tok->line_start), \
+ "'" PREFIX1 "' and '" PREFIX2 "' prefixes are incompatible"); \
+ return -1; \
+ } while (0)
+
+ if (saw_u && saw_b) {
+ RETURN_SYNTAX_ERROR("u", "b");
+ }
+ if (saw_u && saw_r) {
+ RETURN_SYNTAX_ERROR("u", "r");
+ }
+ if (saw_u && saw_f) {
+ RETURN_SYNTAX_ERROR("u", "f");
+ }
+ if (saw_u && saw_t) {
+ RETURN_SYNTAX_ERROR("u", "t");
+ }
+
+ if (saw_b && saw_f) {
+ RETURN_SYNTAX_ERROR("b", "f");
+ }
+ if (saw_b && saw_t) {
+ RETURN_SYNTAX_ERROR("b", "t");
+ }
+
+ if (saw_f && saw_t) {
+ RETURN_SYNTAX_ERROR("f", "t");
+ }
+
+#undef RETURN_SYNTAX_ERROR
+
+ return 0;
+}
+
+int
+_PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c)
+{
+ const char *p_start = NULL;
+ const char *p_end = NULL;
+
+ int quote = c;
+ int quote_size = 1; /* 1 or 3 */
+
+ /* Nodes of type STRING, especially multi line strings
+ must be handled differently in order to get both
+ the starting line number and the column offset right.
+ (cf. issue 16806) */
+ tok->first_lineno = tok->lineno;
+ tok->multi_line_start = tok->line_start;
+
+ /* Find the quote size and start of string */
+ int after_quote = tok_nextc(tok);
+ if (after_quote == quote) {
+ int after_after_quote = tok_nextc(tok);
+ if (after_after_quote == quote) {
+ quote_size = 3;
+ }
+ else {
+ // TODO: Check this
+ tok_backup(tok, after_after_quote);
+ tok_backup(tok, after_quote);
+ }
+ }
+ if (after_quote != quote) {
+ tok_backup(tok, after_quote);
+ }
+
+
+ p_start = tok->start;
+ p_end = tok->cur;
+ if (tok->tok_mode_stack_index + 1 >= MAXFSTRINGLEVEL) {
+ return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "too many nested f-strings or t-strings"));
+ }
+ tokenizer_mode *the_current_tok = TOK_NEXT_MODE(tok);
+ the_current_tok->kind = TOK_FSTRING_MODE;
+ the_current_tok->quote = quote;
+ the_current_tok->quote_size = quote_size;
+ the_current_tok->start = tok->start;
+ the_current_tok->multi_line_start = tok->line_start;
+ the_current_tok->first_line = tok->lineno;
+ the_current_tok->start_offset = -1;
+ the_current_tok->multi_line_start_offset = -1;
+ the_current_tok->last_expr_buffer = NULL;
+ the_current_tok->last_expr_size = 0;
+ the_current_tok->last_expr_end = -1;
+ the_current_tok->in_format_spec = 0;
+ the_current_tok->in_debug = 0;
+
+ enum string_kind_t string_kind = FSTRING;
+ switch (*tok->start) {
+ case 'T':
+ case 't':
+ the_current_tok->raw = Py_TOLOWER(*(tok->start + 1)) == 'r';
+ string_kind = TSTRING;
+ break;
+ case 'F':
+ case 'f':
+ the_current_tok->raw = Py_TOLOWER(*(tok->start + 1)) == 'r';
+ break;
+ case 'R':
+ case 'r':
+ the_current_tok->raw = 1;
+ if (Py_TOLOWER(*(tok->start + 1)) == 't') {
+ string_kind = TSTRING;
+ }
+ break;
+ default:
+ Py_UNREACHABLE();
+ }
+
+ the_current_tok->string_kind = string_kind;
+ the_current_tok->curly_bracket_depth = 0;
+ the_current_tok->curly_bracket_expr_start_depth = -1;
+ return string_kind == TSTRING ? MAKE_TOKEN(TSTRING_START) : MAKE_TOKEN(FSTRING_START);
+}
+
+int
+_PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
+{
+ const char *p_start = NULL;
+ const char *p_end = NULL;
+
+ int quote = c;
+ int quote_size = 1; /* 1 or 3 */
+ int end_quote_size = 0;
+ int has_escaped_quote = 0;
+
+ /* Nodes of type STRING, especially multi line strings
+ must be handled differently in order to get both
+ the starting line number and the column offset right.
+ (cf. issue 16806) */
+ tok->first_lineno = tok->lineno;
+ tok->multi_line_start = tok->line_start;
+
+ /* Find the quote size and start of string */
+ c = tok_nextc(tok);
+ if (c == quote) {
+ c = tok_nextc(tok);
+ if (c == quote) {
+ quote_size = 3;
+ }
+ else {
+ end_quote_size = 1; /* empty string found */
+ }
+ }
+ if (c != quote) {
+ tok_backup(tok, c);
+ }
+
+ /* Get rest of string */
+ while (end_quote_size != quote_size) {
+ c = tok_nextc(tok);
+ if (tok->done == E_ERROR) {
+ return MAKE_TOKEN(ERRORTOKEN);
+ }
+ if (tok->done == E_DECODE) {
+ break;
+ }
+ if (c == EOF || (quote_size == 1 && c == '\n')) {
+ assert(tok->multi_line_start != NULL);
+ // shift the tok_state's location into
+ // the start of string, and report the error
+ // from the initial quote character
+ tok->cur = (char *)tok->start;
+ tok->cur++;
+ tok->line_start = tok->multi_line_start;
+ int start = tok->lineno;
+ tok->lineno = tok->first_lineno;
+
+ if (INSIDE_FSTRING(tok)) {
+ /* When we are in an f-string, before raising the
+ * unterminated string literal error, check whether
+ * does the initial quote matches with f-strings quotes
+ * and if it is, then this must be a missing '}' token
+ * so raise the proper error */
+ tokenizer_mode *the_current_tok = TOK_GET_MODE(tok);
+ if (the_current_tok->quote == quote &&
+ the_current_tok->quote_size == quote_size) {
+ return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
+ "%c-string: expecting '}'", TOK_GET_STRING_PREFIX(tok)));
+ }
+ }
+
+ if (quote_size == 3) {
+ _PyTokenizer_syntaxerror(tok, "unterminated triple-quoted string literal"
+ " (detected at line %d)", start);
+ if (c != '\n') {
+ tok->done = E_EOFS;
+ }
+ return MAKE_TOKEN(ERRORTOKEN);
+ }
+ else {
+ if (has_escaped_quote) {
+ _PyTokenizer_syntaxerror(
+ tok,
+ "unterminated string literal (detected at line %d); "
+ "perhaps you escaped the end quote?",
+ start
+ );
+ } else {
+ _PyTokenizer_syntaxerror(
+ tok, "unterminated string literal (detected at line %d)", start
+ );
+ }
+ if (c != '\n') {
+ tok->done = E_EOLS;
+ }
+ return MAKE_TOKEN(ERRORTOKEN);
+ }
+ }
+ if (c == quote) {
+ end_quote_size += 1;
+ }
+ else {
+ end_quote_size = 0;
+ if (c == '\\') {
+ c = tok_nextc(tok); /* skip escaped char */
+ if (c == quote) { /* but record whether the escaped char was a quote */
+ has_escaped_quote = 1;
+ }
+ if (c == '\r') {
+ c = tok_nextc(tok);
+ }
+ }
+ }
+ }
+
+ p_start = tok->start;
+ p_end = tok->cur;
+ return MAKE_TOKEN(STRING);
+}
+
+int
+_PyLexer_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct token *token)
+{
+ const char *p_start = NULL;
+ const char *p_end = NULL;
+ int end_quote_size = 0;
+ int unicode_escape = 0;
+
+ tok->start = tok->cur;
+ tok->first_lineno = tok->lineno;
+ tok->starting_col_offset = tok->col_offset;
+
+ // If we start with a bracket, we defer to the normal mode as there is nothing for us to tokenize
+ // before it.
+ int start_char = tok_nextc(tok);
+ if (start_char == '{') {
+ int peek1 = tok_nextc(tok);
+ tok_backup(tok, peek1);
+ tok_backup(tok, start_char);
+ if (peek1 != '{') {
+ current_tok->curly_bracket_expr_start_depth++;
+ if (current_tok->curly_bracket_expr_start_depth >= MAX_EXPR_NESTING) {
+ return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
+ "%c-string: expressions nested too deeply", TOK_GET_STRING_PREFIX(tok)));
+ }
+ TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE;
+ return _PyLexer_get_normal_mode(tok, current_tok, token);
+ }
+ }
+ else {
+ tok_backup(tok, start_char);
+ }
+
+ // Check if we are at the end of the string
+ for (int i = 0; i < current_tok->quote_size; i++) {
+ int quote = tok_nextc(tok);
+ if (quote != current_tok->quote) {
+ tok_backup(tok, quote);
+ goto f_string_middle;
+ }
+ }
+
+ if (current_tok->last_expr_buffer != NULL) {
+ PyMem_Free(current_tok->last_expr_buffer);
+ current_tok->last_expr_buffer = NULL;
+ current_tok->last_expr_size = 0;
+ current_tok->last_expr_end = -1;
+ }
+
+ p_start = tok->start;
+ p_end = tok->cur;
+ tok->tok_mode_stack_index--;
+ return MAKE_TOKEN(FTSTRING_END(current_tok));
+
+f_string_middle:
+
+ // TODO: This is a bit of a hack, but it works for now. We need to find a better way to handle
+ // this.
+ tok->multi_line_start = tok->line_start;
+ while (end_quote_size != current_tok->quote_size) {
+ int c = tok_nextc(tok);
+ if (tok->done == E_ERROR || tok->done == E_DECODE) {
+ return MAKE_TOKEN(ERRORTOKEN);
+ }
+ int in_format_spec = (
+ current_tok->in_format_spec
+ &&
+ INSIDE_FSTRING_EXPR(current_tok)
+ );
+
+ if (c == EOF || (current_tok->quote_size == 1 && c == '\n')) {
+ if (tok->decoding_erred) {
+ return MAKE_TOKEN(ERRORTOKEN);
+ }
+
+ // If we are in a format spec and we found a newline,
+ // it means that the format spec ends here and we should
+ // return to the regular mode.
+ if (in_format_spec && c == '\n') {
+ if (current_tok->quote_size == 1) {
+ return MAKE_TOKEN(
+ _PyTokenizer_syntaxerror(
+ tok,
+ "%c-string: newlines are not allowed in format specifiers for single quoted %c-strings",
+ TOK_GET_STRING_PREFIX(tok), TOK_GET_STRING_PREFIX(tok)
+ )
+ );
+ }
+ tok_backup(tok, c);
+ TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE;
+ current_tok->in_format_spec = 0;
+ p_start = tok->start;
+ p_end = tok->cur;
+ return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
+ }
+
+ assert(tok->multi_line_start != NULL);
+ // shift the tok_state's location into
+ // the start of string, and report the error
+ // from the initial quote character
+ tok->cur = (char *)current_tok->start;
+ tok->cur++;
+ tok->line_start = current_tok->multi_line_start;
+ int start = tok->lineno;
+
+ tokenizer_mode *the_current_tok = TOK_GET_MODE(tok);
+ tok->lineno = the_current_tok->first_line;
+
+ if (current_tok->quote_size == 3) {
+ _PyTokenizer_syntaxerror(tok,
+ "unterminated triple-quoted %c-string literal"
+ " (detected at line %d)",
+ TOK_GET_STRING_PREFIX(tok), start);
+ if (c != '\n') {
+ tok->done = E_EOFS;
+ }
+ return MAKE_TOKEN(ERRORTOKEN);
+ }
+ else {
+ return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
+ "unterminated %c-string literal (detected at"
+ " line %d)", TOK_GET_STRING_PREFIX(tok), start));
+ }
+ }
+
+ if (c == current_tok->quote) {
+ end_quote_size += 1;
+ continue;
+ } else {
+ end_quote_size = 0;
+ }
+
+ if (c == '{') {
+ if (!_PyLexer_update_ftstring_expr(tok, c)) {
+ return MAKE_TOKEN(ENDMARKER);
+ }
+ int peek = tok_nextc(tok);
+ if (peek != '{' || in_format_spec) {
+ tok_backup(tok, peek);
+ tok_backup(tok, c);
+ current_tok->curly_bracket_expr_start_depth++;
+ if (current_tok->curly_bracket_expr_start_depth >= MAX_EXPR_NESTING) {
+ return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
+ "%c-string: expressions nested too deeply", TOK_GET_STRING_PREFIX(tok)));
+ }
+ TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE;
+ current_tok->in_format_spec = 0;
+ p_start = tok->start;
+ p_end = tok->cur;
+ } else {
+ p_start = tok->start;
+ p_end = tok->cur - 1;
+ }
+ return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
+ } else if (c == '}') {
+ if (unicode_escape) {
+ p_start = tok->start;
+ p_end = tok->cur;
+ return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
+ }
+ int peek = tok_nextc(tok);
+
+ // The tokenizer can only be in the format spec if we have already completed the expression
+ // scanning (indicated by the end of the expression being set) and we are not at the top level
+ // of the bracket stack (-1 is the top level). Since format specifiers can't legally use double
+ // brackets, we can bypass it here.
+ int cursor = current_tok->curly_bracket_depth;
+ if (peek == '}' && !in_format_spec && cursor == 0) {
+ p_start = tok->start;
+ p_end = tok->cur - 1;
+ } else {
+ tok_backup(tok, peek);
+ tok_backup(tok, c);
+ TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE;
+ current_tok->in_format_spec = 0;
+ p_start = tok->start;
+ p_end = tok->cur;
+ }
+ return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
+ } else if (c == '\\') {
+ int peek = tok_nextc(tok);
+ if (peek == '\r') {
+ peek = tok_nextc(tok);
+ }
+ // Special case when the backslash is right before a curly
+ // brace. We have to restore and return the control back
+ // to the loop for the next iteration.
+ if (peek == '{' || peek == '}') {
+ if (!current_tok->raw) {
+ if (_PyTokenizer_warn_invalid_escape_sequence(tok, peek)) {
+ return MAKE_TOKEN(ERRORTOKEN);
+ }
+ }
+ tok_backup(tok, peek);
+ continue;
+ }
+
+ if (!current_tok->raw) {
+ if (peek == 'N') {
+ /* Handle named unicode escapes (\N{BULLET}) */
+ peek = tok_nextc(tok);
+ if (peek == '{') {
+ unicode_escape = 1;
+ } else {
+ tok_backup(tok, peek);
+ }
+ }
+ } /* else {
+ skip the escaped character
+ }*/
+ }
+ }
+
+ // Backup the f-string quotes to emit a final FSTRING_MIDDLE and
+ // add the quotes to the FSTRING_END in the next tokenizer iteration.
+ for (int i = 0; i < current_tok->quote_size; i++) {
+ tok_backup(tok, current_tok->quote);
+ }
+ p_start = tok->start;
+ p_end = tok->cur;
+ return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
+}
str(MOD_DIR.parent.parent.parent / "Python" / "Python-ast.c"),
str(MOD_DIR.parent.parent.parent / "Python" / "asdl.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "lexer.c"),
+ str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "number.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "state.c"),
+ str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "string.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "lexer" / "buffer.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "string_tokenizer.c"),
str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer" / "file_tokenizer.c"),
projects / thirdparty / Python / cpython.git / commitdiff
