This commit replaces the Python implementation of the tokenize module with an implementation
that reuses the real C tokenizer via a private extension module. The tokenize module now implements
a compatibility layer that transforms tokens from the C tokenizer into Python tokenize tokens for backward
compatibility.
As the C tokenizer does not emit some tokens that the Python tokenizer provides (such as comments and non-semantic newlines), a new special mode has been added to the C tokenizer mode that currently is only used via
the extension module that exposes it to the Python layer. This new mode forces the C tokenizer to emit these new extra tokens and add the appropriate metadata that is needed to match the old Python implementation.
Co-authored-by: Pablo Galindo <pablogsal@gmail.com>
.. data:: FSTRING_END
+.. data:: COMMENT
+
+.. data:: NL
+
.. data:: ERRORTOKEN
.. data:: N_TOKENS
the :mod:`tokenize` module.
.. data:: COMMENT
+ :noindex:
Token value used to indicate a comment.
.. data:: NL
+ :noindex:
Token value used to indicate a non-terminating newline. The
:data:`NEWLINE` token indicates the end of a logical line of Python code;
FSTRING_START
FSTRING_MIDDLE
FSTRING_END
+COMMENT
+NL
ERRORTOKEN
# These aren't used by the C tokenizer but are needed for tokenize.py
-COMMENT
-NL
ENCODING
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exception));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(exp));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(extend));
+ _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(extra_tokens));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(facility));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(factory));
_PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(false));
STRUCT_FOR_ID(exception)
STRUCT_FOR_ID(exp)
STRUCT_FOR_ID(extend)
+ STRUCT_FOR_ID(extra_tokens)
STRUCT_FOR_ID(facility)
STRUCT_FOR_ID(factory)
STRUCT_FOR_ID(false)
INIT_ID(exception), \
INIT_ID(exp), \
INIT_ID(extend), \
+ INIT_ID(extra_tokens), \
INIT_ID(facility), \
INIT_ID(factory), \
INIT_ID(false), \
#define FSTRING_START 61
#define FSTRING_MIDDLE 62
#define FSTRING_END 63
-#define ERRORTOKEN 64
+#define COMMENT 64
+#define NL 65
+#define ERRORTOKEN 66
#define N_TOKENS 68
#define NT_OFFSET 256
string = &_Py_ID(extend);
assert(_PyUnicode_CheckConsistency(string, 1));
_PyUnicode_InternInPlace(interp, &string);
+ string = &_Py_ID(extra_tokens);
+ assert(_PyUnicode_CheckConsistency(string, 1));
+ _PyUnicode_InternInPlace(interp, &string);
string = &_Py_ID(facility);
assert(_PyUnicode_CheckConsistency(string, 1));
_PyUnicode_InternInPlace(interp, &string);
if string == ',':
current_parameter += 1
- if (type == ERRORTOKEN) and (string == '$'):
+ if (type == OP) and (string == '$'):
assert self_parameter is None
self_parameter = current_parameter
continue
add(string)
if (string == ','):
add(' ')
- clean_signature = ''.join(text)
+ clean_signature = ''.join(text).strip()
return clean_signature, self_parameter
errprint("%r: Token Error: %s" % (file, msg))
return
+ except SyntaxError as msg:
+ errprint("%r: Token Error: %s" % (file, msg))
+ return
+
except IndentationError as msg:
errprint("%r: Indentation Error: %s" % (file, msg))
return
return prefix + " " + ', '.join(firsts)
def process_tokens(tokens):
+ try:
+ _process_tokens(tokens)
+ except TabError as e:
+ raise NannyNag(e.lineno, e.msg, e.text)
+
+def _process_tokens(tokens):
INDENT = tokenize.INDENT
DEDENT = tokenize.DEDENT
NEWLINE = tokenize.NEWLINE
with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as file_path:
out = f"{file_path!r}: *** Line 3: trouble in tab city! ***\n"
out += "offending line: '\\tprint(\"world\")\\n'\n"
- out += "indent not equal e.g. at tab size 1\n"
+ out += "inconsistent use of tabs and spaces in indentation\n"
tabnanny.verbose = 1
self.verify_tabnanny_check(file_path, out=out)
def test_with_errored_file(self):
"""Should displays error when errored python file is given."""
with TemporaryPyFile(SOURCE_CODES["wrong_indented"]) as file_path:
- stderr = f"{file_path!r}: Indentation Error: "
+ stderr = f"{file_path!r}: Token Error: "
stderr += ('unindent does not match any outer indentation level'
' (<tokenize>, line 3)')
self.validate_cmd(file_path, stderr=stderr, expect_failure=True)
from tokenize import (tokenize, _tokenize, untokenize, NUMBER, NAME, OP,
STRING, ENDMARKER, ENCODING, tok_name, detect_encoding,
open as tokenize_open, Untokenizer, generate_tokens,
- NEWLINE, _generate_tokens_from_c_tokenizer, DEDENT)
+ NEWLINE, _generate_tokens_from_c_tokenizer, DEDENT, TokenInfo)
from io import BytesIO, StringIO
import unittest
from textwrap import dedent
NAME 'False' (4, 11) (4, 16)
COMMENT '# NEWLINE' (4, 17) (4, 26)
NEWLINE '\\n' (4, 26) (4, 27)
- DEDENT '' (5, 0) (5, 0)
+ DEDENT '' (4, 27) (4, 27)
""")
indent_error_file = b"""\
def k(x):
continue
self.assertEqual(number_token(lit), lit)
for lit in INVALID_UNDERSCORE_LITERALS:
+ try:
+ number_token(lit)
+ except SyntaxError:
+ continue
self.assertNotEqual(number_token(lit), lit)
def test_string(self):
STRING 'rb"\""a\\\\\\nb\\\\\\nc"\""' (1, 0) (3, 4)
""")
self.check_tokenize('f"abc"', """\
- STRING 'f"abc"' (1, 0) (1, 6)
+ FSTRING_START 'f"' (1, 0) (1, 2)
+ FSTRING_MIDDLE 'abc' (1, 2) (1, 5)
+ FSTRING_END '"' (1, 5) (1, 6)
""")
self.check_tokenize('fR"a{b}c"', """\
- STRING 'fR"a{b}c"' (1, 0) (1, 9)
+ FSTRING_START 'fR"' (1, 0) (1, 3)
+ FSTRING_MIDDLE 'a' (1, 3) (1, 4)
+ OP '{' (1, 4) (1, 5)
+ NAME 'b' (1, 5) (1, 6)
+ OP '}' (1, 6) (1, 7)
+ FSTRING_MIDDLE 'c' (1, 7) (1, 8)
+ FSTRING_END '"' (1, 8) (1, 9)
+ """)
+ self.check_tokenize('fR"a{{{b!r}}}c"', """\
+ FSTRING_START 'fR"' (1, 0) (1, 3)
+ FSTRING_MIDDLE 'a{' (1, 3) (1, 5)
+ OP '{' (1, 6) (1, 7)
+ NAME 'b' (1, 7) (1, 8)
+ OP '!' (1, 8) (1, 9)
+ NAME 'r' (1, 9) (1, 10)
+ OP '}' (1, 10) (1, 11)
+ FSTRING_MIDDLE '}' (1, 11) (1, 12)
+ FSTRING_MIDDLE 'c' (1, 13) (1, 14)
+ FSTRING_END '"' (1, 14) (1, 15)
+ """)
+ self.check_tokenize('f"{{{1+1}}}"', """\
+ FSTRING_START 'f"' (1, 0) (1, 2)
+ FSTRING_MIDDLE '{' (1, 2) (1, 3)
+ OP '{' (1, 4) (1, 5)
+ NUMBER '1' (1, 5) (1, 6)
+ OP '+' (1, 6) (1, 7)
+ NUMBER '1' (1, 7) (1, 8)
+ OP '}' (1, 8) (1, 9)
+ FSTRING_MIDDLE '}' (1, 9) (1, 10)
+ FSTRING_END '"' (1, 11) (1, 12)
+ """)
+ self.check_tokenize('f"""{f\'\'\'{f\'{f"{1+1}"}\'}\'\'\'}"""', """\
+ FSTRING_START 'f\"""' (1, 0) (1, 4)
+ OP '{' (1, 4) (1, 5)
+ FSTRING_START "f'''" (1, 5) (1, 9)
+ OP '{' (1, 9) (1, 10)
+ FSTRING_START "f'" (1, 10) (1, 12)
+ OP '{' (1, 12) (1, 13)
+ FSTRING_START 'f"' (1, 13) (1, 15)
+ OP '{' (1, 15) (1, 16)
+ NUMBER '1' (1, 16) (1, 17)
+ OP '+' (1, 17) (1, 18)
+ NUMBER '1' (1, 18) (1, 19)
+ OP '}' (1, 19) (1, 20)
+ FSTRING_END '"' (1, 20) (1, 21)
+ OP '}' (1, 21) (1, 22)
+ FSTRING_END "'" (1, 22) (1, 23)
+ OP '}' (1, 23) (1, 24)
+ FSTRING_END "'''" (1, 24) (1, 27)
+ OP '}' (1, 27) (1, 28)
+ FSTRING_END '\"""' (1, 28) (1, 31)
+ """)
+ self.check_tokenize('f""" x\nstr(data, encoding={invalid!r})\n"""', """\
+ FSTRING_START 'f\"""' (1, 0) (1, 4)
+ FSTRING_MIDDLE ' x\\nstr(data, encoding=' (1, 4) (2, 19)
+ OP '{' (2, 19) (2, 20)
+ NAME 'invalid' (2, 20) (2, 27)
+ OP '!' (2, 27) (2, 28)
+ NAME 'r' (2, 28) (2, 29)
+ OP '}' (2, 29) (2, 30)
+ FSTRING_MIDDLE ')\\n' (2, 30) (3, 0)
+ FSTRING_END '\"""' (3, 0) (3, 3)
+ """)
+ self.check_tokenize('f"""123456789\nsomething{None}bad"""', """\
+ FSTRING_START 'f\"""' (1, 0) (1, 4)
+ FSTRING_MIDDLE '123456789\\nsomething' (1, 4) (2, 9)
+ OP '{' (2, 9) (2, 10)
+ NAME 'None' (2, 10) (2, 14)
+ OP '}' (2, 14) (2, 15)
+ FSTRING_MIDDLE 'bad' (2, 15) (2, 18)
+ FSTRING_END '\"""' (2, 18) (2, 21)
""")
self.check_tokenize('f"""abc"""', """\
- STRING 'f\"\"\"abc\"\"\"' (1, 0) (1, 10)
+ FSTRING_START 'f\"""' (1, 0) (1, 4)
+ FSTRING_MIDDLE 'abc' (1, 4) (1, 7)
+ FSTRING_END '\"""' (1, 7) (1, 10)
""")
self.check_tokenize(r'f"abc\
def"', """\
- STRING 'f"abc\\\\\\ndef"' (1, 0) (2, 4)
+ FSTRING_START 'f"' (1, 0) (1, 2)
+ FSTRING_MIDDLE 'abc\\\\\\ndef' (1, 2) (2, 3)
+ FSTRING_END '"' (2, 3) (2, 4)
""")
self.check_tokenize(r'Rf"abc\
def"', """\
- STRING 'Rf"abc\\\\\\ndef"' (1, 0) (2, 4)
+ FSTRING_START 'Rf"' (1, 0) (1, 3)
+ FSTRING_MIDDLE 'abc\\\\\\ndef' (1, 3) (2, 3)
+ FSTRING_END '"' (2, 3) (2, 4)
+ """)
+ self.check_tokenize("f'some words {a+b:.3f} more words {c+d=} final words'", """\
+ FSTRING_START "f'" (1, 0) (1, 2)
+ FSTRING_MIDDLE 'some words ' (1, 2) (1, 13)
+ OP '{' (1, 13) (1, 14)
+ NAME 'a' (1, 14) (1, 15)
+ OP '+' (1, 15) (1, 16)
+ NAME 'b' (1, 16) (1, 17)
+ OP ':' (1, 17) (1, 18)
+ FSTRING_MIDDLE '.3f' (1, 18) (1, 21)
+ OP '}' (1, 21) (1, 22)
+ FSTRING_MIDDLE ' more words ' (1, 22) (1, 34)
+ OP '{' (1, 34) (1, 35)
+ NAME 'c' (1, 35) (1, 36)
+ OP '+' (1, 36) (1, 37)
+ NAME 'd' (1, 37) (1, 38)
+ OP '=' (1, 38) (1, 39)
+ OP '}' (1, 39) (1, 40)
+ FSTRING_MIDDLE ' final words' (1, 40) (1, 52)
+ FSTRING_END "'" (1, 52) (1, 53)
""")
def test_function(self):
NEWLINE '\\n' (2, 5) (2, 6)
INDENT ' \\t' (3, 0) (3, 9)
NAME 'pass' (3, 9) (3, 13)
- DEDENT '' (4, 0) (4, 0)
- DEDENT '' (4, 0) (4, 0)
+ DEDENT '' (3, 14) (3, 14)
+ DEDENT '' (3, 14) (3, 14)
""")
def test_non_ascii_identifiers(self):
NUMBER '1' (2, 17) (2, 18)
OP ':' (2, 18) (2, 19)
NAME 'pass' (2, 20) (2, 24)
- DEDENT '' (3, 0) (3, 0)
+ DEDENT '' (2, 25) (2, 25)
""")
self.check_tokenize('''async def foo(async): await''', """\
NAME 'await' (6, 2) (6, 7)
OP '=' (6, 8) (6, 9)
NUMBER '2' (6, 10) (6, 11)
- DEDENT '' (7, 0) (7, 0)
+ DEDENT '' (6, 12) (6, 12)
""")
self.check_tokenize('''\
NAME 'await' (6, 2) (6, 7)
OP '=' (6, 8) (6, 9)
NUMBER '2' (6, 10) (6, 11)
- DEDENT '' (7, 0) (7, 0)
+ DEDENT '' (6, 12) (6, 12)
""")
class GenerateTokensTest(TokenizeTest):
])
else:
result.append((toknum, tokval))
- return untokenize(result).decode('utf-8')
+ return untokenize(result).decode('utf-8').strip()
class TestMisc(TestCase):
nonlocal first
if not first:
first = True
- return line
+ yield line
else:
- return b''
+ yield b''
# skip the initial encoding token and the end tokens
- tokens = list(_tokenize(readline, encoding='utf-8'))[1:-2]
- expected_tokens = [(3, '"ЉЊЈЁЂ"', (1, 0), (1, 7), '"ЉЊЈЁЂ"')]
+ tokens = list(_tokenize(readline(), encoding='utf-8'))[:-2]
+ expected_tokens = [TokenInfo(3, '"ЉЊЈЁЂ"', (1, 0), (1, 7), '"ЉЊЈЁЂ"\n')]
self.assertEqual(tokens, expected_tokens,
"bytes not decoded with encoding")
- def test__tokenize_does_not_decode_with_encoding_none(self):
- literal = '"ЉЊЈЁЂ"'
- first = False
- def readline():
- nonlocal first
- if not first:
- first = True
- return literal
- else:
- return b''
-
- # skip the end tokens
- tokens = list(_tokenize(readline, encoding=None))[:-2]
- expected_tokens = [(3, '"ЉЊЈЁЂ"', (1, 0), (1, 7), '"ЉЊЈЁЂ"')]
- self.assertEqual(tokens, expected_tokens,
- "string not tokenized when encoding is None")
-
class TestDetectEncoding(TestCase):
def test_tokenize(self):
import tokenize as tokenize_module
- encoding = object()
+ encoding = "utf-8"
encoding_used = None
def mock_detect_encoding(readline):
return encoding, [b'first', b'second']
encoding_used = encoding
out = []
while True:
- next_line = readline()
+ try:
+ next_line = next(readline)
+ except StopIteration:
+ return out
if next_line:
out.append(next_line)
continue
tokenize_module._tokenize = mock__tokenize
try:
results = tokenize(mock_readline)
- self.assertEqual(list(results),
+ self.assertEqual(list(results)[1:],
[b'first', b'second', b'1', b'2', b'3', b'4'])
finally:
tokenize_module.detect_encoding = orig_detect_encoding
if support.verbose >= 2:
print('tokenize', testfile)
with open(testfile, 'rb') as f:
- with self.subTest(file=testfile):
- self.check_roundtrip(f)
+ # with self.subTest(file=testfile):
+ self.check_roundtrip(f)
def roundtrip(self, code):
def test_unicode(self):
self.check_tokenize("Örter = u'places'\ngrün = U'green'", """\
- NAME 'Örter' (1, 0) (1, 6)
- EQUAL '=' (1, 7) (1, 8)
- STRING "u'places'" (1, 9) (1, 18)
- NEWLINE '' (1, 18) (1, 18)
- NAME 'grün' (2, 0) (2, 5)
- EQUAL '=' (2, 6) (2, 7)
- STRING "U'green'" (2, 8) (2, 16)
+ NAME 'Örter' (1, 0) (1, 5)
+ EQUAL '=' (1, 6) (1, 7)
+ STRING "u'places'" (1, 8) (1, 17)
+ NEWLINE '' (1, 17) (1, 17)
+ NAME 'grün' (2, 0) (2, 4)
+ EQUAL '=' (2, 5) (2, 6)
+ STRING "U'green'" (2, 7) (2, 15)
""")
def test_invalid_syntax(self):
compile(valid, "<string>", "exec")
invalid = generate_source(MAXINDENT)
- tokens = list(_generate_tokens_from_c_tokenizer(invalid))
- self.assertEqual(tokens[-1].type, NEWLINE)
+ self.assertRaises(SyntaxError, lambda: list(_generate_tokens_from_c_tokenizer(invalid)))
self.assertRaises(
IndentationError, compile, invalid, "<string>", "exec"
)
FSTRING_START = 61
FSTRING_MIDDLE = 62
FSTRING_END = 63
+COMMENT = 64
+NL = 65
# These aren't used by the C tokenizer but are needed for tokenize.py
-ERRORTOKEN = 64
-COMMENT = 65
-NL = 66
+ERRORTOKEN = 66
ENCODING = 67
N_TOKENS = 68
# Special definitions for cooperation with parser
else:
return self.type
-def group(*choices): return '(' + '|'.join(choices) + ')'
-def any(*choices): return group(*choices) + '*'
-def maybe(*choices): return group(*choices) + '?'
-
-# Note: we use unicode matching for names ("\w") but ascii matching for
-# number literals.
-Whitespace = r'[ \f\t]*'
-Comment = r'#[^\r\n]*'
-Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
-Name = r'\w+'
-
-Hexnumber = r'0[xX](?:_?[0-9a-fA-F])+'
-Binnumber = r'0[bB](?:_?[01])+'
-Octnumber = r'0[oO](?:_?[0-7])+'
-Decnumber = r'(?:0(?:_?0)*|[1-9](?:_?[0-9])*)'
-Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
-Exponent = r'[eE][-+]?[0-9](?:_?[0-9])*'
-Pointfloat = group(r'[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?',
- r'\.[0-9](?:_?[0-9])*') + maybe(Exponent)
-Expfloat = r'[0-9](?:_?[0-9])*' + Exponent
-Floatnumber = group(Pointfloat, Expfloat)
-Imagnumber = group(r'[0-9](?:_?[0-9])*[jJ]', Floatnumber + r'[jJ]')
-Number = group(Imagnumber, Floatnumber, Intnumber)
-
-# Return the empty string, plus all of the valid string prefixes.
-def _all_string_prefixes():
- # The valid string prefixes. Only contain the lower case versions,
- # and don't contain any permutations (include 'fr', but not
- # 'rf'). The various permutations will be generated.
- _valid_string_prefixes = ['b', 'r', 'u', 'f', 'br', 'fr']
- # if we add binary f-strings, add: ['fb', 'fbr']
- result = {''}
- for prefix in _valid_string_prefixes:
- for t in _itertools.permutations(prefix):
- # create a list with upper and lower versions of each
- # character
- for u in _itertools.product(*[(c, c.upper()) for c in t]):
- result.add(''.join(u))
- return result
-
-@functools.lru_cache
-def _compile(expr):
- return re.compile(expr, re.UNICODE)
-
-# Note that since _all_string_prefixes includes the empty string,
-# StringPrefix can be the empty string (making it optional).
-StringPrefix = group(*_all_string_prefixes())
-
-# Tail end of ' string.
-Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
-# Tail end of " string.
-Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
-# Tail end of ''' string.
-Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
-# Tail end of """ string.
-Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
-Triple = group(StringPrefix + "'''", StringPrefix + '"""')
-# Single-line ' or " string.
-String = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
- StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
-
-# Sorting in reverse order puts the long operators before their prefixes.
-# Otherwise if = came before ==, == would get recognized as two instances
-# of =.
-Special = group(*map(re.escape, sorted(EXACT_TOKEN_TYPES, reverse=True)))
-Funny = group(r'\r?\n', Special)
-
-PlainToken = group(Number, Funny, String, Name)
-Token = Ignore + PlainToken
-
-# First (or only) line of ' or " string.
-ContStr = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
- group("'", r'\\\r?\n'),
- StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
- group('"', r'\\\r?\n'))
-PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple)
-PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
-
-# For a given string prefix plus quotes, endpats maps it to a regex
-# to match the remainder of that string. _prefix can be empty, for
-# a normal single or triple quoted string (with no prefix).
-endpats = {}
-for _prefix in _all_string_prefixes():
- endpats[_prefix + "'"] = Single
- endpats[_prefix + '"'] = Double
- endpats[_prefix + "'''"] = Single3
- endpats[_prefix + '"""'] = Double3
-del _prefix
-
-# A set of all of the single and triple quoted string prefixes,
-# including the opening quotes.
-single_quoted = set()
-triple_quoted = set()
-for t in _all_string_prefixes():
- for u in (t + '"', t + "'"):
- single_quoted.add(u)
- for u in (t + '"""', t + "'''"):
- triple_quoted.add(u)
-del t, u
-
-tabsize = 8
class TokenError(Exception): pass
-class StopTokenizing(Exception): pass
+class StopTokenizing(Exception): pass
class Untokenizer:
self.tokens.append(indent)
self.prev_col = len(indent)
startline = False
+ elif tok_type == FSTRING_MIDDLE:
+ if '{' in token or '}' in token:
+ end_line, end_col = end
+ end = (end_line, end_col + token.count('{') + token.count('}'))
+ token = re.sub('{', '{{', token)
+ token = re.sub('}', '}}', token)
+
+
self.add_whitespace(start)
self.tokens.append(token)
self.prev_row, self.prev_col = end
elif startline and indents:
toks_append(indents[-1])
startline = False
+ elif toknum == FSTRING_MIDDLE:
+ if '{' in tokval or '}' in tokval:
+ tokval = re.sub('{', '{{', tokval)
+ tokval = re.sub('}', '}}', tokval)
+
toks_append(tokval)
buffer.close()
raise
-
def tokenize(readline):
"""
The tokenize() generator requires one argument, readline, which
which tells you which encoding was used to decode the bytes stream.
"""
encoding, consumed = detect_encoding(readline)
- empty = _itertools.repeat(b"")
- rl_gen = _itertools.chain(consumed, iter(readline, b""), empty)
- return _tokenize(rl_gen.__next__, encoding)
-
-
-def _tokenize(readline, encoding):
- lnum = parenlev = continued = 0
- numchars = '0123456789'
- contstr, needcont = '', 0
- contline = None
- indents = [0]
-
+ rl_gen = _itertools.chain(consumed, iter(readline, b""))
if encoding is not None:
if encoding == "utf-8-sig":
# BOM will already have been stripped.
encoding = "utf-8"
yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '')
- last_line = b''
- line = b''
- while True: # loop over lines in stream
- try:
- # We capture the value of the line variable here because
- # readline uses the empty string '' to signal end of input,
- # hence `line` itself will always be overwritten at the end
- # of this loop.
- last_line = line
- line = readline()
- except StopIteration:
- line = b''
-
- if encoding is not None:
- line = line.decode(encoding)
- lnum += 1
- pos, max = 0, len(line)
-
- if contstr: # continued string
- if not line:
- raise TokenError("EOF in multi-line string", strstart)
- endmatch = endprog.match(line)
- if endmatch:
- pos = end = endmatch.end(0)
- yield TokenInfo(STRING, contstr + line[:end],
- strstart, (lnum, end), contline + line)
- contstr, needcont = '', 0
- contline = None
- elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
- yield TokenInfo(ERRORTOKEN, contstr + line,
- strstart, (lnum, len(line)), contline)
- contstr = ''
- contline = None
- continue
- else:
- contstr = contstr + line
- contline = contline + line
- continue
-
- elif parenlev == 0 and not continued: # new statement
- if not line: break
- column = 0
- while pos < max: # measure leading whitespace
- if line[pos] == ' ':
- column += 1
- elif line[pos] == '\t':
- column = (column//tabsize + 1)*tabsize
- elif line[pos] == '\f':
- column = 0
- else:
- break
- pos += 1
- if pos == max:
- break
-
- if line[pos] in '#\r\n': # skip comments or blank lines
- if line[pos] == '#':
- comment_token = line[pos:].rstrip('\r\n')
- yield TokenInfo(COMMENT, comment_token,
- (lnum, pos), (lnum, pos + len(comment_token)), line)
- pos += len(comment_token)
-
- yield TokenInfo(NL, line[pos:],
- (lnum, pos), (lnum, len(line)), line)
- continue
-
- if column > indents[-1]: # count indents or dedents
- indents.append(column)
- yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
- while column < indents[-1]:
- if column not in indents:
- raise IndentationError(
- "unindent does not match any outer indentation level",
- ("<tokenize>", lnum, pos, line))
- indents = indents[:-1]
-
- yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line)
-
- else: # continued statement
- if not line:
- raise TokenError("EOF in multi-line statement", (lnum, 0))
- continued = 0
-
- while pos < max:
- pseudomatch = _compile(PseudoToken).match(line, pos)
- if pseudomatch: # scan for tokens
- start, end = pseudomatch.span(1)
- spos, epos, pos = (lnum, start), (lnum, end), end
- if start == end:
- continue
- token, initial = line[start:end], line[start]
-
- if (initial in numchars or # ordinary number
- (initial == '.' and token != '.' and token != '...')):
- yield TokenInfo(NUMBER, token, spos, epos, line)
- elif initial in '\r\n':
- if parenlev > 0:
- yield TokenInfo(NL, token, spos, epos, line)
- else:
- yield TokenInfo(NEWLINE, token, spos, epos, line)
-
- elif initial == '#':
- assert not token.endswith("\n")
- yield TokenInfo(COMMENT, token, spos, epos, line)
-
- elif token in triple_quoted:
- endprog = _compile(endpats[token])
- endmatch = endprog.match(line, pos)
- if endmatch: # all on one line
- pos = endmatch.end(0)
- token = line[start:pos]
- yield TokenInfo(STRING, token, spos, (lnum, pos), line)
- else:
- strstart = (lnum, start) # multiple lines
- contstr = line[start:]
- contline = line
- break
-
- # Check up to the first 3 chars of the token to see if
- # they're in the single_quoted set. If so, they start
- # a string.
- # We're using the first 3, because we're looking for
- # "rb'" (for example) at the start of the token. If
- # we switch to longer prefixes, this needs to be
- # adjusted.
- # Note that initial == token[:1].
- # Also note that single quote checking must come after
- # triple quote checking (above).
- elif (initial in single_quoted or
- token[:2] in single_quoted or
- token[:3] in single_quoted):
- if token[-1] == '\n': # continued string
- strstart = (lnum, start)
- # Again, using the first 3 chars of the
- # token. This is looking for the matching end
- # regex for the correct type of quote
- # character. So it's really looking for
- # endpats["'"] or endpats['"'], by trying to
- # skip string prefix characters, if any.
- endprog = _compile(endpats.get(initial) or
- endpats.get(token[1]) or
- endpats.get(token[2]))
- contstr, needcont = line[start:], 1
- contline = line
- break
- else: # ordinary string
- yield TokenInfo(STRING, token, spos, epos, line)
-
- elif initial.isidentifier(): # ordinary name
- yield TokenInfo(NAME, token, spos, epos, line)
- elif initial == '\\': # continued stmt
- continued = 1
- else:
- if initial in '([{':
- parenlev += 1
- elif initial in ')]}':
- parenlev -= 1
- yield TokenInfo(OP, token, spos, epos, line)
- else:
- yield TokenInfo(ERRORTOKEN, line[pos],
- (lnum, pos), (lnum, pos+1), line)
- pos += 1
-
- # Add an implicit NEWLINE if the input doesn't end in one
- if last_line and last_line[-1] not in '\r\n' and not last_line.strip().startswith("#"):
- yield TokenInfo(NEWLINE, '', (lnum - 1, len(last_line)), (lnum - 1, len(last_line) + 1), '')
- for indent in indents[1:]: # pop remaining indent levels
- yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '')
- yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '')
+ yield from _tokenize(rl_gen, encoding)
+
+def _tokenize(rl_gen, encoding):
+ source = b"".join(rl_gen).decode(encoding)
+ token = None
+ for token in _generate_tokens_from_c_tokenizer(source, extra_tokens=True):
+ # TODO: Marta -> limpiar esto
+ if 6 < token.type <= 54:
+ token = token._replace(type=OP)
+ if token.type in {ASYNC, AWAIT}:
+ token = token._replace(type=NAME)
+ if token.type == NEWLINE:
+ l_start, c_start = token.start
+ l_end, c_end = token.end
+ token = token._replace(string='\n', start=(l_start, c_start), end=(l_end, c_end+1))
+
+ yield token
+ if token is not None:
+ last_line, _ = token.start
+ yield TokenInfo(ENDMARKER, '', (last_line + 1, 0), (last_line + 1, 0), '')
def generate_tokens(readline):
This has the same API as tokenize(), except that it expects the *readline*
callable to return str objects instead of bytes.
"""
- return _tokenize(readline, None)
+ def _gen():
+ while True:
+ try:
+ line = readline()
+ except StopIteration:
+ return
+ if not line:
+ return
+ yield line.encode()
+ return _tokenize(_gen(), 'utf-8')
def main():
import argparse
tokens = list(tokenize(f.readline))
else:
filename = "<stdin>"
- tokens = _tokenize(sys.stdin.readline, None)
+ tokens = _tokenize(
+ (x.encode('utf-8') for x in iter(sys.stdin.readline, "")
+ ), "utf-8")
+
# Output the tokenization
for token in tokens:
perror("unexpected error: %s" % err)
raise
-def _generate_tokens_from_c_tokenizer(source):
+def _generate_tokens_from_c_tokenizer(source, extra_tokens=False):
"""Tokenize a source reading Python code as unicode strings using the internal C tokenizer"""
import _tokenize as c_tokenizer
- for info in c_tokenizer.TokenizerIter(source):
+ for info in c_tokenizer.TokenizerIter(source, extra_tokens=extra_tokens):
tok, type, lineno, end_lineno, col_off, end_col_off, line = info
yield TokenInfo(type, tok, (lineno, col_off), (end_lineno, end_col_off), line)
--- /dev/null
+Implement PEP 701 changes in the :mod:`tokenize` module. Patch by Marta Gómez Macías and Pablo Galindo Salgado
_PyPegen_fill_token(Parser *p)
{
struct token new_token;
- new_token.metadata = NULL;
+ _PyToken_Init(&new_token);
int type = _PyTokenizer_Get(p->tok, &new_token);
// Record and skip '# type: ignore' comments
Token *t = p->tokens[p->fill];
return initialize_token(p, t, &new_token, type);
error:
- Py_XDECREF(new_token.metadata);
+ _PyToken_Free(&new_token);
return -1;
}
int ret = 0;
struct token new_token;
- new_token.metadata = NULL;
+ _PyToken_Init(&new_token);
for (;;) {
switch (_PyTokenizer_Get(p->tok, &new_token)) {
exit:
- Py_XDECREF(new_token.metadata);
+ _PyToken_Free(&new_token);
// If we're in an f-string, we want the syntax error in the expression part
// to propagate, so that tokenizer errors (like expecting '}') that happen afterwards
// do not swallow it.
"FSTRING_START",
"FSTRING_MIDDLE",
"FSTRING_END",
+ "COMMENT",
+ "NL",
"<ERRORTOKEN>",
- "<COMMENT>",
- "<NL>",
"<ENCODING>",
"<N_TOKENS>",
};
tok->interactive_underflow = IUNDERFLOW_NORMAL;
tok->str = NULL;
tok->report_warnings = 1;
+ tok->tok_extra_tokens = 0;
+ tok->comment_newline = 0;
tok->tok_mode_stack[0] = (tokenizer_mode){.kind =TOK_REGULAR_MODE, .f_string_quote='\0', .f_string_quote_size = 0, .f_string_debug=0};
tok->tok_mode_stack_index = 0;
tok->tok_report_warnings = 1;
PyMem_Free(tok);
}
+void
+_PyToken_Free(struct token *token) {
+ Py_XDECREF(token->metadata);
+}
+
+void
+_PyToken_Init(struct token *token) {
+ token->metadata = NULL;
+}
+
static int
tok_readline_raw(struct tok_state *tok)
{
return type;
}
+
static int
tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct token *token)
{
tok->starting_col_offset = -1;
blankline = 0;
+
/* Get indentation level */
if (tok->atbol) {
int col = 0;
tok->starting_col_offset = tok->col_offset;
/* Return pending indents/dedents */
- if (tok->pendin != 0) {
+ if (tok->pendin != 0) {
if (tok->pendin < 0) {
+ if (tok->tok_extra_tokens) {
+ p_start = tok->cur;
+ p_end = tok->cur;
+ }
tok->pendin++;
return MAKE_TOKEN(DEDENT);
}
else {
+ if (tok->tok_extra_tokens) {
+ p_start = tok->buf;
+ p_end = tok->cur;
+ }
tok->pendin--;
return MAKE_TOKEN(INDENT);
}
return MAKE_TOKEN(syntaxerror(tok, "f-string expression part cannot include '#'"));
}
- const char *prefix, *p, *type_start;
+ const char* p = NULL;
+ const char *prefix, *type_start;
int current_starting_col_offset;
while (c != EOF && c != '\n') {
c = tok_nextc(tok);
}
+ if (tok->tok_extra_tokens) {
+ p = tok->start;
+ }
+
if (tok->type_comments) {
p = tok->start;
current_starting_col_offset = tok->starting_col_offset;
}
}
}
+ if (tok->tok_extra_tokens) {
+ tok_backup(tok, c); /* don't eat the newline or EOF */
+ p_start = p;
+ p_end = tok->cur;
+ tok->comment_newline = blankline;
+ return MAKE_TOKEN(COMMENT);
+ }
}
if (tok->done == E_INTERACT_STOP) {
struct tok_state ahead_tok;
struct token ahead_token;
+ _PyToken_Init(&ahead_token);
int ahead_tok_kind;
memcpy(&ahead_tok, tok, sizeof(ahead_tok));
returning a plain NAME token, return ASYNC. */
tok->async_def_indent = tok->indent;
tok->async_def = 1;
+ _PyToken_Free(&ahead_token);
return MAKE_TOKEN(ASYNC);
}
+ _PyToken_Free(&ahead_token);
}
}
if (c == '\n') {
tok->atbol = 1;
if (blankline || tok->level > 0) {
+ if (tok->tok_extra_tokens) {
+ p_start = tok->start;
+ p_end = tok->cur;
+ return MAKE_TOKEN(NL);
+ }
goto nextline;
}
+ if (tok->comment_newline && tok->tok_extra_tokens) {
+ tok->comment_newline = 0;
+ p_start = tok->start;
+ p_end = tok->cur;
+ return MAKE_TOKEN(NL);
+ }
p_start = tok->start;
p_end = tok->cur - 1; /* Leave '\n' out of the string */
tok->cont_line = 0;
f_string_middle:
+ // TODO: This is a bit of a hack, but it works for now. We need to find a better way to handle
+ // this.
+ tok->multi_line_start = tok->line_start;
while (end_quote_size != current_tok->f_string_quote_size) {
int c = tok_nextc(tok);
if (tok->done == E_ERROR) {
// if fetching the encoding shows a warning.
tok->report_warnings = 0;
while (tok->lineno < 2 && tok->done == E_OK) {
+ _PyToken_Init(&token);
_PyTokenizer_Get(tok, &token);
+ _PyToken_Free(&token);
}
fclose(fp);
if (tok->encoding) {
tokenizer_mode tok_mode_stack[MAXFSTRINGLEVEL];
int tok_mode_stack_index;
int tok_report_warnings;
+ int tok_extra_tokens;
+ int comment_newline;
#ifdef Py_DEBUG
int debug;
#endif
extern struct tok_state *_PyTokenizer_FromFile(FILE *, const char*,
const char *, const char *);
extern void _PyTokenizer_Free(struct tok_state *);
+extern void _PyToken_Free(struct token *);
+extern void _PyToken_Init(struct token *);
extern int _PyTokenizer_Get(struct tok_state *, struct token *);
#define tok_dump _Py_tok_dump
#include "Python.h"
+#include "errcode.h"
#include "../Parser/tokenizer.h"
+#include "../Parser/pegen.h" // _PyPegen_byte_offset_to_character_offset()
+#include "../Parser/pegen.h" // _PyPegen_byte_offset_to_character_offset()
static struct PyModuleDef _tokenizemodule;
_tokenizer.tokenizeriter.__new__ as tokenizeriter_new
source: str
+ *
+ extra_tokens: bool
[clinic start generated code]*/
static PyObject *
-tokenizeriter_new_impl(PyTypeObject *type, const char *source)
-/*[clinic end generated code: output=7fd9f46cf9263cbb input=4384b368407375c6]*/
+tokenizeriter_new_impl(PyTypeObject *type, const char *source,
+ int extra_tokens)
+/*[clinic end generated code: output=f6f9d8b4beec8106 input=90dc5b6a5df180c2]*/
{
tokenizeriterobject *self = (tokenizeriterobject *)type->tp_alloc(type, 0);
if (self == NULL) {
return NULL;
}
self->tok->filename = filename;
+ if (extra_tokens) {
+ self->tok->tok_extra_tokens = 1;
+ }
return (PyObject *)self;
}
+static int
+_tokenizer_error(struct tok_state *tok)
+{
+ if (PyErr_Occurred()) {
+ return -1;
+ }
+
+ const char *msg = NULL;
+ PyObject* errtype = PyExc_SyntaxError;
+ switch (tok->done) {
+ case E_TOKEN:
+ msg = "invalid token";
+ break;
+ case E_EOF:
+ if (tok->level) {
+ PyErr_Format(PyExc_SyntaxError,
+ "parenthesis '%c' was never closed",
+ tok->parenstack[tok->level-1]);
+ } else {
+ PyErr_SetString(PyExc_SyntaxError, "unexpected EOF while parsing");
+ }
+ return -1;
+ case E_DEDENT:
+ PyErr_Format(PyExc_IndentationError,
+ "unindent does not match any outer indentation level "
+ "(<tokenize>, line %d)",
+ tok->lineno);
+ return -1;
+ case E_INTR:
+ if (!PyErr_Occurred()) {
+ PyErr_SetNone(PyExc_KeyboardInterrupt);
+ }
+ return -1;
+ case E_NOMEM:
+ PyErr_NoMemory();
+ return -1;
+ case E_TABSPACE:
+ errtype = PyExc_TabError;
+ msg = "inconsistent use of tabs and spaces in indentation";
+ break;
+ case E_TOODEEP:
+ errtype = PyExc_IndentationError;
+ msg = "too many levels of indentation";
+ break;
+ case E_LINECONT: {
+ msg = "unexpected character after line continuation character";
+ break;
+ }
+ default:
+ msg = "unknown tokenization error";
+ }
+
+ PyObject* errstr = NULL;
+ PyObject* error_line = NULL;
+ PyObject* tmp = NULL;
+ PyObject* value = NULL;
+ int result = 0;
+
+ Py_ssize_t size = tok->inp - tok->buf;
+ error_line = PyUnicode_DecodeUTF8(tok->buf, size, "replace");
+ if (!error_line) {
+ result = -1;
+ goto exit;
+ }
+
+ tmp = Py_BuildValue("(OnnOii)", tok->filename, tok->lineno, 0, error_line, 0, 0);
+ if (!tmp) {
+ result = -1;
+ goto exit;
+ }
+
+ errstr = PyUnicode_FromString(msg);
+ if (!errstr) {
+ result = -1;
+ goto exit;
+ }
+
+ value = PyTuple_Pack(2, errstr, tmp);
+ if (!value) {
+ result = -1;
+ goto exit;
+ }
+
+ PyErr_SetObject(errtype, value);
+
+exit:
+ Py_XDECREF(errstr);
+ Py_XDECREF(error_line);
+ Py_XDECREF(tmp);
+ Py_XDECREF(value);
+ return result;
+}
+
static PyObject *
tokenizeriter_next(tokenizeriterobject *it)
{
+ PyObject* result = NULL;
struct token token;
+ _PyToken_Init(&token);
+
int type = _PyTokenizer_Get(it->tok, &token);
- if (type == ERRORTOKEN && PyErr_Occurred()) {
- return NULL;
+ if (type == ERRORTOKEN) {
+ if(!PyErr_Occurred()) {
+ _tokenizer_error(it->tok);
+ assert(PyErr_Occurred());
+ }
+ goto exit;
}
if (type == ERRORTOKEN || type == ENDMARKER) {
PyErr_SetString(PyExc_StopIteration, "EOF");
- return NULL;
+ goto exit;
}
PyObject *str = NULL;
if (token.start == NULL || token.end == NULL) {
str = PyUnicode_FromStringAndSize(token.start, token.end - token.start);
}
if (str == NULL) {
- return NULL;
+ goto exit;
}
Py_ssize_t size = it->tok->inp - it->tok->buf;
PyObject *line = PyUnicode_DecodeUTF8(it->tok->buf, size, "replace");
if (line == NULL) {
Py_DECREF(str);
- return NULL;
+ goto exit;
}
const char *line_start = ISSTRINGLIT(type) ? it->tok->multi_line_start : it->tok->line_start;
- int lineno = ISSTRINGLIT(type) ? it->tok->first_lineno : it->tok->lineno;
- int end_lineno = it->tok->lineno;
- int col_offset = -1;
- int end_col_offset = -1;
+ Py_ssize_t lineno = ISSTRINGLIT(type) ? it->tok->first_lineno : it->tok->lineno;
+ Py_ssize_t end_lineno = it->tok->lineno;
+ Py_ssize_t col_offset = -1;
+ Py_ssize_t end_col_offset = -1;
if (token.start != NULL && token.start >= line_start) {
- col_offset = (int)(token.start - line_start);
+ col_offset = _PyPegen_byte_offset_to_character_offset(line, token.start - line_start);
}
if (token.end != NULL && token.end >= it->tok->line_start) {
- end_col_offset = (int)(token.end - it->tok->line_start);
+ end_col_offset = _PyPegen_byte_offset_to_character_offset(line, token.end - it->tok->line_start);
}
- return Py_BuildValue("(NiiiiiN)", str, type, lineno, end_lineno, col_offset, end_col_offset, line);
+ result = Py_BuildValue("(NinnnnN)", str, type, lineno, end_lineno, col_offset, end_col_offset, line);
+exit:
+ _PyToken_Free(&token);
+ return result;
}
static void
static PyObject *
-tokenizeriter_new_impl(PyTypeObject *type, const char *source);
+tokenizeriter_new_impl(PyTypeObject *type, const char *source,
+ int extra_tokens);
static PyObject *
tokenizeriter_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
PyObject *return_value = NULL;
#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
- #define NUM_KEYWORDS 1
+ #define NUM_KEYWORDS 2
static struct {
PyGC_Head _this_is_not_used;
PyObject_VAR_HEAD
PyObject *ob_item[NUM_KEYWORDS];
} _kwtuple = {
.ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS)
- .ob_item = { &_Py_ID(source), },
+ .ob_item = { &_Py_ID(source), &_Py_ID(extra_tokens), },
};
#undef NUM_KEYWORDS
#define KWTUPLE (&_kwtuple.ob_base.ob_base)
# define KWTUPLE NULL
#endif // !Py_BUILD_CORE
- static const char * const _keywords[] = {"source", NULL};
+ static const char * const _keywords[] = {"source", "extra_tokens", NULL};
static _PyArg_Parser _parser = {
.keywords = _keywords,
.fname = "tokenizeriter",
.kwtuple = KWTUPLE,
};
#undef KWTUPLE
- PyObject *argsbuf[1];
+ PyObject *argsbuf[2];
PyObject * const *fastargs;
Py_ssize_t nargs = PyTuple_GET_SIZE(args);
const char *source;
+ int extra_tokens;
- fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf);
+ fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 1, argsbuf);
if (!fastargs) {
goto exit;
}
PyErr_SetString(PyExc_ValueError, "embedded null character");
goto exit;
}
- return_value = tokenizeriter_new_impl(type, source);
+ extra_tokens = PyObject_IsTrue(fastargs[1]);
+ if (extra_tokens < 0) {
+ goto exit;
+ }
+ return_value = tokenizeriter_new_impl(type, source, extra_tokens);
exit:
return return_value;
}
-/*[clinic end generated code: output=8c2c09f651961986 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=940b564c67f6e0e2 input=a9049054013a1b77]*/
projects / thirdparty / Python / cpython.git / commitdiff
