From: Miss Islington (bot) <31488909+miss-islington@users.noreply.github.com> Date: Fri, 29 Nov 2024 09:22:27 +0000 (+0100) Subject: [3.12] gh-127303: Add docs for token.EXACT_TOKEN_TYPES (GH-127304) (#127391) X-Git-Tag: v3.12.8~22 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=91399c39845b7ca4a48b5feaec030a349ff3f68e;p=thirdparty%2FPython%2Fcpython.git [3.12] gh-127303: Add docs for token.EXACT_TOKEN_TYPES (GH-127304) (#127391) gh-127303: Add docs for token.EXACT_TOKEN_TYPES (GH-127304) --------- (cherry picked from commit dd3a87d2a8f8750978359a99de2c5cb2168351d1) Co-authored-by: Илья Любавский <100635212+lubaskinc0de@users.noreply.github.com> Co-authored-by: Terry Jan Reedy Co-authored-by: Tomas R. Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- diff --git a/Doc/library/token.rst b/Doc/library/token.rst index e27a3b96d8fa..6d06ae8205dc 100644 --- a/Doc/library/token.rst +++ b/Doc/library/token.rst @@ -79,6 +79,13 @@ the :mod:`tokenize` module. ``type_comments=True``. +.. data:: EXACT_TOKEN_TYPES + + A dictionary mapping the string representation of a token to its numeric code. + + .. versionadded:: 3.8 + + .. versionchanged:: 3.5 Added :data:`AWAIT` and :data:`ASYNC` tokens. diff --git a/Lib/token.py b/Lib/token.py index 487f6edd3c95..e26d36bd64e3 100644 --- a/Lib/token.py +++ b/Lib/token.py @@ -1,7 +1,8 @@ """Token constants.""" # Auto-generated by Tools/build/generate_token.py -__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF'] +__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF', + 'EXACT_TOKEN_TYPES'] ENDMARKER = 0 NAME = 1 diff --git a/Misc/ACKS b/Misc/ACKS index cb9de23eeeac..0f8b79f26fbf 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1139,6 +1139,7 @@ Mark Lutz Taras Lyapun Jim Lynch Mikael Lyngvig +Ilya Lyubavski Jeff MacDonald John Machin Andrew I MacIntyre diff --git a/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst b/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst new file mode 100644 index 000000000000..58ebf5d0abe1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst @@ -0,0 +1 @@ +Publicly expose :data:`~token.EXACT_TOKEN_TYPES` in :attr:`!token.__all__`. diff --git a/Tools/build/generate_token.py b/Tools/build/generate_token.py index 3bd307c17338..6d27bc41ba26 100755 --- a/Tools/build/generate_token.py +++ b/Tools/build/generate_token.py @@ -226,7 +226,8 @@ token_py_template = f'''\ # {AUTO_GENERATED_BY_SCRIPT} ''' token_py_template += ''' -__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF'] +__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF', + 'EXACT_TOKEN_TYPES'] %s N_TOKENS = %d