[3.12] gh-127303: Add docs for token.EXACT_TOKEN_TYPES (GH-127304) (#127391)

gh-127303: Add docs for token.EXACT_TOKEN_TYPES (GH-127304)

---------

(cherry picked from commit dd3a87d2a8f8750978359a99de2c5cb2168351d1)

Co-authored-by: Илья Любавский <100635212+lubaskinc0de@users.noreply.github.com>
Co-authored-by: Terry Jan Reedy <tjreedy@udel.edu>
Co-authored-by: Tomas R. <tomas.roun8@gmail.com>
Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com>

diff --git a/Doc/library/token.rst b/Doc/library/token.rst
index e27a3b96d8fadec56732787cd017bb15a80708e7..6d06ae8205dc015b6f6baa921033ddc5b2a7286a 100644 (file)
--- a/Doc/library/token.rst
+++ b/Doc/library/token.rst
@@ -79,6 +79,13 @@ the :mod:`tokenize` module.
    ``type_comments=True``.
 
 
+.. data:: EXACT_TOKEN_TYPES
+
+   A dictionary mapping the string representation of a token to its numeric code.
+
+   .. versionadded:: 3.8
+
+
 .. versionchanged:: 3.5
    Added :data:`AWAIT` and :data:`ASYNC` tokens.
 

diff --git a/Lib/token.py b/Lib/token.py
index 487f6edd3c951cabb802998c8595f00eff2a17a1..e26d36bd64e3c17208aa3f073eb8cca5165884fd 100644 (file)
--- a/Lib/token.py
+++ b/Lib/token.py
@@ -1,7 +1,8 @@
 """Token constants."""
 # Auto-generated by Tools/build/generate_token.py
 
-__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF']
+__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF',
+           'EXACT_TOKEN_TYPES']
 
 ENDMARKER = 0
 NAME = 1

diff --git a/Misc/ACKS b/Misc/ACKS
index cb9de23eeeac116fc166a8e03384926a24b66e56..0f8b79f26fbf6b04780d6537ad1c2f470923c3d2 100644 (file)
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -1139,6 +1139,7 @@ Mark Lutz
 Taras Lyapun
 Jim Lynch
 Mikael Lyngvig
+Ilya Lyubavski
 Jeff MacDonald
 John Machin
 Andrew I MacIntyre

diff --git a/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst b/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst
new file mode 100644 (file)
index 0000000..58ebf5d
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst
@@ -0,0 +1 @@
+Publicly expose :data:`~token.EXACT_TOKEN_TYPES` in :attr:`!token.__all__`.

diff --git a/Tools/build/generate_token.py b/Tools/build/generate_token.py
index 3bd307c173386732954018be71e401cc46a31bb7..6d27bc41ba2663abbfdf04fddeafce8716eb3a68 100755 (executable)
--- a/Tools/build/generate_token.py
+++ b/Tools/build/generate_token.py
@@ -226,7 +226,8 @@ token_py_template = f'''\
 # {AUTO_GENERATED_BY_SCRIPT}
 '''
 token_py_template += '''
-__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF']
+__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF',
+           'EXACT_TOKEN_TYPES']
 
 %s
 N_TOKENS = %d