From 3160185361a25b493e9757254c8ab55deeb3ba0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=BB=D1=8C=D1=8F=20=D0=9B=D1=8E=D0=B1=D0=B0=D0=B2?= =?UTF-8?q?=D1=81=D0=BA=D0=B8=D0=B9?= <100635212+lubaskinc0de@users.noreply.github.com> Date: Fri, 29 Nov 2024 12:00:50 +0300 Subject: [PATCH] gh-127303: Add docs for token.EXACT_TOKEN_TYPES (GH-127304) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --------- (cherry picked from commit dd3a87d2a8f8750978359a99de2c5cb2168351d1) Co-authored-by: Илья Любавский <100635212+lubaskinc0de@users.noreply.github.com> Co-authored-by: Terry Jan Reedy Co-authored-by: Tomas R. Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/library/token.rst | 7 +++++++ Lib/token.py | 3 ++- Misc/ACKS | 1 + .../Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst | 1 + Tools/build/generate_token.py | 3 ++- 5 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst diff --git a/Doc/library/token.rst b/Doc/library/token.rst index e27a3b96d8fade..6d06ae8205dc01 100644 --- a/Doc/library/token.rst +++ b/Doc/library/token.rst @@ -79,6 +79,13 @@ the :mod:`tokenize` module. ``type_comments=True``. +.. data:: EXACT_TOKEN_TYPES + + A dictionary mapping the string representation of a token to its numeric code. + + .. versionadded:: 3.8 + + .. versionchanged:: 3.5 Added :data:`AWAIT` and :data:`ASYNC` tokens. diff --git a/Lib/token.py b/Lib/token.py index 487f6edd3c951c..e26d36bd64e3c1 100644 --- a/Lib/token.py +++ b/Lib/token.py @@ -1,7 +1,8 @@ """Token constants.""" # Auto-generated by Tools/build/generate_token.py -__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF'] +__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF', + 'EXACT_TOKEN_TYPES'] ENDMARKER = 0 NAME = 1 diff --git a/Misc/ACKS b/Misc/ACKS index cb9de23eeeac11..0f8b79f26fbf6b 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1139,6 +1139,7 @@ Mark Lutz Taras Lyapun Jim Lynch Mikael Lyngvig +Ilya Lyubavski Jeff MacDonald John Machin Andrew I MacIntyre diff --git a/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst b/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst new file mode 100644 index 00000000000000..58ebf5d0abe141 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst @@ -0,0 +1 @@ +Publicly expose :data:`~token.EXACT_TOKEN_TYPES` in :attr:`!token.__all__`. diff --git a/Tools/build/generate_token.py b/Tools/build/generate_token.py index 3bd307c1733867..6d27bc41ba2663 100755 --- a/Tools/build/generate_token.py +++ b/Tools/build/generate_token.py @@ -226,7 +226,8 @@ def make_rst(infile, outfile='Doc/library/token-list.inc'): # {AUTO_GENERATED_BY_SCRIPT} ''' token_py_template += ''' -__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF'] +__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF', + 'EXACT_TOKEN_TYPES'] %s N_TOKENS = %d