Commit

Author: StanFromIreland

Doc/deprecations/pending-removal-in-3.17.rst

@@ -8,3 +8,8 @@ Pending removal in Python 3.17
     but it has been retained for backward compatibility, with removal scheduled for Python
     3.17. Users should use documented introspection helpers like :func:`typing.get_origin`
     and :func:`typing.get_args` instead of relying on private implementation details.
+
+* :mod:`token`:
+
+  - The :func:`token.ISTERMINAL`, :func:`token.ISNONTERMINAL` and
+    :func:`token.ISEOF` tokens.

Doc/library/token.rst

@@ -35,16 +35,22 @@ a ``"match"`` token may be either :data:`NAME` or :data:`SOFT_KEYWORD`.
 
    Return ``True`` for terminal token values.
 
+   .. deprecated-removed:: next 3.17
+
 
 .. function:: ISNONTERMINAL(x)
 
    Return ``True`` for non-terminal token values.
 
+   .. deprecated-removed:: next 3.17
+
 
 .. function:: ISEOF(x)
 
    Return ``True`` if *x* is the marker indicating the end of input.
 
+   .. deprecated-removed:: next 3.17
+
 
 The token constants are:
 

Doc/whatsnew/3.15.rst

@@ -440,6 +440,15 @@ hashlib
   (Contributed by Bénédikt Tran in :gh:`134978`.)
 
 
+token
+-----
+
+* The :func:`token.ISTERMINAL`, :func:`token.ISNONTERMINAL` and
+  :func:`token.ISEOF` tokens are now deprecated, and will be removed
+  in Python 3.17.
+  (Contributed by Stan Ulbrych in :gh:`86353`)
+
+
 .. Add deprecations above alphabetically, not here at the end.
 
 Removed

Include/internal/pycore_token.h

@@ -82,13 +82,13 @@ extern "C" {
 #define NL              66
 #define ERRORTOKEN      67
 #define N_TOKENS        69
-#define NT_OFFSET       256
+#define NT_OFFSET       256 /* Deprecated and will be removed in 3.17 */
 
 /* Special definitions for cooperation with parser */
 
-#define ISTERMINAL(x)           ((x) < NT_OFFSET)
-#define ISNONTERMINAL(x)        ((x) >= NT_OFFSET)
-#define ISEOF(x)                ((x) == ENDMARKER)
+#define ISTERMINAL(x)           ((x) < NT_OFFSET) /* Deprecated and will be removed in 3.17 */
+#define ISNONTERMINAL(x)        ((x) >= NT_OFFSET) /* Deprecated and will be removed in 3.17 */
+#define ISEOF(x)                ((x) == ENDMARKER) /* Deprecated and will be removed in 3.17 */
 #define ISWHITESPACE(x)         ((x) == ENDMARKER || \
                                  (x) == NEWLINE   || \
                                  (x) == INDENT    || \

Lib/token.py

@@ -0,0 +1,2 @@
+Deprecate and schedule for removal in 3.17, :func:`token.ISTERMINAL`,
+:func:`token.ISNONTERMINAL` and :func:`token.ISEOF`.

Misc/NEWS.d/next/Library/2025-08-24-17-54-22.gh-issue-86353.bw0iat.rst

@@ -79,9 +79,9 @@ def update_file(file, content):
 
 /* Special definitions for cooperation with parser */
 
-#define ISTERMINAL(x)           ((x) < NT_OFFSET)
-#define ISNONTERMINAL(x)        ((x) >= NT_OFFSET)
-#define ISEOF(x)                ((x) == ENDMARKER)
+#define ISTERMINAL(x)           ((x) < NT_OFFSET) /* Deprecated and will be removed in 3.17 */
+#define ISNONTERMINAL(x)        ((x) >= NT_OFFSET) /* Deprecated and will be removed in 3.17 */
+#define ISEOF(x)                ((x) == ENDMARKER) /* Deprecated and will be removed in 3.17 */
 #define ISWHITESPACE(x)         ((x) == ENDMARKER || \\
                                  (x) == NEWLINE   || \\
                                  (x) == INDENT    || \\
@@ -279,12 +279,21 @@ def make_rst(infile, outfile='Doc/library/token-list.inc',
 }
 
 def ISTERMINAL(x: int) -> bool:
+    import warnings
+    warnings.warn('token.ISTERMINAL is deprecated and will be removed in 3.17',
+                   DeprecationWarning, stacklevel=2)
     return x < NT_OFFSET
 
 def ISNONTERMINAL(x: int) -> bool:
+    import warnings
+    warnings.warn('token.ISNONTERMINAL is deprecated and will be removed in 3.17',
+                   DeprecationWarning, stacklevel=2)
     return x >= NT_OFFSET
 
 def ISEOF(x: int) -> bool:
+    import warnings
+    warnings.warn('token.ISEOF is deprecated and will be removed in 3.17',
+                   DeprecationWarning, stacklevel=2)
     return x == ENDMARKER
 '''