Deprecate support for instantiating abstract AST nodes

Author: brianschubert

Doc/library/ast.rst

@@ -43,7 +43,7 @@ Node classes
 
 .. class:: AST
 
-   This is the base of all AST node classes.  The actual node classes are
+   This is the abstract base of all AST node classes.  The actual node classes are
    derived from the :file:`Parser/Python.asdl` file, which is reproduced
    :ref:`above <abstract-grammar>`.  They are defined in the :mod:`!_ast` C
    module and re-exported in :mod:`ast`.
@@ -161,6 +161,15 @@ Node classes
    match any of the fields of the AST node. This behavior is deprecated and will
    be removed in Python 3.15.
 
+.. deprecated-removed:: next 3.20
+
+    In the :ref:`grammar above <abstract-grammar>`, the AST node classes that
+    correspond to production rules with variants (aka "sums") are abstract
+    classes. Previous versions of Python allowed for the creation of direct
+    instances of these abstract node classes. This behavior is deprecated and
+    will be removed in Python 3.20.
+
+
 .. note::
     The descriptions of the specific node classes displayed here
     were initially adapted from the fantastic `Green Tree

Doc/whatsnew/3.15.rst

@@ -408,6 +408,14 @@ module_name
 Deprecated
 ==========
 
+ast
+---
+
+* Creating instances of abstract AST nodes (such as :class:`ast.AST`
+  or :class:`!ast.expr`) is deprecated and will raise an error in Python 3.20.
+  (Contributed by Brian Schubert in :gh:`116021`.)
+
+
 hashlib
 -------
 

Include/internal/pycore_ast_state.h

@@ -84,7 +84,9 @@ def _assertTrueorder(self, ast_node, parent_pos):
         self.assertEqual(ast_node._fields, ast_node.__match_args__)
 
     def test_AST_objects(self):
-        x = ast.AST()
+        # Directly instantiating abstract node class AST is allowed (but deprecated)
+        with self.assertWarns(DeprecationWarning):
+            x = ast.AST()
         self.assertEqual(x._fields, ())
         x.foobar = 42
         self.assertEqual(x.foobar, 42)
@@ -93,7 +95,7 @@ def test_AST_objects(self):
         with self.assertRaises(AttributeError):
             x.vararg
 
-        with self.assertRaises(TypeError):
+        with self.assertRaises(TypeError), self.assertWarns(DeprecationWarning):
             # "ast.AST constructor takes 0 positional arguments"
             ast.AST(2)
 
@@ -109,15 +111,15 @@ def cleanup():
 
         msg = "type object 'ast.AST' has no attribute '_fields'"
         # Both examples used to crash:
-        with self.assertRaisesRegex(AttributeError, msg):
+        with self.assertRaisesRegex(AttributeError, msg), self.assertWarns(DeprecationWarning):
             ast.AST(arg1=123)
-        with self.assertRaisesRegex(AttributeError, msg):
+        with self.assertRaisesRegex(AttributeError, msg), self.assertWarns(DeprecationWarning):
             ast.AST()
 
-    def test_AST_garbage_collection(self):
+    def test_node_garbage_collection(self):
         class X:
             pass
-        a = ast.AST()
+        a = ast.Module()
         a.x = X()
         a.x.a = a
         ref = weakref.ref(a.x)
@@ -427,7 +429,12 @@ def _construct_ast_class(self, cls):
             elif typ is object:
                 kwargs[name] = b'capybara'
             elif isinstance(typ, type) and issubclass(typ, ast.AST):
-                kwargs[name] = self._construct_ast_class(typ)
+                if typ._is_abstract():
+                    # Use an arbitrary concrete subclass
+                    concrete = next(sub for sub in typ.__subclasses__() if not sub._is_abstract())
+                    kwargs[name] = self._construct_ast_class(concrete)
+                else:
+                    kwargs[name] = self._construct_ast_class(typ)
         return cls(**kwargs)
 
     def test_arguments(self):
@@ -573,14 +580,20 @@ def test_nodeclasses(self):
             x = ast.BinOp(1, 2, 3, foobarbaz=42)
         self.assertEqual(x.foobarbaz, 42)
 
+        # Directly instantiating abstract node types is allowed (but deprecated)
+        with self.assertWarns(DeprecationWarning):
+            ast.stmt()
+
     def test_no_fields(self):
         # this used to fail because Sub._fields was None
         x = ast.Sub()
         self.assertEqual(x._fields, ())
 
     def test_invalid_sum(self):
         pos = dict(lineno=2, col_offset=3)
-        m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], [])
+        with self.assertWarns(DeprecationWarning):
+            # Creating instances of ast.expr is deprecated
+            m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], [])
         with self.assertRaises(TypeError) as cm:
             compile(m, "<test>", "exec")
         self.assertIn("but got expr()", str(cm.exception))
@@ -1140,6 +1153,9 @@ def do(cls):
                 return
             if cls is ast.Index:
                 return
+            # Don't attempt to create instances of abstract AST nodes
+            if cls._is_abstract():
+                return
 
             yield cls
             for sub in cls.__subclasses__():

Lib/test/test_ast/test_ast.py

@@ -0,0 +1,2 @@
+Support for creating instances of abstract AST nodes from the :mod:`ast` module
+is deprecated and scheduled for removal in Python 3.20. Patch by Brian Schubert.

Misc/NEWS.d/next/Core_and_Builtins/2025-08-16-12-56-08.gh-issue-116021.hMN9yw.rst

@@ -881,6 +881,21 @@ def visitModule(self, mod):
         return -1;
     }
 
+    int contains = PySet_Contains(state->abstract_types, (PyObject *)Py_TYPE(self));
+    if (contains == -1) {
+        return -1;
+    }
+    else if (contains == 1) {
+        if (PyErr_WarnFormat(
+            PyExc_DeprecationWarning, 1,
+            "Instantiating abstract AST node class %T is deprecated. "
+            "This will become an error in Python 3.20",
+            self
+        ) < 0) {
+            return -1;
+        }
+    }
+
     Py_ssize_t i, numfields = 0;
     int res = -1;
     PyObject *key, *value, *fields, *attributes = NULL, *remaining_fields = NULL;
@@ -1443,6 +1458,23 @@ def visitModule(self, mod):
     return result;
 }
 
+/* Helper for checking if a node class is abstract in the tests. */
+PyObject *
+ast_is_abstract(PyObject *cls, void *Py_UNUSED(ignored)) {
+    struct ast_state *state = get_ast_state();
+    if (state == NULL) {
+        return NULL;
+    }
+    int contains = PySet_Contains(state->abstract_types, cls);
+    if (contains == -1) {
+        return NULL;
+    }
+    else if (contains == 1) {
+        Py_RETURN_TRUE;
+    }
+    Py_RETURN_FALSE;
+}
+
 static PyMemberDef ast_type_members[] = {
     {"__dictoffset__", Py_T_PYSSIZET, offsetof(AST_object, dict), Py_READONLY},
     {NULL}  /* Sentinel */
@@ -1454,6 +1486,7 @@ def visitModule(self, mod):
      PyDoc_STR("__replace__($self, /, **fields)\\n--\\n\\n"
                "Return a copy of the AST node with new values "
                "for the specified fields.")},
+    {"_is_abstract", _PyCFunction_CAST(ast_is_abstract), METH_CLASS | METH_NOARGS, NULL},
     {NULL}
 };
 
@@ -1887,6 +1920,13 @@ def visitModule(self, mod):
                 if (!state->AST_type) {
                     return -1;
                 }
+                state->abstract_types = PySet_New(NULL);
+                if (!state->abstract_types) {
+                    return -1;
+                }
+                if (PySet_Add(state->abstract_types, state->AST_type) < 0) {
+                    return -1;
+                }
                 if (add_ast_fields(state) < 0) {
                     return -1;
                 }
@@ -1928,6 +1968,7 @@ def visitSum(self, sum, name):
                             (name, name, len(sum.attributes)), 1)
         else:
             self.emit("if (add_attributes(state, state->%s_type, NULL, 0) < 0) return -1;" % name, 1)
+        self.emit("if (PySet_Add(state->abstract_types, state->%s_type) < 0) return -1;" % name, 1)
         self.emit_defaults(name, sum.attributes, 1)
         simple = is_simple(sum)
         for t in sum.types:
@@ -2289,6 +2330,7 @@ def generate_module_def(mod, metadata, f, internal_h):
         "%s_type" % type
         for type in metadata.types
     )
+    module_state.add("abstract_types")
 
     state_strings = sorted(state_strings)
     module_state = sorted(module_state)