[3.14] gh-144490: Test the internal C API in test_cext (#144678)

Backport changes from the main branch.

Test also datetime.h in test_cppext.

Author: vstinner

Lib/test/test_cext/__init__.py

@@ -12,7 +12,9 @@
 from test import support
 
 
-SOURCE = os.path.join(os.path.dirname(__file__), 'extension.c')
+SOURCES = [
+    os.path.join(os.path.dirname(__file__), 'extension.c'),
+]
 SETUP = os.path.join(os.path.dirname(__file__), 'setup.py')
 
 
@@ -28,29 +30,13 @@
 @support.requires_venv_with_pip()
 @support.requires_subprocess()
 @support.requires_resource('cpu')
-class TestExt(unittest.TestCase):
+class BaseTests:
+    TEST_INTERNAL_C_API = False
+
     # Default build with no options
     def test_build(self):
         self.check_build('_test_cext')
 
-    def test_build_c11(self):
-        self.check_build('_test_c11_cext', std='c11')
-
-    @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c99")
-    def test_build_c99(self):
-        # In public docs, we say C API is compatible with C11. However,
-        # in practice we do maintain C99 compatibility in public headers.
-        # Please ask the C API WG before adding a new C11-only feature.
-        self.check_build('_test_c99_cext', std='c99')
-
-    @support.requires_gil_enabled('incompatible with Free Threading')
-    def test_build_limited(self):
-        self.check_build('_test_limited_cext', limited=True)
-
-    @support.requires_gil_enabled('broken for now with Free Threading')
-    def test_build_limited_c11(self):
-        self.check_build('_test_limited_c11_cext', limited=True, std='c11')
-
     def check_build(self, extension_name, std=None, limited=False):
         venv_dir = 'env'
         with support.setup_venv_with_pip_setuptools(venv_dir) as python_exe:
@@ -61,7 +47,9 @@ def _check_build(self, extension_name, python_exe, std, limited):
         pkg_dir = 'pkg'
         os.mkdir(pkg_dir)
         shutil.copy(SETUP, os.path.join(pkg_dir, os.path.basename(SETUP)))
-        shutil.copy(SOURCE, os.path.join(pkg_dir, os.path.basename(SOURCE)))
+        for source in SOURCES:
+            dest = os.path.join(pkg_dir, os.path.basename(source))
+            shutil.copy(source, dest)
 
         def run_cmd(operation, cmd):
             env = os.environ.copy()
@@ -70,6 +58,7 @@ def run_cmd(operation, cmd):
             if limited:
                 env['CPYTHON_TEST_LIMITED'] = '1'
             env['CPYTHON_TEST_EXT_NAME'] = extension_name
+            env['TEST_INTERNAL_C_API'] = str(int(self.TEST_INTERNAL_C_API))
             if support.verbose:
                 print('Run:', ' '.join(map(shlex.quote, cmd)))
                 subprocess.run(cmd, check=True, env=env)
@@ -110,5 +99,29 @@ def run_cmd(operation, cmd):
         run_cmd('Import', cmd)
 
 
+class TestPublicCAPI(BaseTests, unittest.TestCase):
+    @support.requires_gil_enabled('incompatible with Free Threading')
+    def test_build_limited(self):
+        self.check_build('_test_limited_cext', limited=True)
+
+    @support.requires_gil_enabled('broken for now with Free Threading')
+    def test_build_limited_c11(self):
+        self.check_build('_test_limited_c11_cext', limited=True, std='c11')
+
+    def test_build_c11(self):
+        self.check_build('_test_c11_cext', std='c11')
+
+    @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c99")
+    def test_build_c99(self):
+        # In public docs, we say C API is compatible with C11. However,
+        # in practice we do maintain C99 compatibility in public headers.
+        # Please ask the C API WG before adding a new C11-only feature.
+        self.check_build('_test_c99_cext', std='c99')
+
+
+class TestInteralCAPI(BaseTests, unittest.TestCase):
+    TEST_INTERNAL_C_API = True
+
+
 if __name__ == "__main__":
     unittest.main()

Lib/test/test_cext/extension.c

@@ -1,10 +1,31 @@
 // gh-116869: Basic C test extension to check that the Python C API
 // does not emit C compiler warnings.
+//
+// Test also the internal C API if the TEST_INTERNAL_C_API macro is defined.
 
 // Always enable assertions
 #undef NDEBUG
 
+#ifdef TEST_INTERNAL_C_API
+#  define Py_BUILD_CORE_MODULE 1
+#endif
+
 #include "Python.h"
+#include "datetime.h"
+
+#ifdef TEST_INTERNAL_C_API
+   // gh-135906: Check for compiler warnings in the internal C API.
+   // - Cython uses pycore_frame.h.
+   // - greenlet uses pycore_frame.h, pycore_interpframe_structs.h and
+   //   pycore_interpframe.h.
+#  include "internal/pycore_frame.h"
+#  include "internal/pycore_gc.h"
+#  include "internal/pycore_interp.h"
+#  include "internal/pycore_interpframe.h"
+#  include "internal/pycore_interpframe_structs.h"
+#  include "internal/pycore_object.h"
+#  include "internal/pycore_pystate.h"
+#endif
 
 #ifndef MODULE_NAME
 #  error "MODULE_NAME macro must be defined"
@@ -30,27 +51,43 @@ _testcext_add(PyObject *Py_UNUSED(module), PyObject *args)
 }
 
 
+static PyObject *
+test_datetime(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args))
+{
+    // datetime.h is excluded from the limited C API
+#ifndef Py_LIMITED_API
+    PyDateTime_IMPORT;
+    if (PyErr_Occurred()) {
+        return NULL;
+    }
+#endif
+
+    Py_RETURN_NONE;
+}
+
+
 static PyMethodDef _testcext_methods[] = {
     {"add", _testcext_add, METH_VARARGS, _testcext_add_doc},
+    {"test_datetime", test_datetime, METH_NOARGS, NULL},
     {NULL, NULL, 0, NULL}  // sentinel
 };
 
 
 static int
-_testcext_exec(
-#ifdef __STDC_VERSION__
-    PyObject *module
-#else
-    PyObject *Py_UNUSED(module)
-#endif
-    )
+_testcext_exec(PyObject *module)
 {
+    PyObject *result;
+
 #ifdef __STDC_VERSION__
     if (PyModule_AddIntMacro(module, __STDC_VERSION__) < 0) {
         return -1;
     }
 #endif
 
+    result = PyObject_CallMethod(module, "test_datetime", "");
+    if (!result) return -1;
+    Py_DECREF(result);
+
     // test Py_BUILD_ASSERT() and Py_BUILD_ASSERT_EXPR()
     Py_BUILD_ASSERT(sizeof(int) == sizeof(unsigned int));
     assert(Py_BUILD_ASSERT_EXPR(sizeof(int) == sizeof(unsigned int)) == 0);

Lib/test/test_cext/setup.py

@@ -14,10 +14,15 @@
 
 if not support.MS_WINDOWS:
     # C compiler flags for GCC and clang
-    CFLAGS = [
+    BASE_CFLAGS = [
         # The purpose of test_cext extension is to check that building a C
         # extension using the Python C API does not emit C compiler warnings.
         '-Werror',
+    ]
+
+    # C compiler flags for GCC and clang
+    PUBLIC_CFLAGS = [
+        *BASE_CFLAGS,
 
         # gh-120593: Check the 'const' qualifier
         '-Wcast-qual',
@@ -26,27 +31,42 @@
         '-pedantic-errors',
     ]
     if not support.Py_GIL_DISABLED:
-        CFLAGS.append(
+        PUBLIC_CFLAGS.append(
             # gh-116869: The Python C API must be compatible with building
             # with the -Werror=declaration-after-statement compiler flag.
             '-Werror=declaration-after-statement',
         )
+    INTERNAL_CFLAGS = [*BASE_CFLAGS]
 else:
     # MSVC compiler flags
-    CFLAGS = [
-        # Display warnings level 1 to 4
-        '/W4',
+    BASE_CFLAGS = [
         # Treat all compiler warnings as compiler errors
         '/WX',
     ]
+    PUBLIC_CFLAGS = [
+        *BASE_CFLAGS,
+        # Display warnings level 1 to 4
+        '/W4',
+    ]
+    INTERNAL_CFLAGS = [
+        *BASE_CFLAGS,
+        # Display warnings level 1 to 3
+        '/W3',
+    ]
 
 
 def main():
     std = os.environ.get("CPYTHON_TEST_STD", "")
     module_name = os.environ["CPYTHON_TEST_EXT_NAME"]
     limited = bool(os.environ.get("CPYTHON_TEST_LIMITED", ""))
+    internal = bool(int(os.environ.get("TEST_INTERNAL_C_API", "0")))
 
-    cflags = list(CFLAGS)
+    sources = [SOURCE]
+
+    if not internal:
+        cflags = list(PUBLIC_CFLAGS)
+    else:
+        cflags = list(INTERNAL_CFLAGS)
     cflags.append(f'-DMODULE_NAME={module_name}')
 
     # Add -std=STD or /std:STD (MSVC) compiler flag
@@ -75,6 +95,9 @@ def main():
         version = sys.hexversion
         cflags.append(f'-DPy_LIMITED_API={version:#x}')
 
+    if internal:
+        cflags.append('-DTEST_INTERNAL_C_API=1')
+
     # On Windows, add PCbuild\amd64\ to include and library directories
     include_dirs = []
     library_dirs = []
@@ -90,7 +113,7 @@ def main():
             print(f"Add PCbuild directory: {pcbuild}")
 
     # Display information to help debugging
-    for env_name in ('CC', 'CFLAGS'):
+    for env_name in ('CC', 'CFLAGS', 'CPPFLAGS'):
         if env_name in os.environ:
             print(f"{env_name} env var: {os.environ[env_name]!r}")
         else:
@@ -99,7 +122,7 @@ def main():
 
     ext = Extension(
         module_name,
-        sources=[SOURCE],
+        sources=sources,
         extra_compile_args=cflags,
         include_dirs=include_dirs,
         library_dirs=library_dirs)

Lib/test/test_cppext/extension.cpp

@@ -11,14 +11,16 @@
 #endif
 
 #include "Python.h"
+#include "datetime.h"
 
 #ifdef TEST_INTERNAL_C_API
    // gh-135906: Check for compiler warnings in the internal C API
 #  include "internal/pycore_frame.h"
    // mimalloc emits many compiler warnings when Python is built in debug
    // mode (when MI_DEBUG is not zero).
-   // mimalloc emits compiler warnings when Python is built on Windows.
-#  if !defined(Py_DEBUG) && !defined(MS_WINDOWS)
+   // mimalloc emits compiler warnings when Python is built on Windows
+   // and macOS.
+#  if !defined(Py_DEBUG) && !defined(MS_WINDOWS) && !defined(__APPLE__)
 #    include "internal/pycore_backoff.h"
 #    include "internal/pycore_cell.h"
 #  endif
@@ -230,11 +232,26 @@ test_virtual_object(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args))
     Py_RETURN_NONE;
 }
 
+static PyObject *
+test_datetime(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args))
+{
+    // datetime.h is excluded from the limited C API
+#ifndef Py_LIMITED_API
+    PyDateTime_IMPORT;
+    if (PyErr_Occurred()) {
+        return NULL;
+    }
+#endif
+
+    Py_RETURN_NONE;
+}
+
 static PyMethodDef _testcppext_methods[] = {
     {"add", _testcppext_add, METH_VARARGS, _testcppext_add_doc},
     {"test_api_casts", test_api_casts, METH_NOARGS, _Py_NULL},
     {"test_unicode", test_unicode, METH_NOARGS, _Py_NULL},
     {"test_virtual_object", test_virtual_object, METH_NOARGS, _Py_NULL},
+    {"test_datetime", test_datetime, METH_NOARGS, _Py_NULL},
     // Note: _testcppext_exec currently runs all test functions directly.
     // When adding a new one, add a call there.
 
@@ -263,6 +280,10 @@ _testcppext_exec(PyObject *module)
     if (!result) return -1;
     Py_DECREF(result);
 
+    result = PyObject_CallMethod(module, "test_datetime", "");
+    if (!result) return -1;
+    Py_DECREF(result);
+
     // test Py_BUILD_ASSERT() and Py_BUILD_ASSERT_EXPR()
     Py_BUILD_ASSERT(sizeof(int) == sizeof(unsigned int));
     assert(Py_BUILD_ASSERT_EXPR(sizeof(int) == sizeof(unsigned int)) == 0);

Lib/test/test_cppext/setup.py

@@ -101,7 +101,7 @@ def main():
             print(f"Add PCbuild directory: {pcbuild}")
 
     # Display information to help debugging
-    for env_name in ('CC', 'CFLAGS', 'CPPFLAGS'):
+    for env_name in ('CC', 'CXX', 'CFLAGS', 'CPPFLAGS', 'CXXFLAGS'):
         if env_name in os.environ:
             print(f"{env_name} env var: {os.environ[env_name]!r}")
         else: