Merge remote-tracking branch 'upstream/main' into tsan

Author: crusaderky

.github/CODEOWNERS

@@ -63,8 +63,8 @@
 .azure-pipelines/             @AA-Turner
 
 # GitHub & related scripts
-.github/                               @ezio-melotti @hugovk @AA-Turner
-Tools/build/compute-changes.py         @AA-Turner
+.github/                               @ezio-melotti @hugovk @AA-Turner @webknjaz
+Tools/build/compute-changes.py         @AA-Turner @hugovk @webknjaz
 Tools/build/verify_ensurepip_wheels.py @AA-Turner @pfmoore @pradyunsg
 
 # Pre-commit

.github/workflows/build.yml

@@ -261,7 +261,7 @@ jobs:
         # Keep 1.1.1w in our list despite it being upstream EOL and otherwise
         # unsupported as it most resembles other 1.1.1-work-a-like ssl APIs
         # supported by important vendors such as AWS-LC.
-        openssl_ver: [1.1.1w, 3.0.18, 3.2.6, 3.3.5, 3.4.3, 3.5.4]
+        openssl_ver: [1.1.1w, 3.0.18, 3.3.5, 3.4.3, 3.5.4, 3.6.0]
         # See Tools/ssl/make_ssl_data.py for notes on adding a new version
     env:
       OPENSSL_VER: ${{ matrix.openssl_ver }}
@@ -641,45 +641,45 @@ jobs:
         run: |
           "$BUILD_DIR/cross-python/bin/python3" -m test test_sysconfig test_site test_embed
 
-  # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/
   cifuzz:
-    name: CIFuzz
-    runs-on: ubuntu-latest
-    timeout-minutes: 60
+    # ${{ '' } is a hack to nest jobs under the same sidebar category.
+    name: CIFuzz${{ '' }}  # zizmor: ignore[obfuscation]
     needs: build-context
-    if: needs.build-context.outputs.run-ci-fuzz == 'true'
+    if: >-
+      needs.build-context.outputs.run-ci-fuzz == 'true'
+      || needs.build-context.outputs.run-ci-fuzz-stdlib == 'true'
     permissions:
       security-events: write
     strategy:
       fail-fast: false
       matrix:
-        sanitizer: [address, undefined, memory]
-    steps:
-      - name: Build fuzzers (${{ matrix.sanitizer }})
-        id: build
-        uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
-        with:
-          oss-fuzz-project-name: cpython3
-          sanitizer: ${{ matrix.sanitizer }}
-      - name: Run fuzzers (${{ matrix.sanitizer }})
-        uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
-        with:
-          fuzz-seconds: 600
-          oss-fuzz-project-name: cpython3
-          output-sarif: true
-          sanitizer: ${{ matrix.sanitizer }}
-      - name: Upload crash
-        if: failure() && steps.build.outcome == 'success'
-        uses: actions/upload-artifact@v6
-        with:
-          name: ${{ matrix.sanitizer }}-artifacts
-          path: ./out/artifacts
-      - name: Upload SARIF
-        if: always() && steps.build.outcome == 'success'
-        uses: github/codeql-action/upload-sarif@v4
-        with:
-          sarif_file: cifuzz-sarif/results.sarif
-          checkout_path: cifuzz-sarif
+        sanitizer:
+        - address
+        - undefined
+        - memory
+        oss-fuzz-project-name:
+        - cpython3
+        - python3-libraries
+        exclude:
+        # Note that the 'no-exclude' sentinel below is to prevent
+        # an empty string value from excluding all jobs and causing
+        # GHA to create a 'default' matrix entry with all empty values.
+        - oss-fuzz-project-name: >-
+            ${{
+              needs.build-context.outputs.run-ci-fuzz == 'true'
+              && 'no-exclude'
+              || 'cpython3'
+            }}
+        - oss-fuzz-project-name: >-
+            ${{
+              needs.build-context.outputs.run-ci-fuzz-stdlib == 'true'
+              && 'no-exclude'
+              || 'python3-libraries'
+            }}
+    uses: ./.github/workflows/reusable-cifuzz.yml
+    with:
+      oss-fuzz-project-name: ${{ matrix.oss-fuzz-project-name }}
+      sanitizer: ${{ matrix.sanitizer }}
 
   all-required-green:  # This job does nothing and is only used for the branch protection
     name: All required checks pass
@@ -734,7 +734,12 @@ jobs:
             || ''
           }}
           ${{ !fromJSON(needs.build-context.outputs.run-windows-tests) && 'build-windows,' || '' }}
-          ${{ !fromJSON(needs.build-context.outputs.run-ci-fuzz) && 'cifuzz,' || '' }}
+          ${{
+            !fromJSON(needs.build-context.outputs.run-ci-fuzz)
+            && !fromJSON(needs.build-context.outputs.run-ci-fuzz-stdlib)
+            && 'cifuzz,' ||
+            ''
+          }}
           ${{ !fromJSON(needs.build-context.outputs.run-macos) && 'build-macos,' || '' }}
           ${{
             !fromJSON(needs.build-context.outputs.run-ubuntu)

.github/workflows/jit.yml

@@ -7,6 +7,7 @@ on:
       - 'Python/optimizer*.c'
       - 'Python/executor_cases.c.h'
       - 'Python/optimizer_cases.c.h'
+      - '**_testinternalcapi**'
       - '!Python/perf_jit_trampoline.c'
       - '!**/*.md'
       - '!**/*.ini'
@@ -17,6 +18,7 @@ on:
       - 'Python/optimizer*.c'
       - 'Python/executor_cases.c.h'
       - 'Python/optimizer_cases.c.h'
+      - '**_testinternalcapi**'
       - '!Python/perf_jit_trampoline.c'
       - '!**/*.md'
       - '!**/*.ini'

.github/workflows/reusable-cifuzz.yml

@@ -0,0 +1,46 @@
+# CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/
+name: Reusable CIFuzz
+
+on:
+  workflow_call:
+    inputs:
+      oss-fuzz-project-name:
+        description: OSS-Fuzz project name
+        required: true
+        type: string
+      sanitizer:
+        description: OSS-Fuzz sanitizer
+        required: true
+        type: string
+
+jobs:
+  cifuzz:
+    name: ${{ inputs.oss-fuzz-project-name }} (${{ inputs.sanitizer }})
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Build fuzzers (${{ inputs.sanitizer }})
+        id: build
+        uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
+        with:
+          oss-fuzz-project-name: ${{ inputs.oss-fuzz-project-name }}
+          sanitizer: ${{ inputs.sanitizer }}
+      - name: Run fuzzers (${{ inputs.sanitizer }})
+        uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
+        with:
+          fuzz-seconds: 600
+          oss-fuzz-project-name: ${{ inputs.oss-fuzz-project-name }}
+          output-sarif: true
+          sanitizer: ${{ inputs.sanitizer }}
+      - name: Upload crash
+        if: failure() && steps.build.outcome == 'success'
+        uses: actions/upload-artifact@v6
+        with:
+          name: ${{ inputs.sanitizer }}-artifacts
+          path: ./out/artifacts
+      - name: Upload SARIF
+        if: always() && steps.build.outcome == 'success'
+        uses: github/codeql-action/upload-sarif@v4
+        with:
+          sarif_file: cifuzz-sarif/results.sarif
+          checkout_path: cifuzz-sarif

.github/workflows/reusable-context.yml

@@ -21,8 +21,11 @@ on:  # yamllint disable-line rule:truthy
         description: Whether to run the Android tests
         value: ${{ jobs.compute-changes.outputs.run-android }}  # bool
       run-ci-fuzz:
-        description: Whether to run the CIFuzz job
+        description: Whether to run the CIFuzz job for 'cpython' fuzzer
         value: ${{ jobs.compute-changes.outputs.run-ci-fuzz }}  # bool
+      run-ci-fuzz-stdlib:
+        description: Whether to run the CIFuzz job for 'python3-libraries' fuzzer
+        value: ${{ jobs.compute-changes.outputs.run-ci-fuzz-stdlib }}  # bool
       run-docs:
         description: Whether to build the docs
         value: ${{ jobs.compute-changes.outputs.run-docs }}  # bool
@@ -56,6 +59,7 @@ jobs:
     outputs:
       run-android: ${{ steps.changes.outputs.run-android }}
       run-ci-fuzz: ${{ steps.changes.outputs.run-ci-fuzz }}
+      run-ci-fuzz-stdlib: ${{ steps.changes.outputs.run-ci-fuzz-stdlib }}
       run-docs: ${{ steps.changes.outputs.run-docs }}
       run-ios: ${{ steps.changes.outputs.run-ios }}
       run-macos: ${{ steps.changes.outputs.run-macos }}

Doc/c-api/descriptor.rst

@@ -10,11 +10,6 @@ found in the dictionary of type objects.
 
 .. XXX document these!
 
-.. c:var:: PyTypeObject PyProperty_Type
-
-   The type object for the built-in descriptor types.
-
-
 .. c:function:: PyObject* PyDescr_NewGetSet(PyTypeObject *type, struct PyGetSetDef *getset)
 
 
@@ -74,9 +69,26 @@ found in the dictionary of type objects.
 .. c:function:: PyObject* PyWrapper_New(PyObject *, PyObject *)
 
 
+.. c:macro:: PyDescr_COMMON
+
+   This is a :term:`soft deprecated` macro including the common fields for a
+   descriptor object.
+
+   This was included in Python's C API by mistake; do not use it in extensions.
+   For creating custom descriptor objects, create a class implementing the
+   descriptor protocol (:c:member:`~PyTypeObject.tp_descr_get` and
+   :c:member:`~PyTypeObject.tp_descr_set`).
+
+
 Built-in descriptors
 ^^^^^^^^^^^^^^^^^^^^
 
+.. c:var:: PyTypeObject PyProperty_Type
+
+   The type object for property objects. This is the same object as
+   :class:`property` in the Python layer.
+
+
 .. c:var:: PyTypeObject PySuper_Type
 
    The type object for super objects. This is the same object as

Doc/c-api/exceptions.rst

@@ -793,6 +793,17 @@ Exception Classes
    Return :c:member:`~PyTypeObject.tp_name` of the exception class *ob*.
 
 
+.. c:macro:: PyException_HEAD
+
+   This is a :term:`soft deprecated` macro including the base fields for an
+   exception object.
+
+   This was included in Python's C API by mistake and is not designed for use
+   in extensions. For creating custom exception objects, use
+   :c:func:`PyErr_NewException` or otherwise create a class inheriting from
+   :c:data:`PyExc_BaseException`.
+
+
 Exception Objects
 =================
 

Doc/c-api/lifecycle.rst

@@ -256,6 +256,8 @@ To allocate and free memory, see :ref:`allocating-objects`.
    collection (i.e., the :c:macro:`Py_TPFLAGS_HAVE_GC` flag is set); this may
    change in the future.
 
+   .. versionadded:: 3.4
+
 
 .. c:function:: int PyObject_CallFinalizerFromDealloc(PyObject *op)
 
@@ -266,6 +268,8 @@ To allocate and free memory, see :ref:`allocating-objects`.
    should happen.  Otherwise, this function returns 0 and destruction can
    continue normally.
 
+   .. versionadded:: 3.4
+
    .. seealso::
 
       :c:member:`~PyTypeObject.tp_dealloc` for example code.

Doc/c-api/long.rst

@@ -687,7 +687,7 @@ Export API
 
 .. versionadded:: 3.14
 
-.. c:struct:: PyLongLayout
+.. c:type:: PyLongLayout
 
    Layout of an array of "digits" ("limbs" in the GMP terminology), used to
    represent absolute value for arbitrary precision integers.
@@ -727,15 +727,15 @@ Export API
 
    Get the native layout of Python :class:`int` objects.
 
-   See the :c:struct:`PyLongLayout` structure.
+   See the :c:type:`PyLongLayout` structure.
 
    The function must not be called before Python initialization nor after
    Python finalization. The returned layout is valid until Python is
    finalized. The layout is the same for all Python sub-interpreters
    in a process, and so it can be cached.
 
 
-.. c:struct:: PyLongExport
+.. c:type:: PyLongExport
 
    Export of a Python :class:`int` object.
 
@@ -769,7 +769,7 @@ Export API
 
    Export a Python :class:`int` object.
 
-   *export_long* must point to a :c:struct:`PyLongExport` structure allocated
+   *export_long* must point to a :c:type:`PyLongExport` structure allocated
    by the caller. It must not be ``NULL``.
 
    On success, fill in *\*export_long* and return ``0``.
@@ -799,7 +799,7 @@ The :c:type:`PyLongWriter` API can be used to import an integer.
 
 .. versionadded:: 3.14
 
-.. c:struct:: PyLongWriter
+.. c:type:: PyLongWriter
 
    A Python :class:`int` writer instance.
 
@@ -827,7 +827,7 @@ The :c:type:`PyLongWriter` API can be used to import an integer.
    The layout of *digits* is described by :c:func:`PyLong_GetNativeLayout`.
 
    Digits must be in the range [``0``; ``(1 << bits_per_digit) - 1``]
-   (where the :c:struct:`~PyLongLayout.bits_per_digit` is the number of bits
+   (where the :c:type:`~PyLongLayout.bits_per_digit` is the number of bits
    per digit).
    Any unused most significant digits must be set to ``0``.
 

Doc/c-api/object.rst

@@ -85,7 +85,7 @@ Object Protocol
    instead of the :func:`repr`.
 
 
-.. c:function:: void PyUnstable_Object_Dump(PyObject *op)
+.. c:function:: void PyObject_Dump(PyObject *op)
 
    Dump an object *op* to ``stderr``. This should only be used for debugging.