Merge branch 'main' into argparse-single-dash-long-option-dest

Author: savannahostrowski

Doc/c-api/datetime.rst

@@ -8,11 +8,42 @@ DateTime Objects
 Various date and time objects are supplied by the :mod:`datetime` module.
 Before using any of these functions, the header file :file:`datetime.h` must be
 included in your source (note that this is not included by :file:`Python.h`),
-and the macro :c:macro:`!PyDateTime_IMPORT` must be invoked, usually as part of
+and the macro :c:macro:`PyDateTime_IMPORT` must be invoked, usually as part of
 the module initialisation function.  The macro puts a pointer to a C structure
-into a static variable, :c:data:`!PyDateTimeAPI`, that is used by the following
+into a static variable, :c:data:`PyDateTimeAPI`, that is used by the following
 macros.
 
+.. c:macro:: PyDateTime_IMPORT()
+
+   Import the datetime C API.
+
+   On success, populate the :c:var:`PyDateTimeAPI` pointer.
+   On failure, set :c:var:`PyDateTimeAPI` to ``NULL`` and set an exception.
+   The caller must check if an error occurred via :c:func:`PyErr_Occurred`:
+
+   .. code-block::
+
+      PyDateTime_IMPORT;
+      if (PyErr_Occurred()) { /* cleanup */ }
+
+   .. warning::
+
+      This is not compatible with subinterpreters.
+
+.. c:type:: PyDateTime_CAPI
+
+   Structure containing the fields for the datetime C API.
+
+   The fields of this structure are private and subject to change.
+
+   Do not use this directly; prefer ``PyDateTime_*`` APIs instead.
+
+.. c:var:: PyDateTime_CAPI *PyDateTimeAPI
+
+   Dynamically allocated object containing the datetime C API.
+
+   This variable is only available once :c:macro:`PyDateTime_IMPORT` succeeds.
+
 .. c:type:: PyDateTime_Date
 
    This subtype of :c:type:`PyObject` represents a Python date object.
@@ -325,3 +356,16 @@ Macros for the convenience of modules implementing the DB API:
 
    Create and return a new :class:`datetime.date` object given an argument
    tuple suitable for passing to :meth:`datetime.date.fromtimestamp`.
+
+
+Internal data
+-------------
+
+The following symbols are exposed by the C API but should be considered
+internal-only.
+
+.. c:macro:: PyDateTime_CAPSULE_NAME
+
+   Name of the datetime capsule to pass to :c:func:`PyCapsule_Import`.
+
+   Internal usage only. Use :c:macro:`PyDateTime_IMPORT` instead.

Doc/c-api/gen.rst

@@ -44,3 +44,41 @@ than explicitly calling :c:func:`PyGen_New` or :c:func:`PyGen_NewWithQualName`.
    with ``__name__`` and ``__qualname__`` set to *name* and *qualname*.
    A reference to *frame* is stolen by this function.  The *frame* argument
    must not be ``NULL``.
+
+.. c:function:: PyCodeObject* PyGen_GetCode(PyGenObject *gen)
+
+   Return a new :term:`strong reference` to the code object wrapped by *gen*.
+   This function always succeeds.
+
+
+Asynchronous Generator Objects
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. seealso::
+   :pep:`525`
+
+.. c:var:: PyTypeObject PyAsyncGen_Type
+
+   The type object corresponding to asynchronous generator objects. This is
+   available as :class:`types.AsyncGeneratorType` in the Python layer.
+
+   .. versionadded:: 3.6
+
+.. c:function:: PyObject *PyAsyncGen_New(PyFrameObject *frame, PyObject *name, PyObject *qualname)
+
+   Create a new asynchronous generator wrapping *frame*, with ``__name__`` and
+   ``__qualname__`` set to *name* and *qualname*. *frame* is stolen by this
+   function and must not be ``NULL``.
+
+   On success, this function returns a :term:`strong reference` to the
+   new asynchronous generator. On failure, this function returns ``NULL``
+   with an exception set.
+
+   .. versionadded:: 3.6
+
+.. c:function:: int PyAsyncGen_CheckExact(PyObject *op)
+
+   Return true if *op* is an asynchronous generator object, false otherwise.
+   This function always succeeds.
+
+   .. versionadded:: 3.6

Doc/c-api/init.rst

@@ -1891,6 +1891,25 @@ pointer and a void pointer argument.
       This function now always schedules *func* to be run in the main
       interpreter.
 
+
+.. c:function:: int Py_MakePendingCalls(void)
+
+   Execute all pending calls. This is usually executed automatically by the
+   interpreter.
+
+   This function returns ``0`` on success, and returns ``-1`` with an exception
+   set on failure.
+
+   If this is not called in the main thread of the main
+   interpreter, this function does nothing and returns ``0``.
+   The caller must hold an :term:`attached thread state`.
+
+   .. versionadded:: 3.1
+
+   .. versionchanged:: 3.12
+      This function only runs pending calls in the main interpreter.
+
+
 .. _profiling:
 
 Profiling and Tracing

Include/internal/pycore_global_objects_fini_generated.h

@@ -793,6 +793,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(symmetric_difference_update)
         STRUCT_FOR_ID(tabsize)
         STRUCT_FOR_ID(tag)
+        STRUCT_FOR_ID(take_bytes)
         STRUCT_FOR_ID(target)
         STRUCT_FOR_ID(target_is_directory)
         STRUCT_FOR_ID(task)

Include/internal/pycore_global_strings.h

@@ -1524,6 +1524,32 @@ def test_take_bytes(self):
             self.assertRaises(BufferError, ba.take_bytes)
         self.assertEqual(ba.take_bytes(), b'abc')
 
+    @support.cpython_only  # tests an implementation detail
+    def test_take_bytes_optimization(self):
+        # Validate optimization around taking lots of little chunks out of a
+        # much bigger buffer. Save work by only copying a little rather than
+        # moving a lot.
+        ba = bytearray(b'abcdef' + b'0' * 1000)
+        start_alloc = ba.__alloc__()
+
+        # Take two bytes at a time, checking alloc doesn't change.
+        self.assertEqual(ba.take_bytes(2), b'ab')
+        self.assertEqual(ba.__alloc__(), start_alloc)
+        self.assertEqual(len(ba), 4 + 1000)
+        self.assertEqual(ba.take_bytes(2), b'cd')
+        self.assertEqual(ba.__alloc__(), start_alloc)
+        self.assertEqual(len(ba), 2 + 1000)
+        self.assertEqual(ba.take_bytes(2), b'ef')
+        self.assertEqual(ba.__alloc__(), start_alloc)
+        self.assertEqual(len(ba), 0 + 1000)
+        self.assertEqual(ba.__alloc__(), start_alloc)
+
+        # Take over half, alloc shrinks to exact size.
+        self.assertEqual(ba.take_bytes(501), b'0' * 501)
+        self.assertEqual(len(ba), 499)
+        bytes_header_size = sys.getsizeof(b'')
+        self.assertEqual(ba.__alloc__(), 499 + bytes_header_size)
+
     def test_setitem(self):
         def setitem_as_mapping(b, i, val):
             b[i] = val

Include/internal/pycore_runtime_init_generated.h

@@ -672,7 +672,7 @@ def __eq__(self, other):
     def test_reduce_5tuple(self):
         class C(dict):
             def __reduce__(self):
-                return (C, (), self.__dict__, None, self.items())
+                return (C, (), self.__dict__, None, iter(self.items()))
             def __eq__(self, other):
                 return (dict(self) == dict(other) and
                         self.__dict__ == other.__dict__)

Include/internal/pycore_unicodeobject_generated.h

@@ -609,6 +609,25 @@ def readinto(self, b):
             with self.assertRaises(ValueError):
                 Misbehaved(bad_size).read()
 
+    def test_RawIOBase_read_gh60107(self):
+        # gh-60107: Ensure a "Raw I/O" which keeps a reference to the
+        # mutable memory doesn't allow making a mutable bytes.
+        class RawIOKeepsReference(self.MockRawIOWithoutRead):
+            def __init__(self, *args, **kwargs):
+                self.buf = None
+                super().__init__(*args, **kwargs)
+
+            def readinto(self, buf):
+                # buf is the bytearray so keeping a reference to it doesn't keep
+                # the memory alive; a memoryview does.
+                self.buf = memoryview(buf)
+                buf[0:4] = self._read_stack.pop()
+                return 3
+
+        with self.assertRaises(BufferError):
+            rawio = RawIOKeepsReference([b"1234"])
+            rawio.read(4)
+
     def test_types_have_dict(self):
         test = (
             self.IOBase(),