Replace Py_UCS4 with (const Py_UCS4*, Py_ssize_t)

Author: vstinner

Doc/c-api/unicode.rst

@@ -307,53 +307,61 @@ These APIs can be used for fast direct character conversions:
    possible.  This function does not raise exceptions.
 
 
-.. c:function:: Py_ssize_t PyUCS4_ToLower(Py_UCS4 ch, Py_UCS4 *buffer, Py_ssize_t size)
+.. c:function:: Py_ssize_t PyUCS4_ToLower(const Py_UCS4 *str, Py_ssize_t str_size, Py_UCS4 *buffer, Py_ssize_t buf_size)
 
-   Convert *ch* to lower case, store result in *buffer*, which should be
-   able to hold as many characters needed for *ch* to be lower cased, and
+   Convert *str* characters to lower case, store result in *buffer*, which should be
+   able to hold as many characters needed for *str* to be lower cased, and
    return the number of characters stored. If at some point a buffer overflow
    is detected, an :exc:`ValueError` is raised and ``-1`` is returned.
 
-   In Unicode 16.0, any character can be lowercased into a buffer of *size* ``2``.
+   *str_size*, *buf_size* and the result are number of UCS-4 characters.
+
+   In Unicode 16.0, any character can be lowercased into a buffer of *buf_size* ``2``.
    See also :c:macro:`PyUCS4_CASE_CONVERSION_BUFFER_SIZE`.
 
    .. versionadded:: next
 
 
-.. c:function:: Py_ssize_t PyUCS4_ToUpper(Py_UCS4 ch, Py_UCS4 *buffer, Py_ssize_t size)
+.. c:function:: Py_ssize_t PyUCS4_ToUpper(const Py_UCS4 *str, Py_ssize_t str_size, Py_UCS4 *buffer, Py_ssize_t buf_size)
 
-   Convert *ch* to upper case, store result in *buffer*, which should be
-   able to hold as many characters needed for *ch* to be upper cased, and
+   Convert *str* characters to upper case, store result in *buffer*, which should be
+   able to hold as many characters needed for *str* to be upper cased, and
    return the number of characters stored. If at some point a buffer overflow
    is detected, an :exc:`ValueError` is raised and ``-1`` is returned.
 
-   In Unicode 16.0, any character can be uppercased into a buffer of *size* ``3``.
+   *str_size*, *buf_size* and the result are number of UCS-4 characters.
+
+   In Unicode 16.0, any character can be uppercased into a buffer of *buf_size* ``3``.
    See also :c:macro:`PyUCS4_CASE_CONVERSION_BUFFER_SIZE`.
 
    .. versionadded:: next
 
 
-.. c:function:: Py_ssize_t PyUCS4_ToTitle(Py_UCS4 ch, Py_UCS4 *buffer, Py_ssize_t size)
+.. c:function:: Py_ssize_t PyUCS4_ToTitle(const Py_UCS4 *str, Py_ssize_t str_size, Py_UCS4 *buffer, Py_ssize_t buf_size)
 
-   Convert *ch* to title case, store result in *buffer*, which should be
-   able to hold as many characters needed for *ch* to be title cased, and
+   Convert *str* characters to title case, store result in *buffer*, which should be
+   able to hold as many characters needed for *str* to be title cased, and
    return the number of characters stored. If at some point a buffer overflow
    is detected, an :exc:`ValueError` is raised and ``-1`` is returned.
 
-   In Unicode 16.0, any character can be titlecased into a buffer of *size* ``3``.
+   *str_size*, *buf_size* and the result are number of UCS-4 characters.
+
+   In Unicode 16.0, any character can be titlecased into a buffer of *buf_size* ``3``.
    See also :c:macro:`PyUCS4_CASE_CONVERSION_BUFFER_SIZE`.
 
    .. versionadded:: next
 
 
-.. c:function:: Py_ssize_t PyUCS4_ToFolded(Py_UCS4 ch, Py_UCS4 *buffer, Py_ssize_t size)
+.. c:function:: Py_ssize_t PyUCS4_ToFolded(const Py_UCS4 *str, Py_ssize_t str_size, Py_UCS4 *buffer, Py_ssize_t buf_size)
 
-   Foldcase *ch*, store result in *buffer*, which should be
-   able to hold as many characters needed for *ch* to be foldcased, and
+   Foldcase *str* characters, store result in *buffer*, which should be
+   able to hold as many characters needed for *str* to be foldcased, and
    return the number of characters stored. If at some point a buffer overflow
    is detected, an :exc:`ValueError` is raised and ``-1`` is returned.
 
-   In Unicode 16.0, any character can be foldcased into a buffer of *size* ``3``.
+   *str_size*, *buf_size* and the result are number of UCS-4 characters.
+
+   In Unicode 16.0, any character can be foldcased into a buffer of *buf_size* ``3``.
    See also :c:macro:`PyUCS4_CASE_CONVERSION_BUFFER_SIZE`.
 
    .. versionadded:: next

Include/cpython/unicodeobject.h

@@ -734,28 +734,28 @@ PyAPI_FUNC(int) _PyUnicode_IsAlpha(
     );
 
 PyAPI_FUNC(Py_ssize_t) PyUCS4_ToLower(
-    Py_UCS4 ch,     /* Unicode character */
-    Py_UCS4 *res,   /* Output buffer */
-    Py_ssize_t size        /* Buffer size */
-    );
+    const Py_UCS4 *str,    /* Unicode string */
+    Py_ssize_t str_size,   /* Unicode string size (UCS-4 characters) */
+    Py_UCS4 *buf,          /* Output buffer */
+    Py_ssize_t buf_size);  /* Buffer size (UCS-4 characters) */
 
 PyAPI_FUNC(Py_ssize_t) PyUCS4_ToUpper(
-    Py_UCS4 ch,     /* Unicode character */
-    Py_UCS4 *res,   /* Output buffer */
-    Py_ssize_t size        /* Buffer size */
-    );
+    const Py_UCS4 *str,    /* Unicode string */
+    Py_ssize_t str_size,   /* Unicode string size (UCS-4 characters) */
+    Py_UCS4 *buf,          /* Output buffer */
+    Py_ssize_t buf_size);  /* Buffer size (UCS-4 characters) */
 
 PyAPI_FUNC(Py_ssize_t) PyUCS4_ToTitle(
-    Py_UCS4 ch,     /* Unicode character */
-    Py_UCS4 *res,   /* Output buffer */
-    Py_ssize_t size        /* Buffer size */
-    );
+    const Py_UCS4 *str,    /* Unicode string */
+    Py_ssize_t str_size,   /* Unicode string size (UCS-4 characters) */
+    Py_UCS4 *buf,          /* Output buffer */
+    Py_ssize_t buf_size);  /* Buffer size (UCS-4 characters) */
 
 PyAPI_FUNC(Py_ssize_t) PyUCS4_ToFolded(
-    Py_UCS4 ch,     /* Unicode character */
-    Py_UCS4 *res,   /* Output buffer */
-    Py_ssize_t size        /* Buffer size */
-    );
+    const Py_UCS4 *str,    /* Unicode string */
+    Py_ssize_t str_size,   /* Unicode string size (UCS-4 characters) */
+    Py_UCS4 *buf,          /* Output buffer */
+    Py_ssize_t buf_size);  /* Buffer size (UCS-4 characters) */
 
 
 // Helper array used by Py_UNICODE_ISSPACE().

Lib/test/test_capi/test_unicode.py

@@ -1,5 +1,6 @@
 import unittest
 import sys
+import string
 from test import support
 from test.support import threading_helper
 
@@ -1756,31 +1757,29 @@ def test_GET_CACHED_HASH(self):
     @support.cpython_only
     @unittest.skipIf(_testcapi is None, 'need _testcapi module')
     def test_tolower(self):
-        import string
         from _testcapi import unicode_tolower
 
-        for i, c in enumerate(string.ascii_uppercase):
-            with self.subTest(c):
-                self.assertEqual(unicode_tolower(c), string.ascii_lowercase[i])
+        self.assertEqual(unicode_tolower(string.ascii_uppercase),
+                         string.ascii_lowercase)
 
         # Test unicode character
         self.assertEqual(unicode_tolower("Č"), "č")
         self.assertEqual(unicode_tolower("Σ"), "σ")
+        self.assertEqual(unicode_tolower("ABCΣ"), "abcσ")
 
     @support.cpython_only
     @unittest.skipIf(_testcapi is None, 'need _testcapi module')
     def test_toupper(self):
-        import string
         from _testcapi import unicode_toupper, unicode_toupper_buffer_too_small
 
-        for i, c in enumerate(string.ascii_lowercase):
-            with self.subTest(c):
-                self.assertEqual(unicode_toupper(c), string.ascii_uppercase[i])
+        self.assertEqual(unicode_toupper(string.ascii_lowercase),
+                         string.ascii_uppercase)
 
         # Test unicode character
         self.assertEqual(unicode_toupper("č"), "Č")
         self.assertEqual(unicode_toupper("ß"), "SS")
         self.assertEqual(unicode_toupper("ΐ"), "Ϊ́")
+        self.assertEqual(unicode_toupper("abcß"), "ABCSS")
 
         # Test unicode character with smaller buffer
         with self.assertRaisesRegex(ValueError, "output buffer is too small"):
@@ -1797,6 +1796,7 @@ def test_totitle(self):
         self.assertEqual(unicode_totitle("ł"), "Ł")
         self.assertEqual(unicode_totitle("ß"), "Ss")
         self.assertEqual(unicode_totitle("ΐ"), "Ϊ́")
+        self.assertEqual(unicode_totitle("abcß"), "ABCSs")
 
     @support.cpython_only
     @unittest.skipIf(_testcapi is None, 'need _testcapi module')
@@ -1808,6 +1808,8 @@ def test_tofolded(self):
         # Test unicode character
         self.assertEqual(unicode_tofolded("Ł"), "ł")
         self.assertEqual(unicode_tofolded("Σ"), "σ")
+        self.assertEqual(unicode_tofolded("abcΣ"), "abcσ")
+        self.assertEqual(unicode_tofolded("ABCσ"), "abcσ")
 
         # Test case-ignorable character
         self.assertEqual(unicode_tofolded("👍"), "👍")

Modules/_testcapi/unicode.c

@@ -221,22 +221,36 @@ unicode_copycharacters(PyObject *self, PyObject *args)
 }
 
 static PyObject *
-unicode_case_operation(PyObject *str, Py_ssize_t (*function)(Py_UCS4, Py_UCS4 *, Py_ssize_t),
-                       Py_UCS4 *buf, Py_ssize_t size)
+unicode_case_operation(PyObject *str,
+                       Py_ssize_t (*function)(const Py_UCS4*, Py_ssize_t, Py_UCS4 *, Py_ssize_t),
+                       int buf_too_small)
 {
     if (!PyUnicode_Check(str)) {
         PyErr_Format(PyExc_TypeError, "expect str type, got %T", str);
         return NULL;
     }
+    Py_ssize_t len = PyUnicode_GET_LENGTH(str);
 
-    if (PyUnicode_GET_LENGTH(str) != 1) {
-        PyErr_SetString(PyExc_ValueError, "expecting 1-character strings only");
+    Py_UCS4 *ucs4 = PyUnicode_AsUCS4Copy(str);
+    if (ucs4 == NULL) {
         return NULL;
     }
 
-    Py_UCS4 c = PyUnicode_READ_CHAR(str, 0);
+    Py_ssize_t buf_size;
+    if (!buf_too_small) {
+        buf_size = len * PyUCS4_CASE_CONVERSION_BUFFER_SIZE;
+    }
+    else {
+        buf_size = len * 1;
+    }
+    Py_UCS4 *buf = PyMem_Malloc(buf_size * sizeof(Py_UCS4));
+    if (buf == NULL) {
+        PyMem_Free(ucs4);
+        return NULL;
+    }
 
-    Py_ssize_t chars = function(c, buf, size);
+    Py_ssize_t chars = function(ucs4, len, buf, buf_size);
+    PyMem_Free(ucs4);
     if (chars < 0) {
         return NULL;
     }
@@ -248,41 +262,36 @@ unicode_case_operation(PyObject *str, Py_ssize_t (*function)(Py_UCS4, Py_UCS4 *,
 static PyObject *
 unicode_tolower(PyObject *self, PyObject *arg)
 {
-    Py_UCS4 buf[PyUCS4_CASE_CONVERSION_BUFFER_SIZE];
-    return unicode_case_operation(arg, PyUCS4_ToLower, buf, PyUCS4_CASE_CONVERSION_BUFFER_SIZE);
+    return unicode_case_operation(arg, PyUCS4_ToLower, 0);
 }
 
 
 /* Test PyUCS4_ToUpper() */
 static PyObject *
 unicode_toupper(PyObject *self, PyObject *arg)
 {
-    Py_UCS4 buf[PyUCS4_CASE_CONVERSION_BUFFER_SIZE];
-    return unicode_case_operation(arg, PyUCS4_ToUpper, buf, PyUCS4_CASE_CONVERSION_BUFFER_SIZE);
+    return unicode_case_operation(arg, PyUCS4_ToUpper, 0);
 }
 
 /* Test PyUCS4_ToUpper() with a small buffer */
 static PyObject *
 unicode_toupper_buffer_too_small(PyObject *self, PyObject *arg)
 {
-    Py_UCS4 buf;
-    return unicode_case_operation(arg, PyUCS4_ToUpper, &buf, 1);
+    return unicode_case_operation(arg, PyUCS4_ToUpper, 1);
 }
 
 /* Test PyUCS4_ToLower() */
 static PyObject *
 unicode_totitle(PyObject *self, PyObject *arg)
 {
-    Py_UCS4 buf[PyUCS4_CASE_CONVERSION_BUFFER_SIZE];
-    return unicode_case_operation(arg, PyUCS4_ToTitle, buf, PyUCS4_CASE_CONVERSION_BUFFER_SIZE);
+    return unicode_case_operation(arg, PyUCS4_ToTitle, 0);
 }
 
 /* Test PyUCS4_ToLower() */
 static PyObject *
 unicode_tofolded(PyObject *self, PyObject *arg)
 {
-    Py_UCS4 buf[PyUCS4_CASE_CONVERSION_BUFFER_SIZE];
-    return unicode_case_operation(arg, PyUCS4_ToFolded, buf, PyUCS4_CASE_CONVERSION_BUFFER_SIZE);
+    return unicode_case_operation(arg, PyUCS4_ToFolded, 0);
 }