gh-77188: Add support for pickling private methods and nested classes (GH-21480)

Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>

Author: ZackerySpytz

Doc/whatsnew/3.15.rst

@@ -704,6 +704,13 @@ os.path
   (Contributed by Petr Viktorin for :cve:`2025-4517`.)
 
 
+pickle
+------
+
+* Add support for pickling private methods and nested classes.
+  (Contributed by Zackery Spytz and Serhiy Storchaka in :gh:`77188`.)
+
+
 resource
 --------
 

Include/internal/pycore_symtable.h

@@ -151,7 +151,12 @@ extern int _PySymtable_LookupOptional(struct symtable *, void *, PySTEntryObject
 extern void _PySymtable_Free(struct symtable *);
 
 extern PyObject *_Py_MaybeMangle(PyObject *privateobj, PySTEntryObject *ste, PyObject *name);
-extern PyObject* _Py_Mangle(PyObject *p, PyObject *name);
+
+// Export for '_pickle' shared extension
+PyAPI_FUNC(PyObject *)
+_Py_Mangle(PyObject *, PyObject *);
+PyAPI_FUNC(int)
+_Py_IsPrivateName(PyObject *);
 
 /* Flags for def-use information */
 

Lib/pickle.py

@@ -1175,6 +1175,17 @@ def save_global(self, obj, name=None):
             if name is None:
                 name = obj.__name__
 
+        if '.__' in name:
+            # Mangle names of private attributes.
+            dotted_path = name.split('.')
+            for i, subpath in enumerate(dotted_path):
+                if i and subpath.startswith('__') and not subpath.endswith('__'):
+                    prev = prev.lstrip('_')
+                    if prev:
+                        dotted_path[i] = f"_{prev.lstrip('_')}{subpath}"
+                prev = subpath
+            name = '.'.join(dotted_path)
+
         module_name = whichmodule(obj, name)
         if self.proto >= 2:
             code = _extension_registry.get((module_name, name), _NoValue)

Lib/test/picklecommon.py

@@ -388,3 +388,48 @@ def pie(self):
 class Subclass(tuple):
     class Nested(str):
         pass
+
+# For test_private_methods
+class PrivateMethods:
+    def __init__(self, value):
+        self.value = value
+
+    def __private_method(self):
+        return self.value
+
+    def get_method(self):
+        return self.__private_method
+
+    @classmethod
+    def get_unbound_method(cls):
+        return cls.__private_method
+
+    @classmethod
+    def __private_classmethod(cls):
+        return 43
+
+    @classmethod
+    def get_classmethod(cls):
+        return cls.__private_classmethod
+
+    @staticmethod
+    def __private_staticmethod():
+        return 44
+
+    @classmethod
+    def get_staticmethod(cls):
+        return cls.__private_staticmethod
+
+# For test_private_nested_classes
+class PrivateNestedClasses:
+    @classmethod
+    def get_nested(cls):
+        return cls.__Nested
+
+    class __Nested:
+        @classmethod
+        def get_nested2(cls):
+            return cls.__Nested2
+
+        class __Nested2:
+            pass

Lib/test/pickletester.py

@@ -4118,6 +4118,33 @@ def test_c_methods(self):
                 with self.subTest(proto=proto, descr=descr):
                     self.assertRaises(TypeError, self.dumps, descr, proto)
 
+    def test_private_methods(self):
+        if self.py_version < (3, 15):
+            self.skipTest('not supported in Python < 3.15')
+        obj = PrivateMethods(42)
+        for proto in protocols:
+            with self.subTest(proto=proto):
+                unpickled = self.loads(self.dumps(obj.get_method(), proto))
+                self.assertEqual(unpickled(), 42)
+                unpickled = self.loads(self.dumps(obj.get_unbound_method(), proto))
+                self.assertEqual(unpickled(obj), 42)
+                unpickled = self.loads(self.dumps(obj.get_classmethod(), proto))
+                self.assertEqual(unpickled(), 43)
+                unpickled = self.loads(self.dumps(obj.get_staticmethod(), proto))
+                self.assertEqual(unpickled(), 44)
+
+    def test_private_nested_classes(self):
+        if self.py_version < (3, 15):
+            self.skipTest('not supported in Python < 3.15')
+        cls1 = PrivateNestedClasses.get_nested()
+        cls2 = cls1.get_nested2()
+        for proto in protocols:
+            with self.subTest(proto=proto):
+                unpickled = self.loads(self.dumps(cls1, proto))
+                self.assertIs(unpickled, cls1)
+                unpickled = self.loads(self.dumps(cls2, proto))
+                self.assertIs(unpickled, cls2)
+
     def test_object_with_attrs(self):
         obj = Object()
         obj.a = 1

Misc/NEWS.d/next/Library/2020-07-14-23-54-18.gh-issue-77188.TyI3_Q.rst

@@ -0,0 +1 @@
+The :mod:`pickle` module now properly handles name-mangled private methods.

Modules/_pickle.c

@@ -19,6 +19,7 @@
 #include "pycore_pystate.h"       // _PyThreadState_GET()
 #include "pycore_runtime.h"       // _Py_ID()
 #include "pycore_setobject.h"     // _PySet_NextEntry()
+#include "pycore_symtable.h"      // _Py_Mangle()
 #include "pycore_sysmodule.h"     // _PySys_GetSizeOf()
 #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString()
 
@@ -1928,6 +1929,37 @@ get_dotted_path(PyObject *name)
     return PyUnicode_Split(name, _Py_LATIN1_CHR('.'), -1);
 }
 
+static PyObject *
+join_dotted_path(PyObject *dotted_path)
+{
+    return PyUnicode_Join(_Py_LATIN1_CHR('.'), dotted_path);
+}
+
+/* Returns -1 (with an exception set) on error, 0 if there were no changes,
+ * 1 if some names were mangled. */
+static int
+mangle_dotted_path(PyObject *dotted_path)
+{
+    int rc = 0;
+    Py_ssize_t n = PyList_GET_SIZE(dotted_path);
+    for (Py_ssize_t i = n-1; i > 0; i--) {
+        PyObject *subpath = PyList_GET_ITEM(dotted_path, i);
+        if (_Py_IsPrivateName(subpath)) {
+            PyObject *parent = PyList_GET_ITEM(dotted_path, i-1);
+            PyObject *mangled = _Py_Mangle(parent, subpath);
+            if (mangled == NULL) {
+                return -1;
+            }
+            if (mangled != subpath) {
+                rc = 1;
+            }
+            PyList_SET_ITEM(dotted_path, i, mangled);
+            Py_DECREF(subpath);
+        }
+    }
+    return rc;
+}
+
 static int
 check_dotted_path(PickleState *st, PyObject *obj, PyObject *dotted_path)
 {
@@ -3809,6 +3841,15 @@ save_global(PickleState *st, PicklerObject *self, PyObject *obj,
     dotted_path = get_dotted_path(global_name);
     if (dotted_path == NULL)
         goto error;
+    switch (mangle_dotted_path(dotted_path)) {
+        case -1:
+            goto error;
+        case 1:
+            Py_SETREF(global_name, join_dotted_path(dotted_path));
+            if (global_name == NULL) {
+                goto error;
+            }
+    }
     module_name = whichmodule(st, obj, global_name, dotted_path);
     if (module_name == NULL)
         goto error;

Objects/classobject.c

@@ -7,6 +7,7 @@
 #include "pycore_object.h"
 #include "pycore_pyerrors.h"
 #include "pycore_pystate.h"       // _PyThreadState_GET()
+#include "pycore_symtable.h"      // _Py_Mangle()
 #include "pycore_weakref.h"       // FT_CLEAR_WEAKREFS()
 
 
@@ -143,6 +144,20 @@ method___reduce___impl(PyMethodObject *self)
     if (funcname == NULL) {
         return NULL;
     }
+    if (_Py_IsPrivateName(funcname)) {
+        PyObject *classname = PyType_Check(funcself)
+            ? PyType_GetName((PyTypeObject *)funcself)
+            : PyType_GetName(Py_TYPE(funcself));
+        if (classname == NULL) {
+            Py_DECREF(funcname);
+            return NULL;
+        }
+        Py_SETREF(funcname, _Py_Mangle(classname, funcname));
+        Py_DECREF(classname);
+        if (funcname == NULL) {
+            return NULL;
+        }
+    }
     return Py_BuildValue(
             "N(ON)", _PyEval_GetBuiltin(&_Py_ID(getattr)), funcself, funcname);
 }

Python/symtable.c

@@ -3183,6 +3183,27 @@ _Py_MaybeMangle(PyObject *privateobj, PySTEntryObject *ste, PyObject *name)
     return _Py_Mangle(privateobj, name);
 }
 
+int
+_Py_IsPrivateName(PyObject *ident)
+{
+    if (!PyUnicode_Check(ident)) {
+        return 0;
+    }
+    Py_ssize_t nlen = PyUnicode_GET_LENGTH(ident);
+    if (nlen < 3 ||
+        PyUnicode_READ_CHAR(ident, 0) != '_' ||
+        PyUnicode_READ_CHAR(ident, 1) != '_')
+    {
+        return 0;
+    }
+    if (PyUnicode_READ_CHAR(ident, nlen-1) == '_' &&
+        PyUnicode_READ_CHAR(ident, nlen-2) == '_')
+    {
+        return 0; /* Don't mangle __whatever__ */
+    }
+    return 1;
+}
+
 PyObject *
 _Py_Mangle(PyObject *privateobj, PyObject *ident)
 {