Create new references to fast locals overwritten via f_locals

These may provide support for borrowed references contained in frames
closer to the top of the call stack. Add them to a list attached to the
frame when they are overwritten, to be destroyed when the frame is destroyed.

Author: mpage

Include/internal/pycore_frame.h

@@ -31,6 +31,12 @@ struct _frame {
        PyEval_GetLocals requires a borrowed reference so the actual reference
        is stored here */
     PyObject *f_locals_cache;
+    /* A list containing strong references to fast locals that were overwritten
+     * via f_locals. Borrowed references to these locals may exist in frames
+     * closer to the top of the stack. The references in this list act as
+     * "support" for the borrowed references, ensuring that they remain valid.
+     */
+    PyObject *f_overwritten_fast_locals;
     /* The frame data, if this frame object owns the frame */
     PyObject *_f_frame_data[1];
 };

Lib/test/test_frame.py

@@ -597,6 +597,20 @@ def make_frame():
         with self.assertRaises(TypeError):
             FrameLocalsProxy(frame=sys._getframe())  # no keyword arguments
 
+    def test_overwrite_locals(self):
+        # Verify we do not crash if we overwrite a local passed as an argument
+        # from an ancestor in the call stack.
+        def f():
+            xs = [1, 2, 3]
+            return g(xs)
+
+        def g(xs):
+            f = sys._getframe()
+            f.f_back.f_locals["xs"] = None
+            return xs[1]
+
+        self.assertEqual(f(), 2)
+
 
 class FrameLocalsProxyMappingTests(mapping_tests.TestHashMappingProtocol):
     """Test that FrameLocalsProxy behaves like a Mapping (with exceptions)"""

Lib/test/test_sys.py

@@ -1643,7 +1643,7 @@ def func():
             INTERPRETER_FRAME = '9PihcP'
         else:
             INTERPRETER_FRAME = '9PhcP'
-        check(x, size('3PiccPP' + INTERPRETER_FRAME + 'P'))
+        check(x, size('3PiccPPP' + INTERPRETER_FRAME + 'P'))
         # function
         def func(): pass
         check(func, size('16Pi'))

Objects/frameobject.c

@@ -243,7 +243,19 @@ framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value)
             Py_XINCREF(value);
             PyCell_SetTakeRef((PyCellObject *)cell, value);
         } else if (value != PyStackRef_AsPyObjectBorrow(oldvalue)) {
-            PyStackRef_XCLOSE(fast[i]);
+            if (!PyStackRef_IsNull(fast[i])) {
+                if (frame->f_overwritten_fast_locals == NULL) {
+                    frame->f_overwritten_fast_locals = PyList_New(0);
+                    if (frame->f_overwritten_fast_locals == NULL) {
+                        return -1;
+                    }
+                }
+                PyObject *obj = PyStackRef_AsPyObjectBorrow(fast[i]);
+                if (PyList_Append(frame->f_overwritten_fast_locals, obj) < 0) {
+                    return -1;
+                }
+                PyStackRef_CLOSE(fast[i]);
+            }
             fast[i] = PyStackRef_FromPyObjectNew(value);
         }
         return 0;
@@ -1806,6 +1818,7 @@ frame_dealloc(PyObject *op)
     Py_CLEAR(f->f_trace);
     Py_CLEAR(f->f_extra_locals);
     Py_CLEAR(f->f_locals_cache);
+    Py_CLEAR(f->f_overwritten_fast_locals);
     PyObject_GC_Del(f);
     Py_TRASHCAN_END;
 }
@@ -1818,6 +1831,7 @@ frame_traverse(PyObject *op, visitproc visit, void *arg)
     Py_VISIT(f->f_trace);
     Py_VISIT(f->f_extra_locals);
     Py_VISIT(f->f_locals_cache);
+    Py_VISIT(f->f_overwritten_fast_locals);
     if (f->f_frame->owner != FRAME_OWNED_BY_FRAME_OBJECT) {
         return 0;
     }
@@ -1832,6 +1846,7 @@ frame_tp_clear(PyObject *op)
     Py_CLEAR(f->f_trace);
     Py_CLEAR(f->f_extra_locals);
     Py_CLEAR(f->f_locals_cache);
+    Py_CLEAR(f->f_overwritten_fast_locals);
 
     /* locals and stack */
     _PyStackRef *locals = _PyFrame_GetLocalsArray(f->f_frame);
@@ -1973,6 +1988,7 @@ _PyFrame_New_NoTrack(PyCodeObject *code)
     f->f_lineno = 0;
     f->f_extra_locals = NULL;
     f->f_locals_cache = NULL;
+    f->f_overwritten_fast_locals = NULL;
     return f;
 }