GH-94739: Mark stacks of exception handling blocks for setting frame.f_lineno in the debugger. (GH-94958)

Author: markshannon

Include/internal/pycore_code.h

@@ -390,6 +390,19 @@ read_obj(uint16_t *p)
     return (PyObject *)val;
 }
 
+/* See Objects/exception_handling_notes.txt for details.
+ */
+static inline unsigned char *
+parse_varint(unsigned char *p, int *result) {
+    int val = p[0] & 63;
+    while (p[0] & 64) {
+        p++;
+        val = (val << 6) | (p[0] & 63);
+    }
+    *result = val;
+    return p+1;
+}
+
 static inline int
 write_varint(uint8_t *ptr, unsigned int val)
 {

Lib/test/test_sys_settrace.py

@@ -1879,7 +1879,7 @@ def test_jump_out_of_block_backwards(output):
             output.append(6)
         output.append(7)
 
-    @async_jump_test(4, 5, [3], (ValueError, 'into'))
+    @async_jump_test(4, 5, [3, 5])
     async def test_jump_out_of_async_for_block_forwards(output):
         for i in [1]:
             async for i in asynciter([1, 2]):
@@ -1921,7 +1921,7 @@ def test_jump_in_nested_finally(output):
                 output.append(8)
             output.append(9)
 
-    @jump_test(6, 7, [2], (ValueError, 'within'))
+    @jump_test(6, 7, [2, 7], (ZeroDivisionError, ''))
     def test_jump_in_nested_finally_2(output):
         try:
             output.append(2)
@@ -1932,7 +1932,7 @@ def test_jump_in_nested_finally_2(output):
             output.append(7)
         output.append(8)
 
-    @jump_test(6, 11, [2], (ValueError, 'within'))
+    @jump_test(6, 11, [2, 11], (ZeroDivisionError, ''))
     def test_jump_in_nested_finally_3(output):
         try:
             output.append(2)
@@ -2043,8 +2043,8 @@ def test_jump_backwards_out_of_try_except_block(output):
             output.append(5)
             raise
 
-    @jump_test(5, 7, [4], (ValueError, 'within'))
-    def test_no_jump_between_except_blocks(output):
+    @jump_test(5, 7, [4, 7, 8])
+    def test_jump_between_except_blocks(output):
         try:
             1/0
         except ZeroDivisionError:
@@ -2054,8 +2054,19 @@ def test_no_jump_between_except_blocks(output):
             output.append(7)
         output.append(8)
 
-    @jump_test(5, 6, [4], (ValueError, 'within'))
-    def test_no_jump_within_except_block(output):
+    @jump_test(5, 7, [4, 7, 8])
+    def test_jump_from_except_to_finally(output):
+        try:
+            1/0
+        except ZeroDivisionError:
+            output.append(4)
+            output.append(5)
+        finally:
+            output.append(7)
+        output.append(8)
+
+    @jump_test(5, 6, [4, 6, 7])
+    def test_jump_within_except_block(output):
         try:
             1/0
         except:
@@ -2290,7 +2301,7 @@ def test_no_jump_backwards_into_for_block(output):
             output.append(2)
         output.append(3)
 
-    @async_jump_test(3, 2, [2, 2], (ValueError, 'within'))
+    @async_jump_test(3, 2, [2, 2], (ValueError, "can't jump into the body of a for loop"))
     async def test_no_jump_backwards_into_async_for_block(output):
         async for i in asynciter([1, 2]):
             output.append(2)
@@ -2355,8 +2366,8 @@ def test_jump_backwards_into_try_except_block(output):
         output.append(6)
 
     # 'except' with a variable creates an implicit finally block
-    @jump_test(5, 7, [4], (ValueError, 'within'))
-    def test_no_jump_between_except_blocks_2(output):
+    @jump_test(5, 7, [4, 7, 8])
+    def test_jump_between_except_blocks_2(output):
         try:
             1/0
         except ZeroDivisionError:
@@ -2392,23 +2403,23 @@ def test_jump_out_of_finally_block(output):
         finally:
             output.append(5)
 
-    @jump_test(1, 5, [], (ValueError, "into an exception"))
+    @jump_test(1, 5, [], (ValueError, "can't jump into an 'except' block as there's no exception"))
     def test_no_jump_into_bare_except_block(output):
         output.append(1)
         try:
             output.append(3)
         except:
             output.append(5)
 
-    @jump_test(1, 5, [], (ValueError, "into an exception"))
+    @jump_test(1, 5, [], (ValueError, "can't jump into an 'except' block as there's no exception"))
     def test_no_jump_into_qualified_except_block(output):
         output.append(1)
         try:
             output.append(3)
         except Exception:
             output.append(5)
 
-    @jump_test(3, 6, [2, 5, 6], (ValueError, "into an exception"))
+    @jump_test(3, 6, [2, 5, 6], (ValueError, "can't jump into an 'except' block as there's no exception"))
     def test_no_jump_into_bare_except_block_from_try_block(output):
         try:
             output.append(2)
@@ -2419,7 +2430,7 @@ def test_no_jump_into_bare_except_block_from_try_block(output):
             raise
         output.append(8)
 
-    @jump_test(3, 6, [2], (ValueError, "into an exception"))
+    @jump_test(3, 6, [2], (ValueError, "can't jump into an 'except' block as there's no exception"))
     def test_no_jump_into_qualified_except_block_from_try_block(output):
         try:
             output.append(2)
@@ -2430,8 +2441,8 @@ def test_no_jump_into_qualified_except_block_from_try_block(output):
             raise
         output.append(8)
 
-    @jump_test(7, 1, [1, 3, 6], (ValueError, "within"))
-    def test_no_jump_out_of_bare_except_block(output):
+    @jump_test(7, 1, [1, 3, 6, 1, 3, 6, 7])
+    def test_jump_out_of_bare_except_block(output):
         output.append(1)
         try:
             output.append(3)
@@ -2440,8 +2451,8 @@ def test_no_jump_out_of_bare_except_block(output):
             output.append(6)
             output.append(7)
 
-    @jump_test(7, 1, [1, 3, 6], (ValueError, "within"))
-    def test_no_jump_out_of_qualified_except_block(output):
+    @jump_test(7, 1, [1, 3, 6, 1, 3, 6, 7])
+    def test_jump_out_of_qualified_except_block(output):
         output.append(1)
         try:
             output.append(3)

Misc/NEWS.d/next/Core and Builtins/2022-07-18-14-19-21.gh-issue-94739.NQJQi7.rst

@@ -0,0 +1 @@
+Allow jumping within, out of, and across exception handlers in the debugger.

Objects/frameobject.c

@@ -138,6 +138,7 @@ typedef enum kind {
     Except = 2,
     Object = 3,
     Null = 4,
+    Lasti = 5,
 } Kind;
 
 static int
@@ -162,6 +163,8 @@ compatible_kind(Kind from, Kind to) {
 #define MAX_STACK_ENTRIES (63/BITS_PER_BLOCK)
 #define WILL_OVERFLOW (1ULL<<((MAX_STACK_ENTRIES-1)*BITS_PER_BLOCK))
 
+#define EMPTY_STACK 0
+
 static inline int64_t
 push_value(int64_t stack, Kind kind)
 {
@@ -185,6 +188,69 @@ top_of_stack(int64_t stack)
     return stack & ((1<<BITS_PER_BLOCK)-1);
 }
 
+static int64_t
+pop_to_level(int64_t stack, int level) {
+    if (level == 0) {
+        return EMPTY_STACK;
+    }
+    int64_t max_item = (1<<BITS_PER_BLOCK) - 1;
+    int64_t level_max_stack = max_item << ((level-1) * BITS_PER_BLOCK);
+    while (stack > level_max_stack) {
+        stack = pop_value(stack);
+    }
+    return stack;
+}
+
+#if 0
+/* These functions are useful for debugging the stack marking code */
+
+static char
+tos_char(int64_t stack) {
+    switch(top_of_stack(stack)) {
+        case Iterator:
+            return 'I';
+        case Except:
+            return 'E';
+        case Object:
+            return 'O';
+        case Lasti:
+            return 'L';
+        case Null:
+            return 'N';
+    }
+}
+
+static void
+print_stack(int64_t stack) {
+    if (stack < 0) {
+        if (stack == UNINITIALIZED) {
+            printf("---");
+        }
+        else if (stack == OVERFLOWED) {
+            printf("OVERFLOWED");
+        }
+        else {
+            printf("??");
+        }
+        return;
+    }
+    while (stack) {
+        printf("%c", tos_char(stack));
+        stack = pop_value(stack);
+    }
+}
+
+static void
+print_stacks(int64_t *stacks, int n) {
+    for (int i = 0; i < n; i++) {
+        printf("%d: ", i);
+        print_stack(stacks[i]);
+        printf("\n");
+    }
+}
+
+#endif
+
 static int64_t *
 mark_stacks(PyCodeObject *code_obj, int len)
 {
@@ -204,7 +270,7 @@ mark_stacks(PyCodeObject *code_obj, int len)
     for (int i = 1; i <= len; i++) {
         stacks[i] = UNINITIALIZED;
     }
-    stacks[0] = 0;
+    stacks[0] = EMPTY_STACK;
     if (code_obj->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR))
     {
         // Generators get sent None while starting:
@@ -213,6 +279,7 @@ mark_stacks(PyCodeObject *code_obj, int len)
     int todo = 1;
     while (todo) {
         todo = 0;
+        /* Scan instructions */
         for (i = 0; i < len; i++) {
             int64_t next_stack = stacks[i];
             if (next_stack == UNINITIALIZED) {
@@ -296,23 +363,25 @@ mark_stacks(PyCodeObject *code_obj, int len)
                     break;
                 }
                 case END_ASYNC_FOR:
-                    next_stack = pop_value(pop_value(pop_value(next_stack)));
+                    next_stack = pop_value(pop_value(next_stack));
                     stacks[i+1] = next_stack;
                     break;
                 case PUSH_EXC_INFO:
+                    next_stack = push_value(next_stack, Except);
+                    stacks[i+1] = next_stack;
+                    break;
                 case POP_EXCEPT:
-                    /* These instructions only appear in exception handlers, which
-                     * skip this switch ever since the move to zero-cost exceptions
-                     * (their stack remains UNINITIALIZED because nothing sets it).
-                     *
-                     * Note that explain_incompatible_stack interprets an
-                     * UNINITIALIZED stack as belonging to an exception handler.
-                     */
-                    Py_UNREACHABLE();
+                    next_stack = pop_value(next_stack);
+                    stacks[i+1] = next_stack;
                     break;
                 case RETURN_VALUE:
+                    assert(pop_value(next_stack) == EMPTY_STACK);
+                    assert(top_of_stack(next_stack) == Object);
+                    break;
                 case RAISE_VARARGS:
+                    break;
                 case RERAISE:
+                    assert(top_of_stack(next_stack) == Except);
                     /* End of block */
                     break;
                 case PUSH_NULL:
@@ -331,6 +400,7 @@ mark_stacks(PyCodeObject *code_obj, int len)
                 }
                 case LOAD_ATTR:
                 {
+                    assert(top_of_stack(next_stack) == Object);
                     int j = get_arg(code, i);
                     if (j & 1) {
                         next_stack = pop_value(next_stack);
@@ -340,6 +410,16 @@ mark_stacks(PyCodeObject *code_obj, int len)
                     stacks[i+1] = next_stack;
                     break;
                 }
+                case CALL:
+                {
+                    int args = get_arg(code, i);
+                    for (int j = 0; j < args+2; j++) {
+                        next_stack = pop_value(next_stack);
+                    }
+                    next_stack = push_value(next_stack, Object);
+                    stacks[i+1] = next_stack;
+                    break;
+                }
                 default:
                 {
                     int delta = PyCompile_OpcodeStackEffect(opcode, _Py_OPARG(code[i]));
@@ -355,6 +435,34 @@ mark_stacks(PyCodeObject *code_obj, int len)
                 }
             }
         }
+        /* Scan exception table */
+        unsigned char *start = (unsigned char *)PyBytes_AS_STRING(code_obj->co_exceptiontable);
+        unsigned char *end = start + PyBytes_GET_SIZE(code_obj->co_exceptiontable);
+        unsigned char *scan = start;
+        while (scan < end) {
+            int start_offset, size, handler;
+            scan = parse_varint(scan, &start_offset);
+            assert(start_offset >= 0 && start_offset < len);
+            scan = parse_varint(scan, &size);
+            assert(size >= 0 && start_offset+size <= len);
+            scan = parse_varint(scan, &handler);
+            assert(handler >= 0 && handler < len);
+            int depth_and_lasti;
+            scan = parse_varint(scan, &depth_and_lasti);
+            int level = depth_and_lasti >> 1;
+            int lasti = depth_and_lasti & 1;
+            if (stacks[start_offset] != UNINITIALIZED) {
+                if (stacks[handler] == UNINITIALIZED) {
+                    todo = 1;
+                    uint64_t target_stack = pop_to_level(stacks[start_offset], level);
+                    if (lasti) {
+                        target_stack = push_value(target_stack, Lasti);
+                    }
+                    target_stack = push_value(target_stack, Except);
+                    stacks[handler] = target_stack;
+                }
+            }
+        }
     }
     Py_DECREF(co_code);
     return stacks;
@@ -395,6 +503,8 @@ explain_incompatible_stack(int64_t to_stack)
     switch(target_kind) {
         case Except:
             return "can't jump into an 'except' block as there's no exception";
+        case Lasti:
+            return "can't jump into a re-raising block as there's no location";
         case Object:
         case Null:
             return "incompatible stacks";
@@ -650,7 +760,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore
                     msg = "stack to deep to analyze";
                 }
                 else if (start_stack == UNINITIALIZED) {
-                    msg = "can't jump from within an exception handler";
+                    msg = "can't jump from unreachable code";
                 }
                 else {
                     msg = explain_incompatible_stack(target_stack);

Python/ceval.c

@@ -6089,20 +6089,6 @@ positional_only_passed_as_keyword(PyThreadState *tstate, PyCodeObject *co,
 
 }
 
-/* Exception table parsing code.
- * See Objects/exception_table_notes.txt for details.
- */
-
-static inline unsigned char *
-parse_varint(unsigned char *p, int *result) {
-    int val = p[0] & 63;
-    while (p[0] & 64) {
-        p++;
-        val = (val << 6) | (p[0] & 63);
-    }
-    *result = val;
-    return p+1;
-}
 
 static inline unsigned char *
 scan_back_to_entry_start(unsigned char *p) {