Skip to content

Commit b4aac70

Browse files
committed
refs qoretechnologies/qore#4653 fixed stack reporting in Qore exceptions called from Python; Python stack frames are now included in the Qore stack trace
1 parent 6ea98f9 commit b4aac70

11 files changed

+336
-26
lines changed

CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ set(CPP_SRC
7272
src/PythonQoreClass.cpp
7373
src/PythonQoreCallable.cpp
7474
src/ModuleNamespace.cpp
75+
src/QorePythonStackLocationHelper.cpp
7576
)
7677

7778
qore_wrap_qpp_value(QPP_SOURCES ${QPP_SRC})

docs/mainpage.dox.tmpl

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -500,6 +500,11 @@ PythonProgram::setSaveObjectCallback(callback);
500500
501501
@section pythonreleasenotes python Module Release Notes
502502
503+
@subsection python_1_2 python Module Version 1.2
504+
- enable proper Python stack trace reporting when exceptions are thrown in %Qore code called from %Python; %Python
505+
stack frames are now included in the %Qore stack trace
506+
(<a href="https://github.com/qorelanguage/qore/issues/4653">issue 4653</a>)
507+
503508
@subsection python_1_1_7 python Module Version 1.1.7
504509
- enabled building with standard atomic operations for all platforms
505510

src/PythonQoreClass.cpp

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
/*
33
qore Python module
44
5-
Copyright (C) 2020 - 2021 Qore Technologies, s.r.o.
5+
Copyright (C) 2020 - 2022 Qore Technologies, s.r.o.
66
77
This library is free software; you can redistribute it and/or
88
modify it under the terms of the GNU Lesser General Public
@@ -22,6 +22,7 @@
2222
#include "PythonQoreClass.h"
2323
#include "QoreLoader.h"
2424
#include "QorePythonProgram.h"
25+
#include "QorePythonStackLocationHelper.h"
2526

2627
#include <string.h>
2728
#include <memory>
@@ -435,6 +436,9 @@ PyObject* PythonQoreClass::exec_qore_method(PyObject* method_capsule, PyObject*
435436
ValueHolder rv(&xsink);
436437
{
437438
QorePythonReleaseGilHelper prgh;
439+
440+
QorePythonStackLocationHelper slh(qph.getOldProgram());
441+
438442
rv = obj->evalMethod(*m, *qargs, &xsink);
439443
}
440444
if (!xsink) {
@@ -491,6 +495,9 @@ PyObject* PythonQoreClass::exec_qore_static_method(const QoreMethod& m, PyObject
491495
ValueHolder rv(&xsink);
492496
{
493497
QorePythonReleaseGilHelper prgh;
498+
499+
QorePythonStackLocationHelper slh(qph.getOldProgram());
500+
494501
rv = QoreObject::evalStaticMethod(m, m.getClass(), *qargs, &xsink);
495502
}
496503
if (!xsink) {
@@ -563,6 +570,9 @@ int PythonQoreClass::py_init(PyObject* self, PyObject* args, PyObject* kwds) {
563570
QoreExternalProgramContextHelper pch(&xsink, qore_python_pgm->getQoreProgram());
564571
if (!xsink) {
565572
QorePythonReleaseGilHelper prgh;
573+
574+
QorePythonStackLocationHelper slh(qore_python_pgm);
575+
566576
ReferenceHolder<QoreObject> qobj(constructor_cls->execConstructor(*qcls, *qargs, true, &xsink),
567577
&xsink);
568578
if (!xsink) {
@@ -631,6 +641,9 @@ PyObject* PythonQoreClass::py_getattro(PyObject* self, PyObject* attr) {
631641
ValueHolder v(&xsink);
632642
{
633643
QorePythonReleaseGilHelper prgh;
644+
645+
QorePythonStackLocationHelper slh(qore_python_pgm);
646+
634647
v = obj->evalMember(member, &xsink);
635648
}
636649
printd(5, "PythonQoreClass::py_getattro() obj %p %s.%s = %s\n", obj, qcls->getName(), member, v->getFullTypeName());

src/QC_PythonProgram.qpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
66
Qore Programming Language
77
8-
Copyright 2020 - 2021 Qore Technologies, s.r.o.
8+
Copyright 2020 - 2022 Qore Technologies, s.r.o.
99
1010
This library is free software; you can redistribute it and/or
1111
modify it under the terms of the GNU Lesser General Public

src/QorePythonClass.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
55
Qore Programming Language python Module
66
7-
Copyright (C) 2020 - 2021 Qore Technologies, s.r.o.
7+
Copyright (C) 2020 - 2022 Qore Technologies, s.r.o.
88
99
This library is free software; you can redistribute it and/or
1010
modify it under the terms of the GNU Lesser General Public

src/QorePythonProgram.cpp

Lines changed: 21 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
66
Qore Programming Language
77
8-
Copyright 2020 - 2021 Qore Technologies, s.r.o.
8+
Copyright 2020 - 2022 Qore Technologies, s.r.o.
99
1010
This library is free software; you can redistribute it and/or
1111
modify it under the terms of the GNU Lesser General Public
@@ -31,6 +31,7 @@
3131
#include "PythonCallableCallReferenceNode.h"
3232
#include "PythonQoreCallable.h"
3333
#include "ModuleNamespace.h"
34+
#include "QorePythonStackLocationHelper.h"
3435

3536
#include <structmember.h>
3637
#include <frameobject.h>
@@ -75,8 +76,6 @@ QorePythonProgram::py_global_tid_map_t QorePythonProgram::py_global_tid_map;
7576
QoreThreadLock QorePythonProgram::py_thr_lck;
7677
unsigned QorePythonProgram::pgm_count = 0;
7778

78-
QorePythonReferenceHolder QorePythonProgram::extract_tb;
79-
8079
QorePythonProgram::QorePythonProgram() : save_object_callback(nullptr) {
8180
printd(5, "QorePythonProgram::QorePythonProgram() this: %p\n", this);
8281
assert(PyGILState_Check());
@@ -275,16 +274,6 @@ QorePythonProgram* QorePythonProgram::getContext() {
275274
int QorePythonProgram::staticInit() {
276275
PyDateTime_IMPORT;
277276
assert(PyDateTimeAPI);
278-
QorePythonReferenceHolder traceback_mod(PyImport_ImportModule("traceback"));
279-
if (!traceback_mod) {
280-
PyErr_Clear();
281-
return -1;
282-
}
283-
extract_tb = PyObject_GetAttrString(*traceback_mod, "extract_tb");
284-
if (!extract_tb || !PyCallable_Check(*extract_tb)) {
285-
PyErr_Clear();
286-
return -1;
287-
}
288277
return 0;
289278
}
290279

@@ -1973,22 +1962,38 @@ int QorePythonProgram::checkPythonException(ExceptionSink* xsink) {
19731962
QorePythonReferenceHolder code((PyObject*)PyFrame_GetCode(frame));
19741963
int line = PyCode_Addr2Line((PyCodeObject*)*code, PyFrame_GetLasti(frame));
19751964
QorePythonReferenceHolder filename_obj(PyObject_GetAttrString(*code, "co_filename"));
1976-
const char* filename = PyUnicode_AsUTF8(*filename_obj);
1965+
const char* filename;
1966+
Py_INCREF(*filename_obj);
1967+
QorePythonReferenceHolder np_obj(QorePythonStackLocationHelper::normalizePath(*filename_obj));
1968+
if (np_obj) {
1969+
filename = getCString(*np_obj);
1970+
} else {
1971+
filename = PyUnicode_AsUTF8(*filename_obj);
1972+
}
19771973

19781974
if (frame == tb->tb_frame) {
19791975
loc.set(filename, line, line, nullptr, 0, QORE_PYTHON_LANG_NAME);
19801976
} else {
19811977
callstack.add(CT_USER, filename, line, line, funcname, QORE_PYTHON_LANG_NAME);
19821978
}
1983-
QorePythonReferenceHolder funcname_obj(PyObject_GetAttrString(*code, "co_name"));
1979+
QorePythonReferenceHolder funcname_obj(PyObject_GetAttrString(*code, "co_qualname"));
19841980
funcname = PyUnicode_AsUTF8(*funcname_obj);
19851981

19861982
frame = PyFrame_GetBack(frame);
19871983
}
19881984
#else
19891985
while (frame) {
19901986
int line = PyCode_Addr2Line(frame->f_code, frame->f_lasti);
1991-
const char* filename = getCString(frame->f_code->co_filename);
1987+
const char* filename;
1988+
Py_INCREF(frame->f_code->co_filename);
1989+
QorePythonReferenceHolder np_obj(QorePythonStackLocationHelper::normalizePath(
1990+
frame->f_code->co_filename
1991+
));
1992+
if (np_obj) {
1993+
filename = getCString(*np_obj);
1994+
} else {
1995+
filename = getCString(frame->f_code->co_filename);
1996+
}
19921997

19931998
if (frame == tb->tb_frame) {
19941999
loc.set(filename, line, line, nullptr, 0, QORE_PYTHON_LANG_NAME);

src/QorePythonProgram.h

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
55
Qore Programming Language
66
7-
Copyright (C) 2020 - 2021 Qore Technologies, s.r.o.
7+
Copyright (C) 2020 - 2022 Qore Technologies, s.r.o.
88
99
Permission is hereby granted, free of charge, to any person obtaining a
1010
copy of this software and associated documentation files (the "Software"),
@@ -463,9 +463,6 @@ class QorePythonProgram : public AbstractQoreProgramExternalData {
463463
//! number of program objects; writable only in the py_thr_lck lock
464464
DLLLOCAL static unsigned pgm_count;
465465

466-
//! traceback.extract_tb() method
467-
DLLLOCAL static QorePythonReferenceHolder extract_tb;
468-
469466
// for local program thread management
470467
mutable int pgm_thr_cnt = 0;
471468
mutable int pgm_thr_waiting = 0;
Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
/* -*- mode: c++; indent-tabs-mode: nil -*- */
2+
/** @file QorePythonProgram.cpp defines the QorePythonProgram class */
3+
/*
4+
QorePythonStackLocationHelper.cpp
5+
6+
Qore Programming Language
7+
8+
Copyright 2020 - 2022 Qore Technologies, s.r.o.
9+
10+
This library is free software; you can redistribute it and/or
11+
modify it under the terms of the GNU Lesser General Public
12+
License as published by the Free Software Foundation; either
13+
version 2.1 of the License, or (at your option) any later version.
14+
15+
This library is distributed in the hope that it will be useful,
16+
but WITHOUT ANY WARRANTY; without even the implied warranty of
17+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18+
Lesser General Public License for more details.
19+
20+
You should have received a copy of the GNU Lesser General Public
21+
License along with this library; if not, write to the Free Software
22+
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
23+
*/
24+
25+
#include "QorePythonStackLocationHelper.h"
26+
#include "QorePythonProgram.h"
27+
28+
#include <frameobject.h>
29+
30+
std::string QorePythonStackLocationHelper::python_no_call_name = "<python_module_no_runtime_stack_info>";
31+
QoreExternalProgramLocationWrapper QorePythonStackLocationHelper::python_loc_builtin("<python_module_unknown>", -1,
32+
-1);
33+
34+
QorePythonReferenceHolder QorePythonStackLocationHelper::_getframe;
35+
QorePythonReferenceHolder QorePythonStackLocationHelper::normpath;
36+
37+
int QorePythonStackLocationHelper::staticInit() {
38+
{
39+
QorePythonReferenceHolder sys(PyImport_ImportModule("sys"));
40+
if (!sys) {
41+
PyErr_Clear();
42+
return -1;
43+
}
44+
_getframe = PyObject_GetAttrString(*sys, "_getframe");
45+
if (!_getframe || !PyCallable_Check(*_getframe)) {
46+
PyErr_Clear();
47+
return -1;
48+
}
49+
}
50+
{
51+
QorePythonReferenceHolder path(PyImport_ImportModule("os.path"));
52+
if (!path) {
53+
PyErr_Clear();
54+
return -1;
55+
}
56+
normpath = PyObject_GetAttrString(*path, "normpath");
57+
if (!normpath || !PyCallable_Check(*normpath)) {
58+
PyErr_Clear();
59+
return -1;
60+
}
61+
}
62+
return 0;
63+
}
64+
65+
QorePythonStackLocationHelper::QorePythonStackLocationHelper(QorePythonProgram* py_pgm) : py_pgm(py_pgm) {
66+
}
67+
68+
const std::string& QorePythonStackLocationHelper::getCallName() const {
69+
if (tid != q_gettid()) {
70+
return python_no_call_name;
71+
}
72+
checkInit();
73+
assert((unsigned)current < size());
74+
//printd(5, "QorePythonStackLocationHelper::getCallName() this: %p %d/%d '%s'\n", this, (int)current, (int)size,
75+
// stack_call[current].c_str());
76+
return stack_call[current];
77+
}
78+
79+
qore_call_t QorePythonStackLocationHelper::getCallType() const {
80+
if (tid != q_gettid()) {
81+
return CT_BUILTIN;
82+
}
83+
checkInit();
84+
assert((unsigned)current < size());
85+
return CT_USER;
86+
}
87+
88+
const QoreProgramLocation& QorePythonStackLocationHelper::getLocation() const {
89+
if (tid != q_gettid()) {
90+
return python_loc_builtin.get();
91+
}
92+
checkInit();
93+
assert((unsigned)current < size());
94+
//printd(5, "QorePythonStackLocationHelper::getLocation() %s:%d (%s)\n", stack_loc[current].getFile(), stack_loc[current].getStartLine());
95+
return stack_loc[current].get();
96+
}
97+
98+
const QoreStackLocation* QorePythonStackLocationHelper::getNext() const {
99+
if (tid != q_gettid()) {
100+
return stack_next;
101+
}
102+
checkInit();
103+
assert((unsigned)current < size());
104+
// issue #3169: reset the pointer after iterating all the information in the stack
105+
// the exception stack can be iterated multiple times
106+
++current;
107+
if ((unsigned)current < size()) {
108+
return this;
109+
}
110+
current = 0;
111+
return stack_next;
112+
}
113+
114+
void QorePythonStackLocationHelper::checkInit() const {
115+
assert(tid == q_gettid());
116+
if (init) {
117+
return;
118+
}
119+
init = true;
120+
121+
QorePythonHelper qph(py_pgm);
122+
123+
// start at depth = 1 or the first two entries will be identical
124+
int depth = 1;
125+
while (true) {
126+
QorePythonReferenceHolder args(PyTuple_New(1));
127+
PyTuple_SET_ITEM(*args, 0, PyLong_FromLong(depth));
128+
129+
QorePythonReferenceHolder frame_obj(PyEval_CallObject(*_getframe, *args));
130+
if (PyErr_Occurred()) {
131+
PyErr_Clear();
132+
break;
133+
}
134+
135+
PyFrameObject* frame = reinterpret_cast<PyFrameObject*>(*frame_obj);
136+
137+
const char* filename;
138+
int line;
139+
const char* funcname;
140+
#if PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION > 10
141+
// get line number
142+
QorePythonReferenceHolder code((PyObject*)PyFrame_GetCode(frame));
143+
line = PyCode_Addr2Line((PyCodeObject*)*code, PyFrame_GetLasti(frame));
144+
QorePythonReferenceHolder filename_obj(PyObject_GetAttrString(*code, "co_filename"));
145+
filename = PyUnicode_AsUTF8(*filename_obj);
146+
147+
QorePythonReferenceHolder funcname_obj(PyObject_GetAttrString(*code, "co_qualname"));
148+
funcname = PyUnicode_AsUTF8(*funcname_obj);
149+
#else
150+
line = PyCode_Addr2Line(frame->f_code, frame->f_lasti);
151+
filename = QorePythonProgram::getCString(frame->f_code->co_filename);
152+
funcname = QorePythonProgram::getCString(frame->f_code->co_name);
153+
#endif
154+
// get normalized path
155+
QorePythonReferenceHolder np_obj(normalizePath(filename));
156+
if (!np_obj) {
157+
break;
158+
}
159+
filename = QorePythonProgram::getCString(*np_obj);
160+
161+
QoreExternalProgramLocationWrapper loc(filename, line, line, nullptr, 0, QORE_PYTHON_LANG_NAME);
162+
stack_loc.push_back(loc);
163+
stack_call.push_back(funcname);
164+
++depth;
165+
}
166+
167+
if (!size()) {
168+
stack_call.push_back(python_no_call_name);
169+
stack_loc.push_back(python_loc_builtin);
170+
}
171+
}
172+
173+
PyObject* QorePythonStackLocationHelper::normalizePath(const char* path) {
174+
// normalize path
175+
QorePythonReferenceHolder path_obj(PyUnicode_FromString(path));
176+
return normalizePath(path_obj.release());
177+
}
178+
179+
PyObject* QorePythonStackLocationHelper::normalizePath(PyObject* path_obj) {
180+
// normalize path
181+
QorePythonReferenceHolder normpath_args(PyTuple_New(1));
182+
PyTuple_SET_ITEM(*normpath_args, 0, path_obj);
183+
184+
// get normalized path
185+
QorePythonReferenceHolder np_obj(PyEval_CallObject(*normpath, *normpath_args));
186+
if (PyErr_Occurred()) {
187+
PyErr_Clear();
188+
return nullptr;
189+
}
190+
return np_obj.release();
191+
}

0 commit comments

Comments
 (0)