refs qoretechnologies/qore#4289 fixed additional namespace clashes and namespace allocation issues for imported Python classes in Qore (#53)

refs qoretechnologies/qore#4290 do not shut down the Python library on exit, as it will cause a crash in exit handlers in Python modules that require the library to be in place (ex: h5py module v3.3.0)

Author: davidnich

CMakeLists.txt

@@ -3,7 +3,7 @@ project(qore-python-module)
 
 set (VERSION_MAJOR 1)
 set (VERSION_MINOR 0)
-set (VERSION_PATCH 2)
+set (VERSION_PATCH 3)
 
 set(PROJECT_VERSION "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}")
 

docs/mainpage.dox.tmpl

@@ -500,6 +500,13 @@ PythonProgram::setSaveObjectCallback(callback);
 
     @section pythonreleasenotes python Module Release Notes
 
+    @subsection python_1_0_3 python Module Version 1.0.3
+    - do not shut down the %Python library on exit, as it will cause a crash in exit handlers in %Python modules that
+      require the library to be in place (ex: h5py module v 3.3.0)
+      (<a href="https://github.com/qorelanguage/qore/issues/4290">issue 4290</a>)
+    - fixed additional namespace clashes and namespace allocation issues for imported %Python classes in %Qore
+      (<a href="https://github.com/qorelanguage/qore/issues/4289">issue 4289</a>)
+
     @subsection python_1_0_2 python Module Version 1.0.2
     - fixed issues handling the case when the python module is initialized with no Program context
       (<a href="https://github.com/qorelanguage/qore/issues/4153">issue 4153</a>)

src/QorePythonProgram.cpp

@@ -593,8 +593,8 @@ QorePythonThreadInfo QorePythonProgram::setContext() const {
     if (!python) {
         python = PyThreadState_New(interpreter);
 
-        //printd(5, "QorePythonProgram::setContext() this: %p created new thread context: %p (py_thr_map: %p " \
-        //  "size: %d)\n", this, python, &py_thr_map, (int)py_thr_map.size());
+        printd(5, "QorePythonProgram::setContext() this: %p created new thread context: %p (py_thr_map: %p "
+            "size: %d)\n", this, python, &py_thr_map, (int)py_thr_map.size());
         assert(python);
         assert(python->gilstate_counter == 1);
         // the thread state will be deleted when the thread terminates or the interpreter is deleted
@@ -1179,7 +1179,8 @@ void QorePythonProgram::exportClass(ExceptionSink* xsink, QoreString& arg) {
         ns = ns->findCreateNamespacePathAll(ns_str.c_str());
     }
 
-    addClassToNamespaceIntern(xsink, ns, (PyTypeObject*)*obj, strpath.back().c_str(), i);
+    strset_t nsset;
+    addClassToNamespaceIntern(xsink, ns, (PyTypeObject*)*obj, strpath.back().c_str(), i, nsset);
 }
 
 void QorePythonProgram::addModulePath(ExceptionSink* xsink, QoreString& arg) {
@@ -2068,7 +2069,8 @@ QoreClass* QorePythonProgram::getCreateQorePythonClass(ExceptionSink* xsink, PyT
         return nullptr;
     }
 
-    return getCreateQorePythonClassIntern(xsink, type, nullptr, flags);
+    strset_t nsset;
+    return getCreateQorePythonClassIntern(xsink, type, nsset, nullptr, flags);
 }
 
 QoreNamespace* QorePythonProgram::getNamespaceForObject(PyObject* obj) {
@@ -2115,7 +2117,7 @@ QoreNamespace* QorePythonProgram::getNamespaceForObject(PyObject* obj) {
 }
 
 QoreClass* QorePythonProgram::getCreateQorePythonClassIntern(ExceptionSink* xsink, PyTypeObject* type,
-        const char* cname, int flags) {
+        strset_t& nsset, const char* cname, int flags) {
     //printd(5, "QorePythonProgram::getCreateQorePythonClassIntern() class: '%s'\n", type->tp_name);
     // see if the Python type already represents a Qore class
     if (PyQoreObjectType_Check(type)) {
@@ -2131,11 +2133,11 @@ QoreClass* QorePythonProgram::getCreateQorePythonClassIntern(ExceptionSink* xsin
     }
 
     // get relative path to class and class name
-    std::string rpath_str;
+    QoreString rpath_str;
     if (!cname) {
         const char* p = strrchr(type->tp_name, '.');
         if (p) {
-            rpath_str = std::string(type->tp_name, p - type->tp_name);
+            rpath_str.set(type->tp_name, p - type->tp_name);
             cname = p + 1;
         } else {
             cname = type->tp_name;
@@ -2144,42 +2146,60 @@ QoreClass* QorePythonProgram::getCreateQorePythonClassIntern(ExceptionSink* xsin
 
     // create new QorePythonClass
     QoreNamespace* ns = getNamespaceForObject(reinterpret_cast<PyObject*>(type));
-    return addClassToNamespaceIntern(xsink, ns, type, cname, i, flags);
+    if (!rpath_str.empty() && rpath_str.startsWith(ns->getName())) {
+        size_t len = strlen(ns->getName());
+        if (rpath_str[len] == '.' && rpath_str.size() > len) {
+            rpath_str.replace(0, len + 1, nullptr);
+            rpath_str.replaceAll(".", "::");
+            ns = ns->findCreateNamespacePathAll(rpath_str.c_str());
+        }
+    }
+    return addClassToNamespaceIntern(xsink, ns, type, cname, i, nsset, flags);
 }
 
 QorePythonClass* QorePythonProgram::addClassToNamespaceIntern(ExceptionSink* xsink, QoreNamespace* ns,
-        PyTypeObject* type, const char* cname, clmap_t::iterator i, int flags) {
+        PyTypeObject* type, const char* cname, clmap_t::iterator i, strset_t& nsset, int flags) {
     // get a unique name for the class
     QoreString cname_str = cname;
+    // namespace path
+    std::string ns_path = ns->getPath(true);
+    // full path of class with ns
+    std::string full_path;
+    strset_t::iterator nsi;
     {
         int base = 0;
-        while (ns->findLocalClass(cname_str.c_str())) {
+        while (true) {
+            full_path = ns_path + "::" + cname_str.c_str();
+            nsi = nsset.find(full_path);
+            if (nsi == nsset.end() && !ns->findLocalClass(cname_str.c_str())) {
+                break;
+            }
             cname_str.clear();
             cname_str.sprintf("%s_base_%d", cname, base++);
         }
     }
     cname = cname_str.c_str();
 
     // create new class
-    std::string path = ns->getPath(true);
-    path += "::";
-    path += cname;
-    std::unique_ptr<QorePythonClass> cls(new QorePythonClass(this, cname, path.c_str()));
+    std::unique_ptr<QorePythonClass> cls(new QorePythonClass(this, cname, full_path.c_str()));
+
+    // add to nsset
+    nsset.insert(nsi, full_path);
 
     // insert into map
     clmap.insert(i, clmap_t::value_type(type, cls.get()));
 
     //printd(5, "QorePythonProgram::addClassToNamespaceIntern() ns: '%s' cls: '%s' (%s id: %d)\n", ns->getName(),
     //  cls->getName(), type->tp_name, cls->getID());
 
-    return setupQorePythonClass(xsink, ns, type, cls, flags);
+    return setupQorePythonClass(xsink, ns, type, cls, nsset, flags);
 }
 
 static constexpr int static_meth_flags = QCF_USES_EXTRA_ARGS;
 static constexpr int normal_meth_flags = static_meth_flags | QCF_ABSTRACT_OVERRIDE_ALL;
 
 QorePythonClass* QorePythonProgram::setupQorePythonClass(ExceptionSink* xsink, QoreNamespace* ns, PyTypeObject* type,
-        std::unique_ptr<QorePythonClass>& cls, int flags) {
+        std::unique_ptr<QorePythonClass>& cls, strset_t& nsset, int flags) {
     //printd(5, "QorePythonProgram::setupQorePythonClass() ns: '%s' cls: '%s' (%s) flags: %d\n", ns->getName(),
     //  cls->getName(), type->tp_name, flags);
     cls->addConstructor((void*)type, (q_external_constructor_t)execPythonConstructor, Public,
@@ -2193,7 +2213,7 @@ QorePythonClass* QorePythonProgram::setupQorePythonClass(ExceptionSink* xsink, Q
 
     // add single base class
     if (type->tp_base) {
-        QoreClass* bclass = getCreateQorePythonClassIntern(xsink, type->tp_base);
+        QoreClass* bclass = getCreateQorePythonClassIntern(xsink, type->tp_base, nsset);
         if (!bclass) {
             assert(*xsink);
             return nullptr;
@@ -2737,7 +2757,8 @@ int QorePythonProgram::importSymbol(ExceptionSink* xsink, PyObject* value, const
     if (PyType_Check(value)) {
         //printd(5, "QorePythonProgram::importSymbol() class sym: '%s' -> '%s' (%p)\n", symbol,
         //  reinterpret_cast<PyTypeObject*>(value)->tp_name, value);
-        getCreateQorePythonClassIntern(xsink, reinterpret_cast<PyTypeObject*>(value));
+        strset_t nsset;
+        getCreateQorePythonClassIntern(xsink, reinterpret_cast<PyTypeObject*>(value), nsset);
         if (*xsink) {
             return -1;
         }

src/QorePythonProgram.h

@@ -63,6 +63,8 @@ struct QorePythonThreadStateInfo {
 class QorePythonProgram : public AbstractQoreProgramExternalData {
     friend class PythonModuleContextHelper;
 public:
+    typedef std::set<std::string> strset_t;
+
     //! Python context using the main interpreter
     DLLLOCAL QorePythonProgram();
 
@@ -289,7 +291,7 @@ class QorePythonProgram : public AbstractQoreProgramExternalData {
 
     //! Populates the QoreClass based on the Python class
     DLLLOCAL QorePythonClass* setupQorePythonClass(ExceptionSink* xsink, QoreNamespace* ns, PyTypeObject* type,
-            std::unique_ptr<QorePythonClass>& cls, int flags = 0);
+            std::unique_ptr<QorePythonClass>& cls, strset_t& nsset, int flags = 0);
 
     //! Creates ot retrieves a QoreClass for the given Python type
     DLLLOCAL QoreClass* getCreateQorePythonClass(ExceptionSink* xsink, PyTypeObject* type, int flags = 0);
@@ -429,7 +431,6 @@ class QorePythonProgram : public AbstractQoreProgramExternalData {
     pyobj_set_t mod_set;
 
     //! set of unique strings
-    typedef std::set<std::string> strset_t;
     strset_t strset;
 
     //! mutex for thread state map
@@ -474,11 +475,11 @@ class QorePythonProgram : public AbstractQoreProgramExternalData {
 
     DLLLOCAL QoreNamespace* getNamespaceForObject(PyObject* type);
 
-    DLLLOCAL QoreClass* getCreateQorePythonClassIntern(ExceptionSink* xsink, PyTypeObject* type,
+    DLLLOCAL QoreClass* getCreateQorePythonClassIntern(ExceptionSink* xsink, PyTypeObject* type, strset_t& nsset,
         const char* cls_name = nullptr, int flags = 0);
 
     DLLLOCAL QorePythonClass* addClassToNamespaceIntern(ExceptionSink* xsink, QoreNamespace* ns, PyTypeObject* type,
-        const char* cname, clmap_t::iterator i, int flags = 0);
+        const char* cname, clmap_t::iterator i, strset_t& nsset, int flags = 0);
 
     //! Call a method and and return the result
     DLLLOCAL QoreValue callCFunctionMethod(ExceptionSink* xsink, PyObject* func, const QoreListNode* args,

src/python-module.cpp

@@ -233,7 +233,12 @@ static QoreStringNode* python_module_init_intern(bool repeat) {
         }
 
         Py_InitializeEx(0);
+#ifndef QORE_ALLOW_PYTHON_SHUTDOWN
+        // issue# 4290: if we actively shut down Python on exit, then exit handlers in modules
+        // (such as the h5py module in version 3.3.0) will cause a crash when the process exits,
+        // as it requires the Python library to be still in place and initialized
         python_needs_shutdown = true;
+#endif
         //printd(5, "python_module_init() Python initialized\n");
     }