From eb13ae20ef14d6bb694dea13b46d82fe947c69e7 Mon Sep 17 00:00:00 2001 From: "Robert A. McDougal" Date: Mon, 20 Jun 2022 19:50:42 -0400 Subject: [PATCH 01/30] hoc.Vector distinct from hoc.HocObject --- src/nrnpython/nrnpy_hoc.cpp | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/src/nrnpython/nrnpy_hoc.cpp b/src/nrnpython/nrnpy_hoc.cpp index be5cbafb42..e0acfc6ed8 100644 --- a/src/nrnpython/nrnpy_hoc.cpp +++ b/src/nrnpython/nrnpy_hoc.cpp @@ -157,6 +157,8 @@ static PyObject* get_mech_object_ = NULL; static PyObject* nrnpy_rvp_pyobj_callback = NULL; PyTypeObject* hocobject_type; +PyTypeObject* vectorobject_type; + static PyObject* hocobj_call(PyHocObject* self, PyObject* args, PyObject* kwrds); static PyObject* nrnexec(PyObject* self, PyObject* args) { @@ -503,15 +505,24 @@ PyObject* nrnpy_ho2po(Object* o) { // The return value is None, or the encapsulated PyObject or // an encapsulating PyHocObject PyObject* po; + //printf("inside ho2po\n"); if (!o) { po = Py_BuildValue(""); + //printf("empty po\n"); } else if (o->ctemplate->sym == nrnpy_pyobj_sym_) { po = nrnpy_hoc2pyobject(o); Py_INCREF(po); + //printf("from nrnpy_hoc2pyobject\n"); } else { po = hocobj_new(hocobject_type, 0, 0); ((PyHocObject*) po)->ho_ = o; ((PyHocObject*) po)->type_ = PyHoc::HocObject; + if (o->ctemplate->sym == hoc_vec_template_->sym) { + //printf("syms match\n"); + po->ob_type = vectorobject_type; + } else { + //printf("syms don't match\n"); + } hoc_obj_ref(o); } return po; @@ -676,6 +687,16 @@ static void* fcall(void* vself, void* vargs) { PyHocObject* result = (PyHocObject*) hocobj_new(hocobject_type, 0, 0); result->ho_ = ho; result->type_ = PyHoc::HocObject; + // Note: I think the only reason we're not using ho2po here is because we don't have to + // hocref ho since it was created by hoc_newobj1... but it would be better if we did + // so we could avoid repetitive code + if (ho->ctemplate->sym == hoc_vec_template_->sym) { + //printf("TEMPLATE: syms match\n"); + ((PyObject*) result)->ob_type = vectorobject_type; + } else { + //printf("TEMPLATE: syms don't match\n"); + } + hocobj_pushargs_free_strings(strings_to_free); return result; } else { @@ -3106,9 +3127,18 @@ PyObject* nrnpy_hoc() { if (PyType_Ready(hocobject_type) < 0) goto fail; Py_INCREF(hocobject_type); + //printf("defining vectorobject_type\n"); + vectorobject_type = (PyTypeObject*) PyType_FromSpecWithBases(&nrnpy_HocObjectType_spec, NULL);//(PyObject*) hocobject_type); + if (PyType_Ready(vectorobject_type) < 0) + goto fail; + Py_INCREF(vectorobject_type); + //printf("done\n"); // printf("AddObject HocObject\n"); PyModule_AddObject(m, "HocObject", (PyObject*) hocobject_type); - + //printf("registering Vector\n"); + PyModule_AddObject(m, "Vector", (PyObject*) vectorobject_type); + //printf("done\n"); + topmethdict = PyDict_New(); for (PyMethodDef* meth = toplevel_methods; meth->ml_name != NULL; meth++) { PyObject* descr; From c0eee1eb45cd1f79ee6a28b92bb255160361635c Mon Sep 17 00:00:00 2001 From: "Robert A. McDougal" Date: Mon, 20 Jun 2022 20:45:03 -0400 Subject: [PATCH 02/30] hoc.Vector is a subclass of hoc.HocObject --- src/nrnpython/nrnpy_hoc.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/nrnpython/nrnpy_hoc.cpp b/src/nrnpython/nrnpy_hoc.cpp index e0acfc6ed8..a5a7e61b3d 100644 --- a/src/nrnpython/nrnpy_hoc.cpp +++ b/src/nrnpython/nrnpy_hoc.cpp @@ -3097,6 +3097,7 @@ static char* nrncore_arg(double tstop) { PyObject* nrnpy_hoc() { PyObject* m; + PyObject* bases; nrnpy_vec_from_python_p_ = nrnpy_vec_from_python; nrnpy_vec_to_python_p_ = nrnpy_vec_to_python; nrnpy_vec_as_numpy_helper_ = vec_as_numpy_helper; @@ -3128,7 +3129,10 @@ PyObject* nrnpy_hoc() { goto fail; Py_INCREF(hocobject_type); //printf("defining vectorobject_type\n"); - vectorobject_type = (PyTypeObject*) PyType_FromSpecWithBases(&nrnpy_HocObjectType_spec, NULL);//(PyObject*) hocobject_type); + bases = PyTuple_Pack(1, hocobject_type); + Py_INCREF(bases); + vectorobject_type = (PyTypeObject*) PyType_FromSpecWithBases(&nrnpy_HocObjectType_spec, bases); + Py_DECREF(bases); if (PyType_Ready(vectorobject_type) < 0) goto fail; Py_INCREF(vectorobject_type); From 944bcbc0b0d57cc552df406b0a2b647109553f9b Mon Sep 17 00:00:00 2001 From: "Robert A. McDougal" Date: Mon, 20 Jun 2022 21:32:26 -0400 Subject: [PATCH 03/30] generating specs dynamically allows for different names --- src/nrnpython/nrnpy_hoc.cpp | 18 ++++++++++++++++-- src/nrnpython/nrnpy_hoc.h | 7 ------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/nrnpython/nrnpy_hoc.cpp b/src/nrnpython/nrnpy_hoc.cpp index a5a7e61b3d..d7f6c4cb32 100644 --- a/src/nrnpython/nrnpy_hoc.cpp +++ b/src/nrnpython/nrnpy_hoc.cpp @@ -3095,9 +3095,21 @@ static char* nrncore_arg(double tstop) { return NULL; } + +static PyType_Spec obj_spec_from_name(const char* name) { + return { + name, + sizeof(PyHocObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + nrnpy_HocObjectType_slots, + }; +} + PyObject* nrnpy_hoc() { PyObject* m; PyObject* bases; + PyType_Spec spec; nrnpy_vec_from_python_p_ = nrnpy_vec_from_python; nrnpy_vec_to_python_p_ = nrnpy_vec_to_python; nrnpy_vec_as_numpy_helper_ = vec_as_numpy_helper; @@ -3124,14 +3136,16 @@ PyObject* nrnpy_hoc() { m = PyModule_Create(&hocmodule); assert(m); Symbol* s = NULL; - hocobject_type = (PyTypeObject*) PyType_FromSpec(&nrnpy_HocObjectType_spec); + spec = obj_spec_from_name("hoc.HocObject"); + hocobject_type = (PyTypeObject*) PyType_FromSpec(&spec); if (PyType_Ready(hocobject_type) < 0) goto fail; Py_INCREF(hocobject_type); //printf("defining vectorobject_type\n"); + spec = obj_spec_from_name("hoc.Vector"); bases = PyTuple_Pack(1, hocobject_type); Py_INCREF(bases); - vectorobject_type = (PyTypeObject*) PyType_FromSpecWithBases(&nrnpy_HocObjectType_spec, bases); + vectorobject_type = (PyTypeObject*) PyType_FromSpecWithBases(&spec, bases); Py_DECREF(bases); if (PyType_Ready(vectorobject_type) < 0) goto fail; diff --git a/src/nrnpython/nrnpy_hoc.h b/src/nrnpython/nrnpy_hoc.h index 547bd3cf78..7313cf8cad 100644 --- a/src/nrnpython/nrnpy_hoc.h +++ b/src/nrnpython/nrnpy_hoc.h @@ -27,13 +27,6 @@ static PyType_Slot nrnpy_HocObjectType_slots[] = { {Py_nb_true_divide, (PyObject*) py_hocobj_div}, {0, 0}, }; -static PyType_Spec nrnpy_HocObjectType_spec = { - "hoc.HocObject", - sizeof(PyHocObject), - 0, - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, - nrnpy_HocObjectType_slots, -}; static struct PyModuleDef hocmodule = {PyModuleDef_HEAD_INIT, From 381e6b11f808e55bebf4dea26d83c00d65afed03 Mon Sep 17 00:00:00 2001 From: ramcdougal Date: Tue, 21 Jun 2022 10:57:34 -0400 Subject: [PATCH 04/30] check result checks type --- test/cover/checkresult.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/cover/checkresult.py b/test/cover/checkresult.py index b73c1948cd..a93871f890 100644 --- a/test/cover/checkresult.py +++ b/test/cover/checkresult.py @@ -1,6 +1,6 @@ import json import math -from neuron import h +from neuron import h, hoc class Chk: @@ -39,7 +39,7 @@ def __call__(self, key, value, tol=0.0): """ if key in self.d: - if type(value) == type(h.Vector): # actually hoc.HocObject + if isinstance(value, hoc.Vector): # Convert to list to keep the `equal` method below simple value = list(value) # Hand-rolled comparison that uses `tol` for arithmetic values From 146643d4f32fd6a7324160336ca1280721a44ce5 Mon Sep 17 00:00:00 2001 From: "Robert A. McDougal" Date: Tue, 21 Jun 2022 12:04:54 -0400 Subject: [PATCH 05/30] h.Vector returns hoc.Vector; hclass not needed anymore --- src/nrnpython/nrnpy_hoc.cpp | 75 +++++++++++++++++++++++++------------ 1 file changed, 51 insertions(+), 24 deletions(-) diff --git a/src/nrnpython/nrnpy_hoc.cpp b/src/nrnpython/nrnpy_hoc.cpp index d7f6c4cb32..15b0989ccb 100644 --- a/src/nrnpython/nrnpy_hoc.cpp +++ b/src/nrnpython/nrnpy_hoc.cpp @@ -208,8 +208,12 @@ static void hocobj_dealloc(PyHocObject* self) { static PyObject* hocobj_new(PyTypeObject* subtype, PyObject* args, PyObject* kwds) { PyObject* subself; + PyObject* base; + PyHocObject* hbase; + bool has_base = false; + bool ok = true; subself = subtype->tp_alloc(subtype, 0); - // printf("hocobj_new %s %p\n", subtype->tp_name, subself); + //printf("hocobj_new %s %p %p\n", subtype->tp_name, subtype, subself); if (subself == NULL) { return NULL; } @@ -221,35 +225,49 @@ static PyObject* hocobj_new(PyTypeObject* subtype, PyObject* args, PyObject* kwd self->nindex_ = 0; self->type_ = PyHoc::HocTopLevelInterpreter; self->iteritem_ = 0; + if (PyObject_IsSubclass((PyObject*) subtype, (PyObject*) vectorobject_type)) { + //printf("subtype is vectorobject_type\n"); + PyObject* result = hocobj_new(hocobject_type, 0, 0); + hbase = (PyHocObject*) result; + hbase->type_ = PyHoc::HocFunction; + hbase->sym_ = hoc_vec_template_->sym; + has_base = true; + } else { + //printf("not vectorobject_type\n"); + } if (kwds && PyDict_Check(kwds)) { - PyObject* base = PyDict_GetItemString(kwds, "hocbase"); + base = PyDict_GetItemString(kwds, "hocbase"); if (base) { - int ok = 0; + ok = false; if (PyObject_TypeCheck(base, hocobject_type)) { - PyHocObject* hbase = (PyHocObject*) base; - if (hbase->type_ == PyHoc::HocFunction && hbase->sym_->type == TEMPLATE) { - // printf("hocobj_new base %s\n", hbase->sym_->name); - // remove the hocbase keyword since hocobj_call only allows - // the "sec" keyword argument - PyDict_DelItemString(kwds, "hocbase"); - PyObject* r = hocobj_call(hbase, args, kwds); - if (!r) { - Py_DECREF(subself); - return NULL; - } - PyHocObject* rh = (PyHocObject*) r; - self->type_ = rh->type_; - self->ho_ = rh->ho_; - hoc_obj_ref(self->ho_); - Py_DECREF(r); - ok = 1; - } + hbase = (PyHocObject*) base; + has_base = true; } - if (!ok) { + // TODO: is it a problem doing this before we're done with the base + PyDict_DelItemString(kwds, "hocbase"); + } + } + if (has_base) { + if (hbase->type_ == PyHoc::HocFunction && hbase->sym_->type == TEMPLATE) { + // printf("hocobj_new base %s\n", hbase->sym_->name); + // remove the hocbase keyword since hocobj_call only allows + // the "sec" keyword argument + PyObject* r = hocobj_call(hbase, args, kwds); + if (!r) { Py_DECREF(subself); - PyErr_SetString(PyExc_TypeError, "HOC base class not valid"); return NULL; } + PyHocObject* rh = (PyHocObject*) r; + self->type_ = rh->type_; + self->ho_ = rh->ho_; + hoc_obj_ref(self->ho_); + Py_DECREF(r); + ok = 1; + } + if (!ok) { + Py_DECREF(subself); + PyErr_SetString(PyExc_TypeError, "HOC base class not valid"); + return NULL; } } return subself; @@ -683,6 +701,7 @@ static void* fcall(void* vself, void* vargs) { double d = hoc_call_func(self->sym_, 1); hoc_pushx(d); } else if (self->sym_->type == TEMPLATE) { + //printf("in fcall TEMPLATE case\n"); Object* ho = hoc_newobj1(self->sym_, narg); PyHocObject* result = (PyHocObject*) hocobj_new(hocobject_type, 0, 0); result->ho_ = ho; @@ -978,6 +997,13 @@ static PyObject* hocobj_getattr(PyObject* subself, PyObject* pyname) { return NULL; } + // TODO: this better + if (strcmp(n, "Vector") == 0 && self->type_ == PyHoc::HocTopLevelInterpreter) { + //printf("inside vector case\n"); + Py_INCREF(vectorobject_type); + return (PyObject*) vectorobject_type; + } + Symbol* sym = getsym(n, self->ho_, 0); if (!sym) { if (self->type_ == PyHoc::HocObject && self->ho_->ctemplate->sym == nrnpy_pyobj_sym_) { @@ -1257,13 +1283,14 @@ static PyObject* hocobj_getattr(PyObject* subself, PyObject* pyname) { result = (PyObject*) intermediate(self, sym, -1); } break; + case TEMPLATE: + printf("template stuff\n"); case PROCEDURE: case FUNCTION: case FUN_BLTIN: case BLTIN: case HOCOBJFUNCTION: case STRINGFUNC: - case TEMPLATE: case OBJECTFUNC: { result = hocobj_new(hocobject_type, 0, 0); PyHocObject* po = (PyHocObject*) result; From 476e9c66076e550fcc80c06722b09c63144af74f Mon Sep 17 00:00:00 2001 From: Fernando Pereira Date: Tue, 21 Jun 2022 19:39:26 +0200 Subject: [PATCH 06/30] Changing type lookup by string to symbol pointer --- src/nrnpython/nrnpy_hoc.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/nrnpython/nrnpy_hoc.cpp b/src/nrnpython/nrnpy_hoc.cpp index 15b0989ccb..0191a34d9c 100644 --- a/src/nrnpython/nrnpy_hoc.cpp +++ b/src/nrnpython/nrnpy_hoc.cpp @@ -11,6 +11,8 @@ #include "nrnpy_utils.h" #include "../nrniv/shapeplt.h" #include +#include + #include "nrnwrap_dlfcn.h" #if defined(NRNPYTHON_DYNAMICLOAD) && NRNPYTHON_DYNAMICLOAD > 0 @@ -997,14 +999,13 @@ static PyObject* hocobj_getattr(PyObject* subself, PyObject* pyname) { return NULL; } - // TODO: this better - if (strcmp(n, "Vector") == 0 && self->type_ == PyHoc::HocTopLevelInterpreter) { - //printf("inside vector case\n"); + Symbol* sym = getsym(n, self->ho_, 0); + // Return well known types right away + if (sym == hoc_vec_template_->sym) { Py_INCREF(vectorobject_type); return (PyObject*) vectorobject_type; } - Symbol* sym = getsym(n, self->ho_, 0); if (!sym) { if (self->type_ == PyHoc::HocObject && self->ho_->ctemplate->sym == nrnpy_pyobj_sym_) { PyObject* p = nrnpy_hoc2pyobject(self->ho_); @@ -1283,14 +1284,13 @@ static PyObject* hocobj_getattr(PyObject* subself, PyObject* pyname) { result = (PyObject*) intermediate(self, sym, -1); } break; - case TEMPLATE: - printf("template stuff\n"); case PROCEDURE: case FUNCTION: case FUN_BLTIN: case BLTIN: case HOCOBJFUNCTION: case STRINGFUNC: + case TEMPLATE: case OBJECTFUNC: { result = hocobj_new(hocobject_type, 0, 0); PyHocObject* po = (PyHocObject*) result; From 3958e58441ecc55ccdd8aba87e370ccb4387afa1 Mon Sep 17 00:00:00 2001 From: "Robert A. McDougal" Date: Tue, 21 Jun 2022 14:22:42 -0400 Subject: [PATCH 07/30] generating all HOC classes --- src/nrnpython/nrnpy_hoc.cpp | 33 ++++++++++++++++++++++++++------- src/oc/hoc_oop.cpp | 4 ++++ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/nrnpython/nrnpy_hoc.cpp b/src/nrnpython/nrnpy_hoc.cpp index 0191a34d9c..512f10ea1a 100644 --- a/src/nrnpython/nrnpy_hoc.cpp +++ b/src/nrnpython/nrnpy_hoc.cpp @@ -103,6 +103,9 @@ static cTemplate* hoc_vec_template_; static cTemplate* hoc_list_template_; static cTemplate* hoc_sectionlist_template_; +static std::vector class_name_list; +static std::unordered_map sym_to_type_map; + // typestr returned by Vector.__array_interface__ // byteorder (first element) is modified at import time // to reflect the system byteorder @@ -163,6 +166,11 @@ PyTypeObject* vectorobject_type; static PyObject* hocobj_call(PyHocObject* self, PyObject* args, PyObject* kwrds); +void nrnpy_register_class(const char* name) { + class_name_list.push_back(name); + printf("declared %s\n", name); +} + static PyObject* nrnexec(PyObject* self, PyObject* args) { const char* cmd; if (!PyArg_ParseTuple(args, "s", &cmd)) { @@ -3136,6 +3144,7 @@ static PyType_Spec obj_spec_from_name(const char* name) { PyObject* nrnpy_hoc() { PyObject* m; PyObject* bases; + PyTypeObject* pto; PyType_Spec spec; nrnpy_vec_from_python_p_ = nrnpy_vec_from_python; nrnpy_vec_to_python_p_ = nrnpy_vec_to_python; @@ -3168,20 +3177,30 @@ PyObject* nrnpy_hoc() { if (PyType_Ready(hocobject_type) < 0) goto fail; Py_INCREF(hocobject_type); + PyModule_AddObject(m, "HocObject", (PyObject*) hocobject_type); + + printf("constructing type objects\n"); //printf("defining vectorobject_type\n"); - spec = obj_spec_from_name("hoc.Vector"); bases = PyTuple_Pack(1, hocobject_type); Py_INCREF(bases); - vectorobject_type = (PyTypeObject*) PyType_FromSpecWithBases(&spec, bases); + for (auto name: class_name_list) { + // TODO: obj_spec_from_name needs a hoc. prepended + auto long_name = std::string("hoc.") + name; + spec = obj_spec_from_name(long_name.c_str()); + pto = (PyTypeObject*) PyType_FromSpecWithBases(&spec, bases); + sym_to_type_map[hoc_lookup(name)] = pto; + if (PyType_Ready(pto) < 0) + goto fail; + Py_INCREF(pto); + PyModule_AddObject(m, name, (PyObject*) pto); + } Py_DECREF(bases); - if (PyType_Ready(vectorobject_type) < 0) - goto fail; - Py_INCREF(vectorobject_type); + + vectorobject_type = sym_to_type_map[hoc_lookup("Vector")]; + //printf("done\n"); // printf("AddObject HocObject\n"); - PyModule_AddObject(m, "HocObject", (PyObject*) hocobject_type); //printf("registering Vector\n"); - PyModule_AddObject(m, "Vector", (PyObject*) vectorobject_type); //printf("done\n"); topmethdict = PyDict_New(); diff --git a/src/oc/hoc_oop.cpp b/src/oc/hoc_oop.cpp index 96d6dbf1e6..da5d0aab98 100644 --- a/src/oc/hoc_oop.cpp +++ b/src/oc/hoc_oop.cpp @@ -30,6 +30,7 @@ Symbol* nrnpy_pyobj_sym_; void (*nrnpy_py2n_component)(Object* o, Symbol* s, int nindex, int isfunc); void (*nrnpy_hpoasgn)(Object* o, int type); void* (*nrnpy_opaque_obj2pyobj_p_)(Object*); +extern void nrnpy_register_class(const char* name); #endif #if CABLE @@ -1611,6 +1612,9 @@ void class2oc(const char* name, if (hoc_lookup(name)) { hoc_execerror(name, "already being used as a name"); } + #if USE_PYTHON + nrnpy_register_class(name); + #endif tsym = hoc_install(name, UNDEF, 0.0, &hoc_symlist); tsym->subtype = CPLUSOBJECT; hoc_begintemplate(tsym); From 27481ab44a7f1f70008c0e33987efbcc2162e68e Mon Sep 17 00:00:00 2001 From: "Robert A. McDougal" Date: Tue, 21 Jun 2022 15:14:34 -0400 Subject: [PATCH 08/30] finished generalization to built-in classes --- src/nrnpython/nrnpy_hoc.cpp | 54 +++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/src/nrnpython/nrnpy_hoc.cpp b/src/nrnpython/nrnpy_hoc.cpp index 512f10ea1a..aa59f81035 100644 --- a/src/nrnpython/nrnpy_hoc.cpp +++ b/src/nrnpython/nrnpy_hoc.cpp @@ -105,6 +105,7 @@ static cTemplate* hoc_sectionlist_template_; static std::vector class_name_list; static std::unordered_map sym_to_type_map; +static std::unordered_map type_to_sym_map; // typestr returned by Vector.__array_interface__ // byteorder (first element) is modified at import time @@ -162,13 +163,11 @@ static PyObject* get_mech_object_ = NULL; static PyObject* nrnpy_rvp_pyobj_callback = NULL; PyTypeObject* hocobject_type; -PyTypeObject* vectorobject_type; static PyObject* hocobj_call(PyHocObject* self, PyObject* args, PyObject* kwrds); void nrnpy_register_class(const char* name) { class_name_list.push_back(name); - printf("declared %s\n", name); } static PyObject* nrnexec(PyObject* self, PyObject* args) { @@ -235,6 +234,27 @@ static PyObject* hocobj_new(PyTypeObject* subtype, PyObject* args, PyObject* kwd self->nindex_ = 0; self->type_ = PyHoc::HocTopLevelInterpreter; self->iteritem_ = 0; + + // if subtype is a subclass of some NEURON class, then one of its + // tp_mro's is in sym_to_type_map + + for (Py_ssize_t i = 0; i < PyTuple_Size(subtype->tp_mro); i++) { + PyObject* item = PyTuple_GetItem(subtype->tp_mro, i); + Py_INCREF(item); + auto location = type_to_sym_map.find((PyTypeObject*) item); + if (location != type_to_sym_map.end()) { + PyObject* result = hocobj_new(hocobject_type, 0, 0); + hbase = (PyHocObject*) result; + hbase->type_ = PyHoc::HocFunction; + hbase->sym_ = location->second; //type_to_sym_map[(PyTypeObject*) item]; + has_base = true; + } + Py_DECREF(item); + if (has_base) { + break; + } + } +/* if (PyObject_IsSubclass((PyObject*) subtype, (PyObject*) vectorobject_type)) { //printf("subtype is vectorobject_type\n"); PyObject* result = hocobj_new(hocobject_type, 0, 0); @@ -244,7 +264,7 @@ static PyObject* hocobj_new(PyTypeObject* subtype, PyObject* args, PyObject* kwd has_base = true; } else { //printf("not vectorobject_type\n"); - } + }*/ if (kwds && PyDict_Check(kwds)) { base = PyDict_GetItemString(kwds, "hocbase"); if (base) { @@ -545,11 +565,9 @@ PyObject* nrnpy_ho2po(Object* o) { po = hocobj_new(hocobject_type, 0, 0); ((PyHocObject*) po)->ho_ = o; ((PyHocObject*) po)->type_ = PyHoc::HocObject; - if (o->ctemplate->sym == hoc_vec_template_->sym) { - //printf("syms match\n"); - po->ob_type = vectorobject_type; - } else { - //printf("syms don't match\n"); + auto location = sym_to_type_map.find(o->ctemplate->sym); + if (location != sym_to_type_map.end()) { + po->ob_type = sym_to_type_map[o->ctemplate->sym]; } hoc_obj_ref(o); } @@ -719,11 +737,9 @@ static void* fcall(void* vself, void* vargs) { // Note: I think the only reason we're not using ho2po here is because we don't have to // hocref ho since it was created by hoc_newobj1... but it would be better if we did // so we could avoid repetitive code - if (ho->ctemplate->sym == hoc_vec_template_->sym) { - //printf("TEMPLATE: syms match\n"); - ((PyObject*) result)->ob_type = vectorobject_type; - } else { - //printf("TEMPLATE: syms don't match\n"); + auto location = sym_to_type_map.find(ho->ctemplate->sym); + if (location != sym_to_type_map.end()) { + ((PyObject*) result)->ob_type = location->second; } hocobj_pushargs_free_strings(strings_to_free); @@ -1009,9 +1025,10 @@ static PyObject* hocobj_getattr(PyObject* subself, PyObject* pyname) { Symbol* sym = getsym(n, self->ho_, 0); // Return well known types right away - if (sym == hoc_vec_template_->sym) { - Py_INCREF(vectorobject_type); - return (PyObject*) vectorobject_type; + auto location = sym_to_type_map.find(sym); + if (location != sym_to_type_map.end()) { + Py_INCREF(location->second); + return (PyObject*) location->second; } if (!sym) { @@ -3179,8 +3196,6 @@ PyObject* nrnpy_hoc() { Py_INCREF(hocobject_type); PyModule_AddObject(m, "HocObject", (PyObject*) hocobject_type); - printf("constructing type objects\n"); - //printf("defining vectorobject_type\n"); bases = PyTuple_Pack(1, hocobject_type); Py_INCREF(bases); for (auto name: class_name_list) { @@ -3189,6 +3204,7 @@ PyObject* nrnpy_hoc() { spec = obj_spec_from_name(long_name.c_str()); pto = (PyTypeObject*) PyType_FromSpecWithBases(&spec, bases); sym_to_type_map[hoc_lookup(name)] = pto; + type_to_sym_map[pto] = hoc_lookup(name); if (PyType_Ready(pto) < 0) goto fail; Py_INCREF(pto); @@ -3196,8 +3212,6 @@ PyObject* nrnpy_hoc() { } Py_DECREF(bases); - vectorobject_type = sym_to_type_map[hoc_lookup("Vector")]; - //printf("done\n"); // printf("AddObject HocObject\n"); //printf("registering Vector\n"); From 72b225d276d5bfb546d675b8b634c27e5f9a56d6 Mon Sep 17 00:00:00 2001 From: Robin De Schepper Date: Tue, 21 Jun 2022 22:15:07 +0200 Subject: [PATCH 09/30] addressed review comments --- src/nrnpython/nrnpy_hoc.cpp | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/src/nrnpython/nrnpy_hoc.cpp b/src/nrnpython/nrnpy_hoc.cpp index aa59f81035..ef5207fb23 100644 --- a/src/nrnpython/nrnpy_hoc.cpp +++ b/src/nrnpython/nrnpy_hoc.cpp @@ -246,7 +246,7 @@ static PyObject* hocobj_new(PyTypeObject* subtype, PyObject* args, PyObject* kwd PyObject* result = hocobj_new(hocobject_type, 0, 0); hbase = (PyHocObject*) result; hbase->type_ = PyHoc::HocFunction; - hbase->sym_ = location->second; //type_to_sym_map[(PyTypeObject*) item]; + hbase->sym_ = location->second; has_base = true; } Py_DECREF(item); @@ -254,17 +254,7 @@ static PyObject* hocobj_new(PyTypeObject* subtype, PyObject* args, PyObject* kwd break; } } -/* - if (PyObject_IsSubclass((PyObject*) subtype, (PyObject*) vectorobject_type)) { - //printf("subtype is vectorobject_type\n"); - PyObject* result = hocobj_new(hocobject_type, 0, 0); - hbase = (PyHocObject*) result; - hbase->type_ = PyHoc::HocFunction; - hbase->sym_ = hoc_vec_template_->sym; - has_base = true; - } else { - //printf("not vectorobject_type\n"); - }*/ + if (kwds && PyDict_Check(kwds)) { base = PyDict_GetItemString(kwds, "hocbase"); if (base) { @@ -285,6 +275,7 @@ static PyObject* hocobj_new(PyTypeObject* subtype, PyObject* args, PyObject* kwd PyObject* r = hocobj_call(hbase, args, kwds); if (!r) { Py_DECREF(subself); + PyErr_SetString(PyExc_TypeError, "HOC base class not valid"); return NULL; } PyHocObject* rh = (PyHocObject*) r; @@ -292,7 +283,7 @@ static PyObject* hocobj_new(PyTypeObject* subtype, PyObject* args, PyObject* kwd self->ho_ = rh->ho_; hoc_obj_ref(self->ho_); Py_DECREF(r); - ok = 1; + ok = true; } if (!ok) { Py_DECREF(subself); @@ -567,7 +558,7 @@ PyObject* nrnpy_ho2po(Object* o) { ((PyHocObject*) po)->type_ = PyHoc::HocObject; auto location = sym_to_type_map.find(o->ctemplate->sym); if (location != sym_to_type_map.end()) { - po->ob_type = sym_to_type_map[o->ctemplate->sym]; + po->ob_type = location->second; } hoc_obj_ref(o); } @@ -729,7 +720,6 @@ static void* fcall(void* vself, void* vargs) { double d = hoc_call_func(self->sym_, 1); hoc_pushx(d); } else if (self->sym_->type == TEMPLATE) { - //printf("in fcall TEMPLATE case\n"); Object* ho = hoc_newobj1(self->sym_, narg); PyHocObject* result = (PyHocObject*) hocobj_new(hocobject_type, 0, 0); result->ho_ = ho; @@ -3212,11 +3202,6 @@ PyObject* nrnpy_hoc() { } Py_DECREF(bases); - //printf("done\n"); - // printf("AddObject HocObject\n"); - //printf("registering Vector\n"); - //printf("done\n"); - topmethdict = PyDict_New(); for (PyMethodDef* meth = toplevel_methods; meth->ml_name != NULL; meth++) { PyObject* descr; From 8d4b20a362a5fc54cd5df8ce6f56dc053e171944 Mon Sep 17 00:00:00 2001 From: Robin De Schepper Date: Tue, 21 Jun 2022 22:28:10 +0200 Subject: [PATCH 10/30] fixed typecheck --- test/cover/checkresult.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cover/checkresult.py b/test/cover/checkresult.py index a93871f890..c647bc4b92 100644 --- a/test/cover/checkresult.py +++ b/test/cover/checkresult.py @@ -69,7 +69,7 @@ def equal(a, b): assert match else: print("{} added {}".format(self, key)) - if type(value) == type(h.Vector): # actually hoc.HocObject + if isinstance(value, hoc.Vector): self.d[key] = value.to_python() else: self.d[key] = value From 477130fa34f18e7dd48c247f164209fbe5d00719 Mon Sep 17 00:00:00 2001 From: Fernando Pereira Date: Wed, 22 Jun 2022 16:13:46 +0200 Subject: [PATCH 11/30] Avoid function pointers to register classes. Simply use the Vector defined in hoc_oop.cpp --- src/nrnpython/nrnpy_hoc.cpp | 8 ++------ src/nrnpython/nrnpy_p2h.cpp | 1 + src/oc/hoc_oop.cpp | 12 +++++++----- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/nrnpython/nrnpy_hoc.cpp b/src/nrnpython/nrnpy_hoc.cpp index ef5207fb23..1b86439332 100644 --- a/src/nrnpython/nrnpy_hoc.cpp +++ b/src/nrnpython/nrnpy_hoc.cpp @@ -22,6 +22,7 @@ #endif extern PyTypeObject* psection_type; +extern std::vector class_name_list; // copied from nrnpy_nrn typedef struct { @@ -103,7 +104,6 @@ static cTemplate* hoc_vec_template_; static cTemplate* hoc_list_template_; static cTemplate* hoc_sectionlist_template_; -static std::vector class_name_list; static std::unordered_map sym_to_type_map; static std::unordered_map type_to_sym_map; @@ -166,10 +166,6 @@ PyTypeObject* hocobject_type; static PyObject* hocobj_call(PyHocObject* self, PyObject* args, PyObject* kwrds); -void nrnpy_register_class(const char* name) { - class_name_list.push_back(name); -} - static PyObject* nrnexec(PyObject* self, PyObject* args) { const char* cmd; if (!PyArg_ParseTuple(args, "s", &cmd)) { @@ -3188,7 +3184,7 @@ PyObject* nrnpy_hoc() { bases = PyTuple_Pack(1, hocobject_type); Py_INCREF(bases); - for (auto name: class_name_list) { + for (auto name : class_name_list) { // TODO: obj_spec_from_name needs a hoc. prepended auto long_name = std::string("hoc.") + name; spec = obj_spec_from_name(long_name.c_str()); diff --git a/src/nrnpython/nrnpy_p2h.cpp b/src/nrnpython/nrnpy_p2h.cpp index ac6238f260..47d64b78de 100644 --- a/src/nrnpython/nrnpy_p2h.cpp +++ b/src/nrnpython/nrnpy_p2h.cpp @@ -48,6 +48,7 @@ extern Object* (*nrnpy_pickle2po)(char*, size_t size); extern char* (*nrnpy_callpicklef)(char*, size_t size, int narg, size_t* retsize); extern int (*nrnpy_pysame)(Object*, Object*); // contain same Python object extern Object* (*nrnpympi_alltoall_type)(int, int); + typedef struct { PyObject_HEAD Section* sec_; char* name_; diff --git a/src/oc/hoc_oop.cpp b/src/oc/hoc_oop.cpp index da5d0aab98..8f4eacae8a 100644 --- a/src/oc/hoc_oop.cpp +++ b/src/oc/hoc_oop.cpp @@ -1,6 +1,8 @@ #include <../../nrnconf.h> #include #include +#include + #include "hocstr.h" #include "parse.hpp" #include "hocparse.h" @@ -30,7 +32,6 @@ Symbol* nrnpy_pyobj_sym_; void (*nrnpy_py2n_component)(Object* o, Symbol* s, int nindex, int isfunc); void (*nrnpy_hpoasgn)(Object* o, int type); void* (*nrnpy_opaque_obj2pyobj_p_)(Object*); -extern void nrnpy_register_class(const char* name); #endif #if CABLE @@ -47,8 +48,10 @@ static int connect_obsec_; static void call_constructor(Object*, Symbol*, int); static void free_objectdata(Objectdata*, cTemplate*); -int hoc_print_first_instance = 1; +// The list of classes to be exposed to Python +std::vector class_name_list{}; +int hoc_print_first_instance = 1; int hoc_max_builtin_class_id = -1; static Symbol* hoc_obj_; @@ -1612,9 +1615,8 @@ void class2oc(const char* name, if (hoc_lookup(name)) { hoc_execerror(name, "already being used as a name"); } - #if USE_PYTHON - nrnpy_register_class(name); - #endif + class_name_list.push_back(name); + tsym = hoc_install(name, UNDEF, 0.0, &hoc_symlist); tsym->subtype = CPLUSOBJECT; hoc_begintemplate(tsym); From bc7f04989c0e3d8f249928a7384dbab28e95e84b Mon Sep 17 00:00:00 2001 From: Robin De Schepper Date: Wed, 22 Jun 2022 16:18:41 +0200 Subject: [PATCH 12/30] Ignore egg info --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index c8c043b646..b60c2fe3fe 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ virtualenv .python-version *.o *.lo +*.egg-info # These files are generated at build time # It would be a good idea to create them in the From 75aa55f75a88fc105bb89aa71f00e2688621405c Mon Sep 17 00:00:00 2001 From: Fernando Pereira Date: Wed, 22 Jun 2022 16:31:35 +0200 Subject: [PATCH 13/30] Fix style --- src/nrnpython/nrnpy_hoc.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/nrnpython/nrnpy_hoc.cpp b/src/nrnpython/nrnpy_hoc.cpp index 1b86439332..8a7cadbdff 100644 --- a/src/nrnpython/nrnpy_hoc.cpp +++ b/src/nrnpython/nrnpy_hoc.cpp @@ -218,7 +218,7 @@ static PyObject* hocobj_new(PyTypeObject* subtype, PyObject* args, PyObject* kwd bool has_base = false; bool ok = true; subself = subtype->tp_alloc(subtype, 0); - //printf("hocobj_new %s %p %p\n", subtype->tp_name, subtype, subself); + // printf("hocobj_new %s %p %p\n", subtype->tp_name, subtype, subself); if (subself == NULL) { return NULL; } @@ -540,14 +540,14 @@ PyObject* nrnpy_ho2po(Object* o) { // The return value is None, or the encapsulated PyObject or // an encapsulating PyHocObject PyObject* po; - //printf("inside ho2po\n"); + // printf("inside ho2po\n"); if (!o) { po = Py_BuildValue(""); - //printf("empty po\n"); + // printf("empty po\n"); } else if (o->ctemplate->sym == nrnpy_pyobj_sym_) { po = nrnpy_hoc2pyobject(o); Py_INCREF(po); - //printf("from nrnpy_hoc2pyobject\n"); + // printf("from nrnpy_hoc2pyobject\n"); } else { po = hocobj_new(hocobject_type, 0, 0); ((PyHocObject*) po)->ho_ = o; @@ -3184,7 +3184,7 @@ PyObject* nrnpy_hoc() { bases = PyTuple_Pack(1, hocobject_type); Py_INCREF(bases); - for (auto name : class_name_list) { + for (auto name: class_name_list) { // TODO: obj_spec_from_name needs a hoc. prepended auto long_name = std::string("hoc.") + name; spec = obj_spec_from_name(long_name.c_str()); From 0f89ca2f1d4401002efc3cee759a52476a4b1001 Mon Sep 17 00:00:00 2001 From: Fernando Pereira Date: Wed, 22 Jun 2022 17:11:24 +0200 Subject: [PATCH 14/30] Use a static list of exposed classes to avoid point processes --- src/nrnpython/nrnpy_hoc.cpp | 46 +++++++++++++++++++++++++++++++++++++ src/oc/hoc_oop.cpp | 5 +--- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/src/nrnpython/nrnpy_hoc.cpp b/src/nrnpython/nrnpy_hoc.cpp index 8a7cadbdff..23130135b9 100644 --- a/src/nrnpython/nrnpy_hoc.cpp +++ b/src/nrnpython/nrnpy_hoc.cpp @@ -3145,6 +3145,52 @@ static PyType_Spec obj_spec_from_name(const char* name) { } PyObject* nrnpy_hoc() { + // Neuron classes being exposed to Python as real types (nrn.xxx) + std::vector class_name_list{"SectionList", + "SectionRef", + "PointProcessMark", + "Graph", + "HBox", + "VBox", + "GUIMath", + "PWManager", + "Glyph", + "ValueFieldEditor", + "Timer", + "Deck", + "SymChooser", + "StringFunctions", + "List", + "Vector", + "PtrVector", + "File", + "Pointer", + "Matrix", + "Random", + "Shape", + "PlotShape", + "PPShape", + "RangeVarPlot", + "SectionBrowser", + "MechanismStandard", + "MechanismType", + "NetCon", + "LinearMechanism", + "KSChan", + "KSGate", + "KSState", + "KSTrans", + "Impedance", + "SaveState", + "BBSaveState", + "FInitializeHandler", + "StateTransitionEvent", + "PythonObject", + "CVode", + "TQueue", + "SingleChan", + "ParallelContext"}; + PyObject* m; PyObject* bases; PyTypeObject* pto; diff --git a/src/oc/hoc_oop.cpp b/src/oc/hoc_oop.cpp index 8f4eacae8a..519b447204 100644 --- a/src/oc/hoc_oop.cpp +++ b/src/oc/hoc_oop.cpp @@ -48,9 +48,6 @@ static int connect_obsec_; static void call_constructor(Object*, Symbol*, int); static void free_objectdata(Objectdata*, cTemplate*); -// The list of classes to be exposed to Python -std::vector class_name_list{}; - int hoc_print_first_instance = 1; int hoc_max_builtin_class_id = -1; @@ -1615,12 +1612,12 @@ void class2oc(const char* name, if (hoc_lookup(name)) { hoc_execerror(name, "already being used as a name"); } - class_name_list.push_back(name); tsym = hoc_install(name, UNDEF, 0.0, &hoc_symlist); tsym->subtype = CPLUSOBJECT; hoc_begintemplate(tsym); t = tsym->u.ctemplate; + if (!hoc_main1_inited_ && t->id > hoc_max_builtin_class_id) { hoc_max_builtin_class_id = t->id; } From 199ffddfaaf16b4d00e4dea65e7e66cb16e02f0f Mon Sep 17 00:00:00 2001 From: Robin De Schepper Date: Wed, 22 Jun 2022 17:28:28 +0200 Subject: [PATCH 15/30] fix hoctype validity check in hclass --- share/lib/python/neuron/hclass3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/lib/python/neuron/hclass3.py b/share/lib/python/neuron/hclass3.py index 2ee17f293f..569dae6bac 100644 --- a/share/lib/python/neuron/hclass3.py +++ b/share/lib/python/neuron/hclass3.py @@ -91,7 +91,7 @@ class MyVector(neuron.HocBaseObject, hoc_type=neuron.h.Vector): def __init_subclass__(cls, hoc_type=None, **kwargs): if hoc_type is not None: - if not isinstance(hoc_type, hoc.HocObject): + if not issubclass(hoc_type, hoc.HocObject): raise TypeError( f"Class's `hoc_type` {hoc_type} is not a valid HOC type." ) From cbc54f3ab4dfb7ec5269e133e919d45bac3f64ca Mon Sep 17 00:00:00 2001 From: Robin De Schepper Date: Wed, 22 Jun 2022 17:30:24 +0200 Subject: [PATCH 16/30] tweaked hoctype checking --- share/lib/python/neuron/hclass3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/lib/python/neuron/hclass3.py b/share/lib/python/neuron/hclass3.py index 569dae6bac..80f08d1ed7 100644 --- a/share/lib/python/neuron/hclass3.py +++ b/share/lib/python/neuron/hclass3.py @@ -91,7 +91,7 @@ class MyVector(neuron.HocBaseObject, hoc_type=neuron.h.Vector): def __init_subclass__(cls, hoc_type=None, **kwargs): if hoc_type is not None: - if not issubclass(hoc_type, hoc.HocObject): + if not (isinstance(hoc_type, type) and issubclass(hoc_type, hoc.HocObject)): raise TypeError( f"Class's `hoc_type` {hoc_type} is not a valid HOC type." ) From 560591e0ba383e4b9bbecfd1387a3064a0343031 Mon Sep 17 00:00:00 2001 From: Robin De Schepper Date: Wed, 22 Jun 2022 17:59:41 +0200 Subject: [PATCH 17/30] allow old style hocobjects as hclass base --- share/lib/python/neuron/hclass3.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/share/lib/python/neuron/hclass3.py b/share/lib/python/neuron/hclass3.py index 80f08d1ed7..3cbc302fcb 100644 --- a/share/lib/python/neuron/hclass3.py +++ b/share/lib/python/neuron/hclass3.py @@ -91,7 +91,10 @@ class MyVector(neuron.HocBaseObject, hoc_type=neuron.h.Vector): def __init_subclass__(cls, hoc_type=None, **kwargs): if hoc_type is not None: - if not (isinstance(hoc_type, type) and issubclass(hoc_type, hoc.HocObject)): + if ( + not (isinstance(hoc_type, type) and issubclass(hoc_type, hoc.HocObject) + and not isinstance(hoc_type, hoc.HocObject) + ): raise TypeError( f"Class's `hoc_type` {hoc_type} is not a valid HOC type." ) From 8a5490917948e2d5084df02e2e5ac3d675a4d732 Mon Sep 17 00:00:00 2001 From: Robin De Schepper Date: Wed, 22 Jun 2022 18:13:38 +0200 Subject: [PATCH 18/30] hoc_base should always be oldstyle `h.` --- share/lib/python/neuron/hclass3.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/share/lib/python/neuron/hclass3.py b/share/lib/python/neuron/hclass3.py index 3cbc302fcb..21f1b276d3 100644 --- a/share/lib/python/neuron/hclass3.py +++ b/share/lib/python/neuron/hclass3.py @@ -92,9 +92,14 @@ class MyVector(neuron.HocBaseObject, hoc_type=neuron.h.Vector): def __init_subclass__(cls, hoc_type=None, **kwargs): if hoc_type is not None: if ( - not (isinstance(hoc_type, type) and issubclass(hoc_type, hoc.HocObject) - and not isinstance(hoc_type, hoc.HocObject) + hoc_type is nrn.Section + or ( + isinstance(hoc_type, type) + and issubclass(hoc_type, hoc.HocObject) + ) ): + return hoc_type + elif not isinstance(hoc_type, hoc.HocObject): raise TypeError( f"Class's `hoc_type` {hoc_type} is not a valid HOC type." ) From 0feed68683d98a2565e63e4363c55bb7c9a320da Mon Sep 17 00:00:00 2001 From: Robin De Schepper Date: Wed, 22 Jun 2022 18:46:07 +0200 Subject: [PATCH 19/30] test inheritance basics --- test/pynrn/test_inheritance.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 test/pynrn/test_inheritance.py diff --git a/test/pynrn/test_inheritance.py b/test/pynrn/test_inheritance.py new file mode 100644 index 0000000000..b7e6967134 --- /dev/null +++ b/test/pynrn/test_inheritance.py @@ -0,0 +1,19 @@ +import neuron + +def test_builtin_templates(): + assert isinstance(neuron.hoc.Vector, type), "Type instance expected for hoc.Vector" + assert isinstance(neuron.hoc.CVode, type), "Type instance expected for hoc.CVode" + assert isinstance(neuron.hoc.List, type), "Type instance expected for hoc.List" + assert isinstance(neuron.hoc.Deck, type), "Type instance expected for hoc.Deck" + + assert neuron.h.Vector is neuron.hoc.Vector, "Redirect to hoc.Vector failed" + assert neuron.h.Deck is neuron.hoc.Deck, "Redirect to hoc.Deck failed" + assert neuron.h.List is neuron.hoc.List, "Redirect to hoc.List failed" + +def test_inheritance_builtin(): + v = neuron.h.Vector() + assert isinstance(v, neuron.hoc.HocObject), "hoc.HocObject should be parent." + assert isinstance(v, neuron.hoc.Vector), "Should be instance of its class" + assert isinstance(v, neuron.hoc.Deck), "Should not be instance of another class" + assert type(v) is neuron.hoc.Vector, "Type should be class" + assert type(v) is not neuron.hoc.Deck, "Type should not be another class" From 3292dbdb39afc1471cbf8b33d4cbd694d9ee2a79 Mon Sep 17 00:00:00 2001 From: Fernando Pereira Date: Wed, 22 Jun 2022 18:46:58 +0200 Subject: [PATCH 20/30] Go back to dyn create the list of classes to expose --- src/nrnoc/init.cpp | 16 ++++++------ src/nrnpython/nrnpy_hoc.cpp | 50 ++----------------------------------- src/oc/hoc_oop.cpp | 30 ++++++++++++++++------ 3 files changed, 32 insertions(+), 64 deletions(-) diff --git a/src/nrnoc/init.cpp b/src/nrnoc/init.cpp index 0b0e929c62..4d534047b8 100644 --- a/src/nrnoc/init.cpp +++ b/src/nrnoc/init.cpp @@ -794,13 +794,13 @@ int point_reg_helper(Symbol* s2) { return pointtype++; } -extern void class2oc(const char*, - void* (*cons)(Object*), - void (*destruct)(void*), - Member_func*, - int (*checkpoint)(void**), - Member_ret_obj_func*, - Member_ret_str_func*); +extern void class2oc_base(const char*, + void* (*cons)(Object*), + void (*destruct)(void*), + Member_func*, + int (*checkpoint)(void**), + Member_ret_obj_func*, + Member_ret_str_func*); extern "C" int point_register_mech(const char** m, @@ -819,7 +819,7 @@ extern "C" int point_register_mech(const char** m, Symlist* sl; Symbol *s, *s2; nrn_load_name_check(m[1]); - class2oc(m[1], constructor, destructor, fmember, nullptr, nullptr, nullptr); + class2oc_base(m[1], constructor, destructor, fmember, nullptr, nullptr, nullptr); s = hoc_lookup(m[1]); sl = hoc_symlist; hoc_symlist = s->u.ctemplate->symtable; diff --git a/src/nrnpython/nrnpy_hoc.cpp b/src/nrnpython/nrnpy_hoc.cpp index 23130135b9..c9907527ca 100644 --- a/src/nrnpython/nrnpy_hoc.cpp +++ b/src/nrnpython/nrnpy_hoc.cpp @@ -22,7 +22,7 @@ #endif extern PyTypeObject* psection_type; -extern std::vector class_name_list; +extern std::vector py_exposed_classes; // copied from nrnpy_nrn typedef struct { @@ -3145,52 +3145,6 @@ static PyType_Spec obj_spec_from_name(const char* name) { } PyObject* nrnpy_hoc() { - // Neuron classes being exposed to Python as real types (nrn.xxx) - std::vector class_name_list{"SectionList", - "SectionRef", - "PointProcessMark", - "Graph", - "HBox", - "VBox", - "GUIMath", - "PWManager", - "Glyph", - "ValueFieldEditor", - "Timer", - "Deck", - "SymChooser", - "StringFunctions", - "List", - "Vector", - "PtrVector", - "File", - "Pointer", - "Matrix", - "Random", - "Shape", - "PlotShape", - "PPShape", - "RangeVarPlot", - "SectionBrowser", - "MechanismStandard", - "MechanismType", - "NetCon", - "LinearMechanism", - "KSChan", - "KSGate", - "KSState", - "KSTrans", - "Impedance", - "SaveState", - "BBSaveState", - "FInitializeHandler", - "StateTransitionEvent", - "PythonObject", - "CVode", - "TQueue", - "SingleChan", - "ParallelContext"}; - PyObject* m; PyObject* bases; PyTypeObject* pto; @@ -3230,7 +3184,7 @@ PyObject* nrnpy_hoc() { bases = PyTuple_Pack(1, hocobject_type); Py_INCREF(bases); - for (auto name: class_name_list) { + for (auto name: py_exposed_classes) { // TODO: obj_spec_from_name needs a hoc. prepended auto long_name = std::string("hoc.") + name; spec = obj_spec_from_name(long_name.c_str()); diff --git a/src/oc/hoc_oop.cpp b/src/oc/hoc_oop.cpp index 519b447204..b738a038ae 100644 --- a/src/oc/hoc_oop.cpp +++ b/src/oc/hoc_oop.cpp @@ -48,6 +48,8 @@ static int connect_obsec_; static void call_constructor(Object*, Symbol*, int); static void free_objectdata(Objectdata*, cTemplate*); +std::vector py_exposed_classes{}; + int hoc_print_first_instance = 1; int hoc_max_builtin_class_id = -1; @@ -1597,13 +1599,13 @@ void hoc_endtemplate(Symbol* t) { } } -void class2oc(const char* name, - void* (*cons)(Object*), - void (*destruct)(void*), - Member_func* m, - int (*checkpoint)(void**), - Member_ret_obj_func* mobjret, - Member_ret_str_func* strret) { +void class2oc_base(const char* name, + void* (*cons)(Object*), + void (*destruct)(void*), + Member_func* m, + int (*checkpoint)(void**), + Member_ret_obj_func* mobjret, + Member_ret_str_func* strret) { extern int hoc_main1_inited_; Symbol *tsym, *s; cTemplate* t; @@ -1617,7 +1619,6 @@ void class2oc(const char* name, tsym->subtype = CPLUSOBJECT; hoc_begintemplate(tsym); t = tsym->u.ctemplate; - if (!hoc_main1_inited_ && t->id > hoc_max_builtin_class_id) { hoc_max_builtin_class_id = t->id; } @@ -1625,6 +1626,7 @@ void class2oc(const char* name, t->destructor = destruct; t->steer = 0; t->checkpoint = checkpoint; + if (m) for (i = 0; m[i].name; ++i) { s = hoc_install(m[i].name, FUNCTION, 0.0, &hoc_symlist); @@ -1646,6 +1648,18 @@ void class2oc(const char* name, hoc_endtemplate(tsym); } + +void class2oc(const char* name, + void* (*cons)(Object*), + void (*destruct)(void*), + Member_func* m, + int (*checkpoint)(void**), + Member_ret_obj_func* mobjret, + Member_ret_str_func* strret) { + class2oc_base(name, cons, destruct, m, checkpoint, mobjret, strret); + py_exposed_classes.push_back(name); +} + #if JAVA2NRN Symbol* java2nrn_class(const char* name, int id, const char* meth) { Symbol *tsym, *s; From 5e622af3a95dd43a9f21763a7afc23a9a30227d0 Mon Sep 17 00:00:00 2001 From: Robin De Schepper Date: Wed, 22 Jun 2022 19:42:01 +0200 Subject: [PATCH 21/30] wip --- share/lib/python/neuron/hclass3.py | 64 +++++++++++++++++++----------- 1 file changed, 40 insertions(+), 24 deletions(-) diff --git a/share/lib/python/neuron/hclass3.py b/share/lib/python/neuron/hclass3.py index 21f1b276d3..2266a1ec63 100644 --- a/share/lib/python/neuron/hclass3.py +++ b/share/lib/python/neuron/hclass3.py @@ -12,6 +12,20 @@ import sys +def _is_hoc_pytype(hoc_type): + return ( + hoc_type is nrn.Section + or ( + isinstance(hoc_type, type) + and issubclass(hoc_type, hoc.HocObject) + ) + ) + + +def _rewrite_hoc_pytype_bases(cls, py_type): + pass + + def assert_not_hoc_composite(cls): """ Asserts that a class is not directly composed of multiple HOC types. @@ -64,10 +78,13 @@ class MyVector(myClassTemplate): module_name = __name__ if name is None: name = hoc_type.hname()[:-2] - try: - hc = type(name, (HocBaseObject,), {}, hoc_type=hoc_type) - except TypeError: - raise TypeError("Argument is not a valid HOC type.") from None + if _is_hoc_pytype(hoc_type): + return hoc_type + else: + try: + hc = type(name, (HocBaseObject,), {}, hoc_type=hoc_type) + except TypeError: + raise TypeError("Argument is not a valid HOC type.") from None hc.__module__ = module_name hc.__name__ = name hc.__qualname__ = name @@ -90,31 +107,30 @@ class MyVector(neuron.HocBaseObject, hoc_type=neuron.h.Vector): """ def __init_subclass__(cls, hoc_type=None, **kwargs): - if hoc_type is not None: - if ( - hoc_type is nrn.Section - or ( - isinstance(hoc_type, type) - and issubclass(hoc_type, hoc.HocObject) - ) - ): - return hoc_type - elif not isinstance(hoc_type, hoc.HocObject): + if _is_hoc_pytype(hoc_type): + # The given hoc_type, is actually a pytype: we don't actually need to inherit + # from HocBaseObject in this case, and modify the inheritance. + return _rewrite_hoc_pytype_bases(cls, hoc_type) + if hoc_type is None: + if not hasattr(cls, "_hoc_type"): raise TypeError( - f"Class's `hoc_type` {hoc_type} is not a valid HOC type." + "Class keyword argument `hoc_type` is required for HocBaseObjects." ) - cls._hoc_type = hoc_type - elif not hasattr(cls, "_hoc_type"): + elif not isinstance(hoc_type, hoc.HocObject): raise TypeError( - "Class keyword argument `hoc_type` is required for HocBaseObjects." + f"Class's `hoc_type` {hoc_type} is not a valid HOC type." ) + else: + cls._hoc_type = hoc_type + # HOC type classes may not be composed of multiple hoc types assert_not_hoc_composite(cls) - hobj = hoc.HocObject - hbase = HocBaseObject - if _overrides(cls, hobj, "__init__") and not _overrides(cls, hbase, "__new__"): - # Subclasses that override `__init__` must also implement `__new__` to deal - # with the arguments that have to be passed into `HocObject.__new__`. - # See https://github.com/neuronsimulator/nrn/issues/1129 + # Subclasses that override `__init__` must also implement `__new__` to deal + # with the arguments that have to be passed into `HocObject.__new__`. + # See https://github.com/neuronsimulator/nrn/issues/1129 + if ( + _overrides(cls, hoc.HocObject, "__init__") + and not _overrides(cls, HocBaseObject, "__new__") + ): raise TypeError( f"`{cls.__qualname__}` implements `__init__` but misses `__new__`. " + "Class must implement `__new__`" From 5900f794dddd2b0eeb81e57f0a33f53a294543a6 Mon Sep 17 00:00:00 2001 From: Robin De Schepper Date: Wed, 22 Jun 2022 21:43:06 +0200 Subject: [PATCH 22/30] Errors during hoc object construction were overwritten --- src/nrnpython/nrnpy_hoc.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/nrnpython/nrnpy_hoc.cpp b/src/nrnpython/nrnpy_hoc.cpp index c9907527ca..97c0c5cead 100644 --- a/src/nrnpython/nrnpy_hoc.cpp +++ b/src/nrnpython/nrnpy_hoc.cpp @@ -271,7 +271,6 @@ static PyObject* hocobj_new(PyTypeObject* subtype, PyObject* args, PyObject* kwd PyObject* r = hocobj_call(hbase, args, kwds); if (!r) { Py_DECREF(subself); - PyErr_SetString(PyExc_TypeError, "HOC base class not valid"); return NULL; } PyHocObject* rh = (PyHocObject*) r; From a93e9385fca5187b3ce6fe4c7d8b85b698910f6d Mon Sep 17 00:00:00 2001 From: Robin De Schepper Date: Wed, 22 Jun 2022 22:43:46 +0200 Subject: [PATCH 23/30] fixed/tested all cases of `hclass`/`HocBaseObject` --- share/lib/python/neuron/hclass3.py | 29 +++++------- test/pynrn/test_pyobj.py | 72 ++++++++++++++++++------------ 2 files changed, 56 insertions(+), 45 deletions(-) diff --git a/share/lib/python/neuron/hclass3.py b/share/lib/python/neuron/hclass3.py index 2266a1ec63..853402b298 100644 --- a/share/lib/python/neuron/hclass3.py +++ b/share/lib/python/neuron/hclass3.py @@ -22,10 +22,6 @@ def _is_hoc_pytype(hoc_type): ) -def _rewrite_hoc_pytype_bases(cls, py_type): - pass - - def assert_not_hoc_composite(cls): """ Asserts that a class is not directly composed of multiple HOC types. @@ -72,19 +68,16 @@ class MyVector(myClassTemplate): omitted the name of the HOC type is used. :deprecated: Inherit from :class:`~neuron.HocBaseObject` instead. """ - if hoc_type == h.Section: - return nrn.Section + if _is_hoc_pytype(hoc_type): + return hoc_type if module_name is None: module_name = __name__ if name is None: name = hoc_type.hname()[:-2] - if _is_hoc_pytype(hoc_type): - return hoc_type - else: - try: - hc = type(name, (HocBaseObject,), {}, hoc_type=hoc_type) - except TypeError: - raise TypeError("Argument is not a valid HOC type.") from None + try: + hc = type(name, (HocBaseObject,), {}, hoc_type=hoc_type) + except TypeError: + raise TypeError("Argument is not a valid HOC type.") from None hc.__module__ = module_name hc.__name__ = name hc.__qualname__ = name @@ -108,9 +101,11 @@ class MyVector(neuron.HocBaseObject, hoc_type=neuron.h.Vector): def __init_subclass__(cls, hoc_type=None, **kwargs): if _is_hoc_pytype(hoc_type): - # The given hoc_type, is actually a pytype: we don't actually need to inherit - # from HocBaseObject in this case, and modify the inheritance. - return _rewrite_hoc_pytype_bases(cls, hoc_type) + cls_name = cls.__name__ + raise TypeError( + f"Using HocBaseObject with {cls_name} is deprecated." + f" Inherit directly from {cls_name} instead." + ) if hoc_type is None: if not hasattr(cls, "_hoc_type"): raise TypeError( @@ -143,7 +138,7 @@ def __new__(cls, *args, **kwds): # To construct HOC objects within NEURON from the Python interface, we use the # C-extension module `hoc`. `hoc.HocObject.__new__` both creates an internal # representation of the object in NEURON, and hands us back a Python object that - # is linked to that internal representation. The `__new__` functions takes the + # is linked to that internal representation. The `__new__` function takes the # arguments that HOC objects of that type would take, and uses the `hocbase` # keyword argument to determine which type of HOC object to create. The `sec` # keyword argument can be passed along in case the construction of a HOC object diff --git a/test/pynrn/test_pyobj.py b/test/pynrn/test_pyobj.py index cd4faff413..22b97416c0 100644 --- a/test/pynrn/test_pyobj.py +++ b/test/pynrn/test_pyobj.py @@ -2,13 +2,20 @@ import pytest +def test_builtin(): + with pytest.raises(TypeError): + class MyList(neuron.HocBaseObject, hoc_type=neuron.h.List): + pass + + def test_hocbase(): - class MyList(neuron.HocBaseObject, hoc_type=neuron.h.Vector): + class MyStim(neuron.HocBaseObject, hoc_type=neuron.h.NetStim): pass - assert issubclass(MyList, neuron.hoc.HocObject) - assert issubclass(MyList, neuron.HocBaseObject) - assert MyList._hoc_type == neuron.h.Vector + assert issubclass(MyStim, neuron.hoc.HocObject) + assert issubclass(MyStim, neuron.HocBaseObject) + assert MyStim._hoc_type == neuron.h.NetStim + def test_hoc_template_hclass(): @@ -64,25 +71,25 @@ def test_pyobj_constructor(): # Test that __new__ is required when __init__ is overridden with pytest.raises(TypeError): - class PyObj(neuron.HocBaseObject, hoc_type=neuron.h.List): - def __init__(self, first): + class PyObj(neuron.HocBaseObject, hoc_type=neuron.h.NetStim): + def __init__(self, freq): super().__init__() self.append(first) - class PyObj(neuron.HocBaseObject, hoc_type=neuron.h.List): - def __new__(cls, first): + class PyObj(neuron.HocBaseObject, hoc_type=neuron.h.NetStim): + def __new__(cls, freq): return super().__new__(cls) - def __init__(self, first): + def __init__(self, freq): super().__init__() - self.append(first) + self.interval = 1000 / freq - p = PyObj(neuron.h.List()) - assert p.count() == 1 + p = PyObj(4) + assert p.interval == 250 def test_pyobj_def(): - class PyObj(neuron.HocBaseObject, hoc_type=neuron.h.List): + class PyObj(neuron.HocBaseObject, hoc_type=neuron.h.NetStim): def my_method(self, a): return a * 2 @@ -91,20 +98,29 @@ def my_method(self, a): def test_pyobj_overloading(): - class PyObj(neuron.HocBaseObject, hoc_type=neuron.h.List): - def append(self, i): - self.last_appended = i - return self.baseattr("append")(i) + class PyObj(neuron.HocBaseObject, hoc_type=neuron.h.PatternStim): + def play(self, i): + self.played = True + v = neuron.h.Vector([i]) + return self.baseattr("play")(v, v) + + p = PyObj() + p.play(2) + assert hasattr(p, "played") + + +@pytest.mark.xfail(reason="inf. recursion because baseattr finds Python attrs") +def test_bad_overload(): + class PyObj(neuron.HocBaseObject, hoc_type=neuron.h.PatternStim): + def not_on_base(self): + return p.baseattr("not_on_base")() p = PyObj() - p2 = PyObj() - assert p.append(p) == 1 - assert p.count() == 1 - assert p[0] == p + p.not_on_base() def test_pyobj_inheritance(): - class PyObj(neuron.HocBaseObject, hoc_type=neuron.h.List): + class PyObj(neuron.HocBaseObject, hoc_type=neuron.h.NetStim): pass class MyObj(PyObj): @@ -116,7 +132,7 @@ class MyObj2(PyObj): def __init__(self, arg): pass - class List(neuron.HocBaseObject, hoc_type=neuron.h.List): + class List(neuron.HocBaseObject, hoc_type=neuron.h.NetStim): def __new__(cls, *args, **kwargs): super().__new__(cls) @@ -126,17 +142,17 @@ def __init__(self, *args): for arg in args: self.append(arg) - l = InitList(neuron.h.List(), neuron.h.List()) + l = InitList(neuron.h.NetStim(), neuron.h.NetStim()) def test_pyobj_composition(): - class A(neuron.HocBaseObject, hoc_type=neuron.h.List): + class A(neuron.HocBaseObject, hoc_type=neuron.h.NetStim): pass - class B(neuron.HocBaseObject, hoc_type=neuron.h.List): + class B(neuron.HocBaseObject, hoc_type=neuron.h.NetStim): pass - class C(neuron.HocBaseObject, hoc_type=neuron.h.Vector): + class C(neuron.HocBaseObject, hoc_type=neuron.h.ExpSyn): pass with pytest.raises(TypeError): @@ -147,7 +163,7 @@ class D(A, C): class E(A, B): pass - assert E._hoc_type == neuron.h.List + assert E._hoc_type == neuron.h.NetStim class PickleTest(neuron.HocBaseObject, hoc_type=neuron.h.NetStim): From c631d94be1ccccf06e1de0ee739c4c7488f0b7e7 Mon Sep 17 00:00:00 2001 From: Robin De Schepper Date: Wed, 22 Jun 2022 23:03:36 +0200 Subject: [PATCH 24/30] fixed py inheritance tests --- test/pynrn/test_inheritance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pynrn/test_inheritance.py b/test/pynrn/test_inheritance.py index b7e6967134..8b6042fd9e 100644 --- a/test/pynrn/test_inheritance.py +++ b/test/pynrn/test_inheritance.py @@ -14,6 +14,6 @@ def test_inheritance_builtin(): v = neuron.h.Vector() assert isinstance(v, neuron.hoc.HocObject), "hoc.HocObject should be parent." assert isinstance(v, neuron.hoc.Vector), "Should be instance of its class" - assert isinstance(v, neuron.hoc.Deck), "Should not be instance of another class" + assert not isinstance(v, neuron.hoc.Deck), "Should not be instance of another class" assert type(v) is neuron.hoc.Vector, "Type should be class" assert type(v) is not neuron.hoc.Deck, "Type should not be another class" From b2717b70597b49cf116ed9a392d0dbbb496117f7 Mon Sep 17 00:00:00 2001 From: Fernando Pereira Date: Thu, 23 Jun 2022 15:53:37 +0200 Subject: [PATCH 25/30] Black Style --- share/lib/python/neuron/hclass3.py | 17 +++++------------ test/pynrn/test_inheritance.py | 2 ++ test/pynrn/test_pyobj.py | 2 +- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/share/lib/python/neuron/hclass3.py b/share/lib/python/neuron/hclass3.py index 853402b298..63c3c74389 100644 --- a/share/lib/python/neuron/hclass3.py +++ b/share/lib/python/neuron/hclass3.py @@ -13,12 +13,8 @@ def _is_hoc_pytype(hoc_type): - return ( - hoc_type is nrn.Section - or ( - isinstance(hoc_type, type) - and issubclass(hoc_type, hoc.HocObject) - ) + return hoc_type is nrn.Section or ( + isinstance(hoc_type, type) and issubclass(hoc_type, hoc.HocObject) ) @@ -112,9 +108,7 @@ def __init_subclass__(cls, hoc_type=None, **kwargs): "Class keyword argument `hoc_type` is required for HocBaseObjects." ) elif not isinstance(hoc_type, hoc.HocObject): - raise TypeError( - f"Class's `hoc_type` {hoc_type} is not a valid HOC type." - ) + raise TypeError(f"Class's `hoc_type` {hoc_type} is not a valid HOC type.") else: cls._hoc_type = hoc_type # HOC type classes may not be composed of multiple hoc types @@ -122,9 +116,8 @@ def __init_subclass__(cls, hoc_type=None, **kwargs): # Subclasses that override `__init__` must also implement `__new__` to deal # with the arguments that have to be passed into `HocObject.__new__`. # See https://github.com/neuronsimulator/nrn/issues/1129 - if ( - _overrides(cls, hoc.HocObject, "__init__") - and not _overrides(cls, HocBaseObject, "__new__") + if _overrides(cls, hoc.HocObject, "__init__") and not _overrides( + cls, HocBaseObject, "__new__" ): raise TypeError( f"`{cls.__qualname__}` implements `__init__` but misses `__new__`. " diff --git a/test/pynrn/test_inheritance.py b/test/pynrn/test_inheritance.py index 8b6042fd9e..1fb524e32a 100644 --- a/test/pynrn/test_inheritance.py +++ b/test/pynrn/test_inheritance.py @@ -1,5 +1,6 @@ import neuron + def test_builtin_templates(): assert isinstance(neuron.hoc.Vector, type), "Type instance expected for hoc.Vector" assert isinstance(neuron.hoc.CVode, type), "Type instance expected for hoc.CVode" @@ -10,6 +11,7 @@ def test_builtin_templates(): assert neuron.h.Deck is neuron.hoc.Deck, "Redirect to hoc.Deck failed" assert neuron.h.List is neuron.hoc.List, "Redirect to hoc.List failed" + def test_inheritance_builtin(): v = neuron.h.Vector() assert isinstance(v, neuron.hoc.HocObject), "hoc.HocObject should be parent." diff --git a/test/pynrn/test_pyobj.py b/test/pynrn/test_pyobj.py index 22b97416c0..ad4f3332fa 100644 --- a/test/pynrn/test_pyobj.py +++ b/test/pynrn/test_pyobj.py @@ -4,6 +4,7 @@ def test_builtin(): with pytest.raises(TypeError): + class MyList(neuron.HocBaseObject, hoc_type=neuron.h.List): pass @@ -17,7 +18,6 @@ class MyStim(neuron.HocBaseObject, hoc_type=neuron.h.NetStim): assert MyStim._hoc_type == neuron.h.NetStim - def test_hoc_template_hclass(): neuron.h( """ From ac02b531d5b7b29f59b9adf5bd94443c1184cc75 Mon Sep 17 00:00:00 2001 From: Fernando Pereira Date: Sat, 9 Jul 2022 02:28:35 +0200 Subject: [PATCH 26/30] Keep exposed pytype names alive until end execution --- src/nrnpython/nrnpy_hoc.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/nrnpython/nrnpy_hoc.cpp b/src/nrnpython/nrnpy_hoc.cpp index 97c0c5cead..4d92a3c436 100644 --- a/src/nrnpython/nrnpy_hoc.cpp +++ b/src/nrnpython/nrnpy_hoc.cpp @@ -106,6 +106,7 @@ static cTemplate* hoc_sectionlist_template_; static std::unordered_map sym_to_type_map; static std::unordered_map type_to_sym_map; +static std::vector exposed_py_type_names; // typestr returned by Vector.__array_interface__ // byteorder (first element) is modified at import time @@ -3185,8 +3186,8 @@ PyObject* nrnpy_hoc() { Py_INCREF(bases); for (auto name: py_exposed_classes) { // TODO: obj_spec_from_name needs a hoc. prepended - auto long_name = std::string("hoc.") + name; - spec = obj_spec_from_name(long_name.c_str()); + exposed_py_type_names.push_back(std::string("hoc.") + name); + spec = obj_spec_from_name(exposed_py_type_names.back().c_str()); pto = (PyTypeObject*) PyType_FromSpecWithBases(&spec, bases); sym_to_type_map[hoc_lookup(name)] = pto; type_to_sym_map[pto] = hoc_lookup(name); From c3baa2669f15ef86c31c5084e08cb4d1cb521fd2 Mon Sep 17 00:00:00 2001 From: Fernando Pereira Date: Sat, 9 Jul 2022 04:54:42 +0200 Subject: [PATCH 27/30] Revisiting Ref counts --- src/nrnpython/nrnpy_hoc.cpp | 79 +++++++++++++++---------------------- 1 file changed, 32 insertions(+), 47 deletions(-) diff --git a/src/nrnpython/nrnpy_hoc.cpp b/src/nrnpython/nrnpy_hoc.cpp index 4d92a3c436..1d1005e997 100644 --- a/src/nrnpython/nrnpy_hoc.cpp +++ b/src/nrnpython/nrnpy_hoc.cpp @@ -215,9 +215,8 @@ static void hocobj_dealloc(PyHocObject* self) { static PyObject* hocobj_new(PyTypeObject* subtype, PyObject* args, PyObject* kwds) { PyObject* subself; PyObject* base; - PyHocObject* hbase; - bool has_base = false; - bool ok = true; + PyHocObject* hbase = nullptr; + subself = subtype->tp_alloc(subtype, 0); // printf("hocobj_new %s %p %p\n", subtype->tp_name, subtype, subself); if (subself == NULL) { @@ -234,59 +233,45 @@ static PyObject* hocobj_new(PyTypeObject* subtype, PyObject* args, PyObject* kwd // if subtype is a subclass of some NEURON class, then one of its // tp_mro's is in sym_to_type_map - for (Py_ssize_t i = 0; i < PyTuple_Size(subtype->tp_mro); i++) { PyObject* item = PyTuple_GetItem(subtype->tp_mro, i); - Py_INCREF(item); - auto location = type_to_sym_map.find((PyTypeObject*) item); - if (location != type_to_sym_map.end()) { - PyObject* result = hocobj_new(hocobject_type, 0, 0); - hbase = (PyHocObject*) result; + auto symbol_result = type_to_sym_map.find((PyTypeObject*) item); + if (symbol_result != type_to_sym_map.end()) { + hbase = (PyHocObject*) hocobj_new(hocobject_type, 0, 0); hbase->type_ = PyHoc::HocFunction; - hbase->sym_ = location->second; - has_base = true; - } - Py_DECREF(item); - if (has_base) { + hbase->sym_ = symbol_result->second; break; } } - if (kwds && PyDict_Check(kwds)) { - base = PyDict_GetItemString(kwds, "hocbase"); - if (base) { - ok = false; - if (PyObject_TypeCheck(base, hocobject_type)) { - hbase = (PyHocObject*) base; - has_base = true; - } - // TODO: is it a problem doing this before we're done with the base - PyDict_DelItemString(kwds, "hocbase"); - } - } - if (has_base) { - if (hbase->type_ == PyHoc::HocFunction && hbase->sym_->type == TEMPLATE) { - // printf("hocobj_new base %s\n", hbase->sym_->name); - // remove the hocbase keyword since hocobj_call only allows - // the "sec" keyword argument - PyObject* r = hocobj_call(hbase, args, kwds); - if (!r) { - Py_DECREF(subself); - return NULL; - } - PyHocObject* rh = (PyHocObject*) r; - self->type_ = rh->type_; - self->ho_ = rh->ho_; - hoc_obj_ref(self->ho_); - Py_DECREF(r); - ok = true; - } - if (!ok) { - Py_DECREF(subself); + if (kwds && PyDict_Check(kwds) && (base = PyDict_GetItemString(kwds, "hocbase"))) { + if (PyObject_TypeCheck(base, hocobject_type)) { + hbase = (PyHocObject*) base; + } + else { PyErr_SetString(PyExc_TypeError, "HOC base class not valid"); + Py_DECREF(subself); + return NULL; + } + PyDict_DelItemString(kwds, "hocbase"); + } + + if (hbase and hbase->type_ == PyHoc::HocFunction && hbase->sym_->type == TEMPLATE) { + // printf("hocobj_new base %s\n", hbase->sym_->name); + // remove the hocbase keyword since hocobj_call only allows + // the "sec" keyword argument + PyObject* r = hocobj_call(hbase, args, kwds); + if (!r) { + Py_DECREF(subself); return NULL; } + PyHocObject* rh = (PyHocObject*) r; + self->type_ = rh->type_; + self->ho_ = rh->ho_; + hoc_obj_ref(self->ho_); + Py_DECREF(r); } + return subself; } @@ -554,6 +539,7 @@ PyObject* nrnpy_ho2po(Object* o) { ((PyHocObject*) po)->type_ = PyHoc::HocObject; auto location = sym_to_type_map.find(o->ctemplate->sym); if (location != sym_to_type_map.end()) { + Py_INCREF(location->second); po->ob_type = location->second; } hoc_obj_ref(o); @@ -725,6 +711,7 @@ static void* fcall(void* vself, void* vargs) { // so we could avoid repetitive code auto location = sym_to_type_map.find(ho->ctemplate->sym); if (location != sym_to_type_map.end()) { + Py_INCREF(location->second); ((PyObject*) result)->ob_type = location->second; } @@ -3179,7 +3166,6 @@ PyObject* nrnpy_hoc() { hocobject_type = (PyTypeObject*) PyType_FromSpec(&spec); if (PyType_Ready(hocobject_type) < 0) goto fail; - Py_INCREF(hocobject_type); PyModule_AddObject(m, "HocObject", (PyObject*) hocobject_type); bases = PyTuple_Pack(1, hocobject_type); @@ -3193,7 +3179,6 @@ PyObject* nrnpy_hoc() { type_to_sym_map[pto] = hoc_lookup(name); if (PyType_Ready(pto) < 0) goto fail; - Py_INCREF(pto); PyModule_AddObject(m, name, (PyObject*) pto); } Py_DECREF(bases); From 97976fee39459044925ff92441c289272323c21a Mon Sep 17 00:00:00 2001 From: Fernando Pereira Date: Mon, 25 Sep 2023 22:16:08 -0400 Subject: [PATCH 28/30] Fixes for PR --- src/nrnpython/nrnpy_hoc.cpp | 3 +-- test/pytest_coreneuron/test_basic.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/nrnpython/nrnpy_hoc.cpp b/src/nrnpython/nrnpy_hoc.cpp index f61b9f8516..6414df11a6 100644 --- a/src/nrnpython/nrnpy_hoc.cpp +++ b/src/nrnpython/nrnpy_hoc.cpp @@ -233,8 +233,7 @@ static PyObject* hocobj_new(PyTypeObject* subtype, PyObject* args, PyObject* kwd if (kwds && PyDict_Check(kwds) && (base = PyDict_GetItemString(kwds, "hocbase"))) { if (PyObject_TypeCheck(base, hocobject_type)) { hbase = (PyHocObject*) base; - } - else { + } else { PyErr_SetString(PyExc_TypeError, "HOC base class not valid"); Py_DECREF(subself); return NULL; diff --git a/test/pytest_coreneuron/test_basic.py b/test/pytest_coreneuron/test_basic.py index a8c951616c..029aa2882f 100644 --- a/test/pytest_coreneuron/test_basic.py +++ b/test/pytest_coreneuron/test_basic.py @@ -449,8 +449,8 @@ def test_help(): assert h.Vector().to_python.__doc__.startswith( "Syntax:\n ``pythonlist = vec.to_python()" ) - assert h.Vector().__doc__.startswith("This class was imple") - assert h.Vector.__doc__.startswith("This class was imple") + assert h.Vector().__doc__.startswith("class neuron.hoc.HocObject") + assert h.Vector.__doc__.startswith("class neuron.hoc.HocObject") assert h.finitialize.__doc__.startswith("Syntax:\n ``h.finiti") assert h.__doc__.startswith("\n\nneuron.h\n====") From c106dce42e8dda418b950976e6cd18763c4796b9 Mon Sep 17 00:00:00 2001 From: Fernando Pereira Date: Wed, 27 Sep 2023 11:29:31 -0400 Subject: [PATCH 29/30] Adding docs --- docs/python/index.rst | 1 + docs/python/programming/neuron_classes.rst | 87 ++++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 docs/python/programming/neuron_classes.rst diff --git a/docs/python/index.rst b/docs/python/index.rst index 6064efe542..1817e3b993 100755 --- a/docs/python/index.rst +++ b/docs/python/index.rst @@ -55,6 +55,7 @@ Basic Programming programming/dynamiccode.rst programming/projectmanagement.rst programming/internals.rst + programming/neuron_classes.rst programming/hoc-from-python.rst Model Specification diff --git a/docs/python/programming/neuron_classes.rst b/docs/python/programming/neuron_classes.rst new file mode 100644 index 0000000000..de677f9188 --- /dev/null +++ b/docs/python/programming/neuron_classes.rst @@ -0,0 +1,87 @@ +NEURON Python Classes and Objects +================================= + +Neuron exposes its internal objects and hoc templates as Python objects via an automatic +conversion layer, effectively making all entities from the HOC stack available to a Python program. + +There are basically two main objects which expose most Neuron entities. The first is `hoc` which +exposes a number of internal established classes and functions. + +.. code-block:: + python + + >>> from neuron import hoc + >>> hoc. + hoc.List( + hoc.SectionList( + hoc.SectionRef( + hoc.Vector( + ... + +However, for *dynamic* entities Neuron provides the `h` gateway object. It gives access to internal +classes (templates) and objects, even if they were just created. E.g.: + +.. code-block:: + python + + >>> from neuron import h + >>> # Create objects in the hoc stack + >>> h("objref vec") + >>> h("vec = new Vector(5, 1)") + >>> # Access to objects + >>> h.vec.as_numpy() + array([1., 1., 1., 1., 1.]) + >>> + >>> # Access to exposed types + >>> vv = h.Vector(5, 2) + >>> vv.as_numpy() + array([1., 1., 1., 1., 1.]) + +This is particularly useful as Neuron can dynamically load libraries with more functions and classes. + +Class Hierarchy +--------------- + +All NEURON's internal interpreter objects are instances of a global top-level type: HocObject. +Until very recently they were considered direct instances, without any intermediate hierarchy. + +With #1858 Hoc classes are now associated with actual Python types, created dynamically. Such +change enables type instances to be properly recognized as such, respecting e.g. `isinstance()` +predicates and subclassing. + +.. code-block:: + python + + >>> isinstance(hoc.Vector, type) + True + >>> v = h.Vector() + >>> isinstance(v, hoc.HocObject) + True + >>> isinstance(v, hoc.Vector) + True + >>> type(v) is hoc.Vector # direct subclass + True + >>>isinstance(v, hoc.Deck) # Not instance of other class + False + +Subclasses are also recognized properly. For creating them please inherit from `HocBaseObject` +with `hoc_type` given as argument. E.g.: + +.. code-block:: + python + + >>> class MyStim(neuron.HocBaseObject, hoc_type=h.NetStim): + pass + >>> issubclass(MyStim, hoc.HocObject) + True + >>> issubclass(MyStim, neuron.HocBaseObject) + True + >>> MyStim._hoc_type == h.NetStim + True + >>> stim = MyStim() + >>> isinstance(stim, MyStim) + True + >>> isinstance(stim, h.NetStim) + True + >>> isinstance(stim, h.HocObject) + True From f4be23d02d90b7f4162009fa3e85b7927f3359f0 Mon Sep 17 00:00:00 2001 From: Robert A McDougal Date: Thu, 28 Sep 2023 11:58:43 -0400 Subject: [PATCH 30/30] branding: Neuron -> NEURON --- docs/python/programming/neuron_classes.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/python/programming/neuron_classes.rst b/docs/python/programming/neuron_classes.rst index de677f9188..1a5db25d3e 100644 --- a/docs/python/programming/neuron_classes.rst +++ b/docs/python/programming/neuron_classes.rst @@ -1,7 +1,7 @@ NEURON Python Classes and Objects ================================= -Neuron exposes its internal objects and hoc templates as Python objects via an automatic +NEURON exposes its internal objects and hoc templates as Python objects via an automatic conversion layer, effectively making all entities from the HOC stack available to a Python program. There are basically two main objects which expose most Neuron entities. The first is `hoc` which @@ -18,7 +18,7 @@ exposes a number of internal established classes and functions. hoc.Vector( ... -However, for *dynamic* entities Neuron provides the `h` gateway object. It gives access to internal +However, for *dynamic* entities NEURON provides the `h` gateway object. It gives access to internal classes (templates) and objects, even if they were just created. E.g.: .. code-block:: @@ -37,12 +37,12 @@ classes (templates) and objects, even if they were just created. E.g.: >>> vv.as_numpy() array([1., 1., 1., 1., 1.]) -This is particularly useful as Neuron can dynamically load libraries with more functions and classes. +This is particularly useful as NEURON can dynamically load libraries with more functions and classes. Class Hierarchy --------------- -All NEURON's internal interpreter objects are instances of a global top-level type: HocObject. +All NEURON's internal interpreter objects are instances of a global top-level type: `HocObject`. Until very recently they were considered direct instances, without any intermediate hierarchy. With #1858 Hoc classes are now associated with actual Python types, created dynamically. Such