diff --git a/plugins/python/CMakeLists.txt b/plugins/python/CMakeLists.txt index d21131767..27f41d14a 100644 --- a/plugins/python/CMakeLists.txt +++ b/plugins/python/CMakeLists.txt @@ -7,6 +7,9 @@ if(Python_NumPy_VERSION VERSION_LESS "2.0.0") ) endif() +find_package(PkgConfig REQUIRED) +pkg_check_modules(FFI REQUIRED IMPORTED_TARGET libffi) + # Phlex module to run Python algorithms add_library( pymodule @@ -17,8 +20,9 @@ add_library( src/dciwrap.cpp src/lifelinewrap.cpp src/errorwrap.cpp + src/dyncall.cpp ) -target_link_libraries(pymodule PRIVATE phlex::module Python::Python Python::NumPy) +target_link_libraries(pymodule PRIVATE phlex::module PkgConfig::FFI Python::Python Python::NumPy) target_compile_definitions(pymodule PRIVATE NPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION) install(TARGETS pymodule LIBRARY DESTINATION lib) diff --git a/plugins/python/python/phlex/_typing.py b/plugins/python/python/phlex/_typing.py index 6a38e3bdb..e25b43618 100644 --- a/plugins/python/python/phlex/_typing.py +++ b/plugins/python/python/phlex/_typing.py @@ -14,11 +14,18 @@ import numpy as np +try: + import numba.core.types as nb_types + has_numba = True +except ImportError: + has_numba = False + __all__ = [ "normalize_type", ] -# ctypes and numpy types are likely candidates for use in annotations +# ctypes and numpy types are likely candidates for use in annotations; Numba +# types may appear from callback signatures # TODO: should users be allowed to add to these? _PY2CPP: dict[type, str] = { # numpy types @@ -40,6 +47,22 @@ # np.uintp: "size_t", } +if has_numba: + _PY2CPP.update({ + nb_types.bool: "bool", + nb_types.int8: "int8_t", + nb_types.int16: "int16_t", + nb_types.int32: "int32_t", + nb_types.int64: "int64_t", + nb_types.uint8: "uint8_t", + nb_types.uint16: "uint16_t", + nb_types.uint32: "uint32_t", + nb_types.uint64: "uint64_t", + nb_types.Float: "float", + nb_types.float32: "float", + nb_types.double: "double", + }) + # ctypes types that don't map cleanly to intN_t / uintN_t _CTYPES_SPECIAL: dict[type, str] = {} for _attr, _cpp in [ @@ -96,8 +119,11 @@ def _build_ctypes_map() -> dict[type, str]: "unsigned int": _PY2CPP[ctypes.c_uint], "long": _PY2CPP[ctypes.c_long], "unsigned long": _PY2CPP[ctypes.c_ulong], + # special cases; not necessarily correct but as expected on major platforms "long long": "int64_t", "unsigned long long": "uint64_t", + "float32": "float", + "float64": "double", } diff --git a/plugins/python/src/dyncall.cpp b/plugins/python/src/dyncall.cpp new file mode 100644 index 000000000..c7ca48565 --- /dev/null +++ b/plugins/python/src/dyncall.cpp @@ -0,0 +1,127 @@ +// Dynamic dispatcher from generically packaged args to any C or Python function. +// +// Note: this particular implementation is based on libffi, presumed to be for +// now the minimal dependency, but an alternative could be based on JITing +// using Cling or even Numba's llvmlite. + +#include "dyncall.hpp" +#include +#include + +#include + +using namespace phlex::experimental; + +phlex::experimental::dcarg phlex::experimental::dcarg::from_str(std::string const& stype) +{ + // only types currently used in modulewrap are added, not all ffi types + if (stype == "bool") + return dcarg(false); + else if (stype == "int32_t") + return dcarg(static_cast(0)); + else if (stype == "uint32_t") + return dcarg(static_cast(0)); + else if (stype == "int64_t") + return dcarg(static_cast(0)); + else if (stype == "uint64_t") + return dcarg(static_cast(0)); + else if (stype == "float") + return dcarg(0.0f); + else if (stype == "double") + return dcarg(0.0); + + throw std::invalid_argument("unknown type string: " + stype); +} + +void* phlex::experimental::dcarg::value_ptr() +{ + return std::visit( + [](auto& val) -> void* { + using T = std::decay_t; + if constexpr (std::is_same_v) { + return nullptr; + } else { + return static_cast(&val); + } + }, + m_value); +} + +namespace { + static ffi_type* get_ffi_type(dcarg const& d) + { + return std::visit( + [](auto&& val) -> ffi_type* { + using T = std::decay_t; + + // there are duplicate bodies here b/c bool is represented by uint8, + // just as uint8 is, there being no bool in C; the code is cleaner + // with each type on its own line, however, rather than combining the + // two in a single predicate as a special case + // NOLINTBEGIN(bugprone-branch-clone) + if constexpr (std::is_same_v) + return &ffi_type_void; + else if constexpr (std::is_same_v) + return &ffi_type_pointer; + else if constexpr (std::is_same_v) + return &ffi_type_uint8; + else if constexpr (std::is_same_v) + return &ffi_type_sint8; + else if constexpr (std::is_same_v) + return &ffi_type_uint8; + else if constexpr (std::is_same_v) + return &ffi_type_sint16; + else if constexpr (std::is_same_v) + return &ffi_type_uint16; + else if constexpr (std::is_same_v) + return &ffi_type_sint32; + else if constexpr (std::is_same_v) + return &ffi_type_uint32; + else if constexpr (std::is_same_v) + return &ffi_type_sint64; + else if constexpr (std::is_same_v) + return &ffi_type_uint64; + else if constexpr (std::is_same_v) + return &ffi_type_float; + else if constexpr (std::is_same_v) + return &ffi_type_double; + // NOLINTEND(bugprone-branch-clone) + }, + d.m_value); + } +} + +void phlex::experimental::dyncall(void* fn, dcarg& result, dcargs_t& args, int var_offset) +{ + // Perform a dynamic call of function fn, taking arguments `args` and returning + // `result`. Set `var_offset` to the appropriate number of positional arguments + // if the other arguments are variational. + + // Except for the memory management unique_ptrs, this code is essentially C, + // because libffi is, and that yields a plethora of warnings from clang-tidy, + // none of which warrant actual changes. + // NOLINTBEGIN + std::size_t N = (std::size_t)args.size(); + + auto t = std::make_unique(N); + auto p = std::make_unique(N); + + for (dcargs_t::size_type i = 0; i < N; ++i) { + auto& a = args[i]; + t[i] = get_ffi_type(a); + p[i] = a.value_ptr(); + } + + ffi_cif cif; + ffi_status status; + if (0 < var_offset) + status = ffi_prep_cif_var(&cif, FFI_DEFAULT_ABI, var_offset, N, get_ffi_type(result), t.get()); + else + status = ffi_prep_cif(&cif, FFI_DEFAULT_ABI, N, get_ffi_type(result), t.get()); + + if (status) + throw std::runtime_error("ffi prep failed"); + + ffi_call(&cif, (void (*)())fn, result.value_ptr(), p.get()); + // NOLINTEND +} diff --git a/plugins/python/src/dyncall.hpp b/plugins/python/src/dyncall.hpp new file mode 100644 index 000000000..d3785f031 --- /dev/null +++ b/plugins/python/src/dyncall.hpp @@ -0,0 +1,97 @@ +#ifndef PLUGINS_PYTHON_SRC_DYNCALL_HPP +#define PLUGINS_PYTHON_SRC_DYNCALL_HPP + +// ======================================================================================= +// +// Dynamic dispatcher from generically packaged args to any C or Python function. +// +// Design rationale +// ================ +// +// Python code is inserted in the Phlex execution graph using generic types to avoid a +// combinatorial explosion of types. This way, all template instantiations can be done at +// compile time. Callback wrappers are then needed to either pack from generic to Python +// or to unpack from generic to C/C++ and perform the call. This dynamic dispatcher +// provides that functionality. +// +// ======================================================================================= + +#include "Python.h" // for PyObject* get<> specialization only + +#include +#include +#include +#include + +#if defined(__APPLE__) && defined(__MACH__) +// This is a temporary workaround until we have a solution for handling translation of types +// between C++ and Python. +typedef long ph_long_t; +typedef unsigned long ph_ulong_t; +#else +typedef std::int64_t ph_long_t; +typedef std::uint64_t ph_ulong_t; +#endif + +namespace phlex::experimental { + + struct dcarg { + using FFIType = std::variant; + + FFIType m_value; + + // convenience mapper of human-readable string to dcarg + static dcarg from_str(std::string const& stype); + + // factory-style constructors to guarantee value/type match + dcarg() : m_value(std::monostate{}) {} + explicit dcarg(void* v) : m_value(v) {} + explicit dcarg(bool v) : m_value(v) {} + explicit dcarg(std::int8_t v) : m_value(v) {} + explicit dcarg(std::uint8_t v) : m_value(v) {} + explicit dcarg(std::int16_t v) : m_value(v) {} + explicit dcarg(std::uint16_t v) : m_value(v) {} + explicit dcarg(std::int32_t v) : m_value(v) {} + explicit dcarg(std::uint32_t v) : m_value(v) {} + explicit dcarg(ph_long_t v) : m_value(v) {} + explicit dcarg(ph_ulong_t v) : m_value(v) {} + explicit dcarg(float v) : m_value(v) {} + explicit dcarg(double v) : m_value(v) {} + + // pointer access to payload + void* value_ptr(); + + // value access to payload + template + T get() + { + return std::get(m_value); + } + }; + + // specialization to simplify a very common case + template <> + inline PyObject* dcarg::get() + { + return reinterpret_cast(std::get(m_value)); + } + + typedef std::vector dcargs_t; + + void dyncall(void* fn, dcarg& result, dcargs_t& args, int var_offset = -1); + +} // phlex::experimental + +#endif // PLUGINS_PYTHON_SRC_DYNCALL_HPP diff --git a/plugins/python/src/modulewrap.cpp b/plugins/python/src/modulewrap.cpp index 4b89a8ff5..2c05f7a6d 100644 --- a/plugins/python/src/modulewrap.cpp +++ b/plugins/python/src/modulewrap.cpp @@ -1,3 +1,4 @@ +#include "dyncall.hpp" #include "wrap.hpp" #include "phlex/model/data_cell_index.hpp" @@ -11,6 +12,8 @@ #include #include #include +#include +#include #include #define NO_IMPORT_ARRAY @@ -39,6 +42,13 @@ // need that same input translated. This simplifies memory management, but // can cause a performance bottleneck (since all require the GIL). +// This is dumb, but for now, because all templates need to be instantiated, only +// support up to a fixed compile-timee maximum number of arguments. An alternative +// would be to collect the arguments, but that currently suffers from needing a +// "initial" to create the container to collect arguments into. This may all go +// away once converter nodes have better support in phlex' core +constexpr size_t MAX_SUPPORTED_ARGS = 3; + using namespace phlex::experimental; using namespace phlex; using phlex::concurrency; @@ -95,33 +105,34 @@ namespace { return fmt::format("{}_arg{}_py", algname, arg); } - static inline PyObject* lifeline_transform(intptr_t arg) + static inline dcarg lifeline_transform(dcarg arg) { - PyObject* pyobj = reinterpret_cast(arg); + PyObject* pyobj = arg.get(); if (pyobj && PyObject_TypeCheck(pyobj, &PhlexLifeline_Type)) { - return reinterpret_cast(pyobj)->m_view; + return dcarg{reinterpret_cast(pyobj)->m_view}; } - return pyobj; + return arg; } - // callable object managing the callback - template - struct py_callback { + // callable objects managing the callback + struct py_callback_base { PyObject* m_callable; // owned + void* m_ccallback; // C callable (either dispatcher or direct pointer) - explicit py_callback(PyObject* callable) : m_callable(callable) + py_callback_base(PyObject* callable, void* cb) : m_callable(callable), m_ccallback(cb) { - // callable is always non-null here (validated before py_callback construction) + // callable is always non-null here (validated before construction) PyGILRAII gil; Py_INCREF(m_callable); } - py_callback(py_callback const& pc) : m_callable(pc.m_callable) + py_callback_base(py_callback_base const& pc) : + m_callable(pc.m_callable), m_ccallback(pc.m_ccallback) { // Must hold GIL when manipulating reference counts PyGILRAII gil; Py_INCREF(m_callable); } - py_callback& operator=(py_callback const& pc) + py_callback_base& operator=(py_callback_base const& pc) { if (this != &pc) { // Must hold GIL when manipulating reference counts @@ -129,66 +140,63 @@ namespace { Py_INCREF(pc.m_callable); Py_DECREF(m_callable); m_callable = pc.m_callable; + m_ccallback = pc.m_ccallback; } return *this; } - ~py_callback() + virtual ~py_callback_base() { // TODO: cleanup deferred to Phlex shutdown hook // Cannot safely Py_DECREF during arbitrary destruction due to: // - TOCTOU race on Py_IsInitialized() without GIL // - Module offloading in interpreter cleanup phase 2 } - py_callback(py_callback&&) = default; - py_callback& operator=(py_callback&&) = default; - - template - intptr_t call(Args... args) - { - static_assert(sizeof...(Args) == N, "Argument count mismatch"); - - PyGILRAII gil; - - PyObject* result = - PyObject_CallFunctionObjArgs(m_callable, lifeline_transform(args)..., nullptr); - - std::string error_msg; - if (!result) { - if (!msg_from_py_error(error_msg)) - error_msg = "Unknown python error"; - } + py_callback_base(py_callback_base&&) = default; + py_callback_base& operator=(py_callback_base&&) = default; + }; - decref_all(args...); + // type repeater to automatically instantiate callbacks taking N args + template + using type_repeater = T; - if (!error_msg.empty()) { - throw std::runtime_error(error_msg.c_str()); - } + template + struct py_callback_impl; - return reinterpret_cast(result); + template + struct py_callback_impl> : public py_callback_base { + py_callback_impl(PyObject* callable) : + py_callback_base(callable, reinterpret_cast(PyObject_CallFunctionObjArgs)) + { } - template - void callv(Args... args) + RT operator()(type_repeater... args) { - static_assert(sizeof...(Args) == N, "Argument count mismatch"); + dcargs_t argsv; + argsv.reserve(sizeof...(Is) + 2); + argsv.push_back(dcarg{m_callable}); + (argsv.push_back(lifeline_transform(args)), ...); + argsv.push_back(dcarg{nullptr}); PyGILRAII gil; - PyObject* result = - PyObject_CallFunctionObjArgs(m_callable, lifeline_transform(args)..., nullptr); + dcarg result{nullptr}; + dyncall((void*)m_ccallback, result, argsv, 1); std::string error_msg; - if (!result) { + if (!result.get()) { if (!msg_from_py_error(error_msg)) error_msg = "Unknown python error"; - } else - Py_DECREF(result); + } decref_all(args...); - if (!error_msg.empty()) { + if (!error_msg.empty()) throw std::runtime_error(error_msg.c_str()); - } + + if constexpr (!std::is_void_v) + return result; + else + Py_DECREF(result.get()); } private: @@ -196,60 +204,61 @@ namespace { void decref_all(Args... args) { // helper to decrement reference counts of N arguments - (Py_DECREF((PyObject*)args), ...); + (Py_DECREF(reinterpret_cast(std::get(args.m_value))), ...); } }; - // use explicit instatiations to ensure that the function signature can - // be derived by the graph builder - struct py_callback_1 : public py_callback<1> { - using py_callback<1>::py_callback; - intptr_t operator()(intptr_t arg0) { return call(arg0); } - }; + template + struct jit_callback_impl; - struct py_callback_2 : public py_callback<2> { - using py_callback<2>::py_callback; - intptr_t operator()(intptr_t arg0, intptr_t arg1) { return call(arg0, arg1); } - }; + template + struct jit_callback_impl> : public py_callback_base { + dcarg m_rtype; // dynamic call return type - struct py_callback_3 : public py_callback<3> { - using py_callback<3>::py_callback; - intptr_t operator()(intptr_t arg0, intptr_t arg1, intptr_t arg2) + jit_callback_impl(PyObject* callable, void* cb, std::string const& stype) : + py_callback_base(callable, cb), m_rtype(dcarg::from_str(stype)) { - return call(arg0, arg1, arg2); } - }; - struct py_callback_1v : public py_callback<1> { - using py_callback<1>::py_callback; - void operator()(intptr_t arg0) { callv(arg0); } - }; + RT operator()(type_repeater... args) + { + dcarg result{m_rtype}; + dcargs_t argsv; + argsv.reserve(sizeof...(Is)); + (argsv.push_back(args), ...); - struct py_callback_2v : public py_callback<2> { - using py_callback<2>::py_callback; - void operator()(intptr_t arg0, intptr_t arg1) { callv(arg0, arg1); } - }; + dyncall((void*)m_ccallback, result, argsv); + // TODO: error reporting? - struct py_callback_3v : public py_callback<3> { - using py_callback<3>::py_callback; - void operator()(intptr_t arg0, intptr_t arg1, intptr_t arg2) { callv(arg0, arg1, arg2); } + if constexpr (!std::is_void_v) + return result; + } }; - static inline std::optional validate_query(PyObject* pyquery) + // aliases to reduce typing downstream (explicit instatiations used to ensure + // that the function signature can be derived by the graph builder + template + using py_callback = py_callback_impl>; + + template + using jit_callback = jit_callback_impl>; + + // input/output validation helpers + static inline std::optional validate_selector(PyObject* pysel) { - if (!PyDict_Check(pyquery)) { - PyErr_Format(PyExc_TypeError, "query should be a product specification"); + if (!PyDict_Check(pysel)) { + PyErr_Format(PyExc_TypeError, "selector should be a product specification"); return std::nullopt; } - PyObject* pyc = PyDict_GetItemString(pyquery, "creator"); + PyObject* pyc = PyDict_GetItemString(pysel, "creator"); if (!pyc || !PyUnicode_Check(pyc)) { PyErr_Format(PyExc_TypeError, "missing \"creator\" or not a string"); return std::nullopt; } char const* c = PyUnicode_AsUTF8(pyc); - PyObject* pyl = PyDict_GetItemString(pyquery, "layer"); + PyObject* pyl = PyDict_GetItemString(pysel, "layer"); if (!pyl || !PyUnicode_Check(pyl)) { PyErr_Format(PyExc_TypeError, "missing \"layer\" or not a string"); return std::nullopt; @@ -257,7 +266,7 @@ namespace { char const* l = PyUnicode_AsUTF8(pyl); std::optional s; - PyObject* pys = PyDict_GetItemString(pyquery, "suffix"); + PyObject* pys = PyDict_GetItemString(pysel, "suffix"); if (pys) { if (!PyUnicode_Check(pys)) { PyErr_Format(PyExc_TypeError, "provided \"suffix\" is not a string"); @@ -288,11 +297,11 @@ namespace { for (Py_ssize_t i = 0; i < len; ++i) { PyObject* item = items[i]; // borrowed reference - auto pq = validate_query(item); + auto pq = validate_selector(item); if (pq.has_value()) { cargs.push_back(pq.value()); } else { - // validate_query will have set a python exception + // validate_selection will have set a python exception break; } } @@ -345,6 +354,31 @@ namespace { namespace { + static bool is_numba_cfunc(PyObject* obj) + { + static PyObject* cfunc_type = nullptr; + static bool checked = false; + if (!checked) { + checked = true; + + PyObject* nbmod = PyImport_ImportModule("numba.core.ccallback"); + if (nbmod) { + cfunc_type = PyObject_GetAttrString(nbmod, "CFunc"); + Py_DECREF(nbmod); + } + + if (!cfunc_type) + PyErr_Clear(); + // hard reference to cfunc_type here if not null + } + + if (!cfunc_type) + return false; + + int result = PyObject_IsInstance(obj, cfunc_type); + return result == 1; + } + static std::string annotation_as_text(PyObject* pyobj) { static PyObject* normalizer = nullptr; @@ -381,26 +415,49 @@ namespace { std::vector& input_types, std::vector& output_types) { + bool conversion_ok = false; + PyObject* sann = PyUnicode_FromString("__annotations__"); PyObject* annot = PyObject_GetAttr(callable, sann); if (!annot) { - // the callable may be an instance with a __call__ method - PyErr_Clear(); - PyObject* callm = PyObject_GetAttrString(callable, "__call__"); - if (callm) { - annot = PyObject_GetAttr(callm, sann); - Py_DECREF(callm); + // the callable may be a Numba CFunc and have a declared signature + PyObject* sig = PyObject_GetAttrString(callable, "_sig"); + if (sig) { + PyObject* ret = PyObject_GetAttrString(sig, "return_type"); + PyObject* args = PyObject_GetAttrString(sig, "args"); + + if (ret && args && PyTuple_CheckExact(args)) { + output_types.push_back(annotation_as_text(ret)); + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(args); ++i) { + PyObject* item = PyTuple_GET_ITEM(args, i); + input_types.push_back(annotation_as_text(item)); + } + conversion_ok = true; + } else + PyErr_Clear(); + + Py_XDECREF(args); + Py_XDECREF(ret); + } else { + PyErr_Clear(); + // the callable may be an instance with a __call__ method + PyObject* callm = PyObject_GetAttrString(callable, "__call__"); + if (callm) { + annot = PyObject_GetAttr(callm, sann); + Py_DECREF(callm); + } } } Py_DECREF(sann); - bool conversion_ok = true; - if (annot && PyDict_Check(annot)) { + if (!conversion_ok && annot && PyDict_Check(annot)) { // Variant guarantees OrderedDict with "return" last Py_ssize_t pos = 0; PyObject* key = nullptr; PyObject* value = nullptr; + + conversion_ok = true; while (PyDict_Next(annot, &pos, &key, &value)) { std::string const& ann = annotation_as_text(value); if (ann.empty() && PyErr_Occurred()) { @@ -416,7 +473,6 @@ namespace { } } } else { - conversion_ok = false; if (!PyErr_Occurred()) PyErr_SetString(PyExc_TypeError, "unknown annotation formatting"); } @@ -500,33 +556,38 @@ namespace { // for expressions, but causes havoc with C++ signatures. We suppress this warning for the block // because the use of continuations makes per-line suppression impossible. #define BASIC_CONVERTER(name, cpptype, topy, frompy) \ - static intptr_t name##_to_py(cpptype a) \ + static dcarg name##_to_py(cpptype a) \ { \ PyGILRAII gil; \ - return reinterpret_cast(topy(a)); \ + return dcarg{topy(a)}; \ } \ \ - static cpptype py_to_##name(intptr_t pyobj) \ + static dcarg name##_to_dcarg(cpptype a) { return dcarg{a}; } \ + \ + static cpptype py_to_##name(dcarg a) \ { \ PyGILRAII gil; \ - cpptype i = static_cast(frompy(reinterpret_cast(pyobj))); \ + PyObject* pyobj = a.get(); \ + cpptype i = static_cast(frompy(pyobj)); \ std::string msg; \ if (msg_from_py_error(msg, true)) { \ - Py_DECREF(reinterpret_cast(pyobj)); \ + Py_DECREF(pyobj); \ throw std::runtime_error("Python conversion error for type " #name ": " + msg); \ } \ - Py_DECREF(reinterpret_cast(pyobj)); \ + Py_DECREF(pyobj); \ return i; \ } \ \ - struct provider_cb_##name : public py_callback<1> { \ - using py_callback<1>::py_callback; \ + static cpptype dcarg_to_##name(dcarg a) { return a.get(); } \ + \ + struct provider_cb_##name : public py_callback { \ + using py_callback::py_callback; \ cpptype operator()(data_cell_index const& id) \ { \ PyGILRAII gil; \ PyObject* arg0 = wrap_dci(id); \ - intptr_t const arg0i = reinterpret_cast(arg0); \ - PyObject* pyres = reinterpret_cast(call(arg0i)); /* decrefs arg0 */ \ + dcarg res = this->py_callback::operator()(dcarg{arg0}); /* decrefs arg0 */ \ + PyObject* pyres = res.get(); \ cpptype cres = frompy(pyres); \ Py_DECREF(pyres); \ return cres; \ @@ -536,25 +597,18 @@ namespace { BASIC_CONVERTER(bool, bool, PyBool_FromLong, pylong_as_bool) BASIC_CONVERTER(int, std::int32_t, PyLong_FromLong, PyLong_AsLong) BASIC_CONVERTER(uint, std::uint32_t, PyLong_FromLong, pylong_or_int_as_ulong) -#if defined(__APPLE__) && defined(__MACH__) - // This is a temporary workaround until we have a solution for handling translation of types - // between C++ and Python. - BASIC_CONVERTER(long, long, PyLong_FromLong, pylong_as_strictlong) - BASIC_CONVERTER(ulong, unsigned long, PyLong_FromUnsignedLong, pylong_or_int_as_ulong) -#else - BASIC_CONVERTER(long, std::int64_t, PyLong_FromLong, pylong_as_strictlong) - BASIC_CONVERTER(ulong, std::uint64_t, PyLong_FromUnsignedLong, pylong_or_int_as_ulong) -#endif + BASIC_CONVERTER(long, ph_long_t, PyLong_FromLong, pylong_as_strictlong) + BASIC_CONVERTER(ulong, ph_ulong_t, PyLong_FromUnsignedLong, pylong_or_int_as_ulong) BASIC_CONVERTER(float, float, PyFloat_FromDouble, PyFloat_AsDouble) BASIC_CONVERTER(double, double, PyFloat_FromDouble, PyFloat_AsDouble) #define VECTOR_CONVERTER(name, cpptype, nptype) \ - static intptr_t name##_to_py(std::shared_ptr> const& v) \ + static dcarg name##_to_py(std::shared_ptr> const& v) \ { \ PyGILRAII gil; \ \ if (!v) \ - return 0; \ + return dcarg{nullptr}; \ \ /* use a numpy view with the shared pointer tied up in a lifeline object (note: this */ \ /* is just a demonstrator; alternatives are still being considered) */ \ @@ -567,7 +621,7 @@ namespace { ); \ \ if (!np_view) \ - return 0; \ + return dcarg{nullptr}; \ \ /* make the data read-only by not making it writable */ \ PyArray_CLEARFLAGS(reinterpret_cast(np_view), NPY_ARRAY_WRITEABLE); \ @@ -579,12 +633,12 @@ namespace { PhlexLifeline_Type.tp_new(&PhlexLifeline_Type, nullptr, nullptr)); \ if (!pyll) { \ Py_DECREF(np_view); \ - return 0; \ + return dcarg{nullptr}; \ } \ pyll->m_source = v; \ pyll->m_view = np_view; /* steals reference */ \ \ - return reinterpret_cast(pyll); \ + return dcarg{pyll}; \ } VECTOR_CONVERTER(vint, std::int32_t, NPY_INT32) @@ -595,14 +649,15 @@ namespace { VECTOR_CONVERTER(vdouble, double, NPY_DOUBLE) #define NUMPY_ARRAY_CONVERTER(name, cpptype, nptype, frompy) \ - static std::shared_ptr> py_to_##name(intptr_t pyobj) \ + static std::shared_ptr> py_to_##name(dcarg a) \ { \ PyGILRAII gil; \ \ auto vec = std::make_shared>(); \ + PyObject* pyobj = a.get(); \ \ /* TODO: because of unresolved ownership issues, copy the full array contents */ \ - if (PyArray_Check(reinterpret_cast(pyobj))) { \ + if (PyArray_Check(pyobj)) { \ PyArrayObject* arr = reinterpret_cast(pyobj); \ \ /* TODO: flattening the array here seems to be the only workable solution */ \ @@ -616,11 +671,11 @@ namespace { cpptype* raw = static_cast(PyArray_DATA(arr)); \ vec->reserve(total); \ vec->insert(vec->end(), raw, raw + total); \ - } else if (PyList_Check(reinterpret_cast(pyobj))) { \ - Py_ssize_t total = PyList_Size(reinterpret_cast(pyobj)); \ + } else if (PyList_Check(pyobj)) { \ + Py_ssize_t total = PyList_Size(pyobj); \ vec->reserve(total); \ for (Py_ssize_t i = 0; i < total; ++i) { \ - PyObject* item = PyList_GetItem(reinterpret_cast(pyobj), i); \ + PyObject* item = PyList_GetItem(pyobj, i); \ vec->push_back(static_cast(frompy(item))); \ if (PyErr_Occurred()) { \ PyErr_Clear(); \ @@ -634,18 +689,18 @@ namespace { } \ } \ \ - Py_DECREF(reinterpret_cast(pyobj)); \ + Py_DECREF(pyobj); \ return vec; \ } \ \ - struct provider_cb_##name : public py_callback<1> { \ - using py_callback<1>::py_callback; \ + struct provider_cb_##name : public py_callback { \ + using py_callback::py_callback; \ std::shared_ptr> operator()(data_cell_index const& id) \ { \ PyGILRAII gil; \ PyObject* arg0 = wrap_dci(id); \ - intptr_t pyres = call(reinterpret_cast(arg0)); /* decrefs arg0 */ \ - auto cres = py_to_##name(pyres); /* decrefs pyres */ \ + dcarg pyres = this->py_callback::operator()(dcarg{arg0}); /* decrefs arg0 */ \ + auto cres = py_to_##name(pyres); /* decrefs pyres */ \ return cres; \ } \ }; @@ -666,7 +721,8 @@ namespace { product_selector pq_in, std::string const& output) { - mod->ph_module->transform(name, converter, concurrency::serial) + mod->ph_module + ->transform(name, converter, (concurrency)16) //concurrency::serial) // TODO! .input_family(pq_in) .output_product_suffixes(output); } @@ -676,10 +732,11 @@ namespace { static PyObject* parse_args(PyObject* args, PyObject* kwds, std::string& functor_name, - std::vector& input_queries, + std::vector& input_selectors, std::vector& input_types, std::vector& output_suffixes, - std::vector& output_types) + std::vector& output_types, + concurrency& nconcur) { // Helper function to extract the common names and identifiers needed to insert // any node. (The observer does not require outputs, but they still need to be @@ -689,24 +746,22 @@ static PyObject* parse_args(PyObject* args, kw3[] = "concurrency", kw4[] = "name"; // kwnames can be of type char const*[] once we mandate Python 3.13 or newer static char* kwnames[] = {kw0, kw1, kw2, kw3, kw4, nullptr}; - PyObject *callable = nullptr, *input = nullptr, *output = nullptr, *concurrency = nullptr, - *pyname = nullptr; + PyObject *callable = nullptr, *input = nullptr, *output = nullptr, *pyname = nullptr; + int nconcur_ = -1; if (!PyArg_ParseTupleAndKeywords( - args, kwds, "OO|OOO", kwnames, &callable, &input, &output, &concurrency, &pyname)) { + args, kwds, "OO|OiO", (char**)kwnames, &callable, &input, &output, &nconcur_, &pyname)) { // error already set by argument parser return nullptr; } - if (concurrency && concurrency != Py_None) { - PyErr_SetString(PyExc_TypeError, "only serial concurrency is supported"); - return nullptr; - } - if (!callable || !PyCallable_Check(callable)) { PyErr_SetString(PyExc_TypeError, "provided algorithm is not callable"); return nullptr; } + // set concurrency, or the default of serial if not set + nconcur = nconcur_ > 0 ? (concurrency)nconcur_ : concurrency::serial; + // retrieve function name if (!pyname) { pyname = PyObject_GetAttrString(callable, "__name__"); @@ -727,8 +782,8 @@ static PyObject* parse_args(PyObject* args, } // convert input declarations, to be able to pass them to Phlex - input_queries = validate_input(input); - if (input_queries.empty()) { + input_selectors = validate_input(input); + if (input_selectors.empty()) { if (!PyErr_Occurred()) { PyErr_Format(PyExc_ValueError, "no input provided for %s; node can not be scheduled", @@ -744,8 +799,8 @@ static PyObject* parse_args(PyObject* args, return nullptr; } - // retrieve C++ (matching) types from annotations - input_types.reserve(input_queries.size()); + // retrieve C++ (matching) types if provided + input_types.reserve(input_selectors.size()); if (!annotations_to_strings(callable, input_types, output_types)) return nullptr; // Python error already set @@ -754,12 +809,12 @@ static PyObject* parse_args(PyObject* args, output_types.clear(); // if annotations were correct (and correctly parsed), there should be as many - // input types as input product queries - if (input_types.size() != input_queries.size()) { + // input types as input product selectors + if (input_types.size() != input_selectors.size()) { PyErr_Format(PyExc_TypeError, "number of inputs (%d; %s) does not match number of annotation types (%d; %s)", - input_queries.size(), - stringify(input_queries).c_str(), + input_selectors.size(), + stringify(input_selectors).c_str(), input_types.size(), stringify(input_types).c_str()); return nullptr; @@ -793,12 +848,13 @@ static std::optional collection_dtype(std::string const& type_ static bool insert_input_converters(py_phlex_module* mod, std::string const& cname, // TODO: shared_ptr - std::vector const& input_queries, - std::vector const& input_types) + std::vector const& input_selectors, + std::vector const& input_types, + bool ispy) { // insert input converter nodes into the graph for (auto const [i, inp_pq, inp_type] : - std::views::zip(std::views::iota(size_t{}), input_queries, input_types)) { + std::views::zip(std::views::iota(size_t{}), input_selectors, input_types)) { // TODO: this seems overly verbose and inefficient, but the function needs // to be properly types, so every option is made explicit @@ -807,19 +863,19 @@ static bool insert_input_converters(py_phlex_module* mod, "py_" + (inp_pq.suffix ? std::string{static_cast(*inp_pq.suffix)} : ""); if (inp_type == "bool") - insert_converter(mod, pyname, bool_to_py, inp_pq, output); + insert_converter(mod, pyname, ispy ? bool_to_py : bool_to_dcarg, inp_pq, output); else if (inp_type == "int32_t") - insert_converter(mod, pyname, int_to_py, inp_pq, output); + insert_converter(mod, pyname, ispy ? int_to_py : int_to_dcarg, inp_pq, output); else if (inp_type == "uint32_t") - insert_converter(mod, pyname, uint_to_py, inp_pq, output); + insert_converter(mod, pyname, ispy ? uint_to_py : uint_to_dcarg, inp_pq, output); else if (inp_type == "int64_t") - insert_converter(mod, pyname, long_to_py, inp_pq, output); + insert_converter(mod, pyname, ispy ? long_to_py : long_to_dcarg, inp_pq, output); else if (inp_type == "uint64_t") - insert_converter(mod, pyname, ulong_to_py, inp_pq, output); + insert_converter(mod, pyname, ispy ? ulong_to_py : ulong_to_dcarg, inp_pq, output); else if (inp_type == "float") - insert_converter(mod, pyname, float_to_py, inp_pq, output); + insert_converter(mod, pyname, ispy ? float_to_py : float_to_dcarg, inp_pq, output); else if (inp_type == "double") - insert_converter(mod, pyname, double_to_py, inp_pq, output); + insert_converter(mod, pyname, ispy ? double_to_py : double_to_dcarg, inp_pq, output); else if (inp_type.compare(0, 7, "ndarray") == 0 || inp_type.compare(0, 4, "list") == 0) { // TODO: these are hard-coded std::vector <-> numpy array mappings, which is // way too simplistic for real use. It only exists for demonstration purposes, @@ -858,23 +914,24 @@ static bool insert_output_converter(py_phlex_module* mod, std::string const& cname, product_selector const& out_pq, std::string const& out_type, - std::string const& output) + std::string const& output, + bool ispy) { // insert output converter node into the graph if (out_type == "bool") - insert_converter(mod, cname, py_to_bool, out_pq, output); + insert_converter(mod, cname, ispy ? py_to_bool : dcarg_to_bool, out_pq, output); else if (out_type == "int32_t") - insert_converter(mod, cname, py_to_int, out_pq, output); + insert_converter(mod, cname, ispy ? py_to_int : dcarg_to_int, out_pq, output); else if (out_type == "uint32_t") - insert_converter(mod, cname, py_to_uint, out_pq, output); + insert_converter(mod, cname, ispy ? py_to_uint : dcarg_to_uint, out_pq, output); else if (out_type == "int64_t") - insert_converter(mod, cname, py_to_long, out_pq, output); + insert_converter(mod, cname, ispy ? py_to_long : dcarg_to_long, out_pq, output); else if (out_type == "uint64_t") - insert_converter(mod, cname, py_to_ulong, out_pq, output); + insert_converter(mod, cname, ispy ? py_to_ulong : dcarg_to_ulong, out_pq, output); else if (out_type == "float") - insert_converter(mod, cname, py_to_float, out_pq, output); + insert_converter(mod, cname, ispy ? py_to_float : dcarg_to_float, out_pq, output); else if (out_type == "double") - insert_converter(mod, cname, py_to_double, out_pq, output); + insert_converter(mod, cname, ispy ? py_to_double : dcarg_to_double, out_pq, output); else if (out_type.compare(0, 7, "ndarray") == 0 || out_type.compare(0, 4, "list") == 0) { // TODO: just like for input types, these are hard-coded, but should be handled by // an IDL instead. @@ -907,19 +964,51 @@ static bool insert_output_converter(py_phlex_module* mod, return true; } +template +static bool unroll_switch(size_t rt_size, Cf&& func) +{ + return [&](std::index_sequence) { + // 1-based sequence (all computational nodes have an input, or they can't be scheduled), + // with the fold expression short-circuited using || + + // clang-tidy is incorrect here, b/c the condition "rt_size == (Is + 1)" is only ever + // true once, so the forward is only called once, and func is never used after move + // NOLINTBEGIN(bugprone-use-after-move) + bool matched = (... || ((rt_size == (Is + 1)) + ? (std::forward(func)(std::make_index_sequence{}), true) + : false)); + // NOLINTEND(bugprone-use-after-move) + + return matched; + }(std::make_index_sequence{}); +} + static PyObject* md_transform(py_phlex_module* mod, PyObject* args, PyObject* kwds) { // Register a python algorithm by adding the necessary intermediate converter // nodes going from C++ to PyObject* and back. std::string cname; - std::vector input_queries; + std::vector input_selectors; std::vector input_types, output_suffixes, output_types; - PyObject* callable = - parse_args(args, kwds, cname, input_queries, input_types, output_suffixes, output_types); + concurrency nconcur; + PyObject* callable = parse_args( + args, kwds, cname, input_selectors, input_types, output_suffixes, output_types, nconcur); + if (!callable) return nullptr; // error already set + // detect numba and extract C function pointer if any, else use default Python + // callable dispatcher + void* ccallf = nullptr; + if (is_numba_cfunc(callable)) { + PyObject* pyaddr = PyObject_GetAttrString(callable, "address"); + if (pyaddr) + ccallf = PyLong_AsVoidPtr(pyaddr); + if (!ccallf) + PyErr_Clear(); + } + if (output_types.empty()) { PyErr_Format(PyExc_TypeError, "transform %s should have an output type", cname.c_str()); Py_DECREF(callable); @@ -928,9 +1017,9 @@ static PyObject* md_transform(py_phlex_module* mod, PyObject* args, PyObject* kw // TODO: it's not clear what the output layer will be if the input layers are not // all the same, so for now, simply raise an error if their is any ambiguity - auto output_layer = static_cast(input_queries[0].layer); - if (1 < input_queries.size()) { - for (auto const& iq_pq : input_queries | std::views::drop(1)) { + auto output_layer = static_cast(input_selectors[0].layer); + if (1 < input_selectors.size()) { + for (auto const& iq_pq : input_selectors | std::views::drop(1)) { if (static_cast(iq_pq.layer) != output_layer) { PyErr_Format(PyExc_ValueError, "transform %s output layer is ambiguous", cname.c_str()); Py_DECREF(callable); @@ -939,80 +1028,60 @@ static PyObject* md_transform(py_phlex_module* mod, PyObject* args, PyObject* kw } } - if (!insert_input_converters(mod, cname, input_queries, input_types)) { + if (!insert_input_converters(mod, cname, input_selectors, input_types, !ccallf)) { Py_DECREF(callable); return nullptr; // error already set } - // register Python transform + // register Python transform callbacks // TODO: only support single output type for now, as there has to be a mapping // onto a std::tuple otherwise, which is a typed object, thus complicating the // template instantiation std::string pyname = "py_" + cname; std::string pyoutput = output_suffixes[0] + "_py"; + std::string const& out_type = output_types[0]; - auto pq0 = input_queries[0]; - std::string c0 = input_converter_name(cname, 0); - std::string suff0 = - "py_" + (pq0.suffix ? std::string{static_cast(*pq0.suffix)} : ""); - - switch (input_queries.size()) { - case 1: { - mod->ph_module->transform(pyname, py_callback_1{callable}, concurrency::serial) - .input_family(product_selector{ - .creator = identifier(c0), .layer = pq0.layer, .suffix = identifier(suff0)}) - .output_product_suffixes(pyoutput); - break; - } - case 2: { - auto pq1 = input_queries[1]; - std::string c1 = input_converter_name(cname, 1); - std::string suff1 = - "py_" + (pq1.suffix ? std::string{static_cast(*pq1.suffix)} : ""); - mod->ph_module->transform(pyname, py_callback_2{callable}, concurrency::serial) - .input_family(product_selector{.creator = identifier(c0), - .layer = pq0.layer, - .suffix = identifier(suff0)}, - product_selector{ - .creator = identifier(c1), .layer = pq1.layer, .suffix = identifier(suff1)}) - .output_product_suffixes(pyoutput); - break; - } - case 3: { - auto pq1 = input_queries[1]; - std::string c1 = input_converter_name(cname, 1); - std::string suff1 = - "py_" + (pq1.suffix ? std::string{static_cast(*pq1.suffix)} : ""); - auto pq2 = input_queries[2]; - std::string c2 = input_converter_name(cname, 2); - std::string suff2 = - "py_" + (pq2.suffix ? std::string{static_cast(*pq2.suffix)} : ""); - mod->ph_module->transform(pyname, py_callback_3{callable}, concurrency::serial) - .input_family(product_selector{.creator = identifier(c0), - .layer = pq0.layer, - .suffix = identifier(suff0)}, - product_selector{ - .creator = identifier(c1), .layer = pq1.layer, .suffix = identifier(suff1)}, - product_selector{ - .creator = identifier(c2), .layer = pq2.layer, .suffix = identifier(suff2)}) - .output_product_suffixes(pyoutput); - break; - } - default: { + auto transform_N_args = [&](std::index_sequence) { + constexpr size_t N = sizeof...(Is); + + auto make_product_selector = [&](size_t i) { + auto pq = input_selectors[i]; + std::string c = input_converter_name(cname, i); + std::string suff = + "py_" + (pq.suffix ? std::string{static_cast(*pq.suffix)} : ""); + + return product_selector{ + .creator = identifier(c), .layer = pq.layer, .suffix = identifier(suff)}; + }; + + auto insert_tranform_for_callback = [&](auto& cb) { + mod->ph_module->transform(pyname, cb, nconcur) + .input_family(make_product_selector(Is)...) + .output_product_suffixes(pyoutput); + }; + + if (ccallf) { + jit_callback cb{callable, ccallf, out_type}; + insert_tranform_for_callback(cb); + } else { + py_callback cb{callable}; + insert_tranform_for_callback(cb); + } + }; + + if (!unroll_switch(input_selectors.size(), transform_N_args)) { PyErr_SetString(PyExc_TypeError, "unsupported number of inputs"); Py_DECREF(callable); return nullptr; } - } // insert output converter node into the graph auto out_pq = product_selector{.creator = identifier(pyname), .layer = identifier(output_layer), .suffix = identifier(pyoutput)}; - std::string const& out_type = output_types[0]; std::string const& output = output_suffixes[0]; - if (!insert_output_converter(mod, cname, out_pq, out_type, output)) { + if (!insert_output_converter(mod, cname, out_pq, out_type, output, !ccallf)) { Py_DECREF(callable); return nullptr; // error already set } @@ -1027,10 +1096,12 @@ static PyObject* md_observe(py_phlex_module* mod, PyObject* args, PyObject* kwds // nodes going from C++ to PyObject* and back. std::string cname; - std::vector input_queries; + std::vector input_selectors; std::vector input_types, output_suffixes, output_types; - PyObject* callable = - parse_args(args, kwds, cname, input_queries, input_types, output_suffixes, output_types); + concurrency nconcur; + PyObject* callable = parse_args( + args, kwds, cname, input_selectors, input_types, output_suffixes, output_types, nconcur); + if (!callable) return nullptr; // error already set @@ -1040,62 +1111,34 @@ static PyObject* md_observe(py_phlex_module* mod, PyObject* args, PyObject* kwds return nullptr; } - if (!insert_input_converters(mod, cname, input_queries, input_types)) { + if (!insert_input_converters(mod, cname, input_selectors, input_types, true)) { Py_DECREF(callable); return nullptr; // error already set } - // register Python observer - auto pq0 = input_queries[0]; - std::string c0 = input_converter_name(cname, 0); - std::string suff0 = - "py_" + (pq0.suffix ? std::string{static_cast(*pq0.suffix)} : ""); - - switch (input_queries.size()) { - case 1: { - mod->ph_module->observe(cname, py_callback_1v{callable}, concurrency::serial) - .input_family(product_selector{ - .creator = identifier(c0), .layer = pq0.layer, .suffix = identifier(suff0)}); - break; - } - case 2: { - auto pq1 = input_queries[1]; - std::string c1 = input_converter_name(cname, 1); - std::string suff1 = - "py_" + (pq1.suffix ? std::string{static_cast(*pq1.suffix)} : ""); - mod->ph_module->observe(cname, py_callback_2v{callable}, concurrency::serial) - .input_family(product_selector{.creator = identifier(c0), - .layer = pq0.layer, - .suffix = identifier(suff0)}, - product_selector{ - .creator = identifier(c1), .layer = pq1.layer, .suffix = identifier(suff1)}); - break; - } - case 3: { - auto pq1 = input_queries[1]; - std::string c1 = input_converter_name(cname, 1); - std::string suff1 = - "py_" + (pq1.suffix ? std::string{static_cast(*pq1.suffix)} : ""); - auto pq2 = input_queries[2]; - std::string c2 = input_converter_name(cname, 2); - std::string suff2 = - "py_" + (pq2.suffix ? std::string{static_cast(*pq2.suffix)} : ""); - mod->ph_module->observe(cname, py_callback_3v{callable}, concurrency::serial) - .input_family(product_selector{.creator = identifier(c0), - .layer = pq0.layer, - .suffix = identifier(suff0)}, - product_selector{ - .creator = identifier(c1), .layer = pq1.layer, .suffix = identifier(suff1)}, - product_selector{ - .creator = identifier(c2), .layer = pq2.layer, .suffix = identifier(suff2)}); - break; - } - default: { + // register Python observer callbacks + auto observe_N_args = [&](std::index_sequence) { + constexpr size_t N = sizeof...(Is); + + auto make_product_selector = [&](size_t i) { + auto pq = input_selectors[i]; + std::string c = input_converter_name(cname, i); + std::string suff = + "py_" + (pq.suffix ? std::string{static_cast(*pq.suffix)} : ""); + + return product_selector{ + .creator = identifier(c), .layer = pq.layer, .suffix = identifier(suff)}; + }; + + mod->ph_module->observe(cname, py_callback{callable}, nconcur) + .input_family(make_product_selector(Is)...); + }; + + if (!unroll_switch(input_selectors.size(), observe_N_args)) { PyErr_SetString(PyExc_TypeError, "unsupported number of inputs"); Py_DECREF(callable); return nullptr; } - } Py_DECREF(callable); Py_RETURN_NONE; @@ -1248,11 +1291,12 @@ static PyObject* sc_provide(py_phlex_source* src, PyObject* args, PyObject* kwds PyErr_Clear(); } - // translate and validate the output "query" - // Since a query in Python is just a dictionary, it isn't called out in the user API as a query - auto opq = validate_query(output); + // translate and validate the output "selectors" + // Since a selector in Python is just a dictionary, it isn't called out in the user + // API as a selector + auto opq = validate_selector(output); if (!opq.has_value()) { - // validate_query has set a python exception with details about the error + // validate_selector has set a python exception with details about the error return nullptr; } diff --git a/test/python/CMakeLists.txt b/test/python/CMakeLists.txt index aa79543c4..276180f55 100644 --- a/test/python/CMakeLists.txt +++ b/test/python/CMakeLists.txt @@ -192,6 +192,13 @@ list(APPEND ACTIVE_PY_CPHLEX_TESTS py:reduce) add_test(NAME py:coverage COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pycoverage.jsonnet) list(APPEND ACTIVE_PY_CPHLEX_TESTS py:coverage) +# numba tests if installed +if(HAS_NUMBA) + # phlex-based tests that require numpy support + add_test(NAME py:jited COMMAND phlex::phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pyjited.jsonnet) + list(APPEND ACTIVE_PY_CPHLEX_TESTS py:jited) +endif() + add_test( NAME py:mismatch COMMAND ${PROJECT_BINARY_DIR}/bin/phlex -c ${CMAKE_CURRENT_SOURCE_DIR}/pymismatch.jsonnet diff --git a/test/python/jited.py b/test/python/jited.py new file mode 100644 index 000000000..875a270c5 --- /dev/null +++ b/test/python/jited.py @@ -0,0 +1,61 @@ +"""Basic Numba tests for all supported types. + +Smallest possible tests with a mixture of Python and Numba: Python +providers to produce data, Numba algorithms to transform them, and Python +observers for verification. +""" + +import numba.core.decorators as nb_dec +import numpy as np +from adder import add + +from phlex import Variant + +# arg0 suff, arg1 suff, type, result +specs = ( + ("i", "j", np.int32, 1), + ("u1", "u2", np.uint32, 1), + ("l1", "l2", np.int64, 1), + ("ul1", "ul2", np.uint64, 100), + ("f1", "f2", np.float32, 1.), + ("d1", "d2", np.float64, 1.), +) + + +def PHLEX_REGISTER_ALGORITHMS(m, config): + """Register Numba-jited `add` algorithm variants as a transformation. + + Use the standard Phlex `transform` registration to insert a node in the + execution graph of a Numba-jited Python function that receives two inputs + and produces their sum as an ouput. + + Similarly, use the standard Phlex `observe` to add verifier nodes. + + Args: + m (internal): Phlex registrar representation. + config (internal): Phlex configuration representation. + + Returns: + None + """ + + def new_o(x): + def o(y): + assert y == x + return o + + for arg0, arg1, t, res in specs: + tn = t.__name__ + + f_a = nb_dec.cfunc(f"{tn}({tn}, {tn})", nogil=True, nopython=True, cache=True)(add) + m.transform(f_a, + name="add_"+tn, + input_family=[{"creator": "input", "layer": "event", "suffix": arg0}, + {"creator": "input", "layer": "event", "suffix": arg1}], + output_product_suffixes=["sum_"+tn], + concurrency=4) + + o = Variant(new_o(res), {"y": t, "return": None}, "observe_" + tn) + m.observe(o, + input_family=[{"creator": "add_" + tn, "layer": "event", "suffix": "sum_"+tn}]) + diff --git a/test/python/pyjited.jsonnet b/test/python/pyjited.jsonnet new file mode 100644 index 000000000..317633cc1 --- /dev/null +++ b/test/python/pyjited.jsonnet @@ -0,0 +1,18 @@ +{ + driver: { + cpp: 'generate_layers', + layers: { + event: { parent: 'job', total: 100, starting_number: 1 }, + }, + }, + sources: { + provider: { + cpp: 'cppsource4py', + }, + }, + modules: { + pyadd: { + py: 'jited', + }, + }, +}