diff --git a/tmva/sofie/inc/TMVA/RModel.hxx b/tmva/sofie/inc/TMVA/RModel.hxx index ec4e1115b759d..08ad14149aacf 100644 --- a/tmva/sofie/inc/TMVA/RModel.hxx +++ b/tmva/sofie/inc/TMVA/RModel.hxx @@ -45,6 +45,8 @@ private: MemoryPoolInfo fIntermediateMemoryInfo; /// fIntermediateTensorFrequencyLookup; /// & shapeValues, bool scalar = false); + void AddExtraCodeForDimShapes(const std::string & code) { fExtraCodeForDimShapes += code; } // add and initialize subgraph to the model void InitializeSubGraph(std::shared_ptr graph); @@ -239,7 +242,8 @@ public: bool UseVDT() const { return fUseVDT;} // Use the ClassDef macro to allow definition of custom streaming - ClassDefNV(RModel, 3); + // Use Version 0 since we don't support for time being ROOT I/O streaming of RModel objects + ClassDefNV(RModel, 4); }; // need to implement here templated member functions and its specialization diff --git a/tmva/sofie/inc/TMVA/ROperator_Cast.hxx b/tmva/sofie/inc/TMVA/ROperator_Cast.hxx index cace65040c772..85f7ac40e6aac 100644 --- a/tmva/sofie/inc/TMVA/ROperator_Cast.hxx +++ b/tmva/sofie/inc/TMVA/ROperator_Cast.hxx @@ -66,8 +66,9 @@ public: if (!fIsOutputConstant) model.AddIntermediateTensor(fNY, fType, fShape); if (model.Verbose()) { - std::cout << "Cast : " << ConvertTypeToString(inputType) << " " << fNX << " -> " << ConvertTypeToString(fType) << " for " << fNY - << " shape " << ConvertDimShapeToString(fShape); + std::cout << "Cast : " << ConvertTypeToString(inputType) << " " << fNX << " -> " << ConvertTypeToString(fType); + if (fType == ETensorType::BOOL) std::cout << " (converted from BOOL) "; + std::cout << " for " << fNY << " shape " << ConvertDimShapeToString(fShape); if (fIsOutputConstant) std::cout << " (constant) "; std::cout << std::endl; } @@ -87,7 +88,11 @@ public: out << SP << "for (int id = 0; id < " << length << " ; id++){\n"; - out << SP << SP << "tensor_" << fNY << "[id] = static_cast<"<< ConvertTypeToString(fType) << ">(tensor_" << fNX << "[id]);\n"; + // need to handle bool case separatly since casting to uint8 will not give right result + if (fType == ETensorType::BOOL) + out << SP << SP << "tensor_" << fNY << "[id] = (tensor_" << fNX << "[id] != 0) ? 1 : 0;\n"; + else + out << SP << SP << "tensor_" << fNY << "[id] = static_cast<"<< ConvertTypeToString(fType) << ">(tensor_" << fNX << "[id]);\n"; out << SP << "}\n"; return out.str(); diff --git a/tmva/sofie/inc/TMVA/ROperator_Gather.hxx b/tmva/sofie/inc/TMVA/ROperator_Gather.hxx index ad91d1256ded1..cdc28a6abfcd9 100644 --- a/tmva/sofie/inc/TMVA/ROperator_Gather.hxx +++ b/tmva/sofie/inc/TMVA/ROperator_Gather.hxx @@ -17,6 +17,7 @@ class ROperator_Gather final : public ROperator { private: + bool fIsOutputParamShape = false; // for shape outputs int64_t fAttrAxis = 0; std::string fNX; @@ -26,11 +27,13 @@ private: std::vector fShapeX; std::vector fShapeIndices; std::vector fShapeY; + std::vector fOutputShapeData; std::vector fIndices; // indices vector in case they are known at initialization std::string fType; + public: ROperator_Gather(){} ROperator_Gather(int64_t attrAxis, std::string nameX, std::string nameIndices, std::string nameY): @@ -121,17 +124,17 @@ public: else if (model.IsShapeTensor(fNX) && q <=1 && fIndices.size() > 0) { auto inputData = model.GetShapeTensorValues(fNX); // if r == 1 and q<=1 then output length is 1 (is a scalar or tensor of size1) - std::vector outputData(1); - outputData[0] = inputData[fIndices[0]]; - if (outputData[0].isParam) { - fIsOutputConstant = true; + fOutputShapeData.resize(1); + fOutputShapeData[0] = inputData[fIndices[0]]; + if (fOutputShapeData[0].isParam) { + fIsOutputParamShape = true; // shapeY can be scalar or vector of size1 - model.AddShapeTensor(fNY, outputData, fShapeY.size() == 0); + model.AddShapeTensor(fNY, fOutputShapeData, fShapeY.size() == 0); if (model.Verbose()) std::cout << "Gather: " << fNX << " " << ConvertDimShapeToString(fShapeX) << " -> " << fNY << " with shape " << ConvertDimShapeToString(fShapeY) - << " and values " << ConvertDimShapeToString(outputData) << " (shape) " << std::endl; + << " and values " << ConvertDimShapeToString(fOutputShapeData) << " (shape) " << std::endl; } else { - int64_t value = static_cast(outputData[0].dim); + int64_t value = static_cast(fOutputShapeData[0].dim); auto shapeY = ConvertShapeToInt(fShapeY); model.AddConstantTensor(fNY, shapeY, &value); fIsOutputConstant = true; @@ -140,11 +143,11 @@ public: << " and values {" << value << "} (constant) " << std::endl; } } - if (!fIsOutputConstant) { + if (!fIsOutputConstant && !fIsOutputParamShape) { // Add output tensor model.AddIntermediateTensor(fNY, model.GetTensorType(fNX), fShapeY); fType = ConvertTypeToString(model.GetTensorType(fNX)); - if (model.Verbose()) + //if (model.Verbose()) std::cout << "Gather: input " << fNX << " " << ConvertDimShapeToString(fShapeX) << " indices " << fNIndices << ConvertDimShapeToString(fShapeIndices) << " -> " << fNY << " with shape " << ConvertDimShapeToString(fShapeY) << std::endl; } @@ -159,6 +162,14 @@ public: out << "//--------------------(constant)----------\n"; return out.str(); } + if (fIsOutputParamShape) { + // no code to generate here for param shape output. Tensor output is defined in Session constructor + out << "//--------------------(shape)----------\n"; + for (int i = 0; i < static_cast(fOutputShapeData.size()); i++) { + out << SP << "tensor_" << fNY << "[" << i << " ] = " << fOutputShapeData[i].GetVal() << ";\n"; + } + return out.str(); + } // The shape of the output is q + r - 1 size_t r = fShapeX.size(); // Indices of shape q diff --git a/tmva/sofie/inc/TMVA/ROperator_NonZero.hxx b/tmva/sofie/inc/TMVA/ROperator_NonZero.hxx index 8587035f8d44b..0aebf5b14309b 100644 --- a/tmva/sofie/inc/TMVA/ROperator_NonZero.hxx +++ b/tmva/sofie/inc/TMVA/ROperator_NonZero.hxx @@ -19,6 +19,7 @@ private: std::string fNX; std::string fNY; + std::string fNonZeroParam; // name of the parameter used to store the number of non zero elements when output is not constant std::vector fShapeX; std::vector fShapeY; @@ -93,7 +94,15 @@ public: fShapeY[0] = fShapeX.size(); // identify as -1 since we will declare maximum as size of input - fShapeY[1] = Dim{std::string("v_NonZero_") + fNX, static_cast(-1)}; + // we will compute at run time the actual number of non zero and rearrange the output vector accordingly + fNonZeroParam = "v_NonZero_" + fNX; + fShapeY[1] = Dim{fNonZeroParam, static_cast(-1)}; + + // declare the parameter for number of non zero elements, used when output is not constant + auto inputLength = ConvertDimShapeToLength(fShapeX); + std::string codeDecl = SP + "size_t " + fNonZeroParam + " = " + inputLength + ";\n"; + codeDecl += SP + "fV_NonZero_" + fNX + " = " + fNonZeroParam + ";\n"; + model.AddExtraCodeForDimShapes(codeDecl); model.AddIntermediateTensor(fNY, ETensorType::INT64, fShapeY); if (model.Verbose()) { @@ -104,13 +113,12 @@ public: std::string GenerateSessionMembersCode(std::string /*opName*/) override { if (fIsOutputConstant) return ""; - // define output value used as max non zero with max size = input shape * N - auto inputLength = ConvertDimShapeToLength(fShapeX); std::stringstream out; - out << SP << "size_t fV_NonZero_" << fNX << " = " << inputLength << ";\n"; + out << SP << "size_t fV_NonZero_" << fNX << " = 0;\n"; return out.str(); } + std::string Generate(std::string opName) override { if (fIsOutputConstant) { return ""; @@ -127,9 +135,9 @@ public: inputLength = ConvertShapeToLength(intShapeX); size_t dims = fShapeX.size(); - out << "\n//------ NonZero\n"; + out << "\n//------ NonZero -> " << ConvertDimShapeToString(fShapeY) << "\n"; - std::string vnonzero = "v_NonZero_" + fNX; + std::string vnonzero = fNonZeroParam; // loop on input indices out << SP << "size_t offset_" << opName << " = 0;\n"; diff --git a/tmva/sofie/inc/TMVA/ROperator_Reshape.hxx b/tmva/sofie/inc/TMVA/ROperator_Reshape.hxx index 4168144f2e708..41946a33085b5 100644 --- a/tmva/sofie/inc/TMVA/ROperator_Reshape.hxx +++ b/tmva/sofie/inc/TMVA/ROperator_Reshape.hxx @@ -332,7 +332,7 @@ public: } std::string Generate(std::string opName) override { - if (fIsOutputConstant) return ""; //no op for constant tensors + std::stringstream out; std::string opType = "Reshape"; @@ -345,6 +345,8 @@ public: out << SP << "///--------" << opType << " operator " << opName << " --> " << ConvertDimShapeToString(fShapeOutput) << "\n"; + if (fIsOutputConstant) return out.str(); //no op for constant tensors + // in case of dynamic output shape we need to set the shape value from input shape tensor // and take case of the zero values if (fDynamicShape) { diff --git a/tmva/sofie/inc/TMVA/ROperator_Slice.hxx b/tmva/sofie/inc/TMVA/ROperator_Slice.hxx index d119fa3a29ea1..dfdf492893113 100644 --- a/tmva/sofie/inc/TMVA/ROperator_Slice.hxx +++ b/tmva/sofie/inc/TMVA/ROperator_Slice.hxx @@ -339,7 +339,7 @@ public: } model.AddIntermediateTensor(fNOutput, model.GetTensorType(fNData), fShapeOutput); - if (fIdentitySlice) model.AddAliasTensor(fNOutput, fNData); + //if (fIdentitySlice) model.AddAliasTensor(fNOutput, fNData); if (model.Verbose()) { std::cout << "Slice " << fNData << " " << ConvertDimShapeToString(fShapeInput) @@ -366,8 +366,9 @@ public: size_t ndim = fShapeInput.size(); if (fIdentitySlice) { - out << "/// Slice is just an identity (copy pointers) \n"; - out << SP << "tensor_" << fNOutput << " = tensor_" << fNData << ";\n"; + out << "/// Slice is just an identity (copy) \n"; + //out << SP << "tensor_" << fNOutput << " = const_cast<" << ConvertTypeToString(fOutputType) << " *>(tensor_" << fNData << ");\n"; + out << SP << "std::copy(tensor_" << fNData << ", tensor_" << fNData << " + " << ConvertDimShapeToLength(fShapeInput) << ", tensor_" << fNOutput << ");\n"; return out.str(); } diff --git a/tmva/sofie/inc/TMVA/ROperator_Softmax.hxx b/tmva/sofie/inc/TMVA/ROperator_Softmax.hxx index db79c2b6d0f7d..025d6d678088a 100644 --- a/tmva/sofie/inc/TMVA/ROperator_Softmax.hxx +++ b/tmva/sofie/inc/TMVA/ROperator_Softmax.hxx @@ -62,12 +62,14 @@ public: } } - std::string Generate(std::string OpName) override { - OpName = "op_" + OpName; + std::string Generate(std::string opName) override { + opName = "op_" + opName; if (fShape.empty()) { throw std::runtime_error("TMVA SOFIE Operator Softmax called to Generate without being initialized first"); } std::stringstream out; + out << "///------- Softmax " << opName << " ---> " // << fNY << " " + << ConvertDimShapeToString(fShape) << "\n" << std::endl; size_t size = fShape.size(); auto length_str = ConvertDimShapeToLength(fShape); size_t axis = fAttrAxis < 0 ? size + fAttrAxis : fAttrAxis; @@ -85,7 +87,7 @@ public: num_rows = "(" + length_str + ") / (" + axis_size + ")"; } - out << "\n" << SP << "//------ SOFTMAX - " << size << " " << length_str << " " << axis << "\n"; + out << SP << "//----- softmax axis is last one - " << axis << "\n"; out << SP << "for (int i = 0; i < " << num_rows << "; ++i) {\n"; out << SP << SP << "size_t offset = i * " << axis_size << ";\n"; out << SP << SP << fType << " const * x_ptr = &tensor_" << fNX << "[offset];\n"; @@ -111,6 +113,7 @@ public: out << SP << "}\n"; } else { + // generic case for any axis auto stride = UTILITY::ComputeStrideFromShape(fShape); size_t k = 0; std::vector l(size); @@ -118,7 +121,7 @@ public: if (i != axis) { for (size_t j = 0; j < k; j++) out << SP; l[i] = std::string("i") + std::to_string(i); - out << "for (int " << l[i] << " = 0; " << l[i] << " < " << fShape[i] << "; " << l[i] << "++) {\n"; + out << SP << "for (int " << l[i] << " = 0; " << l[i] << " < " << fShape[i] << "; " << l[i] << "++) {\n"; k++; } } @@ -167,7 +170,8 @@ public: out << "for (int i = 0; i < " << fShape[axis] << "; i++) {\n"; for (size_t j = 0; j < size; j++) out << SP; out << "size_t id = index + i"; - if (stride[axis].GetVal() != "1") out << "*(" << stride[axis] << ");\n"; + if (stride[axis].GetVal() != "1") out << "*(" << stride[axis] << ")"; + out << ";\n"; for (size_t j = 0; j < size; j++) out << SP; out << "tensor_" << fNY << "[id] /= sum;\n"; if (fLogSoftmax) { diff --git a/tmva/sofie/inc/TMVA/ROperator_Where.hxx b/tmva/sofie/inc/TMVA/ROperator_Where.hxx index 4c42ad6d655d9..073c7e1ec19e7 100644 --- a/tmva/sofie/inc/TMVA/ROperator_Where.hxx +++ b/tmva/sofie/inc/TMVA/ROperator_Where.hxx @@ -7,32 +7,36 @@ #include -namespace TMVA { -namespace Experimental { -namespace SOFIE { +namespace TMVA{ +namespace Experimental{ +namespace SOFIE{ -template -class ROperator_Where final : public ROperator { + + +template +class ROperator_Where final : public ROperator{ private: bool fIsInputBoolTensor = false; - // Tensor names: C = condition, X = true branch, Y = false branch, Z = output - std::string fNC; // condition (bool) - std::string fNX; // true-branch values - std::string fNY; // false-branch values - std::string fNZ; // output - std::string fNBroadcastedC; + + std::string fNX; + std::string fNY; + std::string fNC; std::string fNBroadcastedX; std::string fNBroadcastedY; + std::string fNBroadcastedC; + std::string fNZ; - // Static shapes (used when all inputs are non-dynamic) - std::vector fShapeC; + + + // static shapes (used when tensors are not dynamic) ) std::vector fShapeX; std::vector fShapeY; + std::vector fShapeC; std::vector fShapeZ; - // Dynamic shapes (Dim-aware, used when any input is dynamic) + // Dynamic generic shapes std::vector fDimShapeC; std::vector fDimShapeX; std::vector fDimShapeY; @@ -46,47 +50,37 @@ private: int fBroadcastFlag = 0; public: - ROperator_Where() {} - ROperator_Where(const std::string &nameC, - const std::string &nameX, - const std::string &nameY, - const std::string &nameZ) - : fNC(UTILITY::Clean_name(nameC)), - fNX(UTILITY::Clean_name(nameX)), - fNY(UTILITY::Clean_name(nameY)), - fNZ(UTILITY::Clean_name(nameZ)) - { - fInputTensorNames = { fNC, fNX, fNY }; - fOutputTensorNames = { fNZ }; - } + ROperator_Where(){} + ROperator_Where(const std::string & nameC, const std::string & nameX, const std::string & nameY, const std::string & nameZ): + fNX(UTILITY::Clean_name(nameX)), fNY(UTILITY::Clean_name(nameY)), fNC(UTILITY::Clean_name(nameC)), fNZ(UTILITY::Clean_name(nameZ)){ + fInputTensorNames = { fNX, fNY, fNC }; + fOutputTensorNames = { fNZ }; + } // type of output given input - std::vector TypeInference(std::vector input) override - { - // output type follows X (and Y), not C (which is bool) - return { input[1] }; + std::vector TypeInference(std::vector input) override { + return input; } // shape of output tensors given input tensors - std::vector> ShapeInference(std::vector> input) override - { - // conservative: assume same shape (broadcasting resolved in Initialize) - return { input[1] }; + std::vector> ShapeInference(std::vector> input) override { + // assume now inputs have same shape (no broadcasting) + auto ret = std::vector>(1, input[0]); // return vector size 1 with first input + return ret; } - void Initialize(RModel &model) override - { - // ---------------------------------------------------------------- // - // Check all inputs exist - // ---------------------------------------------------------------- // - if (!model.CheckIfTensorAlreadyExist(fNC)) - throw std::runtime_error(std::string("TMVA SOFIE Where Op: condition tensor ") + fNC + " not found in model"); - if (!model.CheckIfTensorAlreadyExist(fNX)) - throw std::runtime_error(std::string("TMVA SOFIE Where Op: X tensor ") + fNX + " not found in model"); - if (!model.CheckIfTensorAlreadyExist(fNY)) - throw std::runtime_error(std::string("TMVA SOFIE Where Op: Y tensor ") + fNY + " not found in model"); - - // condition tensor is bool (uint8) - mark if it is a live input tensor + void Initialize(RModel& model) override { + // input must be a graph input, or already initialized intermediate tensor + if (!model.CheckIfTensorAlreadyExist(fNX)){ + throw std::runtime_error(std::string("TMVA SOFIE Where Op Input Tensor ") + fNX + "is not found in model"); + } + if (!model.CheckIfTensorAlreadyExist(fNY)) { + throw std::runtime_error(std::string("TMVA SOFIE Where Op Input Tensor ") + fNY + "is not found in model"); + } + if (!model.CheckIfTensorAlreadyExist(fNC)) { + throw std::runtime_error(std::string("TMVA SOFIE Where Op Input Tensor ") + fNC + "is not found in model"); + } + // check if fNC input tensor is boolean if (model.IsReadyInputTensor(fNC)) fIsInputBoolTensor = true; @@ -117,13 +111,14 @@ public: fDimShapeY = ConvertShapeToDim(fShapeY); } + if (model.Verbose()) { if (dynamicInputs & 1) std::cout << "Where : condition " << fNC << " is dynamic " << ConvertDimShapeToString(fDimShapeC) << "\n"; if (dynamicInputs & 2) - std::cout << "Where : X " << fNX << " is dynamic " << ConvertDimShapeToString(fDimShapeX) << "\n"; + std::cout << "Where : " << fNX << " is dynamic " << ConvertDimShapeToString(fDimShapeX) << "\n"; if (dynamicInputs & 4) - std::cout << "Where : Y " << fNY << " is dynamic " << ConvertDimShapeToString(fDimShapeY) << "\n"; + std::cout << "Where : Y " << fNZ << " is dynamic " << ConvertDimShapeToString(fDimShapeZ) << "\n"; } // ---------------------------------------------------------------- // @@ -131,79 +126,186 @@ public: // ---------------------------------------------------------------- // if (dynamicInputs == 0) { - // Multidirectional broadcast over all three tensors - auto retXY = UTILITY::MultidirectionalBroadcastShape(fShapeX, fShapeY); - fBroadcastFlag = retXY.first; - fShapeZ = retXY.second; - // also factor in C - auto retCZ = UTILITY::MultidirectionalBroadcastShape(fShapeC, fShapeZ); - fBroadcastFlag |= retCZ.first; - fShapeZ = retCZ.second; - - bool allConstant = model.IsInitializedTensor(fNC) && - model.IsInitializedTensor(fNX) && - model.IsInitializedTensor(fNY); - - if (allConstant) { - // ---------------------------------------------------------- - // Constant folding: evaluate Where at model initialisation - // ---------------------------------------------------------- - auto broadcastIfNeeded = [&](const std::string &name, - const std::vector &shape, - std::string &bcName, - const std::string &prefix) { - if (shape != fShapeZ) { - bcName = prefix + name + "to" + fNZ; - auto data = model.GetInitializedTensorData(name); - std::shared_ptr bcData( - UTILITY::UnidirectionalBroadcast(static_cast(data.get()), shape, fShapeZ), + bool broadcast = !UTILITY::AreSameShape(fShapeX, fShapeY) || !UTILITY::AreSameShape(fShapeX, fShapeC); + if (broadcast) { + // find shape to broadcast between X,Y,C looking for max length + size_t lengthX = ConvertShapeToLength(fShapeX); + size_t lengthY = ConvertShapeToLength(fShapeY); + size_t lengthC = ConvertShapeToLength(fShapeC); + bool broadcastX = false, broadcastY = false, broadcastC = false; + if (lengthX >= lengthY && lengthX >= lengthC) { + fShapeZ = fShapeX; + // broadcast Y and C if different than X + broadcastY = (lengthY != lengthX); + broadcastC = (lengthC != lengthX); + } else if (lengthY >= lengthX && lengthY >= lengthC) { + fShapeZ = fShapeY; + // broadcast X and C if different than Y + broadcastX = (lengthX != lengthY); + broadcastC = (lengthC != lengthY); + } else if (lengthC >= lengthX && lengthC >= lengthY) { + fShapeZ = fShapeC; + // broadcast X and Y if different than C + broadcastX = (lengthX != lengthC); + broadcastY = (lengthY != lengthC); + } + + // Broadcast X to Z + if (broadcastX) { + fNBroadcastedX = "BC_" + fNX + "_to_" + fNZ; + if (model.IsInitializedTensor(fNX)) { + auto data = model.GetInitializedTensorData(fNX); + std::shared_ptr broadcastedData( + UTILITY::UnidirectionalBroadcast(static_cast(data.get()), fShapeX, fShapeZ), std::default_delete()); - model.AddConstantTensor(bcName, model.GetTensorType(name), fShapeZ, bcData); + // Update the data and the shape of X + model.AddConstantTensor(fNBroadcastedX, model.GetTensorType(fNX), fShapeZ, broadcastedData); + fShapeX = fShapeZ; + } else { + // I need to prepend to shape of X the extra dimensions added for broadcasting to Z + if (fShapeX.size() < fShapeZ.size()) { + size_t nPrepend = fShapeZ.size() - fShapeX.size(); + fShapeX.insert(fShapeX.begin(), nPrepend, 1); + } } - }; - - broadcastIfNeeded(fNX, fShapeX, fNBroadcastedX, "BC_"); - broadcastIfNeeded(fNY, fShapeY, fNBroadcastedY, "BC_"); - broadcastIfNeeded(fNC, fShapeC, fNBroadcastedC, "BC_"); + } + // Broadcast Y to Z + if (broadcastY) { + fNBroadcastedY = "BC_" + fNY + "_to_" + fNZ; + if (model.IsInitializedTensor(fNY)) { + auto data = model.GetInitializedTensorData(fNY); + std::shared_ptr broadcastedData( + UTILITY::UnidirectionalBroadcast(static_cast(data.get()), fShapeY, fShapeZ), + std::default_delete()); + // do not update tensor B but add broadcasted one (since it can be input to some other operators) + model.AddConstantTensor(fNBroadcastedY, model.GetTensorType(fNY), fShapeZ, broadcastedData); + fShapeY = fShapeZ; + } else { + // I need to prepend to shape of Y the extra dimensions added for broadcasting to Z + if (fShapeY.size() < fShapeZ.size()) { + size_t nPrepend = fShapeZ.size() - fShapeY.size(); + fShapeY.insert(fShapeY.begin(), nPrepend, 1); + } - const std::string &nameC = fNBroadcastedC.empty() ? fNC : fNBroadcastedC; - const std::string &nameX = fNBroadcastedX.empty() ? fNX : fNBroadcastedX; - const std::string &nameY = fNBroadcastedY.empty() ? fNY : fNBroadcastedY; + } + } + // Broadcast C to Z + if (broadcastC) { + fNBroadcastedC = "BC_" + fNC + "_to_" + fNZ; + if (model.IsInitializedTensor(fNC)) { + auto data = model.GetInitializedTensorData(fNC); + std::shared_ptr broadcastedData( + UTILITY::UnidirectionalBroadcast(static_cast(data.get()), fShapeC, fShapeZ), + std::default_delete()); + // do not update tensor C but add broadcasted one (since it can be input to some other operators) + model.AddConstantTensor(fNBroadcastedC, model.GetTensorType(fNC), fShapeZ, broadcastedData); + fShapeC = fShapeZ; + } else { + // I need to prepend to shape of C the extra dimensions added for broadcasting to Z + if (fShapeC.size() < fShapeZ.size()) { + size_t nPrepend = fShapeZ.size() - fShapeC.size(); + fShapeC.insert(fShapeC.begin(), nPrepend, 1); + } + } + } + } else { + fShapeZ = fShapeX; + } + // check case of constant output (if all inputs are defined) + if (model.IsInitializedTensor(fNC)) { + std::cout << "Where op: " << fNC << " is initialized\n"; + std::string nameC = fNBroadcastedC.empty() ? fNC : fNBroadcastedC; auto dataC = static_cast(model.GetInitializedTensorData(nameC).get()); - auto dataX = static_cast (model.GetInitializedTensorData(nameX).get()); - auto dataY = static_cast (model.GetInitializedTensorData(nameY).get()); - - size_t len = ConvertShapeToLength(fShapeZ); - std::vector dataZ(len); - for (size_t i = 0; i < len; ++i) - dataZ[i] = dataC[i] ? dataX[i] : dataY[i]; - - model.AddConstantTensor(fNZ, fShapeZ, dataZ.data()); model.SetNotWritableInitializedTensor(nameC); - model.SetNotWritableInitializedTensor(nameX); - model.SetNotWritableInitializedTensor(nameY); + T *dataX = nullptr; + T *dataY = nullptr; + std::vector shapeDataX; + std::vector shapeDataY; + if (model.IsInitializedTensor(fNX)) { + std::cout << "Where op: " << fNX << " is initialized\n"; + std::string nameX = fNBroadcastedX.empty() ? fNX : fNBroadcastedX; + dataX = static_cast(model.GetInitializedTensorData(nameX).get()); + // flag tensors to not be written in a file + model.SetNotWritableInitializedTensor(nameX); + } else if (model.IsShapeTensor(fNX)) { + std::cout << "Where op: " << fNX << " is a shape tensor\n"; + shapeDataX = model.GetShapeTensorValues(fNX); + } + if (model.IsInitializedTensor(fNY)) { + std::cout << "Where op: " << fNY << " is initialized\n"; + std::string nameY = fNBroadcastedY.empty() ? fNY : fNBroadcastedY; + dataY = static_cast(model.GetInitializedTensorData(nameY).get()); + model.SetNotWritableInitializedTensor(nameY); + } else if (model.IsShapeTensor(fNY)) { + std::cout << "Where op: " << fNY << " is a shape tensor\n"; + shapeDataY = model.GetShapeTensorValues(fNY); + } + std::vector dataZ; // used in case output is constant tensor + std::vector shapeDataZ; // used in case output is a shape tensor (can be also constant if all + // dimensions are not parametric) + // if fNC (condition) is initialized we know the output is a shape or a constant tensor, + // so we can compute it at initialization and add it as a constant tensor to the model + // (and not add the operator output as intermediate tensor to the model) + bool isOutputConstantTensor = true; + if (dataX && dataY) { + dataZ.resize(ConvertShapeToLength(fShapeZ)); + for (size_t i = 0; i < dataZ.size(); i++) + dataZ[i] = (dataC[i]) ? dataX[i] : dataY[i]; + std::cout << "data A and B : dataZ constant: " << ConvertValuesToString(dataZ) << std::endl; + } else if (dataX && shapeDataY.size() > 0) { + shapeDataZ.resize(ConvertShapeToLength(fShapeZ)); + for (size_t i = 0; i < shapeDataZ.size(); i++) { + shapeDataZ[i] = (dataC[i]) ? Dim{size_t(dataX[i])} : shapeDataY[i]; + isOutputConstantTensor &= !shapeDataZ[i].isParam; + } + std::cout << "data A but shapeB " << ConvertDimShapeToString(shapeDataY) << " " + << isOutputConstantTensor << std::endl; + } else if (dataY && shapeDataX.size() > 0) { + shapeDataZ.resize(ConvertShapeToLength(fShapeZ)); + for (size_t i = 0; i < shapeDataZ.size(); i++) { + shapeDataZ[i] = (dataC[i]) ? shapeDataY[i] : Dim{size_t(dataY[i])}; + isOutputConstantTensor &= !shapeDataZ[i].isParam; + } + std::cout << "data B but shapeA " << ConvertDimShapeToString(shapeDataX) << " " + << isOutputConstantTensor << std::endl; + } else if (shapeDataY.size() > 0 && shapeDataX.size() > 0) { + shapeDataZ.resize(ConvertShapeToLength(fShapeZ)); + for (size_t i = 0; i < shapeDataZ.size(); i++) { + shapeDataZ[i] = (dataC[i]) ? shapeDataX[i] : shapeDataY[i]; + isOutputConstantTensor &= !shapeDataZ[i].isParam; + } + std::cout << " shapeA and B " << ConvertDimShapeToString(shapeDataX) << " shapeB " + << ConvertDimShapeToString(shapeDataY) << " " << isOutputConstantTensor << std::endl; + } fIsOutputConstant = true; - fOutputTensorNames.pop_back(); - - if (model.Verbose()) - std::cout << "Where --> " << fNZ << " " << ConvertShapeToString(fShapeZ) - << " : " << ConvertValuesToString(dataZ) << " (constant)\n"; - } else { - // ---------------------------------------------------------- - // Non-constant static tensors - we don't need to broadcast tensors - // ---------------------------------------------------------- + // add as constant or shape tensor depending on the case + if (dataZ.size() > 0) + model.AddConstantTensor(fNZ, fShapeZ, dataZ.data()); + else if (shapeDataZ.size() > 0) + model.AddShapeTensor(fNZ, shapeDataZ, fShapeZ.size() == 0); + else { + fIsOutputConstant = false; + } + if (fIsOutputConstant && model.Verbose()) + std::cout << "Where op ---> " << fNZ << " " << ConvertShapeToString(fShapeZ) << " : " + << ((dataZ.size() > 0) ? ConvertValuesToString(dataZ) : ConvertDimShapeToString(shapeDataZ)) + << ((dataZ.size() > 0) ? " (constant)" : " (shape)") << std::endl; + + // output is a constant tensor + if (fIsOutputConstant) + fOutputTensorNames.pop_back(); + } + if (!fIsOutputConstant) { fDimShapeZ = ConvertShapeToDim(fShapeZ); model.AddIntermediateTensor(fNZ, model.GetTensorType(fNX), fShapeZ); - if (model.Verbose()) - std::cout << "Where : C=" << fNC << " " << ConvertShapeToString(fShapeC) - << " X=" << fNX << " " << ConvertShapeToString(fShapeX) - << " Y=" << fNY << " " << ConvertShapeToString(fShapeY) - << " --> Z=" << fNZ << " " << ConvertShapeToString(fShapeZ) << "\n"; + std::cout << "Where : condition : " << fNC << " " << ConvertShapeToString(fShapeC) << " X " + << fNX << " " << ConvertShapeToString(fShapeX) << " Y " << fNY << " " + << ConvertShapeToString(fShapeY) << " ---> " << fNZ << " " << ConvertShapeToString(fShapeZ) + << std::endl; } - } else { // ---------------------------------------------------------------- // // Dynamic path: at least one input has a parametric shape @@ -227,7 +329,7 @@ public: for (size_t i = 0; i < fDimShapeZ.size(); i++) { auto &s = fDimShapeZ[i]; if (s.isParam && s.param.find("std::max") != std::string::npos) { - // prefer X dim over Y dim + // prefer A dim over B dim if (i < fDimShapeX.size() && IsInputDimParam(fDimShapeX[i].param)) { s = (fDimShapeX[i].dim != 1) ? fDimShapeX[i] : fDimShapeY[i]; } else if (i < fDimShapeY.size() && IsInputDimParam(fDimShapeY[i].param)) { @@ -236,35 +338,42 @@ public: } } } + // I need to prepend to shape of X,Y,C the extra dimensions added for broadcasting to Z + if (fDimShapeX.size() < fDimShapeZ.size()) { + size_t nPrepend = fDimShapeZ.size() - fDimShapeX.size(); + fDimShapeX.insert(fDimShapeX.begin(), nPrepend, Dim{1}); + } + if (fDimShapeY.size() < fDimShapeZ.size()) { + size_t nPrepend = fDimShapeZ.size() - fDimShapeY.size(); + fDimShapeY.insert(fDimShapeY.begin(), nPrepend, Dim{1}); + } + if (fDimShapeC.size() < fDimShapeZ.size()) { + size_t nPrepend = fDimShapeZ.size() - fDimShapeC.size(); + fDimShapeC.insert(fDimShapeC.begin(), nPrepend, Dim{1}); + } model.AddIntermediateTensor(fNZ, model.GetTensorType(fNX), fDimShapeZ); if (model.Verbose()) std::cout << "Where (dynamic) : C=" << ConvertDimShapeToString(fDimShapeC) - << " X=" << ConvertDimShapeToString(fDimShapeX) - << " Y=" << ConvertDimShapeToString(fDimShapeY) - << " --> Z=" << ConvertDimShapeToString(fDimShapeZ) << "\n"; + << " A=" << ConvertDimShapeToString(fDimShapeX) + << " B=" << ConvertDimShapeToString(fDimShapeY) + << " --> Y=" << ConvertDimShapeToString(fDimShapeZ) << "\n"; } } - std::string GenerateInitCode() override - { + std::string GenerateInitCode() override { std::stringstream out; return out.str(); } - std::string Generate(std::string opName) override - { - if (fIsOutputConstant) return ""; + std::string Generate(std::string opName) override { opName = "op_" + opName; - - if (fDimShapeZ.empty()) { - throw std::runtime_error("TMVA SOFIE Where Op called to Generate without being initialized first"); - } - std::stringstream out; out << SP << "\n//------ WHERE " << opName << " --> " << ConvertDimShapeToString(fDimShapeZ) << "\n"; + if (fIsOutputConstant) return out.str(); + // ---------------------------------------------------------------- // // Runtime broadcast validation (dynamic shapes, flag bit 4) @@ -281,14 +390,14 @@ public: out << SP << SP << "if (" << fDimShapeX[i] << " != 1 && " << fDimShapeX[i] << " != " << fDimShapeZ[i] << ")\n"; out << SP << SP << SP - << "throw std::runtime_error(\"SOFIE Where: cannot broadcast X dim " << i << " in " << opName << "\");\n"; + << "throw std::runtime_error(\"SOFIE Where: cannot broadcast A dim " << i << " in " << opName << "\");\n"; } // validate Y vs Z if (i < fDimShapeY.size() && fDimShapeY[i].isParam) { out << SP << SP << "if (" << fDimShapeY[i] << " != 1 && " << fDimShapeY[i] << " != " << fDimShapeZ[i] << ")\n"; out << SP << SP << SP - << "throw std::runtime_error(\"SOFIE Where: cannot broadcast Y dim " << i << " in " << opName << "\");\n"; + << "throw std::runtime_error(\"SOFIE Where: cannot broadcast B dim " << i << " in " << opName << "\");\n"; } // validate C vs Z if (i < fDimShapeC.size() && fDimShapeC[i].isParam) { @@ -300,10 +409,8 @@ public: } out << SP << "}\n"; } - + // implement now where using teh strides and looping on the different dimensions // ---------------------------------------------------------------- // - // Runtime for non-constant, non-initialised tensors - // // Generate loop(s) with per-dimension stride-based index arithmetic // ---------------------------------------------------------------- // auto stridesX = UTILITY::ComputeStrideFromShape(fDimShapeX); @@ -320,6 +427,7 @@ public: return "0"; std::string expr; size_t offset = rankZ - dimShape.size(); + std::cout << rankZ << " " << dimShape.size() << " " << offset << std::endl; for (size_t i = 0; i < dimShape.size(); ++i) { if (dimShape[i].dim == 1 || dimShape[i].GetVal() == "1") continue; expr += "idx_" + std::to_string(i + offset); @@ -336,9 +444,10 @@ public: std::string idxY = buildIdxExpr(fDimShapeY, stridesY, fDimShapeZ.size()); std::string idxC = buildIdxExpr(fDimShapeC, stridesC, fDimShapeZ.size()); - // Emit nested loops over output shape + // Emit nested loops over output shape int nloop = 0; std::string idxZ; + // case Z is a scalar (all dimensions are 1) or Z has no dimension if (fDimShapeZ.empty() || std::all_of(fDimShapeZ.begin(), fDimShapeZ.end(), [](Dim d) { return d.dim == 1 || d.GetVal() == "1"; })) { @@ -375,10 +484,13 @@ public: return out.str(); } + + }; -} // namespace SOFIE -} // namespace Experimental -} // namespace TMVA +}//SOFIE +}//Experimental +}//TMVA + -#endif // TMVA_SOFIE_ROperator_Where +#endif //TMVA_SOFIE_ROperator_Where diff --git a/tmva/sofie/inc/TMVA/SOFIE_common.hxx b/tmva/sofie/inc/TMVA/SOFIE_common.hxx index 9f35cca5f7db3..ff206db95f981 100644 --- a/tmva/sofie/inc/TMVA/SOFIE_common.hxx +++ b/tmva/sofie/inc/TMVA/SOFIE_common.hxx @@ -228,12 +228,18 @@ std::string ConvertValuesToString(size_t n, const T * data, size_t maxprint = -1 std::stringstream ret; ret << "{ "; for (size_t i = 0; i < std::min(n,maxprint); i++) { - if (std::is_floating_point_v) - ret << std::setprecision(std::numeric_limits::max_digits10) << data[i]; - else - // cast in case of boolean (int8) - ret << data[i]; - + if (std::is_floating_point_v) { + // special case for infinity and Nan + if (std::isinf(data[i])) + ret << (data[i] > 0 ? "std::numeric_limits<" + TensorType::Name() + ">::infinity()" : + "-std::numeric_limits<" + TensorType::Name() + ">::infinity()"); + else if (std::isnan(data[i])) + ret << "std::numeric_limits<" + TensorType::Name() + ">::quiet_NaN()"; + else + ret << std::setprecision(std::numeric_limits::max_digits10) << data[i]; + } else { + ret << std::to_string(data[i]); + } if (i < n-1) ret << ", "; if (i < n-1 && i == maxprint-1) ret << "..... "; } @@ -779,7 +785,8 @@ inline void Fill(float *output, float value, int size) std::fill(output, output + size, value); } -inline void Copy(float *output, float const *input, int size) +template +inline void Copy(T *output, T const *input, int size) { std::copy(input, input + size, output); } diff --git a/tmva/sofie/src/RModel.cxx b/tmva/sofie/src/RModel.cxx index 5c30a42619e55..037c9f292fc3b 100644 --- a/tmva/sofie/src/RModel.cxx +++ b/tmva/sofie/src/RModel.cxx @@ -763,6 +763,7 @@ std::string GenerateConstantTensorCode(const std::pair 1) { size_t idx = 1; @@ -797,6 +798,19 @@ void RModel::GenerateInitializedTensorInfo() size_t length = ConvertShapeToLength(i.second.shape()); if (!fUseWeightFile || i.second.IsConstantTensor() || !i.second.IsWeightTensor() || i.second.type() != ETensorType::FLOAT ) { if (i.second.type() == ETensorType::FLOAT) { + // check if NaN of Inf are inside tensor data + bool hasInfOrNaN = false; + const float *data = i.second.data(); + for (size_t idx = 0; idx < length; idx++) { + if (std::is_floating_point::value) { + if (std::isinf(data[idx]) || std::isnan(data[idx])) { + hasInfOrNaN = true; + break; + } + } + } + if (hasInfOrNaN) + AddNeededStdLib("limits"); fGC += GenerateConstantTensorCode(i); fConstantTensorSize += length * sizeof(float); } else if (i.second.type() == ETensorType::INT64) { @@ -1158,6 +1172,7 @@ void RModel::GenerateOutput() // Use the session member (fXxx) when any dim is a runtime-computed identifier // (e.g. NonZero count). For expression-type dims derived from input shapes // (e.g. "((W+-3)/2+1)"), use the expression directly. + // for input shape parameters we don't need to use the session member since it is passed as argument to the infer function and it is not a runtime computed value bool hasRuntimeParam = false; for (auto const &dim : GetDynamicTensorShape(name)) { if (dim.isParam && IsIdentifier(dim.param) && !IsInputTensorShapeParam(dim.param)) @@ -1304,9 +1319,10 @@ void RModel::GenerateSessionCode() // storing the parameters for future checking to avoid mismatches if (!fDimShapeNames.empty()) { - fGC += "\n\n"; - std::sort(fDimShapeNames.begin(), fDimShapeNames.end()); - for (const auto &p : fDimShapeNames) { + fGC += "\n// dynamic shape parameters\n"; + auto dimShapeNames = fDimShapeNames; + std::sort(dimShapeNames.begin(), dimShapeNames.end()); + for (const auto &p : dimShapeNames) { fGC += "size_t " + memberNameForDimShape(p) + ";\n"; } } @@ -1344,8 +1360,7 @@ void RModel::GenerateSessionCode() // add initialization of shape parameters // assume all parameters are of type size_t if (!fDimShapeNames.empty()) { - // sort first the shape parameters in alphabetical order to avoid a random order - std::sort(fDimShapeNames.begin(), fDimShapeNames.end() ); + // need to use same order as in infer function not alphabetical one for (auto &p : fDimShapeNames) { fGC += ",\n"; fGC += " size_t " + p + " = " + fShapeParams[p]; @@ -1361,6 +1376,8 @@ void RModel::GenerateSessionCode() fGC += " " + memberNameForDimShape(p) + " = " + p + ";\n"; } } + // add some extra code needed for initialization of dynamic parameters + fGC += fExtraCodeForDimShapes; if (fUseWeightFile) { fGC += "\n//--- reading weights from file\n"; @@ -1759,6 +1776,42 @@ void RModel::GenerateRequiredInputTensorInfo() fGC += "\nconstexpr bool hasDynamicInputTensors{" + std::string{hasDynamicInputTensors ? "true" : "false"} + "};\n\n"; + + fGC += "\n// Output tensor dimensions\n"; + bool hasDynamicOutputTensors = false; + for (std::size_t iOutput = 0; iOutput < fOutputTensorNames.size(); ++iOutput) { + auto const &name = fOutputTensorNames[iOutput]; + if (IsDynamicTensor(name)) { + hasDynamicOutputTensors = true; + } + std::vector shape = GetDimTensorShape(name); + fGC += "constexpr std::array dim_" + name + "{"; + for (std::size_t iDim = 0; iDim < shape.size(); ++iDim) { + auto const &dim = shape[iDim]; + if (dim.isParam) { + fGC += "SingleDim{\"" + dim.GetVal() + "\"}"; + } else { + fGC += "SingleDim{" + dim.GetVal() + "}"; + } + if (iDim != shape.size() - 1) { + fGC += ", "; + } + } + fGC += "};\n"; + } + fGC += "\nconstexpr std::array outputTensorDims{\n"; + for (std::size_t iOutput = 0; iOutput < fOutputTensorNames.size(); ++iOutput) { + auto const &name = fOutputTensorNames[iOutput]; + fGC += SP + "makeDims(dim_" + name + ")"; + if (iOutput == fOutputTensorNames.size() - 1) { + fGC += "\n"; + } else { + fGC += ",\n"; + } + } + fGC += "};\n"; + fGC += + "\nconstexpr bool hasDynamicOutputTensors{" + std::string{hasDynamicOutputTensors ? "true" : "false"} + "};\n\n"; } void RModel::PrintRequiredInputTensors() const { diff --git a/tmva/sofie_parsers/src/ParseWhere.cxx b/tmva/sofie_parsers/src/ParseWhere.cxx index 6ebcf161e5012..dc4b436282cab 100644 --- a/tmva/sofie_parsers/src/ParseWhere.cxx +++ b/tmva/sofie_parsers/src/ParseWhere.cxx @@ -12,6 +12,10 @@ ParserFuncSignature ParseWhere = [](RModelParser_ONNX &parser, const onnx::NodeP throw std::runtime_error("TMVA::SOFIE ONNX Parser Where op has invalid input size"); } // condition boolean vector is input 0 + if (!parser.IsRegisteredTensorType(nodeproto.input(0))){ + throw std::runtime_error("TMVA::SOFIE ONNX Parser Where op has input tensor " + nodeproto.input(0) + + " but its type is not yet registered"); + } if (!parser.IsRegisteredTensorType(nodeproto.input(1))){ throw std::runtime_error("TMVA::SOFIE ONNX Parser Where op has input tensor " + nodeproto.input(1) + " but its type is not yet registered"); @@ -31,6 +35,7 @@ ParserFuncSignature ParseWhere = [](RModelParser_ONNX &parser, const onnx::NodeP std::string output_name = nodeproto.output(0); switch (input_type) { + //note ROPeratore_WHere signature takes as first tensor the condition case ETensorType::FLOAT: op.reset(new ROperator_Where(nodeproto.input(0), nodeproto.input(1), nodeproto.input(2), output_name)); break;