ATLAS Offline Software
All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Modules Pages
OnnxRuntimeBase.cxx
Go to the documentation of this file.
2 
3 #include <cassert>
4 #include <stdexcept>
5 #include <iostream>
6 #include <Eigen/Dense>
7 
8 
10 {
11  initialize(std::move(fileName));
12 }
13 
15 
17 {
18  m_fileName = std::move(fileName);
19  //load the onnx model to memory using the path m_path_to_onnx
20  m_env = std::make_unique< Ort::Env >(ORT_LOGGING_LEVEL_WARNING, "");
21 
22  // Set the ONNX runtime session options
23  Ort::SessionOptions session_options;
24  // Set graph optimization level
25  session_options.SetIntraOpNumThreads(1);
26  session_options.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_EXTENDED);
27  // Create the Ort session
28  m_session = std::make_unique< Ort::Session >(*m_env, m_fileName.Data(), session_options);
29  // Default allocator
30  Ort::AllocatorWithDefaultOptions allocator;
31  // Get the names of the input nodes of the model
32  size_t numInputNodes = m_session->GetInputCount();
33  // Iterate over all input nodes and get the name
34  for (size_t i = 0; i < numInputNodes; i++)
35  {
36  auto name = m_session->GetInputNameAllocated(i, allocator);
37  char* input_name = new char[strlen(name.get()) + 1];
38  strcpy(input_name, name.get());
39 
40  m_inputNodeNames.push_back(input_name);
41  // Get the dimensions of the input nodes,
42  // here we assume that all input nodes have the same dimensions
43  Ort::TypeInfo inputTypeInfo = m_session->GetInputTypeInfo(i);
44  auto tensorInfo = inputTypeInfo.GetTensorTypeAndShapeInfo();
45 
46  m_inputNodeDims = tensorInfo.GetShape();
47  }
48  // Get the names of the output nodes
49  size_t numOutputNodes = m_session->GetOutputCount();
50  // Iterate over all output nodes and get the name
51  for (size_t i = 0; i < numOutputNodes; i++)
52  {
53  auto name = m_session->GetOutputNameAllocated(i, allocator);
54  char* output_name = new char[strlen(name.get()) + 1];
55  strcpy(output_name, name.get());
56  m_outputNodeNames.push_back(output_name);
57  // Get the dimensions of the output nodes
58  // here we assume that all output nodes have the dimensions
59  Ort::TypeInfo outputTypeInfo = m_session->GetOutputTypeInfo(i);
60  auto tensorInfo = outputTypeInfo.GetTensorTypeAndShapeInfo();
61  m_outputNodeDims = tensorInfo.GetShape();
62  }
63 }
64 
65 // Inference function using ONNX runtime for one single entry
66 std::vector<float> OnnxRuntimeBase::runONNXInference(std::vector<float>& inputTensorValues) const
67 {
68  NetworkBatchInput vectorInput(1, inputTensorValues.size());
69  for (size_t i = 0; i < inputTensorValues.size(); i++) {
70  vectorInput(0, i) = inputTensorValues[i];
71  }
72  auto vectorOutput = runONNXInference(vectorInput);
73  return vectorOutput[0];
74 }
75 
76 // Inference function using ONNX runtime
77 // the function assumes that the model has 1 input node and 1 output node
78 std::vector<std::vector<float>> OnnxRuntimeBase::runONNXInference(NetworkBatchInput& inputTensorValues) const
79 {
80  int batchSize = inputTensorValues.rows();
81  std::vector<int64_t> inputNodeDims = m_inputNodeDims;
82  std::vector<int64_t> outputNodeDims = m_outputNodeDims; //bad. Assumes they all have the same number of nodes.
83 
84  // The first dim node should correspond to the batch size
85  // If it is -1, it is dynamic and should be set to the input size
86  if (inputNodeDims[0] == -1)
87  {
88  inputNodeDims[0] = batchSize;
89  }
90  if (outputNodeDims[0] == -1)
91  {
92  outputNodeDims[0] = batchSize;
93  }
94 
95  if(inputNodeDims[1]*inputNodeDims[2] != inputTensorValues.cols() && inputNodeDims[1] != inputTensorValues.cols())
96  {
97  throw std::runtime_error("runONNXInference: feature size doesn't match the input size: inputSize required: " + std::to_string(inputNodeDims[1]*inputNodeDims[2]) + " inputSize provided: " + std::to_string(inputTensorValues.cols()));
98  }
99 
100  if (batchSize != 1 && (inputNodeDims[0] != batchSize || outputNodeDims[0] != batchSize))
101  {
102  throw std::runtime_error("runONNXInference: batch size doesn't match the input or output node size");
103  }
104 
105  // Create input tensor object from data values
106  // note: this assumes the model has only 1 input node
107  Ort::MemoryInfo memoryInfo = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);
108  Ort::Value inputTensor = Ort::Value::CreateTensor<float>(memoryInfo, inputTensorValues.data(), inputTensorValues.size(),inputNodeDims.data(), inputNodeDims.size());
109  // Double-check that inputTensor is a Tensor
110  if (!inputTensor.IsTensor())
111  {
112  throw std::runtime_error("runONNXInference: conversion of input to Tensor failed. ");
113  }
114  // Score model on input tensors, get back output tensors
115  Ort::RunOptions run_options;
116  std::vector<Ort::Value> outputTensors =
117  m_session->Run(run_options, m_inputNodeNames.data(), &inputTensor,
118  m_inputNodeNames.size(), m_outputNodeNames.data(),
119  m_outputNodeNames.size());
120  // Double-check that outputTensors contains Tensors and that the count matches
121  // that of output nodes
122  if (!outputTensors[0].IsTensor() || (outputTensors.size() != m_outputNodeNames.size())) {
123  throw std::runtime_error("runONNXInference: calculation of output failed. ");
124  }
125  // Get pointer to output tensor float values
126  // note: this assumes the model has only 1 output value
127  float* outputTensor = outputTensors.front().GetTensorMutableData<float>();
128  // Get the output values
129  std::vector<std::vector<float>> outputTensorValues(batchSize, std::vector<float>(outputNodeDims[1], -1));
130  for (int i = 0; i < outputNodeDims[0]; i++)
131  {
132  for (int j = 0; j < ((outputNodeDims.size() > 1) ? outputNodeDims[1] : 1); j++)
133  {
134  outputTensorValues[i][j] = outputTensor[i * outputNodeDims[1] + j];
135  }
136  }
137 
138  return outputTensorValues;
139 }
140 
141 // Inference function using ONNX runtime
142 // the function assumes that the model has two output layers
143 // should return a map of <int, Eigen::Matrix>
144 std::map<int, Eigen::MatrixXf> OnnxRuntimeBase::runONNXInferenceMultilayerOutput(NetworkBatchInput& inputTensorValues) const
145 {
146  const int batchSize = inputTensorValues.rows();
147  std::vector<int64_t> inputNodeDims = m_inputNodeDims;
148  std::vector<int64_t> outputNodeDims = m_outputNodeDims;
149 
150  // The first dim node should correspond to the batch size
151  // If it is -1, it is dynamic and should be set to the input size
152  if (inputNodeDims[0] == -1)
153  {
154  inputNodeDims[0] = batchSize;
155  }
156  if (outputNodeDims[0] == -1)
157  {
158  outputNodeDims[0] = batchSize;
159  }
160 
161  if(inputNodeDims[1] != inputTensorValues.cols())
162  {
163  throw std::runtime_error("runONNXInference: feature size doesn't match the input size: inputSize required: " + std::to_string(inputNodeDims[1]) + " inputSize provided: " + std::to_string(inputTensorValues.cols()));
164  }
165 
166  if (batchSize != 1 &&(inputNodeDims[0] != batchSize || outputNodeDims[0] != batchSize))
167  {
168  throw std::runtime_error("runONNXInference: batch size doesn't match the input or output node size");
169  }
170  // Create input tensor object from data values
171  // note: this assumes the model has only 1 input node
172  Ort::MemoryInfo memoryInfo = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);
173  Ort::Value inputTensor = Ort::Value::CreateTensor<float>(memoryInfo, inputTensorValues.data(), inputTensorValues.size(), inputNodeDims.data(), inputNodeDims.size());
174  // Double-check that inputTensor is a Tensor
175  if (!inputTensor.IsTensor())
176  {
177  throw std::runtime_error("runONNXInference: conversion of input to Tensor failed. ");
178  }
179  // Score model on input tensors, get back output tensors
180  Ort::RunOptions run_options;
181  std::vector<Ort::Value> outputTensors =
182  m_session->Run(run_options, m_inputNodeNames.data(), &inputTensor,
183  m_inputNodeNames.size(), m_outputNodeNames.data(),
184  m_outputNodeNames.size());
185  // Double-check that outputTensors contains Tensors and that the count matches
186  // that of output nodes
187  if (!outputTensors[0].IsTensor() || (outputTensors.size() != m_outputNodeNames.size())) {
188  throw std::runtime_error("runONNXInference: calculation of output failed. ");
189  }
190  // Get pointers to output tensor float values
191  // note: this assumes the model has multiple output layers
192  std::map<int, Eigen::MatrixXf> outputTensorMap;
193  size_t numOutputNodes = m_session->GetOutputCount();
194  for (size_t i=0; i<numOutputNodes; i++){ // two output layers
195 
196  // retrieve pointer to output float tenor
197  float* output = outputTensors.at(i).GetTensorMutableData<float>();
198  Ort::TypeInfo outputTypeInfo = m_session->GetOutputTypeInfo(i);
199  auto outputTensorInfo = outputTypeInfo.GetTensorTypeAndShapeInfo();
200  // Not all outputNodes have the same shape. Get the new shape.
201  // First dimension should be batch size
202  outputNodeDims = outputTensorInfo.GetShape();
203 
204  int nNodes = outputNodeDims.size() > 1 ? outputNodeDims[1] : 1;
205  Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic> batchMatrix(batchSize, nNodes);
206  for (int j = 0; j < batchSize; j++)
207  {
208  Eigen::VectorXf vec(nNodes);
209  for (int k = 0; k<nNodes; k++)
210  {
211  float val = output[j * outputNodeDims[1] + k];
212  vec(k) = val;
213  }
214  batchMatrix.row(j) = vec;
215  } // batch
216  outputTensorMap[i] = std::move(batchMatrix);
217  } // output layers
218  return outputTensorMap;
219 }
OnnxRuntimeBase::runONNXInference
std::vector< float > runONNXInference(std::vector< float > &inputTensorValues) const
Definition: OnnxRuntimeBase.cxx:66
OnnxRuntimeBase::m_env
std::unique_ptr< Ort::Env > m_env
Definition: OnnxRuntimeBase.h:44
OnnxRuntimeBase::initialize
void initialize(TString)
Definition: OnnxRuntimeBase.cxx:16
OnnxRuntimeBase::m_outputNodeDims
std::vector< int64_t > m_outputNodeDims
Definition: OnnxRuntimeBase.h:42
OnnxRuntimeBase::m_inputNodeDims
std::vector< int64_t > m_inputNodeDims
Definition: OnnxRuntimeBase.h:40
OnnxRuntimeBase::m_session
std::unique_ptr< Ort::Session > m_session
ONNX runtime session / model properties.
Definition: OnnxRuntimeBase.h:33
CxxUtils::vec
typename vecDetail::vec_typedef< T, N >::type vec
Define a nice alias for the vectorized type.
Definition: vec.h:207
OnnxRuntimeBase::m_inputNodeNames
std::vector< const char * > m_inputNodeNames
Definition: OnnxRuntimeBase.h:39
OnnxRuntimeBase.h
FortranAlgorithmOptions.fileName
fileName
Definition: FortranAlgorithmOptions.py:13
OnnxRuntimeBase::runONNXInferenceMultilayerOutput
std::map< int, Eigen::MatrixXf > runONNXInferenceMultilayerOutput(NetworkBatchInput &inputTensorValues) const
Definition: OnnxRuntimeBase.cxx:144
lumiFormat.i
int i
Definition: lumiFormat.py:85
OnnxRuntimeBase::m_fileName
TString m_fileName
Definition: OnnxRuntimeBase.h:17
merge.output
output
Definition: merge.py:17
name
std::string name
Definition: Control/AthContainers/Root/debug.cxx:228
ActsTrk::to_string
std::string to_string(const DetectorType &type)
Definition: GeometryDefs.h:34
OnnxRuntimeBase::m_outputNodeNames
std::vector< const char * > m_outputNodeNames
Definition: OnnxRuntimeBase.h:41
Pythia8_RapidityOrderMPI.val
val
Definition: Pythia8_RapidityOrderMPI.py:14
OnnxRuntimeBase::OnnxRuntimeBase
OnnxRuntimeBase()
Definition: OnnxRuntimeBase.cxx:14
NetworkBatchInput
Eigen::Matrix< float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor > NetworkBatchInput
Definition: OnnxRuntimeBase.h:9
fitman.k
k
Definition: fitman.py:528