OnnxRuntimeBase Class Reference

#include <OnnxRuntimeBase.h>

Inheritance diagram for OnnxRuntimeBase:

Collaboration diagram for OnnxRuntimeBase:

Public Member Functions
	OnnxRuntimeBase (TString fileName)
	OnnxRuntimeBase ()
	~OnnxRuntimeBase ()
void	initialize (TString)
std::vector< float >	runONNXInference (std::vector< float > &inputTensorValues) const
std::vector< std::vector< float > >	runONNXInference (std::vector< std::vector< float > > &inputTensorValues) const
std::vector< std::vector< float > >	runONNXInference (NetworkBatchInput &inputTensorValues) const
std::map< int, Eigen::MatrixXf >	runONNXInferenceMultilayerOutput (NetworkBatchInput &inputTensorValues) const
const std::vector< int64_t > &	getInputNodesDims ()
const std::vector< int64_t > &	getOutputNodesDims ()

Public Attributes
TString	m_fileName

Private Attributes
std::unique_ptr< Ort::Session >	m_session
	ONNX runtime session / model properties. More...
std::vector< const char * >	m_inputNodeNames
std::vector< int64_t >	m_inputNodeDims
std::vector< const char * >	m_outputNodeNames
std::vector< int64_t >	m_outputNodeDims
std::unique_ptr< Ort::Env >	m_env

Detailed Description

Definition at line 13 of file OnnxRuntimeBase.h.

Constructor & Destructor Documentation

OnnxRuntimeBase() [1/2]

OnnxRuntimeBase::OnnxRuntimeBase

(

TString

fileName

)

Definition at line 9 of file OnnxRuntimeBase.cxx.

{
    initialize(std::move(fileName));
}

OnnxRuntimeBase() [2/2]

OnnxRuntimeBase::OnnxRuntimeBase

(

)

Definition at line 14 of file OnnxRuntimeBase.cxx.

{}

~OnnxRuntimeBase()

OnnxRuntimeBase::~OnnxRuntimeBase ( )

inline

Definition at line 25 of file OnnxRuntimeBase.h.

{}

Member Function Documentation

getInputNodesDims()

const std::vector<int64_t>& OnnxRuntimeBase::getInputNodesDims ( )

inline

Definition at line 33 of file OnnxRuntimeBase.h.

{return m_inputNodeDims;};

getOutputNodesDims()

const std::vector<int64_t>& OnnxRuntimeBase::getOutputNodesDims ( )

inline

Definition at line 34 of file OnnxRuntimeBase.h.

{return m_outputNodeDims;};

initialize()

void OnnxRuntimeBase::initialize

(

TString

fileName

)

Definition at line 16 of file OnnxRuntimeBase.cxx.

{
  m_fileName = std::move(fileName);
  //load the onnx model to memory using the path m_path_to_onnx
  m_env = std::make_unique< Ort::Env >(ORT_LOGGING_LEVEL_WARNING, "");
 
  // Set the ONNX runtime session options
  Ort::SessionOptions session_options;
  // Set graph optimization level
  session_options.SetIntraOpNumThreads(1);
  session_options.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_EXTENDED);
  // Create the Ort session
  m_session = std::make_unique< Ort::Session >(*m_env, m_fileName.Data(), session_options);
  // Default allocator
  Ort::AllocatorWithDefaultOptions allocator;
  // Get the names of the input nodes of the model
  size_t numInputNodes = m_session->GetInputCount();
  // Iterate over all input nodes and get the name
  for (size_t i = 0; i < numInputNodes; i++) 
  {
    auto name = m_session->GetInputNameAllocated(i, allocator);
    char* input_name = new char[strlen(name.get()) + 1];
    strcpy(input_name, name.get());
 
    m_inputNodeNames.push_back(input_name);
    // Get the dimensions of the input nodes,
    // here we assume that all input nodes have the same dimensions
    Ort::TypeInfo inputTypeInfo = m_session->GetInputTypeInfo(i);
    auto tensorInfo = inputTypeInfo.GetTensorTypeAndShapeInfo();
 
    m_inputNodeDims = tensorInfo.GetShape();
  }
  // Get the names of the output nodes
  size_t numOutputNodes = m_session->GetOutputCount();
  // Iterate over all output nodes and get the name
  for (size_t i = 0; i < numOutputNodes; i++) 
  {
    auto name = m_session->GetOutputNameAllocated(i, allocator);
    char* output_name = new char[strlen(name.get()) + 1];
    strcpy(output_name, name.get());
    m_outputNodeNames.push_back(output_name);
    // Get the dimensions of the output nodes
    // here we assume that all output nodes have the dimensions
    Ort::TypeInfo outputTypeInfo = m_session->GetOutputTypeInfo(i);
    auto tensorInfo = outputTypeInfo.GetTensorTypeAndShapeInfo();
    m_outputNodeDims = tensorInfo.GetShape();
  }
}

runONNXInference() [1/3]

std::vector< std::vector< float > > OnnxRuntimeBase::runONNXInference

(

NetworkBatchInput &

inputTensorValues

)

const

Definition at line 96 of file OnnxRuntimeBase.cxx.

{
  int batchSize = inputTensorValues.rows();
  std::vector<int64_t> inputNodeDims = m_inputNodeDims;
  std::vector<int64_t> outputNodeDims = m_outputNodeDims; //bad. Assumes they all have the same number of nodes.
 
  // The first dim node should correspond to the batch size
  // If it is -1, it is dynamic and should be set to the input size
  if (inputNodeDims[0] == -1) 
  {
    inputNodeDims[0] = batchSize;
  }
  if (outputNodeDims[0] == -1) 
  {
    outputNodeDims[0] = batchSize;
  }
 
  if(inputNodeDims[1]*inputNodeDims[2] != inputTensorValues.cols() && inputNodeDims[1] != inputTensorValues.cols())
  {
    throw std::runtime_error("runONNXInference: feature size doesn't match the input size: inputSize required: " + std::to_string(inputNodeDims[1]*inputNodeDims[2]) + " inputSize provided: " + std::to_string(inputTensorValues.cols()));
  }
 
  if (batchSize != 1 && (inputNodeDims[0] != batchSize || outputNodeDims[0] != batchSize)) 
  {
    throw std::runtime_error("runONNXInference: batch size doesn't match the input or output node size");
  }
 
  // Create input tensor object from data values
  // note: this assumes the model has only 1 input node
  Ort::MemoryInfo memoryInfo = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);
  Ort::Value inputTensor = Ort::Value::CreateTensor<float>(memoryInfo, inputTensorValues.data(), inputTensorValues.size(),inputNodeDims.data(), inputNodeDims.size());
  // Double-check that inputTensor is a Tensor
  if (!inputTensor.IsTensor()) 
  {
    throw std::runtime_error("runONNXInference: conversion of input to Tensor failed. ");
  }
  // Score model on input tensors, get back output tensors
  Ort::RunOptions run_options;
  std::vector<Ort::Value> outputTensors =
      m_session->Run(run_options, m_inputNodeNames.data(), &inputTensor,
                     m_inputNodeNames.size(), m_outputNodeNames.data(),
                     m_outputNodeNames.size());
  // Double-check that outputTensors contains Tensors and that the count matches
  // that of output nodes
  if (!outputTensors[0].IsTensor() || (outputTensors.size() != m_outputNodeNames.size())) {
    throw std::runtime_error("runONNXInference: calculation of output failed. ");
  }
  // Get pointer to output tensor float values
  // note: this assumes the model has only 1 output value
  float* outputTensor = outputTensors.front().GetTensorMutableData<float>();
  // Get the output values
  std::vector<std::vector<float>> outputTensorValues(batchSize, std::vector<float>(outputNodeDims[1], -1));
  for (int i = 0; i < outputNodeDims[0]; i++) 
  {
    for (int j = 0; j < ((outputNodeDims.size() > 1) ? outputNodeDims[1] : 1); j++) 
    {
      outputTensorValues[i][j] = outputTensor[i * outputNodeDims[1] + j];
    }
  }
 
  return outputTensorValues;
}

runONNXInference() [2/3]

std::vector< float > OnnxRuntimeBase::runONNXInference

(

std::vector< float > &

inputTensorValues

)

const

Definition at line 84 of file OnnxRuntimeBase.cxx.

{
  NetworkBatchInput vectorInput(1, inputTensorValues.size());
  for (size_t i = 0; i < inputTensorValues.size(); i++) {
    vectorInput(0, i) = inputTensorValues[i];
  }
  auto vectorOutput = runONNXInference(vectorInput);
  return vectorOutput[0];
}

runONNXInference() [3/3]

std::vector< std::vector< float > > OnnxRuntimeBase::runONNXInference

(

std::vector< std::vector< float > > &

inputTensorValues

)

const

Definition at line 66 of file OnnxRuntimeBase.cxx.

{
  std::vector<std::vector<float> > output;
  if (inputTensorValues.size() == 0) return output;
 
  NetworkBatchInput vectorInput(inputTensorValues.size(), inputTensorValues[0].size());
  for (size_t i = 0; i < inputTensorValues.size(); i++) { 
    for (size_t j = 0; j < inputTensorValues[i].size(); j++) {
      vectorInput(i,j) = inputTensorValues[i][j];
    }
  }
  output = runONNXInference(vectorInput);
  return output;
}

runONNXInferenceMultilayerOutput()

std::map< int, Eigen::MatrixXf > OnnxRuntimeBase::runONNXInferenceMultilayerOutput

(

NetworkBatchInput &

inputTensorValues

)

const

Definition at line 162 of file OnnxRuntimeBase.cxx.

{
  const int batchSize = inputTensorValues.rows();
  std::vector<int64_t> inputNodeDims = m_inputNodeDims;
  std::vector<int64_t> outputNodeDims = m_outputNodeDims;
 
  // The first dim node should correspond to the batch size
  // If it is -1, it is dynamic and should be set to the input size
  if (inputNodeDims[0] == -1) 
  {
    inputNodeDims[0] = batchSize;
  }
  if (outputNodeDims[0] == -1) 
  {
    outputNodeDims[0] = batchSize;
  }
 
  if(inputNodeDims[1] != inputTensorValues.cols())
  {
    throw std::runtime_error("runONNXInference: feature size doesn't match the input size: inputSize required: " + std::to_string(inputNodeDims[1]) + " inputSize provided: " + std::to_string(inputTensorValues.cols()));
  }
 
  if (batchSize != 1 &&(inputNodeDims[0] != batchSize || outputNodeDims[0] != batchSize)) 
  {
    throw std::runtime_error("runONNXInference: batch size doesn't match the input or output node size");
  }
  // Create input tensor object from data values
  // note: this assumes the model has only 1 input node
  Ort::MemoryInfo memoryInfo = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);
  Ort::Value inputTensor = Ort::Value::CreateTensor<float>(memoryInfo, inputTensorValues.data(), inputTensorValues.size(), inputNodeDims.data(), inputNodeDims.size());
  // Double-check that inputTensor is a Tensor
  if (!inputTensor.IsTensor()) 
  {
    throw std::runtime_error("runONNXInference: conversion of input to Tensor failed. ");
  }
  // Score model on input tensors, get back output tensors
  Ort::RunOptions run_options;
  std::vector<Ort::Value> outputTensors =
      m_session->Run(run_options, m_inputNodeNames.data(), &inputTensor,
                     m_inputNodeNames.size(), m_outputNodeNames.data(),
                     m_outputNodeNames.size());
  // Double-check that outputTensors contains Tensors and that the count matches
  // that of output nodes
  if (!outputTensors[0].IsTensor() || (outputTensors.size() != m_outputNodeNames.size())) {
    throw std::runtime_error("runONNXInference: calculation of output failed. ");
  }
  // Get pointers to output tensor float values
  // note: this assumes the model has multiple output layers
  std::map<int, Eigen::MatrixXf> outputTensorMap;
  size_t numOutputNodes = m_session->GetOutputCount();
  for (size_t i=0; i<numOutputNodes; i++){ // two output layers
 
    // retrieve pointer to output float tenor
    float* output = outputTensors.at(i).GetTensorMutableData<float>();
    Ort::TypeInfo outputTypeInfo = m_session->GetOutputTypeInfo(i);
    auto outputTensorInfo = outputTypeInfo.GetTensorTypeAndShapeInfo();
    // Not all outputNodes have the same shape. Get the new shape.
    // First dimension should be batch size
    outputNodeDims = outputTensorInfo.GetShape();
 
    int nNodes = outputNodeDims.size() > 1 ? outputNodeDims[1] : 1;
    Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic> batchMatrix(batchSize, nNodes);
    for (int j = 0; j < batchSize; j++) 
    {
      Eigen::VectorXf vec(nNodes);
      for (int k = 0; k<nNodes; k++) 
      {
        float val = output[j * outputNodeDims[1] + k];
        vec(k) = val;
      }
      batchMatrix.row(j) = vec;
    } // batch
    outputTensorMap[i] = std::move(batchMatrix);
  } // output layers
  return outputTensorMap;
}

Member Data Documentation

m_env

std::unique_ptr< Ort::Env > OnnxRuntimeBase::m_env

private

Definition at line 45 of file OnnxRuntimeBase.h.

m_fileName

TString OnnxRuntimeBase::m_fileName

Definition at line 17 of file OnnxRuntimeBase.h.

m_inputNodeDims

std::vector<int64_t> OnnxRuntimeBase::m_inputNodeDims

private

Definition at line 41 of file OnnxRuntimeBase.h.

m_inputNodeNames

std::vector<const char*> OnnxRuntimeBase::m_inputNodeNames

private

Definition at line 40 of file OnnxRuntimeBase.h.

m_outputNodeDims

std::vector<int64_t> OnnxRuntimeBase::m_outputNodeDims

private

Definition at line 43 of file OnnxRuntimeBase.h.

m_outputNodeNames

std::vector<const char*> OnnxRuntimeBase::m_outputNodeNames

private

Definition at line 42 of file OnnxRuntimeBase.h.

m_session

std::unique_ptr<Ort::Session> OnnxRuntimeBase::m_session

private

ONNX runtime session / model properties.

Definition at line 38 of file OnnxRuntimeBase.h.

---

The documentation for this class was generated from the following files:

• OnnxRuntimeBase.h
• OnnxRuntimeBase.cxx