onnxruntime_cxx_api.h

// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
 
// Summary: The Ort C++ API is a header only wrapper around the Ort C API.
//
// The C++ API simplifies usage by returning values directly instead of error codes, throwing exceptions on errors
// and automatically releasing resources in the destructors.
//
// Each of the C++ wrapper classes holds only a pointer to the C internal object. Treat them like smart pointers.
// To create an empty object, pass 'nullptr' to the constructor (for example, Env e{nullptr};).
//
// Only move assignment between objects is allowed, there are no copy constructors. Some objects have explicit 'Clone'
// methods for this purpose.
 
#pragma once
#include "onnxruntime_c_api.h"
#include <cstddef>
#include <array>
#include <memory>
#include <stdexcept>
#include <string>
#include <vector>
#include <unordered_map>
#include <utility>
#include <type_traits>
 
#ifdef ORT_NO_EXCEPTIONS
#include <iostream>
#endif
 
namespace Ort {
 
struct Exception : std::exception {
  Exception(std::string&& string, OrtErrorCode code) : message_{std::move(string)}, code_{code} {}
 
  OrtErrorCode GetOrtErrorCode() const { return code_; }
  const char* what() const noexcept override { return message_.c_str(); }
 
 private:
  std::string message_;
  OrtErrorCode code_;
};
 
#ifdef ORT_NO_EXCEPTIONS
// The #ifndef is for the very special case where the user of this library wants to define their own way of handling errors.
// NOTE: This header expects control flow to not continue after calling ORT_CXX_API_THROW
#ifndef ORT_CXX_API_THROW
#define ORT_CXX_API_THROW(string, code)       \
  do {                                        \
    std::cerr << Ort::Exception(string, code) \
                     .what()                  \
              << std::endl;                   \
    abort();                                  \
  } while (false)
#endif
#else
#define ORT_CXX_API_THROW(string, code) \
  throw Ort::Exception(string, code)
#endif
 
// This is used internally by the C++ API. This class holds the global variable that points to the OrtApi, it's in a template so that we can define a global variable in a header and make
// it transparent to the users of the API.
template <typename T>
struct Global {
  static const OrtApi* api_;
};
 
// If macro ORT_API_MANUAL_INIT is defined, no static initialization will be performed. Instead, user must call InitApi() before using it.
template <typename T>
#ifdef ORT_API_MANUAL_INIT
const OrtApi* Global<T>::api_{};
inline void InitApi() { Global<void>::api_ = OrtGetApiBase()->GetApi(ORT_API_VERSION); }
#else
#if defined(_MSC_VER) && !defined(__clang__)
#pragma warning(push)
// "Global initializer calls a non-constexpr function." Therefore you can't use ORT APIs in the other global initializers.
// Please define ORT_API_MANUAL_INIT if it conerns you.
#pragma warning(disable : 26426)
#endif
const OrtApi* Global<T>::api_ = OrtGetApiBase()->GetApi(ORT_API_VERSION);
#if defined(_MSC_VER) && !defined(__clang__)
#pragma warning(pop)
#endif
#endif
 
inline const OrtApi& GetApi() { return *Global<void>::api_; }
 
std::vector<std::string> GetAvailableProviders();
 
// This is used internally by the C++ API. This macro is to make it easy to generate overloaded methods for all of the various OrtRelease* functions for every Ort* type
// This can't be done in the C API since C doesn't have function overloading.
#define ORT_DEFINE_RELEASE(NAME) \
  inline void OrtRelease(Ort##NAME* ptr) { GetApi().Release##NAME(ptr); }
 
ORT_DEFINE_RELEASE(Allocator);
ORT_DEFINE_RELEASE(MemoryInfo);
ORT_DEFINE_RELEASE(CustomOpDomain);
ORT_DEFINE_RELEASE(Env);
ORT_DEFINE_RELEASE(RunOptions);
ORT_DEFINE_RELEASE(Session);
ORT_DEFINE_RELEASE(SessionOptions);
ORT_DEFINE_RELEASE(TensorTypeAndShapeInfo);
ORT_DEFINE_RELEASE(SequenceTypeInfo);
ORT_DEFINE_RELEASE(MapTypeInfo);
ORT_DEFINE_RELEASE(TypeInfo);
ORT_DEFINE_RELEASE(Value);
ORT_DEFINE_RELEASE(ModelMetadata);
ORT_DEFINE_RELEASE(ThreadingOptions);
ORT_DEFINE_RELEASE(IoBinding);
ORT_DEFINE_RELEASE(ArenaCfg);
 
#undef ORT_DEFINE_RELEASE
 
struct Float16_t {
  uint16_t value;
  constexpr Float16_t() noexcept : value(0) {}
  constexpr Float16_t(uint16_t v) noexcept : value(v) {}
  constexpr operator uint16_t() const noexcept { return value; }
  constexpr bool operator==(const Float16_t& rhs) const noexcept { return value == rhs.value; };
  constexpr bool operator!=(const Float16_t& rhs) const noexcept { return value != rhs.value; };
};
 
static_assert(sizeof(Float16_t) == sizeof(uint16_t), "Sizes must match");
 
struct BFloat16_t {
  uint16_t value;
  constexpr BFloat16_t() noexcept : value(0) {}
  constexpr BFloat16_t(uint16_t v) noexcept : value(v) {}
  constexpr operator uint16_t() const noexcept { return value; }
  constexpr bool operator==(const BFloat16_t& rhs) const noexcept { return value == rhs.value; };
  constexpr bool operator!=(const BFloat16_t& rhs) const noexcept { return value != rhs.value; };
};
 
static_assert(sizeof(BFloat16_t) == sizeof(uint16_t), "Sizes must match");
 
template <typename T>
struct Base {
  using contained_type = T;
 
  Base() = default;
  Base(T* p) : p_{p} {
    if (!p)
      ORT_CXX_API_THROW("Allocation failure", ORT_FAIL);
  }
  ~Base() { OrtRelease(p_); }
 
  operator T*() { return p_; }
  operator const T*() const { return p_; }
 
  T* release() {
    T* p = p_;
    p_ = nullptr;
    return p;
  }
 
 protected:
  Base(const Base&) = delete;
  Base& operator=(const Base&) = delete;
  Base(Base&& v) noexcept : p_{v.p_} { v.p_ = nullptr; }
  void operator=(Base&& v) noexcept {
    OrtRelease(p_);
    p_ = v.release();
  }
 
  T* p_{};
 
  template <typename>
  friend struct Unowned;  // This friend line is needed to keep the centos C++ compiler from giving an error
};
 
template <typename T>
struct Unowned : T {
  Unowned(typename T::contained_type* p) : T{p} {}
  Unowned(Unowned&& v) : T{v.p_} {}
  ~Unowned() { this->release(); }
};
 
struct AllocatorWithDefaultOptions;
struct MemoryInfo;
struct Env;
struct TypeInfo;
struct Value;
struct ModelMetadata;
 
namespace detail {
// Light functor to release memory with OrtAllocator
struct AllocatedFree {
  OrtAllocator* allocator_;
  explicit AllocatedFree(OrtAllocator* allocator)
      : allocator_(allocator) {}
  void operator()(void* ptr) const {
    if (ptr) allocator_->Free(allocator_, ptr);
  }
};
}  // namespace detail
 
using AllocatedStringPtr = std::unique_ptr<char, detail::AllocatedFree>;
 
struct Env : Base<OrtEnv> {
  explicit Env(std::nullptr_t) {}  
 
  Env(OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = "");
 
  Env(OrtLoggingLevel logging_level, const char* logid, OrtLoggingFunction logging_function, void* logger_param);
 
  Env(const OrtThreadingOptions* tp_options, OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = "");
 
  Env(const OrtThreadingOptions* tp_options, OrtLoggingFunction logging_function, void* logger_param,
      OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING, _In_ const char* logid = "");
 
  explicit Env(OrtEnv* p) : Base<OrtEnv>{p} {}
 
  Env& EnableTelemetryEvents();   
  Env& DisableTelemetryEvents();  
 
  Env& CreateAndRegisterAllocator(const OrtMemoryInfo* mem_info, const OrtArenaCfg* arena_cfg);  
};
 
struct CustomOpDomain : Base<OrtCustomOpDomain> {
  explicit CustomOpDomain(std::nullptr_t) {}  
 
  explicit CustomOpDomain(const char* domain);
 
  void Add(OrtCustomOp* op);  
};
 
struct RunOptions : Base<OrtRunOptions> {
  explicit RunOptions(std::nullptr_t) {}  
  RunOptions();                           
 
  RunOptions& SetRunLogVerbosityLevel(int);  
  int GetRunLogVerbosityLevel() const;       
 
  RunOptions& SetRunLogSeverityLevel(int);  
  int GetRunLogSeverityLevel() const;       
 
  RunOptions& SetRunTag(const char* run_tag);  
  const char* GetRunTag() const;               
 
  RunOptions& AddConfigEntry(const char* config_key, const char* config_value);  
 
  RunOptions& SetTerminate();
 
  RunOptions& UnsetTerminate();
};
 
struct SessionOptions : Base<OrtSessionOptions> {
  explicit SessionOptions(std::nullptr_t) {}                                     
  SessionOptions();                                                              
  explicit SessionOptions(OrtSessionOptions* p) : Base<OrtSessionOptions>{p} {}  
 
  SessionOptions Clone() const;  
 
  SessionOptions& SetIntraOpNumThreads(int intra_op_num_threads);                              
  SessionOptions& SetInterOpNumThreads(int inter_op_num_threads);                              
  SessionOptions& SetGraphOptimizationLevel(GraphOptimizationLevel graph_optimization_level);  
 
  SessionOptions& EnableCpuMemArena();   
  SessionOptions& DisableCpuMemArena();  
 
  SessionOptions& SetOptimizedModelFilePath(const ORTCHAR_T* optimized_model_file);  
 
  SessionOptions& EnableProfiling(const ORTCHAR_T* profile_file_prefix);  
  SessionOptions& DisableProfiling();                                     
 
  SessionOptions& EnableOrtCustomOps();  
 
  SessionOptions& EnableMemPattern();   
  SessionOptions& DisableMemPattern();  
 
  SessionOptions& SetExecutionMode(ExecutionMode execution_mode);  
 
  SessionOptions& SetLogId(const char* logid);     
  SessionOptions& SetLogSeverityLevel(int level);  
 
  SessionOptions& Add(OrtCustomOpDomain* custom_op_domain);  
 
  SessionOptions& DisablePerSessionThreads();  
 
  SessionOptions& AddConfigEntry(const char* config_key, const char* config_value);                                      
  SessionOptions& AddInitializer(const char* name, const OrtValue* ort_val);                                             
  SessionOptions& AddExternalInitializers(const std::vector<std::string>& names, const std::vector<Value>& ort_values);  
 
  SessionOptions& AppendExecutionProvider_CUDA(const OrtCUDAProviderOptions& provider_options);               
  SessionOptions& AppendExecutionProvider_CUDA_V2(const OrtCUDAProviderOptionsV2& provider_options);          
  SessionOptions& AppendExecutionProvider_ROCM(const OrtROCMProviderOptions& provider_options);               
  SessionOptions& AppendExecutionProvider_OpenVINO(const OrtOpenVINOProviderOptions& provider_options);       
  SessionOptions& AppendExecutionProvider_TensorRT(const OrtTensorRTProviderOptions& provider_options);       
  SessionOptions& AppendExecutionProvider_TensorRT_V2(const OrtTensorRTProviderOptionsV2& provider_options);  
  SessionOptions& AppendExecutionProvider_MIGraphX(const OrtMIGraphXProviderOptions& provider_options);       
  SessionOptions& AppendExecutionProvider(const std::string& provider_name,
                                          const std::unordered_map<std::string, std::string>& provider_options = {});
 
  SessionOptions& SetCustomCreateThreadFn(OrtCustomCreateThreadFn ort_custom_create_thread_fn);  
  SessionOptions& SetCustomThreadCreationOptions(void* ort_custom_thread_creation_options);      
  SessionOptions& SetCustomJoinThreadFn(OrtCustomJoinThreadFn ort_custom_join_thread_fn);        
};
 
struct ModelMetadata : Base<OrtModelMetadata> {
  explicit ModelMetadata(std::nullptr_t) {}                                   
  explicit ModelMetadata(OrtModelMetadata* p) : Base<OrtModelMetadata>{p} {}  
 
  char* GetProducerName(OrtAllocator* allocator) const;  
 
  AllocatedStringPtr GetProducerNameAllocated(OrtAllocator* allocator) const;  
 
  char* GetGraphName(OrtAllocator* allocator) const;  
 
  AllocatedStringPtr GetGraphNameAllocated(OrtAllocator* allocator) const;  
 
  char* GetDomain(OrtAllocator* allocator) const;  
 
  AllocatedStringPtr GetDomainAllocated(OrtAllocator* allocator) const;  
 
  char* GetDescription(OrtAllocator* allocator) const;  
 
  AllocatedStringPtr GetDescriptionAllocated(OrtAllocator* allocator) const;  
 
  char* GetGraphDescription(OrtAllocator* allocator) const;  
 
  AllocatedStringPtr GetGraphDescriptionAllocated(OrtAllocator* allocator) const;  
 
  char** GetCustomMetadataMapKeys(OrtAllocator* allocator, _Out_ int64_t& num_keys) const;  
 
  std::vector<AllocatedStringPtr> GetCustomMetadataMapKeysAllocated(OrtAllocator* allocator) const;  
 
  char* LookupCustomMetadataMap(const char* key, OrtAllocator* allocator) const;  
 
  AllocatedStringPtr LookupCustomMetadataMapAllocated(const char* key, OrtAllocator* allocator) const;  
 
  int64_t GetVersion() const;  
};
 
struct Session : Base<OrtSession> {
  explicit Session(std::nullptr_t) {}                                                                                                        
  Session(Env& env, const ORTCHAR_T* model_path, const SessionOptions& options);                                                             
  Session(Env& env, const ORTCHAR_T* model_path, const SessionOptions& options, OrtPrepackedWeightsContainer* prepacked_weights_container);  
  Session(Env& env, const void* model_data, size_t model_data_length, const SessionOptions& options);                                        
  Session(Env& env, const void* model_data, size_t model_data_length, const SessionOptions& options,
          OrtPrepackedWeightsContainer* prepacked_weights_container);  
 
  std::vector<Value> Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count,
                         const char* const* output_names, size_t output_count);
 
  void Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count,
           const char* const* output_names, Value* output_values, size_t output_count);
 
  void Run(const RunOptions& run_options, const struct IoBinding&);  
 
  size_t GetInputCount() const;                   
  size_t GetOutputCount() const;                  
  size_t GetOverridableInitializerCount() const;  
 
  char* GetInputName(size_t index, OrtAllocator* allocator) const;  
 
  AllocatedStringPtr GetInputNameAllocated(size_t index, OrtAllocator* allocator) const;
 
  char* GetOutputName(size_t index, OrtAllocator* allocator) const;  
 
  AllocatedStringPtr GetOutputNameAllocated(size_t index, OrtAllocator* allocator) const;
 
  char* GetOverridableInitializerName(size_t index, OrtAllocator* allocator) const;  
 
  AllocatedStringPtr GetOverridableInitializerNameAllocated(size_t index, OrtAllocator* allocator) const;  
 
  char* EndProfiling(OrtAllocator* allocator) const;  
 
  AllocatedStringPtr EndProfilingAllocated(OrtAllocator* allocator) const;  
  uint64_t GetProfilingStartTimeNs() const;                                 
  ModelMetadata GetModelMetadata() const;                                   
 
  TypeInfo GetInputTypeInfo(size_t index) const;                   
  TypeInfo GetOutputTypeInfo(size_t index) const;                  
  TypeInfo GetOverridableInitializerTypeInfo(size_t index) const;  
};
 
struct TensorTypeAndShapeInfo : Base<OrtTensorTypeAndShapeInfo> {
  explicit TensorTypeAndShapeInfo(std::nullptr_t) {}                                                     
  explicit TensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo* p) : Base<OrtTensorTypeAndShapeInfo>{p} {}  
 
  ONNXTensorElementDataType GetElementType() const;  
  size_t GetElementCount() const;                    
 
  size_t GetDimensionsCount() const;                                           
  void GetDimensions(int64_t* values, size_t values_count) const;              
  void GetSymbolicDimensions(const char** values, size_t values_count) const;  
 
  std::vector<int64_t> GetShape() const;  
};
 
struct SequenceTypeInfo : Base<OrtSequenceTypeInfo> {
  explicit SequenceTypeInfo(std::nullptr_t) {}                                         
  explicit SequenceTypeInfo(OrtSequenceTypeInfo* p) : Base<OrtSequenceTypeInfo>{p} {}  
 
  TypeInfo GetSequenceElementType() const;  
};
 
struct MapTypeInfo : Base<OrtMapTypeInfo> {
  explicit MapTypeInfo(std::nullptr_t) {}                               
  explicit MapTypeInfo(OrtMapTypeInfo* p) : Base<OrtMapTypeInfo>{p} {}  
 
  ONNXTensorElementDataType GetMapKeyType() const;  
  TypeInfo GetMapValueType() const;                 
};
 
struct TypeInfo : Base<OrtTypeInfo> {
  explicit TypeInfo(std::nullptr_t) {}                         
  explicit TypeInfo(OrtTypeInfo* p) : Base<OrtTypeInfo>{p} {}  
 
  Unowned<TensorTypeAndShapeInfo> GetTensorTypeAndShapeInfo() const;  
  Unowned<SequenceTypeInfo> GetSequenceTypeInfo() const;              
  Unowned<MapTypeInfo> GetMapTypeInfo() const;                        
 
  ONNXType GetONNXType() const;
};
 
struct Value : Base<OrtValue> {
  // This structure is used to feed  sparse tensor values
  // information for use with FillSparseTensor<Format>() API
  // if the data type for the sparse tensor values is numeric
  // use data.p_data, otherwise, use data.str pointer to feed
  // values. data.str is an array of const char* that are zero terminated.
  // number of strings in the array must match shape size.
  // For fully sparse tensors use shape {0} and set p_data/str
  // to nullptr.
  struct OrtSparseValuesParam {
    const int64_t* values_shape;
    size_t values_shape_len;
    union {
      const void* p_data;
      const char** str;
    } data;
  };
 
  // Provides a way to pass shape in a single
  // argument
  struct Shape {
    const int64_t* shape;
    size_t shape_len;
  };
 
  template <typename T>
  static Value CreateTensor(const OrtMemoryInfo* info, T* p_data, size_t p_data_element_count, const int64_t* shape, size_t shape_len);
 
  static Value CreateTensor(const OrtMemoryInfo* info, void* p_data, size_t p_data_byte_count, const int64_t* shape, size_t shape_len,
                            ONNXTensorElementDataType type);
 
#if !defined(DISABLE_SPARSE_TENSORS)
  template <typename T>
  static Value CreateSparseTensor(const OrtMemoryInfo* info, T* p_data, const Shape& dense_shape,
                                  const Shape& values_shape);
 
  static Value CreateSparseTensor(const OrtMemoryInfo* info, void* p_data, const Shape& dense_shape,
                                  const Shape& values_shape, ONNXTensorElementDataType type);
 
  void UseCooIndices(int64_t* indices_data, size_t indices_num);
 
  void UseCsrIndices(int64_t* inner_data, size_t inner_num, int64_t* outer_data, size_t outer_num);
 
  void UseBlockSparseIndices(const Shape& indices_shape, int32_t* indices_data);
 
#endif  // !defined(DISABLE_SPARSE_TENSORS)
 
  template <typename T>
  static Value CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len);
 
  static Value CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type);
 
#if !defined(DISABLE_SPARSE_TENSORS)
  template <typename T>
  static Value CreateSparseTensor(OrtAllocator* allocator, const Shape& dense_shape);
 
  static Value CreateSparseTensor(OrtAllocator* allocator, const Shape& dense_shape, ONNXTensorElementDataType type);
 
  void FillSparseTensorCoo(const OrtMemoryInfo* data_mem_info, const OrtSparseValuesParam& values_param,
                           const int64_t* indices_data, size_t indices_num);
 
  void FillSparseTensorCsr(const OrtMemoryInfo* data_mem_info,
                           const OrtSparseValuesParam& values,
                           const int64_t* inner_indices_data, size_t inner_indices_num,
                           const int64_t* outer_indices_data, size_t outer_indices_num);
 
  void FillSparseTensorBlockSparse(const OrtMemoryInfo* data_mem_info,
                                   const OrtSparseValuesParam& values,
                                   const Shape& indices_shape,
                                   const int32_t* indices_data);
 
  OrtSparseFormat GetSparseFormat() const;
 
  TensorTypeAndShapeInfo GetSparseTensorValuesTypeAndShapeInfo() const;
 
  TensorTypeAndShapeInfo GetSparseTensorIndicesTypeShapeInfo(OrtSparseIndicesFormat format) const;
 
  template <typename T>
  const T* GetSparseTensorIndicesData(OrtSparseIndicesFormat indices_format, size_t& num_indices) const;
 
#endif  // !defined(DISABLE_SPARSE_TENSORS)
 
  static Value CreateMap(Value& keys, Value& values);       
  static Value CreateSequence(std::vector<Value>& values);  
 
  template <typename T>
  static Value CreateOpaque(const char* domain, const char* type_name, const T&);  
 
  template <typename T>
  void GetOpaqueData(const char* domain, const char* type_name, T&) const;  
 
  explicit Value(std::nullptr_t) {}                   
  explicit Value(OrtValue* p) : Base<OrtValue>{p} {}  
  Value(Value&&) = default;
  Value& operator=(Value&&) = default;
 
  bool IsTensor() const;  
  bool HasValue() const;  
 
#if !defined(DISABLE_SPARSE_TENSORS)
  bool IsSparseTensor() const;
#endif
 
  size_t GetCount() const;  // If a non tensor, returns 2 for map and N for sequence, where N is the number of elements
  Value GetValue(int index, OrtAllocator* allocator) const;
 
  size_t GetStringTensorDataLength() const;
 
  void GetStringTensorContent(void* buffer, size_t buffer_length, size_t* offsets, size_t offsets_count) const;
 
  template <typename T>
  T* GetTensorMutableData();  
 
  template <typename T>
  const T* GetTensorData() const;  
 
#if !defined(DISABLE_SPARSE_TENSORS)
  template <typename T>
  const T* GetSparseTensorValues() const;
#endif
 
  template <typename T>
  T& At(const std::vector<int64_t>& location);
 
  TypeInfo GetTypeInfo() const;
 
  TensorTypeAndShapeInfo GetTensorTypeAndShapeInfo() const;
 
  size_t GetStringTensorElementLength(size_t element_index) const;
 
  void GetStringTensorElement(size_t buffer_length, size_t element_index, void* buffer) const;
 
  void FillStringTensor(const char* const* s, size_t s_len);
  void FillStringTensorElement(const char* s, size_t index);
};
 
// Represents native memory allocation
struct MemoryAllocation {
  MemoryAllocation(OrtAllocator* allocator, void* p, size_t size);
  ~MemoryAllocation();
  MemoryAllocation(const MemoryAllocation&) = delete;
  MemoryAllocation& operator=(const MemoryAllocation&) = delete;
  MemoryAllocation(MemoryAllocation&&) noexcept;
  MemoryAllocation& operator=(MemoryAllocation&&) noexcept;
 
  void* get() { return p_; }
  size_t size() const { return size_; }
 
 private:
  OrtAllocator* allocator_;
  void* p_;
  size_t size_;
};
 
struct AllocatorWithDefaultOptions {
  AllocatorWithDefaultOptions();
 
  operator OrtAllocator*() { return p_; }
  operator const OrtAllocator*() const { return p_; }
 
  void* Alloc(size_t size);
  // The return value will own the allocation
  MemoryAllocation GetAllocation(size_t size);
  void Free(void* p);
 
  const OrtMemoryInfo* GetInfo() const;
 
 private:
  OrtAllocator* p_{};
};
 
struct MemoryInfo : Base<OrtMemoryInfo> {
  static MemoryInfo CreateCpu(OrtAllocatorType type, OrtMemType mem_type1);
 
  explicit MemoryInfo(std::nullptr_t) {}
  explicit MemoryInfo(OrtMemoryInfo* p) : Base<OrtMemoryInfo>{p} {}  
  MemoryInfo(const char* name, OrtAllocatorType type, int id, OrtMemType mem_type);
 
  std::string GetAllocatorName() const;
  OrtAllocatorType GetAllocatorType() const;
  int GetDeviceId() const;
  OrtMemType GetMemoryType() const;
 
  bool operator==(const MemoryInfo& o) const;
};
 
struct Allocator : public Base<OrtAllocator> {
  Allocator(const Session& session, const MemoryInfo&);
 
  void* Alloc(size_t size) const;
  // The return value will own the allocation
  MemoryAllocation GetAllocation(size_t size);
  void Free(void* p) const;
  Unowned<const MemoryInfo> GetInfo() const;
};
 
struct IoBinding : public Base<OrtIoBinding> {
  explicit IoBinding(Session& session);
  void BindInput(const char* name, const Value&);
  void BindOutput(const char* name, const Value&);
  void BindOutput(const char* name, const MemoryInfo&);
  std::vector<std::string> GetOutputNames() const;
  std::vector<std::string> GetOutputNames(Allocator&) const;
  std::vector<Value> GetOutputValues() const;
  std::vector<Value> GetOutputValues(Allocator&) const;
  void ClearBoundInputs();
  void ClearBoundOutputs();
  void SynchronizeInputs();
  void SynchronizeOutputs();
 
 private:
  std::vector<std::string> GetOutputNamesHelper(OrtAllocator*) const;
  std::vector<Value> GetOutputValuesHelper(OrtAllocator*) const;
};
 
struct ArenaCfg : Base<OrtArenaCfg> {
  explicit ArenaCfg(std::nullptr_t) {}  
  ArenaCfg(size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, int max_dead_bytes_per_chunk);
};
 
//
// Custom OPs (only needed to implement custom OPs)
//
 
struct CustomOpApi {
  CustomOpApi(const OrtApi& api) : api_(api) {}
 
  template <typename T>  // T is only implemented for std::vector<float>, std::vector<int64_t>, float, int64_t, and string
  T KernelInfoGetAttribute(_In_ const OrtKernelInfo* info, _In_ const char* name);
 
  OrtTensorTypeAndShapeInfo* GetTensorTypeAndShape(_In_ const OrtValue* value);
  size_t GetTensorShapeElementCount(_In_ const OrtTensorTypeAndShapeInfo* info);
  ONNXTensorElementDataType GetTensorElementType(const OrtTensorTypeAndShapeInfo* info);
  size_t GetDimensionsCount(_In_ const OrtTensorTypeAndShapeInfo* info);
  void GetDimensions(_In_ const OrtTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values, size_t dim_values_length);
  void SetDimensions(OrtTensorTypeAndShapeInfo* info, _In_ const int64_t* dim_values, size_t dim_count);
 
  template <typename T>
  T* GetTensorMutableData(_Inout_ OrtValue* value);
  template <typename T>
  const T* GetTensorData(_Inout_ const OrtValue* value);
 
  const OrtMemoryInfo* GetTensorMemoryInfo(_In_ const OrtValue* value);
 
  std::vector<int64_t> GetTensorShape(const OrtTensorTypeAndShapeInfo* info);
  void ReleaseTensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo* input);
  size_t KernelContext_GetInputCount(const OrtKernelContext* context);
  const OrtValue* KernelContext_GetInput(const OrtKernelContext* context, _In_ size_t index);
  size_t KernelContext_GetOutputCount(const OrtKernelContext* context);
  OrtValue* KernelContext_GetOutput(OrtKernelContext* context, _In_ size_t index, _In_ const int64_t* dim_values, size_t dim_count);
  void* KernelContext_GetGPUComputeStream(const OrtKernelContext* context);
 
  void ThrowOnError(OrtStatus* result);
 
  OrtOpAttr* CreateOpAttr(_In_ const char* name,
                          _In_ const void* data,
                          _In_ int len,
                          _In_ OrtOpAttrType type);
 
  void ReleaseOpAttr(_Frees_ptr_opt_ OrtOpAttr* op_attr);
 
  OrtOp* CreateOp(_In_ const OrtKernelInfo* info,
                  _In_ const char* op_name,
                  _In_ const char* domain,
                  _In_ int version,
                  _In_opt_ const char** type_constraint_names,
                  _In_opt_ const ONNXTensorElementDataType* type_constraint_values,
                  _In_opt_ int type_constraint_count,
                  _In_opt_ const OrtOpAttr* const* attr_values,
                  _In_opt_ int attr_count,
                  _In_ int input_count,
                  _In_ int output_count);
 
  void InvokeOp(_In_ const OrtKernelContext* context,
                _In_ const OrtOp* ort_op,
                _In_ const OrtValue* const* input_values,
                _In_ int input_count,
                _Inout_ OrtValue* const* output_values,
                _In_ int output_count);
 
  void ReleaseOp(_Frees_ptr_opt_ OrtOp* ort_op);
 
  OrtKernelInfo* CopyKernelInfo(_In_ const OrtKernelInfo* info);
 
  void ReleaseKernelInfo(_Frees_ptr_opt_ OrtKernelInfo* info_copy);
 
 private:
  const OrtApi& api_;
};
 
template <typename TOp, typename TKernel>
struct CustomOpBase : OrtCustomOp {
  CustomOpBase() {
    OrtCustomOp::version = ORT_API_VERSION;
    OrtCustomOp::CreateKernel = [](const OrtCustomOp* this_, const OrtApi* api, const OrtKernelInfo* info) { return static_cast<const TOp*>(this_)->CreateKernel(*api, info); };
    OrtCustomOp::GetName = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetName(); };
 
    OrtCustomOp::GetExecutionProviderType = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetExecutionProviderType(); };
 
    OrtCustomOp::GetInputTypeCount = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetInputTypeCount(); };
    OrtCustomOp::GetInputType = [](const OrtCustomOp* this_, size_t index) { return static_cast<const TOp*>(this_)->GetInputType(index); };
 
    OrtCustomOp::GetOutputTypeCount = [](const OrtCustomOp* this_) { return static_cast<const TOp*>(this_)->GetOutputTypeCount(); };
    OrtCustomOp::GetOutputType = [](const OrtCustomOp* this_, size_t index) { return static_cast<const TOp*>(this_)->GetOutputType(index); };
 
    OrtCustomOp::KernelCompute = [](void* op_kernel, OrtKernelContext* context) { static_cast<TKernel*>(op_kernel)->Compute(context); };
#if defined(_MSC_VER) && !defined(__clang__)
#pragma warning(push)
#pragma warning(disable : 26409)
#endif
    OrtCustomOp::KernelDestroy = [](void* op_kernel) { delete static_cast<TKernel*>(op_kernel); };
#if defined(_MSC_VER) && !defined(__clang__)
#pragma warning(pop)
#endif
    OrtCustomOp::GetInputCharacteristic = [](const OrtCustomOp* this_, size_t index) { return static_cast<const TOp*>(this_)->GetInputCharacteristic(index); };
    OrtCustomOp::GetOutputCharacteristic = [](const OrtCustomOp* this_, size_t index) { return static_cast<const TOp*>(this_)->GetOutputCharacteristic(index); };
  }
 
  // Default implementation of GetExecutionProviderType that returns nullptr to default to the CPU provider
  const char* GetExecutionProviderType() const { return nullptr; }
 
  // Default implementations of GetInputCharacteristic() and GetOutputCharacteristic() below
  // (inputs and outputs are required by default)
  OrtCustomOpInputOutputCharacteristic GetInputCharacteristic(size_t /*index*/) const {
    return OrtCustomOpInputOutputCharacteristic::INPUT_OUTPUT_REQUIRED;
  }
 
  OrtCustomOpInputOutputCharacteristic GetOutputCharacteristic(size_t /*index*/) const {
    return OrtCustomOpInputOutputCharacteristic::INPUT_OUTPUT_REQUIRED;
  }
};
 
}  // namespace Ort
 
#include "onnxruntime_cxx_inline.h"