microsoft/onnxruntime-extensions
Publicmirrored from https://github.com/microsoft/onnxruntime-extensionsAvailable
includes/onnxruntime/onnxruntime_cxx_inline.h
981lines · modecode
| 1 | // Copyright (c) Microsoft Corporation. All rights reserved. |
| 2 | // Licensed under the MIT License. |
| 3 | |
| 4 | // Do not include this file directly. Please include "onnxruntime_cxx_api.h" instead. |
| 5 | // If interested in trying out features of the new experimental C++ API, include "experimental_onnxruntime_cxx_api.h" instead. |
| 6 | // |
| 7 | // These are the inline implementations of the C++ header APIs. They're in this separate file as to not clutter |
| 8 | // the main C++ file with implementation details. |
| 9 | |
| 10 | namespace Ort { |
| 11 | |
| 12 | inline void ThrowOnError(const OrtApi& ort, OrtStatus* status) { |
| 13 | if (status) { |
| 14 | std::string error_message = ort.GetErrorMessage(status); |
| 15 | OrtErrorCode error_code = ort.GetErrorCode(status); |
| 16 | ort.ReleaseStatus(status); |
| 17 | ORT_CXX_API_THROW(std::move(error_message), error_code); |
| 18 | } |
| 19 | } |
| 20 | |
| 21 | inline void ThrowOnError(OrtStatus* status) { |
| 22 | ThrowOnError(GetApi(), status); |
| 23 | } |
| 24 | |
| 25 | // This template converts a C++ type into it's ONNXTensorElementDataType |
| 26 | template <typename T> |
| 27 | struct TypeToTensorType; |
| 28 | template <> |
| 29 | struct TypeToTensorType<float> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT; }; |
| 30 | template <> |
| 31 | struct TypeToTensorType<Float16_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16; }; |
| 32 | template <> |
| 33 | struct TypeToTensorType<BFloat16_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16; }; |
| 34 | template <> |
| 35 | struct TypeToTensorType<double> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE; }; |
| 36 | template <> |
| 37 | struct TypeToTensorType<int8_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8; }; |
| 38 | template <> |
| 39 | struct TypeToTensorType<int16_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16; }; |
| 40 | template <> |
| 41 | struct TypeToTensorType<int32_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32; }; |
| 42 | template <> |
| 43 | struct TypeToTensorType<int64_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64; }; |
| 44 | template <> |
| 45 | struct TypeToTensorType<uint8_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8; }; |
| 46 | template <> |
| 47 | struct TypeToTensorType<uint16_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16; }; |
| 48 | template <> |
| 49 | struct TypeToTensorType<uint32_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32; }; |
| 50 | template <> |
| 51 | struct TypeToTensorType<uint64_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64; }; |
| 52 | template <> |
| 53 | struct TypeToTensorType<bool> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL; }; |
| 54 | |
| 55 | inline MemoryAllocation::MemoryAllocation(OrtAllocator* allocator, void* p, size_t size) |
| 56 | : allocator_(allocator), p_(p), size_(size) { |
| 57 | } |
| 58 | |
| 59 | inline MemoryAllocation::~MemoryAllocation() { |
| 60 | if (p_ != nullptr) { |
| 61 | // We do not throw out of destructor |
| 62 | auto ret = GetApi().AllocatorFree(allocator_, p_); |
| 63 | static_cast<void>(ret); |
| 64 | } |
| 65 | } |
| 66 | |
| 67 | inline MemoryAllocation::MemoryAllocation(MemoryAllocation&& o) : allocator_(nullptr), p_(nullptr), size_(0) { |
| 68 | *this = std::move(o); |
| 69 | } |
| 70 | |
| 71 | inline MemoryAllocation& MemoryAllocation::operator=(MemoryAllocation&& o) { |
| 72 | OrtAllocator* alloc = nullptr; |
| 73 | void* p = nullptr; |
| 74 | size_t sz = 0; |
| 75 | |
| 76 | // Swap out this |
| 77 | std::swap(alloc, allocator_); |
| 78 | std::swap(p, p_); |
| 79 | std::swap(sz, size_); |
| 80 | |
| 81 | // Swap with incoming |
| 82 | std::swap(allocator_, o.allocator_); |
| 83 | std::swap(p_, o.p_); |
| 84 | std::swap(size_, o.size_); |
| 85 | |
| 86 | // Destroy this instance if needed |
| 87 | MemoryAllocation this_alloc(alloc, p, sz); |
| 88 | return *this; |
| 89 | } |
| 90 | |
| 91 | inline AllocatorWithDefaultOptions::AllocatorWithDefaultOptions() { |
| 92 | ThrowOnError(GetApi().GetAllocatorWithDefaultOptions(&p_)); |
| 93 | } |
| 94 | |
| 95 | inline void* AllocatorWithDefaultOptions::Alloc(size_t size) { |
| 96 | void* out; |
| 97 | ThrowOnError(GetApi().AllocatorAlloc(p_, size, &out)); |
| 98 | return out; |
| 99 | } |
| 100 | |
| 101 | inline MemoryAllocation Ort::AllocatorWithDefaultOptions::GetAllocation(size_t size) { |
| 102 | void* out; |
| 103 | ThrowOnError(GetApi().AllocatorAlloc(p_, size, &out)); |
| 104 | MemoryAllocation result(p_, out, size); |
| 105 | return result; |
| 106 | } |
| 107 | |
| 108 | inline void AllocatorWithDefaultOptions::Free(void* p) { |
| 109 | ThrowOnError(GetApi().AllocatorFree(p_, p)); |
| 110 | } |
| 111 | |
| 112 | inline const OrtMemoryInfo* AllocatorWithDefaultOptions::GetInfo() const { |
| 113 | const OrtMemoryInfo* out; |
| 114 | ThrowOnError(GetApi().AllocatorGetInfo(p_, &out)); |
| 115 | return out; |
| 116 | } |
| 117 | |
| 118 | template <typename B> |
| 119 | inline std::string BaseMemoryInfo<B>::GetAllocatorName() const { |
| 120 | const char* name = nullptr; |
| 121 | ThrowOnError(GetApi().MemoryInfoGetName(*this, &name)); |
| 122 | return std::string(name); |
| 123 | } |
| 124 | |
| 125 | template <typename B> |
| 126 | inline OrtAllocatorType BaseMemoryInfo<B>::GetAllocatorType() const { |
| 127 | OrtAllocatorType type; |
| 128 | ThrowOnError(GetApi().MemoryInfoGetType(*this, &type)); |
| 129 | return type; |
| 130 | } |
| 131 | |
| 132 | template <typename B> |
| 133 | int BaseMemoryInfo<B>::GetDeviceId() const { |
| 134 | int id = 0; |
| 135 | ThrowOnError(GetApi().MemoryInfoGetId(*this, &id)); |
| 136 | return id; |
| 137 | } |
| 138 | |
| 139 | template <typename B> |
| 140 | inline OrtMemType BaseMemoryInfo<B>::GetMemoryType() const { |
| 141 | OrtMemType type; |
| 142 | ThrowOnError(GetApi().MemoryInfoGetMemType(*this, &type)); |
| 143 | return type; |
| 144 | } |
| 145 | |
| 146 | template <typename B> |
| 147 | template <typename U> |
| 148 | inline bool BaseMemoryInfo<B>::operator==(const BaseMemoryInfo<U>& o) const { |
| 149 | int comp_result = 0; |
| 150 | ThrowOnError(Ort::GetApi().CompareMemoryInfo(*this, o, &comp_result)); |
| 151 | return comp_result == 0; |
| 152 | } |
| 153 | |
| 154 | inline MemoryInfo MemoryInfo::CreateCpu(OrtAllocatorType type, OrtMemType mem_type) { |
| 155 | OrtMemoryInfo* p; |
| 156 | ThrowOnError(GetApi().CreateCpuMemoryInfo(type, mem_type, &p)); |
| 157 | return MemoryInfo(p); |
| 158 | } |
| 159 | |
| 160 | inline MemoryInfo::MemoryInfo(const char* name, OrtAllocatorType type, int id, OrtMemType mem_type) { |
| 161 | ThrowOnError(GetApi().CreateMemoryInfo(name, type, id, mem_type, &p_)); |
| 162 | } |
| 163 | |
| 164 | inline Allocator::Allocator(const Session& sess, const MemoryInfo& mem_info) { |
| 165 | ThrowOnError(GetApi().CreateAllocator(sess, mem_info, &p_)); |
| 166 | } |
| 167 | |
| 168 | inline void* Allocator::Alloc(size_t size) const { |
| 169 | void* out = nullptr; |
| 170 | ThrowOnError(GetApi().AllocatorAlloc(p_, size, &out)); |
| 171 | return out; |
| 172 | } |
| 173 | |
| 174 | inline MemoryAllocation Ort::Allocator::GetAllocation(size_t size) { |
| 175 | void* out = nullptr; |
| 176 | ThrowOnError(GetApi().AllocatorAlloc(p_, size, &out)); |
| 177 | MemoryAllocation result(p_, out, size); |
| 178 | return result; |
| 179 | } |
| 180 | |
| 181 | inline void Allocator::Free(void* p) const { |
| 182 | ThrowOnError(GetApi().AllocatorFree(p_, p)); |
| 183 | } |
| 184 | |
| 185 | inline UnownedMemoryInfo Allocator::GetInfo() const { |
| 186 | const OrtMemoryInfo* out = nullptr; |
| 187 | ThrowOnError(GetApi().AllocatorGetInfo(p_, &out)); |
| 188 | return UnownedMemoryInfo(out); |
| 189 | } |
| 190 | |
| 191 | inline IoBinding::IoBinding(Session& session) { |
| 192 | ThrowOnError(GetApi().CreateIoBinding(session, &p_)); |
| 193 | } |
| 194 | |
| 195 | inline void IoBinding::BindInput(const char* name, const Value& value) { |
| 196 | ThrowOnError(GetApi().BindInput(p_, name, value)); |
| 197 | } |
| 198 | |
| 199 | inline void IoBinding::BindOutput(const char* name, const Value& value) { |
| 200 | ThrowOnError(GetApi().BindOutput(p_, name, value)); |
| 201 | } |
| 202 | |
| 203 | inline void IoBinding::BindOutput(const char* name, const MemoryInfo& mem_info) { |
| 204 | ThrowOnError(GetApi().BindOutputToDevice(p_, name, mem_info)); |
| 205 | } |
| 206 | |
| 207 | inline std::vector<std::string> IoBinding::GetOutputNamesHelper(OrtAllocator* allocator) const { |
| 208 | std::vector<std::string> result; |
| 209 | auto free_fn = [allocator](void* p) { if (p) allocator->Free(allocator, p); }; |
| 210 | using Ptr = std::unique_ptr<void, decltype(free_fn)>; |
| 211 | |
| 212 | char* buffer = nullptr; |
| 213 | size_t* lengths = nullptr; |
| 214 | size_t count = 0; |
| 215 | ThrowOnError(GetApi().GetBoundOutputNames(p_, allocator, &buffer, &lengths, &count)); |
| 216 | |
| 217 | if (count == 0) { |
| 218 | return result; |
| 219 | } |
| 220 | |
| 221 | Ptr buffer_g(buffer, free_fn); |
| 222 | Ptr lengths_g(lengths, free_fn); |
| 223 | |
| 224 | result.reserve(count); |
| 225 | for (size_t i = 0; i < count; ++i) { |
| 226 | auto sz = *lengths; |
| 227 | result.emplace_back(buffer, sz); |
| 228 | buffer += sz; |
| 229 | ++lengths; |
| 230 | } |
| 231 | return result; |
| 232 | } |
| 233 | |
| 234 | inline std::vector<std::string> IoBinding::GetOutputNames() const { |
| 235 | AllocatorWithDefaultOptions allocator; |
| 236 | return GetOutputNamesHelper(allocator); |
| 237 | } |
| 238 | |
| 239 | inline std::vector<std::string> IoBinding::GetOutputNames(Allocator& allocator) const { |
| 240 | return GetOutputNamesHelper(allocator); |
| 241 | } |
| 242 | |
| 243 | inline std::vector<Value> Ort::IoBinding::GetOutputValuesHelper(OrtAllocator* allocator) const { |
| 244 | std::vector<Value> result; |
| 245 | size_t owned = 0; |
| 246 | size_t output_count = 0; |
| 247 | // Lambda to release the buffer when no longer needed and |
| 248 | // make sure that we destroy all instances on exception |
| 249 | auto free_fn = [&owned, &output_count, allocator](OrtValue** buffer) { |
| 250 | if (buffer) { |
| 251 | while (owned < output_count) { |
| 252 | auto* p = buffer + owned++; |
| 253 | GetApi().ReleaseValue(*p); |
| 254 | } |
| 255 | allocator->Free(allocator, buffer); |
| 256 | } |
| 257 | }; |
| 258 | using Ptr = std::unique_ptr<OrtValue*, decltype(free_fn)>; |
| 259 | |
| 260 | OrtValue** output_buffer = nullptr; |
| 261 | ThrowOnError(GetApi().GetBoundOutputValues(p_, allocator, &output_buffer, &output_count)); |
| 262 | if (output_count == 0) { |
| 263 | return result; |
| 264 | } |
| 265 | |
| 266 | Ptr buffer_g(output_buffer, free_fn); |
| 267 | |
| 268 | result.reserve(output_count); |
| 269 | for (size_t i = 0; i < output_count; ++i) { |
| 270 | result.emplace_back(output_buffer[i]); |
| 271 | ++owned; |
| 272 | } |
| 273 | return result; |
| 274 | } |
| 275 | |
| 276 | inline std::vector<Value> Ort::IoBinding::GetOutputValues(Allocator& allocator) const { |
| 277 | return GetOutputValuesHelper(allocator); |
| 278 | } |
| 279 | |
| 280 | inline std::vector<Value> Ort::IoBinding::GetOutputValues() const { |
| 281 | AllocatorWithDefaultOptions allocator; |
| 282 | return GetOutputValuesHelper(allocator); |
| 283 | } |
| 284 | |
| 285 | inline void IoBinding::ClearBoundInputs() { |
| 286 | GetApi().ClearBoundInputs(p_); |
| 287 | } |
| 288 | |
| 289 | inline void IoBinding::ClearBoundOutputs() { |
| 290 | GetApi().ClearBoundOutputs(p_); |
| 291 | } |
| 292 | |
| 293 | inline ArenaCfg::ArenaCfg(size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, int max_dead_bytes_per_chunk) { |
| 294 | ThrowOnError(GetApi().CreateArenaCfg(max_mem, arena_extend_strategy, initial_chunk_size_bytes, max_dead_bytes_per_chunk, &p_)); |
| 295 | } |
| 296 | |
| 297 | inline Env::Env(OrtLoggingLevel logging_level, _In_ const char* logid) { |
| 298 | ThrowOnError(GetApi().CreateEnv(logging_level, logid, &p_)); |
| 299 | if (strcmp(logid, "onnxruntime-node") == 0) { |
| 300 | ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS)); |
| 301 | } else { |
| 302 | ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS)); |
| 303 | } |
| 304 | } |
| 305 | |
| 306 | inline Env::Env(OrtLoggingLevel logging_level, const char* logid, OrtLoggingFunction logging_function, void* logger_param) { |
| 307 | ThrowOnError(GetApi().CreateEnvWithCustomLogger(logging_function, logger_param, logging_level, logid, &p_)); |
| 308 | if (strcmp(logid, "onnxruntime-node") == 0) { |
| 309 | ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS)); |
| 310 | } else { |
| 311 | ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS)); |
| 312 | } |
| 313 | } |
| 314 | |
| 315 | inline Env::Env(const OrtThreadingOptions* tp_options, OrtLoggingLevel logging_level, _In_ const char* logid) { |
| 316 | ThrowOnError(GetApi().CreateEnvWithGlobalThreadPools(logging_level, logid, tp_options, &p_)); |
| 317 | if (strcmp(logid, "onnxruntime-node") == 0) { |
| 318 | ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS)); |
| 319 | } else { |
| 320 | ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS)); |
| 321 | } |
| 322 | } |
| 323 | |
| 324 | inline Env::Env(const OrtThreadingOptions* tp_options, OrtLoggingFunction logging_function, void* logger_param, |
| 325 | OrtLoggingLevel logging_level, _In_ const char* logid) { |
| 326 | ThrowOnError(GetApi().CreateEnvWithCustomLoggerAndGlobalThreadPools(logging_function, logger_param, logging_level, logid, tp_options, &p_)); |
| 327 | if (strcmp(logid, "onnxruntime-node") == 0) { |
| 328 | ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_NODEJS)); |
| 329 | } else { |
| 330 | ThrowOnError(GetApi().SetLanguageProjection(p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS)); |
| 331 | } |
| 332 | } |
| 333 | |
| 334 | inline Env& Env::EnableTelemetryEvents() { |
| 335 | ThrowOnError(GetApi().EnableTelemetryEvents(p_)); |
| 336 | return *this; |
| 337 | } |
| 338 | |
| 339 | inline Env& Env::DisableTelemetryEvents() { |
| 340 | ThrowOnError(GetApi().DisableTelemetryEvents(p_)); |
| 341 | return *this; |
| 342 | } |
| 343 | |
| 344 | inline Env& Env::CreateAndRegisterAllocator(const OrtMemoryInfo* mem_info, const OrtArenaCfg* arena_cfg) { |
| 345 | ThrowOnError(GetApi().CreateAndRegisterAllocator(p_, mem_info, arena_cfg)); |
| 346 | return *this; |
| 347 | } |
| 348 | |
| 349 | inline CustomOpDomain::CustomOpDomain(const char* domain) { |
| 350 | ThrowOnError(GetApi().CreateCustomOpDomain(domain, &p_)); |
| 351 | } |
| 352 | |
| 353 | inline void CustomOpDomain::Add(OrtCustomOp* op) { |
| 354 | ThrowOnError(GetApi().CustomOpDomain_Add(p_, op)); |
| 355 | } |
| 356 | |
| 357 | inline RunOptions::RunOptions() { |
| 358 | ThrowOnError(GetApi().CreateRunOptions(&p_)); |
| 359 | } |
| 360 | |
| 361 | inline RunOptions& RunOptions::SetRunLogVerbosityLevel(int level) { |
| 362 | ThrowOnError(GetApi().RunOptionsSetRunLogVerbosityLevel(p_, level)); |
| 363 | return *this; |
| 364 | } |
| 365 | |
| 366 | inline RunOptions& RunOptions::SetRunLogSeverityLevel(int level) { |
| 367 | ThrowOnError(GetApi().RunOptionsSetRunLogSeverityLevel(p_, level)); |
| 368 | return *this; |
| 369 | } |
| 370 | |
| 371 | inline int RunOptions::GetRunLogVerbosityLevel() const { |
| 372 | int out; |
| 373 | ThrowOnError(GetApi().RunOptionsGetRunLogVerbosityLevel(p_, &out)); |
| 374 | return out; |
| 375 | } |
| 376 | |
| 377 | inline RunOptions& RunOptions::SetRunTag(const char* run_tag) { |
| 378 | ThrowOnError(GetApi().RunOptionsSetRunTag(p_, run_tag)); |
| 379 | return *this; |
| 380 | } |
| 381 | |
| 382 | inline const char* RunOptions::GetRunTag() const { |
| 383 | const char* out; |
| 384 | ThrowOnError(GetApi().RunOptionsGetRunTag(p_, &out)); |
| 385 | return out; |
| 386 | } |
| 387 | |
| 388 | inline RunOptions& RunOptions::SetTerminate() { |
| 389 | ThrowOnError(GetApi().RunOptionsSetTerminate(p_)); |
| 390 | return *this; |
| 391 | } |
| 392 | |
| 393 | inline RunOptions& RunOptions::UnsetTerminate() { |
| 394 | ThrowOnError(GetApi().RunOptionsUnsetTerminate(p_)); |
| 395 | return *this; |
| 396 | } |
| 397 | |
| 398 | inline SessionOptions::SessionOptions() { |
| 399 | ThrowOnError(GetApi().CreateSessionOptions(&p_)); |
| 400 | } |
| 401 | |
| 402 | inline SessionOptions SessionOptions::Clone() const { |
| 403 | OrtSessionOptions* out; |
| 404 | ThrowOnError(GetApi().CloneSessionOptions(p_, &out)); |
| 405 | return SessionOptions{out}; |
| 406 | } |
| 407 | |
| 408 | inline SessionOptions& SessionOptions::SetIntraOpNumThreads(int intra_op_num_threads) { |
| 409 | ThrowOnError(GetApi().SetIntraOpNumThreads(p_, intra_op_num_threads)); |
| 410 | return *this; |
| 411 | } |
| 412 | |
| 413 | inline SessionOptions& SessionOptions::SetInterOpNumThreads(int inter_op_num_threads) { |
| 414 | ThrowOnError(GetApi().SetInterOpNumThreads(p_, inter_op_num_threads)); |
| 415 | return *this; |
| 416 | } |
| 417 | |
| 418 | inline SessionOptions& SessionOptions::SetGraphOptimizationLevel(GraphOptimizationLevel graph_optimization_level) { |
| 419 | ThrowOnError(GetApi().SetSessionGraphOptimizationLevel(p_, graph_optimization_level)); |
| 420 | return *this; |
| 421 | } |
| 422 | |
| 423 | inline SessionOptions& SessionOptions::SetOptimizedModelFilePath(const ORTCHAR_T* optimized_model_filepath) { |
| 424 | ThrowOnError(GetApi().SetOptimizedModelFilePath(p_, optimized_model_filepath)); |
| 425 | return *this; |
| 426 | } |
| 427 | |
| 428 | inline SessionOptions& SessionOptions::EnableProfiling(const ORTCHAR_T* profile_file_prefix) { |
| 429 | ThrowOnError(GetApi().EnableProfiling(p_, profile_file_prefix)); |
| 430 | return *this; |
| 431 | } |
| 432 | |
| 433 | inline SessionOptions& SessionOptions::DisableProfiling() { |
| 434 | ThrowOnError(GetApi().DisableProfiling(p_)); |
| 435 | return *this; |
| 436 | } |
| 437 | |
| 438 | inline SessionOptions& SessionOptions::EnableMemPattern() { |
| 439 | ThrowOnError(GetApi().EnableMemPattern(p_)); |
| 440 | return *this; |
| 441 | } |
| 442 | |
| 443 | inline SessionOptions& SessionOptions::DisableMemPattern() { |
| 444 | ThrowOnError(GetApi().DisableMemPattern(p_)); |
| 445 | return *this; |
| 446 | } |
| 447 | |
| 448 | inline SessionOptions& SessionOptions::EnableCpuMemArena() { |
| 449 | ThrowOnError(GetApi().EnableCpuMemArena(p_)); |
| 450 | return *this; |
| 451 | } |
| 452 | |
| 453 | inline SessionOptions& SessionOptions::DisableCpuMemArena() { |
| 454 | ThrowOnError(GetApi().DisableCpuMemArena(p_)); |
| 455 | return *this; |
| 456 | } |
| 457 | |
| 458 | inline SessionOptions& SessionOptions::SetExecutionMode(ExecutionMode execution_mode) { |
| 459 | ThrowOnError(GetApi().SetSessionExecutionMode(p_, execution_mode)); |
| 460 | return *this; |
| 461 | } |
| 462 | |
| 463 | inline SessionOptions& SessionOptions::SetLogId(const char* logid) { |
| 464 | ThrowOnError(GetApi().SetSessionLogId(p_, logid)); |
| 465 | return *this; |
| 466 | } |
| 467 | |
| 468 | inline SessionOptions& SessionOptions::SetLogSeverityLevel(int level) { |
| 469 | ThrowOnError(GetApi().SetSessionLogSeverityLevel(p_, level)); |
| 470 | return *this; |
| 471 | } |
| 472 | |
| 473 | inline SessionOptions& SessionOptions::Add(OrtCustomOpDomain* custom_op_domain) { |
| 474 | ThrowOnError(GetApi().AddCustomOpDomain(p_, custom_op_domain)); |
| 475 | return *this; |
| 476 | } |
| 477 | |
| 478 | inline SessionOptions& SessionOptions::AddConfigEntry(const char* config_key, const char* config_value) { |
| 479 | ThrowOnError(GetApi().AddSessionConfigEntry(p_, config_key, config_value)); |
| 480 | return *this; |
| 481 | } |
| 482 | |
| 483 | inline SessionOptions& SessionOptions::AddInitializer(const char* name, const OrtValue* ort_val) { |
| 484 | ThrowOnError(GetApi().AddInitializer(p_, name, ort_val)); |
| 485 | return *this; |
| 486 | } |
| 487 | |
| 488 | inline SessionOptions& SessionOptions::AppendExecutionProvider_CUDA(const OrtCUDAProviderOptions& provider_options) { |
| 489 | ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_CUDA(p_, &provider_options)); |
| 490 | return *this; |
| 491 | } |
| 492 | |
| 493 | inline SessionOptions& SessionOptions::AppendExecutionProvider_OpenVINO(const OrtOpenVINOProviderOptions& provider_options) { |
| 494 | ThrowOnError(GetApi().SessionOptionsAppendExecutionProvider_OpenVINO(p_, &provider_options)); |
| 495 | return *this; |
| 496 | } |
| 497 | |
| 498 | inline Session::Session(Env& env, const ORTCHAR_T* model_path, const SessionOptions& options) { |
| 499 | ThrowOnError(GetApi().CreateSession(env, model_path, options, &p_)); |
| 500 | } |
| 501 | |
| 502 | inline Session::Session(Env& env, const void* model_data, size_t model_data_length, const SessionOptions& options) { |
| 503 | ThrowOnError(GetApi().CreateSessionFromArray(env, model_data, model_data_length, options, &p_)); |
| 504 | } |
| 505 | |
| 506 | inline std::vector<Value> Session::Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count, |
| 507 | const char* const* output_names, size_t output_names_count) { |
| 508 | std::vector<Ort::Value> output_values; |
| 509 | for (size_t i = 0; i < output_names_count; i++) |
| 510 | output_values.emplace_back(nullptr); |
| 511 | Run(run_options, input_names, input_values, input_count, output_names, output_values.data(), output_names_count); |
| 512 | return output_values; |
| 513 | } |
| 514 | |
| 515 | inline void Session::Run(const RunOptions& run_options, const char* const* input_names, const Value* input_values, size_t input_count, |
| 516 | const char* const* output_names, Value* output_values, size_t output_count) { |
| 517 | static_assert(sizeof(Value) == sizeof(OrtValue*), "Value is really just an array of OrtValue* in memory, so we can reinterpret_cast safely"); |
| 518 | auto ort_input_values = reinterpret_cast<const OrtValue**>(const_cast<Value*>(input_values)); |
| 519 | auto ort_output_values = reinterpret_cast<OrtValue**>(output_values); |
| 520 | ThrowOnError(GetApi().Run(p_, run_options, input_names, ort_input_values, input_count, output_names, output_count, ort_output_values)); |
| 521 | } |
| 522 | |
| 523 | inline void Session::Run(const RunOptions& run_options, const IoBinding& io_binding) { |
| 524 | ThrowOnError(GetApi().RunWithBinding(p_, run_options, io_binding)); |
| 525 | } |
| 526 | |
| 527 | inline size_t Session::GetInputCount() const { |
| 528 | size_t out; |
| 529 | ThrowOnError(GetApi().SessionGetInputCount(p_, &out)); |
| 530 | return out; |
| 531 | } |
| 532 | |
| 533 | inline size_t Session::GetOutputCount() const { |
| 534 | size_t out; |
| 535 | ThrowOnError(GetApi().SessionGetOutputCount(p_, &out)); |
| 536 | return out; |
| 537 | } |
| 538 | |
| 539 | inline size_t Session::GetOverridableInitializerCount() const { |
| 540 | size_t out; |
| 541 | ThrowOnError(GetApi().SessionGetOverridableInitializerCount(p_, &out)); |
| 542 | return out; |
| 543 | } |
| 544 | |
| 545 | inline char* Session::GetInputName(size_t index, OrtAllocator* allocator) const { |
| 546 | char* out; |
| 547 | ThrowOnError(GetApi().SessionGetInputName(p_, index, allocator, &out)); |
| 548 | return out; |
| 549 | } |
| 550 | |
| 551 | inline char* Session::GetOutputName(size_t index, OrtAllocator* allocator) const { |
| 552 | char* out; |
| 553 | ThrowOnError(GetApi().SessionGetOutputName(p_, index, allocator, &out)); |
| 554 | return out; |
| 555 | } |
| 556 | |
| 557 | inline char* Session::GetOverridableInitializerName(size_t index, OrtAllocator* allocator) const { |
| 558 | char* out; |
| 559 | ThrowOnError(GetApi().SessionGetOverridableInitializerName(p_, index, allocator, &out)); |
| 560 | return out; |
| 561 | } |
| 562 | |
| 563 | inline char* Session::EndProfiling(OrtAllocator* allocator) const { |
| 564 | char* out; |
| 565 | ThrowOnError(GetApi().SessionEndProfiling(p_, allocator, &out)); |
| 566 | return out; |
| 567 | } |
| 568 | |
| 569 | inline uint64_t Session::GetProfilingStartTimeNs() const { |
| 570 | uint64_t out; |
| 571 | ThrowOnError(GetApi().SessionGetProfilingStartTimeNs(p_, &out)); |
| 572 | return out; |
| 573 | } |
| 574 | |
| 575 | inline ModelMetadata Session::GetModelMetadata() const { |
| 576 | OrtModelMetadata* out; |
| 577 | ThrowOnError(GetApi().SessionGetModelMetadata(p_, &out)); |
| 578 | return ModelMetadata{out}; |
| 579 | } |
| 580 | |
| 581 | inline char* ModelMetadata::GetProducerName(OrtAllocator* allocator) const { |
| 582 | char* out; |
| 583 | ThrowOnError(GetApi().ModelMetadataGetProducerName(p_, allocator, &out)); |
| 584 | return out; |
| 585 | } |
| 586 | |
| 587 | inline char* ModelMetadata::GetGraphName(OrtAllocator* allocator) const { |
| 588 | char* out; |
| 589 | ThrowOnError(GetApi().ModelMetadataGetGraphName(p_, allocator, &out)); |
| 590 | return out; |
| 591 | } |
| 592 | |
| 593 | inline char* ModelMetadata::GetDomain(OrtAllocator* allocator) const { |
| 594 | char* out; |
| 595 | ThrowOnError(GetApi().ModelMetadataGetDomain(p_, allocator, &out)); |
| 596 | return out; |
| 597 | } |
| 598 | |
| 599 | inline char* ModelMetadata::GetDescription(OrtAllocator* allocator) const { |
| 600 | char* out; |
| 601 | ThrowOnError(GetApi().ModelMetadataGetDescription(p_, allocator, &out)); |
| 602 | return out; |
| 603 | } |
| 604 | |
| 605 | inline char* ModelMetadata::LookupCustomMetadataMap(const char* key, OrtAllocator* allocator) const { |
| 606 | char* out; |
| 607 | ThrowOnError(GetApi().ModelMetadataLookupCustomMetadataMap(p_, allocator, key, &out)); |
| 608 | return out; |
| 609 | } |
| 610 | |
| 611 | inline char** ModelMetadata::GetCustomMetadataMapKeys(OrtAllocator* allocator, _Out_ int64_t& num_keys) const { |
| 612 | char** out; |
| 613 | ThrowOnError(GetApi().ModelMetadataGetCustomMetadataMapKeys(p_, allocator, &out, &num_keys)); |
| 614 | return out; |
| 615 | } |
| 616 | |
| 617 | inline int64_t ModelMetadata::GetVersion() const { |
| 618 | int64_t out; |
| 619 | ThrowOnError(GetApi().ModelMetadataGetVersion(p_, &out)); |
| 620 | return out; |
| 621 | } |
| 622 | |
| 623 | inline TypeInfo Session::GetInputTypeInfo(size_t index) const { |
| 624 | OrtTypeInfo* out; |
| 625 | ThrowOnError(GetApi().SessionGetInputTypeInfo(p_, index, &out)); |
| 626 | return TypeInfo{out}; |
| 627 | } |
| 628 | |
| 629 | inline TypeInfo Session::GetOutputTypeInfo(size_t index) const { |
| 630 | OrtTypeInfo* out; |
| 631 | ThrowOnError(GetApi().SessionGetOutputTypeInfo(p_, index, &out)); |
| 632 | return TypeInfo{out}; |
| 633 | } |
| 634 | |
| 635 | inline TypeInfo Session::GetOverridableInitializerTypeInfo(size_t index) const { |
| 636 | OrtTypeInfo* out; |
| 637 | ThrowOnError(GetApi().SessionGetOverridableInitializerTypeInfo(p_, index, &out)); |
| 638 | return TypeInfo{out}; |
| 639 | } |
| 640 | |
| 641 | inline ONNXTensorElementDataType TensorTypeAndShapeInfo::GetElementType() const { |
| 642 | ONNXTensorElementDataType out; |
| 643 | ThrowOnError(GetApi().GetTensorElementType(p_, &out)); |
| 644 | return out; |
| 645 | } |
| 646 | |
| 647 | inline size_t TensorTypeAndShapeInfo::GetElementCount() const { |
| 648 | size_t out; |
| 649 | ThrowOnError(GetApi().GetTensorShapeElementCount(p_, &out)); |
| 650 | return static_cast<size_t>(out); |
| 651 | } |
| 652 | |
| 653 | inline size_t TensorTypeAndShapeInfo::GetDimensionsCount() const { |
| 654 | size_t out; |
| 655 | ThrowOnError(GetApi().GetDimensionsCount(p_, &out)); |
| 656 | return out; |
| 657 | } |
| 658 | |
| 659 | inline void TensorTypeAndShapeInfo::GetDimensions(int64_t* values, size_t values_count) const { |
| 660 | ThrowOnError(GetApi().GetDimensions(p_, values, values_count)); |
| 661 | } |
| 662 | |
| 663 | inline void TensorTypeAndShapeInfo::GetSymbolicDimensions(const char** values, size_t values_count) const { |
| 664 | ThrowOnError(GetApi().GetSymbolicDimensions(p_, values, values_count)); |
| 665 | } |
| 666 | |
| 667 | inline std::vector<int64_t> TensorTypeAndShapeInfo::GetShape() const { |
| 668 | std::vector<int64_t> out(GetDimensionsCount(), 0); |
| 669 | GetDimensions(out.data(), out.size()); |
| 670 | return out; |
| 671 | } |
| 672 | |
| 673 | inline Unowned<TensorTypeAndShapeInfo> TypeInfo::GetTensorTypeAndShapeInfo() const { |
| 674 | const OrtTensorTypeAndShapeInfo* out; |
| 675 | ThrowOnError(GetApi().CastTypeInfoToTensorInfo(p_, &out)); |
| 676 | return Unowned<TensorTypeAndShapeInfo>(const_cast<OrtTensorTypeAndShapeInfo*>(out)); |
| 677 | } |
| 678 | |
| 679 | inline Unowned<SequenceTypeInfo> TypeInfo::GetSequenceTypeInfo() const { |
| 680 | const OrtSequenceTypeInfo* out; |
| 681 | ThrowOnError(GetApi().CastTypeInfoToSequenceTypeInfo(p_, &out)); |
| 682 | return Unowned<SequenceTypeInfo>{const_cast<OrtSequenceTypeInfo*>(out)}; |
| 683 | } |
| 684 | |
| 685 | inline TypeInfo SequenceTypeInfo::GetSequenceElementType() const { |
| 686 | OrtTypeInfo* output; |
| 687 | ThrowOnError(GetApi().GetSequenceElementType(p_, &output)); |
| 688 | return TypeInfo{output}; |
| 689 | } |
| 690 | |
| 691 | inline Unowned<MapTypeInfo> TypeInfo::GetMapTypeInfo() const { |
| 692 | const OrtMapTypeInfo* out; |
| 693 | ThrowOnError(GetApi().CastTypeInfoToMapTypeInfo(p_, &out)); |
| 694 | return Unowned<MapTypeInfo>{const_cast<OrtMapTypeInfo*>(out)}; |
| 695 | } |
| 696 | |
| 697 | inline ONNXTensorElementDataType MapTypeInfo::GetMapKeyType() const { |
| 698 | ONNXTensorElementDataType out; |
| 699 | ThrowOnError(GetApi().GetMapKeyType(p_, &out)); |
| 700 | return out; |
| 701 | } |
| 702 | |
| 703 | inline TypeInfo MapTypeInfo::GetMapValueType() const { |
| 704 | OrtTypeInfo* output; |
| 705 | ThrowOnError(GetApi().GetMapValueType(p_, &output)); |
| 706 | return TypeInfo{output}; |
| 707 | } |
| 708 | |
| 709 | inline ONNXType TypeInfo::GetONNXType() const { |
| 710 | ONNXType out; |
| 711 | ThrowOnError(GetApi().GetOnnxTypeFromTypeInfo(p_, &out)); |
| 712 | return out; |
| 713 | } |
| 714 | |
| 715 | template <typename T> |
| 716 | inline Value Value::CreateTensor(const OrtMemoryInfo* info, T* p_data, size_t p_data_element_count, const int64_t* shape, size_t shape_len) { |
| 717 | return CreateTensor(info, p_data, p_data_element_count * sizeof(T), shape, shape_len, TypeToTensorType<T>::type); |
| 718 | } |
| 719 | |
| 720 | inline Value Value::CreateTensor(const OrtMemoryInfo* info, void* p_data, size_t p_data_byte_count, const int64_t* shape, size_t shape_len, |
| 721 | ONNXTensorElementDataType type) { |
| 722 | OrtValue* out; |
| 723 | ThrowOnError(GetApi().CreateTensorWithDataAsOrtValue(info, p_data, p_data_byte_count, shape, shape_len, type, &out)); |
| 724 | return Value{out}; |
| 725 | } |
| 726 | |
| 727 | template <typename T> |
| 728 | inline Value Value::CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len) { |
| 729 | return CreateTensor(allocator, shape, shape_len, TypeToTensorType<T>::type); |
| 730 | } |
| 731 | |
| 732 | inline Value Value::CreateTensor(OrtAllocator* allocator, const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type) { |
| 733 | OrtValue* out; |
| 734 | ThrowOnError(GetApi().CreateTensorAsOrtValue(allocator, shape, shape_len, type, &out)); |
| 735 | return Value{out}; |
| 736 | } |
| 737 | |
| 738 | inline Value Value::CreateMap(Value& keys, Value& values) { |
| 739 | OrtValue* out; |
| 740 | OrtValue* inputs[2] = {keys, values}; |
| 741 | ThrowOnError(GetApi().CreateValue(inputs, 2, ONNX_TYPE_MAP, &out)); |
| 742 | return Value{out}; |
| 743 | } |
| 744 | |
| 745 | inline Value Value::CreateSequence(std::vector<Value>& values) { |
| 746 | OrtValue* out; |
| 747 | std::vector<OrtValue*> values_ort{values.data(), values.data() + values.size()}; |
| 748 | ThrowOnError(GetApi().CreateValue(values_ort.data(), values_ort.size(), ONNX_TYPE_SEQUENCE, &out)); |
| 749 | return Value{out}; |
| 750 | } |
| 751 | |
| 752 | template <typename T> |
| 753 | inline Value Value::CreateOpaque(const char* domain, const char* type_name, const T& data_container) { |
| 754 | OrtValue* out; |
| 755 | ThrowOnError(GetApi().CreateOpaqueValue(domain, type_name, &data_container, sizeof(T), &out)); |
| 756 | return Value{out}; |
| 757 | } |
| 758 | |
| 759 | template <typename T> |
| 760 | inline void Value::GetOpaqueData(const char* domain, const char* type_name, T& out) const { |
| 761 | ThrowOnError(GetApi().GetOpaqueValue(domain, type_name, p_, &out, sizeof(T))); |
| 762 | } |
| 763 | |
| 764 | inline bool Value::IsTensor() const { |
| 765 | int out; |
| 766 | ThrowOnError(GetApi().IsTensor(p_, &out)); |
| 767 | return out != 0; |
| 768 | } |
| 769 | |
| 770 | inline size_t Value::GetCount() const { |
| 771 | size_t out; |
| 772 | ThrowOnError(GetApi().GetValueCount(p_, &out)); |
| 773 | return out; |
| 774 | } |
| 775 | |
| 776 | inline Value Value::GetValue(int index, OrtAllocator* allocator) const { |
| 777 | OrtValue* out; |
| 778 | ThrowOnError(GetApi().GetValue(p_, index, allocator, &out)); |
| 779 | return Value{out}; |
| 780 | } |
| 781 | |
| 782 | inline size_t Value::GetStringTensorDataLength() const { |
| 783 | size_t out; |
| 784 | ThrowOnError(GetApi().GetStringTensorDataLength(p_, &out)); |
| 785 | return out; |
| 786 | } |
| 787 | |
| 788 | inline size_t Value::GetStringTensorElementLength(size_t element_index) const { |
| 789 | size_t out; |
| 790 | ThrowOnError(GetApi().GetStringTensorElementLength(p_, element_index, &out)); |
| 791 | return out; |
| 792 | } |
| 793 | |
| 794 | inline void Value::GetStringTensorContent(void* buffer, size_t buffer_length, size_t* offsets, size_t offsets_count) const { |
| 795 | ThrowOnError(GetApi().GetStringTensorContent(p_, buffer, buffer_length, offsets, offsets_count)); |
| 796 | } |
| 797 | |
| 798 | inline void Value::GetStringTensorElement(size_t buffer_length, size_t element_index, void* buffer) const { |
| 799 | ThrowOnError(GetApi().GetStringTensorElement(p_, buffer_length, element_index, buffer)); |
| 800 | } |
| 801 | |
| 802 | inline void Value::FillStringTensor(const char* const* s, size_t s_len) { |
| 803 | ThrowOnError(GetApi().FillStringTensor(p_, s, s_len)); |
| 804 | } |
| 805 | |
| 806 | inline void Value::FillStringTensorElement(const char* s, size_t index) { |
| 807 | ThrowOnError(GetApi().FillStringTensorElement(p_, s, index)); |
| 808 | } |
| 809 | |
| 810 | template <typename T> |
| 811 | T* Value::GetTensorMutableData() { |
| 812 | T* out; |
| 813 | ThrowOnError(GetApi().GetTensorMutableData(p_, (void**)&out)); |
| 814 | return out; |
| 815 | } |
| 816 | |
| 817 | template <typename T> |
| 818 | const T* Value::GetTensorData() const { |
| 819 | T* out; |
| 820 | ThrowOnError(GetApi().GetTensorMutableData(p_, (void**)&out)); |
| 821 | return out; |
| 822 | } |
| 823 | |
| 824 | template <typename T> |
| 825 | inline T& Value::At(const std::vector<int64_t>& location) { |
| 826 | static_assert(!std::is_same<T, std::string>::value, "this api does not support std::string"); |
| 827 | T* out; |
| 828 | ThrowOnError(GetApi().TensorAt(p_, location.data(), location.size(), (void**)&out)); |
| 829 | return *out; |
| 830 | } |
| 831 | |
| 832 | inline TypeInfo Value::GetTypeInfo() const { |
| 833 | OrtTypeInfo* output; |
| 834 | ThrowOnError(GetApi().GetTypeInfo(p_, &output)); |
| 835 | return TypeInfo{output}; |
| 836 | } |
| 837 | |
| 838 | inline TensorTypeAndShapeInfo Value::GetTensorTypeAndShapeInfo() const { |
| 839 | OrtTensorTypeAndShapeInfo* output; |
| 840 | ThrowOnError(GetApi().GetTensorTypeAndShape(p_, &output)); |
| 841 | return TensorTypeAndShapeInfo{output}; |
| 842 | } |
| 843 | |
| 844 | // |
| 845 | // Custom OP API Inlines |
| 846 | // |
| 847 | inline void CustomOpApi::ThrowOnError(OrtStatus* status) { |
| 848 | Ort::ThrowOnError(api_, status); |
| 849 | } |
| 850 | |
| 851 | template <> |
| 852 | inline float CustomOpApi::KernelInfoGetAttribute<float>(_In_ const OrtKernelInfo* info, _In_ const char* name) { |
| 853 | float out; |
| 854 | ThrowOnError(api_.KernelInfoGetAttribute_float(info, name, &out)); |
| 855 | return out; |
| 856 | } |
| 857 | |
| 858 | template <> |
| 859 | inline int64_t CustomOpApi::KernelInfoGetAttribute<int64_t>(_In_ const OrtKernelInfo* info, _In_ const char* name) { |
| 860 | int64_t out; |
| 861 | ThrowOnError(api_.KernelInfoGetAttribute_int64(info, name, &out)); |
| 862 | return out; |
| 863 | } |
| 864 | |
| 865 | template <> |
| 866 | inline std::string CustomOpApi::KernelInfoGetAttribute<std::string>(_In_ const OrtKernelInfo* info, _In_ const char* name) { |
| 867 | size_t size = 0; |
| 868 | std::string out; |
| 869 | OrtStatus* status = api_.KernelInfoGetAttribute_string(info, name, nullptr, &size); |
| 870 | |
| 871 | // The status should be ORT_INVALID_ARGUMENT because the size is insufficient to hold the string |
| 872 | if (status == nullptr || api_.GetErrorCode(status) == ORT_INVALID_ARGUMENT) { |
| 873 | if (status != nullptr) { |
| 874 | api_.ReleaseStatus(status); |
| 875 | } |
| 876 | out.resize(size); |
| 877 | ThrowOnError(api_.KernelInfoGetAttribute_string(info, name, &out[0], &size)); |
| 878 | out.resize(size - 1); // remove the terminating character '\0' |
| 879 | } else { |
| 880 | ThrowOnError(status); |
| 881 | } |
| 882 | return out; |
| 883 | } |
| 884 | |
| 885 | inline OrtTensorTypeAndShapeInfo* CustomOpApi::GetTensorTypeAndShape(_In_ const OrtValue* value) { |
| 886 | OrtTensorTypeAndShapeInfo* out; |
| 887 | ThrowOnError(api_.GetTensorTypeAndShape(value, &out)); |
| 888 | return out; |
| 889 | } |
| 890 | |
| 891 | inline size_t CustomOpApi::GetTensorShapeElementCount(_In_ const OrtTensorTypeAndShapeInfo* info) { |
| 892 | size_t out; |
| 893 | ThrowOnError(api_.GetTensorShapeElementCount(info, &out)); |
| 894 | return out; |
| 895 | } |
| 896 | |
| 897 | inline ONNXTensorElementDataType CustomOpApi::GetTensorElementType(const OrtTensorTypeAndShapeInfo* info) { |
| 898 | ONNXTensorElementDataType out; |
| 899 | ThrowOnError(api_.GetTensorElementType(info, &out)); |
| 900 | return out; |
| 901 | } |
| 902 | |
| 903 | inline size_t CustomOpApi::GetDimensionsCount(_In_ const OrtTensorTypeAndShapeInfo* info) { |
| 904 | size_t out; |
| 905 | ThrowOnError(api_.GetDimensionsCount(info, &out)); |
| 906 | return out; |
| 907 | } |
| 908 | |
| 909 | inline void CustomOpApi::GetDimensions(_In_ const OrtTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values, size_t dim_values_length) { |
| 910 | ThrowOnError(api_.GetDimensions(info, dim_values, dim_values_length)); |
| 911 | } |
| 912 | |
| 913 | inline void CustomOpApi::SetDimensions(OrtTensorTypeAndShapeInfo* info, _In_ const int64_t* dim_values, size_t dim_count) { |
| 914 | ThrowOnError(api_.SetDimensions(info, dim_values, dim_count)); |
| 915 | } |
| 916 | |
| 917 | template <typename T> |
| 918 | inline T* CustomOpApi::GetTensorMutableData(_Inout_ OrtValue* value) { |
| 919 | T* data; |
| 920 | ThrowOnError(api_.GetTensorMutableData(value, reinterpret_cast<void**>(&data))); |
| 921 | return data; |
| 922 | } |
| 923 | |
| 924 | template <typename T> |
| 925 | inline const T* CustomOpApi::GetTensorData(_Inout_ const OrtValue* value) { |
| 926 | return GetTensorMutableData<T>(const_cast<OrtValue*>(value)); |
| 927 | } |
| 928 | |
| 929 | inline std::vector<int64_t> CustomOpApi::GetTensorShape(const OrtTensorTypeAndShapeInfo* info) { |
| 930 | std::vector<int64_t> output(GetDimensionsCount(info)); |
| 931 | GetDimensions(info, output.data(), output.size()); |
| 932 | return output; |
| 933 | } |
| 934 | |
| 935 | inline void CustomOpApi::ReleaseTensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo* input) { |
| 936 | api_.ReleaseTensorTypeAndShapeInfo(input); |
| 937 | } |
| 938 | |
| 939 | inline size_t CustomOpApi::KernelContext_GetInputCount(const OrtKernelContext* context) { |
| 940 | size_t out; |
| 941 | ThrowOnError(api_.KernelContext_GetInputCount(context, &out)); |
| 942 | return out; |
| 943 | } |
| 944 | |
| 945 | inline const OrtValue* CustomOpApi::KernelContext_GetInput(const OrtKernelContext* context, _In_ size_t index) { |
| 946 | const OrtValue* out; |
| 947 | ThrowOnError(api_.KernelContext_GetInput(context, index, &out)); |
| 948 | return out; |
| 949 | } |
| 950 | |
| 951 | inline size_t CustomOpApi::KernelContext_GetOutputCount(const OrtKernelContext* context) { |
| 952 | size_t out; |
| 953 | ThrowOnError(api_.KernelContext_GetOutputCount(context, &out)); |
| 954 | return out; |
| 955 | } |
| 956 | |
| 957 | inline OrtValue* CustomOpApi::KernelContext_GetOutput(OrtKernelContext* context, _In_ size_t index, |
| 958 | _In_ const int64_t* dim_values, size_t dim_count) { |
| 959 | OrtValue* out; |
| 960 | ThrowOnError(api_.KernelContext_GetOutput(context, index, dim_values, dim_count, &out)); |
| 961 | return out; |
| 962 | } |
| 963 | |
| 964 | inline SessionOptions& SessionOptions::DisablePerSessionThreads() { |
| 965 | ThrowOnError(GetApi().DisablePerSessionThreads(p_)); |
| 966 | return *this; |
| 967 | } |
| 968 | |
| 969 | inline std::vector<std::string> GetAvailableProviders() { |
| 970 | int len; |
| 971 | char** providers; |
| 972 | const OrtApi& api = GetApi(); |
| 973 | ThrowOnError(api.GetAvailableProviders(&providers, &len)); |
| 974 | std::vector<std::string> available_providers(providers, providers + len); |
| 975 | ThrowOnError(api.ReleaseAvailableProviders(providers, len)); |
| 976 | return available_providers; |
| 977 | } |
| 978 | |
| 979 | SessionOptions& AddInitializer(const char* name, const OrtValue* ort_val); |
| 980 | |
| 981 | } // namespace Ort |
| 982 | |