GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/onnxruntime-extensions

Public

mirrored from https://github.com/microsoft/onnxruntime-extensionsAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
rel-0.5

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

onnxruntime-extensions/includes/onnxruntime

includes/onnxruntime/onnxruntime_c_api.h

1188lines · modecode

RawDownload
Latest commit unavailable.
1// Copyright (c) Microsoft Corporation. All rights reserved.
2// Licensed under the MIT License.
3
4#pragma once
5#include <stdlib.h>
6#include <stdint.h>
7#include <string.h>
8
9// This value is used in structures passed to ORT so that a newer version of ORT will still work with them
10#define ORT_API_VERSION 6
11
12#ifdef __cplusplus
13extern "C" {
14#endif
15
16// SAL2 Definitions
17#ifndef _WIN32
18#define _In_
19#define _In_z_
20#define _In_opt_
21#define _In_opt_z_
22#define _Out_
23#define _Outptr_
24#define _Out_opt_
25#define _Inout_
26#define _Inout_opt_
27#define _Frees_ptr_opt_
28#define _Ret_maybenull_
29#define _Ret_notnull_
30#define _Check_return_
31#define _Outptr_result_maybenull_
32#define _In_reads_(X)
33#define _Inout_updates_all_(X)
34#define _Out_writes_bytes_all_(X)
35#define _Out_writes_all_(X)
36#define _Success_(X)
37#define _Outptr_result_buffer_maybenull_(X)
38#define ORT_ALL_ARGS_NONNULL __attribute__((nonnull))
39#else
40#if defined(__MINGW32__)
41#define _Frees_ptr_opt_
42#endif
43#include <specstrings.h>
44#define ORT_ALL_ARGS_NONNULL
45#endif
46
47#ifdef _WIN32
48// Define ORT_DLL_IMPORT if your program is dynamically linked to Ort.
49// dllexport is not used, we use a .def file.
50#ifdef ORT_DLL_IMPORT
51#define ORT_EXPORT __declspec(dllimport)
52#else
53#define ORT_EXPORT
54#endif
55#if defined(__MINGW32__)
56#define ORT_API_CALL __stdcall
57#else
58#define ORT_API_CALL _stdcall
59#endif
60#define ORT_MUST_USE_RESULT
61#define ORTCHAR_T wchar_t
62#else
63#define ORT_EXPORT
64#define ORT_API_CALL
65#define ORT_MUST_USE_RESULT __attribute__((warn_unused_result))
66#define ORTCHAR_T char
67#endif
68
69#ifndef ORT_TSTR
70#ifdef _WIN32
71#define ORT_TSTR(X) L##X
72#else
73#define ORT_TSTR(X) X
74#endif
75#endif
76
77// Any pointer marked with _In_ or _Out_, cannot be NULL.
78
79// Windows users should use unicode paths when possible to bypass the MAX_PATH limitation
80// Every pointer marked with _In_ or _Out_, cannot be NULL. Caller should ensure that.
81// for ReleaseXXX(...) functions, they can accept NULL pointer.
82
83#ifdef __cplusplus
84// For any compiler with C++11 support, MSVC 2015 and greater, or Clang version supporting noexcept.
85// Such complex condition is needed because compilers set __cplusplus value differently.
86#ifndef __has_feature
87#define __has_feature(x) 0
88#endif
89#if ((__cplusplus >= 201103L) || (_MSC_VER >= 1900) || (defined(__has_feature) && __has_feature(cxx_noexcept)))
90#define NO_EXCEPTION noexcept
91#else
92#define NO_EXCEPTION throw()
93#endif
94#else
95#define NO_EXCEPTION
96#endif
97
98// Copied from TensorProto::DataType
99// Currently, Ort doesn't support complex64, complex128
100typedef enum ONNXTensorElementDataType {
101 ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED,
102 ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, // maps to c type float
103 ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8, // maps to c type uint8_t
104 ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8, // maps to c type int8_t
105 ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16, // maps to c type uint16_t
106 ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16, // maps to c type int16_t
107 ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32, // maps to c type int32_t
108 ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64, // maps to c type int64_t
109 ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING, // maps to c++ type std::string
110 ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL,
111 ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16,
112 ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE, // maps to c type double
113 ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32, // maps to c type uint32_t
114 ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64, // maps to c type uint64_t
115 ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64, // complex with float32 real and imaginary components
116 ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128, // complex with float64 real and imaginary components
117 ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16 // Non-IEEE floating-point format based on IEEE754 single-precision
118} ONNXTensorElementDataType;
119
120// Synced with onnx TypeProto oneof
121typedef enum ONNXType {
122 ONNX_TYPE_UNKNOWN,
123 ONNX_TYPE_TENSOR,
124 ONNX_TYPE_SEQUENCE,
125 ONNX_TYPE_MAP,
126 ONNX_TYPE_OPAQUE,
127 ONNX_TYPE_SPARSETENSOR,
128} ONNXType;
129
130typedef enum OrtLoggingLevel {
131 ORT_LOGGING_LEVEL_VERBOSE,
132 ORT_LOGGING_LEVEL_INFO,
133 ORT_LOGGING_LEVEL_WARNING,
134 ORT_LOGGING_LEVEL_ERROR,
135 ORT_LOGGING_LEVEL_FATAL,
136} OrtLoggingLevel;
137
138typedef enum OrtErrorCode {
139 ORT_OK,
140 ORT_FAIL,
141 ORT_INVALID_ARGUMENT,
142 ORT_NO_SUCHFILE,
143 ORT_NO_MODEL,
144 ORT_ENGINE_ERROR,
145 ORT_RUNTIME_EXCEPTION,
146 ORT_INVALID_PROTOBUF,
147 ORT_MODEL_LOADED,
148 ORT_NOT_IMPLEMENTED,
149 ORT_INVALID_GRAPH,
150 ORT_EP_FAIL,
151} OrtErrorCode;
152
153#define ORT_RUNTIME_CLASS(X) \
154 struct Ort##X; \
155 typedef struct Ort##X Ort##X;
156
157// The actual types defined have an Ort prefix
158ORT_RUNTIME_CLASS(Env);
159ORT_RUNTIME_CLASS(Status); // nullptr for Status* indicates success
160ORT_RUNTIME_CLASS(MemoryInfo);
161ORT_RUNTIME_CLASS(IoBinding);
162ORT_RUNTIME_CLASS(Session); //Don't call OrtReleaseSession from Dllmain (because session owns a thread pool)
163ORT_RUNTIME_CLASS(Value);
164ORT_RUNTIME_CLASS(RunOptions);
165ORT_RUNTIME_CLASS(TypeInfo);
166ORT_RUNTIME_CLASS(TensorTypeAndShapeInfo);
167ORT_RUNTIME_CLASS(SessionOptions);
168ORT_RUNTIME_CLASS(CustomOpDomain);
169ORT_RUNTIME_CLASS(MapTypeInfo);
170ORT_RUNTIME_CLASS(SequenceTypeInfo);
171ORT_RUNTIME_CLASS(ModelMetadata);
172ORT_RUNTIME_CLASS(ThreadPoolParams);
173ORT_RUNTIME_CLASS(ThreadingOptions);
174ORT_RUNTIME_CLASS(ArenaCfg);
175
176#ifdef _WIN32
177typedef _Return_type_success_(return == 0) OrtStatus* OrtStatusPtr;
178#else
179typedef OrtStatus* OrtStatusPtr;
180#endif
181
182// __VA_ARGS__ on Windows and Linux are different
183#define ORT_API(RETURN_TYPE, NAME, ...) ORT_EXPORT RETURN_TYPE ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION
184
185#define ORT_API_STATUS(NAME, ...) \
186 ORT_EXPORT _Check_return_ _Ret_maybenull_ OrtStatusPtr ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION ORT_MUST_USE_RESULT
187
188// XXX: Unfortunately, SAL annotations are known to not work with function pointers
189#define ORT_API2_STATUS(NAME, ...) \
190 _Check_return_ _Ret_maybenull_ OrtStatusPtr(ORT_API_CALL* NAME)(__VA_ARGS__) NO_EXCEPTION ORT_MUST_USE_RESULT
191
192// Used in *.cc files. Almost as same as ORT_API_STATUS, except without ORT_MUST_USE_RESULT and ORT_EXPORT
193#define ORT_API_STATUS_IMPL(NAME, ...) \
194 _Check_return_ _Ret_maybenull_ OrtStatusPtr ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION
195
196#define ORT_CLASS_RELEASE(X) void(ORT_API_CALL * Release##X)(_Frees_ptr_opt_ Ort##X * input)
197
198// When passing in an allocator to any ORT function, be sure that the allocator object
199// is not destroyed until the last allocated object using it is freed.
200typedef struct OrtAllocator {
201 uint32_t version; // Initialize to ORT_API_VERSION
202 void*(ORT_API_CALL* Alloc)(struct OrtAllocator* this_, size_t size);
203 void(ORT_API_CALL* Free)(struct OrtAllocator* this_, void* p);
204 const struct OrtMemoryInfo*(ORT_API_CALL* Info)(const struct OrtAllocator* this_);
205} OrtAllocator;
206
207typedef void(ORT_API_CALL* OrtLoggingFunction)(
208 void* param, OrtLoggingLevel severity, const char* category, const char* logid, const char* code_location,
209 const char* message);
210
211// Set Graph optimization level.
212// Refer https://github.com/microsoft/onnxruntime/blob/master/docs/ONNX_Runtime_Graph_Optimizations.md
213// for in-depth undersrtanding of Graph Optimizations in ORT
214typedef enum GraphOptimizationLevel {
215 ORT_DISABLE_ALL = 0,
216 ORT_ENABLE_BASIC = 1,
217 ORT_ENABLE_EXTENDED = 2,
218 ORT_ENABLE_ALL = 99
219} GraphOptimizationLevel;
220
221typedef enum ExecutionMode {
222 ORT_SEQUENTIAL = 0,
223 ORT_PARALLEL = 1,
224} ExecutionMode;
225
226// Set the language projection, default is C, which means it will classify the language not in the list to C also.
227typedef enum OrtLanguageProjection {
228 ORT_PROJECTION_C = 0, // default
229 ORT_PROJECTION_CPLUSPLUS = 1,
230 ORT_PROJECTION_CSHARP = 2,
231 ORT_PROJECTION_PYTHON = 3,
232 ORT_PROJECTION_JAVA = 4,
233 ORT_PROJECTION_WINML = 5,
234 ORT_PROJECTION_NODEJS = 6,
235} OrtLanguageProjection;
236
237struct OrtKernelInfo;
238typedef struct OrtKernelInfo OrtKernelInfo;
239struct OrtKernelContext;
240typedef struct OrtKernelContext OrtKernelContext;
241struct OrtCustomOp;
242typedef struct OrtCustomOp OrtCustomOp;
243
244typedef enum OrtAllocatorType {
245 Invalid = -1,
246 OrtDeviceAllocator = 0,
247 OrtArenaAllocator = 1
248} OrtAllocatorType;
249
250/**
251 * memory types for allocator, exec provider specific types should be extended in each provider
252 * Whenever this struct is updated, please also update the MakeKey function in onnxruntime/core/framework/execution_provider.cc
253*/
254typedef enum OrtMemType {
255 OrtMemTypeCPUInput = -2, // Any CPU memory used by non-CPU execution provider
256 OrtMemTypeCPUOutput = -1, // CPU accessible memory outputted by non-CPU execution provider, i.e. CUDA_PINNED
257 OrtMemTypeCPU = OrtMemTypeCPUOutput, // temporary CPU accessible memory allocated by non-CPU execution provider, i.e. CUDA_PINNED
258 OrtMemTypeDefault = 0, // the default allocator for execution provider
259} OrtMemType;
260
261typedef enum OrtCudnnConvAlgoSearch {
262 EXHAUSTIVE, // expensive exhaustive benchmarking using cudnnFindConvolutionForwardAlgorithmEx
263 HEURISTIC, // lightweight heuristic based search using cudnnGetConvolutionForwardAlgorithm_v7
264 DEFAULT, // default algorithm using CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM
265} OrtCudnnConvAlgoSearch;
266
267/// <summary>
268/// Options for the CUDA provider that are passed to SessionOptionsAppendExecutionProvider_CUDA
269/// </summary>
270typedef struct OrtCUDAProviderOptions {
271 int device_id; // cuda device with id=0 as default device.
272 OrtCudnnConvAlgoSearch cudnn_conv_algo_search; // cudnn conv algo search option
273 size_t cuda_mem_limit; // default cuda memory limitation to maximum finite value of size_t.
274 int arena_extend_strategy; // default area extend strategy to KNextPowerOfTwo.
275 int do_copy_in_default_stream;
276} OrtCUDAProviderOptions;
277
278/// <summary>
279/// Options for the OpenVINO provider that are passed to SessionOptionsAppendExecutionProvider_OpenVINO
280/// </summary>
281typedef struct OrtOpenVINOProviderOptions {
282#ifdef __cplusplus
283 OrtOpenVINOProviderOptions() : device_type{}, enable_vpu_fast_compile{}, device_id{}, num_of_threads{} {}
284#endif
285 const char* device_type; // CPU_FP32, GPU_FP32, GPU_FP16, MYRIAD_FP16, VAD-M_FP16 or VAD-F_FP32
286 unsigned char enable_vpu_fast_compile; // 0 = false, nonzero = true
287 const char* device_id;
288 size_t num_of_threads; // 0 uses default number of threads
289} OrtOpenVINOProviderOptions;
290
291struct OrtApi;
292typedef struct OrtApi OrtApi;
293
294struct OrtApiBase {
295 const OrtApi*(ORT_API_CALL* GetApi)(uint32_t version)NO_EXCEPTION; // Pass in ORT_API_VERSION
296 // nullptr will be returned if the version is unsupported, for example when using a runtime older than this header file
297
298 const char*(ORT_API_CALL* GetVersionString)() NO_EXCEPTION;
299};
300typedef struct OrtApiBase OrtApiBase;
301
302ORT_EXPORT const OrtApiBase* ORT_API_CALL OrtGetApiBase(void) NO_EXCEPTION;
303
304struct OrtApi {
305 /**
306* \param msg A null-terminated string. Its content will be copied into the newly created OrtStatus
307*/
308 OrtStatus*(ORT_API_CALL* CreateStatus)(OrtErrorCode code, _In_ const char* msg)NO_EXCEPTION ORT_ALL_ARGS_NONNULL;
309
310 OrtErrorCode(ORT_API_CALL* GetErrorCode)(_In_ const OrtStatus* status) NO_EXCEPTION ORT_ALL_ARGS_NONNULL;
311
312 /**
313 * \param status must not be NULL
314 * \return The error message inside the `status`. Do not free the returned value.
315 */
316 const char*(ORT_API_CALL* GetErrorMessage)(_In_ const OrtStatus* status)NO_EXCEPTION ORT_ALL_ARGS_NONNULL;
317
318 /**
319 * \param out Should be freed by `OrtReleaseEnv` after use
320 */
321 ORT_API2_STATUS(CreateEnv, OrtLoggingLevel logging_level, _In_ const char* logid, _Outptr_ OrtEnv** out);
322
323 /**
324 * \param out Should be freed by `OrtReleaseEnv` after use
325 */
326 ORT_API2_STATUS(CreateEnvWithCustomLogger, OrtLoggingFunction logging_function, _In_opt_ void* logger_param,
327 OrtLoggingLevel logging_level, _In_ const char* logid, _Outptr_ OrtEnv** out);
328
329 // Platform telemetry events are on by default since they are lightweight. You can manually turn them off.
330 ORT_API2_STATUS(EnableTelemetryEvents, _In_ const OrtEnv* env);
331 ORT_API2_STATUS(DisableTelemetryEvents, _In_ const OrtEnv* env);
332
333 // TODO: document the path separator convention? '/' vs '\'
334 // TODO: should specify the access characteristics of model_path. Is this read only during the
335 // execution of CreateSession, or does the OrtSession retain a handle to the file/directory
336 // and continue to access throughout the OrtSession lifetime?
337 // What sort of access is needed to model_path : read or read/write?
338 ORT_API2_STATUS(CreateSession, _In_ const OrtEnv* env, _In_ const ORTCHAR_T* model_path,
339 _In_ const OrtSessionOptions* options, _Outptr_ OrtSession** out);
340
341 ORT_API2_STATUS(CreateSessionFromArray, _In_ const OrtEnv* env, _In_ const void* model_data, size_t model_data_length,
342 _In_ const OrtSessionOptions* options, _Outptr_ OrtSession** out);
343
344 ORT_API2_STATUS(Run, _Inout_ OrtSession* sess, _In_opt_ const OrtRunOptions* run_options,
345 _In_reads_(input_len) const char* const* input_names,
346 _In_reads_(input_len) const OrtValue* const* input, size_t input_len,
347 _In_reads_(output_names_len) const char* const* output_names1, size_t output_names_len,
348 _Inout_updates_all_(output_names_len) OrtValue** output);
349
350 /**
351 * \return A pointer of the newly created object. The pointer should be freed by OrtReleaseSessionOptions after use
352 */
353 ORT_API2_STATUS(CreateSessionOptions, _Outptr_ OrtSessionOptions** options);
354
355 // Set filepath to save optimized model after graph level transformations.
356 ORT_API2_STATUS(SetOptimizedModelFilePath, _Inout_ OrtSessionOptions* options,
357 _In_ const ORTCHAR_T* optimized_model_filepath);
358
359 // create a copy of an existing OrtSessionOptions
360 ORT_API2_STATUS(CloneSessionOptions, _In_ const OrtSessionOptions* in_options,
361 _Outptr_ OrtSessionOptions** out_options);
362
363 // Controls whether you want to execute operators in your graph sequentially or in parallel. Usually when the model
364 // has many branches, setting this option to ExecutionMode.ORT_PARALLEL will give you better performance.
365 // See [docs/ONNX_Runtime_Perf_Tuning.md] for more details.
366 ORT_API2_STATUS(SetSessionExecutionMode, _Inout_ OrtSessionOptions* options, ExecutionMode execution_mode);
367
368 // Enable profiling for this session.
369 ORT_API2_STATUS(EnableProfiling, _Inout_ OrtSessionOptions* options, _In_ const ORTCHAR_T* profile_file_prefix);
370 ORT_API2_STATUS(DisableProfiling, _Inout_ OrtSessionOptions* options);
371
372 // Enable the memory pattern optimization.
373 // The idea is if the input shapes are the same, we could trace the internal memory allocation
374 // and generate a memory pattern for future request. So next time we could just do one allocation
375 // with a big chunk for all the internal memory allocation.
376 // Note: memory pattern optimization is only available when SequentialExecution enabled.
377 ORT_API2_STATUS(EnableMemPattern, _Inout_ OrtSessionOptions* options);
378 ORT_API2_STATUS(DisableMemPattern, _Inout_ OrtSessionOptions* options);
379
380 // Enable the memory arena on CPU
381 // Arena may pre-allocate memory for future usage.
382 // set this option to false if you don't want it.
383 ORT_API2_STATUS(EnableCpuMemArena, _Inout_ OrtSessionOptions* options);
384 ORT_API2_STATUS(DisableCpuMemArena, _Inout_ OrtSessionOptions* options);
385
386 // < logger id to use for session output
387 ORT_API2_STATUS(SetSessionLogId, _Inout_ OrtSessionOptions* options, const char* logid);
388
389 // < applies to session load, initialization, etc
390 ORT_API2_STATUS(SetSessionLogVerbosityLevel, _Inout_ OrtSessionOptions* options, int session_log_verbosity_level);
391 ORT_API2_STATUS(SetSessionLogSeverityLevel, _Inout_ OrtSessionOptions* options, int session_log_severity_level);
392
393 ORT_API2_STATUS(SetSessionGraphOptimizationLevel, _Inout_ OrtSessionOptions* options,
394 GraphOptimizationLevel graph_optimization_level);
395
396 // Sets the number of threads used to parallelize the execution within nodes
397 // A value of 0 means ORT will pick a default
398 // Note: If you've built ORT with OpenMP, this API has no effect on the number of threads used. In this case
399 // use the OpenMP env variables to configure the number of intra op num threads.
400 ORT_API2_STATUS(SetIntraOpNumThreads, _Inout_ OrtSessionOptions* options, int intra_op_num_threads);
401
402 // Sets the number of threads used to parallelize the execution of the graph (across nodes)
403 // If sequential execution is enabled this value is ignored
404 // A value of 0 means ORT will pick a default
405 ORT_API2_STATUS(SetInterOpNumThreads, _Inout_ OrtSessionOptions* options, int inter_op_num_threads);
406
407 /*
408 Create a custom op domain. After all sessions using it are released, call OrtReleaseCustomOpDomain
409 */
410 ORT_API2_STATUS(CreateCustomOpDomain, _In_ const char* domain, _Outptr_ OrtCustomOpDomain** out);
411
412 /*
413 * Add custom ops to the OrtCustomOpDomain
414 * Note: The OrtCustomOp* pointer must remain valid until the OrtCustomOpDomain using it is released
415 */
416 ORT_API2_STATUS(CustomOpDomain_Add, _Inout_ OrtCustomOpDomain* custom_op_domain, _In_ const OrtCustomOp* op);
417
418 /*
419 * Add a custom op domain to the OrtSessionOptions
420 * Note: The OrtCustomOpDomain* must not be deleted until the sessions using it are released
421 */
422 ORT_API2_STATUS(AddCustomOpDomain, _Inout_ OrtSessionOptions* options, _In_ OrtCustomOpDomain* custom_op_domain);
423
424 /*
425 * Loads a DLL named 'library_path' and looks for this entry point:
426 * OrtStatus* RegisterCustomOps(OrtSessionOptions * options, const OrtApiBase* api);
427 * It then passes in the provided session options to this function along with the api base.
428 * The handle to the loaded library is returned in library_handle. It can be freed by the caller after all sessions using the passed in
429 * session options are destroyed, or if an error occurs and it is non null.
430 */
431 ORT_API2_STATUS(RegisterCustomOpsLibrary, _Inout_ OrtSessionOptions* options, _In_ const char* library_path,
432 void** library_handle);
433
434 /**
435 * To use additional providers, you must build ORT with the extra providers enabled. Then call one of these
436 * functions to enable them in the session:
437 * OrtSessionOptionsAppendExecutionProvider_CPU
438 * OrtSessionOptionsAppendExecutionProvider_CUDA
439 * OrtSessionOptionsAppendExecutionProvider_<remaining providers...>
440 * The order they are called indicates the preference order as well. In other words call this method
441 * on your most preferred execution provider first followed by the less preferred ones.
442 * If none are called Ort will use its internal CPU execution provider.
443 */
444
445 ORT_API2_STATUS(SessionGetInputCount, _In_ const OrtSession* sess, _Out_ size_t* out);
446 ORT_API2_STATUS(SessionGetOutputCount, _In_ const OrtSession* sess, _Out_ size_t* out);
447 ORT_API2_STATUS(SessionGetOverridableInitializerCount, _In_ const OrtSession* sess, _Out_ size_t* out);
448
449 /**
450 * \param out should be freed by OrtReleaseTypeInfo after use
451 */
452 ORT_API2_STATUS(SessionGetInputTypeInfo, _In_ const OrtSession* sess, size_t index, _Outptr_ OrtTypeInfo** type_info);
453
454 /**
455 * \param out should be freed by OrtReleaseTypeInfo after use
456 */
457 ORT_API2_STATUS(SessionGetOutputTypeInfo, _In_ const OrtSession* sess, size_t index,
458 _Outptr_ OrtTypeInfo** type_info);
459
460 /**
461 * \param out should be freed by OrtReleaseTypeInfo after use
462 */
463 ORT_API2_STATUS(SessionGetOverridableInitializerTypeInfo, _In_ const OrtSession* sess, size_t index,
464 _Outptr_ OrtTypeInfo** type_info);
465
466 /**
467 * \param value is set to a null terminated string allocated using 'allocator'. The caller is responsible for freeing it.
468 */
469 ORT_API2_STATUS(SessionGetInputName, _In_ const OrtSession* sess, size_t index, _Inout_ OrtAllocator* allocator,
470 _Outptr_ char** value);
471 ORT_API2_STATUS(SessionGetOutputName, _In_ const OrtSession* sess, size_t index, _Inout_ OrtAllocator* allocator,
472 _Outptr_ char** value);
473 ORT_API2_STATUS(SessionGetOverridableInitializerName, _In_ const OrtSession* sess, size_t index,
474 _Inout_ OrtAllocator* allocator, _Outptr_ char** value);
475
476 /**
477 * \return A pointer to the newly created object. The pointer should be freed by OrtReleaseRunOptions after use
478 */
479 ORT_API2_STATUS(CreateRunOptions, _Outptr_ OrtRunOptions** out);
480
481 ORT_API2_STATUS(RunOptionsSetRunLogVerbosityLevel, _Inout_ OrtRunOptions* options, int value);
482 ORT_API2_STATUS(RunOptionsSetRunLogSeverityLevel, _Inout_ OrtRunOptions* options, int value);
483 ORT_API2_STATUS(RunOptionsSetRunTag, _Inout_ OrtRunOptions*, _In_ const char* run_tag);
484
485 ORT_API2_STATUS(RunOptionsGetRunLogVerbosityLevel, _In_ const OrtRunOptions* options, _Out_ int* out);
486 ORT_API2_STATUS(RunOptionsGetRunLogSeverityLevel, _In_ const OrtRunOptions* options, _Out_ int* out);
487 ORT_API2_STATUS(RunOptionsGetRunTag, _In_ const OrtRunOptions*, _Out_ const char** out);
488
489 // Set a flag so that ALL incomplete OrtRun calls that are using this instance of OrtRunOptions
490 // will exit as soon as possible.
491 ORT_API2_STATUS(RunOptionsSetTerminate, _Inout_ OrtRunOptions* options);
492 // Unset the terminate flag to enable this OrtRunOptions instance being used in new OrtRun calls.
493 ORT_API2_STATUS(RunOptionsUnsetTerminate, _Inout_ OrtRunOptions* options);
494
495 /**
496 * Create a tensor from an allocator. OrtReleaseValue will also release the buffer inside the output value
497 * \param out Should be freed by calling OrtReleaseValue
498 * \param type must be one of TENSOR_ELEMENT_DATA_TYPE_xxxx
499 */
500 ORT_API2_STATUS(CreateTensorAsOrtValue, _Inout_ OrtAllocator* allocator, _In_ const int64_t* shape, size_t shape_len,
501 ONNXTensorElementDataType type, _Outptr_ OrtValue** out);
502
503 /**
504 * Create a tensor with user's buffer. You can fill the buffer either before calling this function or after.
505 * p_data is owned by caller. OrtReleaseValue won't release p_data.
506 * \param out Should be freed by calling OrtReleaseValue
507 */
508 ORT_API2_STATUS(CreateTensorWithDataAsOrtValue, _In_ const OrtMemoryInfo* info, _Inout_ void* p_data,
509 size_t p_data_len, _In_ const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type,
510 _Outptr_ OrtValue** out);
511
512 /**
513 * \Sets *out to 1 iff an OrtValue is a tensor, 0 otherwise
514 */
515 ORT_API2_STATUS(IsTensor, _In_ const OrtValue* value, _Out_ int* out);
516
517 // This function doesn't work with string tensor
518 // this is a no-copy method whose pointer is only valid until the backing OrtValue is free'd.
519 ORT_API2_STATUS(GetTensorMutableData, _Inout_ OrtValue* value, _Outptr_ void** out);
520
521 /**
522 * \param value A tensor created from OrtCreateTensor... function.
523 * \param s each A string array. Each string in this array must be null terminated.
524 * \param s_len length of s
525 */
526 ORT_API2_STATUS(FillStringTensor, _Inout_ OrtValue* value, _In_ const char* const* s, size_t s_len);
527
528 /**
529 * \param value A tensor created from OrtCreateTensor... function.
530 * \param len total data length, not including the trailing '\0' chars.
531 */
532 ORT_API2_STATUS(GetStringTensorDataLength, _In_ const OrtValue* value, _Out_ size_t* len);
533
534 /**
535 * \param s string contents. Each string is NOT null-terminated.
536 * \param value A tensor created from OrtCreateTensor... function.
537 * \param s_len total data length, get it from OrtGetStringTensorDataLength
538 */
539 ORT_API2_STATUS(GetStringTensorContent, _In_ const OrtValue* value, _Out_writes_bytes_all_(s_len) void* s,
540 size_t s_len, _Out_writes_all_(offsets_len) size_t* offsets, size_t offsets_len);
541
542 /**
543 * Don't free the 'out' value
544 */
545 ORT_API2_STATUS(CastTypeInfoToTensorInfo, _In_ const OrtTypeInfo*,
546 _Outptr_result_maybenull_ const OrtTensorTypeAndShapeInfo** out);
547
548 /**
549 * Return OnnxType from OrtTypeInfo
550 */
551 ORT_API2_STATUS(GetOnnxTypeFromTypeInfo, _In_ const OrtTypeInfo*, _Out_ enum ONNXType* out);
552
553 /**
554 * The 'out' value should be released by calling OrtReleaseTensorTypeAndShapeInfo
555 */
556 ORT_API2_STATUS(CreateTensorTypeAndShapeInfo, _Outptr_ OrtTensorTypeAndShapeInfo** out);
557
558 ORT_API2_STATUS(SetTensorElementType, _Inout_ OrtTensorTypeAndShapeInfo*, enum ONNXTensorElementDataType type);
559
560 /**
561 * \param info Created from CreateTensorTypeAndShapeInfo() function
562 * \param dim_values An array with length of `dim_count`. Its elements can contain negative values.
563 * \param dim_count length of dim_values
564 */
565 ORT_API2_STATUS(SetDimensions, OrtTensorTypeAndShapeInfo* info, _In_ const int64_t* dim_values, size_t dim_count);
566
567 ORT_API2_STATUS(GetTensorElementType, _In_ const OrtTensorTypeAndShapeInfo*,
568 _Out_ enum ONNXTensorElementDataType* out);
569 ORT_API2_STATUS(GetDimensionsCount, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ size_t* out);
570 ORT_API2_STATUS(GetDimensions, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values,
571 size_t dim_values_length);
572 ORT_API2_STATUS(GetSymbolicDimensions, _In_ const OrtTensorTypeAndShapeInfo* info,
573 _Out_writes_all_(dim_params_length) const char* dim_params[], size_t dim_params_length);
574
575 /**
576 * Return the number of elements specified by the tensor shape.
577 * Return a negative value if unknown (i.e., any dimension is negative.)
578 * e.g.
579 * [] -> 1
580 * [1,3,4] -> 12
581 * [2,0,4] -> 0
582 * [-1,3,4] -> -1
583 */
584 ORT_API2_STATUS(GetTensorShapeElementCount, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ size_t* out);
585
586 /**
587 * \param out Should be freed by OrtReleaseTensorTypeAndShapeInfo after use
588 */
589 ORT_API2_STATUS(GetTensorTypeAndShape, _In_ const OrtValue* value, _Outptr_ OrtTensorTypeAndShapeInfo** out);
590
591 /**
592 * Get the type information of an OrtValue
593 * \param value
594 * \param out The returned value should be freed by OrtReleaseTypeInfo after use
595 */
596 ORT_API2_STATUS(GetTypeInfo, _In_ const OrtValue* value, _Outptr_result_maybenull_ OrtTypeInfo** out);
597
598 ORT_API2_STATUS(GetValueType, _In_ const OrtValue* value, _Out_ enum ONNXType* out);
599
600 ORT_API2_STATUS(CreateMemoryInfo, _In_ const char* name1, enum OrtAllocatorType type, int id1,
601 enum OrtMemType mem_type1, _Outptr_ OrtMemoryInfo** out);
602
603 /**
604 * Convenience function for special case of CreateMemoryInfo, for the CPU allocator. Uses name = "Cpu" and id = 0.
605 */
606 ORT_API2_STATUS(CreateCpuMemoryInfo, enum OrtAllocatorType type, enum OrtMemType mem_type1,
607 _Outptr_ OrtMemoryInfo** out);
608
609 /**
610 * Test if two memory info are equal
611 * \Sets 'out' to 0 if equal, -1 if not equal
612 */
613 ORT_API2_STATUS(CompareMemoryInfo, _In_ const OrtMemoryInfo* info1, _In_ const OrtMemoryInfo* info2, _Out_ int* out);
614
615 /**
616 * Do not free the returned value
617 */
618 ORT_API2_STATUS(MemoryInfoGetName, _In_ const OrtMemoryInfo* ptr, _Out_ const char** out);
619 ORT_API2_STATUS(MemoryInfoGetId, _In_ const OrtMemoryInfo* ptr, _Out_ int* out);
620 ORT_API2_STATUS(MemoryInfoGetMemType, _In_ const OrtMemoryInfo* ptr, _Out_ OrtMemType* out);
621 ORT_API2_STATUS(MemoryInfoGetType, _In_ const OrtMemoryInfo* ptr, _Out_ OrtAllocatorType* out);
622
623 ORT_API2_STATUS(AllocatorAlloc, _Inout_ OrtAllocator* ptr, size_t size, _Outptr_ void** out);
624 ORT_API2_STATUS(AllocatorFree, _Inout_ OrtAllocator* ptr, void* p);
625 ORT_API2_STATUS(AllocatorGetInfo, _In_ const OrtAllocator* ptr, _Outptr_ const struct OrtMemoryInfo** out);
626
627 // The returned pointer doesn't have to be freed.
628 // Always returns the same instance on every invocation.
629 // Please note that this is a non-arena based allocator.
630 ORT_API2_STATUS(GetAllocatorWithDefaultOptions, _Outptr_ OrtAllocator** out);
631
632 // Override symbolic dimensions (by specific denotation strings) with actual values if known at session initialization time to enable
633 // optimizations that can take advantage of fixed values (such as memory planning, etc)
634 ORT_API2_STATUS(AddFreeDimensionOverride, _Inout_ OrtSessionOptions* options, _In_ const char* dim_denotation,
635 _In_ int64_t dim_value);
636
637 /**
638 * APIs to support non-tensor types - map and sequence.
639 * Currently only the following types are supported
640 * Note: the following types should be kept in sync with data_types.h
641 * Map types
642 * =========
643 * std::map<std::string, std::string>
644 * std::map<std::string, int64_t>
645 * std::map<std::string, float>
646 * std::map<std::string, double>
647 * std::map<int64_t, std::string>
648 * std::map<int64_t, int64_t>
649 * std::map<int64_t, float>
650 * std::map<int64_t, double>
651 *
652 * Sequence types
653 * ==============
654 * std::vector<std::string>
655 * std::vector<int64_t>
656 * std::vector<float>
657 * std::vector<double>
658 * std::vector<std::map<std::string, float>>
659 * std::vector<std::map<int64_t, float>
660 */
661
662 /**
663 * If input OrtValue represents a map, you need to retrieve the keys and values
664 * separately. Use index=0 to retrieve keys and index=1 to retrieve values.
665 * If input OrtValue represents a sequence, use index to retrieve the index'th element
666 * of the sequence.
667 */
668 ORT_API2_STATUS(GetValue, _In_ const OrtValue* value, int index, _Inout_ OrtAllocator* allocator,
669 _Outptr_ OrtValue** out);
670
671 /**
672 * Returns 2 for type map and N for sequence where N is the number of elements
673 * in the sequence.
674 */
675 ORT_API2_STATUS(GetValueCount, _In_ const OrtValue* value, _Out_ size_t* out);
676
677 /**
678 * To construct a map, use num_values = 2 and 'in' should be an arrary of 2 OrtValues
679 * representing keys and values.
680 * To construct a sequence, use num_values = N where N is the number of the elements in the
681 * sequence. 'in' should be an arrary of N OrtValues.
682 * \value_type should be either map or sequence.
683 */
684 ORT_API2_STATUS(CreateValue, _In_reads_(num_values) const OrtValue* const* in, size_t num_values,
685 enum ONNXType value_type, _Outptr_ OrtValue** out);
686
687 /**
688 * Construct OrtValue that contains a value of non-standard type created for
689 * experiments or while awaiting standardization. OrtValue in this case would contain
690 * an internal representation of the Opaque type. Opaque types are distinguished between
691 * each other by two strings 1) domain and 2) type name. The combination of the two
692 * must be unique, so the type representation is properly identified internally. The combination
693 * must be properly registered from within ORT at both compile/run time or by another API.
694 *
695 * To construct the OrtValue pass domain and type names, also a pointer to a data container
696 * the type of which must be know to both ORT and the client program. That data container may or may
697 * not match the internal representation of the Opaque type. The sizeof(data_container) is passed for
698 * verification purposes.
699 *
700 * \domain_name - domain name for the Opaque type, null terminated.
701 * \type_name - type name for the Opaque type, null terminated.
702 * \data_contianer - data to populate OrtValue
703 * \data_container_size - sizeof() of the data container. Must match the sizeof() of the expected
704 * data_container size internally.
705 */
706 ORT_API2_STATUS(CreateOpaqueValue, _In_z_ const char* domain_name, _In_z_ const char* type_name,
707 _In_ const void* data_container, size_t data_container_size, _Outptr_ OrtValue** out);
708
709 /**
710 * Fetch data from an OrtValue that contains a value of non-standard type created for
711 * experiments or while awaiting standardization.
712 * \domain_name - domain name for the Opaque type, null terminated.
713 * \type_name - type name for the Opaque type, null terminated.
714 * \data_contianer - data to populate OrtValue
715 * \data_container_size - sizeof() of the data container. Must match the sizeof() of the expected
716 * data_container size internally.
717 */
718
719 ORT_API2_STATUS(GetOpaqueValue, _In_ const char* domain_name, _In_ const char* type_name, _In_ const OrtValue* in,
720 _Out_ void* data_container, size_t data_container_size);
721
722 ORT_API2_STATUS(KernelInfoGetAttribute_float, _In_ const OrtKernelInfo* info, _In_ const char* name,
723 _Out_ float* out);
724 ORT_API2_STATUS(KernelInfoGetAttribute_int64, _In_ const OrtKernelInfo* info, _In_ const char* name,
725 _Out_ int64_t* out);
726 ORT_API2_STATUS(KernelInfoGetAttribute_string, _In_ const OrtKernelInfo* info, _In_ const char* name, _Out_ char* out,
727 _Inout_ size_t* size);
728
729 ORT_API2_STATUS(KernelContext_GetInputCount, _In_ const OrtKernelContext* context, _Out_ size_t* out);
730 ORT_API2_STATUS(KernelContext_GetOutputCount, _In_ const OrtKernelContext* context, _Out_ size_t* out);
731 ORT_API2_STATUS(KernelContext_GetInput, _In_ const OrtKernelContext* context, _In_ size_t index,
732 _Out_ const OrtValue** out);
733 ORT_API2_STATUS(KernelContext_GetOutput, _Inout_ OrtKernelContext* context, _In_ size_t index,
734 _In_ const int64_t* dim_values, size_t dim_count, _Outptr_ OrtValue** out);
735
736 ORT_CLASS_RELEASE(Env);
737 ORT_CLASS_RELEASE(Status); // nullptr for Status* indicates success
738 ORT_CLASS_RELEASE(MemoryInfo);
739 ORT_CLASS_RELEASE(Session); //Don't call OrtReleaseSession from Dllmain (because session owns a thread pool)
740 ORT_CLASS_RELEASE(Value);
741 ORT_CLASS_RELEASE(RunOptions);
742 ORT_CLASS_RELEASE(TypeInfo);
743 ORT_CLASS_RELEASE(TensorTypeAndShapeInfo);
744 ORT_CLASS_RELEASE(SessionOptions);
745 ORT_CLASS_RELEASE(CustomOpDomain);
746
747 // End of Version 1 - DO NOT MODIFY ABOVE (see above text for more information)
748
749 // Version 2 - In development, feel free to add/remove/rearrange here
750
751 /**
752 * GetDenotationFromTypeInfo
753 * This api augments OrtTypeInfo to return denotations on the type.
754 * This is used by WinML to determine if an input/output is intended to be an Image or a Tensor.
755 */
756 ORT_API2_STATUS(GetDenotationFromTypeInfo, _In_ const OrtTypeInfo*, _Out_ const char** const denotation,
757 _Out_ size_t* len);
758
759 // OrtTypeInfo Casting methods
760
761 /**
762 * CastTypeInfoToMapTypeInfo
763 * This api augments OrtTypeInfo to return an OrtMapTypeInfo when the type is a map.
764 * The OrtMapTypeInfo has additional information about the map's key type and value type.
765 * This is used by WinML to support model reflection APIs.
766 * This is used by WinML to support model reflection APIs.
767 *
768 * Don't free the 'out' value
769 */
770 ORT_API2_STATUS(CastTypeInfoToMapTypeInfo, _In_ const OrtTypeInfo* type_info,
771 _Outptr_result_maybenull_ const OrtMapTypeInfo** out);
772
773 /**
774 * CastTypeInfoToSequenceTypeInfo
775 * This api augments OrtTypeInfo to return an OrtSequenceTypeInfo when the type is a sequence.
776 * The OrtSequenceTypeInfo has additional information about the sequence's element type.
777 * This is used by WinML to support model reflection APIs.
778 *
779 * Don't free the 'out' value
780 */
781 ORT_API2_STATUS(CastTypeInfoToSequenceTypeInfo, _In_ const OrtTypeInfo* type_info,
782 _Outptr_result_maybenull_ const OrtSequenceTypeInfo** out);
783
784 // OrtMapTypeInfo Accessors
785
786 /**
787 * GetMapKeyType
788 * This api augments get the key type of a map. Key types are restricted to being scalar types and use ONNXTensorElementDataType.
789 * This is used by WinML to support model reflection APIs.
790 */
791 ORT_API2_STATUS(GetMapKeyType, _In_ const OrtMapTypeInfo* map_type_info, _Out_ enum ONNXTensorElementDataType* out);
792
793 /**
794 * GetMapValueType
795 * This api augments get the value type of a map.
796 */
797 ORT_API2_STATUS(GetMapValueType, _In_ const OrtMapTypeInfo* map_type_info, _Outptr_ OrtTypeInfo** type_info);
798
799 // OrtSequenceTypeInfo Accessors
800
801 /**
802 * GetSequenceElementType
803 * This api augments get the element type of a sequence.
804 * This is used by WinML to support model reflection APIs.
805 */
806 ORT_API2_STATUS(GetSequenceElementType, _In_ const OrtSequenceTypeInfo* sequence_type_info,
807 _Outptr_ OrtTypeInfo** type_info);
808
809 ORT_CLASS_RELEASE(MapTypeInfo);
810 ORT_CLASS_RELEASE(SequenceTypeInfo);
811
812 /**
813 * \param out is set to a null terminated string allocated using 'allocator'. The caller is responsible for freeing it.
814 * Profiling is turned ON automatically if enabled for the particular session by invoking EnableProfiling()
815 * on the SessionOptions instance used to create the session.
816 */
817 ORT_API2_STATUS(SessionEndProfiling, _In_ OrtSession* sess, _Inout_ OrtAllocator* allocator, _Outptr_ char** out);
818
819 /**
820 * \param out is a pointer to the newly created object. The pointer should be freed by calling ReleaseModelMetadata after use.
821 */
822 ORT_API2_STATUS(SessionGetModelMetadata, _In_ const OrtSession* sess, _Outptr_ OrtModelMetadata** out);
823
824 /**
825 * \param value is set to a null terminated string allocated using 'allocator'. The caller is responsible for freeing it.
826 */
827 ORT_API2_STATUS(ModelMetadataGetProducerName, _In_ const OrtModelMetadata* model_metadata,
828 _Inout_ OrtAllocator* allocator, _Outptr_ char** value);
829 ORT_API2_STATUS(ModelMetadataGetGraphName, _In_ const OrtModelMetadata* model_metadata,
830 _Inout_ OrtAllocator* allocator, _Outptr_ char** value);
831 ORT_API2_STATUS(ModelMetadataGetDomain, _In_ const OrtModelMetadata* model_metadata, _Inout_ OrtAllocator* allocator,
832 _Outptr_ char** value);
833 ORT_API2_STATUS(ModelMetadataGetDescription, _In_ const OrtModelMetadata* model_metadata,
834 _Inout_ OrtAllocator* allocator, _Outptr_ char** value);
835 /**
836 * \param value is set to a null terminated string allocated using 'allocator'. The caller is responsible for freeing it.
837 * 'value' will be a nullptr if the given key is not found in the custom metadata map.
838 */
839 ORT_API2_STATUS(ModelMetadataLookupCustomMetadataMap, _In_ const OrtModelMetadata* model_metadata,
840 _Inout_ OrtAllocator* allocator, _In_ const char* key, _Outptr_result_maybenull_ char** value);
841
842 ORT_API2_STATUS(ModelMetadataGetVersion, _In_ const OrtModelMetadata* model_metadata, _Out_ int64_t* value);
843
844 ORT_CLASS_RELEASE(ModelMetadata);
845
846 /*
847 * Creates an environment with global threadpools that will be shared across sessions.
848 * Use this in conjunction with DisablePerSessionThreads API or else the session will use
849 * its own thread pools.
850 */
851 ORT_API2_STATUS(CreateEnvWithGlobalThreadPools, OrtLoggingLevel logging_level, _In_ const char* logid,
852 _In_ const OrtThreadingOptions* t_options, _Outptr_ OrtEnv** out);
853
854 /*
855 * Calling this API will make the session use the global threadpools shared across sessions.
856 * This API should be used in conjunction with CreateEnvWithGlobalThreadPools API.
857 */
858 ORT_API2_STATUS(DisablePerSessionThreads, _Inout_ OrtSessionOptions* options);
859
860 ORT_API2_STATUS(CreateThreadingOptions, _Outptr_ OrtThreadingOptions** out);
861
862 ORT_CLASS_RELEASE(ThreadingOptions);
863
864 /**
865 * \param num_keys contains the number of keys in the custom metadata map
866 * \param keys is an array of null terminated strings (array count = num_keys) allocated using 'allocator'.
867 * The caller is responsible for freeing each string and the pointer array.
868 * 'keys' will be a nullptr if custom metadata map is empty.
869 */
870 ORT_API2_STATUS(ModelMetadataGetCustomMetadataMapKeys, _In_ const OrtModelMetadata* model_metadata,
871 _Inout_ OrtAllocator* allocator, _Outptr_result_buffer_maybenull_(*num_keys) char*** keys, _Out_ int64_t* num_keys);
872
873 // Override symbolic dimensions (by specific name strings) with actual values
874 // if known at session initialization time to enable optimizations that can
875 // take advantage of fixed values (such as memory planning, etc)
876 ORT_API2_STATUS(AddFreeDimensionOverrideByName,
877 _Inout_ OrtSessionOptions* options, _In_ const char* dim_name,
878 _In_ int64_t dim_value);
879
880 /**
881 * \param out_ptr will hold a pointer to the array of char *
882 * representing available providers.
883 * \param provider_length is a pointer to an int variable where
884 * the number of available providers will be added.
885 * The caller is responsible for freeing each char * and the pointer
886 * array by calling ReleaseAvailableProviders().
887 */
888 ORT_API2_STATUS(GetAvailableProviders, _Outptr_ char*** out_ptr,
889 _In_ int* provider_length);
890
891 /**
892 * \param ptr is the pointer to an array of available providers you
893 * get after calling GetAvailableProviders().
894 * \param providers_length is the number of available providers.
895 */
896 ORT_API2_STATUS(ReleaseAvailableProviders, _In_ char** ptr,
897 _In_ int providers_length);
898
899 /**
900 * \param value - A tensor created from OrtCreateTensor... function.
901 * \param index - index of string tensor element, length of element at index will be returned.
902 * \param out - number of UTF-8 bytes that the string contains
903 */
904 ORT_API2_STATUS(GetStringTensorElementLength, _In_ const OrtValue* value, size_t index, _Out_ size_t* out);
905
906 /**
907 * \param s string element contents in UTF-8 encoding. The string is NOT null-terminated.
908 * \param value A tensor created from OrtCreateTensor... function.
909 * \param s_len element length, get it from OrtGetStringTensorElementLength.
910 * \param index offset of element of tensor to return.
911 */
912 ORT_API2_STATUS(GetStringTensorElement, _In_ const OrtValue* value, size_t s_len, size_t index, _Out_writes_bytes_all_(s_len) void* s);
913
914 /**
915 * \param value - A tensor created from OrtCreateTensor... function.
916 * \param s - A null terminated UTF-8 encoded string.
917 * \param index - index of string tensor element to fill
918 */
919 ORT_API2_STATUS(FillStringTensorElement, _Inout_ OrtValue* value, _In_ const char* s, size_t index);
920
921 /**
922 * Set a single session configuration entry as a pair of strings
923 * If a configuration with same key exists, this will overwrite the configuration with the given config_value
924 * \param config_key A null terminated string representation of the config key
925 * \param config_value A null terminated string representation of the config value
926 * The config_key and the format of config_value are defined in onnxruntime_session_options_config_keys.h
927 */
928 ORT_API2_STATUS(AddSessionConfigEntry, _Inout_ OrtSessionOptions* options,
929 _In_z_ const char* config_key, _In_z_ const char* config_value);
930
931 /**
932 * \param sess valid OrtSession instance
933 * \param mem_info - valid OrtMemoryInfo instance
934 * \param - out a ptr to a new instance of OrtAllocator according to the spec within mem_info
935 * if successful
936 * \return OrtStatus or nullptr if successful
937 */
938 ORT_API2_STATUS(CreateAllocator, _In_ const OrtSession* sess, _In_ const OrtMemoryInfo* mem_info,
939 _Outptr_ OrtAllocator** out);
940
941 // Release instance of OrtAllocator obtained from CreateAllocator API
942 ORT_CLASS_RELEASE(Allocator);
943
944 ORT_API2_STATUS(RunWithBinding, _Inout_ OrtSession* sess, _In_opt_ const OrtRunOptions* run_options, _In_ const OrtIoBinding* binding_ptr);
945
946 // Creates an IoBinding instance that allows one to bind pre-allocated OrtValues
947 // to input names. Thus if you want to use a raw on device buffer as input or output
948 // you can avoid extra copy during runtime.
949 ORT_API2_STATUS(CreateIoBinding, _Inout_ OrtSession* sess, _Outptr_ OrtIoBinding** out);
950
951 // Release instance or OrtIoBinding obtained from CreateIoBinding API
952 ORT_CLASS_RELEASE(IoBinding);
953
954 /**
955 * The function will bind the OrtValue to a specified input name.
956 * The OrtValue must be a Tensor. ORT would use that value in place of input for the specified name.
957 * \param binding_ptr - an instance of OrtIoBinding created by CreateIoBinding()
958 * \param name - name for the model input
959 * \param val_ptr - OrtValue of Tensor type.
960 * \return OrtStatus instance on error which the caller is responsible to free or nullptr on success
961 */
962 ORT_API2_STATUS(BindInput, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtValue* val_ptr);
963
964 /**
965 * The function will bind the OrtValue to the specified output name.
966 * The OrtValue must be a Tensor. ORT would use that value in place of output for the specified name.
967 *
968 * \param binding_ptr - an instance of OrtIoBinding created by CreateIoBinding()
969 * \param name - name for the model output
970 * \param val_ptr - OrtValue of Tensor type.
971 * \return OrtStatus instance on error which the caller is responsible to free or nullptr on success
972 */
973 ORT_API2_STATUS(BindOutput, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtValue* val_ptr);
974
975 /**
976 * The function will bind the OrtValue to a device which specification is contained within OrtMemoryInfo
977 * You can either create an instance of OrtMemoryInfo with a device id or obtain one from the allocator that you are created/using
978 * This is useful when one or more outputs have dynamic shapes and, it is hard to pre-allocated and bind a chunk of
979 * memory within OrtValue ahead of time.
980 *
981 * \param binding_ptr - an instance of OrtIoBinding created by CreateIoBinding()
982 * \param name - name for the model output
983 * \param mem_info_ptr - OrtMemoryInfo
984 * \return OrtStatus instance on error which the caller is responsible to free or nullptr on success
985 */
986 ORT_API2_STATUS(BindOutputToDevice, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtMemoryInfo* val_ptr);
987
988 /**
989 * The function returns the names of the outputs in the order they were bound. This is useful after running the model
990 * with bound outputs because the returned names are in order in which output OrtValues are returned. This API is optional
991 * to use. If you knew the order of outputs and its names you used for binding you would not need to use this API.
992 *
993 * \param binding_ptr - a ptr to an instance of OrtIoBinding created obtained from CreateIoBinding()
994 * \param allocator - a ptr to an instance of OrtAllocator obtained with CreateAllocator() or GetAllocatorWithDefaultOptions()
995 * the specified allocator will be used to allocate continuous buffers for output strings and lengths.
996 * \param buffer - pointer to a continuous buffer of non-zero terminated UTF-8 encoded strings. The number of strings stored is returned count parameter.
997 * this buffer will be allocated with the specified allocator and must be freed after it is no longer needed.
998 * \param lengths - a pointer to a continuous buffer of size_t lengths of strings returned in the buffer. The number of items is returned
999 * in the count. This buffer is allocated with the specified allocator and must be freed after it is no longer needed.
1000 * \para count - is the number of strings returned. If the instance of OrtIoBiding has no bound outputs, zero is returned,
1001 * no memory allocation is performed and buffer and lengths are nullptr on return.
1002 */
1003 ORT_API2_STATUS(GetBoundOutputNames, _In_ const OrtIoBinding* binding_ptr, _In_ OrtAllocator* allocator,
1004 _Out_ char** buffer, _Out_writes_all_(count) size_t** lengths, _Out_ size_t* count);
1005
1006 /**
1007 * The function returns an array of pointers to individually allocated OrtValues that contain results of a model execution with RunWithBinding()
1008 * The array contains the same number of OrtValues and they are in the same order as they were bound with BindOutput()
1009 * or BindOutputToDevice().
1010 * The returned OrtValues must be individually released after they are no longer needed.
1011 * The array is allocated using the specified instance of the allocator and must be freed using the same allocator after
1012 * all the OrtValues contained therein are individually released.
1013 *
1014 * \param binding_ptr - instance of OrtIoBidning
1015 * \param allocator - instance of allocator to allocate output array
1016 * \param output - pointer to the allocated buffer. Returns nullptr if no outputs.
1017 * \param output_count - pointer to the number of OrtValues returned. Zero if no outputs.
1018 */
1019 ORT_API2_STATUS(GetBoundOutputValues, _In_ const OrtIoBinding* binding_ptr, _In_ OrtAllocator* allocator,
1020 _Out_writes_all_(output_count) OrtValue*** output, _Out_ size_t* output_count);
1021
1022 /** Clears any previously specified bindings for inputs/outputs
1023 */
1024 void(ORT_API_CALL* ClearBoundInputs)(_Inout_ OrtIoBinding* binding_ptr) NO_EXCEPTION ORT_ALL_ARGS_NONNULL;
1025 void(ORT_API_CALL* ClearBoundOutputs)(_Inout_ OrtIoBinding* binding_ptr) NO_EXCEPTION ORT_ALL_ARGS_NONNULL;
1026
1027 /**
1028 * Provides element-level access into a tensor.
1029 * \param location_values a pointer to an array of index values that specify an element's location in the tensor data blob
1030 * \param location_values_count length of location_values
1031 * \param out a pointer to the element specified by location_values
1032 * e.g.
1033 * Given a tensor with overall shape [3,224,224], an element at
1034 * location [2,150,128] can be accessed directly.
1035 *
1036 * This function only works for numeric tensors.
1037 * This is a no-copy method whose pointer is only valid until the backing OrtValue is free'd.
1038 */
1039 ORT_API2_STATUS(TensorAt, _Inout_ OrtValue* value, const int64_t* location_values, size_t location_values_count, _Outptr_ void** out);
1040
1041 /**
1042 * Creates an allocator instance and registers it with the env to enable
1043 * sharing between multiple sessions that use the same env instance.
1044 * Lifetime of the created allocator will be valid for the duration of the environment.
1045 * Returns an error if an allocator with the same OrtMemoryInfo is already registered.
1046 * \param mem_info must be non-null.
1047 * \param arena_cfg if nullptr defaults will be used.
1048 * See docs/C_API.md for details.
1049 */
1050 ORT_API2_STATUS(CreateAndRegisterAllocator, _Inout_ OrtEnv* env, _In_ const OrtMemoryInfo* mem_info,
1051 _In_ const OrtArenaCfg* arena_cfg);
1052
1053 /**
1054 * Set the language projection for collecting telemetry data when Env is created
1055 * \param projection the source projected language.
1056 */
1057 ORT_API2_STATUS(SetLanguageProjection, _In_ const OrtEnv* ort_env, _In_ OrtLanguageProjection projection);
1058
1059 /**
1060 * On some platforms, this timer may not be as precise as nanoseconds
1061 * For instance, on Windows and MacOS, the precision will be ~100ns
1062 * \param out is set to the nanoseconds of profiling's start time
1063 */
1064 ORT_API2_STATUS(SessionGetProfilingStartTimeNs, _In_ const OrtSession* sess, _Outptr_ uint64_t* out);
1065
1066 /**
1067 * Use this API to configure the global thread pool options to be used in the call to CreateEnvWithGlobalThreadPools.
1068 * A value of 0 means ORT will pick the default.
1069 * A value of 1 means the invoking thread will be used; no threads will be created in the thread pool.
1070 */
1071 ORT_API2_STATUS(SetGlobalIntraOpNumThreads, _Inout_ OrtThreadingOptions* tp_options, int intra_op_num_threads);
1072 ORT_API2_STATUS(SetGlobalInterOpNumThreads, _Inout_ OrtThreadingOptions* tp_options, int inter_op_num_threads);
1073
1074 /**
1075 * Use this API to configure the global thread pool options to be used in the call to CreateEnvWithGlobalThreadPools.
1076 * Allow spinning of thread pools when their queues are empty. This API will set the value for both
1077 * inter_op and intra_op threadpools.
1078 * \param allow_spinning valid values are 1 and 0.
1079 * 1: threadpool will spin to wait for queue to become non-empty, 0: it won't spin.
1080 * Prefer a value of 0 if your CPU usage is very high.
1081 */
1082 ORT_API2_STATUS(SetGlobalSpinControl, _Inout_ OrtThreadingOptions* tp_options, int allow_spinning);
1083
1084 /**
1085 * Add a pre-allocated initializer to a session. If a model contains an initializer with a name
1086 * that is same as the name passed to this API call, ORT will use this initializer instance
1087 * instead of deserializing one from the model file. This is useful when you want to share
1088 * the same initializer across sessions.
1089 * \param name name of the initializer
1090 * \param val OrtValue containing the initializer. Lifetime of 'val' and the underlying initializer buffer must be
1091 * managed by the user (created using the CreateTensorWithDataAsOrtValue API) and it must outlive the session object
1092 * to which it is added.
1093 */
1094 ORT_API2_STATUS(AddInitializer, _Inout_ OrtSessionOptions* options, _In_z_ const char* name,
1095 _In_ const OrtValue* val);
1096
1097 /**
1098 * Creates a custom environment with global threadpools and logger that will be shared across sessions.
1099 * Use this in conjunction with DisablePerSessionThreads API or else the session will use
1100 * its own thread pools.
1101 *
1102 * \param out should be freed by `OrtReleaseEnv` after use
1103 */
1104 ORT_API2_STATUS(CreateEnvWithCustomLoggerAndGlobalThreadPools, OrtLoggingFunction logging_function, _In_opt_ void* logger_param, OrtLoggingLevel logging_level,
1105 _In_ const char* logid, _In_ const struct OrtThreadingOptions* tp_options, _Outptr_ OrtEnv** out);
1106
1107 /**
1108 * Append CUDA execution provider to the session options
1109 * If CUDA is not available (due to a non cuda enabled build), this function will return failure.
1110 */
1111 ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_CUDA,
1112 _In_ OrtSessionOptions* options, _In_ const OrtCUDAProviderOptions* cuda_options);
1113
1114 /**
1115 * Append OpenVINO execution provider to the session options
1116 * If OpenVINO is not available (due to the OpenVINO provider shared library or its dependencies not being installed), this function will fail.
1117 */
1118 ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_OpenVINO,
1119 _In_ OrtSessionOptions* options, _In_ const OrtOpenVINOProviderOptions* provider_options);
1120
1121 /**
1122 * Use this API to configure the global thread pool options to be used in the call to CreateEnvWithGlobalThreadPools.
1123 * When this API is called, flush-to-zero and denormal-as-zero are applied to threads in both intra and inter global thread pool.
1124 * Note that an alternative way not using this option at runtime is to train and export a model without denormals
1125 * and that's recommended because turning this option on may hurt model accuracy.
1126 */
1127 ORT_API2_STATUS(SetGlobalDenormalAsZero, _Inout_ OrtThreadingOptions* tp_options);
1128
1129 /**
1130 * Use this API to create the configuration of an arena that can eventually be used to define
1131 * an arena based allocator's behavior
1132 * \param max_mem - use 0 to allow ORT to choose the default
1133 * \param arena_extend_strategy - use -1 to allow ORT to choose the default, 0 = kNextPowerOfTwo, 1 = kSameAsRequested
1134 * \param initial_chunk_size_bytes - use -1 to allow ORT to choose the default
1135 * \param max_dead_bytes_per_chunk - use -1 to allow ORT to choose the default
1136 * \param out - a pointer to an OrtArenaCfg instance
1137 * \return a nullptr in case of success or a pointer to an OrtStatus instance in case of failure
1138 * See docs/C_API.md for details on what the following parameters mean and how to choose these values
1139 */
1140 ORT_API2_STATUS(CreateArenaCfg, _In_ size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes,
1141 int max_dead_bytes_per_chunk, _Outptr_ OrtArenaCfg** out);
1142
1143 ORT_CLASS_RELEASE(ArenaCfg);
1144};
1145
1146/*
1147 * Steps to use a custom op:
1148 * 1 Create an OrtCustomOpDomain with the domain name used by the custom ops
1149 * 2 Create an OrtCustomOp structure for each op and add them to the domain
1150 * 3 Call OrtAddCustomOpDomain to add the custom domain of ops to the session options
1151*/
1152#define OrtCustomOpApi OrtApi
1153
1154/*
1155 * The OrtCustomOp structure defines a custom op's schema and its kernel callbacks. The callbacks are filled in by
1156 * the implementor of the custom op.
1157*/
1158struct OrtCustomOp {
1159 uint32_t version; // Initialize to ORT_API_VERSION
1160
1161 // This callback creates the kernel, which is a user defined parameter that is passed to the Kernel* callbacks below.
1162 void*(ORT_API_CALL* CreateKernel)(_In_ const struct OrtCustomOp* op, _In_ const OrtApi* api,
1163 _In_ const OrtKernelInfo* info);
1164
1165 // Returns the name of the op
1166 const char*(ORT_API_CALL* GetName)(_In_ const struct OrtCustomOp* op);
1167
1168 // Returns the type of the execution provider, return nullptr to use CPU execution provider
1169 const char*(ORT_API_CALL* GetExecutionProviderType)(_In_ const struct OrtCustomOp* op);
1170
1171 // Returns the count and types of the input & output tensors
1172 ONNXTensorElementDataType(ORT_API_CALL* GetInputType)(_In_ const struct OrtCustomOp* op, _In_ size_t index);
1173 size_t(ORT_API_CALL* GetInputTypeCount)(_In_ const struct OrtCustomOp* op);
1174 ONNXTensorElementDataType(ORT_API_CALL* GetOutputType)(_In_ const struct OrtCustomOp* op, _In_ size_t index);
1175 size_t(ORT_API_CALL* GetOutputTypeCount)(_In_ const struct OrtCustomOp* op);
1176
1177 // Op kernel callbacks
1178 void(ORT_API_CALL* KernelCompute)(_In_ void* op_kernel, _In_ OrtKernelContext* context);
1179 void(ORT_API_CALL* KernelDestroy)(_In_ void* op_kernel);
1180};
1181
1182/*
1183 * END EXPERIMENTAL
1184*/
1185
1186#ifdef __cplusplus
1187}
1188#endif