GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/onnxruntime-extensions

Public

mirrored fromhttps://github.com/microsoft/onnxruntime-extensionsAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
1ae69c0f7aeaab9911cf8ebf86ee92b34dadd26e

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

onnxruntime-extensions/includes/onnxruntime

includes/onnxruntime/onnxruntime_c_api.h

1181lines · modecode

RawDownload
Latest commit unavailable.
1// Copyright (c) Microsoft Corporation. All rights reserved.
2// Licensed under the MIT License.
3
4#pragma once
5#include <stdlib.h>
6#include <stdint.h>
7#include <string.h>
8
9// This value is used in structures passed to ORT so that a newer version of ORT will still work with them
10#define ORT_API_VERSION 6
11
12#ifdef __cplusplus
13extern "C" {
14#endif
15
16// SAL2 Definitions
17#ifndef _WIN32
18#define _In_
19#define _In_z_
20#define _In_opt_
21#define _In_opt_z_
22#define _Out_
23#define _Outptr_
24#define _Out_opt_
25#define _Inout_
26#define _Inout_opt_
27#define _Frees_ptr_opt_
28#define _Ret_maybenull_
29#define _Ret_notnull_
30#define _Check_return_
31#define _Outptr_result_maybenull_
32#define _In_reads_(X)
33#define _Inout_updates_all_(X)
34#define _Out_writes_bytes_all_(X)
35#define _Out_writes_all_(X)
36#define _Success_(X)
37#define _Outptr_result_buffer_maybenull_(X)
38#define ORT_ALL_ARGS_NONNULL __attribute__((nonnull))
39#else
40#include <specstrings.h>
41#define ORT_ALL_ARGS_NONNULL
42#endif
43
44#ifdef _WIN32
45// Define ORT_DLL_IMPORT if your program is dynamically linked to Ort.
46// dllexport is not used, we use a .def file.
47#ifdef ORT_DLL_IMPORT
48#define ORT_EXPORT __declspec(dllimport)
49#else
50#define ORT_EXPORT
51#endif
52#define ORT_API_CALL _stdcall
53#define ORT_MUST_USE_RESULT
54#define ORTCHAR_T wchar_t
55#else
56#define ORT_EXPORT
57#define ORT_API_CALL
58#define ORT_MUST_USE_RESULT __attribute__((warn_unused_result))
59#define ORTCHAR_T char
60#endif
61
62#ifndef ORT_TSTR
63#ifdef _WIN32
64#define ORT_TSTR(X) L##X
65#else
66#define ORT_TSTR(X) X
67#endif
68#endif
69
70// Any pointer marked with _In_ or _Out_, cannot be NULL.
71
72// Windows users should use unicode paths when possible to bypass the MAX_PATH limitation
73// Every pointer marked with _In_ or _Out_, cannot be NULL. Caller should ensure that.
74// for ReleaseXXX(...) functions, they can accept NULL pointer.
75
76#ifdef __cplusplus
77// For any compiler with C++11 support, MSVC 2015 and greater, or Clang version supporting noexcept.
78// Such complex condition is needed because compilers set __cplusplus value differently.
79#ifndef __has_feature
80#define __has_feature(x) 0
81#endif
82#if ((__cplusplus >= 201103L) || (_MSC_VER >= 1900) || (defined(__has_feature) && __has_feature(cxx_noexcept)))
83#define NO_EXCEPTION noexcept
84#else
85#define NO_EXCEPTION throw()
86#endif
87#else
88#define NO_EXCEPTION
89#endif
90
91// Copied from TensorProto::DataType
92// Currently, Ort doesn't support complex64, complex128
93typedef enum ONNXTensorElementDataType {
94 ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED,
95 ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, // maps to c type float
96 ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8, // maps to c type uint8_t
97 ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8, // maps to c type int8_t
98 ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16, // maps to c type uint16_t
99 ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16, // maps to c type int16_t
100 ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32, // maps to c type int32_t
101 ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64, // maps to c type int64_t
102 ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING, // maps to c++ type std::string
103 ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL,
104 ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16,
105 ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE, // maps to c type double
106 ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32, // maps to c type uint32_t
107 ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64, // maps to c type uint64_t
108 ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64, // complex with float32 real and imaginary components
109 ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128, // complex with float64 real and imaginary components
110 ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16 // Non-IEEE floating-point format based on IEEE754 single-precision
111} ONNXTensorElementDataType;
112
113// Synced with onnx TypeProto oneof
114typedef enum ONNXType {
115 ONNX_TYPE_UNKNOWN,
116 ONNX_TYPE_TENSOR,
117 ONNX_TYPE_SEQUENCE,
118 ONNX_TYPE_MAP,
119 ONNX_TYPE_OPAQUE,
120 ONNX_TYPE_SPARSETENSOR,
121} ONNXType;
122
123typedef enum OrtLoggingLevel {
124 ORT_LOGGING_LEVEL_VERBOSE,
125 ORT_LOGGING_LEVEL_INFO,
126 ORT_LOGGING_LEVEL_WARNING,
127 ORT_LOGGING_LEVEL_ERROR,
128 ORT_LOGGING_LEVEL_FATAL,
129} OrtLoggingLevel;
130
131typedef enum OrtErrorCode {
132 ORT_OK,
133 ORT_FAIL,
134 ORT_INVALID_ARGUMENT,
135 ORT_NO_SUCHFILE,
136 ORT_NO_MODEL,
137 ORT_ENGINE_ERROR,
138 ORT_RUNTIME_EXCEPTION,
139 ORT_INVALID_PROTOBUF,
140 ORT_MODEL_LOADED,
141 ORT_NOT_IMPLEMENTED,
142 ORT_INVALID_GRAPH,
143 ORT_EP_FAIL,
144} OrtErrorCode;
145
146#define ORT_RUNTIME_CLASS(X) \
147 struct Ort##X; \
148 typedef struct Ort##X Ort##X;
149
150// The actual types defined have an Ort prefix
151ORT_RUNTIME_CLASS(Env);
152ORT_RUNTIME_CLASS(Status); // nullptr for Status* indicates success
153ORT_RUNTIME_CLASS(MemoryInfo);
154ORT_RUNTIME_CLASS(IoBinding);
155ORT_RUNTIME_CLASS(Session); //Don't call OrtReleaseSession from Dllmain (because session owns a thread pool)
156ORT_RUNTIME_CLASS(Value);
157ORT_RUNTIME_CLASS(RunOptions);
158ORT_RUNTIME_CLASS(TypeInfo);
159ORT_RUNTIME_CLASS(TensorTypeAndShapeInfo);
160ORT_RUNTIME_CLASS(SessionOptions);
161ORT_RUNTIME_CLASS(CustomOpDomain);
162ORT_RUNTIME_CLASS(MapTypeInfo);
163ORT_RUNTIME_CLASS(SequenceTypeInfo);
164ORT_RUNTIME_CLASS(ModelMetadata);
165ORT_RUNTIME_CLASS(ThreadPoolParams);
166ORT_RUNTIME_CLASS(ThreadingOptions);
167ORT_RUNTIME_CLASS(ArenaCfg);
168
169#ifdef _WIN32
170typedef _Return_type_success_(return == 0) OrtStatus* OrtStatusPtr;
171#else
172typedef OrtStatus* OrtStatusPtr;
173#endif
174
175// __VA_ARGS__ on Windows and Linux are different
176#define ORT_API(RETURN_TYPE, NAME, ...) ORT_EXPORT RETURN_TYPE ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION
177
178#define ORT_API_STATUS(NAME, ...) \
179 ORT_EXPORT _Check_return_ _Ret_maybenull_ OrtStatusPtr ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION ORT_MUST_USE_RESULT
180
181// XXX: Unfortunately, SAL annotations are known to not work with function pointers
182#define ORT_API2_STATUS(NAME, ...) \
183 _Check_return_ _Ret_maybenull_ OrtStatusPtr(ORT_API_CALL* NAME)(__VA_ARGS__) NO_EXCEPTION ORT_MUST_USE_RESULT
184
185// Used in *.cc files. Almost as same as ORT_API_STATUS, except without ORT_MUST_USE_RESULT and ORT_EXPORT
186#define ORT_API_STATUS_IMPL(NAME, ...) \
187 _Check_return_ _Ret_maybenull_ OrtStatusPtr ORT_API_CALL NAME(__VA_ARGS__) NO_EXCEPTION
188
189#define ORT_CLASS_RELEASE(X) void(ORT_API_CALL * Release##X)(_Frees_ptr_opt_ Ort##X * input)
190
191// When passing in an allocator to any ORT function, be sure that the allocator object
192// is not destroyed until the last allocated object using it is freed.
193typedef struct OrtAllocator {
194 uint32_t version; // Initialize to ORT_API_VERSION
195 void*(ORT_API_CALL* Alloc)(struct OrtAllocator* this_, size_t size);
196 void(ORT_API_CALL* Free)(struct OrtAllocator* this_, void* p);
197 const struct OrtMemoryInfo*(ORT_API_CALL* Info)(const struct OrtAllocator* this_);
198} OrtAllocator;
199
200typedef void(ORT_API_CALL* OrtLoggingFunction)(
201 void* param, OrtLoggingLevel severity, const char* category, const char* logid, const char* code_location,
202 const char* message);
203
204// Set Graph optimization level.
205// Refer https://github.com/microsoft/onnxruntime/blob/master/docs/ONNX_Runtime_Graph_Optimizations.md
206// for in-depth undersrtanding of Graph Optimizations in ORT
207typedef enum GraphOptimizationLevel {
208 ORT_DISABLE_ALL = 0,
209 ORT_ENABLE_BASIC = 1,
210 ORT_ENABLE_EXTENDED = 2,
211 ORT_ENABLE_ALL = 99
212} GraphOptimizationLevel;
213
214typedef enum ExecutionMode {
215 ORT_SEQUENTIAL = 0,
216 ORT_PARALLEL = 1,
217} ExecutionMode;
218
219// Set the language projection, default is C, which means it will classify the language not in the list to C also.
220typedef enum OrtLanguageProjection {
221 ORT_PROJECTION_C = 0, // default
222 ORT_PROJECTION_CPLUSPLUS = 1,
223 ORT_PROJECTION_CSHARP = 2,
224 ORT_PROJECTION_PYTHON = 3,
225 ORT_PROJECTION_JAVA = 4,
226 ORT_PROJECTION_WINML = 5,
227 ORT_PROJECTION_NODEJS = 6,
228} OrtLanguageProjection;
229
230struct OrtKernelInfo;
231typedef struct OrtKernelInfo OrtKernelInfo;
232struct OrtKernelContext;
233typedef struct OrtKernelContext OrtKernelContext;
234struct OrtCustomOp;
235typedef struct OrtCustomOp OrtCustomOp;
236
237typedef enum OrtAllocatorType {
238 Invalid = -1,
239 OrtDeviceAllocator = 0,
240 OrtArenaAllocator = 1
241} OrtAllocatorType;
242
243/**
244 * memory types for allocator, exec provider specific types should be extended in each provider
245 * Whenever this struct is updated, please also update the MakeKey function in onnxruntime/core/framework/execution_provider.cc
246*/
247typedef enum OrtMemType {
248 OrtMemTypeCPUInput = -2, // Any CPU memory used by non-CPU execution provider
249 OrtMemTypeCPUOutput = -1, // CPU accessible memory outputted by non-CPU execution provider, i.e. CUDA_PINNED
250 OrtMemTypeCPU = OrtMemTypeCPUOutput, // temporary CPU accessible memory allocated by non-CPU execution provider, i.e. CUDA_PINNED
251 OrtMemTypeDefault = 0, // the default allocator for execution provider
252} OrtMemType;
253
254typedef enum OrtCudnnConvAlgoSearch {
255 EXHAUSTIVE, // expensive exhaustive benchmarking using cudnnFindConvolutionForwardAlgorithmEx
256 HEURISTIC, // lightweight heuristic based search using cudnnGetConvolutionForwardAlgorithm_v7
257 DEFAULT, // default algorithm using CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM
258} OrtCudnnConvAlgoSearch;
259
260/// <summary>
261/// Options for the CUDA provider that are passed to SessionOptionsAppendExecutionProvider_CUDA
262/// </summary>
263typedef struct OrtCUDAProviderOptions {
264 int device_id; // cuda device with id=0 as default device.
265 OrtCudnnConvAlgoSearch cudnn_conv_algo_search; // cudnn conv algo search option
266 size_t cuda_mem_limit; // default cuda memory limitation to maximum finite value of size_t.
267 int arena_extend_strategy; // default area extend strategy to KNextPowerOfTwo.
268 int do_copy_in_default_stream;
269} OrtCUDAProviderOptions;
270
271/// <summary>
272/// Options for the OpenVINO provider that are passed to SessionOptionsAppendExecutionProvider_OpenVINO
273/// </summary>
274typedef struct OrtOpenVINOProviderOptions {
275#ifdef __cplusplus
276 OrtOpenVINOProviderOptions() : device_type{}, enable_vpu_fast_compile{}, device_id{}, num_of_threads{} {}
277#endif
278 const char* device_type; // CPU_FP32, GPU_FP32, GPU_FP16, MYRIAD_FP16, VAD-M_FP16 or VAD-F_FP32
279 unsigned char enable_vpu_fast_compile; // 0 = false, nonzero = true
280 const char* device_id;
281 size_t num_of_threads; // 0 uses default number of threads
282} OrtOpenVINOProviderOptions;
283
284struct OrtApi;
285typedef struct OrtApi OrtApi;
286
287struct OrtApiBase {
288 const OrtApi*(ORT_API_CALL* GetApi)(uint32_t version)NO_EXCEPTION; // Pass in ORT_API_VERSION
289 // nullptr will be returned if the version is unsupported, for example when using a runtime older than this header file
290
291 const char*(ORT_API_CALL* GetVersionString)() NO_EXCEPTION;
292};
293typedef struct OrtApiBase OrtApiBase;
294
295ORT_EXPORT const OrtApiBase* ORT_API_CALL OrtGetApiBase(void) NO_EXCEPTION;
296
297struct OrtApi {
298 /**
299* \param msg A null-terminated string. Its content will be copied into the newly created OrtStatus
300*/
301 OrtStatus*(ORT_API_CALL* CreateStatus)(OrtErrorCode code, _In_ const char* msg)NO_EXCEPTION ORT_ALL_ARGS_NONNULL;
302
303 OrtErrorCode(ORT_API_CALL* GetErrorCode)(_In_ const OrtStatus* status) NO_EXCEPTION ORT_ALL_ARGS_NONNULL;
304
305 /**
306 * \param status must not be NULL
307 * \return The error message inside the `status`. Do not free the returned value.
308 */
309 const char*(ORT_API_CALL* GetErrorMessage)(_In_ const OrtStatus* status)NO_EXCEPTION ORT_ALL_ARGS_NONNULL;
310
311 /**
312 * \param out Should be freed by `OrtReleaseEnv` after use
313 */
314 ORT_API2_STATUS(CreateEnv, OrtLoggingLevel logging_level, _In_ const char* logid, _Outptr_ OrtEnv** out);
315
316 /**
317 * \param out Should be freed by `OrtReleaseEnv` after use
318 */
319 ORT_API2_STATUS(CreateEnvWithCustomLogger, OrtLoggingFunction logging_function, _In_opt_ void* logger_param,
320 OrtLoggingLevel logging_level, _In_ const char* logid, _Outptr_ OrtEnv** out);
321
322 // Platform telemetry events are on by default since they are lightweight. You can manually turn them off.
323 ORT_API2_STATUS(EnableTelemetryEvents, _In_ const OrtEnv* env);
324 ORT_API2_STATUS(DisableTelemetryEvents, _In_ const OrtEnv* env);
325
326 // TODO: document the path separator convention? '/' vs '\'
327 // TODO: should specify the access characteristics of model_path. Is this read only during the
328 // execution of CreateSession, or does the OrtSession retain a handle to the file/directory
329 // and continue to access throughout the OrtSession lifetime?
330 // What sort of access is needed to model_path : read or read/write?
331 ORT_API2_STATUS(CreateSession, _In_ const OrtEnv* env, _In_ const ORTCHAR_T* model_path,
332 _In_ const OrtSessionOptions* options, _Outptr_ OrtSession** out);
333
334 ORT_API2_STATUS(CreateSessionFromArray, _In_ const OrtEnv* env, _In_ const void* model_data, size_t model_data_length,
335 _In_ const OrtSessionOptions* options, _Outptr_ OrtSession** out);
336
337 ORT_API2_STATUS(Run, _Inout_ OrtSession* sess, _In_opt_ const OrtRunOptions* run_options,
338 _In_reads_(input_len) const char* const* input_names,
339 _In_reads_(input_len) const OrtValue* const* input, size_t input_len,
340 _In_reads_(output_names_len) const char* const* output_names1, size_t output_names_len,
341 _Inout_updates_all_(output_names_len) OrtValue** output);
342
343 /**
344 * \return A pointer of the newly created object. The pointer should be freed by OrtReleaseSessionOptions after use
345 */
346 ORT_API2_STATUS(CreateSessionOptions, _Outptr_ OrtSessionOptions** options);
347
348 // Set filepath to save optimized model after graph level transformations.
349 ORT_API2_STATUS(SetOptimizedModelFilePath, _Inout_ OrtSessionOptions* options,
350 _In_ const ORTCHAR_T* optimized_model_filepath);
351
352 // create a copy of an existing OrtSessionOptions
353 ORT_API2_STATUS(CloneSessionOptions, _In_ const OrtSessionOptions* in_options,
354 _Outptr_ OrtSessionOptions** out_options);
355
356 // Controls whether you want to execute operators in your graph sequentially or in parallel. Usually when the model
357 // has many branches, setting this option to ExecutionMode.ORT_PARALLEL will give you better performance.
358 // See [docs/ONNX_Runtime_Perf_Tuning.md] for more details.
359 ORT_API2_STATUS(SetSessionExecutionMode, _Inout_ OrtSessionOptions* options, ExecutionMode execution_mode);
360
361 // Enable profiling for this session.
362 ORT_API2_STATUS(EnableProfiling, _Inout_ OrtSessionOptions* options, _In_ const ORTCHAR_T* profile_file_prefix);
363 ORT_API2_STATUS(DisableProfiling, _Inout_ OrtSessionOptions* options);
364
365 // Enable the memory pattern optimization.
366 // The idea is if the input shapes are the same, we could trace the internal memory allocation
367 // and generate a memory pattern for future request. So next time we could just do one allocation
368 // with a big chunk for all the internal memory allocation.
369 // Note: memory pattern optimization is only available when SequentialExecution enabled.
370 ORT_API2_STATUS(EnableMemPattern, _Inout_ OrtSessionOptions* options);
371 ORT_API2_STATUS(DisableMemPattern, _Inout_ OrtSessionOptions* options);
372
373 // Enable the memory arena on CPU
374 // Arena may pre-allocate memory for future usage.
375 // set this option to false if you don't want it.
376 ORT_API2_STATUS(EnableCpuMemArena, _Inout_ OrtSessionOptions* options);
377 ORT_API2_STATUS(DisableCpuMemArena, _Inout_ OrtSessionOptions* options);
378
379 // < logger id to use for session output
380 ORT_API2_STATUS(SetSessionLogId, _Inout_ OrtSessionOptions* options, const char* logid);
381
382 // < applies to session load, initialization, etc
383 ORT_API2_STATUS(SetSessionLogVerbosityLevel, _Inout_ OrtSessionOptions* options, int session_log_verbosity_level);
384 ORT_API2_STATUS(SetSessionLogSeverityLevel, _Inout_ OrtSessionOptions* options, int session_log_severity_level);
385
386 ORT_API2_STATUS(SetSessionGraphOptimizationLevel, _Inout_ OrtSessionOptions* options,
387 GraphOptimizationLevel graph_optimization_level);
388
389 // Sets the number of threads used to parallelize the execution within nodes
390 // A value of 0 means ORT will pick a default
391 // Note: If you've built ORT with OpenMP, this API has no effect on the number of threads used. In this case
392 // use the OpenMP env variables to configure the number of intra op num threads.
393 ORT_API2_STATUS(SetIntraOpNumThreads, _Inout_ OrtSessionOptions* options, int intra_op_num_threads);
394
395 // Sets the number of threads used to parallelize the execution of the graph (across nodes)
396 // If sequential execution is enabled this value is ignored
397 // A value of 0 means ORT will pick a default
398 ORT_API2_STATUS(SetInterOpNumThreads, _Inout_ OrtSessionOptions* options, int inter_op_num_threads);
399
400 /*
401 Create a custom op domain. After all sessions using it are released, call OrtReleaseCustomOpDomain
402 */
403 ORT_API2_STATUS(CreateCustomOpDomain, _In_ const char* domain, _Outptr_ OrtCustomOpDomain** out);
404
405 /*
406 * Add custom ops to the OrtCustomOpDomain
407 * Note: The OrtCustomOp* pointer must remain valid until the OrtCustomOpDomain using it is released
408 */
409 ORT_API2_STATUS(CustomOpDomain_Add, _Inout_ OrtCustomOpDomain* custom_op_domain, _In_ const OrtCustomOp* op);
410
411 /*
412 * Add a custom op domain to the OrtSessionOptions
413 * Note: The OrtCustomOpDomain* must not be deleted until the sessions using it are released
414 */
415 ORT_API2_STATUS(AddCustomOpDomain, _Inout_ OrtSessionOptions* options, _In_ OrtCustomOpDomain* custom_op_domain);
416
417 /*
418 * Loads a DLL named 'library_path' and looks for this entry point:
419 * OrtStatus* RegisterCustomOps(OrtSessionOptions * options, const OrtApiBase* api);
420 * It then passes in the provided session options to this function along with the api base.
421 * The handle to the loaded library is returned in library_handle. It can be freed by the caller after all sessions using the passed in
422 * session options are destroyed, or if an error occurs and it is non null.
423 */
424 ORT_API2_STATUS(RegisterCustomOpsLibrary, _Inout_ OrtSessionOptions* options, _In_ const char* library_path,
425 void** library_handle);
426
427 /**
428 * To use additional providers, you must build ORT with the extra providers enabled. Then call one of these
429 * functions to enable them in the session:
430 * OrtSessionOptionsAppendExecutionProvider_CPU
431 * OrtSessionOptionsAppendExecutionProvider_CUDA
432 * OrtSessionOptionsAppendExecutionProvider_<remaining providers...>
433 * The order they are called indicates the preference order as well. In other words call this method
434 * on your most preferred execution provider first followed by the less preferred ones.
435 * If none are called Ort will use its internal CPU execution provider.
436 */
437
438 ORT_API2_STATUS(SessionGetInputCount, _In_ const OrtSession* sess, _Out_ size_t* out);
439 ORT_API2_STATUS(SessionGetOutputCount, _In_ const OrtSession* sess, _Out_ size_t* out);
440 ORT_API2_STATUS(SessionGetOverridableInitializerCount, _In_ const OrtSession* sess, _Out_ size_t* out);
441
442 /**
443 * \param out should be freed by OrtReleaseTypeInfo after use
444 */
445 ORT_API2_STATUS(SessionGetInputTypeInfo, _In_ const OrtSession* sess, size_t index, _Outptr_ OrtTypeInfo** type_info);
446
447 /**
448 * \param out should be freed by OrtReleaseTypeInfo after use
449 */
450 ORT_API2_STATUS(SessionGetOutputTypeInfo, _In_ const OrtSession* sess, size_t index,
451 _Outptr_ OrtTypeInfo** type_info);
452
453 /**
454 * \param out should be freed by OrtReleaseTypeInfo after use
455 */
456 ORT_API2_STATUS(SessionGetOverridableInitializerTypeInfo, _In_ const OrtSession* sess, size_t index,
457 _Outptr_ OrtTypeInfo** type_info);
458
459 /**
460 * \param value is set to a null terminated string allocated using 'allocator'. The caller is responsible for freeing it.
461 */
462 ORT_API2_STATUS(SessionGetInputName, _In_ const OrtSession* sess, size_t index, _Inout_ OrtAllocator* allocator,
463 _Outptr_ char** value);
464 ORT_API2_STATUS(SessionGetOutputName, _In_ const OrtSession* sess, size_t index, _Inout_ OrtAllocator* allocator,
465 _Outptr_ char** value);
466 ORT_API2_STATUS(SessionGetOverridableInitializerName, _In_ const OrtSession* sess, size_t index,
467 _Inout_ OrtAllocator* allocator, _Outptr_ char** value);
468
469 /**
470 * \return A pointer to the newly created object. The pointer should be freed by OrtReleaseRunOptions after use
471 */
472 ORT_API2_STATUS(CreateRunOptions, _Outptr_ OrtRunOptions** out);
473
474 ORT_API2_STATUS(RunOptionsSetRunLogVerbosityLevel, _Inout_ OrtRunOptions* options, int value);
475 ORT_API2_STATUS(RunOptionsSetRunLogSeverityLevel, _Inout_ OrtRunOptions* options, int value);
476 ORT_API2_STATUS(RunOptionsSetRunTag, _Inout_ OrtRunOptions*, _In_ const char* run_tag);
477
478 ORT_API2_STATUS(RunOptionsGetRunLogVerbosityLevel, _In_ const OrtRunOptions* options, _Out_ int* out);
479 ORT_API2_STATUS(RunOptionsGetRunLogSeverityLevel, _In_ const OrtRunOptions* options, _Out_ int* out);
480 ORT_API2_STATUS(RunOptionsGetRunTag, _In_ const OrtRunOptions*, _Out_ const char** out);
481
482 // Set a flag so that ALL incomplete OrtRun calls that are using this instance of OrtRunOptions
483 // will exit as soon as possible.
484 ORT_API2_STATUS(RunOptionsSetTerminate, _Inout_ OrtRunOptions* options);
485 // Unset the terminate flag to enable this OrtRunOptions instance being used in new OrtRun calls.
486 ORT_API2_STATUS(RunOptionsUnsetTerminate, _Inout_ OrtRunOptions* options);
487
488 /**
489 * Create a tensor from an allocator. OrtReleaseValue will also release the buffer inside the output value
490 * \param out Should be freed by calling OrtReleaseValue
491 * \param type must be one of TENSOR_ELEMENT_DATA_TYPE_xxxx
492 */
493 ORT_API2_STATUS(CreateTensorAsOrtValue, _Inout_ OrtAllocator* allocator, _In_ const int64_t* shape, size_t shape_len,
494 ONNXTensorElementDataType type, _Outptr_ OrtValue** out);
495
496 /**
497 * Create a tensor with user's buffer. You can fill the buffer either before calling this function or after.
498 * p_data is owned by caller. OrtReleaseValue won't release p_data.
499 * \param out Should be freed by calling OrtReleaseValue
500 */
501 ORT_API2_STATUS(CreateTensorWithDataAsOrtValue, _In_ const OrtMemoryInfo* info, _Inout_ void* p_data,
502 size_t p_data_len, _In_ const int64_t* shape, size_t shape_len, ONNXTensorElementDataType type,
503 _Outptr_ OrtValue** out);
504
505 /**
506 * \Sets *out to 1 iff an OrtValue is a tensor, 0 otherwise
507 */
508 ORT_API2_STATUS(IsTensor, _In_ const OrtValue* value, _Out_ int* out);
509
510 // This function doesn't work with string tensor
511 // this is a no-copy method whose pointer is only valid until the backing OrtValue is free'd.
512 ORT_API2_STATUS(GetTensorMutableData, _Inout_ OrtValue* value, _Outptr_ void** out);
513
514 /**
515 * \param value A tensor created from OrtCreateTensor... function.
516 * \param s each A string array. Each string in this array must be null terminated.
517 * \param s_len length of s
518 */
519 ORT_API2_STATUS(FillStringTensor, _Inout_ OrtValue* value, _In_ const char* const* s, size_t s_len);
520
521 /**
522 * \param value A tensor created from OrtCreateTensor... function.
523 * \param len total data length, not including the trailing '\0' chars.
524 */
525 ORT_API2_STATUS(GetStringTensorDataLength, _In_ const OrtValue* value, _Out_ size_t* len);
526
527 /**
528 * \param s string contents. Each string is NOT null-terminated.
529 * \param value A tensor created from OrtCreateTensor... function.
530 * \param s_len total data length, get it from OrtGetStringTensorDataLength
531 */
532 ORT_API2_STATUS(GetStringTensorContent, _In_ const OrtValue* value, _Out_writes_bytes_all_(s_len) void* s,
533 size_t s_len, _Out_writes_all_(offsets_len) size_t* offsets, size_t offsets_len);
534
535 /**
536 * Don't free the 'out' value
537 */
538 ORT_API2_STATUS(CastTypeInfoToTensorInfo, _In_ const OrtTypeInfo*,
539 _Outptr_result_maybenull_ const OrtTensorTypeAndShapeInfo** out);
540
541 /**
542 * Return OnnxType from OrtTypeInfo
543 */
544 ORT_API2_STATUS(GetOnnxTypeFromTypeInfo, _In_ const OrtTypeInfo*, _Out_ enum ONNXType* out);
545
546 /**
547 * The 'out' value should be released by calling OrtReleaseTensorTypeAndShapeInfo
548 */
549 ORT_API2_STATUS(CreateTensorTypeAndShapeInfo, _Outptr_ OrtTensorTypeAndShapeInfo** out);
550
551 ORT_API2_STATUS(SetTensorElementType, _Inout_ OrtTensorTypeAndShapeInfo*, enum ONNXTensorElementDataType type);
552
553 /**
554 * \param info Created from CreateTensorTypeAndShapeInfo() function
555 * \param dim_values An array with length of `dim_count`. Its elements can contain negative values.
556 * \param dim_count length of dim_values
557 */
558 ORT_API2_STATUS(SetDimensions, OrtTensorTypeAndShapeInfo* info, _In_ const int64_t* dim_values, size_t dim_count);
559
560 ORT_API2_STATUS(GetTensorElementType, _In_ const OrtTensorTypeAndShapeInfo*,
561 _Out_ enum ONNXTensorElementDataType* out);
562 ORT_API2_STATUS(GetDimensionsCount, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ size_t* out);
563 ORT_API2_STATUS(GetDimensions, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ int64_t* dim_values,
564 size_t dim_values_length);
565 ORT_API2_STATUS(GetSymbolicDimensions, _In_ const OrtTensorTypeAndShapeInfo* info,
566 _Out_writes_all_(dim_params_length) const char* dim_params[], size_t dim_params_length);
567
568 /**
569 * Return the number of elements specified by the tensor shape.
570 * Return a negative value if unknown (i.e., any dimension is negative.)
571 * e.g.
572 * [] -> 1
573 * [1,3,4] -> 12
574 * [2,0,4] -> 0
575 * [-1,3,4] -> -1
576 */
577 ORT_API2_STATUS(GetTensorShapeElementCount, _In_ const OrtTensorTypeAndShapeInfo* info, _Out_ size_t* out);
578
579 /**
580 * \param out Should be freed by OrtReleaseTensorTypeAndShapeInfo after use
581 */
582 ORT_API2_STATUS(GetTensorTypeAndShape, _In_ const OrtValue* value, _Outptr_ OrtTensorTypeAndShapeInfo** out);
583
584 /**
585 * Get the type information of an OrtValue
586 * \param value
587 * \param out The returned value should be freed by OrtReleaseTypeInfo after use
588 */
589 ORT_API2_STATUS(GetTypeInfo, _In_ const OrtValue* value, _Outptr_result_maybenull_ OrtTypeInfo** out);
590
591 ORT_API2_STATUS(GetValueType, _In_ const OrtValue* value, _Out_ enum ONNXType* out);
592
593 ORT_API2_STATUS(CreateMemoryInfo, _In_ const char* name1, enum OrtAllocatorType type, int id1,
594 enum OrtMemType mem_type1, _Outptr_ OrtMemoryInfo** out);
595
596 /**
597 * Convenience function for special case of CreateMemoryInfo, for the CPU allocator. Uses name = "Cpu" and id = 0.
598 */
599 ORT_API2_STATUS(CreateCpuMemoryInfo, enum OrtAllocatorType type, enum OrtMemType mem_type1,
600 _Outptr_ OrtMemoryInfo** out);
601
602 /**
603 * Test if two memory info are equal
604 * \Sets 'out' to 0 if equal, -1 if not equal
605 */
606 ORT_API2_STATUS(CompareMemoryInfo, _In_ const OrtMemoryInfo* info1, _In_ const OrtMemoryInfo* info2, _Out_ int* out);
607
608 /**
609 * Do not free the returned value
610 */
611 ORT_API2_STATUS(MemoryInfoGetName, _In_ const OrtMemoryInfo* ptr, _Out_ const char** out);
612 ORT_API2_STATUS(MemoryInfoGetId, _In_ const OrtMemoryInfo* ptr, _Out_ int* out);
613 ORT_API2_STATUS(MemoryInfoGetMemType, _In_ const OrtMemoryInfo* ptr, _Out_ OrtMemType* out);
614 ORT_API2_STATUS(MemoryInfoGetType, _In_ const OrtMemoryInfo* ptr, _Out_ OrtAllocatorType* out);
615
616 ORT_API2_STATUS(AllocatorAlloc, _Inout_ OrtAllocator* ptr, size_t size, _Outptr_ void** out);
617 ORT_API2_STATUS(AllocatorFree, _Inout_ OrtAllocator* ptr, void* p);
618 ORT_API2_STATUS(AllocatorGetInfo, _In_ const OrtAllocator* ptr, _Outptr_ const struct OrtMemoryInfo** out);
619
620 // The returned pointer doesn't have to be freed.
621 // Always returns the same instance on every invocation.
622 // Please note that this is a non-arena based allocator.
623 ORT_API2_STATUS(GetAllocatorWithDefaultOptions, _Outptr_ OrtAllocator** out);
624
625 // Override symbolic dimensions (by specific denotation strings) with actual values if known at session initialization time to enable
626 // optimizations that can take advantage of fixed values (such as memory planning, etc)
627 ORT_API2_STATUS(AddFreeDimensionOverride, _Inout_ OrtSessionOptions* options, _In_ const char* dim_denotation,
628 _In_ int64_t dim_value);
629
630 /**
631 * APIs to support non-tensor types - map and sequence.
632 * Currently only the following types are supported
633 * Note: the following types should be kept in sync with data_types.h
634 * Map types
635 * =========
636 * std::map<std::string, std::string>
637 * std::map<std::string, int64_t>
638 * std::map<std::string, float>
639 * std::map<std::string, double>
640 * std::map<int64_t, std::string>
641 * std::map<int64_t, int64_t>
642 * std::map<int64_t, float>
643 * std::map<int64_t, double>
644 *
645 * Sequence types
646 * ==============
647 * std::vector<std::string>
648 * std::vector<int64_t>
649 * std::vector<float>
650 * std::vector<double>
651 * std::vector<std::map<std::string, float>>
652 * std::vector<std::map<int64_t, float>
653 */
654
655 /**
656 * If input OrtValue represents a map, you need to retrieve the keys and values
657 * separately. Use index=0 to retrieve keys and index=1 to retrieve values.
658 * If input OrtValue represents a sequence, use index to retrieve the index'th element
659 * of the sequence.
660 */
661 ORT_API2_STATUS(GetValue, _In_ const OrtValue* value, int index, _Inout_ OrtAllocator* allocator,
662 _Outptr_ OrtValue** out);
663
664 /**
665 * Returns 2 for type map and N for sequence where N is the number of elements
666 * in the sequence.
667 */
668 ORT_API2_STATUS(GetValueCount, _In_ const OrtValue* value, _Out_ size_t* out);
669
670 /**
671 * To construct a map, use num_values = 2 and 'in' should be an arrary of 2 OrtValues
672 * representing keys and values.
673 * To construct a sequence, use num_values = N where N is the number of the elements in the
674 * sequence. 'in' should be an arrary of N OrtValues.
675 * \value_type should be either map or sequence.
676 */
677 ORT_API2_STATUS(CreateValue, _In_reads_(num_values) const OrtValue* const* in, size_t num_values,
678 enum ONNXType value_type, _Outptr_ OrtValue** out);
679
680 /**
681 * Construct OrtValue that contains a value of non-standard type created for
682 * experiments or while awaiting standardization. OrtValue in this case would contain
683 * an internal representation of the Opaque type. Opaque types are distinguished between
684 * each other by two strings 1) domain and 2) type name. The combination of the two
685 * must be unique, so the type representation is properly identified internally. The combination
686 * must be properly registered from within ORT at both compile/run time or by another API.
687 *
688 * To construct the OrtValue pass domain and type names, also a pointer to a data container
689 * the type of which must be know to both ORT and the client program. That data container may or may
690 * not match the internal representation of the Opaque type. The sizeof(data_container) is passed for
691 * verification purposes.
692 *
693 * \domain_name - domain name for the Opaque type, null terminated.
694 * \type_name - type name for the Opaque type, null terminated.
695 * \data_contianer - data to populate OrtValue
696 * \data_container_size - sizeof() of the data container. Must match the sizeof() of the expected
697 * data_container size internally.
698 */
699 ORT_API2_STATUS(CreateOpaqueValue, _In_z_ const char* domain_name, _In_z_ const char* type_name,
700 _In_ const void* data_container, size_t data_container_size, _Outptr_ OrtValue** out);
701
702 /**
703 * Fetch data from an OrtValue that contains a value of non-standard type created for
704 * experiments or while awaiting standardization.
705 * \domain_name - domain name for the Opaque type, null terminated.
706 * \type_name - type name for the Opaque type, null terminated.
707 * \data_contianer - data to populate OrtValue
708 * \data_container_size - sizeof() of the data container. Must match the sizeof() of the expected
709 * data_container size internally.
710 */
711
712 ORT_API2_STATUS(GetOpaqueValue, _In_ const char* domain_name, _In_ const char* type_name, _In_ const OrtValue* in,
713 _Out_ void* data_container, size_t data_container_size);
714
715 ORT_API2_STATUS(KernelInfoGetAttribute_float, _In_ const OrtKernelInfo* info, _In_ const char* name,
716 _Out_ float* out);
717 ORT_API2_STATUS(KernelInfoGetAttribute_int64, _In_ const OrtKernelInfo* info, _In_ const char* name,
718 _Out_ int64_t* out);
719 ORT_API2_STATUS(KernelInfoGetAttribute_string, _In_ const OrtKernelInfo* info, _In_ const char* name, _Out_ char* out,
720 _Inout_ size_t* size);
721
722 ORT_API2_STATUS(KernelContext_GetInputCount, _In_ const OrtKernelContext* context, _Out_ size_t* out);
723 ORT_API2_STATUS(KernelContext_GetOutputCount, _In_ const OrtKernelContext* context, _Out_ size_t* out);
724 ORT_API2_STATUS(KernelContext_GetInput, _In_ const OrtKernelContext* context, _In_ size_t index,
725 _Out_ const OrtValue** out);
726 ORT_API2_STATUS(KernelContext_GetOutput, _Inout_ OrtKernelContext* context, _In_ size_t index,
727 _In_ const int64_t* dim_values, size_t dim_count, _Outptr_ OrtValue** out);
728
729 ORT_CLASS_RELEASE(Env);
730 ORT_CLASS_RELEASE(Status); // nullptr for Status* indicates success
731 ORT_CLASS_RELEASE(MemoryInfo);
732 ORT_CLASS_RELEASE(Session); //Don't call OrtReleaseSession from Dllmain (because session owns a thread pool)
733 ORT_CLASS_RELEASE(Value);
734 ORT_CLASS_RELEASE(RunOptions);
735 ORT_CLASS_RELEASE(TypeInfo);
736 ORT_CLASS_RELEASE(TensorTypeAndShapeInfo);
737 ORT_CLASS_RELEASE(SessionOptions);
738 ORT_CLASS_RELEASE(CustomOpDomain);
739
740 // End of Version 1 - DO NOT MODIFY ABOVE (see above text for more information)
741
742 // Version 2 - In development, feel free to add/remove/rearrange here
743
744 /**
745 * GetDenotationFromTypeInfo
746 * This api augments OrtTypeInfo to return denotations on the type.
747 * This is used by WinML to determine if an input/output is intended to be an Image or a Tensor.
748 */
749 ORT_API2_STATUS(GetDenotationFromTypeInfo, _In_ const OrtTypeInfo*, _Out_ const char** const denotation,
750 _Out_ size_t* len);
751
752 // OrtTypeInfo Casting methods
753
754 /**
755 * CastTypeInfoToMapTypeInfo
756 * This api augments OrtTypeInfo to return an OrtMapTypeInfo when the type is a map.
757 * The OrtMapTypeInfo has additional information about the map's key type and value type.
758 * This is used by WinML to support model reflection APIs.
759 * This is used by WinML to support model reflection APIs.
760 *
761 * Don't free the 'out' value
762 */
763 ORT_API2_STATUS(CastTypeInfoToMapTypeInfo, _In_ const OrtTypeInfo* type_info,
764 _Outptr_result_maybenull_ const OrtMapTypeInfo** out);
765
766 /**
767 * CastTypeInfoToSequenceTypeInfo
768 * This api augments OrtTypeInfo to return an OrtSequenceTypeInfo when the type is a sequence.
769 * The OrtSequenceTypeInfo has additional information about the sequence's element type.
770 * This is used by WinML to support model reflection APIs.
771 *
772 * Don't free the 'out' value
773 */
774 ORT_API2_STATUS(CastTypeInfoToSequenceTypeInfo, _In_ const OrtTypeInfo* type_info,
775 _Outptr_result_maybenull_ const OrtSequenceTypeInfo** out);
776
777 // OrtMapTypeInfo Accessors
778
779 /**
780 * GetMapKeyType
781 * This api augments get the key type of a map. Key types are restricted to being scalar types and use ONNXTensorElementDataType.
782 * This is used by WinML to support model reflection APIs.
783 */
784 ORT_API2_STATUS(GetMapKeyType, _In_ const OrtMapTypeInfo* map_type_info, _Out_ enum ONNXTensorElementDataType* out);
785
786 /**
787 * GetMapValueType
788 * This api augments get the value type of a map.
789 */
790 ORT_API2_STATUS(GetMapValueType, _In_ const OrtMapTypeInfo* map_type_info, _Outptr_ OrtTypeInfo** type_info);
791
792 // OrtSequenceTypeInfo Accessors
793
794 /**
795 * GetSequenceElementType
796 * This api augments get the element type of a sequence.
797 * This is used by WinML to support model reflection APIs.
798 */
799 ORT_API2_STATUS(GetSequenceElementType, _In_ const OrtSequenceTypeInfo* sequence_type_info,
800 _Outptr_ OrtTypeInfo** type_info);
801
802 ORT_CLASS_RELEASE(MapTypeInfo);
803 ORT_CLASS_RELEASE(SequenceTypeInfo);
804
805 /**
806 * \param out is set to a null terminated string allocated using 'allocator'. The caller is responsible for freeing it.
807 * Profiling is turned ON automatically if enabled for the particular session by invoking EnableProfiling()
808 * on the SessionOptions instance used to create the session.
809 */
810 ORT_API2_STATUS(SessionEndProfiling, _In_ OrtSession* sess, _Inout_ OrtAllocator* allocator, _Outptr_ char** out);
811
812 /**
813 * \param out is a pointer to the newly created object. The pointer should be freed by calling ReleaseModelMetadata after use.
814 */
815 ORT_API2_STATUS(SessionGetModelMetadata, _In_ const OrtSession* sess, _Outptr_ OrtModelMetadata** out);
816
817 /**
818 * \param value is set to a null terminated string allocated using 'allocator'. The caller is responsible for freeing it.
819 */
820 ORT_API2_STATUS(ModelMetadataGetProducerName, _In_ const OrtModelMetadata* model_metadata,
821 _Inout_ OrtAllocator* allocator, _Outptr_ char** value);
822 ORT_API2_STATUS(ModelMetadataGetGraphName, _In_ const OrtModelMetadata* model_metadata,
823 _Inout_ OrtAllocator* allocator, _Outptr_ char** value);
824 ORT_API2_STATUS(ModelMetadataGetDomain, _In_ const OrtModelMetadata* model_metadata, _Inout_ OrtAllocator* allocator,
825 _Outptr_ char** value);
826 ORT_API2_STATUS(ModelMetadataGetDescription, _In_ const OrtModelMetadata* model_metadata,
827 _Inout_ OrtAllocator* allocator, _Outptr_ char** value);
828 /**
829 * \param value is set to a null terminated string allocated using 'allocator'. The caller is responsible for freeing it.
830 * 'value' will be a nullptr if the given key is not found in the custom metadata map.
831 */
832 ORT_API2_STATUS(ModelMetadataLookupCustomMetadataMap, _In_ const OrtModelMetadata* model_metadata,
833 _Inout_ OrtAllocator* allocator, _In_ const char* key, _Outptr_result_maybenull_ char** value);
834
835 ORT_API2_STATUS(ModelMetadataGetVersion, _In_ const OrtModelMetadata* model_metadata, _Out_ int64_t* value);
836
837 ORT_CLASS_RELEASE(ModelMetadata);
838
839 /*
840 * Creates an environment with global threadpools that will be shared across sessions.
841 * Use this in conjunction with DisablePerSessionThreads API or else the session will use
842 * its own thread pools.
843 */
844 ORT_API2_STATUS(CreateEnvWithGlobalThreadPools, OrtLoggingLevel logging_level, _In_ const char* logid,
845 _In_ const OrtThreadingOptions* t_options, _Outptr_ OrtEnv** out);
846
847 /*
848 * Calling this API will make the session use the global threadpools shared across sessions.
849 * This API should be used in conjunction with CreateEnvWithGlobalThreadPools API.
850 */
851 ORT_API2_STATUS(DisablePerSessionThreads, _Inout_ OrtSessionOptions* options);
852
853 ORT_API2_STATUS(CreateThreadingOptions, _Outptr_ OrtThreadingOptions** out);
854
855 ORT_CLASS_RELEASE(ThreadingOptions);
856
857 /**
858 * \param num_keys contains the number of keys in the custom metadata map
859 * \param keys is an array of null terminated strings (array count = num_keys) allocated using 'allocator'.
860 * The caller is responsible for freeing each string and the pointer array.
861 * 'keys' will be a nullptr if custom metadata map is empty.
862 */
863 ORT_API2_STATUS(ModelMetadataGetCustomMetadataMapKeys, _In_ const OrtModelMetadata* model_metadata,
864 _Inout_ OrtAllocator* allocator, _Outptr_result_buffer_maybenull_(*num_keys) char*** keys, _Out_ int64_t* num_keys);
865
866 // Override symbolic dimensions (by specific name strings) with actual values
867 // if known at session initialization time to enable optimizations that can
868 // take advantage of fixed values (such as memory planning, etc)
869 ORT_API2_STATUS(AddFreeDimensionOverrideByName,
870 _Inout_ OrtSessionOptions* options, _In_ const char* dim_name,
871 _In_ int64_t dim_value);
872
873 /**
874 * \param out_ptr will hold a pointer to the array of char *
875 * representing available providers.
876 * \param provider_length is a pointer to an int variable where
877 * the number of available providers will be added.
878 * The caller is responsible for freeing each char * and the pointer
879 * array by calling ReleaseAvailableProviders().
880 */
881 ORT_API2_STATUS(GetAvailableProviders, _Outptr_ char*** out_ptr,
882 _In_ int* provider_length);
883
884 /**
885 * \param ptr is the pointer to an array of available providers you
886 * get after calling GetAvailableProviders().
887 * \param providers_length is the number of available providers.
888 */
889 ORT_API2_STATUS(ReleaseAvailableProviders, _In_ char** ptr,
890 _In_ int providers_length);
891
892 /**
893 * \param value - A tensor created from OrtCreateTensor... function.
894 * \param index - index of string tensor element, length of element at index will be returned.
895 * \param out - number of UTF-8 bytes that the string contains
896 */
897 ORT_API2_STATUS(GetStringTensorElementLength, _In_ const OrtValue* value, size_t index, _Out_ size_t* out);
898
899 /**
900 * \param s string element contents in UTF-8 encoding. The string is NOT null-terminated.
901 * \param value A tensor created from OrtCreateTensor... function.
902 * \param s_len element length, get it from OrtGetStringTensorElementLength.
903 * \param index offset of element of tensor to return.
904 */
905 ORT_API2_STATUS(GetStringTensorElement, _In_ const OrtValue* value, size_t s_len, size_t index, _Out_writes_bytes_all_(s_len) void* s);
906
907 /**
908 * \param value - A tensor created from OrtCreateTensor... function.
909 * \param s - A null terminated UTF-8 encoded string.
910 * \param index - index of string tensor element to fill
911 */
912 ORT_API2_STATUS(FillStringTensorElement, _Inout_ OrtValue* value, _In_ const char* s, size_t index);
913
914 /**
915 * Set a single session configuration entry as a pair of strings
916 * If a configuration with same key exists, this will overwrite the configuration with the given config_value
917 * \param config_key A null terminated string representation of the config key
918 * \param config_value A null terminated string representation of the config value
919 * The config_key and the format of config_value are defined in onnxruntime_session_options_config_keys.h
920 */
921 ORT_API2_STATUS(AddSessionConfigEntry, _Inout_ OrtSessionOptions* options,
922 _In_z_ const char* config_key, _In_z_ const char* config_value);
923
924 /**
925 * \param sess valid OrtSession instance
926 * \param mem_info - valid OrtMemoryInfo instance
927 * \param - out a ptr to a new instance of OrtAllocator according to the spec within mem_info
928 * if successful
929 * \return OrtStatus or nullptr if successful
930 */
931 ORT_API2_STATUS(CreateAllocator, _In_ const OrtSession* sess, _In_ const OrtMemoryInfo* mem_info,
932 _Outptr_ OrtAllocator** out);
933
934 // Release instance of OrtAllocator obtained from CreateAllocator API
935 ORT_CLASS_RELEASE(Allocator);
936
937 ORT_API2_STATUS(RunWithBinding, _Inout_ OrtSession* sess, _In_opt_ const OrtRunOptions* run_options, _In_ const OrtIoBinding* binding_ptr);
938
939 // Creates an IoBinding instance that allows one to bind pre-allocated OrtValues
940 // to input names. Thus if you want to use a raw on device buffer as input or output
941 // you can avoid extra copy during runtime.
942 ORT_API2_STATUS(CreateIoBinding, _Inout_ OrtSession* sess, _Outptr_ OrtIoBinding** out);
943
944 // Release instance or OrtIoBinding obtained from CreateIoBinding API
945 ORT_CLASS_RELEASE(IoBinding);
946
947 /**
948 * The function will bind the OrtValue to a specified input name.
949 * The OrtValue must be a Tensor. ORT would use that value in place of input for the specified name.
950 * \param binding_ptr - an instance of OrtIoBinding created by CreateIoBinding()
951 * \param name - name for the model input
952 * \param val_ptr - OrtValue of Tensor type.
953 * \return OrtStatus instance on error which the caller is responsible to free or nullptr on success
954 */
955 ORT_API2_STATUS(BindInput, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtValue* val_ptr);
956
957 /**
958 * The function will bind the OrtValue to the specified output name.
959 * The OrtValue must be a Tensor. ORT would use that value in place of output for the specified name.
960 *
961 * \param binding_ptr - an instance of OrtIoBinding created by CreateIoBinding()
962 * \param name - name for the model output
963 * \param val_ptr - OrtValue of Tensor type.
964 * \return OrtStatus instance on error which the caller is responsible to free or nullptr on success
965 */
966 ORT_API2_STATUS(BindOutput, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtValue* val_ptr);
967
968 /**
969 * The function will bind the OrtValue to a device which specification is contained within OrtMemoryInfo
970 * You can either create an instance of OrtMemoryInfo with a device id or obtain one from the allocator that you are created/using
971 * This is useful when one or more outputs have dynamic shapes and, it is hard to pre-allocated and bind a chunk of
972 * memory within OrtValue ahead of time.
973 *
974 * \param binding_ptr - an instance of OrtIoBinding created by CreateIoBinding()
975 * \param name - name for the model output
976 * \param mem_info_ptr - OrtMemoryInfo
977 * \return OrtStatus instance on error which the caller is responsible to free or nullptr on success
978 */
979 ORT_API2_STATUS(BindOutputToDevice, _Inout_ OrtIoBinding* binding_ptr, _In_ const char* name, _In_ const OrtMemoryInfo* val_ptr);
980
981 /**
982 * The function returns the names of the outputs in the order they were bound. This is useful after running the model
983 * with bound outputs because the returned names are in order in which output OrtValues are returned. This API is optional
984 * to use. If you knew the order of outputs and its names you used for binding you would not need to use this API.
985 *
986 * \param binding_ptr - a ptr to an instance of OrtIoBinding created obtained from CreateIoBinding()
987 * \param allocator - a ptr to an instance of OrtAllocator obtained with CreateAllocator() or GetAllocatorWithDefaultOptions()
988 * the specified allocator will be used to allocate continuous buffers for output strings and lengths.
989 * \param buffer - pointer to a continuous buffer of non-zero terminated UTF-8 encoded strings. The number of strings stored is returned count parameter.
990 * this buffer will be allocated with the specified allocator and must be freed after it is no longer needed.
991 * \param lengths - a pointer to a continuous buffer of size_t lengths of strings returned in the buffer. The number of items is returned
992 * in the count. This buffer is allocated with the specified allocator and must be freed after it is no longer needed.
993 * \para count - is the number of strings returned. If the instance of OrtIoBiding has no bound outputs, zero is returned,
994 * no memory allocation is performed and buffer and lengths are nullptr on return.
995 */
996 ORT_API2_STATUS(GetBoundOutputNames, _In_ const OrtIoBinding* binding_ptr, _In_ OrtAllocator* allocator,
997 _Out_ char** buffer, _Out_writes_all_(count) size_t** lengths, _Out_ size_t* count);
998
999 /**
1000 * The function returns an array of pointers to individually allocated OrtValues that contain results of a model execution with RunWithBinding()
1001 * The array contains the same number of OrtValues and they are in the same order as they were bound with BindOutput()
1002 * or BindOutputToDevice().
1003 * The returned OrtValues must be individually released after they are no longer needed.
1004 * The array is allocated using the specified instance of the allocator and must be freed using the same allocator after
1005 * all the OrtValues contained therein are individually released.
1006 *
1007 * \param binding_ptr - instance of OrtIoBidning
1008 * \param allocator - instance of allocator to allocate output array
1009 * \param output - pointer to the allocated buffer. Returns nullptr if no outputs.
1010 * \param output_count - pointer to the number of OrtValues returned. Zero if no outputs.
1011 */
1012 ORT_API2_STATUS(GetBoundOutputValues, _In_ const OrtIoBinding* binding_ptr, _In_ OrtAllocator* allocator,
1013 _Out_writes_all_(output_count) OrtValue*** output, _Out_ size_t* output_count);
1014
1015 /** Clears any previously specified bindings for inputs/outputs
1016 */
1017 void(ORT_API_CALL* ClearBoundInputs)(_Inout_ OrtIoBinding* binding_ptr) NO_EXCEPTION ORT_ALL_ARGS_NONNULL;
1018 void(ORT_API_CALL* ClearBoundOutputs)(_Inout_ OrtIoBinding* binding_ptr) NO_EXCEPTION ORT_ALL_ARGS_NONNULL;
1019
1020 /**
1021 * Provides element-level access into a tensor.
1022 * \param location_values a pointer to an array of index values that specify an element's location in the tensor data blob
1023 * \param location_values_count length of location_values
1024 * \param out a pointer to the element specified by location_values
1025 * e.g.
1026 * Given a tensor with overall shape [3,224,224], an element at
1027 * location [2,150,128] can be accessed directly.
1028 *
1029 * This function only works for numeric tensors.
1030 * This is a no-copy method whose pointer is only valid until the backing OrtValue is free'd.
1031 */
1032 ORT_API2_STATUS(TensorAt, _Inout_ OrtValue* value, const int64_t* location_values, size_t location_values_count, _Outptr_ void** out);
1033
1034 /**
1035 * Creates an allocator instance and registers it with the env to enable
1036 * sharing between multiple sessions that use the same env instance.
1037 * Lifetime of the created allocator will be valid for the duration of the environment.
1038 * Returns an error if an allocator with the same OrtMemoryInfo is already registered.
1039 * \param mem_info must be non-null.
1040 * \param arena_cfg if nullptr defaults will be used.
1041 * See docs/C_API.md for details.
1042 */
1043 ORT_API2_STATUS(CreateAndRegisterAllocator, _Inout_ OrtEnv* env, _In_ const OrtMemoryInfo* mem_info,
1044 _In_ const OrtArenaCfg* arena_cfg);
1045
1046 /**
1047 * Set the language projection for collecting telemetry data when Env is created
1048 * \param projection the source projected language.
1049 */
1050 ORT_API2_STATUS(SetLanguageProjection, _In_ const OrtEnv* ort_env, _In_ OrtLanguageProjection projection);
1051
1052 /**
1053 * On some platforms, this timer may not be as precise as nanoseconds
1054 * For instance, on Windows and MacOS, the precision will be ~100ns
1055 * \param out is set to the nanoseconds of profiling's start time
1056 */
1057 ORT_API2_STATUS(SessionGetProfilingStartTimeNs, _In_ const OrtSession* sess, _Outptr_ uint64_t* out);
1058
1059 /**
1060 * Use this API to configure the global thread pool options to be used in the call to CreateEnvWithGlobalThreadPools.
1061 * A value of 0 means ORT will pick the default.
1062 * A value of 1 means the invoking thread will be used; no threads will be created in the thread pool.
1063 */
1064 ORT_API2_STATUS(SetGlobalIntraOpNumThreads, _Inout_ OrtThreadingOptions* tp_options, int intra_op_num_threads);
1065 ORT_API2_STATUS(SetGlobalInterOpNumThreads, _Inout_ OrtThreadingOptions* tp_options, int inter_op_num_threads);
1066
1067 /**
1068 * Use this API to configure the global thread pool options to be used in the call to CreateEnvWithGlobalThreadPools.
1069 * Allow spinning of thread pools when their queues are empty. This API will set the value for both
1070 * inter_op and intra_op threadpools.
1071 * \param allow_spinning valid values are 1 and 0.
1072 * 1: threadpool will spin to wait for queue to become non-empty, 0: it won't spin.
1073 * Prefer a value of 0 if your CPU usage is very high.
1074 */
1075 ORT_API2_STATUS(SetGlobalSpinControl, _Inout_ OrtThreadingOptions* tp_options, int allow_spinning);
1076
1077 /**
1078 * Add a pre-allocated initializer to a session. If a model contains an initializer with a name
1079 * that is same as the name passed to this API call, ORT will use this initializer instance
1080 * instead of deserializing one from the model file. This is useful when you want to share
1081 * the same initializer across sessions.
1082 * \param name name of the initializer
1083 * \param val OrtValue containing the initializer. Lifetime of 'val' and the underlying initializer buffer must be
1084 * managed by the user (created using the CreateTensorWithDataAsOrtValue API) and it must outlive the session object
1085 * to which it is added.
1086 */
1087 ORT_API2_STATUS(AddInitializer, _Inout_ OrtSessionOptions* options, _In_z_ const char* name,
1088 _In_ const OrtValue* val);
1089
1090 /**
1091 * Creates a custom environment with global threadpools and logger that will be shared across sessions.
1092 * Use this in conjunction with DisablePerSessionThreads API or else the session will use
1093 * its own thread pools.
1094 *
1095 * \param out should be freed by `OrtReleaseEnv` after use
1096 */
1097 ORT_API2_STATUS(CreateEnvWithCustomLoggerAndGlobalThreadPools, OrtLoggingFunction logging_function, _In_opt_ void* logger_param, OrtLoggingLevel logging_level,
1098 _In_ const char* logid, _In_ const struct OrtThreadingOptions* tp_options, _Outptr_ OrtEnv** out);
1099
1100 /**
1101 * Append CUDA execution provider to the session options
1102 * If CUDA is not available (due to a non cuda enabled build), this function will return failure.
1103 */
1104 ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_CUDA,
1105 _In_ OrtSessionOptions* options, _In_ const OrtCUDAProviderOptions* cuda_options);
1106
1107 /**
1108 * Append OpenVINO execution provider to the session options
1109 * If OpenVINO is not available (due to the OpenVINO provider shared library or its dependencies not being installed), this function will fail.
1110 */
1111 ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_OpenVINO,
1112 _In_ OrtSessionOptions* options, _In_ const OrtOpenVINOProviderOptions* provider_options);
1113
1114 /**
1115 * Use this API to configure the global thread pool options to be used in the call to CreateEnvWithGlobalThreadPools.
1116 * When this API is called, flush-to-zero and denormal-as-zero are applied to threads in both intra and inter global thread pool.
1117 * Note that an alternative way not using this option at runtime is to train and export a model without denormals
1118 * and that's recommended because turning this option on may hurt model accuracy.
1119 */
1120 ORT_API2_STATUS(SetGlobalDenormalAsZero, _Inout_ OrtThreadingOptions* tp_options);
1121
1122 /**
1123 * Use this API to create the configuration of an arena that can eventually be used to define
1124 * an arena based allocator's behavior
1125 * \param max_mem - use 0 to allow ORT to choose the default
1126 * \param arena_extend_strategy - use -1 to allow ORT to choose the default, 0 = kNextPowerOfTwo, 1 = kSameAsRequested
1127 * \param initial_chunk_size_bytes - use -1 to allow ORT to choose the default
1128 * \param max_dead_bytes_per_chunk - use -1 to allow ORT to choose the default
1129 * \param out - a pointer to an OrtArenaCfg instance
1130 * \return a nullptr in case of success or a pointer to an OrtStatus instance in case of failure
1131 * See docs/C_API.md for details on what the following parameters mean and how to choose these values
1132 */
1133 ORT_API2_STATUS(CreateArenaCfg, _In_ size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes,
1134 int max_dead_bytes_per_chunk, _Outptr_ OrtArenaCfg** out);
1135
1136 ORT_CLASS_RELEASE(ArenaCfg);
1137};
1138
1139/*
1140 * Steps to use a custom op:
1141 * 1 Create an OrtCustomOpDomain with the domain name used by the custom ops
1142 * 2 Create an OrtCustomOp structure for each op and add them to the domain
1143 * 3 Call OrtAddCustomOpDomain to add the custom domain of ops to the session options
1144*/
1145#define OrtCustomOpApi OrtApi
1146
1147/*
1148 * The OrtCustomOp structure defines a custom op's schema and its kernel callbacks. The callbacks are filled in by
1149 * the implementor of the custom op.
1150*/
1151struct OrtCustomOp {
1152 uint32_t version; // Initialize to ORT_API_VERSION
1153
1154 // This callback creates the kernel, which is a user defined parameter that is passed to the Kernel* callbacks below.
1155 void*(ORT_API_CALL* CreateKernel)(_In_ const struct OrtCustomOp* op, _In_ const OrtApi* api,
1156 _In_ const OrtKernelInfo* info);
1157
1158 // Returns the name of the op
1159 const char*(ORT_API_CALL* GetName)(_In_ const struct OrtCustomOp* op);
1160
1161 // Returns the type of the execution provider, return nullptr to use CPU execution provider
1162 const char*(ORT_API_CALL* GetExecutionProviderType)(_In_ const struct OrtCustomOp* op);
1163
1164 // Returns the count and types of the input & output tensors
1165 ONNXTensorElementDataType(ORT_API_CALL* GetInputType)(_In_ const struct OrtCustomOp* op, _In_ size_t index);
1166 size_t(ORT_API_CALL* GetInputTypeCount)(_In_ const struct OrtCustomOp* op);
1167 ONNXTensorElementDataType(ORT_API_CALL* GetOutputType)(_In_ const struct OrtCustomOp* op, _In_ size_t index);
1168 size_t(ORT_API_CALL* GetOutputTypeCount)(_In_ const struct OrtCustomOp* op);
1169
1170 // Op kernel callbacks
1171 void(ORT_API_CALL* KernelCompute)(_In_ void* op_kernel, _In_ OrtKernelContext* context);
1172 void(ORT_API_CALL* KernelDestroy)(_In_ void* op_kernel);
1173};
1174
1175/*
1176 * END EXPERIMENTAL
1177*/
1178
1179#ifdef __cplusplus
1180}
1181#endif