GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/onnxruntime-extensions

Public

mirrored from https://github.com/microsoft/onnxruntime-extensionsAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
v0.3.0

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

CMakeLists.txt

303lines · modecode

RawDownload
Latest commit unavailable.
1cmake_minimum_required(VERSION 3.16.0)
2project(onnxruntime_extensions LANGUAGES C CXX)
3# set(CMAKE_VERBOSE_MAKEFILE ON)
4
5if(NOT CMAKE_BUILD_TYPE)
6 message(STATUS "Build type not set - using RelWithDebInfo")
7 set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING "Choose build type: Debug Release RelWithDebInfo." FORCE)
8endif()
9
10
11set(CPACK_PACKAGE_NAME "onnxruntime_extensions")
12set(CPACK_PACKAGE_VERSION_MAJOR "0")
13set(CPACK_PACKAGE_VERSION_MINOR "3")
14set(CPACK_PACKAGE_VERSION_PATCH "0")
15set(VERSION ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH})
16
17
18set(CMAKE_CXX_STANDARD 17)
19set(CMAKE_CXX_STANDARD_REQUIRED ON)
20set(CMAKE_CXX_EXTENSIONS OFF)
21include(CheckCXXCompilerFlag)
22include(CheckLanguage)
23
24option(CC_OPTIMIZE "Allow compiler optimizations, Set to OFF to disable" ON)
25option(OCOS_ENABLE_PYTHON "Enable Python component building" OFF)
26option(OCOS_ENABLE_CTEST "Enable C++ test" OFF)
27option(OCOS_ENABLE_TF_STRING "Enable String Operator Set" ON)
28option(OCOS_ENABLE_GPT2_TOKENIZER "Enable the GPT2 tokenizer building" ON)
29option(OCOS_ENABLE_SPM_TOKENIZER "Enable the SentencePiece tokenizer building" ON)
30option(OCOS_ENABLE_BERT_TOKENIZER "Enable the BertTokenizer building" ON)
31option(OCOS_ENABLE_MATH "Enable the math tensor operators building" ON)
32option(OCOS_ENABLE_STATIC_LIB "Enable generating static library" OFF)
33
34
35if(NOT CC_OPTIMIZE)
36 message("!!!THE COMPILER OPTIMIZATION HAS BEEN DISABLED, DEBUG-ONLY!!!")
37 string(REGEX REPLACE "([\-\/]O[123])" "" CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO}")
38 string(REGEX REPLACE "([\-\/]O[123])" "" CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}")
39 string(REGEX REPLACE "([\-\/]O[123])" "" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}")
40 string(REGEX REPLACE "([\-\/]O[123])" "" CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}")
41
42 if (NOT WIN32)
43 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0")
44 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O0")
45 else()
46 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Od")
47 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /Od")
48 endif()
49endif()
50
51# Build the libraries with -fPIC
52set(CMAKE_POSITION_INDEPENDENT_CODE ON)
53
54set(CMAKE_FIND_FRAMEWORK NEVER CACHE STRING "...")
55if(NOT "${CMAKE_FIND_FRAMEWORK}" STREQUAL "NEVER")
56 message(FATAL_ERROR "CMAKE_FIND_FRAMEWORK is not NEVER")
57endif()
58
59# External dependencies
60list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/externals)
61include(FetchContent)
62if (OCOS_ENABLE_TF_STRING)
63 if (NOT TARGET re2::re2)
64 set(RE2_BUILD_TESTING OFF CACHE INTERNAL "")
65 message(STATUS "Fetch googlere2")
66 include(googlere2)
67 FetchContent_GetProperties(googlere2)
68 endif()
69
70 if (NOT TARGET farmhash)
71 message(STATUS "Fetch farmhash")
72 include(farmhash)
73 FetchContent_GetProperties(farmhash)
74 endif()
75
76 if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten")
77 set_property(TARGET re2 PROPERTY COMPILE_OPTIONS )
78 endif()
79endif()
80
81file(GLOB TARGET_SRC "operators/*.cc" "operators/*.h")
82if (OCOS_ENABLE_TF_STRING)
83 file(GLOB TARGET_SRC_KERNELS "operators/text/*.cc" "operators/text/*.h*")
84 file(GLOB TARGET_SRC_HASH "${farmhash_SOURCE_DIR}/src/farmhash.*")
85 list(APPEND TARGET_SRC ${TARGET_SRC_KERNELS} ${TARGET_SRC_HASH})
86endif()
87
88if (OCOS_ENABLE_MATH)
89 set(DLIB_NO_GUI_SUPPORT ON CACHE INTERNAL "")
90 set(DLIB_USE_CUDA OFF CACHE INTERNAL "")
91 set(DLIB_USE_LAPACK OFF CACHE INTERNAL "")
92 set(DLIB_USE_BLAS OFF CACHE INTERNAL "")
93 include(dlib)
94 file(GLOB TARGET_SRC_MATH "operators/math/*.cc" "operators/math/*.h*")
95 list(APPEND TARGET_SRC ${TARGET_SRC_MATH})
96endif()
97
98if (OCOS_ENABLE_GPT2_TOKENIZER)
99 # GPT2
100 if (NOT TARGET nlohmann_json)
101 set(JSON_BuildTests OFF CACHE INTERNAL "")
102 message(STATUS "Fetch json")
103 include(json)
104 endif()
105
106 file(GLOB tok_TARGET_SRC "operators/tokenizer/gpt*.cc" "operators/tokenizer/unicode*.*")
107 list(APPEND TARGET_SRC ${tok_TARGET_SRC})
108endif()
109
110if (OCOS_ENABLE_SPM_TOKENIZER)
111 # SentencePiece
112 set(SPM_ENABLE_TCMALLOC OFF CACHE INTERNAL "")
113 set(SPM_ENABLE_SHARED OFF CACHE INTERNAL "")
114 message(STATUS "Fetch sentencepiece")
115 include(sentencepieceproject)
116 file(GLOB stpiece_TARGET_SRC "operators/tokenizer/sentencepiece/*.cc" "operators/tokenizer/sentencepiece*")
117 list(REMOVE_ITEM stpiece_TARGET_SRC INCLUDE REGEX ".*((spm)|(train)).*")
118 list(APPEND TARGET_SRC ${stpiece_TARGET_SRC})
119endif()
120
121if (OCOS_ENABLE_BERT_TOKENIZER)
122 # Bert
123 file(GLOB bert_TARGET_SRC "operators/tokenizer/wordpiece*.*")
124 list(APPEND TARGET_SRC ${bert_TARGET_SRC})
125endif()
126
127add_compile_options("$<$<C_COMPILER_ID:MSVC>:/utf-8>")
128add_compile_options("$<$<CXX_COMPILER_ID:MSVC>:/utf-8>")
129add_library(ocos_operators STATIC ${TARGET_SRC})
130target_include_directories(ocos_operators PUBLIC operators/tokenizer)
131
132set(ocos_libraries ocos_operators)
133if (OCOS_ENABLE_TF_STRING)
134 list(APPEND ocos_libraries re2)
135endif()
136
137target_include_directories(ocos_operators PUBLIC
138 ${PROJECT_SOURCE_DIR}/includes
139 ${PROJECT_SOURCE_DIR}/includes/onnxruntime
140 ${PROJECT_SOURCE_DIR}/operators)
141
142set(OCOS_COMPILE_DEFINITIONS "")
143
144if (OCOS_ENABLE_TF_STRING)
145 target_include_directories(ocos_operators PUBLIC
146 ${googlere2_SOURCE_DIR}
147 ${farmhash_SOURCE_DIR}/src)
148 list(APPEND OCOS_COMPILE_DEFINITIONS ENABLE_TF_STRING)
149endif()
150
151if (OCOS_ENABLE_MATH)
152 target_include_directories(ocos_operators PUBLIC ${dlib_SOURCE_DIR})
153 list(APPEND OCOS_COMPILE_DEFINITIONS ENABLE_MATH)
154 # The dlib matrix implementation is all in the headers, no library compiling needed.
155 if (WIN32)
156 list(APPEND ocos_libraries dlib::dlib)
157 endif()
158endif()
159
160if (OCOS_ENABLE_GPT2_TOKENIZER)
161 # GPT2
162 target_include_directories(ocos_operators PRIVATE ${json_SOURCE_DIR}/single_include)
163 list(APPEND OCOS_COMPILE_DEFINITIONS ENABLE_GPT2_TOKENIZER)
164 list(APPEND ocos_libraries nlohmann_json::nlohmann_json)
165endif()
166
167if (OCOS_ENABLE_SPM_TOKENIZER)
168 # SentencePiece
169 target_include_directories(ocos_operators PUBLIC ${sentencepieceproject_INCLUDE_DIRS})
170 list(APPEND OCOS_COMPILE_DEFINITIONS ENABLE_SPM_TOKENIZER)
171 list(APPEND ocos_libraries sentencepiece-static)
172endif()
173
174if (OCOS_ENABLE_TF_STRING)
175 target_compile_definitions(ocos_operators PRIVATE
176 NOMINMAX
177 FARMHASH_NO_BUILTIN_EXPECT)
178endif()
179
180if (OCOS_ENABLE_BERT_TOKENIZER)
181 list(APPEND OCOS_COMPILE_DEFINITIONS ENABLE_BERT_TOKENIZER)
182endif()
183
184target_compile_definitions(ocos_operators PRIVATE ${OCOS_COMPILE_DEFINITIONS})
185
186file(GLOB shared_TARGET_SRC "shared/*.cc" "shared/*.h")
187if(OCOS_ENABLE_PYTHON)
188 set(Python3_FIND_REGISTRY NEVER CACHE STRING "...")
189 if(NOT "${Python3_FIND_REGISTRY}" STREQUAL "NEVER")
190 message(FATAL_ERROR "Python3_FIND_REGISTRY is not NEVER")
191 endif()
192 find_package(Python3 COMPONENTS Interpreter Development.Module NumPy)
193 if (WIN32)
194 list(APPEND shared_TARGET_SRC "${PROJECT_SOURCE_DIR}/onnxruntime_extensions/ortcustomops.def")
195 endif()
196
197 file(GLOB TARGET_SRC_PYOPS "pyop/*.cc" "pyop/*.h")
198 add_library(ortcustomops SHARED ${TARGET_SRC_PYOPS} ${shared_TARGET_SRC})
199 list(APPEND OCOS_COMPILE_DEFINITIONS PYTHON_OP_SUPPORT)
200 # building static lib has higher priority
201elseif(OCOS_ENABLE_STATIC_LIB)
202 add_library(ortcustomops STATIC ${shared_TARGET_SRC})
203elseif(CMAKE_SYSTEM_NAME STREQUAL "Emscripten")
204 add_executable(ortcustomops ${shared_TARGET_SRC})
205 set_target_properties(ortcustomops PROPERTIES LINK_FLAGS " \
206 -s WASM=1 \
207 -s NO_EXIT_RUNTIME=0 \
208 -s ALLOW_MEMORY_GROWTH=1 \
209 -s SAFE_HEAP=0 \
210 -s MODULARIZE=1 \
211 -s SAFE_HEAP_LOG=0 \
212 -s STACK_OVERFLOW_CHECK=0 \
213 -s EXPORT_ALL=0 \
214 -s VERBOSE=0 \
215 --no-entry")
216 if (CMAKE_BUILD_TYPE STREQUAL "Debug")
217 set_property(TARGET ortcustomops APPEND_STRING PROPERTY LINK_FLAGS " -s ASSERTIONS=1 -s DEMANGLE_SUPPORT=1")
218 else()
219 set_property(TARGET ortcustomops APPEND_STRING PROPERTY LINK_FLAGS " -s ASSERTIONS=0 -s DEMANGLE_SUPPORT=0")
220 endif()
221else()
222 list(APPEND shared_TARGET_SRC "${PROJECT_SOURCE_DIR}/shared/ortcustomops.def")
223 add_library(ortcustomops SHARED ${shared_TARGET_SRC})
224endif()
225
226target_compile_definitions(ortcustomops PRIVATE ${OCOS_COMPILE_DEFINITIONS})
227target_link_libraries(ortcustomops PRIVATE ${ocos_libraries})
228
229if(OCOS_ENABLE_PYTHON)
230 message(STATUS "Fetch pybind11")
231 include(pybind11)
232 target_include_directories(ortcustomops PRIVATE
233 $<TARGET_PROPERTY:Python3::Module,INTERFACE_INCLUDE_DIRECTORIES>
234 $<TARGET_PROPERTY:Python3::NumPy,INTERFACE_INCLUDE_DIRECTORIES>
235 ${pybind11_INCLUDE_DIRS}
236 )
237
238 target_compile_definitions(ortcustomops PRIVATE
239 $<TARGET_PROPERTY:Python3::Module,INTERFACE_COMPILE_DEFINITIONS>)
240
241 target_link_libraries(ortcustomops PRIVATE Python3::Module)
242
243 if(NOT "${OCOS_EXTENTION_NAME}" STREQUAL "")
244 if(NOT WIN32)
245 set_target_properties(ortcustomops PROPERTIES
246 LIBRARY_OUTPUT_NAME ${OCOS_EXTENTION_NAME}
247 PREFIX ""
248 SUFFIX "")
249 endif()
250 endif()
251endif()
252
253
254# test section
255if (OCOS_ENABLE_CTEST)
256 # Enable CTest
257 find_library(ONNXRUNTIME onnxruntime HINTS "${ONNXRUNTIME_LIB_DIR}")
258 if (NOT ONNXRUNTIME)
259 message(FATAL_ERROR "The ctest needs the prebuilt onnxruntime libraries directory, please specify it by ONNXRUNTIME_LIB_DIR.")
260 endif()
261
262 enable_testing()
263 message(STATUS "Fetch CTest")
264 include(CTest)
265
266 set(TEST_SRC_DIR ${PROJECT_SOURCE_DIR}/test)
267 message(STATUS "Fetch googletest")
268 include(googletest)
269 file(GLOB static_TEST_SRC "${TEST_SRC_DIR}/static_test/*.cc")
270 add_executable(operators_test ${static_TEST_SRC})
271 target_link_libraries(operators_test gtest_main ${ocos_libraries})
272 add_test(NAME operators_test COMMAND $<TARGET_FILE:operators_test>)
273
274 # needs to link with stdc++fs in Linux
275 if(UNIX AND NOT APPLE)
276 set(FS_STDLIB stdc++fs)
277 endif()
278
279 file(GLOB shared_TEST_SRC "${TEST_SRC_DIR}/shared_test/*.cc")
280 add_executable(ortcustomops_test ${shared_TEST_SRC})
281 if (ONNXRUNTIME_LIB_DIR)
282 target_link_directories(ortcustomops_test PRIVATE ${ONNXRUNTIME_LIB_DIR})
283 endif()
284 target_link_libraries(ortcustomops_test ortcustomops onnxruntime gtest_main ${ocos_libraries} ${FS_STDLIB})
285 if (WIN32)
286 file(TO_CMAKE_PATH "${ONNXRUNTIME_LIB_DIR}/*" ONNXRUNTIME_LIB_FILEPATTERN)
287 file(GLOB ONNXRUNTIME_LIB_FILES CONFIGURE_DEPENDS "${ONNXRUNTIME_LIB_FILEPATTERN}")
288 add_custom_command(
289 TARGET ortcustomops_test POST_BUILD
290 COMMAND ${CMAKE_COMMAND} -E copy ${ONNXRUNTIME_LIB_FILES} $<TARGET_FILE_DIR:ortcustomops_test>)
291 endif()
292
293 set(TEST_DATA_SRC ${TEST_SRC_DIR}/data)
294 set(TEST_DATA_DES ${onnxruntime_extensions_BINARY_DIR}/data)
295
296 # Copy test data from source to destination.
297 add_custom_command(
298 TARGET ortcustomops_test POST_BUILD
299 COMMAND ${CMAKE_COMMAND} -E copy_directory
300 ${TEST_DATA_SRC}
301 ${TEST_DATA_DES})
302 add_test(NAME ortcustomops_test COMMAND $<TARGET_FILE:ortcustomops_test>)
303endif()
304