GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/onnxruntime-extensions

Public

mirrored fromhttps://github.com/microsoft/onnxruntime-extensionsAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
v0.7.0

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

onnxruntime-extensions/.pipelines

.pipelines/ci.yml

496lines · modecode

RawDownload
Fix the build breaks the release pipeline and some C++ warnings (#372)0a9a3a9
1# IMPORTANT: READ ME BEFORE MAKING ANY CHANGES
2#
3# Supported matrix should follow these conventions strictly and
4# for any exceptions to the rules, please document them in comments.
5#
6# * When adding new configurations to the matrix, update all platforms simultaneously.
7# * Onnxruntime version should be one release ahead of torch, torchvision, and torchaudio.
8#
9# The rationale is that the onnxruntime release process would have verified the
10# last available version of its dependencies (including the three listed above)
11# and so it's likely to be _better_ supported relative to any older versions.
12
13jobs:
14
15 #######
16 # Linux
17 #######
18 - job: Linux
19 pool:
20 vmImage: 'ubuntu-latest'
21
22 strategy:
23 matrix:
24 py310-1131:
25 python.version: '3.10'
26 torch.version: 'torch torchvision torchaudio'
27 ort.version: '1.13.1'
28 py39-1121:
29 python.version: '3.9'
30 torch.version: 'torch==1.12.1+cpu torchvision==0.13.1+cpu torchaudio==0.12.1+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html'
31 ort.version: '1.12.0'
32 py38-1110:
33 python.version: '3.8'
34 torch.version: 'torch==1.11.0+cpu torchvision==0.12.0+cpu torchaudio==0.11.0+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html'
35 ort.version: '1.11.0'
36 py37-1100:
37 python.version: '3.7'
38 torch.version: 'torch==1.10.0+cpu torchvision==0.11.1+cpu torchaudio==0.10.0+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html'
39 ort.version: '1.10.0'
40
41 steps:
42 - task: DownloadGitHubRelease@0
43 inputs:
44 connection: 'microsoft'
45 userRepository: 'microsoft/onnxruntime'
46 defaultVersionType: 'specificTag'
47 version: 'v$(ort.version)'
48 itemPattern: '*-linux-x64-$(ort.version)*'
49 downloadPath: '$(Build.SourcesDirectory)'
50 displayName: Download the ONNXRuntime prebuilt package.
51
52 - task: ExtractFiles@1
53 inputs:
54 archiveFilePatterns: '**/*.tgz'
55 destinationFolder: '$(Build.SourcesDirectory)'
56 cleanDestinationFolder: false
57 overwriteExistingFiles: true
58 displayName: Unpack ONNXRuntime package.
59
60 - script: |
61 CPU_NUMBER=2 sh ./build.sh -DOCOS_ENABLE_CTEST=ON -DONNXRUNTIME_PKG_DIR=$(Build.SourcesDirectory)/onnxruntime-linux-x64-$(ort.version)
62 displayName: build the customop library with onnxruntime
63
64 - script: |
65 cd out/Linux/RelWithDebInfo
66 ctest -C RelWithDebInfo --output-on-failure
67 displayName: Run C++ native tests
68
69 - task: UsePythonVersion@0
70 inputs:
71 versionSpec: '$(python.version)'
72 addToPath: true
73
74 - script: |
75 python -m pip install --upgrade pip
76 python -m pip install --upgrade setuptools
77 python -m pip install onnxruntime==$(ort.version)
78 python -m pip install -r requirements.txt
79 displayName: Install requirements.txt
80
81 - script: |
82 CPU_NUMBER=2 python -m pip install -e .
83 displayName: Build the library and tests
84
85 - script: python -m pip install $(torch.version)
86 displayName: Install pytorch
87
88 - script: |
89 python -m pip install -r requirements-dev.txt
90 displayName: Install requirements-dev.txt
91
92 - script: python -m pytest test --verbose
93 displayName: Run python test
94
95 ###########
96 # macOS C++
97 ###########
98 - job: MacOSX
99 pool:
100 vmImage: 'macOS-12'
101
102 strategy:
103 matrix:
104 ort-1131:
105 ort.version: '1.13.1'
106 ort.dirname: 'onnxruntime-osx-x86_64-$(ort.version)'
107 ort-1121:
108 ort.version: '1.12.1'
109 ort.dirname: 'onnxruntime-osx-x86_64-$(ort.version)'
110 ort-1110:
111 ort.version: '1.11.0'
112 ort.dirname: 'onnxruntime-osx-x86_64-$(ort.version)'
113 ort-1100:
114 ort.version: '1.10.0'
115 ort.dirname: 'onnxruntime-osx-x86_64-$(ort.version)'
116
117 steps:
118 # needed for onnxruntime
119 - script: brew install libomp
120 displayName: 'Install omp'
121
122 - task: DownloadGitHubRelease@0
123 inputs:
124 connection: 'microsoft'
125 userRepository: 'microsoft/onnxruntime'
126 defaultVersionType: 'specificTag'
127 version: 'v$(ort.version)'
128 itemPattern: '$(ort.dirname)*'
129 downloadPath: '$(Build.SourcesDirectory)'
130 displayName: Download the ONNXRuntime prebuilt package.
131
132 - task: ExtractFiles@1
133 inputs:
134 archiveFilePatterns: '**/*.tgz'
135 destinationFolder: '$(Build.SourcesDirectory)'
136 cleanDestinationFolder: false
137 overwriteExistingFiles: true
138 displayName: Unpack ONNXRuntime package.
139
140 - script: |
141 sh ./build.sh -DOCOS_ENABLE_CTEST=ON -DONNXRUNTIME_PKG_DIR=$(Build.SourcesDirectory)/$(ort.dirname)
142 displayName: build the customop library with onnxruntime
143
144 - script: |
145 cd out/Darwin/RelWithDebInfo
146 ctest -C RelWithDebInfo --output-on-failure
147 displayName: Run C++ native tests
148
149 #############
150 # macOS Python
151 #############
152 - job: MacOSPython
153 pool:
154 vmImage: 'macOS-12'
155
156 strategy:
157 matrix:
158 py310-1131:
159 python.version: '3.10'
160 torch.version: 'torch torchvision torchaudio'
161 ort.version: '1.13.1'
162 py310-1121:
163 python.version: '3.10'
164 torch.version: 'torch==1.12.1 torchvision==0.13.1 torchaudio==0.12.1 -f https://download.pytorch.org/whl/torch_stable.html'
165 ort.version: '1.12.1'
166 py39-1110:
167 python.version: '3.9'
168 torch.version: 'torch==1.11.0 torchvision==0.12.0 torchaudio==0.11.0 -f https://download.pytorch.org/whl/torch_stable.html'
169 ort.version: '1.11.0'
170 py38-1100:
171 python.version: '3.8'
172 torch.version: 'torch==1.10.0 torchvision==0.11.1 torchaudio==0.10.0 -f https://download.pytorch.org/whl/torch_stable.html'
173 ort.version: '1.10.0'
174
175 steps:
176 # needed for onnxruntime
177 - script: brew install libomp
178 displayName: 'Install omp'
179
180 - task: UsePythonVersion@0
181 inputs:
182 versionSpec: '$(python.version)'
183 addToPath: true
184
185 - script: |
186 python -m pip install --upgrade pip
187 python -m pip install --upgrade setuptools
188 python -m pip install --upgrade wheel
189 python -m pip install onnxruntime==$(ort.version)
190 python -m pip install -r requirements.txt
191 displayName: Install requirements.txt
192
193 - script: |
194 python -c "import onnxruntime;print(onnxruntime.__version__)"
195 displayName: Check installation
196
197 - script: |
198 python -m pip install -e .
199 displayName: Build and install the wheel
200
201 - script: python -m pip install -r requirements-dev.txt
202 displayName: Install requirements-dev.txt
203
204 - script: python -m pip install $(torch.version)
205 displayName: Install pytorch
206
207 - script: cd test && python -m pytest . --verbose
208 displayName: Run python test
209
210 #########
211 # Windows C++
212 #########
213 - job: WindowsC
214 pool:
215 vmImage: 'windows-2019'
216
217 strategy:
218 matrix:
219 ort-1131:
220 ort.version: '1.13.1'
221 ort-1121:
222 ort.version: '1.12.1'
223 ort-1110:
224 ort.version: '1.11.0'
225 ort-1100:
226 ort.version: '1.10.0'
227
228 steps:
229 - task: DownloadGitHubRelease@0
230 inputs:
231 connection: 'microsoft'
232 userRepository: 'microsoft/onnxruntime'
233 defaultVersionType: 'specificTag'
234 version: 'v$(ort.version)'
235 itemPattern: '*-win-x64-$(ort.version)*'
236 downloadPath: '$(Build.SourcesDirectory)'
237 displayName: Download the ONNXRuntime prebuilt package.
238
239 - task: ExtractFiles@1
240 inputs:
241 archiveFilePatterns: '**/*.zip'
242 destinationFolder: '$(Build.SourcesDirectory)'
243 cleanDestinationFolder: false
244 overwriteExistingFiles: true
245 displayName: Unpack ONNXRuntime package.
246
247 - script: |
248 @echo off
249 set vswherepath="%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe"
250 for /f "usebackq delims=" %%i in (`%vswherepath% -latest -property installationPath`) do (
251 if exist "%%i\Common7\Tools\vsdevcmd.bat" (
252 set vsdevcmd="%%i\Common7\Tools\vsdevcmd.bat"
253 )
254 )
255
256 @echo %vsdevcmd% will be used as the VC compiler
257 @echo ##vso[task.setvariable variable=vsdevcmd]%vsdevcmd%
258 displayName: 'locate vsdevcmd via vswhere'
259
260 - script: |
261 call $(vsdevcmd)
262 call .\build.bat -DOCOS_ENABLE_CTEST=ON -DONNXRUNTIME_PKG_DIR=.\onnxruntime-win-x64-$(ort.version)
263 displayName: build the customop library with onnxruntime
264
265 - script: |
266 cd out/Windows
267 ctest -C RelWithDebInfo --output-on-failure
268 displayName: Run C++ native tests
269
270 ################
271 # Windows Python
272 ################
273 - job: WindowsPython
274 pool:
275 vmImage: 'windows-latest'
276
277 strategy:
278 matrix:
279 py310-1131:
280 python.version: '3.10'
281 torch.version: 'torch torchvision torchaudio'
282 ort.version: '1.13.1'
283 py39-1121:
284 python.version: '3.9'
285 torch.version: 'torch==1.12.1+cpu torchvision==0.13.1+cpu torchaudio==0.12.1+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html'
286 ort.version: '1.12.1'
287 py38-1110:
288 python.version: '3.8'
289 torch.version: 'torch==1.11.0+cpu torchvision==0.12.0+cpu torchaudio==0.11.0+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html'
290 ort.version: '1.11.0'
291 py37-1100:
292 python.version: '3.7'
293 torch.version: 'torch==1.10.0+cpu torchvision==0.11.1+cpu torchaudio==0.10.0+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html'
294 ort.version: '1.10.0'
295
296 steps:
297 - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
298 displayName: Add conda to PATH
299
300 - script: conda create --yes --quiet --name pyenv -c conda-forge python=$(python.version) numpy
301 displayName: Create Anaconda environment
302
303 - script: |
304 call activate pyenv
305 python -m pip install --upgrade pip
306 python -m pip install onnxruntime==$(ort.version)
307 python -m pip install -r requirements.txt
308 python -m pip install -r requirements-dev.txt
309 displayName: Install requirements.txt
310
311 - script: |
312 call activate pyenv
313 python -m pip install .
314 displayName: Build the wheel
315
316 - script: |
317 call activate pyenv
318 python -m pip install $(torch.version)
319 displayName: Install pytorch
320
321 - script: |
322 call activate pyenv
323 pytest test
324 displayName: Run python test
325
326 #############
327 # WebAssembly
328 #############
329 - job: WebAssembly
330 pool:
331 vmImage: 'ubuntu-latest'
332
333 steps:
334 - script: |
335 cd $(Build.BinariesDirectory)
336 git clone https://github.com/emscripten-core/emsdk
337 emsdk/emsdk install latest
338 emsdk/emsdk activate latest
339 displayName: Setup emscripten pipeline
340
341 - script: |
342 bash ./build.sh \
343 -DCMAKE_TOOLCHAIN_FILE=$(Build.BinariesDirectory)/emsdk/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake \
344 -DOCOS_ENABLE_SPM_TOKENIZER=ON \
345 -DOCOS_BUILD_PYTHON=OFF \
346 -DOCOS_ENABLE_CV2=OFF \
347 -DOCOS_ENABLE_VISION=OFF
348 displayName: build the customop library with onnxruntime
349 # TODO add unittest for webassembly
350
351 #############
352 # Android
353 #############
354 - job: AndroidPackage
355 pool:
356 vmImage: 'macOS-12'
357 timeoutInMinutes: 120
358 steps:
359 - task: UsePythonVersion@0
360 inputs:
361 versionSpec: "3.9"
362 addToPath: true
363 architecture: "x64"
364 displayName: "Use Python 3.9"
365
366 - task: JavaToolInstaller@0
367 displayName: Use jdk 11
368 inputs:
369 versionSpec: '11'
370 jdkArchitectureOption: 'x64'
371 jdkSourceOption: 'PreInstalled'
372
373 - script: brew install coreutils ninja
374 displayName: Install coreutils and ninja
375
376 - template: templates/run-with-android-emulator-steps.yml
377 parameters:
378 steps:
379
380 - bash: |
381 set -e -x
382
383 _BUILD_CFG="x86_64 $(Build.BinariesDirectory)/android_aar" ./build.android
384
385 VERSION=$(cat ./version.txt)
386 AAR_PATH="$(Build.BinariesDirectory)/android_aar/aar_out/com/microsoft/onnxruntime/onnxruntime-extensions-android/${VERSION}/onnxruntime-extensions-android-${VERSION}.aar"
387
388 echo "##vso[task.setvariable variable=ORT_EXTENSIONS_AAR_PATH]${AAR_PATH}"
389 displayName: Build onnxruntime-extensions AAR package
390
391 - bash: |
392 set -e -x
393
394 cp -r $(Build.SourcesDirectory)/java/src/test/android $(Build.BinariesDirectory)/android_test
395
396 cd $(Build.BinariesDirectory)/android_test
397
398 ./gradlew connectedDebugAndroidTest --no-daemon -DortExtensionsAarLocalPath="${ORT_EXTENSIONS_AAR_PATH}"
399 displayName: Build and run onnxruntime-extensions Android test with Android Emulator
400
401 #############
402 # iOS
403 #############
404 - job: IosPackage
405 pool:
406 vmImage: 'macOS-12'
407 timeoutInMinutes: 120
408 steps:
409 - task: UsePythonVersion@0
410 inputs:
411 versionSpec: '3.9'
412 addToPath: true
413 architecture: 'x64'
414 displayName: "Use Python 3.9"
415
416 # iOS xcframework build doesn't work with CMake 3.25.1, pin to 3.25.0
417 - script: |
418 python -m pip install cmake==3.25.0
419 displayName: "Install CMake 3.25.0"
420
421 - template: templates/set-package-version-variable-step.yml
422 parameters:
423 PackageVersionVariableName: ORT_EXTENSIONS_POD_VERSION
424
425 - script: |
426 python ./tools/ios/build_xcframework.py \
427 --output-dir $(Build.BinariesDirectory)/xcframework_out \
428 --platform-arch iphonesimulator x86_64 \
429 --config Release
430 displayName: Build xcframework for iphonesimulator x86_64
431
432 - script: |
433 python ./tools/ios/assemble_pod_package.py \
434 --staging-dir $(Build.BinariesDirectory)/pod_staging \
435 --xcframework-output-dir $(Build.BinariesDirectory)/xcframework_out \
436 --pod-version ${ORT_EXTENSIONS_POD_VERSION}
437 displayName: Assemble pod
438
439 - script: |
440 ORT_EXTENSIONS_LOCAL_POD_PATH=$(Build.BinariesDirectory)/pod_staging \
441 pod install
442 displayName: Install pods for OrtExtensionsUsage
443 workingDirectory: $(Build.SourcesDirectory)/test/ios/OrtExtensionsUsage
444
445 - template: templates/run-with-ios-simulator-steps.yml
446 parameters:
447 steps:
448 - template: templates/xcode-build-and-test-step.yml
449 parameters:
450 xcWorkspacePath: '$(Build.SourcesDirectory)/test/ios/OrtExtensionsUsage/OrtExtensionsUsage.xcworkspace'
451 scheme: 'OrtExtensionsUsage'
452
453
454 #####################################
455 # Linux prevent exception propagation
456 #####################################
457 - job: Linux_Prevent_Exception_Propagation
458 pool:
459 vmImage: 'ubuntu-latest'
460
461 steps:
462 # Simulate an embedded build as part of ORT with exceptions disabled by manually setting CMAKE_CXX_FLAGS and
463 # using _OCOS_PREVENT_EXCEPTION_PROPAGATION_OVERRIDE. The build should re-enable exceptions within ort-ext
464 # but prevent them from propagating. Unit tests are run to validate this.
465 - script: '
466 ./build_lib.sh --enable_cxx_tests --onnxruntime_version 1.14.0 --config RelWithDebInfo
467 --cmake_extra_defines
468 _OCOS_PREVENT_EXCEPTION_PROPAGATION_OVERRIDE=ON OCOS_ENABLE_CPP_EXCEPTIONS=OFF
469 CMAKE_CXX_FLAGS="-fno-exceptions -fno-unwind-tables -fno-asynchronous-unwind-tables"
470 '
471
472 displayName: Build ort-ext with exception propagation disabled
473
474 # As an extra validation check CMakeCache.txt as well
475 - script: |
476 grep "^_OCOS_PREVENT_EXCEPTION_PROPAGATION.*ON$" build/Linux/RelWithDebInfo/CMakeCache.txt
477 if [ $? -ne 0 ]; then
478 echo "Exception propogation was not enabled correctly."
479 exit 1
480 fi
481
482
483 ##############################
484 # Linux for selected_ops build
485 ##############################
486 - job: Linux_SelectedOpsBuild
487 pool:
488 vmImage: 'ubuntu-latest'
489
490 steps:
491 # compiled as only one operator selected.
492 - bash: |
493 set -e -x -u
494 echo 'set (OCOS_ENABLE_BERT_TOKENIZER ON CACHE BOOL "" FORCE)' > cmake/_selectedoplist.cmake
495 ./build.sh -DOCOS_ENABLE_CPP_EXCEPTIONS=OFF -DOCOS_ENABLE_SELECTED_OPLIST=ON
496 displayName: Build ort-extensions with only one operator was selected