GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/onnxruntime-extensions

Public

mirrored fromhttps://github.com/microsoft/onnxruntime-extensionsAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
1f0c76cefaa9359b9fc74ff4b9ed93214486205a

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

onnxruntime-extensions/.pipelines

.pipelines/ci.yml

498lines · modecode

RawDownload
Latest commit unavailable.
1# IMPORTANT: READ ME BEFORE MAKING ANY CHANGES
2#
3# Supported matrix should follow these conventions strictly and
4# for any exceptions to the rules, please document them in comments.
5#
6# * When adding new configurations to the matrix, update all platforms simultaneously.
7# * Onnxruntime version should be one release ahead of torch, torchvision, and torchaudio.
8#
9# The rationale is that the onnxruntime release process would have verified the
10# last available version of its dependencies (including the three listed above)
11# and so it's likely to be _better_ supported relative to any older versions.
12
13jobs:
14
15 #######
16 # Linux
17 #######
18 - job: Linux
19 pool:
20 vmImage: 'ubuntu-latest'
21
22 strategy:
23 matrix:
24 py311-1150:
25 python.version: '3.11'
26 torch.version: 'torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu'
27 ort.version: '1.15.0'
28 py310-1141:
29 python.version: '3.10'
30 torch.version: 'torch==1.13.1+cpu torchvision==0.14.1+cpu torchaudio==0.13.1 --index-url https://download.pytorch.org/whl/cpu'
31 ort.version: '1.14.1'
32 py39-1131:
33 python.version: '3.9'
34 torch.version: 'torch==1.12.1+cpu torchvision==0.13.1+cpu torchaudio==0.12.1+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html'
35 ort.version: '1.13.1'
36 py38-1121:
37 python.version: '3.8'
38 torch.version: 'torch==1.11.0+cpu torchvision==0.12.0+cpu torchaudio==0.11.0+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html'
39 ort.version: '1.12.1'
40
41 steps:
42 - task: DownloadGitHubRelease@0
43 inputs:
44 connection: 'microsoft'
45 userRepository: 'microsoft/onnxruntime'
46 defaultVersionType: 'specificTag'
47 version: 'v$(ort.version)'
48 itemPattern: '*-linux-x64-$(ort.version)*'
49 downloadPath: '$(Build.SourcesDirectory)'
50 displayName: Download the ONNXRuntime prebuilt package.
51
52 - task: ExtractFiles@1
53 inputs:
54 archiveFilePatterns: '**/*.tgz'
55 destinationFolder: '$(Build.SourcesDirectory)'
56 cleanDestinationFolder: false
57 overwriteExistingFiles: true
58 displayName: Unpack ONNXRuntime package.
59
60 - script: |
61 CPU_NUMBER=2 sh ./build.sh -DOCOS_ENABLE_CTEST=ON -DONNXRUNTIME_PKG_DIR=$(Build.SourcesDirectory)/onnxruntime-linux-x64-$(ort.version)
62 displayName: build the customop library with onnxruntime
63
64 - script: |
65 cd out/Linux/RelWithDebInfo
66 ctest -C RelWithDebInfo --output-on-failure
67 displayName: Run C++ native tests
68
69 - task: UsePythonVersion@0
70 inputs:
71 versionSpec: '$(python.version)'
72 addToPath: true
73
74 - script: |
75 python -m pip install --upgrade pip
76 python -m pip install --upgrade setuptools
77 python -m pip install onnxruntime==$(ort.version)
78 python -m pip install -r requirements.txt
79 displayName: Install requirements.txt
80
81 - script: |
82 CPU_NUMBER=2 python -m pip install -e .
83 displayName: Build the library and tests
84
85 - script: python -m pip install $(torch.version)
86 displayName: Install pytorch
87
88 - script: |
89 python -m pip install -r requirements-dev.txt
90 displayName: Install requirements-dev.txt
91
92 - script: python -m pytest test --verbose
93 displayName: Run python test
94
95 ###########
96 # macOS C++
97 ###########
98 - job: MacOSX
99 pool:
100 vmImage: 'macOS-12'
101
102 strategy:
103 matrix:
104 ort-1150:
105 ort.version: '1.15.0'
106 ort.dirname: 'onnxruntime-osx-x86_64-$(ort.version)'
107 ort-1141:
108 ort.version: '1.14.1'
109 ort.dirname: 'onnxruntime-osx-x86_64-$(ort.version)'
110 ort-1131:
111 ort.version: '1.13.1'
112 ort.dirname: 'onnxruntime-osx-x86_64-$(ort.version)'
113 ort-1121:
114 ort.version: '1.12.1'
115 ort.dirname: 'onnxruntime-osx-x86_64-$(ort.version)'
116
117 steps:
118 # needed for onnxruntime
119 - script: brew install libomp
120 displayName: 'Install omp'
121
122 - task: DownloadGitHubRelease@0
123 inputs:
124 connection: 'microsoft'
125 userRepository: 'microsoft/onnxruntime'
126 defaultVersionType: 'specificTag'
127 version: 'v$(ort.version)'
128 itemPattern: '$(ort.dirname)*'
129 downloadPath: '$(Build.SourcesDirectory)'
130 displayName: Download the ONNXRuntime prebuilt package.
131
132 - task: ExtractFiles@1
133 inputs:
134 archiveFilePatterns: '**/*.tgz'
135 destinationFolder: '$(Build.SourcesDirectory)'
136 cleanDestinationFolder: false
137 overwriteExistingFiles: true
138 displayName: Unpack ONNXRuntime package.
139
140 - script: |
141 sh ./build.sh -DOCOS_ENABLE_CTEST=ON -DONNXRUNTIME_PKG_DIR=$(Build.SourcesDirectory)/$(ort.dirname)
142 displayName: build the customop library with onnxruntime
143
144 - script: |
145 cd out/Darwin/RelWithDebInfo
146 ctest -C RelWithDebInfo --output-on-failure
147 displayName: Run C++ native tests
148
149 #############
150 # macOS Python
151 #############
152 - job: MacOSPython
153 pool:
154 vmImage: 'macOS-12'
155
156 strategy:
157 matrix:
158 py311-1150:
159 python.version: '3.11'
160 torch.version: 'torch torchvision torchaudio'
161 ort.version: '1.15.0'
162 py310-1141:
163 python.version: '3.10'
164 torch.version: 'torch==1.13.1 torchvision==0.14.1 torchaudio==0.13.1 --index-url https://download.pytorch.org/whl/cpu'
165 ort.version: '1.14.1'
166 py39-1131:
167 python.version: '3.9'
168 torch.version: 'torch==1.12.1 torchvision==0.13.1 torchaudio==0.12.1 -f https://download.pytorch.org/whl/torch_stable.html'
169 ort.version: '1.13.1'
170 py38-1120:
171 python.version: '3.8'
172 torch.version: 'torch==1.11.0 torchvision==0.12.0 torchaudio==0.11.0 -f https://download.pytorch.org/whl/torch_stable.html'
173 ort.version: '1.12.0'
174
175 steps:
176 # needed for onnxruntime
177 - script: brew install libomp
178 displayName: 'Install omp'
179
180 - task: UsePythonVersion@0
181 inputs:
182 versionSpec: '$(python.version)'
183 addToPath: true
184
185 - script: |
186 python -m pip install --upgrade pip
187 python -m pip install --upgrade setuptools
188 python -m pip install --upgrade wheel
189 python -m pip install onnxruntime==$(ort.version)
190 python -m pip install -r requirements.txt
191 displayName: Install requirements.txt
192
193 - script: |
194 python -c "import onnxruntime;print(onnxruntime.__version__)"
195 displayName: Check installation
196
197 - script: |
198 python -m pip install -e .
199 displayName: Build and install the wheel
200
201 - script: python -m pip install -r requirements-dev.txt
202 displayName: Install requirements-dev.txt
203
204 - script: python -m pip install $(torch.version)
205 displayName: Install pytorch
206
207 - script: cd test && python -m pytest . --verbose
208 displayName: Run python test
209
210 #########
211 # Windows C++
212 #########
213 - job: WindowsC
214 pool:
215 vmImage: 'windows-latest'
216
217 strategy:
218 matrix:
219 ort-1150:
220 ort.version: '1.15.0'
221 ort-1141:
222 ort.version: '1.14.1'
223 ort-1131:
224 ort.version: '1.13.1'
225 ort-1121:
226 ort.version: '1.12.1'
227
228 steps:
229 - task: DownloadGitHubRelease@0
230 inputs:
231 connection: 'microsoft'
232 userRepository: 'microsoft/onnxruntime'
233 defaultVersionType: 'specificTag'
234 version: 'v$(ort.version)'
235 itemPattern: '*-win-x64-$(ort.version)*'
236 downloadPath: '$(Build.SourcesDirectory)'
237 displayName: Download the ONNXRuntime prebuilt package.
238
239 - task: ExtractFiles@1
240 inputs:
241 archiveFilePatterns: '**/*.zip'
242 destinationFolder: '$(Build.SourcesDirectory)'
243 cleanDestinationFolder: false
244 overwriteExistingFiles: true
245 displayName: Unpack ONNXRuntime package.
246
247 - script: |
248 @echo off
249 set vswherepath="%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe"
250 for /f "usebackq delims=" %%i in (`%vswherepath% -latest -property installationPath`) do (
251 if exist "%%i\Common7\Tools\vsdevcmd.bat" (
252 set vsdevcmd="%%i\Common7\Tools\vsdevcmd.bat"
253 )
254 )
255
256 @echo %vsdevcmd% will be used as the VC compiler
257 @echo ##vso[task.setvariable variable=vsdevcmd]%vsdevcmd%
258 displayName: 'locate vsdevcmd via vswhere'
259
260 - script: |
261 call $(vsdevcmd)
262 call .\build.bat -DOCOS_ENABLE_CTEST=ON -DONNXRUNTIME_PKG_DIR=.\onnxruntime-win-x64-$(ort.version)
263 displayName: build the customop library with onnxruntime
264
265 - script: |
266 cd out/Windows
267 ctest -C RelWithDebInfo --output-on-failure
268 displayName: Run C++ native tests
269
270 ################
271 # Windows Python
272 ################
273 - job: WindowsPython
274 pool:
275 vmImage: 'windows-latest'
276
277 strategy:
278 matrix:
279 py311-1150:
280 python.version: '3.11'
281 torch.version: 'torch torchvision torchaudio'
282 ort.version: '1.15.0'
283 py310-1141:
284 python.version: '3.10'
285 torch.version: 'torch torchvision torchaudio'
286 ort.version: '1.14.1'
287 py39-1131:
288 python.version: '3.9'
289 torch.version: 'torch==1.12.1+cpu torchvision==0.13.1+cpu torchaudio==0.12.1+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html'
290 ort.version: '1.13.1'
291 py38-1121:
292 python.version: '3.8'
293 torch.version: 'torch==1.11.0+cpu torchvision==0.12.0+cpu torchaudio==0.11.0+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html'
294 ort.version: '1.12.1'
295
296 steps:
297 - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
298 displayName: Add conda to PATH
299
300 - script: conda create --yes --quiet --name pyenv -c conda-forge python=$(python.version) numpy
301 displayName: Create Anaconda environment
302
303 - script: |
304 call activate pyenv
305 python -m pip install --upgrade pip
306 python -m pip install onnxruntime==$(ort.version)
307 python -m pip install -r requirements.txt
308 python -m pip install -r requirements-dev.txt
309 displayName: Install requirements.txt
310
311 - script: |
312 call activate pyenv
313 python -m pip install .
314 displayName: Build the wheel
315
316 - script: |
317 call activate pyenv
318 python -m pip install $(torch.version)
319 displayName: Install pytorch
320
321 - script: |
322 call activate pyenv
323 pytest test
324 displayName: Run python test
325
326 #############
327 # WebAssembly
328 #############
329 - job: WebAssembly
330 pool:
331 vmImage: 'ubuntu-latest'
332
333 steps:
334 - script: |
335 cd $(Build.BinariesDirectory)
336 git clone https://github.com/emscripten-core/emsdk
337 emsdk/emsdk install latest
338 emsdk/emsdk activate latest
339 displayName: Setup emscripten pipeline
340
341 - script: |
342 bash ./build.sh \
343 -DCMAKE_TOOLCHAIN_FILE=$(Build.BinariesDirectory)/emsdk/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake \
344 -DOCOS_ENABLE_SPM_TOKENIZER=ON \
345 -DOCOS_BUILD_PYTHON=OFF \
346 -DOCOS_ENABLE_CV2=OFF \
347 -DOCOS_ENABLE_VISION=OFF
348 displayName: build the customop library with onnxruntime
349 # TODO add unittest for webassembly
350
351 #############
352 # Android
353 #############
354 - job: AndroidPackage
355 pool:
356 vmImage: 'macOS-12'
357 timeoutInMinutes: 120
358 steps:
359 - task: UsePythonVersion@0
360 inputs:
361 versionSpec: "3.9"
362 addToPath: true
363 architecture: "x64"
364 displayName: "Use Python 3.9"
365
366 - task: JavaToolInstaller@0
367 displayName: Use jdk 11
368 inputs:
369 versionSpec: '11'
370 jdkArchitectureOption: 'x64'
371 jdkSourceOption: 'PreInstalled'
372
373 - script: brew install coreutils ninja
374 displayName: Install coreutils and ninja
375
376 - template: templates/run-with-android-emulator-steps.yml
377 parameters:
378 steps:
379
380 - bash: |
381 set -e -x
382
383 _BUILD_CFG="x86_64 $(Build.BinariesDirectory)/android_aar" ./build.android
384
385 VERSION=$(cat ./version.txt)
386 AAR_PATH="$(Build.BinariesDirectory)/android_aar/aar_out/com/microsoft/onnxruntime/onnxruntime-extensions-android/${VERSION}/onnxruntime-extensions-android-${VERSION}.aar"
387
388 # Do not output ##vso[] commands with `set -x` or they may be parsed again and include a trailing quote.
389 set +x
390 echo "##vso[task.setvariable variable=ORT_EXTENSIONS_AAR_PATH]${AAR_PATH}"
391 displayName: Build onnxruntime-extensions AAR package
392
393 - bash: |
394 set -e -x
395
396 cp -r $(Build.SourcesDirectory)/java/src/test/android $(Build.BinariesDirectory)/android_test
397
398 cd $(Build.BinariesDirectory)/android_test
399
400 ./gradlew connectedDebugAndroidTest --no-daemon -DortExtensionsAarLocalPath="${ORT_EXTENSIONS_AAR_PATH}"
401 displayName: Build and run onnxruntime-extensions Android test with Android Emulator
402
403 #############
404 # iOS
405 #############
406 - job: IosPackage
407 pool:
408 vmImage: 'macOS-12'
409 timeoutInMinutes: 120
410 steps:
411 - task: UsePythonVersion@0
412 inputs:
413 versionSpec: '3.9'
414 addToPath: true
415 architecture: 'x64'
416 displayName: "Use Python 3.9"
417
418 # iOS xcframework build doesn't work with CMake 3.25.1, pin to 3.25.0
419 - script: |
420 python -m pip install cmake==3.25.0
421 displayName: "Install CMake 3.25.0"
422
423 - template: templates/set-package-version-variable-step.yml
424 parameters:
425 PackageVersionVariableName: ORT_EXTENSIONS_POD_VERSION
426
427 - script: |
428 python ./tools/ios/build_xcframework.py \
429 --output_dir $(Build.BinariesDirectory)/xcframework_out \
430 --platform_arch iphonesimulator x86_64 \
431 --config Release
432 displayName: Build xcframework for iphonesimulator x86_64
433
434 - script: |
435 python ./tools/ios/assemble_pod_package.py \
436 --staging-dir $(Build.BinariesDirectory)/pod_staging \
437 --xcframework-output-dir $(Build.BinariesDirectory)/xcframework_out \
438 --pod-version ${ORT_EXTENSIONS_POD_VERSION}
439 displayName: Assemble pod
440
441 - script: |
442 ORT_EXTENSIONS_LOCAL_POD_PATH=$(Build.BinariesDirectory)/pod_staging \
443 pod install
444 displayName: Install pods for OrtExtensionsUsage
445 workingDirectory: $(Build.SourcesDirectory)/test/ios/OrtExtensionsUsage
446
447 - template: templates/run-with-ios-simulator-steps.yml
448 parameters:
449 steps:
450 - template: templates/xcode-build-and-test-step.yml
451 parameters:
452 xcWorkspacePath: '$(Build.SourcesDirectory)/test/ios/OrtExtensionsUsage/OrtExtensionsUsage.xcworkspace'
453 scheme: 'OrtExtensionsUsage'
454
455
456 #####################################
457 # Linux prevent exception propagation
458 #####################################
459 - job: Linux_Prevent_Exception_Propagation
460 pool:
461 vmImage: 'ubuntu-latest'
462
463 steps:
464 # Simulate an embedded build as part of ORT with exceptions disabled by manually setting CMAKE_CXX_FLAGS and
465 # using _OCOS_PREVENT_EXCEPTION_PROPAGATION_OVERRIDE. The build should re-enable exceptions within ort-ext
466 # but prevent them from propagating. Unit tests are run to validate this.
467 - script: '
468 ./build_lib.sh --enable_cxx_tests --onnxruntime_version 1.14.0 --config RelWithDebInfo
469 --cmake_extra_defines
470 _OCOS_PREVENT_EXCEPTION_PROPAGATION_OVERRIDE=ON OCOS_ENABLE_CPP_EXCEPTIONS=OFF
471 CMAKE_CXX_FLAGS="-fno-exceptions -fno-unwind-tables -fno-asynchronous-unwind-tables"
472 '
473
474 displayName: Build ort-ext with exception propagation disabled
475
476 # As an extra validation check CMakeCache.txt as well
477 - script: |
478 grep "^_OCOS_PREVENT_EXCEPTION_PROPAGATION.*ON$" build/Linux/RelWithDebInfo/CMakeCache.txt
479 if [ $? -ne 0 ]; then
480 echo "Exception propogation was not enabled correctly."
481 exit 1
482 fi
483
484
485 ##############################
486 # Linux for selected_ops build
487 ##############################
488 - job: Linux_SelectedOpsBuild
489 pool:
490 vmImage: 'ubuntu-latest'
491
492 steps:
493 # compiled as only one operator selected.
494 - bash: |
495 set -e -x -u
496 echo 'set (OCOS_ENABLE_BERT_TOKENIZER ON CACHE BOOL "" FORCE)' > cmake/_selectedoplist.cmake
497 ./build.sh -DOCOS_ENABLE_CPP_EXCEPTIONS=OFF -DOCOS_ENABLE_SELECTED_OPLIST=ON
498 displayName: Build ort-extensions with only one operator was selected
499