GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/onnxruntime-extensions

Public

mirrored from https://github.com/microsoft/onnxruntime-extensionsAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
nobf

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

onnxruntime-extensions/.pipelines

.pipelines/ci.yml

915lines · modecode

RawDownload
Latest commit unavailable.
1# Pipeline trigger settings
2trigger:
3 branches:
4 include:
5 - main
6 - rel-*
7 paths:
8 exclude:
9 - docs/**
10 - README.md
11 - tutorials/**
12pr:
13 branches:
14 include:
15 - main
16 - rel-*
17 paths:
18 exclude:
19 - docs/**
20 - README.md
21 - tutorials/**
22
23stages:
24
25- stage: LinuxBuilds
26 dependsOn: []
27 jobs:
28
29 #######
30 # Linux
31 #######
32 - job: Linux
33 pool:
34 name: 'onnxruntime-extensions-Linux-CPU'
35
36 strategy:
37 matrix:
38 py312-1171:
39 python.version: '3.12'
40 torch.version: 'torch torchvision torchaudio'
41 ort.version: '1.17.1'
42 py311-1163:
43 python.version: '3.11'
44 torch.version: 'torch torchvision torchaudio'
45 ort.version: '1.16.3'
46 py310-1151:
47 python.version: '3.10'
48 torch.version: 'torch torchvision torchaudio'
49 ort.version: '1.15.1'
50 py39-1141:
51 python.version: '3.9'
52 torch.version: 'torch==1.13.1+cpu torchvision==0.14.1+cpu torchaudio==0.13.1 --index-url https://download.pytorch.org/whl/cpu'
53 ort.version: '1.14.1'
54 py38-1131:
55 python.version: '3.8'
56 torch.version: 'torch==1.12.1 torchvision==0.13.1 torchaudio==0.12.1 -f https://download.pytorch.org/whl/torch_stable.html'
57 ort.version: '1.13.1'
58
59 steps:
60 - task: DownloadGitHubRelease@0
61 inputs:
62 connection: 'GitHub - Release'
63 userRepository: 'microsoft/onnxruntime'
64 defaultVersionType: 'specificTag'
65 version: 'v$(ort.version)'
66 itemPattern: '*-linux-x64-$(ort.version)*'
67 downloadPath: '$(Build.SourcesDirectory)'
68 displayName: Download the ONNXRuntime prebuilt package.
69
70 - task: ExtractFiles@1
71 inputs:
72 archiveFilePatterns: '**/*.tgz'
73 destinationFolder: '$(Build.SourcesDirectory)'
74 cleanDestinationFolder: false
75 overwriteExistingFiles: true
76 displayName: Unpack ONNXRuntime package.
77
78 - script: |
79 CPU_NUMBER=8 sh ./build.sh -DOCOS_ENABLE_CTEST=ON -DOCOS_ONNXRUNTIME_VERSION="$(ort.version)" -DONNXRUNTIME_PKG_DIR=$(Build.SourcesDirectory)/onnxruntime-linux-x64-$(ort.version)
80 displayName: build the customop library with onnxruntime
81
82 - script: |
83 cd out/Linux/RelWithDebInfo
84 ctest -C RelWithDebInfo --output-on-failure
85 displayName: Run C++ native tests
86
87 - task: UsePythonVersion@0
88 inputs:
89 versionSpec: '$(python.version)'
90 addToPath: true
91
92 - script: |
93 python -m pip install --upgrade pip
94 python -m pip install --upgrade setuptools
95 python -m pip install onnxruntime==$(ort.version)
96 displayName: Install requirements
97
98 - script: |
99 CPU_NUMBER=8 python -m pip install .
100 displayName: Build the library and tests
101
102 - script: python -m pip install $(torch.version)
103 displayName: Install pytorch
104
105 - script: |
106 python -m pip install -r requirements-dev.txt
107 displayName: Install requirements-dev.txt
108
109 - script: cd test && python -m pytest . --verbose
110 displayName: Run python test
111
112 ###############
113 # Linux PyDebug
114 ###############
115 - job: LinuxPyDbg
116 pool:
117 name: 'onnxruntime-extensions-Linux-CPU'
118
119 steps:
120 - task: UsePythonVersion@0
121 inputs:
122 versionSpec: '3.x'
123 disableDownloadFromRegistry: true
124 addToPath: true
125 architecture: 'x64'
126
127 - script: |
128 python -m pip install --upgrade setuptools pip
129 python -m pip install 'numpy < 2.0.0'
130 export OCOS_NO_OPENCV=1
131 export OCOS_SCB_DEBUG=1
132 CPU_NUMBER=8 python -m pip install -e .
133 displayName: Build the python library in editable mode
134
135 - script: |
136 python -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
137 python -m pip install -r requirements-dev.txt
138 displayName: Install requirements-dev.txt
139
140 - script: |
141 cd test
142 python -m pytest --ignore=test_cv2.py --ignore=test_tools_add_pre_post_processing_to_model.py . --verbose
143 displayName: Run python test
144
145 #####################################
146 # Linux prevent exception propagation
147 #####################################
148 - job: Linux_Prevent_Exception_Propagation
149 pool:
150 name: 'onnxruntime-extensions-Linux-CPU'
151
152 steps:
153 # Simulate an embedded build as part of ORT with exceptions disabled by manually setting CMAKE_CXX_FLAGS and
154 # using _OCOS_PREVENT_EXCEPTION_PROPAGATION_OVERRIDE. The build should re-enable exceptions within ort-ext
155 # but prevent them from propagating. Unit tests are run to validate this.
156 - script: '
157 ./build_lib.sh --enable_cxx_tests --onnxruntime_version 1.14.0 --config RelWithDebInfo
158 --cmake_extra_defines
159 _OCOS_PREVENT_EXCEPTION_PROPAGATION_OVERRIDE=ON OCOS_ENABLE_CPP_EXCEPTIONS=OFF
160 CMAKE_CXX_FLAGS="-fno-exceptions -fno-unwind-tables -fno-asynchronous-unwind-tables"
161 '
162
163 displayName: Build ort-ext with exception propagation disabled
164
165 # As an extra validation check CMakeCache.txt as well
166 - script: |
167 grep "^_OCOS_PREVENT_EXCEPTION_PROPAGATION.*ON$" build/Linux/RelWithDebInfo/CMakeCache.txt
168 if [ $? -ne 0 ]; then
169 echo "Exception propogation was not enabled correctly."
170 exit 1
171 fi
172
173
174 ##############################
175 # Linux for selected_ops build
176 ##############################
177 - job: Linux_SelectedOpsBuild
178 pool:
179 name: 'onnxruntime-extensions-Linux-CPU'
180
181 steps:
182 # compiled as only one operator selected.
183 - bash: |
184 set -e -x -u
185 echo 'set (OCOS_ENABLE_BERT_TOKENIZER ON CACHE BOOL "" FORCE)' > cmake/_selectedoplist.cmake
186 ./build.sh -DOCOS_ENABLE_CPP_EXCEPTIONS=OFF -DOCOS_ENABLE_SELECTED_OPLIST=ON -DOCOS_ENABLE_CTEST=OFF
187 displayName: Build ort-extensions with only one operator was selected
188
189 ##############################
190 # Linux for pre-processing API
191 ##############################
192 - job: Linux_PPApiBuild
193 pool:
194 name: 'onnxruntime-extensions-Linux-CPU'
195
196 steps:
197 # compiled as only one operator selected.
198 - bash: |
199 set -e -x -u
200 ./build.sh -DOCOS_ENABLE_C_API=ON
201 cd out/Linux
202 ctest -C RelWithDebInfo --output-on-failure
203 displayName: Build ort-extensions with API enabled and run tests
204
205
206- stage: MacOSBuilds
207 dependsOn: []
208 jobs:
209
210 ###########
211 # macOS C++
212 ###########
213 - job: MacOSX
214 pool:
215 vmImage: 'macOS-13'
216
217 strategy:
218 matrix:
219 ort-1171:
220 ort.version: '1.17.1'
221 ort.dirname: 'onnxruntime-osx-x86_64-$(ort.version)'
222 ort-1163:
223 ort.version: '1.16.3'
224 ort.dirname: 'onnxruntime-osx-x86_64-$(ort.version)'
225 ort-1151:
226 ort.version: '1.15.1'
227 ort.dirname: 'onnxruntime-osx-x86_64-$(ort.version)'
228 ort-1141:
229 ort.version: '1.14.1'
230 ort.dirname: 'onnxruntime-osx-x86_64-$(ort.version)'
231
232 steps:
233 - template: templates/use-xcode-version.yml
234
235 # needed for onnxruntime
236 - script: brew install libomp
237 displayName: 'Install omp'
238
239 - task: DownloadGitHubRelease@0
240 inputs:
241 connection: 'GitHub - Release'
242 userRepository: 'microsoft/onnxruntime'
243 defaultVersionType: 'specificTag'
244 version: 'v$(ort.version)'
245 itemPattern: '$(ort.dirname)*'
246 downloadPath: '$(Build.SourcesDirectory)'
247 displayName: Download the ONNXRuntime prebuilt package.
248
249 - task: ExtractFiles@1
250 inputs:
251 archiveFilePatterns: '**/*.tgz'
252 destinationFolder: '$(Build.SourcesDirectory)'
253 cleanDestinationFolder: false
254 overwriteExistingFiles: true
255 displayName: Unpack ONNXRuntime package.
256
257 - script: |
258 sh ./build.sh -DOCOS_ENABLE_CTEST=ON -DONNXRUNTIME_PKG_DIR=$(Build.SourcesDirectory)/$(ort.dirname)
259 displayName: build the customop library with onnxruntime
260
261 - script: |
262 cd out/Darwin/RelWithDebInfo
263 ctest -C RelWithDebInfo --output-on-failure
264 displayName: Run C++ native tests
265
266 ##############################
267 # MacOS for pre-processing API
268 ##############################
269 - job: MacOS_PPApiBuild
270 pool:
271 vmImage: 'macOS-13'
272
273 steps:
274 # compiled as only one operator selected.
275 - bash: |
276 set -e -x -u
277 ./build.sh -DOCOS_ENABLE_C_API=ON
278 cd out/Darwin/RelWithDebInfo
279 ctest -C RelWithDebInfo --output-on-failure
280 displayName: Build ort-extensions with API enabled and run tests
281
282 #############
283 # macOS Python
284 #############
285 - job: MacOSPython
286 pool:
287 vmImage: 'macOS-13'
288
289 strategy:
290 matrix:
291 py312-1171:
292 python.version: '3.12'
293 torch.version: 'torch torchvision torchaudio'
294 ort.version: '1.17.1'
295 py311-1163:
296 python.version: '3.11'
297 torch.version: 'torch torchvision torchaudio'
298 ort.version: '1.16.3'
299 py310-1151:
300 python.version: '3.10'
301 torch.version: 'torch torchvision torchaudio'
302 ort.version: '1.15.1'
303 py39-1141:
304 python.version: '3.9'
305 torch.version: 'torch==1.13.1 torchvision==0.14.1 torchaudio==0.13.1 --index-url https://download.pytorch.org/whl/cpu'
306 ort.version: '1.14.1'
307 py38-1131:
308 python.version: '3.8'
309 torch.version: 'torch==1.12.1 torchvision==0.13.1 torchaudio==0.12.1 -f https://download.pytorch.org/whl/torch_stable.html'
310 ort.version: '1.13.1'
311
312 steps:
313 - template: templates/use-xcode-version.yml
314
315 - task: UsePythonVersion@0
316 inputs:
317 versionSpec: '$(python.version)'
318 disableDownloadFromRegistry: true
319 addToPath: true
320
321 - script: |
322 python -m pip install --upgrade pip
323 python -m pip install --upgrade setuptools
324 python -m pip install --upgrade wheel
325 python -m pip install 'numpy < 2.0.0'
326 python -m pip install onnxruntime==$(ort.version)
327 displayName: Install requirements
328
329 - script: |
330 python -c "import onnxruntime;print(onnxruntime.__version__)"
331 displayName: Check installation
332
333 - script: |
334 python -m pip install -e .
335 displayName: Build and install the wheel
336
337 - script: python -m pip install -r requirements-dev.txt
338 displayName: Install requirements-dev.txt
339
340 - script: python -m pip install $(torch.version)
341 displayName: Install pytorch
342
343 - script: cd test && python -m pytest . --verbose
344 displayName: Run python test
345
346- stage: WindowsBuilds
347 dependsOn: []
348 jobs:
349
350 #########
351 # Windows C++
352 #########
353 - job: WindowsC
354 pool:
355 name: 'onnxruntime-extensions-Windows-CPU'
356
357 strategy:
358 matrix:
359 ort-1171:
360 ort.version: '1.17.1'
361 ort-1163:
362 ort.version: '1.16.3'
363 ort-1151:
364 ort.version: '1.15.1'
365 ort-1141:
366 ort.version: '1.14.1'
367
368 steps:
369 - task: DownloadGitHubRelease@0
370 inputs:
371 connection: 'GitHub - Release'
372 userRepository: 'microsoft/onnxruntime'
373 defaultVersionType: 'specificTag'
374 version: 'v$(ort.version)'
375 itemPattern: '*-win-x64-$(ort.version)*'
376 downloadPath: '$(Build.SourcesDirectory)'
377 displayName: Download the ONNXRuntime prebuilt package.
378
379 - task: ExtractFiles@1
380 inputs:
381 archiveFilePatterns: '**/*.zip'
382 destinationFolder: '$(Build.SourcesDirectory)'
383 cleanDestinationFolder: false
384 overwriteExistingFiles: true
385 displayName: Unpack ONNXRuntime package.
386
387 - script: |
388 @echo off
389 set vswherepath="%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe"
390 for /f "usebackq delims=" %%i in (`%vswherepath% -latest -property installationPath`) do (
391 if exist "%%i\Common7\Tools\vsdevcmd.bat" (
392 set vsdevcmd="%%i\Common7\Tools\vsdevcmd.bat"
393 )
394 )
395
396 @echo %vsdevcmd% will be used as the VC compiler
397 @echo ##vso[task.setvariable variable=vsdevcmd]%vsdevcmd%
398 displayName: 'locate vsdevcmd via vswhere'
399
400 - script: |
401 call $(vsdevcmd)
402 call .\build.bat -DOCOS_ENABLE_CTEST=ON -DOCOS_ONNXRUNTIME_VERSION="$(ort.version)" -DONNXRUNTIME_PKG_DIR=.\onnxruntime-win-x64-$(ort.version)
403 displayName: build the customop library with onnxruntime
404
405 - script: |
406 cd out/Windows
407 ctest -C RelWithDebInfo --output-on-failure
408 displayName: Run C++ native tests
409
410 - job: WindowsStaticVC
411 pool:
412 name: 'onnxruntime-extensions-Windows-CPU'
413
414 steps:
415 # the vcpkg build requires a cmake python module
416 - script: python -m pip install cmake
417 displayName: Install cmake python module
418
419 - script: |
420 call .\build.bat -DOCOS_ENABLE_CTEST=ON -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded
421 cd out/Windows
422 ctest -C RelWithDebInfo --output-on-failure
423 displayName: build and test ort-extensions with VC static runtime.
424
425 - job: Windows_PPApiBuild
426 pool:
427 name: 'onnxruntime-extensions-Windows-CPU'
428
429 steps:
430 - script: |
431 call .\build.bat -DOCOS_ENABLE_C_API=ON
432 cd out\Windows
433 ctest -C RelWithDebInfo --output-on-failure
434 displayName: Build ort-extensions with API enabled and run tests
435
436 ################
437 # Windows Python
438 ################
439 - job: WindowsPython
440 pool:
441 name: 'onnxruntime-extensions-Windows-CPU'
442
443 strategy:
444 matrix:
445 py312-1171:
446 python.version: '3.12'
447 torch.version: 'torch torchvision torchaudio'
448 ort.version: '1.17.1'
449 py311-1163:
450 python.version: '3.11'
451 torch.version: 'torch torchvision torchaudio'
452 ort.version: '1.16.3'
453 py310-1151:
454 python.version: '3.10'
455 torch.version: 'torch torchvision torchaudio'
456 ort.version: '1.15.1'
457 py39-1141:
458 python.version: '3.9'
459 torch.version: 'torch==1.13.1 torchvision==0.14.1 torchaudio==0.13.1 --index-url https://download.pytorch.org/whl/cpu'
460 ort.version: '1.14.1'
461 py38-1131:
462 python.version: '3.8'
463 torch.version: 'torch==1.12.1 torchvision==0.13.1 torchaudio==0.12.1 -f https://download.pytorch.org/whl/torch_stable.html'
464 ort.version: '1.13.1'
465
466 steps:
467 - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
468 displayName: Add conda to PATH
469
470 - script: conda create --yes --quiet --name pyenv -c conda-forge python=$(python.version) numpy
471 displayName: Create Anaconda environment
472
473 - script: |
474 call activate pyenv
475 python -m pip install --upgrade pip
476 python -m pip install onnxruntime==$(ort.version)
477 python -m pip install -r requirements-dev.txt
478 displayName: Install requirements{-dev}.txt and cmake python modules
479 condition: ne(variables['python.version'], '3.12')
480
481 - script: |
482 call activate pyenv
483 set CMAKE_ARGS=-DOCOS_ONNXRUNTIME_VERSION=$(ort.version)
484 python -m pip install .
485 displayName: Build the wheel
486
487 - script: |
488 call activate pyenv
489 python -m pip install $(torch.version)
490 displayName: Install pytorch
491 condition: ne(variables['python.version'], '3.12')
492
493 - script: |
494 call activate pyenv
495 pytest test
496 displayName: Run python test
497 condition: ne(variables['python.version'], '3.12')
498
499 #################
500 # Windows PyDebug
501 #################
502 - job: WinPyDbgBuild
503 pool:
504 name: 'onnxruntime-extensions-Windows-CPU'
505
506 steps:
507 - task: UsePythonVersion@0
508 inputs:
509 versionSpec: '3.x'
510 disableDownloadFromRegistry: true
511 addToPath: true
512 architecture: 'x64'
513 displayName: Use ADO python task
514
515 - script: |
516 python -m pip install --upgrade setuptools pip
517 python -m pip install "numpy < 2.0.0"
518 set OCOS_NO_OPENCV=1
519 set OCOS_SCB_DEBUG=1
520 python -m pip install -v -e .
521 displayName: Build onnxruntime-extensions in editable mode.
522
523 - script: |
524 python -m pip install -r requirements-dev.txt
525 python -m pip install torch torchvision torchaudio
526 displayName: Install dependencies for pytest
527
528 - script: |
529 cd test
530 python -m pytest --ignore=test_cv2.py --ignore=test_tools_add_pre_post_processing_to_model.py . --verbose
531 displayName: Run python test
532
533- stage: WindowsCUDABuilds
534 dependsOn: []
535 jobs:
536 - job: WindowsCUDABoth
537 pool:
538 name: 'onnxruntime-extensions-Win2022-GPU-A10'
539 variables:
540 ORT_VERSION: '1.17.1'
541 timeoutInMinutes: 120
542 steps:
543 - template: templates/set_winenv.yml
544 parameters:
545 EnvSetupScript: 'set_env_cuda.bat'
546 DownloadCUDA: true
547
548 - script: |
549 nvidia-smi
550 nvcc --version
551 where nvcc
552 displayName: check cuda version
553
554 - task: DownloadGitHubRelease@0
555 inputs:
556 connection: 'GitHub - Release'
557 userRepository: 'microsoft/onnxruntime'
558 defaultVersionType: 'specificTag'
559 version: 'v$(ORT_VERSION)'
560 itemPattern: '*-win-x64-gpu-$(ORT_VERSION)*'
561 downloadPath: '$(Build.SourcesDirectory)'
562 displayName: Download the ONNXRuntime prebuilt package.
563
564 - task: ExtractFiles@1
565 inputs:
566 archiveFilePatterns: '**/*.zip'
567 destinationFolder: '$(Build.SourcesDirectory)'
568 cleanDestinationFolder: false
569 overwriteExistingFiles: true
570 displayName: Unpack ONNXRuntime package.
571
572 - script: |
573 set CUDA_PATH=$(Agent.TempDirectory)\v11.8
574 call .\build.bat -T cuda="%CUDA_PATH%" -DOCOS_ENABLE_CTEST=ON^
575 -DCMAKE_CUDA_FLAGS_INIT=-allow-unsupported-compiler^
576 -DOCOS_USE_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=70;86^
577 -DOCOS_ONNXRUNTIME_VERSION="$(ORT_VERSION)" -DONNXRUNTIME_PKG_DIR=.\onnxruntime-win-x64-gpu-$(ORT_VERSION)
578 displayName: build the customop library with onnxruntime
579
580 - script: |
581 cd out/Windows
582 ctest -C RelWithDebInfo --output-on-failure
583 displayName: Run C++ native tests
584
585 - task: UsePythonVersion@0
586 inputs:
587 versionSpec: '3.x'
588 disableDownloadFromRegistry: true
589 addToPath: true
590 architecture: 'x64'
591 displayName: Use ADO python task
592
593 - script: |
594 set CUDA_PATH=$(Agent.TempDirectory)\v11.8
595 python -m pip install --upgrade setuptools pip
596 python -m pip install "numpy < 2.0.0" coloredlogs flatbuffers packaging protobuf sympy
597 python -m pip install onnxruntime-gpu==$(ORT_VERSION)
598 python -m pip install -v --config-settings "ortx-user-option=use-cuda,cuda_archs=70;86" .
599 displayName: Build and install onnxruntime-extensions CUDA package.
600
601 - script: |
602 python -m pip install -r requirements-dev.txt
603 python -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
604 displayName: Install dependencies for Python unit tests
605
606 - script: |
607 cd test
608 python -m pytest . --verbose
609 cd cuda
610 python -m pytest . --verbose
611 displayName: Run python test for CPU and CUDA kernels
612
613- stage: LinuxCUDABuilds
614 dependsOn: []
615 jobs:
616 - job: LinuxGPU
617 pool:
618 name: 'onnxruntime-extensions-Linux-GPU-A10'
619 timeoutInMinutes: 120
620 variables:
621 ORT_VERSION: '1.17.1'
622 TORCH_VERSION: 'torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118'
623 steps:
624 - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
625 displayName: 'Clean Agent Directories'
626 condition: always()
627
628 - script: |
629 nvidia-smi
630 displayName: check cuda version
631
632 - checkout: self
633 clean: true
634 submodules: none
635
636 - task: UsePythonVersion@0
637 inputs:
638 versionSpec: '3.12'
639 addToPath: true
640
641 - task: DownloadGitHubRelease@0
642 inputs:
643 connection: 'GitHub - Release'
644 userRepository: 'microsoft/onnxruntime'
645 defaultVersionType: 'specificTag'
646 version: 'v$(ORT_VERSION)'
647 itemPattern: '*-linux-x64-gpu-$(ORT_VERSION)*'
648 downloadPath: '$(Build.SourcesDirectory)'
649 displayName: Download the ONNXRuntime prebuilt package.
650
651 - task: ExtractFiles@1
652 inputs:
653 archiveFilePatterns: '**/*.tgz'
654 destinationFolder: '$(Build.SourcesDirectory)'
655 cleanDestinationFolder: false
656 overwriteExistingFiles: true
657 displayName: Unpack ONNXRuntime package.
658
659 - template: ../tools/ci_build/github/azure-pipeline/templates/get-docker-image-steps.yml
660 parameters:
661 Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda11_8_tensorrt8_6
662 Context: tools/ci_build/github/linux/docker
663 DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
664 Repository: onnxruntime-extensionscuda11build
665 UpdateDepsTxt: false
666
667 - task: CmdLine@2
668 inputs:
669 script: |
670 docker run --gpus all --rm \
671 --volume $(Build.SourcesDirectory):/onnxruntime-extensions \
672 --volume $(Build.SourcesDirectory)/onnxruntime-linux-x64-gpu-$(ORT_VERSION):/onnxruntime \
673 -e CUDA_PATH=/usr/local/cuda-11.8 \
674 onnxruntime-extensionscuda11build \
675 /bin/bash -c "
676 set -ex; \
677 pushd /onnxruntime-extensions; \
678 sh ./build.sh -DOCOS_ENABLE_CTEST=ON -DOCOS_USE_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=86 -DOCOS_ONNXRUNTIME_VERSION="$(ORT_VERSION)" -DONNXRUNTIME_PKG_DIR=/onnxruntime; \
679 popd; \
680 "
681 workingDirectory: $(Build.SourcesDirectory)
682 displayName: build the customop library with onnxruntime
683
684 - task: CmdLine@2
685 inputs:
686 script: |
687 docker run --gpus all --rm \
688 --volume $(Build.SourcesDirectory):/onnxruntime-extensions \
689 --volume $(Build.SourcesDirectory)/onnxruntime-linux-x64-gpu-$(ORT_VERSION):/onnxruntime \
690 -e CUDA_PATH=/usr/local/cuda-11.8 \
691 onnxruntime-extensionscuda11build \
692 /bin/bash -c "
693 set -ex; \
694 pushd /onnxruntime-extensions; \
695 cd out/Linux/RelWithDebInfo; \
696 ctest -C RelWithDebInfo --output-on-failure; \
697 popd; \
698 "
699 workingDirectory: $(Build.SourcesDirectory)
700 displayName: Run C++ native tests
701
702 - task: CmdLine@2
703 inputs:
704 script: |
705 docker run --gpus all --rm \
706 --volume $(Build.SourcesDirectory):/onnxruntime-extensions \
707 --volume $(Build.SourcesDirectory)/onnxruntime-linux-x64-gpu-$(ORT_VERSION):/onnxruntime \
708 -e CUDA_PATH=/usr/local/cuda-11.8 \
709 onnxruntime-extensionscuda11build \
710 /bin/bash -c "
711 set -ex; \
712 pushd /onnxruntime-extensions; \
713 python3 -m pip install --upgrade pip; \
714 python3 -m pip install --upgrade setuptools; \
715 python3 -m pip install onnxruntime-gpu==$(ORT_VERSION); \
716 python3 -m pip install -v --config-settings 'ortx-user-option=use-cuda,cuda_archs=70;86' . ; \
717 python3 -m pip install $(TORCH_VERSION) ; \
718 python3 -m pip install -r requirements-dev.txt; \
719 cd test && python -m pytest . --verbose; \
720 cd cuda && python -m pytest . --verbose; \
721 popd; \
722 "
723 workingDirectory: $(Build.SourcesDirectory)
724 displayName: Build the library and Python unit tests
725
726- stage: WebAssemblyBuilds
727 dependsOn: []
728 jobs:
729
730 #############
731 # WebAssembly
732 #############
733 - job: WebAssembly
734 pool:
735 vmImage: 'ubuntu-latest'
736
737 steps:
738 - script: |
739 cd $(Build.BinariesDirectory)
740 git clone https://github.com/emscripten-core/emsdk
741 emsdk/emsdk install latest
742 emsdk/emsdk activate latest
743 displayName: Setup emscripten pipeline
744
745 - script: |
746 bash ./build.sh \
747 -DCMAKE_TOOLCHAIN_FILE=$(Build.BinariesDirectory)/emsdk/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake \
748 -DOCOS_ENABLE_SPM_TOKENIZER=ON \
749 -DOCOS_BUILD_PYTHON=OFF \
750 -DOCOS_ENABLE_CV2=OFF \
751 -DOCOS_ENABLE_VISION=OFF \
752 -DOCOS_ENABLE_CTEST=OFF
753 displayName: build the customop library with onnxruntime
754 # TODO add unittest for webassembly
755
756- stage: AndroidBuilds
757 dependsOn: []
758 jobs:
759
760 #############
761 # Android
762 #############
763 - job: AndroidPackage_BuildOnly
764 pool:
765 vmImage: 'macOS-13'
766 timeoutInMinutes: 120
767 steps:
768 - task: UsePythonVersion@0
769 inputs:
770 versionSpec: "3.12"
771 addToPath: true
772 architecture: "x64"
773 displayName: "Use Python 3.12"
774
775 - task: JavaToolInstaller@0
776 displayName: Use jdk 17
777 inputs:
778 versionSpec: '17'
779 jdkArchitectureOption: 'x64'
780 jdkSourceOption: 'PreInstalled'
781
782 - script: brew install coreutils ninja
783 displayName: Install coreutils and ninja
784
785 - bash: |
786 set -e -x
787
788 _BUILD_CFG="x86_64 $(Build.BinariesDirectory)/android_aar" ./build.android
789
790 VERSION=$(cat ./version.txt)
791 AAR_PATH="$(Build.BinariesDirectory)/android_aar/aar_out/com/microsoft/onnxruntime/onnxruntime-extensions-android/${VERSION}/onnxruntime-extensions-android-${VERSION}.aar"
792
793 # Do not output ##vso[] commands with `set -x` or they may be parsed again and include a trailing quote.
794 set +x
795 echo "##vso[task.setvariable variable=ORT_EXTENSIONS_AAR_PATH]${AAR_PATH}"
796 displayName: Build onnxruntime-extensions AAR package
797
798 - job: AndroidCpp_BuildOnly
799 pool:
800 vmImage: 'macOS-13'
801 timeoutInMinutes: 30
802 steps:
803 - task: UsePythonVersion@0
804 inputs:
805 versionSpec: "3.12"
806 addToPath: true
807 architecture: "x64"
808 displayName: "Use Python 3.12"
809
810 - task: JavaToolInstaller@0
811 displayName: Use jdk 17
812 inputs:
813 versionSpec: '17'
814 jdkArchitectureOption: 'x64'
815 jdkSourceOption: 'PreInstalled'
816
817 - script: brew install ninja
818 displayName: Install ninja
819
820 - bash: |
821 python ./tools/build.py \
822 --config RelWithDebInfo \
823 --android \
824 --android_abi x86_64 \
825 --enable_cxx_tests \
826 --update --build --parallel
827 displayName: Build onnxruntime-extensions for Android
828
829- stage: IosBuilds
830 dependsOn: []
831 jobs:
832
833 #############
834 # iOS
835 #############
836 - job: IosPackage
837 pool:
838 vmImage: 'macOS-13'
839 timeoutInMinutes: 120
840 steps:
841 - template: templates/use-xcode-version.yml
842
843 - task: UsePythonVersion@0
844 inputs:
845 versionSpec: '3.12'
846 disableDownloadFromRegistry: true
847 addToPath: true
848 architecture: 'x64'
849 displayName: "Use Python 3.12"
850
851 # iOS xcframework build doesn't work with CMake 3.25.1, pin to 3.25.0
852 - script: |
853 python -m pip install cmake==3.25.0
854 displayName: "Install CMake 3.25.0"
855
856 - template: templates/set-package-version-variable-step.yml
857 parameters:
858 PackageVersionVariableName: ORT_EXTENSIONS_POD_VERSION
859
860 - script: |
861 python ./tools/ios/build_xcframework.py \
862 --output_dir $(Build.BinariesDirectory)/xcframework_out \
863 --platform_arch iphonesimulator x86_64 \
864 --config RelWithDebInfo \
865 --ios_deployment_target 13.0 \
866 -- \
867 --enable_cxx_tests
868 displayName: "Build xcframework for iphonesimulator x86_64"
869
870 - script: |
871 python ./tools/ios/assemble_pod_package.py \
872 --staging-dir $(Build.BinariesDirectory)/pod_staging \
873 --xcframework-output-dir $(Build.BinariesDirectory)/xcframework_out \
874 --pod-version ${ORT_EXTENSIONS_POD_VERSION}
875 displayName: "Assemble pod"
876
877 # Note: In this CI, we only specify to build for iphonesimulator x86_64 arch in build_framework.py command however
878 # this test app's podfile by default is setup for all platforms, and due to that we have to explicitly exclude the
879 # macos target below when installing the pod for the test app.
880 - script: |
881 ORT_EXTENSIONS_LOCAL_POD_PATH=$(Build.BinariesDirectory)/pod_staging \
882 EXCLUDE_MACOS_TARGET=true \
883 pod install
884 displayName: "Install pods for OrtExtensionsUsage"
885 workingDirectory: $(Build.SourcesDirectory)/test/ios/OrtExtensionsUsage
886
887 - script: |
888 set -e
889
890 SIMULATOR_DEVICE_INFO=$(python ./tools/ios/get_simulator_device_info.py)
891
892 echo "Simulator device info:"
893 echo "${SIMULATOR_DEVICE_INFO}"
894
895 SIMULATOR_DEVICE_ID=$(jq --raw-output '.device_udid' <<< "${SIMULATOR_DEVICE_INFO}")
896
897 # Do not output ##vso[] commands with `set -x` or they may be parsed again and include a trailing quote.
898 set +x
899 echo "##vso[task.setvariable variable=ORT_EXTENSIONS_SIMULATOR_DEVICE_ID]${SIMULATOR_DEVICE_ID}"
900 displayName: "Get simulator device info"
901
902 - script: |
903 xcrun simctl bootstatus ${ORT_EXTENSIONS_SIMULATOR_DEVICE_ID} -b
904 displayName: "Wait for simulator device to boot"
905
906 - script: |
907 xcrun xcodebuild \
908 -sdk iphonesimulator \
909 -configuration Debug \
910 -parallel-testing-enabled NO \
911 -workspace $(Build.SourcesDirectory)/test/ios/OrtExtensionsUsage/OrtExtensionsUsage.xcworkspace \
912 -scheme OrtExtensionsUsage \
913 -destination "platform=iOS Simulator,id=${ORT_EXTENSIONS_SIMULATOR_DEVICE_ID}" \
914 test CODE_SIGNING_ALLOWED=NO
915 displayName: "Build and test OrtExtensionsUsage"
916