microsoft/onnxruntime-extensions

Public

mirrored fromhttps://github.com/microsoft/onnxruntime-extensionsAvailable

Watch0 Fork0 Star0

Code Commits Issues Pull requests Actions Insights Security

27132ced71e5e35e3ee706398a316010a5ada1d9

Find a branch or tag

Branches

27132ced71e5e35e3ee706398a316010a5ada1d9

Clone

HTTPS

Download ZIP

onnxruntime-extensions/.pipelines

.pipelines/ci.yml

533lines · modecode

Raw Download

Latest commit unavailable.

unknown

1	`# IMPORTANT: READ ME BEFORE MAKING ANY CHANGES`
2	`#`
3	`# Supported matrix should follow these conventions strictly and`
4	`# for any exceptions to the rules, please document them in comments.`
5	`#`
6	`# * When adding new configurations to the matrix, update all platforms simultaneously.`
7	`# * Onnxruntime version should be one release ahead of torch, torchvision, and torchaudio.`
8	`#`
9	`# The rationale is that the onnxruntime release process would have verified the`
10	`# last available version of its dependencies (including the three listed above)`
11	`# and so it's likely to be _better_ supported relative to any older versions.`
12
13	`jobs:`
14
15	`#######`
16	`# Linux`
17	`#######`
18	`- job: Linux`
19	`pool:`
20	`vmImage: 'ubuntu-latest'`
21
22	`strategy:`
23	`matrix:`
24	`py311-1150:`
25	`python.version: '3.11'`
26	`torch.version: 'torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu'`
27	`ort.version: '1.15.0'`
28	`py310-1141:`
29	`python.version: '3.10'`
30	`torch.version: 'torch==1.13.1+cpu torchvision==0.14.1+cpu torchaudio==0.13.1 --index-url https://download.pytorch.org/whl/cpu'`
31	`ort.version: '1.14.1'`
32	`py39-1131:`
33	`python.version: '3.9'`
34	`torch.version: 'torch==1.12.1+cpu torchvision==0.13.1+cpu torchaudio==0.12.1+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html'`
35	`ort.version: '1.13.1'`
36	`py38-1121:`
37	`python.version: '3.8'`
38	`torch.version: 'torch==1.11.0+cpu torchvision==0.12.0+cpu torchaudio==0.11.0+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html'`
39	`ort.version: '1.12.1'`
40
41	`steps:`
42	`- task: DownloadGitHubRelease@0`
43	`inputs:`
44	`connection: 'microsoft'`
45	`userRepository: 'microsoft/onnxruntime'`
46	`defaultVersionType: 'specificTag'`
47	`version: 'v$(ort.version)'`
48	`itemPattern: '-linux-x64-$(ort.version)'`
49	`downloadPath: '$(Build.SourcesDirectory)'`
50	`displayName: Download the ONNXRuntime prebuilt package.`
51
52	`- task: ExtractFiles@1`
53	`inputs:`
54	`archiveFilePatterns: '*/.tgz'`
55	`destinationFolder: '$(Build.SourcesDirectory)'`
56	`cleanDestinationFolder: false`
57	`overwriteExistingFiles: true`
58	`displayName: Unpack ONNXRuntime package.`
59
60	`- script: \|`
61	`wget https://github.com/Tencent/rapidjson/archive/refs/tags/v1.1.0.tar.gz`
62	`tar zxvf v1.1.0.tar.gz`
63	`cd rapidjson-1.1.0`
64	`mkdir build`
65	`cd build`
66	`cmake ..`
67	`sudo cmake --install .`
68	`cd ../..`
69	`git clone https://github.com/triton-inference-server/client.git --branch r23.05 ~/client`
70	`sudo ln -s ~/client/src/c++/library/libhttpclient.ldscript /lib/libhttpclient.ldscript`
71	`displayName: install deps for azure invokers`
72
73	`- script: \|`
74	`CPU_NUMBER=2 sh ./build.sh -DOCOS_ENABLE_CTEST=ON -DONNXRUNTIME_PKG_DIR=$(Build.SourcesDirectory)/onnxruntime-linux-x64-$(ort.version) -DOCOS_ENABLE_AZURE=ON`
75	`displayName: build the customop library with onnxruntime`
76
77	`- script: \|`
78	`cd out/Linux/RelWithDebInfo`
79	`ctest -C RelWithDebInfo --output-on-failure`
80	`displayName: Run C++ native tests`
81
82	`- task: UsePythonVersion@0`
83	`inputs:`
84	`versionSpec: '$(python.version)'`
85	`addToPath: true`
86
87	`- script: \|`
88	`python -m pip install --upgrade pip`
89	`python -m pip install --upgrade setuptools`
90	`python -m pip install onnxruntime==$(ort.version)`
91	`python -m pip install -r requirements.txt`
92	`displayName: Install requirements.txt`
93
94	`- script: \|`
95	`CPU_NUMBER=2 python -m pip install -e .`
96	`displayName: Build the library and tests`
97
98	`- script: python -m pip install $(torch.version)`
99	`displayName: Install pytorch`
100
101	`- script: \|`
102	`python -m pip install -r requirements-dev.txt`
103	`displayName: Install requirements-dev.txt`
104
105	`- script: python -m pytest test --verbose`
106	`displayName: Run python test`
107
108	`###########`
109	`# macOS C++`
110	`###########`
111	`- job: MacOSX`
112	`pool:`
113	`vmImage: 'macOS-13'`
114
115	`strategy:`
116	`matrix:`
117	`ort-1150:`
118	`ort.version: '1.15.0'`
119	`ort.dirname: 'onnxruntime-osx-x86_64-$(ort.version)'`
120	`ort-1141:`
121	`ort.version: '1.14.1'`
122	`ort.dirname: 'onnxruntime-osx-x86_64-$(ort.version)'`
123	`ort-1131:`
124	`ort.version: '1.13.1'`
125	`ort.dirname: 'onnxruntime-osx-x86_64-$(ort.version)'`
126	`ort-1121:`
127	`ort.version: '1.12.1'`
128	`ort.dirname: 'onnxruntime-osx-x86_64-$(ort.version)'`
129
130	`steps:`
131	`- template: templates/use-xcode-version.yml`
132
133	`# needed for onnxruntime`
134	`- script: brew install libomp`
135	`displayName: 'Install omp'`
136
137	`- task: DownloadGitHubRelease@0`
138	`inputs:`
139	`connection: 'microsoft'`
140	`userRepository: 'microsoft/onnxruntime'`
141	`defaultVersionType: 'specificTag'`
142	`version: 'v$(ort.version)'`
143	`itemPattern: '$(ort.dirname)*'`
144	`downloadPath: '$(Build.SourcesDirectory)'`
145	`displayName: Download the ONNXRuntime prebuilt package.`
146
147	`- task: ExtractFiles@1`
148	`inputs:`
149	`archiveFilePatterns: '*/.tgz'`
150	`destinationFolder: '$(Build.SourcesDirectory)'`
151	`cleanDestinationFolder: false`
152	`overwriteExistingFiles: true`
153	`displayName: Unpack ONNXRuntime package.`
154
155	`- script: \|`
156	`sh ./build.sh -DOCOS_ENABLE_CTEST=ON -DONNXRUNTIME_PKG_DIR=$(Build.SourcesDirectory)/$(ort.dirname)`
157	`displayName: build the customop library with onnxruntime`
158
159	`- script: \|`
160	`cd out/Darwin/RelWithDebInfo`
161	`ctest -C RelWithDebInfo --output-on-failure`
162	`displayName: Run C++ native tests`
163
164	`#############`
165	`# macOS Python`
166	`#############`
167	`- job: MacOSPython`
168	`pool:`
169	`vmImage: 'macOS-13'`
170
171	`strategy:`
172	`matrix:`
173	`py311-1150:`
174	`python.version: '3.11'`
175	`torch.version: 'torch torchvision torchaudio'`
176	`ort.version: '1.15.0'`
177	`py310-1141:`
178	`python.version: '3.10'`
179	`torch.version: 'torch==1.13.1 torchvision==0.14.1 torchaudio==0.13.1 --index-url https://download.pytorch.org/whl/cpu'`
180	`ort.version: '1.14.1'`
181	`py39-1131:`
182	`python.version: '3.9'`
183	`torch.version: 'torch==1.12.1 torchvision==0.13.1 torchaudio==0.12.1 -f https://download.pytorch.org/whl/torch_stable.html'`
184	`ort.version: '1.13.1'`
185	`py38-1120:`
186	`python.version: '3.8'`
187	`torch.version: 'torch==1.11.0 torchvision==0.12.0 torchaudio==0.11.0 -f https://download.pytorch.org/whl/torch_stable.html'`
188	`ort.version: '1.12.0'`
189
190	`steps:`
191	`- template: templates/use-xcode-version.yml`
192
193	`# needed for onnxruntime`
194	`- script: brew install libomp`
195	`displayName: 'Install omp'`
196
197	`- task: UsePythonVersion@0`
198	`inputs:`
199	`versionSpec: '$(python.version)'`
200	`addToPath: true`
201
202	`- script: \|`
203	`python -m pip install --upgrade pip`
204	`python -m pip install --upgrade setuptools`
205	`python -m pip install --upgrade wheel`
206	`python -m pip install onnxruntime==$(ort.version)`
207	`python -m pip install -r requirements.txt`
208	`displayName: Install requirements.txt`
209
210	`- script: \|`
211	`python -c "import onnxruntime;print(onnxruntime.__version__)"`
212	`displayName: Check installation`
213
214	`- script: \|`
215	`python -m pip install -e .`
216	`displayName: Build and install the wheel`
217
218	`- script: python -m pip install -r requirements-dev.txt`
219	`displayName: Install requirements-dev.txt`
220
221	`- script: python -m pip install $(torch.version)`
222	`displayName: Install pytorch`
223
224	`- script: cd test && python -m pytest . --verbose`
225	`displayName: Run python test`
226
227	`#########`
228	`# Windows C++`
229	`#########`
230	`- job: WindowsC`
231	`pool:`
232	`vmImage: 'windows-latest'`
233
234	`strategy:`
235	`matrix:`
236	`ort-1150:`
237	`ort.version: '1.15.0'`
238	`ort-1141:`
239	`ort.version: '1.14.1'`
240	`ort-1131:`
241	`ort.version: '1.13.1'`
242	`ort-1121:`
243	`ort.version: '1.12.1'`
244
245	`steps:`
246	`- task: DownloadGitHubRelease@0`
247	`inputs:`
248	`connection: 'microsoft'`
249	`userRepository: 'microsoft/onnxruntime'`
250	`defaultVersionType: 'specificTag'`
251	`version: 'v$(ort.version)'`
252	`itemPattern: '-win-x64-$(ort.version)'`
253	`downloadPath: '$(Build.SourcesDirectory)'`
254	`displayName: Download the ONNXRuntime prebuilt package.`
255
256	`- task: ExtractFiles@1`
257	`inputs:`
258	`archiveFilePatterns: '*/.zip'`
259	`destinationFolder: '$(Build.SourcesDirectory)'`
260	`cleanDestinationFolder: false`
261	`overwriteExistingFiles: true`
262	`displayName: Unpack ONNXRuntime package.`
263
264	`- script: \|`
265	`@echo off`
266	`set vswherepath="%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe"`
267	for /f "usebackq delims=" %%i in (`%vswherepath% -latest -property installationPath`) do (
268	`if exist "%%i\Common7\Tools\vsdevcmd.bat" (`
269	`set vsdevcmd="%%i\Common7\Tools\vsdevcmd.bat"`
270	`)`
271	`)`
272
273	`@echo %vsdevcmd% will be used as the VC compiler`
274	`@echo ##vso[task.setvariable variable=vsdevcmd]%vsdevcmd%`
275	`displayName: 'locate vsdevcmd via vswhere'`
276
277	`- script: \|`
278	`call $(vsdevcmd)`
279	`call .\build.bat -DOCOS_ENABLE_CTEST=ON -DONNXRUNTIME_PKG_DIR=.\onnxruntime-win-x64-$(ort.version) -DOCOS_ENABLE_AZURE=ON`
280	`displayName: build the customop library with onnxruntime`
281
282	`- script: \|`
283	`cd out/Windows`
284	`ctest -C RelWithDebInfo --output-on-failure`
285	`displayName: Run C++ native tests`
286
287	`- job: WindowsStaticVC`
288	`pool:`
289	`vmImage: 'windows-latest'`
290
291	`steps:`
292	`- script: \|`
293	`call .\build.bat -DOCOS_ENABLE_CTEST=ON -DCMAKE_MSVC_RUNTIME_LIBRARY="MultiThreaded$<$<CONFIG:Debug>:Debug>"`
294	`cd out/Windows`
295	`ctest -C RelWithDebInfo --output-on-failure`
296	`displayName: build and test ort-extensions with VC static runtime.`
297
298	`################`
299	`# Windows Python`
300	`################`
301	`- job: WindowsPython`
302	`pool:`
303	`vmImage: 'windows-latest'`
304
305	`strategy:`
306	`matrix:`
307	`py311-1150:`
308	`python.version: '3.11'`
309	`torch.version: 'torch torchvision torchaudio'`
310	`ort.version: '1.15.0'`
311	`py310-1141:`
312	`python.version: '3.10'`
313	`torch.version: 'torch torchvision torchaudio'`
314	`ort.version: '1.14.1'`
315	`py39-1131:`
316	`python.version: '3.9'`
317	`torch.version: 'torch==1.12.1+cpu torchvision==0.13.1+cpu torchaudio==0.12.1+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html'`
318	`ort.version: '1.13.1'`
319	`py38-1121:`
320	`python.version: '3.8'`
321	`torch.version: 'torch==1.11.0+cpu torchvision==0.12.0+cpu torchaudio==0.11.0+cpu -f https://download.pytorch.org/whl/cpu/torch_stable.html'`
322	`ort.version: '1.12.1'`
323
324	`steps:`
325	`- powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"`
326	`displayName: Add conda to PATH`
327
328	`- script: conda create --yes --quiet --name pyenv -c conda-forge python=$(python.version) numpy`
329	`displayName: Create Anaconda environment`
330
331	`- script: \|`
332	`call activate pyenv`
333	`python -m pip install --upgrade pip`
334	`python -m pip install onnxruntime==$(ort.version)`
335	`python -m pip install -r requirements.txt`
336	`python -m pip install -r requirements-dev.txt`
337	`displayName: Install requirements.txt`
338
339	`- script: \|`
340	`call activate pyenv`
341	`python -m pip install .`
342	`displayName: Build the wheel`
343
344	`- script: \|`
345	`call activate pyenv`
346	`python -m pip install $(torch.version)`
347	`displayName: Install pytorch`
348
349	`- script: \|`
350	`call activate pyenv`
351	`pytest test`
352	`displayName: Run python test`
353
354	`#############`
355	`# WebAssembly`
356	`#############`
357	`- job: WebAssembly`
358	`pool:`
359	`vmImage: 'ubuntu-latest'`
360
361	`steps:`
362	`- script: \|`
363	`cd $(Build.BinariesDirectory)`
364	`git clone https://github.com/emscripten-core/emsdk`
365	`emsdk/emsdk install latest`
366	`emsdk/emsdk activate latest`
367	`displayName: Setup emscripten pipeline`
368
369	`- script: \|`
370	`bash ./build.sh \`
371	`-DCMAKE_TOOLCHAIN_FILE=$(Build.BinariesDirectory)/emsdk/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake \`
372	`-DOCOS_ENABLE_SPM_TOKENIZER=ON \`
373	`-DOCOS_BUILD_PYTHON=OFF \`
374	`-DOCOS_ENABLE_CV2=OFF \`
375	`-DOCOS_ENABLE_VISION=OFF`
376	`displayName: build the customop library with onnxruntime`
377	`# TODO add unittest for webassembly`
378
379	`#############`
380	`# Android`
381	`#############`
382	`- job: AndroidPackage`
383	`pool:`
384	`vmImage: 'macOS-13'`
385	`timeoutInMinutes: 120`
386	`steps:`
387	`- task: UsePythonVersion@0`
388	`inputs:`
389	`versionSpec: "3.9"`
390	`addToPath: true`
391	`architecture: "x64"`
392	`displayName: "Use Python 3.9"`
393
394	`- task: JavaToolInstaller@0`
395	`displayName: Use jdk 11`
396	`inputs:`
397	`versionSpec: '11'`
398	`jdkArchitectureOption: 'x64'`
399	`jdkSourceOption: 'PreInstalled'`
400
401	`- script: brew install coreutils ninja`
402	`displayName: Install coreutils and ninja`
403
404	`- template: templates/run-with-android-emulator-steps.yml`
405	`parameters:`
406	`steps:`
407
408	`- bash: \|`
409	`set -e -x`
410
411	`_BUILD_CFG="x86_64 $(Build.BinariesDirectory)/android_aar" ./build.android`
412
413	`VERSION=$(cat ./version.txt)`
414	`AAR_PATH="$(Build.BinariesDirectory)/android_aar/aar_out/com/microsoft/onnxruntime/onnxruntime-extensions-android/${VERSION}/onnxruntime-extensions-android-${VERSION}.aar"`
415
416	# Do not output ##vso[] commands with `set -x` or they may be parsed again and include a trailing quote.
417	`set +x`
418	`echo "##vso[task.setvariable variable=ORT_EXTENSIONS_AAR_PATH]${AAR_PATH}"`
419	`displayName: Build onnxruntime-extensions AAR package`
420
421	`- bash: \|`
422	`set -e -x`
423
424	`cp -r $(Build.SourcesDirectory)/java/src/test/android $(Build.BinariesDirectory)/android_test`
425
426	`cd $(Build.BinariesDirectory)/android_test`
427
428	`./gradlew connectedDebugAndroidTest --no-daemon -DortExtensionsAarLocalPath="${ORT_EXTENSIONS_AAR_PATH}"`
429	`displayName: Build and run onnxruntime-extensions Android test with Android Emulator`
430
431	`#############`
432	`# iOS`
433	`#############`
434	`- job: IosPackage`
435	`pool:`
436	`vmImage: 'macOS-13'`
437	`timeoutInMinutes: 120`
438	`steps:`
439	`- template: templates/use-xcode-version.yml`
440
441	`- task: UsePythonVersion@0`
442	`inputs:`
443	`versionSpec: '3.9'`
444	`addToPath: true`
445	`architecture: 'x64'`
446	`displayName: "Use Python 3.9"`
447
448	`# iOS xcframework build doesn't work with CMake 3.25.1, pin to 3.25.0`
449	`- script: \|`
450	`python -m pip install cmake==3.25.0`
451	`displayName: "Install CMake 3.25.0"`
452
453	`- template: templates/set-package-version-variable-step.yml`
454	`parameters:`
455	`PackageVersionVariableName: ORT_EXTENSIONS_POD_VERSION`
456
457	`- script: \|`
458	`python ./tools/ios/build_xcframework.py \`
459	`--output_dir $(Build.BinariesDirectory)/xcframework_out \`
460	`--platform_arch iphonesimulator x86_64 \`
461	`--config Release`
462	`displayName: Build xcframework for iphonesimulator x86_64`
463
464	`- script: \|`
465	`python ./tools/ios/assemble_pod_package.py \`
466	`--staging-dir $(Build.BinariesDirectory)/pod_staging \`
467	`--xcframework-output-dir $(Build.BinariesDirectory)/xcframework_out \`
468	`--pod-version ${ORT_EXTENSIONS_POD_VERSION}`
469	`displayName: Assemble pod`
470
471	`- script: \|`
472	`ORT_EXTENSIONS_LOCAL_POD_PATH=$(Build.BinariesDirectory)/pod_staging \`
473	`pod install`
474	`displayName: Install pods for OrtExtensionsUsage`
475	`workingDirectory: $(Build.SourcesDirectory)/test/ios/OrtExtensionsUsage`
476
477	`- template: templates/run-with-ios-simulator-steps.yml`
478	`parameters:`
479	`steps:`
480	`- script: \|`
481	`xcrun xcodebuild \`
482	`-sdk iphonesimulator \`
483	`-configuration Debug \`
484	`-workspace $(Build.SourcesDirectory)/test/ios/OrtExtensionsUsage/OrtExtensionsUsage.xcworkspace \`
485	`-scheme OrtExtensionsUsage \`
486	`-destination "platform=iOS Simulator,id=${ORT_EXTENSIONS_BUILD_SIMULATOR_ID}" \`
487	`test CODE_SIGNING_ALLOWED=NO`
488	`displayName: Build and test OrtExtensionsUsage`
489
490
491	`#####################################`
492	`# Linux prevent exception propagation`
493	`#####################################`
494	`- job: Linux_Prevent_Exception_Propagation`
495	`pool:`
496	`vmImage: 'ubuntu-latest'`
497
498	`steps:`
499	`# Simulate an embedded build as part of ORT with exceptions disabled by manually setting CMAKE_CXX_FLAGS and`
500	`# using _OCOS_PREVENT_EXCEPTION_PROPAGATION_OVERRIDE. The build should re-enable exceptions within ort-ext`
501	`# but prevent them from propagating. Unit tests are run to validate this.`
502	`- script: '`
503	`./build_lib.sh --enable_cxx_tests --onnxruntime_version 1.14.0 --config RelWithDebInfo`
504	`--cmake_extra_defines`
505	`_OCOS_PREVENT_EXCEPTION_PROPAGATION_OVERRIDE=ON OCOS_ENABLE_CPP_EXCEPTIONS=OFF`
506	`CMAKE_CXX_FLAGS="-fno-exceptions -fno-unwind-tables -fno-asynchronous-unwind-tables"`
507	`'`
508
509	`displayName: Build ort-ext with exception propagation disabled`
510
511	`# As an extra validation check CMakeCache.txt as well`
512	`- script: \|`
513	`grep "^_OCOS_PREVENT_EXCEPTION_PROPAGATION.*ON$" build/Linux/RelWithDebInfo/CMakeCache.txt`
514	`if [ $? -ne 0 ]; then`
515	`echo "Exception propogation was not enabled correctly."`
516	`exit 1`
517	`fi`
518
519
520	`##############################`
521	`# Linux for selected_ops build`
522	`##############################`
523	`- job: Linux_SelectedOpsBuild`
524	`pool:`
525	`vmImage: 'ubuntu-latest'`
526
527	`steps:`
528	`# compiled as only one operator selected.`
529	`- bash: \|`
530	`set -e -x -u`
531	`echo 'set (OCOS_ENABLE_BERT_TOKENIZER ON CACHE BOOL "" FORCE)' > cmake/_selectedoplist.cmake`
532	`./build.sh -DOCOS_ENABLE_CPP_EXCEPTIONS=OFF -DOCOS_ENABLE_SELECTED_OPLIST=ON`
533	`displayName: Build ort-extensions with only one operator was selected`
534

microsoft/onnxruntime-extensions

Branches

Tags

Clone