microsoft/mu_tiano_platforms

Public

mirrored fromhttps://github.com/microsoft/mu_tiano_platformsAvailable

CodeCommitsIssuesPull requestsActionsInsightsSecurity
ace7fc035221a41dc04ae7b40887cd71ef9da7c6

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

.github/workflows/codeql-platform.yml

563lines · modecode

1# This workflow runs CodeQL against platform builds in the repository.
2#
3# Any platform that supports the `--codeql` parameter will be built and the
4# results will be uploaded to GitHub Code Scanning.
5#
6# Note: Important: This file only works with "platform" builds. "CI" builds are
7# supported with the codeql.yml file.
8#
9# Note: This workflow only supports Windows as CodeQL CLI has confirmed issues running
10# against edk2-style codebases on Linux (only tested on Ubuntu). Therefore, this
11# workflow is written only for Windows but could easily be adapted to run on Linux
12# in the future if needed (e.g. swap out "windows" with agent OS var value, etc.)
13#
14# For details about the Linux issue see: https://github.com/github/codeql-action/issues/1338
15#
16# NOTE: This file is automatically synchronized from Mu DevOps. Update the original file there
17# instead of the file in this repo.
18#
19# - Mu DevOps Repo: https://github.com/microsoft/mu_devops
20# - File Sync Settings: https://github.com/microsoft/mu_devops/blob/main/.sync/Files.yml
21#
22# Copyright (c) Microsoft Corporation.
23# SPDX-License-Identifier: BSD-2-Clause-Patent
24
25
26name: "CodeQL - Platform"
27
28on:
29 push:
30 branches:
31 - main
32 - release/*
33 pull_request:
34 branches:
35 - main
36 - release/*
37 paths-ignore:
38 - '!**.c'
39 - '!**.h'
40
41jobs:
42 gather_build_files:
43 name: Gather Platform Build Files
44 runs-on: ubuntu-latest
45 permissions:
46 contents: read
47 outputs:
48 platform_build_files: ${{ steps.generate_matrix.outputs.platform_build_files }}
49
50 steps:
51 - name: Checkout repository
52 uses: actions/checkout@v6
53
54
55
56 - name: Install Python
57 uses: actions/setup-python@v6
58 with:
59 python-version: '3.12'
60 cache: 'pip'
61 cache-dependency-path: 'pip-requirements.txt'
62
63
64 - name: Install/Upgrade pip Modules
65 run: pip install -r pip-requirements.txt --upgrade
66
67 - name: Generate Package Matrix
68 id: generate_matrix
69 shell: python
70 run: |
71 import os
72 import json
73 from pathlib import Path
74
75
76 def supports_parameter(script_path: str, parameter: str):
77 import subprocess
78
79 try:
80 # Run the script with the --help parameter and capture the output
81 # Note: subprocess.run() failed in the GitHub workflow
82 process = subprocess.Popen(['python', script_path, '--help'],
83 stdout=subprocess.PIPE,
84 stderr=subprocess.PIPE)
85
86 output, error = process.communicate()
87 if process.returncode != 0:
88 print(f"::error title=CodeQL Check Failed!::Failed to determine if the platform is CodeQL compatible! Return code {process.returncode}. {error}")
89 return False
90
91 return parameter in output.decode('utf-8')
92 except Exception as e:
93 print(f"::error title=CodeQL Check Exception!::Exception occurred while checking if the platform is CodeQL compatible! {e}")
94 return False
95
96
97 root_dir = Path(os.environ['GITHUB_WORKSPACE'])
98 packages = [d for d in root_dir.rglob('*') if d.is_dir() and d.name.strip().lower().endswith('pkg')]
99
100 platform_build_files = []
101 # This can be made more robust than just checking if a PlatformBuild.py
102 # file exists in the package directory, but the additional complexity is
103 # not needed right now given current package construction conventions.
104 for package in packages:
105 platform_build_file = package / "PlatformBuild.py"
106 if platform_build_file.exists() and platform_build_file.is_file() \
107 and any(file.endswith('.dsc') for file in os.listdir(package)) \
108 and supports_parameter(str(platform_build_file), "--codeql"):
109 platform_build_files.append(str(platform_build_file.relative_to(root_dir)))
110
111 platform_build_files.sort()
112
113 print(platform_build_files)
114
115 with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
116 print(f'platform_build_files={json.dumps(platform_build_files)}', file=fh)
117
118 analyze:
119 name: Analyze
120 runs-on: windows-2022
121 needs:
122 - gather_build_files
123 permissions:
124 actions: read
125 contents: read
126 security-events: write
127
128 strategy:
129 fail-fast: false
130 matrix:
131 build_file: ${{ fromJson(needs.gather_build_files.outputs.platform_build_files) }}
132 include:
133 - tool_chain_tag: VS2022
134
135 steps:
136 - name: Checkout repository
137 uses: actions/checkout@v6
138
139
140
141 - name: Install Python
142 uses: actions/setup-python@v6
143 with:
144 python-version: '3.12'
145 cache: 'pip'
146 cache-dependency-path: 'pip-requirements.txt'
147
148
149 - name: Use Git Long Paths on Windows
150 if: runner.os == 'Windows'
151 shell: pwsh
152 run: |
153 git config --system core.longpaths true
154
155 - name: Install/Upgrade pip Modules
156 run: pip install -r pip-requirements.txt --upgrade requests
157
158
159
160 - name: Get Cargo Tool Details
161 id: get_cargo_tool_details
162 shell: python
163 env:
164 AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
165 run: |
166 import os
167 import requests
168 import sys
169 import time
170
171 def get_response_with_retries(url, headers, retries=5, wait_time=10):
172 for attempt in range(retries):
173 response = requests.get(url, headers=headers)
174 if response.status_code == 200:
175 return response
176 print(f"::warning title=GitHub API Access Error!::Attempt {attempt + 1} failed ({response.status_code}). Retrying in {wait_time} seconds...")
177 time.sleep(wait_time)
178 return response
179
180 GITHUB_REPO = "sagiegurari/cargo-make"
181 api_url = f"https://api.github.com/repos/{GITHUB_REPO}/releases/tags/0.37.24"
182 headers = {
183 "Authorization": f"Bearer {os.environ['AUTH_TOKEN']}",
184 "Accept": "application/vnd.github.v3+json"
185 }
186
187 response = get_response_with_retries(api_url, headers)
188 if response.status_code == 200:
189 build_release_id = response.json()["id"]
190 else:
191 print(f"::error title=GitHub Release Error!::Failed to get cargo-make release ID! ({response.status_code})")
192 sys.exit(1)
193
194 api_url = f"https://api.github.com/repos/{GITHUB_REPO}/releases/{build_release_id}"
195
196 response = get_response_with_retries(api_url, headers)
197 if response.status_code == 200:
198 latest_cargo_make_version = response.json()["tag_name"]
199 else:
200 print(f"::error title=GitHub Release Error!::Failed to get cargo-make! ({response.status_code})")
201 sys.exit(1)
202
203 cache_key = f'cargo-make-{latest_cargo_make_version}'
204
205 with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
206 print(f'cargo_bin_path={os.path.join(os.environ["USERPROFILE"], ".cargo", "bin")}', file=fh)
207 print(f'cargo_make_cache_key={cache_key}', file=fh)
208 print(f'cargo_make_version={latest_cargo_make_version}', file=fh)
209
210
211 # Temporarily disable caching cargo-make as it stopped working in some repos recently
212 # and need to be investigated
213 # - name: Attempt to Load cargo-make From Cache
214 # id: cargo_make_cache
215 # uses: actions/cache@v5
216 # with:
217 # path: ${{ steps.get_cargo_tool_details.outputs.cargo_bin_path }}
218 # key: ${{ steps.get_cargo_tool_details.outputs.cargo_make_cache_key }}
219
220 - name: Force cargo-make cache miss
221 id: cargo_make_cache
222 run: echo "cache-hit=false" >> $GITHUB_OUTPUT
223
224 - name: Download cargo-make
225 if: steps.cargo_make_cache.outputs.cache-hit != 'true'
226 uses: robinraju/release-downloader@v1.12
227 with:
228 repository: 'sagiegurari/cargo-make'
229 tag: '${{ steps.get_cargo_tool_details.outputs.cargo_make_version }}'
230 fileName: 'cargo-make-v${{ steps.get_cargo_tool_details.outputs.cargo_make_version }}-x86_64-pc-windows-msvc.zip'
231 out-file-path: 'cargo-make-download'
232 token: ${{ secrets.GITHUB_TOKEN }}
233
234 - name: Extract cargo-make
235 if: steps.cargo_make_cache.outputs.cache-hit != 'true'
236 env:
237 CARGO_MAKE_VERSION: ${{ steps.get_cargo_tool_details.outputs.cargo_make_version }}
238 DEST_DIR: ${{steps.get_cargo_tool_details.outputs.cargo_bin_path }}
239 shell: python
240 run: |
241 import os
242 import shutil
243 import zipfile
244 from pathlib import Path
245
246 DOWNLOAD_DIR = Path(os.environ["GITHUB_WORKSPACE"], "cargo-make-download")
247 ZIP_FILE_NAME = f"cargo-make-v{os.environ['CARGO_MAKE_VERSION']}-x86_64-pc-windows-msvc.zip"
248 ZIP_FILE_PATH = Path(DOWNLOAD_DIR, ZIP_FILE_NAME)
249 EXTRACT_DIR = Path(DOWNLOAD_DIR, "cargo-make-contents")
250
251 with zipfile.ZipFile(ZIP_FILE_PATH, 'r') as zip_ref:
252 zip_ref.extractall(EXTRACT_DIR)
253
254 for extracted_file in EXTRACT_DIR.iterdir():
255 if extracted_file.name == "cargo-make.exe":
256 shutil.copy2(extracted_file, os.environ["DEST_DIR"])
257 break
258
259 - name: Rust Prep
260 run: rustup component add rust-src
261
262 - name: Get Platform Information
263 id: get_platform_info
264 env:
265 BUILD_FILE_PATH: ${{ matrix.build_file }}
266 shell: python
267 run: |
268 import importlib
269 import inspect
270 import os
271 import sys
272 from pathlib import Path
273 from edk2toolext.invocables.edk2_platform_build import BuildSettingsManager
274 from edk2toolext.invocables.edk2_ci_setup import CiSetupSettingsManager
275 from edk2toolext.invocables.edk2_setup import SetupSettingsManager
276
277 platform_build_file = Path(os.environ['BUILD_FILE_PATH'])
278 if not platform_build_file.is_file():
279 print(f"::error title=Invalid Build File!::Failed to find {str(platform_build_file)}!")
280 sys.exit(1)
281
282 # Load the module
283 module_name = 'platform_settings'
284 spec = importlib.util.spec_from_file_location(module_name, platform_build_file)
285 module = importlib.util.module_from_spec(spec)
286 spec.loader.exec_module(module)
287
288 # Get info from the platform build file
289 pkg_name = "UnknownPkg"
290 ci_setup_supported = False
291 setup_supported = False
292 for name, obj in inspect.getmembers(module):
293 if inspect.isclass(obj):
294 if issubclass(obj, CiSetupSettingsManager):
295 ci_setup_supported = True
296 if issubclass(obj, SetupSettingsManager):
297 setup_supported = True
298 if issubclass(obj, BuildSettingsManager):
299 try:
300 pkg_name = obj().GetName()
301 except AttributeError:
302 print(f"::error title=Invalid Package name!::Failed to get package name in {str(platform_build_file)}!")
303 sys.exit(1)
304
305 with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
306 print(f'ci_setup_supported={str(ci_setup_supported).lower()}', file=fh)
307 print(f'setup_supported={str(setup_supported).lower()}', file=fh)
308 print(f'pkg_name={pkg_name}', file=fh)
309
310 - name: Assign Temp Drive Letter
311 if: runner.os == 'Windows'
312 shell: pwsh
313 run: |
314 subst Z: ${{ github.workspace }}
315
316 - name: Setup
317 if: steps.get_platform_info.outputs.setup_supported == 'true'
318 shell: pwsh
319 working-directory: "Z:"
320 run: stuart_setup -c ${{ matrix.build_file }} -t DEBUG TOOL_CHAIN_TAG=${{ matrix.tool_chain_tag }}
321
322 - name: Upload Setup Log As An Artifact
323 uses: actions/upload-artifact@v6
324 if: (success() || failure()) && steps.get_platform_info.outputs.setup_supported == 'true'
325 with:
326 name: ${{ steps.get_platform_info.outputs.pkg_name }}-Setup-Log
327 path: |
328 **/SETUPLOG.txt
329 retention-days: 7
330 if-no-files-found: ignore
331
332 - name: CI Setup
333 if: steps.get_platform_info.outputs.ci_setup_supported == 'true'
334 shell: pwsh
335 working-directory: "Z:"
336 run: stuart_ci_setup -c ${{ matrix.build_file }} -t DEBUG TOOL_CHAIN_TAG=${{ matrix.tool_chain_tag }}
337
338 - name: Upload CI Setup Log As An Artifact
339 uses: actions/upload-artifact@v6
340 if: (success() || failure()) && steps.get_platform_info.outputs.ci_setup_supported == 'true'
341 with:
342 name: ${{ steps.get_platform_info.outputs.pkg_name }}-CI-Setup-Log
343 path: |
344 **/CISETUP.txt
345 retention-days: 7
346 if-no-files-found: ignore
347
348 - name: Update
349 shell: pwsh
350 working-directory: "Z:"
351 run: stuart_update -c ${{ matrix.build_file }} -t DEBUG TOOL_CHAIN_TAG=${{ matrix.tool_chain_tag }}
352
353 - name: Upload Update Log As An Artifact
354 uses: actions/upload-artifact@v6
355 if: success() || failure()
356 with:
357 name: ${{ steps.get_platform_info.outputs.pkg_name }}-Update-Log
358 path: |
359 **/UPDATE_LOG.txt
360 retention-days: 7
361 if-no-files-found: ignore
362
363 - name: Find CodeQL Plugin Directory
364 id: find_dir
365 shell: python
366 run: |
367 import os
368 import sys
369 from pathlib import Path
370
371 #
372 # Find the plugin directory that contains the CodeQL plugin.
373 #
374 # Prior to Mu Basecore 202311, the CodeQL plugin was located in .pytool. After it
375 # is located in BaseTools. First check BaseTools, but consider .pytool as a backup
376 # for backward compatibility. The .pytool backup can be removed when no longer needed
377 # for supported branches.
378 #
379 plugin_dir = list(Path(os.environ['GITHUB_WORKSPACE']).rglob('BaseTools/Plugin/CodeQL'))
380 if not plugin_dir:
381 plugin_dir = list(Path(os.environ['GITHUB_WORKSPACE']).rglob('.pytool/Plugin/CodeQL'))
382
383 # This should only be found once
384 if len(plugin_dir) == 1:
385 plugin_dir = str(plugin_dir[0])
386
387 with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
388 print(f'codeql_plugin_dir={plugin_dir}', file=fh)
389 else:
390 print("::error title=Workspace Error!::Failed to find Mu Basecore plugin directory!")
391 sys.exit(1)
392
393 - name: Get CodeQL CLI Cache Data
394 id: cache_key_gen
395 env:
396 CODEQL_PLUGIN_DIR: ${{ steps.find_dir.outputs.codeql_plugin_dir }}
397 shell: python
398 run: |
399 import os
400 import yaml
401
402 codeql_cli_ext_dep_name = 'codeqlcli_windows_ext_dep'
403 codeql_plugin_file = os.path.join(os.environ['CODEQL_PLUGIN_DIR'], codeql_cli_ext_dep_name + '.yaml')
404
405 with open (codeql_plugin_file) as pf:
406 codeql_cli_ext_dep = yaml.safe_load(pf)
407
408 cache_key_name = codeql_cli_ext_dep['name']
409 cache_key_version = codeql_cli_ext_dep['version']
410 cache_key = f'{cache_key_name}-{cache_key_version}'
411
412 codeql_plugin_cli_ext_dep_dir = os.path.join(os.environ['CODEQL_PLUGIN_DIR'], codeql_cli_ext_dep['name'].strip() + '_extdep')
413
414 with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
415 print(f'codeql_cli_cache_key={cache_key}', file=fh)
416 print(f'codeql_cli_ext_dep_dir={codeql_plugin_cli_ext_dep_dir}', file=fh)
417
418 - name: Attempt to Load CodeQL CLI From Cache
419 id: codeqlcli_cache
420 uses: actions/cache@v5
421 with:
422 path: ${{ steps.cache_key_gen.outputs.codeql_cli_ext_dep_dir }}
423 key: ${{ steps.cache_key_gen.outputs.codeql_cli_cache_key }}
424
425 - name: Download CodeQL CLI
426 if: steps.codeqlcli_cache.outputs.cache-hit != 'true'
427 shell: pwsh
428 working-directory: "Z:"
429 run: stuart_update -c ${{ matrix.build_file }} -t DEBUG TOOL_CHAIN_TAG=${{ matrix.tool_chain_tag }} --codeql
430
431 - name: Find pytool Plugin Directory
432 id: find_pytool_dir
433 shell: python
434 run: |
435 import os
436 import sys
437 from pathlib import Path
438
439 # Find the plugin directory that contains the Compiler plugin
440 plugin_dir = list(Path(os.environ['GITHUB_WORKSPACE']).rglob('.pytool/Plugin/CompilerPlugin'))
441
442 # This should only be found once
443 if len(plugin_dir) == 1:
444 # If the directory is found get the parent Plugin directory
445 plugin_dir = str(plugin_dir[0].parent)
446
447 with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
448 print(f'pytool_plugin_dir={plugin_dir}', file=fh)
449 else:
450 print("::error title=Workspace Error!::Failed to find Mu Basecore .pytool/Plugin directory!")
451 sys.exit(1)
452
453 - name: Remove CI Plugins Irrelevant to CodeQL
454 shell: python
455 env:
456 PYTOOL_PLUGIN_DIR: ${{ steps.find_pytool_dir.outputs.pytool_plugin_dir }}
457 run: |
458 import os
459 import shutil
460 from pathlib import Path
461
462 # Only these two plugins are needed for CodeQL.
463 #
464 # CodeQL build time is reduced by removing other plugins that are not needed for the CodeQL
465 # build in the .pytool directory. The CompilerPlugin is required to compile code for CodeQL
466 # to extract results from and the CodeQL plugin is necessary to to analyze the results and
467 # build the CodeQL database from them. The CodeQL plugin should be in BaseTools moving forward
468 # but still might be in .pytool in older branches so it is kept here as an exception.
469 #
470 plugins_to_keep = ['CodeQL', 'CompilerPlugin']
471
472 plugin_dir = Path(os.environ['PYTOOL_PLUGIN_DIR']).absolute()
473 if plugin_dir.is_dir():
474 for dir in plugin_dir.iterdir():
475 if str(dir.stem) not in plugins_to_keep:
476 shutil.rmtree(str(dir.absolute()), ignore_errors=True)
477
478 - name: Platform Build
479 shell: pwsh
480 working-directory: "Z:"
481 env:
482 RUST_ENV_CHECK_TOOL_EXCLUSIONS: "cargo fmt, cargo tarpaulin"
483 STUART_CODEQL_PATH: ${{ steps.cache_key_gen.outputs.codeql_cli_ext_dep_dir }}
484 run: stuart_build -c ${{ matrix.build_file }} TARGET=DEBUG TOOL_CHAIN_TAG=${{ matrix.tool_chain_tag }} --codeql
485
486 - name: Build Cleanup
487 id: build_cleanup
488 shell: python
489 run: |
490 import os
491 import shutil
492 from pathlib import Path
493
494 dirs_to_delete = ['ia32', 'x64', 'arm', 'aarch64']
495
496 def delete_dirs(path: Path):
497 if path.exists() and path.is_dir():
498 if path.name.lower() in dirs_to_delete:
499 print(f'Removed {str(path)}')
500 shutil.rmtree(path)
501 return
502
503 for child_dir in path.iterdir():
504 delete_dirs(child_dir)
505
506 build_path = Path(os.environ['GITHUB_WORKSPACE'], 'Build')
507 delete_dirs(build_path)
508
509 - name: Upload Build Logs As An Artifact
510 uses: actions/upload-artifact@v6
511 if: success() || failure()
512 with:
513 name: ${{ steps.get_platform_info.outputs.pkg_name }}-Build-Logs
514 path: |
515 **/BUILD_REPORT.TXT
516 **/OVERRIDELOG.TXT
517 **/BUILDLOG_*.md
518 **/BUILDLOG_*.txt
519 **/CI_*.md
520 **/CI_*.txt
521 retention-days: 7
522 if-no-files-found: ignore
523
524 - name: Prepare Env Data for CodeQL Upload
525 id: env_data
526 env:
527 PACKAGE_NAME: ${{ steps.get_platform_info.outputs.pkg_name }}
528 shell: python
529 run: |
530 import os
531
532 package = os.environ['PACKAGE_NAME'].strip().lower()
533 directory_name = 'codeql-analysis-' + package + '-debug'
534 file_name = 'codeql-db-' + package + '-debug-0.sarif'
535 sarif_path = os.path.join('Build', directory_name, file_name)
536
537 with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
538 print(f'sarif_file_path={sarif_path}', file=fh)
539
540 - name: Upload CodeQL Results (SARIF) As An Artifact
541 uses: actions/upload-artifact@v6
542 with:
543 name: ${{ steps.get_platform_info.outputs.pkg_name }}-CodeQL-SARIF
544 path: ${{ steps.env_data.outputs.sarif_file_path }}
545 retention-days: 14
546 if-no-files-found: warn
547
548 - name: Upload CodeQL Results (SARIF) To GitHub Code Scanning
549 uses: github/codeql-action/upload-sarif@v4
550 with:
551 # Path to SARIF file relative to the root of the repository.
552 sarif_file: ${{ steps.env_data.outputs.sarif_file_path }}
553 # Optional category for the results. Used to differentiate multiple results for one commit.
554 # Each package is a separate category.
555 category: ${{ steps.get_platform_info.outputs.pkg_name }}
556
557 - name: Remove Temp Drive Letter
558 if: runner.os == 'Windows'
559 shell: pwsh
560 run: |
561 subst Z: /D
562
563
564