microsoft/qdk
Publicmirrored from https://github.com/microsoft/qdkAvailable
build.py
507lines · modecode
| 1 | #!/usr/bin/env python3 |
| 2 | |
| 3 | # Copyright (c) Microsoft Corporation. |
| 4 | # Licensed under the MIT License. |
| 5 | |
| 6 | import argparse |
| 7 | import os |
| 8 | import platform |
| 9 | import sys |
| 10 | import time |
| 11 | import venv |
| 12 | import shutil |
| 13 | import subprocess |
| 14 | import functools |
| 15 | |
| 16 | from prereqs import check_prereqs |
| 17 | |
| 18 | # Disable buffered output so that the log statements and subprocess output get interleaved in proper order |
| 19 | print = functools.partial(print, flush=True) |
| 20 | |
| 21 | parser = argparse.ArgumentParser( |
| 22 | description="Builds all projects in the repo, unless specific projects to build are passed " |
| 23 | "as options, in which case only those projects are built." |
| 24 | ) |
| 25 | |
| 26 | parser.add_argument( |
| 27 | "--cli", action="store_true", help="Build the command-line compiler" |
| 28 | ) |
| 29 | parser.add_argument("--pip", action="store_true", help="Build the pip wheel") |
| 30 | parser.add_argument("--widgets", action="store_true", help="Build the Python widgets") |
| 31 | parser.add_argument("--wasm", action="store_true", help="Build the WebAssembly files") |
| 32 | parser.add_argument("--npm", action="store_true", help="Build the npm package") |
| 33 | parser.add_argument("--play", action="store_true", help="Build the web playground") |
| 34 | parser.add_argument("--samples", action="store_true", help="Compile the Q# samples") |
| 35 | parser.add_argument("--vscode", action="store_true", help="Build the VS Code extension") |
| 36 | parser.add_argument( |
| 37 | "--jupyterlab", action="store_true", help="Build the JupyterLab extension" |
| 38 | ) |
| 39 | |
| 40 | parser.add_argument( |
| 41 | "--debug", action="store_true", help="Create a debug build (default is release)" |
| 42 | ) |
| 43 | parser.add_argument( |
| 44 | "--test", |
| 45 | action=argparse.BooleanOptionalAction, |
| 46 | default=True, |
| 47 | help="Run the tests (default is --test)", |
| 48 | ) |
| 49 | |
| 50 | # Below allows for passing --no-check to avoid the default of True |
| 51 | parser.add_argument( |
| 52 | "--check", |
| 53 | action=argparse.BooleanOptionalAction, |
| 54 | default=True, |
| 55 | help="Run the linting and formatting checks (default is --check)", |
| 56 | ) |
| 57 | |
| 58 | parser.add_argument( |
| 59 | "--check-prereqs", |
| 60 | action=argparse.BooleanOptionalAction, |
| 61 | default=True, |
| 62 | help="Run the prerequisites check (default is --check-prereqs)", |
| 63 | ) |
| 64 | |
| 65 | parser.add_argument( |
| 66 | "--integration-tests", |
| 67 | action=argparse.BooleanOptionalAction, |
| 68 | default=False, |
| 69 | help="Build and run the integration tests (default is --no-integration-tests)", |
| 70 | ) |
| 71 | |
| 72 | args = parser.parse_args() |
| 73 | |
| 74 | if args.check_prereqs: |
| 75 | check_prereqs() |
| 76 | |
| 77 | # If no specific project given then build all |
| 78 | build_all = ( |
| 79 | not args.cli |
| 80 | and not args.pip |
| 81 | and not args.widgets |
| 82 | and not args.wasm |
| 83 | and not args.samples |
| 84 | and not args.npm |
| 85 | and not args.play |
| 86 | and not args.vscode |
| 87 | and not args.jupyterlab |
| 88 | ) |
| 89 | build_cli = build_all or args.cli |
| 90 | build_pip = build_all or args.pip |
| 91 | build_widgets = build_all or args.widgets |
| 92 | build_wasm = build_all or args.wasm |
| 93 | build_samples = build_all or args.samples |
| 94 | build_npm = build_all or args.npm |
| 95 | build_play = build_all or args.play |
| 96 | build_vscode = build_all or args.vscode |
| 97 | build_jupyterlab = build_all or args.jupyterlab |
| 98 | |
| 99 | # JavaScript projects and eslint, prettier depend on npm_install |
| 100 | # However the JupyterLab extension uses yarn in a separate workspace |
| 101 | npm_install_needed = ( |
| 102 | build_npm or build_play or build_vscode or build_jupyterlab or args.check |
| 103 | ) |
| 104 | npm_cmd = "npm.cmd" if platform.system() == "Windows" else "npm" |
| 105 | |
| 106 | build_type = "debug" if args.debug else "release" |
| 107 | run_tests = args.test |
| 108 | |
| 109 | root_dir = os.path.dirname(os.path.abspath(__file__)) |
| 110 | wasm_src = os.path.join(root_dir, "wasm") |
| 111 | wasm_bld = os.path.join(root_dir, "target", "wasm32", build_type) |
| 112 | samples_src = os.path.join(root_dir, "samples") |
| 113 | npm_src = os.path.join(root_dir, "npm") |
| 114 | play_src = os.path.join(root_dir, "playground") |
| 115 | pip_src = os.path.join(root_dir, "pip") |
| 116 | widgets_src = os.path.join(root_dir, "widgets") |
| 117 | wheels_dir = os.path.join(root_dir, "target", "wheels") |
| 118 | vscode_src = os.path.join(root_dir, "vscode") |
| 119 | jupyterlab_src = os.path.join(root_dir, "jupyterlab") |
| 120 | |
| 121 | |
| 122 | def step_start(description): |
| 123 | global start_time |
| 124 | prefix = "::group::" if os.getenv("GITHUB_ACTIONS") == "true" else "" |
| 125 | print(f"{prefix}build.py: {description}") |
| 126 | start_time = time.time() |
| 127 | |
| 128 | |
| 129 | def step_end(): |
| 130 | global start_time |
| 131 | duration = time.time() - start_time |
| 132 | print(f"build.py: Finished in {duration:.3f}s.") |
| 133 | if os.getenv("GITHUB_ACTIONS") == "true": |
| 134 | print(f"::endgroup::") |
| 135 | |
| 136 | |
| 137 | def use_python_env(folder): |
| 138 | # Check if in a virtual environment |
| 139 | if ( |
| 140 | os.environ.get("VIRTUAL_ENV") is None |
| 141 | and os.environ.get("CONDA_PREFIX") is None |
| 142 | and os.environ.get("CI") is None |
| 143 | ): |
| 144 | print("Not in a virtual python environment") |
| 145 | |
| 146 | venv_dir = os.path.join(folder, ".venv") |
| 147 | # Create virtual environment under repo root |
| 148 | if not os.path.exists(venv_dir): |
| 149 | print(f"Creating a virtual environment under {venv_dir}") |
| 150 | venv.main([venv_dir]) |
| 151 | |
| 152 | # Check if .venv/bin/python exists, otherwise use .venv/Scripts/python.exe (Windows) |
| 153 | python_bin = os.path.join(venv_dir, "bin", "python") |
| 154 | if not os.path.exists(python_bin): |
| 155 | python_bin = os.path.join(venv_dir, "Scripts", "python.exe") |
| 156 | print(f"Using python from {python_bin}") |
| 157 | else: |
| 158 | # Already in a virtual environment, use current Python |
| 159 | python_bin = sys.executable |
| 160 | |
| 161 | return python_bin |
| 162 | |
| 163 | |
| 164 | if npm_install_needed: |
| 165 | step_start("Running npm install") |
| 166 | subprocess.run([npm_cmd, "install"], check=True, text=True, cwd=root_dir) |
| 167 | step_end() |
| 168 | |
| 169 | if args.check: |
| 170 | step_start("Running eslint and prettier checks") |
| 171 | subprocess.run([npm_cmd, "run", "check"], check=True, text=True, cwd=root_dir) |
| 172 | |
| 173 | if build_wasm or build_cli: |
| 174 | # If we're going to check the Rust code, do this before we try to compile it |
| 175 | print("Running the cargo fmt and clippy checks") |
| 176 | subprocess.run( |
| 177 | ["cargo", "fmt", "--all", "--", "--check"], |
| 178 | check=True, |
| 179 | text=True, |
| 180 | cwd=root_dir, |
| 181 | ) |
| 182 | subprocess.run( |
| 183 | [ |
| 184 | "cargo", |
| 185 | "clippy", |
| 186 | "--all-targets", |
| 187 | "--all-features", |
| 188 | "--", |
| 189 | "-D", |
| 190 | "warnings", |
| 191 | ], |
| 192 | check=True, |
| 193 | text=True, |
| 194 | cwd=root_dir, |
| 195 | ) |
| 196 | step_end() |
| 197 | |
| 198 | if build_cli: |
| 199 | step_start("Building the command line compiler") |
| 200 | cargo_build_args = ["cargo", "build"] |
| 201 | if build_type == "release": |
| 202 | cargo_build_args.append("--release") |
| 203 | subprocess.run(cargo_build_args, check=True, text=True, cwd=root_dir) |
| 204 | |
| 205 | if run_tests: |
| 206 | print("Running tests for the command line compiler") |
| 207 | cargo_test_args = ["cargo", "test"] |
| 208 | if build_type == "release": |
| 209 | cargo_test_args.append("--release") |
| 210 | subprocess.run(cargo_test_args, check=True, text=True, cwd=root_dir) |
| 211 | step_end() |
| 212 | |
| 213 | if build_pip: |
| 214 | step_start("Building the pip package") |
| 215 | |
| 216 | python_bin = use_python_env(pip_src) |
| 217 | |
| 218 | # copy the process env vars |
| 219 | pip_env: dict[str, str] = os.environ.copy() |
| 220 | if platform.system() == "Darwin": |
| 221 | # if on mac, add the arch flags for universal binary |
| 222 | pip_env["ARCHFLAGS"] = "-arch x86_64 -arch arm64" |
| 223 | |
| 224 | pip_build_args = [ |
| 225 | python_bin, |
| 226 | "-m", |
| 227 | "pip", |
| 228 | "wheel", |
| 229 | "--wheel-dir", |
| 230 | wheels_dir, |
| 231 | "-v", |
| 232 | pip_src, |
| 233 | ] |
| 234 | subprocess.run(pip_build_args, check=True, text=True, cwd=pip_src, env=pip_env) |
| 235 | |
| 236 | if run_tests: |
| 237 | print("Running tests for the pip package") |
| 238 | |
| 239 | pip_install_args = [ |
| 240 | python_bin, |
| 241 | "-m", |
| 242 | "pip", |
| 243 | "install", |
| 244 | "-r", |
| 245 | "test_requirements.txt", |
| 246 | ] |
| 247 | subprocess.run(pip_install_args, check=True, text=True, cwd=pip_src) |
| 248 | pip_install_args = [ |
| 249 | python_bin, |
| 250 | "-m", |
| 251 | "pip", |
| 252 | "install", |
| 253 | "--force-reinstall", |
| 254 | "--no-index", |
| 255 | "--find-links=" + wheels_dir, |
| 256 | f"qsharp", |
| 257 | ] |
| 258 | subprocess.run(pip_install_args, check=True, text=True, cwd=pip_src) |
| 259 | pytest_args = [python_bin, "-m", "pytest"] |
| 260 | subprocess.run( |
| 261 | pytest_args, check=True, text=True, cwd=os.path.join(pip_src, "tests") |
| 262 | ) |
| 263 | |
| 264 | qir_test_dir = os.path.join(pip_src, "tests-qir") |
| 265 | # Try to install PyQIR and if successful, run additional tests. |
| 266 | qir_install_args = [ |
| 267 | python_bin, |
| 268 | "-m", |
| 269 | "pip", |
| 270 | "install", |
| 271 | "-r", |
| 272 | "test_requirements.txt", |
| 273 | ] |
| 274 | subprocess.run(qir_install_args, check=True, text=True, cwd=qir_test_dir) |
| 275 | pyqir_check_args = [python_bin, "-c", "import pyqir"] |
| 276 | if ( |
| 277 | subprocess.run( |
| 278 | pyqir_check_args, check=False, text=True, cwd=qir_test_dir |
| 279 | ).returncode |
| 280 | == 0 |
| 281 | ): |
| 282 | print("Running tests for the pip package with PyQIR") |
| 283 | pytest_args = [python_bin, "-m", "pytest"] |
| 284 | subprocess.run(pytest_args, check=True, text=True, cwd=qir_test_dir) |
| 285 | else: |
| 286 | print("Could not import PyQIR, skipping tests") |
| 287 | step_end() |
| 288 | |
| 289 | if build_widgets: |
| 290 | step_start("Building the Python widgets") |
| 291 | |
| 292 | widgets_build_args = [ |
| 293 | sys.executable, |
| 294 | "-m", |
| 295 | "pip", |
| 296 | "wheel", |
| 297 | "--no-deps", |
| 298 | "--wheel-dir", |
| 299 | wheels_dir, |
| 300 | widgets_src, |
| 301 | ] |
| 302 | subprocess.run(widgets_build_args, check=True, text=True, cwd=widgets_src) |
| 303 | |
| 304 | step_end() |
| 305 | |
| 306 | if build_wasm: |
| 307 | step_start("Building the wasm crate") |
| 308 | # wasm-pack can't build for web and node in the same build, so need to run twice. |
| 309 | # Hopefully not needed if https://github.com/rustwasm/wasm-pack/issues/313 lands. |
| 310 | build_flag = "--release" if build_type == "release" else "--dev" |
| 311 | |
| 312 | wasm_pack_args = ["wasm-pack", "build", build_flag] |
| 313 | web_build_args = ["--target", "web", "--out-dir", os.path.join(wasm_bld, "web")] |
| 314 | node_build_args = [ |
| 315 | "--target", |
| 316 | "nodejs", |
| 317 | "--out-dir", |
| 318 | os.path.join(wasm_bld, "node"), |
| 319 | ] |
| 320 | subprocess.run(wasm_pack_args + web_build_args, check=True, text=True, cwd=wasm_src) |
| 321 | subprocess.run( |
| 322 | wasm_pack_args + node_build_args, check=True, text=True, cwd=wasm_src |
| 323 | ) |
| 324 | step_end() |
| 325 | |
| 326 | if build_samples: |
| 327 | step_start("Building qsharp samples") |
| 328 | project_directories = [ |
| 329 | dir for dir in os.walk(samples_src) if "qsharp.json" in dir[2] |
| 330 | ] |
| 331 | single_file_directories = [ |
| 332 | candidate |
| 333 | for candidate in os.walk(samples_src) |
| 334 | if all([not proj_dir[0] in candidate[0] for proj_dir in project_directories]) |
| 335 | ] |
| 336 | |
| 337 | files = [ |
| 338 | os.path.join(dp, f) |
| 339 | for dp, _, filenames in single_file_directories |
| 340 | for f in filenames |
| 341 | if os.path.splitext(f)[1] == ".qs" |
| 342 | ] |
| 343 | projects = [ |
| 344 | os.path.join(dp, f) |
| 345 | for dp, _, filenames in project_directories |
| 346 | for f in filenames |
| 347 | if f == "qsharp.json" |
| 348 | ] |
| 349 | cargo_args = ["cargo", "run", "--bin", "qsc"] |
| 350 | if build_type == "release": |
| 351 | cargo_args.append("--release") |
| 352 | for file in files: |
| 353 | subprocess.run((cargo_args + ["--", file]), check=True, text=True, cwd=root_dir) |
| 354 | for project in projects: |
| 355 | subprocess.run( |
| 356 | (cargo_args + ["--", "--qsharp-json", project]), |
| 357 | check=True, |
| 358 | text=True, |
| 359 | cwd=root_dir, |
| 360 | ) |
| 361 | step_end() |
| 362 | |
| 363 | if build_npm: |
| 364 | step_start("Building the npm package") |
| 365 | # Copy the wasm build files over for web and node targets |
| 366 | for target in ["web", "node"]: |
| 367 | lib_dir = os.path.join(npm_src, "lib", target) |
| 368 | os.makedirs(lib_dir, exist_ok=True) |
| 369 | |
| 370 | for filename in ["qsc_wasm_bg.wasm", "qsc_wasm.d.ts", "qsc_wasm.js"]: |
| 371 | fullpath = os.path.join(wasm_bld, target, filename) |
| 372 | |
| 373 | # To make the node files CommonJS modules, the extension needs to change |
| 374 | # (This is because the package is set to ECMAScript modules by default) |
| 375 | if target == "node" and filename == "qsc_wasm.js": |
| 376 | filename = "qsc_wasm.cjs" |
| 377 | if target == "node" and filename == "qsc_wasm.d.ts": |
| 378 | filename = "qsc_wasm.d.cts" |
| 379 | |
| 380 | shutil.copy2(fullpath, os.path.join(lib_dir, filename)) |
| 381 | |
| 382 | npm_args = [npm_cmd, "run", "build"] |
| 383 | subprocess.run(npm_args, check=True, text=True, cwd=npm_src) |
| 384 | |
| 385 | if run_tests: |
| 386 | print("Running tests for the npm package") |
| 387 | npm_test_args = ["node", "--test"] |
| 388 | subprocess.run(npm_test_args, check=True, text=True, cwd=npm_src) |
| 389 | step_end() |
| 390 | |
| 391 | if build_play: |
| 392 | step_start("Building the playground") |
| 393 | play_args = [npm_cmd, "run", "build"] |
| 394 | subprocess.run(play_args, check=True, text=True, cwd=play_src) |
| 395 | step_end() |
| 396 | |
| 397 | if build_vscode: |
| 398 | step_start("Building the VS Code extension") |
| 399 | vscode_args = [npm_cmd, "run", "build"] |
| 400 | subprocess.run(vscode_args, check=True, text=True, cwd=vscode_src) |
| 401 | step_end() |
| 402 | if args.integration_tests: |
| 403 | step_start("Running the VS Code integration tests") |
| 404 | vscode_args = [npm_cmd, "test"] |
| 405 | subprocess.run(vscode_args, check=True, text=True, cwd=vscode_src) |
| 406 | step_end() |
| 407 | |
| 408 | |
| 409 | if build_jupyterlab: |
| 410 | step_start("Building the JupyterLab extension") |
| 411 | |
| 412 | python_bin = use_python_env(jupyterlab_src) |
| 413 | |
| 414 | pip_build_args = [ |
| 415 | python_bin, |
| 416 | "-m", |
| 417 | "pip", |
| 418 | "wheel", |
| 419 | "--wheel-dir", |
| 420 | wheels_dir, |
| 421 | jupyterlab_src, |
| 422 | ] |
| 423 | subprocess.run(pip_build_args, check=True, text=True, cwd=jupyterlab_src) |
| 424 | step_end() |
| 425 | |
| 426 | if build_pip and build_widgets and args.integration_tests: |
| 427 | step_start("Running notebook samples integration tests") |
| 428 | # Find all notebooks in the samples directory. Skip the basic sample and the azure submission sample, since those won't run |
| 429 | # nicely in automation. |
| 430 | notebook_files = [ |
| 431 | os.path.join(dp, f) |
| 432 | for dp, _, filenames in os.walk(samples_src) |
| 433 | for f in filenames |
| 434 | if f.endswith(".ipynb") |
| 435 | and not (f.startswith("sample.") or f.startswith("azure_submission.")) |
| 436 | ] |
| 437 | python_bin = use_python_env(samples_src) |
| 438 | |
| 439 | # copy the process env vars |
| 440 | pip_env: dict[str, str] = os.environ.copy() |
| 441 | |
| 442 | # Install the qsharp package |
| 443 | pip_install_args = [ |
| 444 | python_bin, |
| 445 | "-m", |
| 446 | "pip", |
| 447 | "install", |
| 448 | "--force-reinstall", |
| 449 | "--no-index", |
| 450 | "--find-links=" + wheels_dir, |
| 451 | f"qsharp", |
| 452 | ] |
| 453 | subprocess.run(pip_install_args, check=True, text=True, cwd=pip_src) |
| 454 | |
| 455 | # Install the widgets package from the folder so dependencies are installed properly |
| 456 | pip_install_args = [ |
| 457 | python_bin, |
| 458 | "-m", |
| 459 | "pip", |
| 460 | "install", |
| 461 | widgets_src, |
| 462 | ] |
| 463 | subprocess.run( |
| 464 | pip_install_args, check=True, text=True, cwd=widgets_src, env=pip_env |
| 465 | ) |
| 466 | |
| 467 | # Install other dependencies |
| 468 | pip_install_args = [ |
| 469 | python_bin, |
| 470 | "-m", |
| 471 | "pip", |
| 472 | "install", |
| 473 | "ipykernel", |
| 474 | "nbconvert", |
| 475 | "pandas", |
| 476 | ] |
| 477 | subprocess.run(pip_install_args, check=True, text=True, cwd=root_dir, env=pip_env) |
| 478 | |
| 479 | for notebook in notebook_files: |
| 480 | print(f"Running {notebook}") |
| 481 | # Run the notebook process, capturing stdout and only displaying it if there is an error |
| 482 | result = subprocess.run( |
| 483 | [ |
| 484 | python_bin, |
| 485 | "-m", |
| 486 | "nbconvert", |
| 487 | "--to", |
| 488 | "notebook", |
| 489 | "--stdout", |
| 490 | "--ExecutePreprocessor.timeout=60", |
| 491 | "--sanitize-html", |
| 492 | "--execute", |
| 493 | notebook, |
| 494 | ], |
| 495 | check=False, |
| 496 | text=True, |
| 497 | cwd=root_dir, |
| 498 | env=pip_env, |
| 499 | stdout=subprocess.PIPE, |
| 500 | stderr=subprocess.STDOUT, |
| 501 | encoding="utf-8", |
| 502 | ) |
| 503 | if result.returncode != 0: |
| 504 | print(result.stdout) |
| 505 | raise Exception(f"Error running {notebook}") |
| 506 | |
| 507 | step_end() |
| 508 | |