microsoft/openvmm
Publicmirrored fromhttps://github.com/microsoft/openvmmAvailable
flowey/flowey_cli/src/pipeline_resolver/ado_yaml.rs
989lines · modecode
| 1 | // Copyright (c) Microsoft Corporation. |
| 2 | // Licensed under the MIT License. |
| 3 | |
| 4 | use super::common_yaml::BashCommands; |
| 5 | use super::common_yaml::FloweySource; |
| 6 | use super::common_yaml::check_generated_yaml_and_json; |
| 7 | use super::common_yaml::job_flowey_bootstrap_source; |
| 8 | use super::common_yaml::write_generated_yaml_and_json; |
| 9 | use super::generic::ResolvedJobArtifact; |
| 10 | use super::generic::ResolvedJobUseParameter; |
| 11 | use crate::cli::exec_snippet::FloweyPipelineStaticDb; |
| 12 | use crate::cli::exec_snippet::VAR_DB_SEEDVAR_FLOWEY_WORKING_DIR; |
| 13 | use crate::cli::pipeline::CheckMode; |
| 14 | use crate::cli::var_db::VarDbRequestBuilder; |
| 15 | use crate::flow_resolver::stage1_dag::OutputGraphEntry; |
| 16 | use crate::flow_resolver::stage1_dag::Step; |
| 17 | use crate::pipeline_resolver::generic::ResolvedPipeline; |
| 18 | use crate::pipeline_resolver::generic::ResolvedPipelineJob; |
| 19 | use anyhow::Context; |
| 20 | use flowey_core::node::FlowArch; |
| 21 | use flowey_core::node::FlowBackend; |
| 22 | use flowey_core::node::FlowPlatform; |
| 23 | use flowey_core::node::FlowPlatformKind; |
| 24 | use flowey_core::node::NodeHandle; |
| 25 | use flowey_core::pipeline::internal::AdoPool; |
| 26 | use flowey_core::pipeline::internal::InternalAdoResourcesRepository; |
| 27 | use std::collections::BTreeMap; |
| 28 | use std::collections::BTreeSet; |
| 29 | use std::fmt::Write; |
| 30 | use std::path::Path; |
| 31 | |
| 32 | /// We use $(Build.StagingDirectory)/.flowey-internal instead |
| 33 | /// of $(Agent.TempDirectory) to (hopefully) guarantee that this folder |
| 34 | /// resides on the same mount-point as the repos being cloned. |
| 35 | /// |
| 36 | /// violating this property would result in calls to `fs::rename` in |
| 37 | /// downstream flowey nodes to fail. |
| 38 | const FLOWEY_TEMP_DIR: &str = "$(Build.StagingDirectory)/.flowey-internal"; |
| 39 | |
| 40 | /// Emit a pipeline as a single self-contained ADO yaml file |
| 41 | pub fn ado_yaml( |
| 42 | pipeline: ResolvedPipeline, |
| 43 | runtime_debug_log: bool, |
| 44 | repo_root: &Path, |
| 45 | pipeline_file: &Path, |
| 46 | flowey_crate: &str, |
| 47 | check: CheckMode, |
| 48 | ) -> anyhow::Result<()> { |
| 49 | if pipeline_file.extension().and_then(|s| s.to_str()) != Some("yaml") { |
| 50 | anyhow::bail!("pipeline name must end with .yaml") |
| 51 | } |
| 52 | |
| 53 | let ResolvedPipeline { |
| 54 | graph, |
| 55 | order, |
| 56 | parameters, |
| 57 | ado_name, |
| 58 | ado_schedule_triggers, |
| 59 | ado_ci_triggers, |
| 60 | ado_pr_triggers, |
| 61 | ado_bootstrap_template, |
| 62 | ado_resources_repository, |
| 63 | ado_post_process_yaml_cb, |
| 64 | ado_variables, |
| 65 | ref ado_job_id_overrides, |
| 66 | gh_name: _, |
| 67 | gh_schedule_triggers: _, |
| 68 | gh_ci_triggers: _, |
| 69 | gh_pr_triggers: _, |
| 70 | gh_bootstrap_template: _, |
| 71 | } = pipeline; |
| 72 | |
| 73 | let mut job_flowey_source: BTreeMap<petgraph::prelude::NodeIndex, FloweySource> = |
| 74 | job_flowey_bootstrap_source(&graph, &order); |
| 75 | |
| 76 | let mut pipeline_static_db = FloweyPipelineStaticDb { |
| 77 | flow_backend: crate::cli::FlowBackendCli::Ado, |
| 78 | var_db_backend_kind: crate::cli::exec_snippet::VarDbBackendKind::Json, |
| 79 | job_reqs: BTreeMap::new(), |
| 80 | }; |
| 81 | |
| 82 | let mut ado_jobs = Vec::new(); |
| 83 | |
| 84 | for job_idx in order { |
| 85 | let ResolvedPipelineJob { |
| 86 | ref root_nodes, |
| 87 | ref patches, |
| 88 | ref label, |
| 89 | platform, |
| 90 | arch, |
| 91 | cond_param_idx, |
| 92 | ref ado_pool, |
| 93 | timeout_minutes, |
| 94 | gh_override_if: _, |
| 95 | gh_global_env: _, |
| 96 | gh_pool: _, |
| 97 | gh_permissions: _, |
| 98 | ref external_read_vars, |
| 99 | ref parameters_used, |
| 100 | ref artifacts_used, |
| 101 | ref artifacts_published, |
| 102 | ref ado_variables, |
| 103 | } = graph[job_idx]; |
| 104 | |
| 105 | let flowey_source = job_flowey_source.remove(&job_idx).unwrap(); |
| 106 | |
| 107 | let (steps, req_db) = resolve_flow_as_ado_yaml_steps( |
| 108 | root_nodes |
| 109 | .clone() |
| 110 | .into_iter() |
| 111 | .map(|(node, requests)| (node, (true, requests))) |
| 112 | .collect(), |
| 113 | patches.clone(), |
| 114 | external_read_vars.clone(), |
| 115 | platform, |
| 116 | arch, |
| 117 | job_idx.index(), |
| 118 | ) |
| 119 | .context(format!("in job '{label}'"))?; |
| 120 | |
| 121 | { |
| 122 | let existing = pipeline_static_db.job_reqs.insert(job_idx.index(), req_db); |
| 123 | assert!(existing.is_none()) |
| 124 | } |
| 125 | |
| 126 | let mut ado_steps = Vec::new(); |
| 127 | |
| 128 | if let FloweySource::Bootstrap(artifact, publish) = &flowey_source { |
| 129 | // actual artifact publish happens at the end of the job |
| 130 | let _ = (artifact, publish); |
| 131 | |
| 132 | if ado_bootstrap_template.is_empty() { |
| 133 | anyhow::bail!( |
| 134 | "Did not specify flowey bootstrap template. Please provide one using `Pipeline::ado_set_flowey_bootstrap_template`" |
| 135 | ) |
| 136 | } |
| 137 | |
| 138 | let ado_bootstrap_template = ado_bootstrap_template |
| 139 | .replace("{{FLOWEY_BIN_EXTENSION}}", platform.exe_suffix()) |
| 140 | .replace("{{FLOWEY_CRATE}}", flowey_crate) |
| 141 | .replace( |
| 142 | "{{FLOWEY_PIPELINE_PATH}}", |
| 143 | &pipeline_file.with_extension("").display().to_string(), |
| 144 | ) |
| 145 | .replace( |
| 146 | "{{FLOWEY_TARGET}}", |
| 147 | match platform { |
| 148 | FlowPlatform::Windows => "x86_64-pc-windows-msvc", |
| 149 | FlowPlatform::Linux(_) => "x86_64-unknown-linux-gnu", |
| 150 | platform => anyhow::bail!("unsupported ADO platform {platform:?}"), |
| 151 | }, |
| 152 | ) |
| 153 | .replace( |
| 154 | "{{FLOWEY_OUTDIR}}", |
| 155 | "$(FLOWEY_TEMP_DIR)/bootstrapped-flowey", |
| 156 | ); |
| 157 | |
| 158 | let bootstrap_steps: serde_yaml::Sequence = |
| 159 | serde_yaml::from_str(&ado_bootstrap_template) |
| 160 | .context("malformed flowey bootstrap template")?; |
| 161 | |
| 162 | ado_steps.push({ |
| 163 | let mut map = serde_yaml::Mapping::new(); |
| 164 | map.insert("bash".into(), "echo \"injected!\"".into()); |
| 165 | map.insert("displayName".into(), "🌼🥾 Bootstrap flowey".into()); |
| 166 | map.into() |
| 167 | }); |
| 168 | ado_steps.extend(bootstrap_steps); |
| 169 | } |
| 170 | |
| 171 | // the first few steps in any job are some "artisan" code, which |
| 172 | // downloads the previously bootstrapped flowey artifact and set up |
| 173 | // various vars that flowey will then rely on throughout the rest |
| 174 | // of the job |
| 175 | |
| 176 | // download previously bootstrapped flowey |
| 177 | if let FloweySource::Consume(artifact) = &flowey_source { |
| 178 | ado_steps.push({ |
| 179 | let map: serde_yaml::Mapping = serde_yaml::from_str(&format!( |
| 180 | r#" |
| 181 | task: DownloadPipelineArtifact@2 |
| 182 | displayName: '🌼🥾 Download bootstrapped flowey' |
| 183 | inputs: |
| 184 | artifact: {artifact} |
| 185 | path: $(FLOWEY_TEMP_DIR)/bootstrapped-flowey |
| 186 | "# |
| 187 | )) |
| 188 | .unwrap(); |
| 189 | map.into() |
| 190 | }); |
| 191 | } |
| 192 | |
| 193 | // also download any artifacts that'll be used |
| 194 | for ResolvedJobArtifact { |
| 195 | flowey_var: _, |
| 196 | name, |
| 197 | } in artifacts_used |
| 198 | { |
| 199 | ado_steps.push({ |
| 200 | let map: serde_yaml::Mapping = serde_yaml::from_str(&format!( |
| 201 | r#" |
| 202 | task: DownloadPipelineArtifact@2 |
| 203 | displayName: '🌼📦 Download {name}' |
| 204 | inputs: |
| 205 | artifact: {name} |
| 206 | path: $(FLOWEY_TEMP_DIR)/used_artifacts/{name} |
| 207 | "# |
| 208 | )) |
| 209 | .unwrap(); |
| 210 | map.into() |
| 211 | }); |
| 212 | } |
| 213 | |
| 214 | let flowey_bin = platform.binary("flowey"); |
| 215 | let flowey_executable_bash = format!( |
| 216 | r###" |
| 217 | set -e |
| 218 | AgentTempDirNormal="$(FLOWEY_TEMP_DIR)" |
| 219 | AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') |
| 220 | echo "##vso[task.setvariable variable=AgentTempDirNormal;]$AgentTempDirNormal" |
| 221 | |
| 222 | chmod +x $AgentTempDirNormal/bootstrapped-flowey/{flowey_bin} |
| 223 | FLOWEY_BIN="$AgentTempDirNormal/bootstrapped-flowey/{flowey_bin}" |
| 224 | echo "##vso[task.setvariable variable=FLOWEY_BIN;]$FLOWEY_BIN" |
| 225 | "### |
| 226 | ).trim_start().to_string(); |
| 227 | |
| 228 | ado_steps.push({ |
| 229 | let mut map = serde_yaml::Mapping::new(); |
| 230 | map.insert( |
| 231 | "bash".into(), |
| 232 | serde_yaml::Value::String(flowey_executable_bash), |
| 233 | ); |
| 234 | map.insert("displayName".into(), "Set flowey path".into()); |
| 235 | map.into() |
| 236 | }); |
| 237 | |
| 238 | let mut flowey_bootstrap_bash = String::new(); |
| 239 | |
| 240 | let var_db = VarDbRequestBuilder::new("$FLOWEY_BIN", job_idx.index()); |
| 241 | |
| 242 | let bootstrap_bash_var_db_inject = |var, is_raw_string| { |
| 243 | var_db |
| 244 | .update_from_stdin(var, false) |
| 245 | .raw_string(is_raw_string) |
| 246 | .to_string() |
| 247 | }; |
| 248 | |
| 249 | // and now use those vars to do some flowey bootstrap |
| 250 | writeln!(flowey_bootstrap_bash, "{}", { |
| 251 | let runtime_debug_level = if runtime_debug_log { "debug" } else { "info" }; |
| 252 | |
| 253 | let var_db_insert_runtime_debug_level = |
| 254 | bootstrap_bash_var_db_inject("FLOWEY_LOG", false); |
| 255 | let var_db_insert_working_dir = |
| 256 | bootstrap_bash_var_db_inject(VAR_DB_SEEDVAR_FLOWEY_WORKING_DIR, true); |
| 257 | |
| 258 | // Need to use "normalized" path in cases where the path is being |
| 259 | // used directly from a bash context, as is the case when we are |
| 260 | // trying to invoke `flowey.exe` in argv0 position) |
| 261 | // |
| 262 | // https://github.com/microsoft/azure-pipelines-tasks/issues/10653#issuecomment-585669089 |
| 263 | format!( |
| 264 | r###" |
| 265 | set -e |
| 266 | echo '"{runtime_debug_level}"' | {var_db_insert_runtime_debug_level} |
| 267 | echo "$(FLOWEY_TEMP_DIR)/work" | {var_db_insert_working_dir} |
| 268 | "### |
| 269 | ) |
| 270 | .trim_start() |
| 271 | .to_owned() |
| 272 | })?; |
| 273 | |
| 274 | // import pipeline vars being used by the job into flowey |
| 275 | for ResolvedJobUseParameter { |
| 276 | flowey_var, |
| 277 | pipeline_param_idx, |
| 278 | } in parameters_used |
| 279 | { |
| 280 | let is_string = matches!( |
| 281 | parameters[*pipeline_param_idx], |
| 282 | flowey_core::pipeline::internal::Parameter::String { .. } |
| 283 | ); |
| 284 | let is_bool = matches!( |
| 285 | parameters[*pipeline_param_idx], |
| 286 | flowey_core::pipeline::internal::Parameter::Bool { .. } |
| 287 | ); |
| 288 | |
| 289 | let name = parameters[*pipeline_param_idx].name(); |
| 290 | |
| 291 | // ADO resolves bools as `True` and `False`, _sigh_ |
| 292 | let with_lowercase = if is_bool { |
| 293 | r#" | tr '[:upper:]' '[:lower:]'"# |
| 294 | } else { |
| 295 | "" |
| 296 | }; |
| 297 | |
| 298 | let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, is_string); |
| 299 | |
| 300 | let cmd = format!( |
| 301 | r#" |
| 302 | cat <<'EOF'{with_lowercase} | {var_db_inject_cmd} |
| 303 | ${{{{ parameters.{name} }}}} |
| 304 | EOF |
| 305 | "# |
| 306 | ) |
| 307 | .trim() |
| 308 | .to_string(); |
| 309 | writeln!(flowey_bootstrap_bash, "{}", cmd)?; |
| 310 | } |
| 311 | |
| 312 | // next, emit ado steps to create dirs for artifacts which will be |
| 313 | // published |
| 314 | for ResolvedJobArtifact { flowey_var, name } in artifacts_published { |
| 315 | writeln!( |
| 316 | flowey_bootstrap_bash, |
| 317 | r#"mkdir -p "$(AgentTempDirNormal)/publish_artifacts/{name}""# |
| 318 | )?; |
| 319 | let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, true); |
| 320 | writeln!( |
| 321 | flowey_bootstrap_bash, |
| 322 | r#"echo "$(FLOWEY_TEMP_DIR)/publish_artifacts/{name}" | {var_db_inject_cmd}"#, |
| 323 | )?; |
| 324 | } |
| 325 | |
| 326 | // lastly, emit ado steps that report the dirs for any artifacts which |
| 327 | // are used by this job |
| 328 | for ResolvedJobArtifact { flowey_var, name } in artifacts_used { |
| 329 | // do NOT use ADO macro syntax $(...), since this is in the same |
| 330 | // bootstrap block as where those ADO vars get defined, meaning it's |
| 331 | // not available yet! |
| 332 | let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, true); |
| 333 | writeln!( |
| 334 | flowey_bootstrap_bash, |
| 335 | r#"echo "$(FLOWEY_TEMP_DIR)/used_artifacts/{name}" | {var_db_inject_cmd}"#, |
| 336 | )?; |
| 337 | } |
| 338 | |
| 339 | // if this was a bootstrap job, also take a moment to run a "self check" |
| 340 | // to make sure that the current checked-in template matches the one it |
| 341 | // expected |
| 342 | if let FloweySource::Bootstrap(..) = &flowey_source { |
| 343 | let mut current_invocation = std::env::args().collect::<Vec<_>>(); |
| 344 | |
| 345 | current_invocation[0] = "$(FLOWEY_BIN)".into(); |
| 346 | |
| 347 | // if this code path is run while generating the YAML to compare the |
| 348 | // check against, we want to remove the --runtime or --check param from the |
| 349 | // current call, or else there'll be a dupe |
| 350 | let mut strip_parameter = |prefix: &str| { |
| 351 | if let Some(i) = current_invocation |
| 352 | .iter() |
| 353 | .position(|s| s.starts_with(prefix)) |
| 354 | { |
| 355 | current_invocation.remove(i); |
| 356 | if !current_invocation[i].starts_with(prefix) { |
| 357 | current_invocation.remove(i); |
| 358 | } |
| 359 | } |
| 360 | }; |
| 361 | |
| 362 | strip_parameter("--runtime"); |
| 363 | strip_parameter("--check"); |
| 364 | |
| 365 | // insert the --check bit of the call alongside the --out param |
| 366 | { |
| 367 | let i = current_invocation |
| 368 | .iter() |
| 369 | .position(|s| s.starts_with("--out")) |
| 370 | .unwrap(); |
| 371 | |
| 372 | let current_yaml = match platform.kind() { |
| 373 | FlowPlatformKind::Windows => { |
| 374 | r#"$ESCAPED_AGENT_TEMPDIR\\bootstrapped-flowey\\pipeline.yaml"# |
| 375 | } |
| 376 | FlowPlatformKind::Unix => { |
| 377 | r#"$ESCAPED_AGENT_TEMPDIR/bootstrapped-flowey/pipeline.yaml"# |
| 378 | } |
| 379 | }; |
| 380 | |
| 381 | current_invocation.insert(i, current_yaml.into()); |
| 382 | current_invocation.insert(i, "--runtime".into()); |
| 383 | } |
| 384 | |
| 385 | // Need to use an escaped version of the "true" windows/linux path |
| 386 | // here, or else the --check will fail. |
| 387 | let cmd = format!( |
| 388 | r###" |
| 389 | ESCAPED_AGENT_TEMPDIR=$( |
| 390 | cat <<'EOF' | sed 's/\\/\\\\/g' |
| 391 | $(FLOWEY_TEMP_DIR) |
| 392 | EOF |
| 393 | ) |
| 394 | {} |
| 395 | "###, |
| 396 | current_invocation.join(" ") |
| 397 | ); |
| 398 | |
| 399 | ado_steps.push({ |
| 400 | let mut map = serde_yaml::Mapping::new(); |
| 401 | map.insert( |
| 402 | "bash".into(), |
| 403 | serde_yaml::Value::String(cmd.trim().to_string()), |
| 404 | ); |
| 405 | map.insert("displayName".into(), "🌼🔎 Self-check YAML".into()); |
| 406 | map.into() |
| 407 | }) |
| 408 | } |
| 409 | |
| 410 | ado_steps.push({ |
| 411 | let mut map = serde_yaml::Mapping::new(); |
| 412 | map.insert( |
| 413 | "bash".into(), |
| 414 | serde_yaml::Value::String(flowey_bootstrap_bash), |
| 415 | ); |
| 416 | map.insert("displayName".into(), "🌼🛫 Initialize job".into()); |
| 417 | map.into() |
| 418 | }); |
| 419 | |
| 420 | // now that we've done all the job-level bootstrapping, we can emit all |
| 421 | // the actual steps the user cares about |
| 422 | ado_steps.extend(steps); |
| 423 | |
| 424 | // ..and once that's done, the last order of business is to emit some |
| 425 | // ado steps to publish the various artifacts created by this job |
| 426 | for ResolvedJobArtifact { |
| 427 | flowey_var: _, |
| 428 | name, |
| 429 | } in artifacts_published |
| 430 | { |
| 431 | ado_steps.push({ |
| 432 | let map: serde_yaml::Mapping = serde_yaml::from_str(&format!( |
| 433 | r#" |
| 434 | publish: $(FLOWEY_TEMP_DIR)/publish_artifacts/{name} |
| 435 | displayName: '🌼📦 Publish {name}' |
| 436 | artifact: {name} |
| 437 | "# |
| 438 | )) |
| 439 | .unwrap(); |
| 440 | map.into() |
| 441 | }); |
| 442 | } |
| 443 | |
| 444 | // also, if this job also bootstrapped flowey that other nodes depend |
| 445 | // on, make sure to publish it! |
| 446 | if let FloweySource::Bootstrap(artifact, true) = flowey_source { |
| 447 | // don't leak the bootstrap job's runtime var db |
| 448 | ado_steps.push({ |
| 449 | let mut map = serde_yaml::Mapping::new(); |
| 450 | map.insert( |
| 451 | "bash".into(), |
| 452 | serde_yaml::Value::String(format!( |
| 453 | "rm $(AgentTempDirNormal)/bootstrapped-flowey/job{}.json", |
| 454 | job_idx.index() |
| 455 | )), |
| 456 | ); |
| 457 | map.insert("displayName".into(), "🌼🧼 Redact bootstrap var db".into()); |
| 458 | map.into() |
| 459 | }); |
| 460 | |
| 461 | ado_steps.push({ |
| 462 | let map: serde_yaml::Mapping = serde_yaml::from_str(&format!( |
| 463 | r#" |
| 464 | publish: $(FLOWEY_TEMP_DIR)/bootstrapped-flowey |
| 465 | displayName: '🌼🥾 Publish bootstrapped flowey' |
| 466 | artifact: {artifact} |
| 467 | "# |
| 468 | )) |
| 469 | .unwrap(); |
| 470 | map.into() |
| 471 | }); |
| 472 | } |
| 473 | |
| 474 | // ADO has this "helpful" default behavior where if you don't explicitly |
| 475 | // include a checkout step, it'll just auto-checkout the current repo. |
| 476 | // |
| 477 | // Work around this nonsense by doing a pre-pass over the emitted |
| 478 | // steps to enumerate how many steps start with `- checkout:`, and |
| 479 | // if the number is zero, emit an explicit `- checkout: none`. |
| 480 | { |
| 481 | let mut found = false; |
| 482 | for val in &ado_steps { |
| 483 | if let Some((key, _val)) = val.as_mapping().unwrap().iter().next() { |
| 484 | let Some(key) = key.as_str() else { continue }; |
| 485 | if key == "checkout" { |
| 486 | found = true; |
| 487 | break; |
| 488 | } |
| 489 | } |
| 490 | } |
| 491 | if !found { |
| 492 | ado_steps.insert(0, { |
| 493 | let map: serde_yaml::Mapping = serde_yaml::from_str("checkout: none").unwrap(); |
| 494 | map.into() |
| 495 | }); |
| 496 | } |
| 497 | } |
| 498 | |
| 499 | // Convert the pool information to the structured format |
| 500 | let AdoPool { |
| 501 | name: pool_name, |
| 502 | demands, |
| 503 | } = ado_pool |
| 504 | .clone() |
| 505 | .context(format!("must specify ADO pool for job '{label}'"))?; |
| 506 | let pool = if demands.is_empty() { |
| 507 | schema_ado_yaml::Pool::Pool(pool_name) |
| 508 | } else { |
| 509 | schema_ado_yaml::Pool::PoolWithMetadata( |
| 510 | [ |
| 511 | ("name".into(), pool_name.into()), |
| 512 | ("demands".into(), demands.into()), |
| 513 | ] |
| 514 | .into(), |
| 515 | ) |
| 516 | }; |
| 517 | |
| 518 | let get_job_id = |id: usize| { |
| 519 | ado_job_id_overrides |
| 520 | .get(&id) |
| 521 | .cloned() |
| 522 | .unwrap_or_else(|| format!("job{}", id.clone())) |
| 523 | }; |
| 524 | ado_jobs.push(schema_ado_yaml::Job { |
| 525 | job: get_job_id(job_idx.index()), |
| 526 | display_name: label.clone(), |
| 527 | pool, |
| 528 | timeout_in_minutes: timeout_minutes, |
| 529 | depends_on: { |
| 530 | graph |
| 531 | .edges_directed(job_idx, petgraph::Direction::Incoming) |
| 532 | .map(|e| { |
| 533 | use petgraph::prelude::*; |
| 534 | get_job_id(e.source().index()) |
| 535 | }) |
| 536 | .collect() |
| 537 | }, |
| 538 | variables: { |
| 539 | let mut ado_variables: Vec<schema_ado_yaml::Variable> = ado_variables |
| 540 | .clone() |
| 541 | .into_iter() |
| 542 | .map(|(name, value)| schema_ado_yaml::Variable { name, value }) |
| 543 | .collect(); |
| 544 | |
| 545 | ado_variables.push(schema_ado_yaml::Variable { |
| 546 | name: "FLOWEY_TEMP_DIR".into(), |
| 547 | value: FLOWEY_TEMP_DIR.into(), |
| 548 | }); |
| 549 | |
| 550 | Some(ado_variables) |
| 551 | }, |
| 552 | steps: ado_steps, |
| 553 | condition: Some(if let Some(cond_param_idx) = cond_param_idx { |
| 554 | format!( |
| 555 | "and(eq('${{{{ parameters.{} }}}}', 'true'), succeeded(), not(canceled()))", |
| 556 | parameters[cond_param_idx].name() |
| 557 | ) |
| 558 | } else { |
| 559 | "and(succeeded(), not(canceled()))".into() |
| 560 | }), |
| 561 | }) |
| 562 | } |
| 563 | |
| 564 | let ado_pipeline = schema_ado_yaml::Pipeline { |
| 565 | name: ado_name, |
| 566 | trigger: Some(match ado_ci_triggers { |
| 567 | None => schema_ado_yaml::CiTrigger::None(()), |
| 568 | Some(t) => { |
| 569 | let flowey_core::pipeline::AdoCiTriggers { |
| 570 | branches, |
| 571 | exclude_branches, |
| 572 | tags, |
| 573 | exclude_tags, |
| 574 | batch, |
| 575 | } = t; |
| 576 | |
| 577 | if branches.is_empty() && tags.is_empty() { |
| 578 | anyhow::bail!("branches and tags cannot both be empty") |
| 579 | } |
| 580 | |
| 581 | schema_ado_yaml::CiTrigger::Some { |
| 582 | batch, |
| 583 | branches: if branches.is_empty() { |
| 584 | if !exclude_branches.is_empty() { |
| 585 | anyhow::bail!("empty branch trigger with non-empty exclude") |
| 586 | } |
| 587 | |
| 588 | None |
| 589 | } else { |
| 590 | Some(schema_ado_yaml::TriggerBranches { |
| 591 | include: branches, |
| 592 | exclude: if exclude_branches.is_empty() { |
| 593 | None |
| 594 | } else { |
| 595 | Some(exclude_branches) |
| 596 | }, |
| 597 | }) |
| 598 | }, |
| 599 | tags: if tags.is_empty() { |
| 600 | if !exclude_tags.is_empty() { |
| 601 | anyhow::bail!("empty tags trigger with non-empty exclude") |
| 602 | } |
| 603 | |
| 604 | None |
| 605 | } else { |
| 606 | Some(schema_ado_yaml::TriggerTags { |
| 607 | include: tags, |
| 608 | exclude: if exclude_tags.is_empty() { |
| 609 | None |
| 610 | } else { |
| 611 | Some(exclude_tags) |
| 612 | }, |
| 613 | }) |
| 614 | }, |
| 615 | } |
| 616 | } |
| 617 | }), |
| 618 | pr: Some(match ado_pr_triggers { |
| 619 | None => schema_ado_yaml::PrTrigger::None(()), |
| 620 | Some(t) => { |
| 621 | let flowey_core::pipeline::AdoPrTriggers { |
| 622 | branches, |
| 623 | exclude_branches, |
| 624 | run_on_draft, |
| 625 | auto_cancel, |
| 626 | } = t; |
| 627 | |
| 628 | schema_ado_yaml::PrTrigger::Some { |
| 629 | auto_cancel, |
| 630 | drafts: run_on_draft, |
| 631 | branches: schema_ado_yaml::TriggerBranches { |
| 632 | include: branches, |
| 633 | exclude: if exclude_branches.is_empty() { |
| 634 | None |
| 635 | } else { |
| 636 | Some(exclude_branches) |
| 637 | }, |
| 638 | }, |
| 639 | } |
| 640 | } |
| 641 | }), |
| 642 | schedules: if ado_schedule_triggers.is_empty() { |
| 643 | None |
| 644 | } else { |
| 645 | Some( |
| 646 | ado_schedule_triggers |
| 647 | .into_iter() |
| 648 | .map(|t| { |
| 649 | let flowey_core::pipeline::AdoScheduleTriggers { |
| 650 | display_name, |
| 651 | branches, |
| 652 | exclude_branches, |
| 653 | cron, |
| 654 | } = t; |
| 655 | |
| 656 | schema_ado_yaml::Schedule { |
| 657 | cron, |
| 658 | display_name, |
| 659 | branches: schema_ado_yaml::TriggerBranches { |
| 660 | include: branches, |
| 661 | exclude: if exclude_branches.is_empty() { |
| 662 | None |
| 663 | } else { |
| 664 | Some(exclude_branches) |
| 665 | }, |
| 666 | }, |
| 667 | batch: false, |
| 668 | } |
| 669 | }) |
| 670 | .collect(), |
| 671 | ) |
| 672 | }, |
| 673 | variables: if !ado_variables.is_empty() { |
| 674 | Some( |
| 675 | ado_variables |
| 676 | .into_iter() |
| 677 | .map(|(name, value)| schema_ado_yaml::Variable { name, value }) |
| 678 | .collect(), |
| 679 | ) |
| 680 | } else { |
| 681 | None |
| 682 | }, |
| 683 | stages: None, |
| 684 | jobs: Some(ado_jobs), |
| 685 | parameters: if !parameters.is_empty() { |
| 686 | Some( |
| 687 | parameters |
| 688 | .clone() |
| 689 | .into_iter() |
| 690 | .map(|param| match param { |
| 691 | flowey_core::pipeline::internal::Parameter::Bool { |
| 692 | name, |
| 693 | description, |
| 694 | kind: _, |
| 695 | default, |
| 696 | } => schema_ado_yaml::Parameter { |
| 697 | name, |
| 698 | display_name: description, |
| 699 | ty: schema_ado_yaml::ParameterType::Boolean { default }, |
| 700 | }, |
| 701 | flowey_core::pipeline::internal::Parameter::String { |
| 702 | name, |
| 703 | description, |
| 704 | kind: _, |
| 705 | default, |
| 706 | possible_values, |
| 707 | } => schema_ado_yaml::Parameter { |
| 708 | name, |
| 709 | display_name: description, |
| 710 | ty: schema_ado_yaml::ParameterType::String { |
| 711 | default, |
| 712 | values: possible_values, |
| 713 | }, |
| 714 | }, |
| 715 | flowey_core::pipeline::internal::Parameter::Num { |
| 716 | name, |
| 717 | description, |
| 718 | kind: _, |
| 719 | default, |
| 720 | possible_values, |
| 721 | } => schema_ado_yaml::Parameter { |
| 722 | name, |
| 723 | display_name: description, |
| 724 | ty: schema_ado_yaml::ParameterType::Number { |
| 725 | default, |
| 726 | values: possible_values, |
| 727 | }, |
| 728 | }, |
| 729 | }) |
| 730 | .collect(), |
| 731 | ) |
| 732 | } else { |
| 733 | None |
| 734 | }, |
| 735 | resources: { |
| 736 | if ado_resources_repository.is_empty() { |
| 737 | None |
| 738 | } else { |
| 739 | Some(schema_ado_yaml::Resources { |
| 740 | repositories: ado_resources_repository |
| 741 | .into_iter() |
| 742 | .map( |
| 743 | |InternalAdoResourcesRepository { |
| 744 | repo_id, |
| 745 | repo_type, |
| 746 | name, |
| 747 | git_ref, |
| 748 | endpoint, |
| 749 | }| { |
| 750 | use flowey_core::pipeline::AdoResourcesRepositoryRef; |
| 751 | use flowey_core::pipeline::AdoResourcesRepositoryType; |
| 752 | |
| 753 | schema_ado_yaml::ResourcesRepository { |
| 754 | repository: repo_id, |
| 755 | endpoint, |
| 756 | name, |
| 757 | r#ref: match git_ref { |
| 758 | AdoResourcesRepositoryRef::Fixed(s) => s, |
| 759 | AdoResourcesRepositoryRef::Parameter(idx) => { |
| 760 | let name = parameters[idx].name(); |
| 761 | format!("${{{{ parameters.{name} }}}}") |
| 762 | } |
| 763 | }, |
| 764 | r#type: match repo_type { |
| 765 | AdoResourcesRepositoryType::AzureReposGit => { |
| 766 | schema_ado_yaml::ResourcesRepositoryType::Git |
| 767 | } |
| 768 | AdoResourcesRepositoryType::GitHub => { |
| 769 | schema_ado_yaml::ResourcesRepositoryType::GitHub |
| 770 | } |
| 771 | }, |
| 772 | } |
| 773 | }, |
| 774 | ) |
| 775 | .collect::<Vec<_>>(), |
| 776 | }) |
| 777 | } |
| 778 | }, |
| 779 | extends: None, |
| 780 | }; |
| 781 | |
| 782 | match check { |
| 783 | CheckMode::Check(_) | CheckMode::Runtime(_) => check_generated_yaml_and_json( |
| 784 | &ado_pipeline, |
| 785 | &pipeline_static_db, |
| 786 | check, |
| 787 | repo_root, |
| 788 | pipeline_file, |
| 789 | ado_post_process_yaml_cb, |
| 790 | ), |
| 791 | CheckMode::None => write_generated_yaml_and_json( |
| 792 | &ado_pipeline, |
| 793 | &pipeline_static_db, |
| 794 | repo_root, |
| 795 | pipeline_file, |
| 796 | ado_post_process_yaml_cb, |
| 797 | ), |
| 798 | } |
| 799 | } |
| 800 | |
| 801 | /// Resolve a flow as a sequence of ADO YAML steps. |
| 802 | /// |
| 803 | /// These steps can then be marshalled into a well-formed ADO pipeline yaml |
| 804 | /// using a separate ADO pipeline yaml builder |
| 805 | // pub(crate) so that internal debug CLI tooling can use it |
| 806 | pub(crate) fn resolve_flow_as_ado_yaml_steps( |
| 807 | seed_nodes: BTreeMap<NodeHandle, (bool, Vec<Box<[u8]>>)>, |
| 808 | resolved_patches: flowey_core::patch::ResolvedPatches, |
| 809 | external_read_vars: BTreeSet<String>, |
| 810 | platform: FlowPlatform, |
| 811 | arch: FlowArch, |
| 812 | job_idx: usize, |
| 813 | ) -> anyhow::Result<( |
| 814 | Vec<serde_yaml::Value>, |
| 815 | BTreeMap<String, Vec<crate::cli::exec_snippet::SerializedRequest>>, |
| 816 | )> { |
| 817 | let mut output_steps = Vec::new(); |
| 818 | |
| 819 | let (mut output_graph, request_db, err_unreachable_nodes) = |
| 820 | crate::flow_resolver::stage1_dag::stage1_dag( |
| 821 | FlowBackend::Ado, |
| 822 | platform, |
| 823 | arch, |
| 824 | resolved_patches, |
| 825 | seed_nodes, |
| 826 | external_read_vars, |
| 827 | // TODO: support ADO agents with persistent storage |
| 828 | None, |
| 829 | )?; |
| 830 | |
| 831 | if err_unreachable_nodes.is_some() { |
| 832 | anyhow::bail!("detected unreachable nodes") |
| 833 | } |
| 834 | |
| 835 | let output_order = petgraph::algo::toposort(&output_graph, None) |
| 836 | .expect("runtime variables cannot introduce a DAG cycle"); |
| 837 | |
| 838 | let var_db = VarDbRequestBuilder::new("$FLOWEY_BIN", job_idx); |
| 839 | |
| 840 | let mut bash_commands = BashCommands::new_ado(); |
| 841 | for idx in output_order.into_iter().rev() { |
| 842 | let OutputGraphEntry { node_handle, step } = output_graph[idx].1.take().unwrap(); |
| 843 | |
| 844 | let node_modpath = node_handle.modpath(); |
| 845 | |
| 846 | match step { |
| 847 | Step::Anchor { .. } => {} |
| 848 | Step::Rust { |
| 849 | idx, |
| 850 | can_merge, |
| 851 | label, |
| 852 | code: _, |
| 853 | } => { |
| 854 | output_steps.extend(bash_commands.push( |
| 855 | Some(label), |
| 856 | can_merge, |
| 857 | crate::cli::exec_snippet::construct_exec_snippet_cli( |
| 858 | "$(FLOWEY_BIN)", |
| 859 | node_modpath, |
| 860 | idx, |
| 861 | job_idx, |
| 862 | ), |
| 863 | )); |
| 864 | } |
| 865 | Step::AdoYaml { |
| 866 | label, |
| 867 | raw_yaml, |
| 868 | ado_to_rust, |
| 869 | rust_to_ado, |
| 870 | condvar, |
| 871 | code_idx, |
| 872 | code, |
| 873 | } => { |
| 874 | for (rust_var, ado_var) in rust_to_ado { |
| 875 | // flowey considers all ADO vars to be typed as raw strings |
| 876 | let read_rust_var = var_db |
| 877 | .write_to_ado_env(&rust_var, &ado_var) |
| 878 | .raw_string(true) |
| 879 | .condvar(condvar.as_deref()); |
| 880 | |
| 881 | bash_commands.push_minor(format!("{read_rust_var}\n")); |
| 882 | } |
| 883 | |
| 884 | if !raw_yaml.is_empty() { |
| 885 | if let Some(condvar) = &condvar { |
| 886 | // guaranteed to be a bare bool `true`/`false`, hence |
| 887 | // is_raw_string = false |
| 888 | let read_condvar = var_db.write_to_ado_env(condvar, "FLOWEY_CONDITION"); |
| 889 | |
| 890 | bash_commands.push_minor(format!("{read_condvar}\n")); |
| 891 | } |
| 892 | |
| 893 | let raw_yaml = if code.lock().is_some() { |
| 894 | let inline_snippet = crate::cli::exec_snippet::construct_exec_snippet_cli( |
| 895 | "$(FLOWEY_BIN)", |
| 896 | node_modpath, |
| 897 | code_idx, |
| 898 | job_idx, |
| 899 | ); |
| 900 | let post_process = |
| 901 | raw_yaml.replace("{{FLOWEY_INLINE_SCRIPT}}", &inline_snippet); |
| 902 | if raw_yaml == post_process { |
| 903 | return Err(anyhow::anyhow!("if using inlins-enippet, YAML must include {{{{FLOWEY_INLINE_SCRIPT}}}}").context(format!( |
| 904 | "invalid yaml in node {node_modpath}: {raw_yaml}" |
| 905 | ))); |
| 906 | } |
| 907 | post_process |
| 908 | } else { |
| 909 | raw_yaml |
| 910 | }; |
| 911 | |
| 912 | let step: serde_yaml::Value = serde_yaml::from_str(&raw_yaml) |
| 913 | .context(format!("invalid yaml in node {node_modpath}: {raw_yaml}"))?; |
| 914 | let step = { |
| 915 | let mut step = step; |
| 916 | let seq = step |
| 917 | .as_sequence_mut() |
| 918 | .context("yaml snippet did not parse as a sequence")?; |
| 919 | |
| 920 | if seq.len() != 1 { |
| 921 | anyhow::bail!("yaml snippet contained more than one sequence element") |
| 922 | } |
| 923 | |
| 924 | let map = seq |
| 925 | .first_mut() |
| 926 | .unwrap() |
| 927 | .as_mapping_mut() |
| 928 | .context("yaml snippet did not parse as a map")?; |
| 929 | let existing = map.insert("displayName".into(), label.into()); |
| 930 | if existing.is_some() { |
| 931 | anyhow::bail!("yaml snippet included `displayName`") |
| 932 | } |
| 933 | if condvar.is_some() { |
| 934 | let existing = map.insert( |
| 935 | "condition".into(), |
| 936 | "and(eq(variables['FLOWEY_CONDITION'], true), succeeded(), not(canceled()))".into(), |
| 937 | ); |
| 938 | if existing.is_some() { |
| 939 | anyhow::bail!("yaml snippet included `condition`") |
| 940 | } |
| 941 | } |
| 942 | |
| 943 | step |
| 944 | }; |
| 945 | output_steps.extend(bash_commands.flush()); |
| 946 | output_steps.push(step.as_sequence().unwrap().first().unwrap().clone()); |
| 947 | } |
| 948 | |
| 949 | for (ado_var, rust_var, is_secret) in ado_to_rust { |
| 950 | // flowey considers all ADO vars to be typed as raw strings |
| 951 | let write_rust_var = var_db |
| 952 | .update_from_stdin(&rust_var, is_secret) |
| 953 | .raw_string(true) |
| 954 | .condvar(condvar.as_deref()) |
| 955 | .env_source(Some(&ado_var)); |
| 956 | |
| 957 | let cmd = format!( |
| 958 | r#" |
| 959 | {write_rust_var} <<'EOF' |
| 960 | $({ado_var}) |
| 961 | EOF |
| 962 | "# |
| 963 | ) |
| 964 | .trim() |
| 965 | .to_string(); |
| 966 | |
| 967 | bash_commands.push_minor(cmd); |
| 968 | } |
| 969 | } |
| 970 | Step::GitHubYaml { .. } => anyhow::bail!("GitHub YAML not supported in ADO"), |
| 971 | } |
| 972 | } |
| 973 | |
| 974 | output_steps.extend(bash_commands.flush()); |
| 975 | |
| 976 | let request_db = request_db |
| 977 | .into_iter() |
| 978 | .map(|(node_handle, reqs)| { |
| 979 | ( |
| 980 | node_handle.modpath().to_owned(), |
| 981 | reqs.into_iter() |
| 982 | .map(crate::cli::exec_snippet::SerializedRequest) |
| 983 | .collect(), |
| 984 | ) |
| 985 | }) |
| 986 | .collect(); |
| 987 | |
| 988 | Ok((output_steps, request_db)) |
| 989 | } |
| 990 | |