GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/openvmm

Public

mirrored fromhttps://github.com/microsoft/openvmmAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
fe2bbc9829558a07c5e06d2b6ececc383b6593bf

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openvmm/flowey/flowey_cli/src/pipeline_resolver

flowey/flowey_cli/src/pipeline_resolver/ado_yaml.rs

1056lines · modecode

RawDownload
Latest commit unavailable.
1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4use super::common_yaml::BashCommands;
5use super::common_yaml::FloweySource;
6use super::common_yaml::check_generated_yaml_and_json;
7use super::common_yaml::job_flowey_bootstrap_source;
8use super::common_yaml::write_generated_yaml_and_json;
9use super::generic::ResolvedJobArtifact;
10use super::generic::ResolvedJobUseParameter;
11use crate::cli::exec_snippet::FloweyPipelineStaticDb;
12use crate::cli::exec_snippet::VAR_DB_SEEDVAR_FLOWEY_WORKING_DIR;
13use crate::cli::pipeline::CheckMode;
14use crate::cli::var_db::VarDbRequestBuilder;
15use crate::flow_resolver::stage1_dag::OutputGraphEntry;
16use crate::flow_resolver::stage1_dag::Step;
17use crate::pipeline_resolver::generic::ResolvedPipeline;
18use crate::pipeline_resolver::generic::ResolvedPipelineJob;
19use anyhow::Context;
20use flowey_core::node::FlowArch;
21use flowey_core::node::FlowBackend;
22use flowey_core::node::FlowPlatform;
23use flowey_core::node::FlowPlatformKind;
24use flowey_core::node::NodeHandle;
25use flowey_core::pipeline::AdoPool;
26use flowey_core::pipeline::internal::InternalAdoResourcesRepository;
27use std::collections::BTreeMap;
28use std::collections::BTreeSet;
29use std::fmt::Write;
30use std::path::Path;
31
32/// We use $(Build.StagingDirectory)/.flowey-internal instead
33/// of $(Agent.TempDirectory) to (hopefully) guarantee that this folder
34/// resides on the same mount-point as the repos being cloned.
35///
36/// violating this property would result in calls to `fs::rename` in
37/// downstream flowey nodes to fail.
38const FLOWEY_TEMP_DIR: &str = "$(Build.StagingDirectory)/.flowey-internal";
39
40/// Emit a pipeline as a single self-contained ADO yaml file
41pub fn ado_yaml(
42 pipeline: ResolvedPipeline,
43 runtime_debug_log: bool,
44 repo_root: &Path,
45 pipeline_file: &Path,
46 flowey_crate: &str,
47 check: CheckMode,
48) -> anyhow::Result<()> {
49 if pipeline_file.extension().and_then(|s| s.to_str()) != Some("yaml") {
50 anyhow::bail!("pipeline name must end with .yaml")
51 }
52
53 let ResolvedPipeline {
54 graph,
55 order,
56 parameters,
57 ado_name,
58 ado_schedule_triggers,
59 ado_ci_triggers,
60 ado_pr_triggers,
61 ado_bootstrap_template,
62 ado_resources_repository,
63 ado_post_process_yaml_cb,
64 ado_variables,
65 ref ado_job_id_overrides,
66 gh_name: _,
67 gh_schedule_triggers: _,
68 gh_ci_triggers: _,
69 gh_pr_triggers: _,
70 gh_bootstrap_template: _,
71 } = pipeline;
72
73 let mut job_flowey_source: BTreeMap<petgraph::prelude::NodeIndex, FloweySource> =
74 job_flowey_bootstrap_source(&graph, &order);
75
76 let mut pipeline_static_db = FloweyPipelineStaticDb {
77 flow_backend: crate::cli::FlowBackendCli::Ado,
78 var_db_backend_kind: crate::cli::exec_snippet::VarDbBackendKind::Json,
79 job_reqs: BTreeMap::new(),
80 job_configs: BTreeMap::new(),
81 job_command_wrappers: BTreeMap::new(),
82 job_platforms: BTreeMap::new(),
83 job_archs: BTreeMap::new(),
84 };
85
86 let mut ado_jobs = Vec::new();
87
88 for job_idx in order {
89 let ResolvedPipelineJob {
90 ref root_nodes,
91 ref root_configs,
92 ref patches,
93 ref label,
94 platform,
95 arch,
96 cond_param_idx,
97 ref ado_pool,
98 timeout_minutes,
99 command_wrapper: ref command_wrapper_kind,
100 gh_override_if: _,
101 gh_global_env: _,
102 gh_pool: _,
103 gh_permissions: _,
104 ref external_read_vars,
105 ref parameters_used,
106 ref artifacts_used,
107 ref artifacts_published,
108 ref ado_variables,
109 } = graph[job_idx];
110
111 let flowey_source = job_flowey_source.remove(&job_idx).unwrap();
112
113 let super::common_yaml::ResolvedFlowSteps {
114 steps,
115 request_db: req_db,
116 config_db: cfg_db,
117 } = resolve_flow_as_ado_yaml_steps(
118 root_nodes
119 .clone()
120 .into_iter()
121 .map(|(node, requests)| (node, (true, requests)))
122 .collect(),
123 root_configs.clone(),
124 patches.clone(),
125 external_read_vars.clone(),
126 platform,
127 arch,
128 job_idx.index(),
129 )
130 .context(format!("in job '{label}'"))?;
131
132 {
133 let existing = pipeline_static_db.job_reqs.insert(job_idx.index(), req_db);
134 assert!(existing.is_none())
135 }
136
137 if !cfg_db.is_empty() {
138 pipeline_static_db
139 .job_configs
140 .insert(job_idx.index(), cfg_db);
141 }
142
143 if let Some(wrapper_kind) = command_wrapper_kind {
144 pipeline_static_db
145 .job_command_wrappers
146 .insert(job_idx.index(), wrapper_kind.clone());
147 }
148
149 pipeline_static_db
150 .job_platforms
151 .insert(job_idx.index(), platform);
152 pipeline_static_db.job_archs.insert(job_idx.index(), arch);
153
154 let mut ado_steps = Vec::new();
155
156 if let FloweySource::Bootstrap(artifact, publish) = &flowey_source {
157 // actual artifact publish happens at the end of the job
158 let _ = (artifact, publish);
159
160 if ado_bootstrap_template.is_empty() {
161 anyhow::bail!(
162 "Did not specify flowey bootstrap template. Please provide one using `Pipeline::ado_set_flowey_bootstrap_template`"
163 )
164 }
165
166 let ado_bootstrap_template = ado_bootstrap_template
167 .replace("{{FLOWEY_BIN_EXTENSION}}", platform.exe_suffix())
168 .replace("{{FLOWEY_CRATE}}", flowey_crate)
169 .replace(
170 "{{FLOWEY_PIPELINE_PATH}}",
171 &pipeline_file.with_extension("").display().to_string(),
172 )
173 .replace(
174 "{{FLOWEY_TARGET}}",
175 match (platform, arch) {
176 (FlowPlatform::Windows, FlowArch::X86_64) => "x86_64-pc-windows-msvc",
177 (FlowPlatform::Windows, FlowArch::Aarch64) => "aarch64-pc-windows-msvc",
178 (FlowPlatform::Linux(_), FlowArch::X86_64) => "x86_64-unknown-linux-gnu",
179 (FlowPlatform::Linux(_), FlowArch::Aarch64) => "aarch64-unknown-linux-gnu",
180 (platform, arch) => anyhow::bail!(
181 "unsupported ADO platform/arch combo {platform:?}/{arch:?}"
182 ),
183 },
184 )
185 .replace(
186 "{{FLOWEY_OUTDIR}}",
187 "$(FLOWEY_TEMP_DIR)/bootstrapped-flowey",
188 );
189
190 let bootstrap_steps: serde_yaml::Sequence =
191 serde_yaml::from_str(&ado_bootstrap_template)
192 .context("malformed flowey bootstrap template")?;
193
194 ado_steps.extend(bootstrap_steps);
195 }
196
197 // the first few steps in any job are some "artisan" code, which
198 // downloads the previously bootstrapped flowey artifact and set up
199 // various vars that flowey will then rely on throughout the rest
200 // of the job
201
202 // download previously bootstrapped flowey
203 if let FloweySource::Consume(artifact) = &flowey_source {
204 ado_steps.push({
205 let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
206 r#"
207 task: DownloadPipelineArtifact@2
208 displayName: '🌼🥾 Download bootstrapped flowey'
209 inputs:
210 artifact: {artifact}
211 path: $(FLOWEY_TEMP_DIR)/bootstrapped-flowey
212 "#
213 ))
214 .unwrap();
215 map.into()
216 });
217 }
218
219 // also download any artifacts that'll be used
220 for ResolvedJobArtifact {
221 flowey_var: _,
222 name,
223 } in artifacts_used
224 {
225 ado_steps.push({
226 let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
227 r#"
228 task: DownloadPipelineArtifact@2
229 displayName: '🌼📦 Download {name}'
230 inputs:
231 artifact: {name}
232 path: $(FLOWEY_TEMP_DIR)/used_artifacts/{name}
233 "#
234 ))
235 .unwrap();
236 map.into()
237 });
238 }
239
240 let flowey_bin = platform.binary("flowey");
241 let flowey_executable_bash = format!(
242 r###"
243set -e
244AgentTempDirNormal="$(FLOWEY_TEMP_DIR)"
245AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|')
246echo "##vso[task.setvariable variable=AgentTempDirNormal;]$AgentTempDirNormal"
247
248chmod +x $AgentTempDirNormal/bootstrapped-flowey/{flowey_bin}
249FLOWEY_BIN="$AgentTempDirNormal/bootstrapped-flowey/{flowey_bin}"
250echo "##vso[task.setvariable variable=FLOWEY_BIN;]$FLOWEY_BIN"
251"###
252 ).trim_start().to_string();
253
254 ado_steps.push({
255 let mut map = serde_yaml::Mapping::new();
256 map.insert(
257 "bash".into(),
258 serde_yaml::Value::String(flowey_executable_bash),
259 );
260 map.insert("displayName".into(), "Set flowey path".into());
261 map.into()
262 });
263
264 let mut flowey_bootstrap_bash = String::new();
265
266 let var_db = VarDbRequestBuilder::new("$FLOWEY_BIN", job_idx.index());
267
268 let bootstrap_bash_var_db_inject = |var, is_raw_string| {
269 var_db
270 .update_from_stdin(var, false)
271 .raw_string(is_raw_string)
272 .to_string()
273 };
274
275 // and now use those vars to do some flowey bootstrap
276 writeln!(flowey_bootstrap_bash, "{}", {
277 let runtime_debug_level = if runtime_debug_log { "debug" } else { "info" };
278
279 let var_db_insert_runtime_debug_level =
280 bootstrap_bash_var_db_inject("FLOWEY_LOG", false);
281 let var_db_insert_working_dir =
282 bootstrap_bash_var_db_inject(VAR_DB_SEEDVAR_FLOWEY_WORKING_DIR, true);
283
284 // Need to use "normalized" path in cases where the path is being
285 // used directly from a bash context, as is the case when we are
286 // trying to invoke `flowey.exe` in argv0 position)
287 //
288 // https://github.com/microsoft/azure-pipelines-tasks/issues/10653#issuecomment-585669089
289 format!(
290 r###"
291set -e
292echo '"{runtime_debug_level}"' | {var_db_insert_runtime_debug_level}
293echo "$(FLOWEY_TEMP_DIR)/work" | {var_db_insert_working_dir}
294"###
295 )
296 .trim_start()
297 .to_owned()
298 })?;
299
300 // import pipeline vars being used by the job into flowey
301 for ResolvedJobUseParameter {
302 flowey_var,
303 pipeline_param_idx,
304 } in parameters_used
305 {
306 let is_string = matches!(
307 parameters[*pipeline_param_idx],
308 flowey_core::pipeline::internal::Parameter::String { .. }
309 );
310 let is_bool = matches!(
311 parameters[*pipeline_param_idx],
312 flowey_core::pipeline::internal::Parameter::Bool { .. }
313 );
314
315 let name = parameters[*pipeline_param_idx].name();
316
317 // ADO resolves bools as `True` and `False`, _sigh_
318 let with_lowercase = if is_bool {
319 r#" | tr '[:upper:]' '[:lower:]'"#
320 } else {
321 ""
322 };
323
324 let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, is_string);
325
326 let cmd = format!(
327 r#"
328cat <<'EOF'{with_lowercase} | {var_db_inject_cmd}
329${{{{ parameters.{name} }}}}
330EOF
331"#
332 )
333 .trim()
334 .to_string();
335 writeln!(flowey_bootstrap_bash, "{}", cmd)?;
336 }
337
338 // next, emit ado steps to create dirs for artifacts which will be
339 // published
340 for ResolvedJobArtifact { flowey_var, name } in artifacts_published {
341 writeln!(
342 flowey_bootstrap_bash,
343 r#"mkdir -p "$(AgentTempDirNormal)/publish_artifacts/{name}""#
344 )?;
345 let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, true);
346 writeln!(
347 flowey_bootstrap_bash,
348 r#"echo "$(FLOWEY_TEMP_DIR)/publish_artifacts/{name}" | {var_db_inject_cmd}"#,
349 )?;
350 }
351
352 // lastly, emit ado steps that report the dirs for any artifacts which
353 // are used by this job
354 for ResolvedJobArtifact { flowey_var, name } in artifacts_used {
355 // do NOT use ADO macro syntax $(...), since this is in the same
356 // bootstrap block as where those ADO vars get defined, meaning it's
357 // not available yet!
358 let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, true);
359 writeln!(
360 flowey_bootstrap_bash,
361 r#"echo "$(FLOWEY_TEMP_DIR)/used_artifacts/{name}" | {var_db_inject_cmd}"#,
362 )?;
363 }
364
365 // if this was a bootstrap job, also take a moment to run a "self check"
366 // to make sure that the current checked-in template matches the one it
367 // expected
368 if let FloweySource::Bootstrap(..) = &flowey_source {
369 let mut current_invocation = std::env::args().collect::<Vec<_>>();
370
371 current_invocation[0] = "$(FLOWEY_BIN)".into();
372
373 // if this code path is run while generating the YAML to compare the
374 // check against, we want to remove the --runtime or --check param from the
375 // current call, or else there'll be a dupe
376 let mut strip_parameter = |prefix: &str| {
377 if let Some(i) = current_invocation
378 .iter()
379 .position(|s| s.starts_with(prefix))
380 {
381 current_invocation.remove(i);
382 if !current_invocation[i].starts_with(prefix) {
383 current_invocation.remove(i);
384 }
385 }
386 };
387
388 strip_parameter("--runtime");
389 strip_parameter("--check");
390
391 // insert the --check bit of the call alongside the --out param
392 {
393 let i = current_invocation
394 .iter()
395 .position(|s| s.starts_with("--out"))
396 .unwrap();
397
398 let current_yaml = match platform.kind() {
399 FlowPlatformKind::Windows => {
400 r#"$ESCAPED_AGENT_TEMPDIR\\bootstrapped-flowey\\pipeline.yaml"#
401 }
402 FlowPlatformKind::Unix => {
403 r#"$ESCAPED_AGENT_TEMPDIR/bootstrapped-flowey/pipeline.yaml"#
404 }
405 };
406
407 current_invocation.insert(i, current_yaml.into());
408 current_invocation.insert(i, "--runtime".into());
409 }
410
411 // Need to use an escaped version of the "true" windows/linux path
412 // here, or else the --check will fail.
413 let cmd = format!(
414 r###"
415ESCAPED_AGENT_TEMPDIR=$(
416cat <<'EOF' | sed 's/\\/\\\\/g'
417$(FLOWEY_TEMP_DIR)
418EOF
419)
420{}
421"###,
422 current_invocation.join(" ")
423 );
424
425 ado_steps.push({
426 let mut map = serde_yaml::Mapping::new();
427 map.insert(
428 "bash".into(),
429 serde_yaml::Value::String(cmd.trim().to_string()),
430 );
431 map.insert("displayName".into(), "🌼🔎 Self-check YAML".into());
432 map.into()
433 })
434 }
435
436 ado_steps.push({
437 let mut map = serde_yaml::Mapping::new();
438 map.insert(
439 "bash".into(),
440 serde_yaml::Value::String(flowey_bootstrap_bash),
441 );
442 map.insert("displayName".into(), "🌼🛫 Initialize job".into());
443 map.into()
444 });
445
446 // now that we've done all the job-level bootstrapping, we can emit all
447 // the actual steps the user cares about
448 ado_steps.extend(steps);
449
450 // ..and once that's done, the last order of business is to emit some
451 // ado steps to publish the various artifacts created by this job
452 for ResolvedJobArtifact {
453 flowey_var: _,
454 name,
455 } in artifacts_published
456 {
457 ado_steps.push({
458 let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
459 r#"
460 publish: $(FLOWEY_TEMP_DIR)/publish_artifacts/{name}
461 displayName: '🌼📦 Publish {name}'
462 artifact: {name}
463 "#
464 ))
465 .unwrap();
466 map.into()
467 });
468 }
469
470 // also, if this job also bootstrapped flowey that other nodes depend
471 // on, make sure to publish it!
472 if let FloweySource::Bootstrap(artifact, true) = flowey_source {
473 // don't leak the bootstrap job's runtime var db
474 ado_steps.push({
475 let mut map = serde_yaml::Mapping::new();
476 map.insert(
477 "bash".into(),
478 serde_yaml::Value::String(format!(
479 "rm $(AgentTempDirNormal)/bootstrapped-flowey/job{}.json",
480 job_idx.index()
481 )),
482 );
483 map.insert("displayName".into(), "🌼🧼 Redact bootstrap var db".into());
484 map.into()
485 });
486
487 ado_steps.push({
488 let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
489 r#"
490 publish: $(FLOWEY_TEMP_DIR)/bootstrapped-flowey
491 displayName: '🌼🥾 Publish bootstrapped flowey'
492 artifact: {artifact}
493 "#
494 ))
495 .unwrap();
496 map.into()
497 });
498 }
499
500 // ADO has this "helpful" default behavior where if you don't explicitly
501 // include a checkout step, it'll just auto-checkout the current repo.
502 //
503 // Work around this nonsense by doing a pre-pass over the emitted
504 // steps to enumerate how many steps start with `- checkout:`, and
505 // if the number is zero, emit an explicit `- checkout: none`.
506 {
507 let mut found = false;
508 for val in &ado_steps {
509 if let Some((key, _val)) = val.as_mapping().unwrap().iter().next() {
510 let Some(key) = key.as_str() else { continue };
511 if key == "checkout" {
512 found = true;
513 break;
514 }
515 }
516 }
517 if !found {
518 ado_steps.insert(0, {
519 let map: serde_yaml::Mapping = serde_yaml::from_str("checkout: none").unwrap();
520 map.into()
521 });
522 }
523 }
524
525 // Convert the pool information to the structured format
526 let AdoPool {
527 name: pool_name,
528 demands,
529 } = ado_pool
530 .clone()
531 .context(format!("must specify ADO pool for job '{label}'"))?;
532 let pool = if demands.is_empty() {
533 schema_ado_yaml::Pool::Pool(pool_name)
534 } else {
535 schema_ado_yaml::Pool::PoolWithMetadata(
536 [
537 ("name".into(), pool_name.into()),
538 ("demands".into(), demands.into()),
539 ]
540 .into(),
541 )
542 };
543
544 let get_job_id = |id: usize| {
545 ado_job_id_overrides
546 .get(&id)
547 .cloned()
548 .unwrap_or_else(|| format!("job{}", id.clone()))
549 };
550 ado_jobs.push(schema_ado_yaml::Job {
551 job: get_job_id(job_idx.index()),
552 display_name: label.clone(),
553 pool,
554 timeout_in_minutes: timeout_minutes,
555 depends_on: {
556 graph
557 .edges_directed(job_idx, petgraph::Direction::Incoming)
558 .map(|e| {
559 use petgraph::prelude::*;
560 get_job_id(e.source().index())
561 })
562 .collect()
563 },
564 variables: {
565 let mut ado_variables: Vec<schema_ado_yaml::Variable> = ado_variables
566 .clone()
567 .into_iter()
568 .map(|(name, value)| schema_ado_yaml::Variable { name, value })
569 .collect();
570
571 ado_variables.push(schema_ado_yaml::Variable {
572 name: "FLOWEY_TEMP_DIR".into(),
573 value: FLOWEY_TEMP_DIR.into(),
574 });
575
576 Some(ado_variables)
577 },
578 steps: ado_steps,
579 condition: Some(if let Some(cond_param_idx) = cond_param_idx {
580 format!(
581 "and(eq('${{{{ parameters.{} }}}}', 'true'), succeeded(), not(canceled()))",
582 parameters[cond_param_idx].name()
583 )
584 } else {
585 "and(succeeded(), not(canceled()))".into()
586 }),
587 })
588 }
589
590 fn resolve_trigger_paths(
591 paths: Vec<String>,
592 exclude_paths: Vec<String>,
593 ) -> anyhow::Result<Option<schema_ado_yaml::TriggerPaths>> {
594 match (paths.is_empty(), exclude_paths.is_empty()) {
595 (true, true) => Ok(None),
596 (true, false) | (false, _) => Ok(Some(schema_ado_yaml::TriggerPaths {
597 include: paths,
598 exclude: (!exclude_paths.is_empty()).then_some(exclude_paths),
599 })),
600 }
601 }
602
603 let ado_pipeline = schema_ado_yaml::Pipeline {
604 name: ado_name,
605 trigger: Some(match ado_ci_triggers {
606 None => schema_ado_yaml::CiTrigger::None(()),
607 Some(t) => {
608 let flowey_core::pipeline::AdoCiTriggers {
609 branches,
610 exclude_branches,
611 tags,
612 exclude_tags,
613 batch,
614 paths,
615 exclude_paths,
616 } = t;
617
618 if branches.is_empty() && tags.is_empty() {
619 anyhow::bail!("branches and tags cannot both be empty")
620 }
621
622 schema_ado_yaml::CiTrigger::Some {
623 batch,
624 branches: if branches.is_empty() {
625 if !exclude_branches.is_empty() {
626 anyhow::bail!("empty branch trigger with non-empty exclude")
627 }
628
629 None
630 } else {
631 Some(schema_ado_yaml::TriggerBranches {
632 include: branches,
633 exclude: if exclude_branches.is_empty() {
634 None
635 } else {
636 Some(exclude_branches)
637 },
638 })
639 },
640 tags: if tags.is_empty() {
641 if !exclude_tags.is_empty() {
642 anyhow::bail!("empty tags trigger with non-empty exclude")
643 }
644
645 None
646 } else {
647 Some(schema_ado_yaml::TriggerTags {
648 include: tags,
649 exclude: if exclude_tags.is_empty() {
650 None
651 } else {
652 Some(exclude_tags)
653 },
654 })
655 },
656 paths: resolve_trigger_paths(paths, exclude_paths)?,
657 }
658 }
659 }),
660 pr: Some(match ado_pr_triggers {
661 None => schema_ado_yaml::PrTrigger::None(()),
662 Some(t) => {
663 let flowey_core::pipeline::AdoPrTriggers {
664 branches,
665 exclude_branches,
666 run_on_draft,
667 auto_cancel,
668 paths,
669 exclude_paths,
670 } = t;
671
672 schema_ado_yaml::PrTrigger::Some {
673 auto_cancel,
674 drafts: run_on_draft,
675 branches: schema_ado_yaml::TriggerBranches {
676 include: branches,
677 exclude: if exclude_branches.is_empty() {
678 None
679 } else {
680 Some(exclude_branches)
681 },
682 },
683 paths: resolve_trigger_paths(paths, exclude_paths)?,
684 }
685 }
686 }),
687 schedules: if ado_schedule_triggers.is_empty() {
688 None
689 } else {
690 Some(
691 ado_schedule_triggers
692 .into_iter()
693 .map(|t| {
694 let flowey_core::pipeline::AdoScheduleTriggers {
695 display_name,
696 branches,
697 exclude_branches,
698 cron,
699 } = t;
700
701 schema_ado_yaml::Schedule {
702 cron,
703 display_name,
704 branches: schema_ado_yaml::TriggerBranches {
705 include: branches,
706 exclude: if exclude_branches.is_empty() {
707 None
708 } else {
709 Some(exclude_branches)
710 },
711 },
712 batch: false,
713 }
714 })
715 .collect(),
716 )
717 },
718 variables: if !ado_variables.is_empty() {
719 Some(
720 ado_variables
721 .into_iter()
722 .map(|(name, value)| schema_ado_yaml::Variable { name, value })
723 .collect(),
724 )
725 } else {
726 None
727 },
728 stages: None,
729 jobs: Some(ado_jobs),
730 parameters: if !parameters.is_empty() {
731 Some(
732 parameters
733 .clone()
734 .into_iter()
735 .map(|param| match param {
736 flowey_core::pipeline::internal::Parameter::Bool {
737 name,
738 description,
739 kind: _,
740 default,
741 } => schema_ado_yaml::Parameter {
742 name,
743 display_name: description,
744 ty: schema_ado_yaml::ParameterType::Boolean { default },
745 },
746 flowey_core::pipeline::internal::Parameter::String {
747 name,
748 description,
749 kind: _,
750 default,
751 possible_values,
752 } => schema_ado_yaml::Parameter {
753 name,
754 display_name: description,
755 ty: schema_ado_yaml::ParameterType::String {
756 default,
757 values: possible_values,
758 },
759 },
760 flowey_core::pipeline::internal::Parameter::Num {
761 name,
762 description,
763 kind: _,
764 default,
765 possible_values,
766 } => schema_ado_yaml::Parameter {
767 name,
768 display_name: description,
769 ty: schema_ado_yaml::ParameterType::Number {
770 default,
771 values: possible_values,
772 },
773 },
774 })
775 .collect(),
776 )
777 } else {
778 None
779 },
780 resources: {
781 if ado_resources_repository.is_empty() {
782 None
783 } else {
784 Some(schema_ado_yaml::Resources {
785 repositories: ado_resources_repository
786 .into_iter()
787 .map(
788 |InternalAdoResourcesRepository {
789 repo_id,
790 repo_type,
791 name,
792 git_ref,
793 endpoint,
794 }| {
795 use flowey_core::pipeline::AdoResourcesRepositoryRef;
796 use flowey_core::pipeline::AdoResourcesRepositoryType;
797
798 schema_ado_yaml::ResourcesRepository {
799 repository: repo_id,
800 endpoint,
801 name,
802 r#ref: match git_ref {
803 AdoResourcesRepositoryRef::Fixed(s) => s,
804 AdoResourcesRepositoryRef::Parameter(idx) => {
805 let name = parameters[idx].name();
806 format!("${{{{ parameters.{name} }}}}")
807 }
808 },
809 r#type: match repo_type {
810 AdoResourcesRepositoryType::AzureReposGit => {
811 schema_ado_yaml::ResourcesRepositoryType::Git
812 }
813 AdoResourcesRepositoryType::GitHub => {
814 schema_ado_yaml::ResourcesRepositoryType::GitHub
815 }
816 },
817 }
818 },
819 )
820 .collect::<Vec<_>>(),
821 })
822 }
823 },
824 extends: None,
825 };
826
827 match check {
828 CheckMode::Check(_) | CheckMode::Runtime(_) => check_generated_yaml_and_json(
829 &ado_pipeline,
830 &pipeline_static_db,
831 check,
832 repo_root,
833 pipeline_file,
834 ado_post_process_yaml_cb,
835 ),
836 CheckMode::None => write_generated_yaml_and_json(
837 &ado_pipeline,
838 &pipeline_static_db,
839 repo_root,
840 pipeline_file,
841 ado_post_process_yaml_cb,
842 ),
843 }
844}
845
846/// Resolve a flow as a sequence of ADO YAML steps.
847///
848/// These steps can then be marshalled into a well-formed ADO pipeline yaml
849/// using a separate ADO pipeline yaml builder
850// pub(crate) so that internal debug CLI tooling can use it
851pub(crate) fn resolve_flow_as_ado_yaml_steps(
852 seed_nodes: BTreeMap<NodeHandle, (bool, Vec<Box<[u8]>>)>,
853 seed_configs: BTreeMap<NodeHandle, Vec<Box<[u8]>>>,
854 resolved_patches: flowey_core::patch::ResolvedPatches,
855 external_read_vars: BTreeSet<String>,
856 platform: FlowPlatform,
857 arch: FlowArch,
858 job_idx: usize,
859) -> anyhow::Result<super::common_yaml::ResolvedFlowSteps> {
860 let mut output_steps = Vec::new();
861
862 let crate::flow_resolver::stage1_dag::Stage1DagOutput {
863 mut output_graph,
864 request_db,
865 config_db,
866 found_unreachable_nodes,
867 } = crate::flow_resolver::stage1_dag::stage1_dag(
868 FlowBackend::Ado,
869 platform,
870 arch,
871 resolved_patches,
872 seed_nodes,
873 seed_configs,
874 external_read_vars,
875 // TODO: support ADO agents with persistent storage
876 None,
877 )?;
878
879 if found_unreachable_nodes {
880 anyhow::bail!("detected unreachable nodes")
881 }
882
883 let output_order = petgraph::algo::toposort(&output_graph, None)
884 .expect("runtime variables cannot introduce a DAG cycle");
885
886 let var_db = VarDbRequestBuilder::new("$FLOWEY_BIN", job_idx);
887
888 let mut bash_commands = BashCommands::new_ado();
889 for idx in output_order.into_iter().rev() {
890 let OutputGraphEntry { node_handle, step } = output_graph[idx].1.take().unwrap();
891
892 let node_modpath = node_handle.modpath();
893
894 match step {
895 Step::Anchor { .. } => {}
896 Step::Rust {
897 idx,
898 can_merge,
899 label,
900 code: _,
901 } => {
902 output_steps.extend(bash_commands.push(
903 Some(label),
904 can_merge,
905 crate::cli::exec_snippet::construct_exec_snippet_cli(
906 "$(FLOWEY_BIN)",
907 node_modpath,
908 idx,
909 job_idx,
910 ),
911 ));
912 }
913 Step::AdoYaml {
914 label,
915 raw_yaml,
916 ado_to_rust,
917 rust_to_ado,
918 condvar,
919 code_idx,
920 code,
921 } => {
922 for (rust_var, ado_var) in rust_to_ado {
923 // flowey considers all ADO vars to be typed as raw strings
924 let read_rust_var = var_db
925 .write_to_ado_env(&rust_var, &ado_var)
926 .raw_string(true)
927 .condvar(condvar.as_deref());
928
929 bash_commands.push_minor(format!("{read_rust_var}\n"));
930 }
931
932 if !raw_yaml.is_empty() {
933 if let Some(condvar) = &condvar {
934 // guaranteed to be a bare bool `true`/`false`, hence
935 // is_raw_string = false
936 let read_condvar = var_db.write_to_ado_env(condvar, "FLOWEY_CONDITION");
937
938 bash_commands.push_minor(format!("{read_condvar}\n"));
939 }
940
941 let raw_yaml = if code.lock().is_some() {
942 let inline_snippet = crate::cli::exec_snippet::construct_exec_snippet_cli(
943 "$(FLOWEY_BIN)",
944 node_modpath,
945 code_idx,
946 job_idx,
947 );
948 let post_process =
949 raw_yaml.replace("{{FLOWEY_INLINE_SCRIPT}}", &inline_snippet);
950 if raw_yaml == post_process {
951 return Err(anyhow::anyhow!("if using inlins-enippet, YAML must include {{{{FLOWEY_INLINE_SCRIPT}}}}").context(format!(
952 "invalid yaml in node {node_modpath}: {raw_yaml}"
953 )));
954 }
955 post_process
956 } else {
957 raw_yaml
958 };
959
960 let step: serde_yaml::Value = serde_yaml::from_str(&raw_yaml)
961 .context(format!("invalid yaml in node {node_modpath}: {raw_yaml}"))?;
962 let step = {
963 let mut step = step;
964 let seq = step
965 .as_sequence_mut()
966 .context("yaml snippet did not parse as a sequence")?;
967
968 if seq.len() != 1 {
969 anyhow::bail!("yaml snippet contained more than one sequence element")
970 }
971
972 let map = seq
973 .first_mut()
974 .unwrap()
975 .as_mapping_mut()
976 .context("yaml snippet did not parse as a map")?;
977 let existing = map.insert("displayName".into(), label.into());
978 if existing.is_some() {
979 anyhow::bail!("yaml snippet included `displayName`")
980 }
981 if condvar.is_some() {
982 let existing = map.insert(
983 "condition".into(),
984 "and(eq(variables['FLOWEY_CONDITION'], true), succeeded(), not(canceled()))".into(),
985 );
986 if existing.is_some() {
987 anyhow::bail!("yaml snippet included `condition`")
988 }
989 }
990
991 step
992 };
993 output_steps.extend(bash_commands.flush());
994 output_steps.push(step.as_sequence().unwrap().first().unwrap().clone());
995 }
996
997 for (ado_var, rust_var, is_secret) in ado_to_rust {
998 // flowey considers all ADO vars to be typed as raw strings
999 let write_rust_var = var_db
1000 .update_from_stdin(&rust_var, is_secret)
1001 .raw_string(true)
1002 .condvar(condvar.as_deref())
1003 .env_source(Some(&ado_var));
1004
1005 let cmd = format!(
1006 r#"
1007{write_rust_var} <<'EOF'
1008$({ado_var})
1009EOF
1010"#
1011 )
1012 .trim()
1013 .to_string();
1014
1015 bash_commands.push_minor(cmd);
1016 }
1017 }
1018 Step::GitHubYaml { label, .. } => {
1019 anyhow::bail!("GitHub YAML not supported in ADO: {label}")
1020 }
1021 }
1022 }
1023
1024 output_steps.extend(bash_commands.flush());
1025
1026 let request_db = request_db
1027 .into_iter()
1028 .map(|(node_handle, reqs)| {
1029 (
1030 node_handle.modpath().to_owned(),
1031 reqs.into_iter()
1032 .map(crate::cli::exec_snippet::SerializedRequest)
1033 .collect(),
1034 )
1035 })
1036 .collect();
1037
1038 let config_db = config_db
1039 .into_iter()
1040 .map(|(node_handle, configs)| {
1041 (
1042 node_handle.modpath().to_owned(),
1043 configs
1044 .into_iter()
1045 .map(crate::cli::exec_snippet::SerializedRequest)
1046 .collect(),
1047 )
1048 })
1049 .collect();
1050
1051 Ok(super::common_yaml::ResolvedFlowSteps {
1052 steps: output_steps,
1053 request_db,
1054 config_db,
1055 })
1056}
1057