GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/openvmm

Public

mirrored from https://github.com/microsoft/openvmmAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
copilot/identify-fixed-issues

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openvmm/flowey/flowey_cli/src/pipeline_resolver

flowey/flowey_cli/src/pipeline_resolver/ado_yaml.rs

997lines · modecode

RawDownload
Latest commit unavailable.
1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4use super::common_yaml::BashCommands;
5use super::common_yaml::FloweySource;
6use super::common_yaml::check_generated_yaml_and_json;
7use super::common_yaml::job_flowey_bootstrap_source;
8use super::common_yaml::write_generated_yaml_and_json;
9use super::generic::ResolvedJobArtifact;
10use super::generic::ResolvedJobUseParameter;
11use crate::cli::exec_snippet::FloweyPipelineStaticDb;
12use crate::cli::exec_snippet::VAR_DB_SEEDVAR_FLOWEY_WORKING_DIR;
13use crate::cli::pipeline::CheckMode;
14use crate::cli::var_db::VarDbRequestBuilder;
15use crate::flow_resolver::stage1_dag::OutputGraphEntry;
16use crate::flow_resolver::stage1_dag::Step;
17use crate::pipeline_resolver::generic::ResolvedPipeline;
18use crate::pipeline_resolver::generic::ResolvedPipelineJob;
19use anyhow::Context;
20use flowey_core::node::FlowArch;
21use flowey_core::node::FlowBackend;
22use flowey_core::node::FlowPlatform;
23use flowey_core::node::FlowPlatformKind;
24use flowey_core::node::NodeHandle;
25use flowey_core::pipeline::internal::AdoPool;
26use flowey_core::pipeline::internal::InternalAdoResourcesRepository;
27use std::collections::BTreeMap;
28use std::collections::BTreeSet;
29use std::fmt::Write;
30use std::path::Path;
31
32/// We use $(Build.StagingDirectory)/.flowey-internal instead
33/// of $(Agent.TempDirectory) to (hopefully) guarantee that this folder
34/// resides on the same mount-point as the repos being cloned.
35///
36/// violating this property would result in calls to `fs::rename` in
37/// downstream flowey nodes to fail.
38const FLOWEY_TEMP_DIR: &str = "$(Build.StagingDirectory)/.flowey-internal";
39
40/// Emit a pipeline as a single self-contained ADO yaml file
41pub fn ado_yaml(
42 pipeline: ResolvedPipeline,
43 runtime_debug_log: bool,
44 repo_root: &Path,
45 pipeline_file: &Path,
46 flowey_crate: &str,
47 check: CheckMode,
48) -> anyhow::Result<()> {
49 if pipeline_file.extension().and_then(|s| s.to_str()) != Some("yaml") {
50 anyhow::bail!("pipeline name must end with .yaml")
51 }
52
53 let ResolvedPipeline {
54 graph,
55 order,
56 parameters,
57 ado_name,
58 ado_schedule_triggers,
59 ado_ci_triggers,
60 ado_pr_triggers,
61 ado_bootstrap_template,
62 ado_resources_repository,
63 ado_post_process_yaml_cb,
64 ado_variables,
65 ref ado_job_id_overrides,
66 gh_name: _,
67 gh_schedule_triggers: _,
68 gh_ci_triggers: _,
69 gh_pr_triggers: _,
70 gh_bootstrap_template: _,
71 } = pipeline;
72
73 let mut job_flowey_source: BTreeMap<petgraph::prelude::NodeIndex, FloweySource> =
74 job_flowey_bootstrap_source(&graph, &order);
75
76 let mut pipeline_static_db = FloweyPipelineStaticDb {
77 flow_backend: crate::cli::FlowBackendCli::Ado,
78 var_db_backend_kind: crate::cli::exec_snippet::VarDbBackendKind::Json,
79 job_reqs: BTreeMap::new(),
80 job_command_wrappers: BTreeMap::new(),
81 };
82
83 let mut ado_jobs = Vec::new();
84
85 for job_idx in order {
86 let ResolvedPipelineJob {
87 ref root_nodes,
88 ref patches,
89 ref label,
90 platform,
91 arch,
92 cond_param_idx,
93 ref ado_pool,
94 timeout_minutes,
95 command_wrapper: ref command_wrapper_kind,
96 gh_override_if: _,
97 gh_global_env: _,
98 gh_pool: _,
99 gh_permissions: _,
100 ref external_read_vars,
101 ref parameters_used,
102 ref artifacts_used,
103 ref artifacts_published,
104 ref ado_variables,
105 } = graph[job_idx];
106
107 let flowey_source = job_flowey_source.remove(&job_idx).unwrap();
108
109 let (steps, req_db) = resolve_flow_as_ado_yaml_steps(
110 root_nodes
111 .clone()
112 .into_iter()
113 .map(|(node, requests)| (node, (true, requests)))
114 .collect(),
115 patches.clone(),
116 external_read_vars.clone(),
117 platform,
118 arch,
119 job_idx.index(),
120 )
121 .context(format!("in job '{label}'"))?;
122
123 {
124 let existing = pipeline_static_db.job_reqs.insert(job_idx.index(), req_db);
125 assert!(existing.is_none())
126 }
127
128 if let Some(wrapper_kind) = command_wrapper_kind {
129 pipeline_static_db
130 .job_command_wrappers
131 .insert(job_idx.index(), wrapper_kind.clone());
132 }
133
134 let mut ado_steps = Vec::new();
135
136 if let FloweySource::Bootstrap(artifact, publish) = &flowey_source {
137 // actual artifact publish happens at the end of the job
138 let _ = (artifact, publish);
139
140 if ado_bootstrap_template.is_empty() {
141 anyhow::bail!(
142 "Did not specify flowey bootstrap template. Please provide one using `Pipeline::ado_set_flowey_bootstrap_template`"
143 )
144 }
145
146 let ado_bootstrap_template = ado_bootstrap_template
147 .replace("{{FLOWEY_BIN_EXTENSION}}", platform.exe_suffix())
148 .replace("{{FLOWEY_CRATE}}", flowey_crate)
149 .replace(
150 "{{FLOWEY_PIPELINE_PATH}}",
151 &pipeline_file.with_extension("").display().to_string(),
152 )
153 .replace(
154 "{{FLOWEY_TARGET}}",
155 match (platform, arch) {
156 (FlowPlatform::Windows, FlowArch::X86_64) => "x86_64-pc-windows-msvc",
157 (FlowPlatform::Windows, FlowArch::Aarch64) => "aarch64-pc-windows-msvc",
158 (FlowPlatform::Linux(_), FlowArch::X86_64) => "x86_64-unknown-linux-gnu",
159 (FlowPlatform::Linux(_), FlowArch::Aarch64) => "aarch64-unknown-linux-gnu",
160 (platform, arch) => anyhow::bail!(
161 "unsupported ADO platform/arch combo {platform:?}/{arch:?}"
162 ),
163 },
164 )
165 .replace(
166 "{{FLOWEY_OUTDIR}}",
167 "$(FLOWEY_TEMP_DIR)/bootstrapped-flowey",
168 );
169
170 let bootstrap_steps: serde_yaml::Sequence =
171 serde_yaml::from_str(&ado_bootstrap_template)
172 .context("malformed flowey bootstrap template")?;
173
174 ado_steps.extend(bootstrap_steps);
175 }
176
177 // the first few steps in any job are some "artisan" code, which
178 // downloads the previously bootstrapped flowey artifact and set up
179 // various vars that flowey will then rely on throughout the rest
180 // of the job
181
182 // download previously bootstrapped flowey
183 if let FloweySource::Consume(artifact) = &flowey_source {
184 ado_steps.push({
185 let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
186 r#"
187 task: DownloadPipelineArtifact@2
188 displayName: '🌼🥾 Download bootstrapped flowey'
189 inputs:
190 artifact: {artifact}
191 path: $(FLOWEY_TEMP_DIR)/bootstrapped-flowey
192 "#
193 ))
194 .unwrap();
195 map.into()
196 });
197 }
198
199 // also download any artifacts that'll be used
200 for ResolvedJobArtifact {
201 flowey_var: _,
202 name,
203 } in artifacts_used
204 {
205 ado_steps.push({
206 let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
207 r#"
208 task: DownloadPipelineArtifact@2
209 displayName: '🌼📦 Download {name}'
210 inputs:
211 artifact: {name}
212 path: $(FLOWEY_TEMP_DIR)/used_artifacts/{name}
213 "#
214 ))
215 .unwrap();
216 map.into()
217 });
218 }
219
220 let flowey_bin = platform.binary("flowey");
221 let flowey_executable_bash = format!(
222 r###"
223set -e
224AgentTempDirNormal="$(FLOWEY_TEMP_DIR)"
225AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|')
226echo "##vso[task.setvariable variable=AgentTempDirNormal;]$AgentTempDirNormal"
227
228chmod +x $AgentTempDirNormal/bootstrapped-flowey/{flowey_bin}
229FLOWEY_BIN="$AgentTempDirNormal/bootstrapped-flowey/{flowey_bin}"
230echo "##vso[task.setvariable variable=FLOWEY_BIN;]$FLOWEY_BIN"
231"###
232 ).trim_start().to_string();
233
234 ado_steps.push({
235 let mut map = serde_yaml::Mapping::new();
236 map.insert(
237 "bash".into(),
238 serde_yaml::Value::String(flowey_executable_bash),
239 );
240 map.insert("displayName".into(), "Set flowey path".into());
241 map.into()
242 });
243
244 let mut flowey_bootstrap_bash = String::new();
245
246 let var_db = VarDbRequestBuilder::new("$FLOWEY_BIN", job_idx.index());
247
248 let bootstrap_bash_var_db_inject = |var, is_raw_string| {
249 var_db
250 .update_from_stdin(var, false)
251 .raw_string(is_raw_string)
252 .to_string()
253 };
254
255 // and now use those vars to do some flowey bootstrap
256 writeln!(flowey_bootstrap_bash, "{}", {
257 let runtime_debug_level = if runtime_debug_log { "debug" } else { "info" };
258
259 let var_db_insert_runtime_debug_level =
260 bootstrap_bash_var_db_inject("FLOWEY_LOG", false);
261 let var_db_insert_working_dir =
262 bootstrap_bash_var_db_inject(VAR_DB_SEEDVAR_FLOWEY_WORKING_DIR, true);
263
264 // Need to use "normalized" path in cases where the path is being
265 // used directly from a bash context, as is the case when we are
266 // trying to invoke `flowey.exe` in argv0 position)
267 //
268 // https://github.com/microsoft/azure-pipelines-tasks/issues/10653#issuecomment-585669089
269 format!(
270 r###"
271set -e
272echo '"{runtime_debug_level}"' | {var_db_insert_runtime_debug_level}
273echo "$(FLOWEY_TEMP_DIR)/work" | {var_db_insert_working_dir}
274"###
275 )
276 .trim_start()
277 .to_owned()
278 })?;
279
280 // import pipeline vars being used by the job into flowey
281 for ResolvedJobUseParameter {
282 flowey_var,
283 pipeline_param_idx,
284 } in parameters_used
285 {
286 let is_string = matches!(
287 parameters[*pipeline_param_idx],
288 flowey_core::pipeline::internal::Parameter::String { .. }
289 );
290 let is_bool = matches!(
291 parameters[*pipeline_param_idx],
292 flowey_core::pipeline::internal::Parameter::Bool { .. }
293 );
294
295 let name = parameters[*pipeline_param_idx].name();
296
297 // ADO resolves bools as `True` and `False`, _sigh_
298 let with_lowercase = if is_bool {
299 r#" | tr '[:upper:]' '[:lower:]'"#
300 } else {
301 ""
302 };
303
304 let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, is_string);
305
306 let cmd = format!(
307 r#"
308cat <<'EOF'{with_lowercase} | {var_db_inject_cmd}
309${{{{ parameters.{name} }}}}
310EOF
311"#
312 )
313 .trim()
314 .to_string();
315 writeln!(flowey_bootstrap_bash, "{}", cmd)?;
316 }
317
318 // next, emit ado steps to create dirs for artifacts which will be
319 // published
320 for ResolvedJobArtifact { flowey_var, name } in artifacts_published {
321 writeln!(
322 flowey_bootstrap_bash,
323 r#"mkdir -p "$(AgentTempDirNormal)/publish_artifacts/{name}""#
324 )?;
325 let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, true);
326 writeln!(
327 flowey_bootstrap_bash,
328 r#"echo "$(FLOWEY_TEMP_DIR)/publish_artifacts/{name}" | {var_db_inject_cmd}"#,
329 )?;
330 }
331
332 // lastly, emit ado steps that report the dirs for any artifacts which
333 // are used by this job
334 for ResolvedJobArtifact { flowey_var, name } in artifacts_used {
335 // do NOT use ADO macro syntax $(...), since this is in the same
336 // bootstrap block as where those ADO vars get defined, meaning it's
337 // not available yet!
338 let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, true);
339 writeln!(
340 flowey_bootstrap_bash,
341 r#"echo "$(FLOWEY_TEMP_DIR)/used_artifacts/{name}" | {var_db_inject_cmd}"#,
342 )?;
343 }
344
345 // if this was a bootstrap job, also take a moment to run a "self check"
346 // to make sure that the current checked-in template matches the one it
347 // expected
348 if let FloweySource::Bootstrap(..) = &flowey_source {
349 let mut current_invocation = std::env::args().collect::<Vec<_>>();
350
351 current_invocation[0] = "$(FLOWEY_BIN)".into();
352
353 // if this code path is run while generating the YAML to compare the
354 // check against, we want to remove the --runtime or --check param from the
355 // current call, or else there'll be a dupe
356 let mut strip_parameter = |prefix: &str| {
357 if let Some(i) = current_invocation
358 .iter()
359 .position(|s| s.starts_with(prefix))
360 {
361 current_invocation.remove(i);
362 if !current_invocation[i].starts_with(prefix) {
363 current_invocation.remove(i);
364 }
365 }
366 };
367
368 strip_parameter("--runtime");
369 strip_parameter("--check");
370
371 // insert the --check bit of the call alongside the --out param
372 {
373 let i = current_invocation
374 .iter()
375 .position(|s| s.starts_with("--out"))
376 .unwrap();
377
378 let current_yaml = match platform.kind() {
379 FlowPlatformKind::Windows => {
380 r#"$ESCAPED_AGENT_TEMPDIR\\bootstrapped-flowey\\pipeline.yaml"#
381 }
382 FlowPlatformKind::Unix => {
383 r#"$ESCAPED_AGENT_TEMPDIR/bootstrapped-flowey/pipeline.yaml"#
384 }
385 };
386
387 current_invocation.insert(i, current_yaml.into());
388 current_invocation.insert(i, "--runtime".into());
389 }
390
391 // Need to use an escaped version of the "true" windows/linux path
392 // here, or else the --check will fail.
393 let cmd = format!(
394 r###"
395ESCAPED_AGENT_TEMPDIR=$(
396cat <<'EOF' | sed 's/\\/\\\\/g'
397$(FLOWEY_TEMP_DIR)
398EOF
399)
400{}
401"###,
402 current_invocation.join(" ")
403 );
404
405 ado_steps.push({
406 let mut map = serde_yaml::Mapping::new();
407 map.insert(
408 "bash".into(),
409 serde_yaml::Value::String(cmd.trim().to_string()),
410 );
411 map.insert("displayName".into(), "🌼🔎 Self-check YAML".into());
412 map.into()
413 })
414 }
415
416 ado_steps.push({
417 let mut map = serde_yaml::Mapping::new();
418 map.insert(
419 "bash".into(),
420 serde_yaml::Value::String(flowey_bootstrap_bash),
421 );
422 map.insert("displayName".into(), "🌼🛫 Initialize job".into());
423 map.into()
424 });
425
426 // now that we've done all the job-level bootstrapping, we can emit all
427 // the actual steps the user cares about
428 ado_steps.extend(steps);
429
430 // ..and once that's done, the last order of business is to emit some
431 // ado steps to publish the various artifacts created by this job
432 for ResolvedJobArtifact {
433 flowey_var: _,
434 name,
435 } in artifacts_published
436 {
437 ado_steps.push({
438 let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
439 r#"
440 publish: $(FLOWEY_TEMP_DIR)/publish_artifacts/{name}
441 displayName: '🌼📦 Publish {name}'
442 artifact: {name}
443 "#
444 ))
445 .unwrap();
446 map.into()
447 });
448 }
449
450 // also, if this job also bootstrapped flowey that other nodes depend
451 // on, make sure to publish it!
452 if let FloweySource::Bootstrap(artifact, true) = flowey_source {
453 // don't leak the bootstrap job's runtime var db
454 ado_steps.push({
455 let mut map = serde_yaml::Mapping::new();
456 map.insert(
457 "bash".into(),
458 serde_yaml::Value::String(format!(
459 "rm $(AgentTempDirNormal)/bootstrapped-flowey/job{}.json",
460 job_idx.index()
461 )),
462 );
463 map.insert("displayName".into(), "🌼🧼 Redact bootstrap var db".into());
464 map.into()
465 });
466
467 ado_steps.push({
468 let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
469 r#"
470 publish: $(FLOWEY_TEMP_DIR)/bootstrapped-flowey
471 displayName: '🌼🥾 Publish bootstrapped flowey'
472 artifact: {artifact}
473 "#
474 ))
475 .unwrap();
476 map.into()
477 });
478 }
479
480 // ADO has this "helpful" default behavior where if you don't explicitly
481 // include a checkout step, it'll just auto-checkout the current repo.
482 //
483 // Work around this nonsense by doing a pre-pass over the emitted
484 // steps to enumerate how many steps start with `- checkout:`, and
485 // if the number is zero, emit an explicit `- checkout: none`.
486 {
487 let mut found = false;
488 for val in &ado_steps {
489 if let Some((key, _val)) = val.as_mapping().unwrap().iter().next() {
490 let Some(key) = key.as_str() else { continue };
491 if key == "checkout" {
492 found = true;
493 break;
494 }
495 }
496 }
497 if !found {
498 ado_steps.insert(0, {
499 let map: serde_yaml::Mapping = serde_yaml::from_str("checkout: none").unwrap();
500 map.into()
501 });
502 }
503 }
504
505 // Convert the pool information to the structured format
506 let AdoPool {
507 name: pool_name,
508 demands,
509 } = ado_pool
510 .clone()
511 .context(format!("must specify ADO pool for job '{label}'"))?;
512 let pool = if demands.is_empty() {
513 schema_ado_yaml::Pool::Pool(pool_name)
514 } else {
515 schema_ado_yaml::Pool::PoolWithMetadata(
516 [
517 ("name".into(), pool_name.into()),
518 ("demands".into(), demands.into()),
519 ]
520 .into(),
521 )
522 };
523
524 let get_job_id = |id: usize| {
525 ado_job_id_overrides
526 .get(&id)
527 .cloned()
528 .unwrap_or_else(|| format!("job{}", id.clone()))
529 };
530 ado_jobs.push(schema_ado_yaml::Job {
531 job: get_job_id(job_idx.index()),
532 display_name: label.clone(),
533 pool,
534 timeout_in_minutes: timeout_minutes,
535 depends_on: {
536 graph
537 .edges_directed(job_idx, petgraph::Direction::Incoming)
538 .map(|e| {
539 use petgraph::prelude::*;
540 get_job_id(e.source().index())
541 })
542 .collect()
543 },
544 variables: {
545 let mut ado_variables: Vec<schema_ado_yaml::Variable> = ado_variables
546 .clone()
547 .into_iter()
548 .map(|(name, value)| schema_ado_yaml::Variable { name, value })
549 .collect();
550
551 ado_variables.push(schema_ado_yaml::Variable {
552 name: "FLOWEY_TEMP_DIR".into(),
553 value: FLOWEY_TEMP_DIR.into(),
554 });
555
556 Some(ado_variables)
557 },
558 steps: ado_steps,
559 condition: Some(if let Some(cond_param_idx) = cond_param_idx {
560 format!(
561 "and(eq('${{{{ parameters.{} }}}}', 'true'), succeeded(), not(canceled()))",
562 parameters[cond_param_idx].name()
563 )
564 } else {
565 "and(succeeded(), not(canceled()))".into()
566 }),
567 })
568 }
569
570 let ado_pipeline = schema_ado_yaml::Pipeline {
571 name: ado_name,
572 trigger: Some(match ado_ci_triggers {
573 None => schema_ado_yaml::CiTrigger::None(()),
574 Some(t) => {
575 let flowey_core::pipeline::AdoCiTriggers {
576 branches,
577 exclude_branches,
578 tags,
579 exclude_tags,
580 batch,
581 } = t;
582
583 if branches.is_empty() && tags.is_empty() {
584 anyhow::bail!("branches and tags cannot both be empty")
585 }
586
587 schema_ado_yaml::CiTrigger::Some {
588 batch,
589 branches: if branches.is_empty() {
590 if !exclude_branches.is_empty() {
591 anyhow::bail!("empty branch trigger with non-empty exclude")
592 }
593
594 None
595 } else {
596 Some(schema_ado_yaml::TriggerBranches {
597 include: branches,
598 exclude: if exclude_branches.is_empty() {
599 None
600 } else {
601 Some(exclude_branches)
602 },
603 })
604 },
605 tags: if tags.is_empty() {
606 if !exclude_tags.is_empty() {
607 anyhow::bail!("empty tags trigger with non-empty exclude")
608 }
609
610 None
611 } else {
612 Some(schema_ado_yaml::TriggerTags {
613 include: tags,
614 exclude: if exclude_tags.is_empty() {
615 None
616 } else {
617 Some(exclude_tags)
618 },
619 })
620 },
621 }
622 }
623 }),
624 pr: Some(match ado_pr_triggers {
625 None => schema_ado_yaml::PrTrigger::None(()),
626 Some(t) => {
627 let flowey_core::pipeline::AdoPrTriggers {
628 branches,
629 exclude_branches,
630 run_on_draft,
631 auto_cancel,
632 } = t;
633
634 schema_ado_yaml::PrTrigger::Some {
635 auto_cancel,
636 drafts: run_on_draft,
637 branches: schema_ado_yaml::TriggerBranches {
638 include: branches,
639 exclude: if exclude_branches.is_empty() {
640 None
641 } else {
642 Some(exclude_branches)
643 },
644 },
645 }
646 }
647 }),
648 schedules: if ado_schedule_triggers.is_empty() {
649 None
650 } else {
651 Some(
652 ado_schedule_triggers
653 .into_iter()
654 .map(|t| {
655 let flowey_core::pipeline::AdoScheduleTriggers {
656 display_name,
657 branches,
658 exclude_branches,
659 cron,
660 } = t;
661
662 schema_ado_yaml::Schedule {
663 cron,
664 display_name,
665 branches: schema_ado_yaml::TriggerBranches {
666 include: branches,
667 exclude: if exclude_branches.is_empty() {
668 None
669 } else {
670 Some(exclude_branches)
671 },
672 },
673 batch: false,
674 }
675 })
676 .collect(),
677 )
678 },
679 variables: if !ado_variables.is_empty() {
680 Some(
681 ado_variables
682 .into_iter()
683 .map(|(name, value)| schema_ado_yaml::Variable { name, value })
684 .collect(),
685 )
686 } else {
687 None
688 },
689 stages: None,
690 jobs: Some(ado_jobs),
691 parameters: if !parameters.is_empty() {
692 Some(
693 parameters
694 .clone()
695 .into_iter()
696 .map(|param| match param {
697 flowey_core::pipeline::internal::Parameter::Bool {
698 name,
699 description,
700 kind: _,
701 default,
702 } => schema_ado_yaml::Parameter {
703 name,
704 display_name: description,
705 ty: schema_ado_yaml::ParameterType::Boolean { default },
706 },
707 flowey_core::pipeline::internal::Parameter::String {
708 name,
709 description,
710 kind: _,
711 default,
712 possible_values,
713 } => schema_ado_yaml::Parameter {
714 name,
715 display_name: description,
716 ty: schema_ado_yaml::ParameterType::String {
717 default,
718 values: possible_values,
719 },
720 },
721 flowey_core::pipeline::internal::Parameter::Num {
722 name,
723 description,
724 kind: _,
725 default,
726 possible_values,
727 } => schema_ado_yaml::Parameter {
728 name,
729 display_name: description,
730 ty: schema_ado_yaml::ParameterType::Number {
731 default,
732 values: possible_values,
733 },
734 },
735 })
736 .collect(),
737 )
738 } else {
739 None
740 },
741 resources: {
742 if ado_resources_repository.is_empty() {
743 None
744 } else {
745 Some(schema_ado_yaml::Resources {
746 repositories: ado_resources_repository
747 .into_iter()
748 .map(
749 |InternalAdoResourcesRepository {
750 repo_id,
751 repo_type,
752 name,
753 git_ref,
754 endpoint,
755 }| {
756 use flowey_core::pipeline::AdoResourcesRepositoryRef;
757 use flowey_core::pipeline::AdoResourcesRepositoryType;
758
759 schema_ado_yaml::ResourcesRepository {
760 repository: repo_id,
761 endpoint,
762 name,
763 r#ref: match git_ref {
764 AdoResourcesRepositoryRef::Fixed(s) => s,
765 AdoResourcesRepositoryRef::Parameter(idx) => {
766 let name = parameters[idx].name();
767 format!("${{{{ parameters.{name} }}}}")
768 }
769 },
770 r#type: match repo_type {
771 AdoResourcesRepositoryType::AzureReposGit => {
772 schema_ado_yaml::ResourcesRepositoryType::Git
773 }
774 AdoResourcesRepositoryType::GitHub => {
775 schema_ado_yaml::ResourcesRepositoryType::GitHub
776 }
777 },
778 }
779 },
780 )
781 .collect::<Vec<_>>(),
782 })
783 }
784 },
785 extends: None,
786 };
787
788 match check {
789 CheckMode::Check(_) | CheckMode::Runtime(_) => check_generated_yaml_and_json(
790 &ado_pipeline,
791 &pipeline_static_db,
792 check,
793 repo_root,
794 pipeline_file,
795 ado_post_process_yaml_cb,
796 ),
797 CheckMode::None => write_generated_yaml_and_json(
798 &ado_pipeline,
799 &pipeline_static_db,
800 repo_root,
801 pipeline_file,
802 ado_post_process_yaml_cb,
803 ),
804 }
805}
806
807/// Resolve a flow as a sequence of ADO YAML steps.
808///
809/// These steps can then be marshalled into a well-formed ADO pipeline yaml
810/// using a separate ADO pipeline yaml builder
811// pub(crate) so that internal debug CLI tooling can use it
812pub(crate) fn resolve_flow_as_ado_yaml_steps(
813 seed_nodes: BTreeMap<NodeHandle, (bool, Vec<Box<[u8]>>)>,
814 resolved_patches: flowey_core::patch::ResolvedPatches,
815 external_read_vars: BTreeSet<String>,
816 platform: FlowPlatform,
817 arch: FlowArch,
818 job_idx: usize,
819) -> anyhow::Result<(
820 Vec<serde_yaml::Value>,
821 BTreeMap<String, Vec<crate::cli::exec_snippet::SerializedRequest>>,
822)> {
823 let mut output_steps = Vec::new();
824
825 let (mut output_graph, request_db, err_unreachable_nodes) =
826 crate::flow_resolver::stage1_dag::stage1_dag(
827 FlowBackend::Ado,
828 platform,
829 arch,
830 resolved_patches,
831 seed_nodes,
832 external_read_vars,
833 // TODO: support ADO agents with persistent storage
834 None,
835 )?;
836
837 if err_unreachable_nodes.is_some() {
838 anyhow::bail!("detected unreachable nodes")
839 }
840
841 let output_order = petgraph::algo::toposort(&output_graph, None)
842 .expect("runtime variables cannot introduce a DAG cycle");
843
844 let var_db = VarDbRequestBuilder::new("$FLOWEY_BIN", job_idx);
845
846 let mut bash_commands = BashCommands::new_ado();
847 for idx in output_order.into_iter().rev() {
848 let OutputGraphEntry { node_handle, step } = output_graph[idx].1.take().unwrap();
849
850 let node_modpath = node_handle.modpath();
851
852 match step {
853 Step::Anchor { .. } => {}
854 Step::Rust {
855 idx,
856 can_merge,
857 label,
858 code: _,
859 } => {
860 output_steps.extend(bash_commands.push(
861 Some(label),
862 can_merge,
863 crate::cli::exec_snippet::construct_exec_snippet_cli(
864 "$(FLOWEY_BIN)",
865 node_modpath,
866 idx,
867 job_idx,
868 ),
869 ));
870 }
871 Step::AdoYaml {
872 label,
873 raw_yaml,
874 ado_to_rust,
875 rust_to_ado,
876 condvar,
877 code_idx,
878 code,
879 } => {
880 for (rust_var, ado_var) in rust_to_ado {
881 // flowey considers all ADO vars to be typed as raw strings
882 let read_rust_var = var_db
883 .write_to_ado_env(&rust_var, &ado_var)
884 .raw_string(true)
885 .condvar(condvar.as_deref());
886
887 bash_commands.push_minor(format!("{read_rust_var}\n"));
888 }
889
890 if !raw_yaml.is_empty() {
891 if let Some(condvar) = &condvar {
892 // guaranteed to be a bare bool `true`/`false`, hence
893 // is_raw_string = false
894 let read_condvar = var_db.write_to_ado_env(condvar, "FLOWEY_CONDITION");
895
896 bash_commands.push_minor(format!("{read_condvar}\n"));
897 }
898
899 let raw_yaml = if code.lock().is_some() {
900 let inline_snippet = crate::cli::exec_snippet::construct_exec_snippet_cli(
901 "$(FLOWEY_BIN)",
902 node_modpath,
903 code_idx,
904 job_idx,
905 );
906 let post_process =
907 raw_yaml.replace("{{FLOWEY_INLINE_SCRIPT}}", &inline_snippet);
908 if raw_yaml == post_process {
909 return Err(anyhow::anyhow!("if using inlins-enippet, YAML must include {{{{FLOWEY_INLINE_SCRIPT}}}}").context(format!(
910 "invalid yaml in node {node_modpath}: {raw_yaml}"
911 )));
912 }
913 post_process
914 } else {
915 raw_yaml
916 };
917
918 let step: serde_yaml::Value = serde_yaml::from_str(&raw_yaml)
919 .context(format!("invalid yaml in node {node_modpath}: {raw_yaml}"))?;
920 let step = {
921 let mut step = step;
922 let seq = step
923 .as_sequence_mut()
924 .context("yaml snippet did not parse as a sequence")?;
925
926 if seq.len() != 1 {
927 anyhow::bail!("yaml snippet contained more than one sequence element")
928 }
929
930 let map = seq
931 .first_mut()
932 .unwrap()
933 .as_mapping_mut()
934 .context("yaml snippet did not parse as a map")?;
935 let existing = map.insert("displayName".into(), label.into());
936 if existing.is_some() {
937 anyhow::bail!("yaml snippet included `displayName`")
938 }
939 if condvar.is_some() {
940 let existing = map.insert(
941 "condition".into(),
942 "and(eq(variables['FLOWEY_CONDITION'], true), succeeded(), not(canceled()))".into(),
943 );
944 if existing.is_some() {
945 anyhow::bail!("yaml snippet included `condition`")
946 }
947 }
948
949 step
950 };
951 output_steps.extend(bash_commands.flush());
952 output_steps.push(step.as_sequence().unwrap().first().unwrap().clone());
953 }
954
955 for (ado_var, rust_var, is_secret) in ado_to_rust {
956 // flowey considers all ADO vars to be typed as raw strings
957 let write_rust_var = var_db
958 .update_from_stdin(&rust_var, is_secret)
959 .raw_string(true)
960 .condvar(condvar.as_deref())
961 .env_source(Some(&ado_var));
962
963 let cmd = format!(
964 r#"
965{write_rust_var} <<'EOF'
966$({ado_var})
967EOF
968"#
969 )
970 .trim()
971 .to_string();
972
973 bash_commands.push_minor(cmd);
974 }
975 }
976 Step::GitHubYaml { label, .. } => {
977 anyhow::bail!("GitHub YAML not supported in ADO: {label}")
978 }
979 }
980 }
981
982 output_steps.extend(bash_commands.flush());
983
984 let request_db = request_db
985 .into_iter()
986 .map(|(node_handle, reqs)| {
987 (
988 node_handle.modpath().to_owned(),
989 reqs.into_iter()
990 .map(crate::cli::exec_snippet::SerializedRequest)
991 .collect(),
992 )
993 })
994 .collect();
995
996 Ok((output_steps, request_db))
997}
998