GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/openvmm

Public

mirrored from https://github.com/microsoft/openvmmAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
release/2411-fork

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openvmm/flowey/flowey_cli/src/pipeline_resolver

flowey/flowey_cli/src/pipeline_resolver/ado_yaml.rs

991lines · modecode

RawDownload
Latest commit unavailable.
1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4use super::common_yaml::check_generated_yaml_and_json;
5use super::common_yaml::job_flowey_bootstrap_source;
6use super::common_yaml::write_generated_yaml_and_json;
7use super::common_yaml::FloweySource;
8use super::generic::ResolvedJobArtifact;
9use super::generic::ResolvedJobUseParameter;
10use crate::cli::exec_snippet::FloweyPipelineStaticDb;
11use crate::cli::exec_snippet::VAR_DB_SEEDVAR_FLOWEY_WORKING_DIR;
12use crate::flow_resolver::stage1_dag::OutputGraphEntry;
13use crate::flow_resolver::stage1_dag::Step;
14use crate::pipeline_resolver::generic::ResolvedPipeline;
15use crate::pipeline_resolver::generic::ResolvedPipelineJob;
16use anyhow::Context;
17use flowey_core::node::FlowArch;
18use flowey_core::node::FlowBackend;
19use flowey_core::node::FlowPlatform;
20use flowey_core::node::FlowPlatformKind;
21use flowey_core::node::NodeHandle;
22use flowey_core::pipeline::internal::AdoPool;
23use flowey_core::pipeline::internal::InternalAdoResourcesRepository;
24use std::collections::BTreeMap;
25use std::collections::BTreeSet;
26use std::fmt::Write;
27use std::path::Path;
28use std::path::PathBuf;
29
30/// We use $(Build.StagingDirectory)/.flowey-internal instead
31/// of $(Agent.TempDirectory) to (hopefully) guarantee that this folder
32/// resides on the same mount-point as the repos being cloned.
33///
34/// violating this property would result in calls to `fs::rename` in
35/// downstream flowey nodes to fail.
36const FLOWEY_TEMP_DIR: &str = "$(Build.StagingDirectory)/.flowey-internal";
37
38/// Emit a pipeline as a single self-contained ADO yaml file
39pub fn ado_yaml(
40 pipeline: ResolvedPipeline,
41 runtime_debug_log: bool,
42 repo_root: &Path,
43 pipeline_file: &Path,
44 flowey_crate: &str,
45 check: Option<PathBuf>,
46) -> anyhow::Result<()> {
47 if pipeline_file.extension().and_then(|s| s.to_str()) != Some("yaml") {
48 anyhow::bail!("pipeline name must end with .yaml")
49 }
50
51 let ResolvedPipeline {
52 graph,
53 order,
54 parameters,
55 ado_name,
56 ado_schedule_triggers,
57 ado_ci_triggers,
58 ado_pr_triggers,
59 ado_bootstrap_template,
60 ado_resources_repository,
61 ado_post_process_yaml_cb,
62 ado_variables,
63 gh_name: _,
64 gh_schedule_triggers: _,
65 gh_ci_triggers: _,
66 gh_pr_triggers: _,
67 gh_bootstrap_template: _,
68 } = pipeline;
69
70 let mut job_flowey_source: BTreeMap<petgraph::prelude::NodeIndex, FloweySource> =
71 job_flowey_bootstrap_source(&graph, &order);
72
73 let mut pipeline_static_db = FloweyPipelineStaticDb {
74 flow_backend: crate::cli::FlowBackendCli::Ado,
75 var_db_backend_kind: crate::cli::exec_snippet::VarDbBackendKind::Json,
76 job_reqs: BTreeMap::new(),
77 };
78
79 let mut ado_jobs = Vec::new();
80
81 for job_idx in order {
82 let ResolvedPipelineJob {
83 ref root_nodes,
84 ref patches,
85 ref label,
86 platform,
87 arch,
88 cond_param_idx,
89 ref ado_pool,
90 gh_override_if: _,
91 gh_global_env: _,
92 gh_pool: _,
93 gh_permissions: _,
94 ref external_read_vars,
95 ref parameters_used,
96 ref artifacts_used,
97 ref artifacts_published,
98 ref ado_variables,
99 } = graph[job_idx];
100
101 let flowey_source = job_flowey_source.remove(&job_idx).unwrap();
102
103 let (steps, req_db) = resolve_flow_as_ado_yaml_steps(
104 root_nodes
105 .clone()
106 .into_iter()
107 .map(|(node, requests)| (node, (true, requests)))
108 .collect(),
109 patches.clone(),
110 external_read_vars.clone(),
111 platform,
112 arch,
113 job_idx.index(),
114 )
115 .context(format!("in job '{label}'"))?;
116
117 {
118 let existing = pipeline_static_db.job_reqs.insert(job_idx.index(), req_db);
119 assert!(existing.is_none())
120 }
121
122 let mut ado_steps = Vec::new();
123
124 if let FloweySource::Bootstrap(artifact, publish) = &flowey_source {
125 // actual artifact publish happens at the end of the job
126 let _ = (artifact, publish);
127
128 if ado_bootstrap_template.is_empty() {
129 anyhow::bail!("Did not specify flowey bootstrap template. Please provide one using `Pipeline::ado_set_flowey_bootstrap_template`")
130 }
131
132 let ado_bootstrap_template = ado_bootstrap_template
133 .replace("{{FLOWEY_BIN_EXTENSION}}", platform.exe_suffix())
134 .replace("{{FLOWEY_CRATE}}", flowey_crate)
135 .replace(
136 "{{FLOWEY_PIPELINE_PATH}}",
137 &pipeline_file.with_extension("").display().to_string(),
138 )
139 .replace(
140 "{{FLOWEY_TARGET}}",
141 match platform {
142 FlowPlatform::Windows => "x86_64-pc-windows-msvc",
143 FlowPlatform::Linux(_) => "x86_64-unknown-linux-gnu",
144 platform => anyhow::bail!("unsupported ADO platform {platform:?}"),
145 },
146 )
147 .replace(
148 "{{FLOWEY_OUTDIR}}",
149 "$(FLOWEY_TEMP_DIR)/bootstrapped-flowey",
150 );
151
152 let bootstrap_steps: serde_yaml::Sequence =
153 serde_yaml::from_str(&ado_bootstrap_template)
154 .context("malformed flowey bootstrap template")?;
155
156 ado_steps.push({
157 let mut map = serde_yaml::Mapping::new();
158 map.insert("bash".into(), "echo \"injected!\"".into());
159 map.insert("displayName".into(), "🌼🥾 Bootstrap flowey".into());
160 map.into()
161 });
162 ado_steps.extend(bootstrap_steps);
163 }
164
165 // the first few steps in any job are some "artisan" code, which
166 // downloads the previously bootstrapped flowey artifact and set up
167 // various vars that flowey will then rely on throughout the rest
168 // of the job
169
170 // download previously bootstrapped flowey
171 if let FloweySource::Consume(artifact) = &flowey_source {
172 ado_steps.push({
173 let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
174 r#"
175 task: DownloadPipelineArtifact@2
176 displayName: '🌼🥾 Download bootstrapped flowey'
177 inputs:
178 artifact: {artifact}
179 path: $(FLOWEY_TEMP_DIR)/bootstrapped-flowey
180 "#
181 ))
182 .unwrap();
183 map.into()
184 });
185 }
186
187 // also download any artifacts that'll be used
188 for ResolvedJobArtifact {
189 flowey_var: _,
190 name,
191 } in artifacts_used
192 {
193 ado_steps.push({
194 let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
195 r#"
196 task: DownloadPipelineArtifact@2
197 displayName: '🌼📦 Download {name}'
198 inputs:
199 artifact: {name}
200 path: $(FLOWEY_TEMP_DIR)/used_artifacts/{name}
201 "#
202 ))
203 .unwrap();
204 map.into()
205 });
206 }
207
208 let mut flowey_bootstrap_bash = String::new();
209
210 let bootstrap_bash_var_db_inject = |var, is_raw_string| {
211 crate::cli::var_db::construct_var_db_cli(
212 "$FLOWEY_BIN",
213 job_idx.index(),
214 var,
215 false,
216 true,
217 None,
218 is_raw_string,
219 None,
220 )
221 };
222
223 // and now use those vars to do some flowey bootstrap
224 writeln!(flowey_bootstrap_bash, "{}", {
225 let flowey_bin = platform.binary("flowey");
226
227 let runtime_debug_level = if runtime_debug_log { "debug" } else { "info" };
228
229 let var_db_insert_runtime_debug_level =
230 bootstrap_bash_var_db_inject("FLOWEY_LOG", false);
231 let var_db_insert_working_dir =
232 bootstrap_bash_var_db_inject(VAR_DB_SEEDVAR_FLOWEY_WORKING_DIR, true);
233
234 // Need to use "normalized" path in cases where the path is being
235 // used directly from a bash context, as is the case when we are
236 // trying to invoke `flowey.exe` in argv0 position)
237 //
238 // https://github.com/microsoft/azure-pipelines-tasks/issues/10653#issuecomment-585669089
239 format!(
240 r###"
241AgentTempDirNormal="$(FLOWEY_TEMP_DIR)"
242AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|')
243echo "##vso[task.setvariable variable=AgentTempDirNormal;]$AgentTempDirNormal"
244
245chmod +x $AgentTempDirNormal/bootstrapped-flowey/{flowey_bin}
246FLOWEY_BIN="$AgentTempDirNormal/bootstrapped-flowey/{flowey_bin}"
247echo "##vso[task.setvariable variable=FLOWEY_BIN;]$FLOWEY_BIN"
248
249echo '"{runtime_debug_level}"' | {var_db_insert_runtime_debug_level}
250echo "$(FLOWEY_TEMP_DIR)/work" | {var_db_insert_working_dir}
251"###
252 )
253 .trim_start()
254 })?;
255
256 // import pipeline vars being used by the job into flowey
257 for ResolvedJobUseParameter {
258 flowey_var,
259 pipeline_param_idx,
260 } in parameters_used
261 {
262 let is_string = matches!(
263 parameters[*pipeline_param_idx],
264 flowey_core::pipeline::internal::Parameter::String { .. }
265 );
266 let is_bool = matches!(
267 parameters[*pipeline_param_idx],
268 flowey_core::pipeline::internal::Parameter::Bool { .. }
269 );
270
271 // ADO resolves bools as `True` and `False`, _sigh_
272 let with_lowercase = if is_bool {
273 r#" | tr '[:upper:]' '[:lower:]'"#
274 } else {
275 ""
276 };
277
278 let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, is_string);
279
280 let cmd = format!(
281 r#"
282cat <<'EOF'{with_lowercase} | {var_db_inject_cmd}
283${{{{ parameters.param{pipeline_param_idx} }}}}
284EOF
285"#
286 )
287 .trim()
288 .to_string();
289 writeln!(flowey_bootstrap_bash, "{}", cmd)?;
290 }
291
292 // next, emit ado steps to create dirs for artifacts which will be
293 // published
294 for ResolvedJobArtifact { flowey_var, name } in artifacts_published {
295 // do NOT use ADO macro syntax $(...), since this is in the same
296 // bootstrap block as where those ADO vars get defined, meaning it's
297 // not available yet!
298 writeln!(
299 flowey_bootstrap_bash,
300 r#"mkdir -p "$AgentTempDirNormal/publish_artifacts/{name}""#
301 )?;
302 let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, true);
303 writeln!(
304 flowey_bootstrap_bash,
305 r#"echo "$(FLOWEY_TEMP_DIR)/publish_artifacts/{name}" | {var_db_inject_cmd}"#,
306 )?;
307 }
308
309 // lastly, emit ado steps that report the dirs for any artifacts which
310 // are used by this job
311 for ResolvedJobArtifact { flowey_var, name } in artifacts_used {
312 // do NOT use ADO macro syntax $(...), since this is in the same
313 // bootstrap block as where those ADO vars get defined, meaning it's
314 // not available yet!
315 let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, true);
316 writeln!(
317 flowey_bootstrap_bash,
318 r#"echo "$(FLOWEY_TEMP_DIR)/used_artifacts/{name}" | {var_db_inject_cmd}"#,
319 )?;
320 }
321
322 ado_steps.push({
323 let mut map = serde_yaml::Mapping::new();
324 map.insert(
325 "bash".into(),
326 serde_yaml::Value::String(flowey_bootstrap_bash),
327 );
328 map.insert("displayName".into(), "🌼🛫 Initialize job".into());
329 map.into()
330 });
331
332 // if this was a bootstrap job, also take a moment to run a "self check"
333 // to make sure that the current checked-in template matches the one it
334 // expected
335 if let FloweySource::Bootstrap(..) = &flowey_source {
336 let mut current_invocation = std::env::args().collect::<Vec<_>>();
337
338 current_invocation[0] = "$(FLOWEY_BIN)".into();
339
340 // if this code path is run while generating the YAML to compare the
341 // check against, we want to remove the --check param from the
342 // current call, or else there'll be a dupe
343 if let Some(i) = current_invocation
344 .iter()
345 .position(|s| s.starts_with("--check"))
346 {
347 // remove the --check param
348 let s = current_invocation.remove(i);
349 if !s.starts_with("--check=") {
350 // remove its freestanding argument
351 current_invocation.remove(i);
352 }
353 }
354
355 // insert the --check bit of the call alongside the --out param
356 {
357 let i = current_invocation
358 .iter()
359 .position(|s| s.starts_with("--out"))
360 .unwrap();
361
362 let current_yaml = match platform.kind() {
363 FlowPlatformKind::Windows => {
364 r#"$ESCAPED_AGENT_TEMPDIR\\bootstrapped-flowey\\pipeline.yaml"#
365 }
366 FlowPlatformKind::Unix => {
367 r#"$ESCAPED_AGENT_TEMPDIR/bootstrapped-flowey/pipeline.yaml"#
368 }
369 };
370
371 current_invocation.insert(i, current_yaml.into());
372 current_invocation.insert(i, "--check".into());
373 }
374
375 // Need to use an escaped version of the "true" windows/linux path
376 // here, or else the --check will fail.
377 let cmd = format!(
378 r###"
379ESCAPED_AGENT_TEMPDIR=$(
380cat <<'EOF' | sed 's/\\/\\\\/g'
381$(FLOWEY_TEMP_DIR)
382EOF
383)
384{}
385"###,
386 current_invocation.join(" ")
387 );
388
389 ado_steps.push({
390 let mut map = serde_yaml::Mapping::new();
391 map.insert(
392 "bash".into(),
393 serde_yaml::Value::String(cmd.trim().to_string()),
394 );
395 map.insert("displayName".into(), "🌼🔎 Self-check YAML".into());
396 map.into()
397 })
398 }
399
400 // now that we've done all the job-level bootstrapping, we can emit all
401 // the actual steps the user cares about
402 ado_steps.extend(steps);
403
404 // ..and once that's done, the last order of business is to emit some
405 // ado steps to publish the various artifacts created by this job
406 for ResolvedJobArtifact {
407 flowey_var: _,
408 name,
409 } in artifacts_published
410 {
411 ado_steps.push({
412 let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
413 r#"
414 publish: $(FLOWEY_TEMP_DIR)/publish_artifacts/{name}
415 displayName: '🌼📦 Publish {name}'
416 artifact: {name}
417 "#
418 ))
419 .unwrap();
420 map.into()
421 });
422 }
423
424 // also, if this job also bootstrapped flowey that other nodes depend
425 // on, make sure to publish it!
426 if let FloweySource::Bootstrap(artifact, true) = flowey_source {
427 // don't leak the bootstrap job's runtime var db
428 ado_steps.push({
429 let mut map = serde_yaml::Mapping::new();
430 map.insert(
431 "bash".into(),
432 serde_yaml::Value::String(format!(
433 "rm $(AgentTempDirNormal)/bootstrapped-flowey/job{}.json",
434 job_idx.index()
435 )),
436 );
437 map.insert("displayName".into(), "🌼🧼 Redact bootstrap var db".into());
438 map.into()
439 });
440
441 ado_steps.push({
442 let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
443 r#"
444 publish: $(FLOWEY_TEMP_DIR)/bootstrapped-flowey
445 displayName: '🌼🥾 Publish bootstrapped flowey'
446 artifact: {artifact}
447 "#
448 ))
449 .unwrap();
450 map.into()
451 });
452 }
453
454 // ADO has this "helpful" default behavior where if you don't explicitly
455 // include a checkout step, it'll just auto-checkout the current repo.
456 //
457 // Work around this nonsense by doing a pre-pass over the emitted
458 // steps to enumerate how many steps start with `- checkout:`, and
459 // if the number is zero, emit an explicit `- checkout: none`.
460 {
461 let mut found = false;
462 for val in &ado_steps {
463 if let Some((key, _val)) = val.as_mapping().unwrap().iter().next() {
464 let Some(key) = key.as_str() else { continue };
465 if key == "checkout" {
466 found = true;
467 break;
468 }
469 }
470 }
471 if !found {
472 ado_steps.insert(0, {
473 let map: serde_yaml::Mapping = serde_yaml::from_str("checkout: none").unwrap();
474 map.into()
475 });
476 }
477 }
478
479 // Convert the pool information to the structured format
480 let AdoPool {
481 name: pool_name,
482 demands,
483 } = ado_pool
484 .clone()
485 .context(format!("must specify ADO pool for job '{label}'"))?;
486 let pool = if demands.is_empty() {
487 schema_ado_yaml::Pool::Pool(pool_name)
488 } else {
489 schema_ado_yaml::Pool::PoolWithMetadata(
490 [
491 ("name".into(), pool_name.into()),
492 ("demands".into(), demands.into()),
493 ]
494 .into(),
495 )
496 };
497
498 ado_jobs.push(schema_ado_yaml::Job {
499 job: format!("job{}", job_idx.index()),
500 display_name: label.clone(),
501 pool,
502 depends_on: {
503 graph
504 .edges_directed(job_idx, petgraph::Direction::Incoming)
505 .map(|e| {
506 use petgraph::prelude::*;
507 format!("job{}", e.source().index())
508 })
509 .collect()
510 },
511 variables: {
512 let mut ado_variables: Vec<schema_ado_yaml::Variable> = ado_variables.clone().into_iter()
513 .map(|(name, value)| schema_ado_yaml::Variable { name, value })
514 .collect();
515
516 ado_variables.push(schema_ado_yaml::Variable {name: "FLOWEY_TEMP_DIR".into(), value: FLOWEY_TEMP_DIR.into()});
517
518 Some(
519 ado_variables
520 )
521 },
522 steps: ado_steps,
523 condition: Some(if let Some(cond_param_idx) = cond_param_idx {
524 format!("and(eq('${{{{ parameters.param{cond_param_idx} }}}}', 'true'), succeeded(), not(canceled()))")
525 } else {
526 "and(succeeded(), not(canceled()))".into()
527 }),
528 })
529 }
530
531 let ado_pipeline = schema_ado_yaml::Pipeline {
532 name: ado_name,
533 trigger: Some(match ado_ci_triggers {
534 None => schema_ado_yaml::CiTrigger::None(()),
535 Some(t) => {
536 let flowey_core::pipeline::AdoCiTriggers {
537 branches,
538 exclude_branches,
539 batch,
540 } = t;
541
542 schema_ado_yaml::CiTrigger::Some {
543 batch,
544 branches: schema_ado_yaml::TriggerBranches {
545 include: branches,
546 exclude: if exclude_branches.is_empty() {
547 None
548 } else {
549 Some(exclude_branches)
550 },
551 },
552 }
553 }
554 }),
555 pr: Some(match ado_pr_triggers {
556 None => schema_ado_yaml::PrTrigger::None(()),
557 Some(t) => {
558 let flowey_core::pipeline::AdoPrTriggers {
559 branches,
560 exclude_branches,
561 run_on_draft,
562 auto_cancel,
563 } = t;
564
565 schema_ado_yaml::PrTrigger::Some {
566 auto_cancel,
567 drafts: run_on_draft,
568 branches: schema_ado_yaml::TriggerBranches {
569 include: branches,
570 exclude: if exclude_branches.is_empty() {
571 None
572 } else {
573 Some(exclude_branches)
574 },
575 },
576 }
577 }
578 }),
579 schedules: if ado_schedule_triggers.is_empty() {
580 None
581 } else {
582 Some(
583 ado_schedule_triggers
584 .into_iter()
585 .map(|t| {
586 let flowey_core::pipeline::AdoScheduleTriggers {
587 display_name,
588 branches,
589 exclude_branches,
590 cron,
591 } = t;
592
593 schema_ado_yaml::Schedule {
594 cron,
595 display_name,
596 branches: schema_ado_yaml::TriggerBranches {
597 include: branches,
598 exclude: if exclude_branches.is_empty() {
599 None
600 } else {
601 Some(exclude_branches)
602 },
603 },
604 batch: false,
605 }
606 })
607 .collect(),
608 )
609 },
610 variables: if !ado_variables.is_empty() {
611 Some(
612 ado_variables
613 .into_iter()
614 .map(|(name, value)| schema_ado_yaml::Variable { name, value })
615 .collect(),
616 )
617 } else {
618 None
619 },
620 stages: None,
621 jobs: Some(ado_jobs),
622 parameters: if !parameters.is_empty() {
623 Some(
624 parameters
625 .into_iter()
626 .enumerate()
627 .map(|(idx, param)| match param {
628 flowey_core::pipeline::internal::Parameter::Bool {
629 description,
630 default,
631 } => schema_ado_yaml::Parameter {
632 name: format!("param{idx}"),
633 display_name: description,
634 ty: schema_ado_yaml::ParameterType::Boolean { default },
635 },
636 flowey_core::pipeline::internal::Parameter::String {
637 description,
638 default,
639 possible_values,
640 } => schema_ado_yaml::Parameter {
641 name: format!("param{idx}"),
642 display_name: description,
643 ty: schema_ado_yaml::ParameterType::String {
644 default,
645 values: possible_values,
646 },
647 },
648 flowey_core::pipeline::internal::Parameter::Num {
649 description,
650 default,
651 possible_values,
652 } => schema_ado_yaml::Parameter {
653 name: format!("param{idx}"),
654 display_name: description,
655 ty: schema_ado_yaml::ParameterType::Number {
656 default,
657 values: possible_values,
658 },
659 },
660 })
661 .collect(),
662 )
663 } else {
664 None
665 },
666 resources: {
667 if ado_resources_repository.is_empty() {
668 None
669 } else {
670 Some(schema_ado_yaml::Resources {
671 repositories: ado_resources_repository
672 .into_iter()
673 .map(
674 |InternalAdoResourcesRepository {
675 repo_id,
676 repo_type,
677 name,
678 git_ref,
679 endpoint,
680 }| {
681 use flowey_core::pipeline::AdoResourcesRepositoryRef;
682 use flowey_core::pipeline::AdoResourcesRepositoryType;
683
684 schema_ado_yaml::ResourcesRepository {
685 repository: repo_id,
686 endpoint,
687 name,
688 r#ref: match git_ref {
689 AdoResourcesRepositoryRef::Fixed(s) => s,
690 AdoResourcesRepositoryRef::Parameter(idx) => {
691 format!("${{{{ parameters.param{idx} }}}}")
692 }
693 },
694 r#type: match repo_type {
695 AdoResourcesRepositoryType::AzureReposGit => {
696 schema_ado_yaml::ResourcesRepositoryType::Git
697 }
698 AdoResourcesRepositoryType::GitHub => {
699 schema_ado_yaml::ResourcesRepositoryType::GitHub
700 }
701 },
702 }
703 },
704 )
705 .collect::<Vec<_>>(),
706 })
707 }
708 },
709 extends: None,
710 };
711
712 if let Some(check) = check {
713 check_generated_yaml_and_json(
714 &ado_pipeline,
715 &pipeline_static_db,
716 check,
717 repo_root,
718 pipeline_file,
719 ado_post_process_yaml_cb,
720 )
721 } else {
722 write_generated_yaml_and_json(
723 &ado_pipeline,
724 &pipeline_static_db,
725 repo_root,
726 pipeline_file,
727 ado_post_process_yaml_cb,
728 )
729 }
730}
731
732/// Resolve a flow as a sequence of ADO YAML steps.
733///
734/// These steps can then be marshalled into a well-formed ADO pipeline yaml
735/// using a separate ADO pipeline yaml builder
736// pub(crate) so that internal debug CLI tooling can use it
737pub(crate) fn resolve_flow_as_ado_yaml_steps(
738 seed_nodes: BTreeMap<NodeHandle, (bool, Vec<Box<[u8]>>)>,
739 resolved_patches: flowey_core::patch::ResolvedPatches,
740 external_read_vars: BTreeSet<String>,
741 platform: FlowPlatform,
742 arch: FlowArch,
743 job_idx: usize,
744) -> anyhow::Result<(
745 Vec<serde_yaml::Value>,
746 BTreeMap<String, Vec<crate::cli::exec_snippet::SerializedRequest>>,
747)> {
748 let mut output_steps = Vec::new();
749
750 let (mut output_graph, request_db, err_unreachable_nodes) =
751 crate::flow_resolver::stage1_dag::stage1_dag(
752 FlowBackend::Ado,
753 platform,
754 arch,
755 resolved_patches,
756 seed_nodes,
757 external_read_vars,
758 // TODO: support ADO agents with persistent storage
759 None,
760 )?;
761
762 if err_unreachable_nodes.is_some() {
763 anyhow::bail!("detected unreachable nodes")
764 }
765
766 let output_order = petgraph::algo::toposort(&output_graph, None)
767 .expect("runtime variables cannot introduce a DAG cycle");
768
769 let mut rust_step_bash_buffer = Vec::new();
770 for idx in output_order.into_iter().rev() {
771 let OutputGraphEntry { node_handle, step } = output_graph[idx].1.take().unwrap();
772
773 let node_modpath = node_handle.modpath();
774
775 match step {
776 Step::Anchor { .. } => {}
777 Step::Rust {
778 idx,
779 label: _,
780 code: _,
781 } => {
782 rust_step_bash_buffer.push(crate::cli::exec_snippet::StepIdx {
783 node_modpath,
784 snippet_idx: idx,
785 });
786 }
787 Step::AdoYaml {
788 label,
789 raw_yaml,
790 ado_to_rust,
791 rust_to_ado,
792 condvar,
793 code_idx,
794 code,
795 } => {
796 if !rust_step_bash_buffer.is_empty() {
797 let rust_step_bash_buffer = std::mem::take(&mut rust_step_bash_buffer);
798 let mut map = serde_yaml::Mapping::new();
799 map.insert(
800 "bash".into(),
801 serde_yaml::Value::String(
802 crate::cli::exec_snippet::construct_exec_snippet_cli_multi(
803 "$(FLOWEY_BIN)",
804 job_idx,
805 rust_step_bash_buffer,
806 ),
807 ),
808 );
809 map.insert("displayName".into(), "🦀 flowey rust steps".into());
810 output_steps.push(map.into());
811 }
812
813 let var_db_cmd = |var: &str, is_secret, update_from_stdin, is_raw_string| {
814 crate::cli::var_db::construct_var_db_cli(
815 "$(FLOWEY_BIN)",
816 job_idx,
817 var,
818 is_secret,
819 update_from_stdin,
820 None,
821 is_raw_string,
822 None,
823 )
824 };
825
826 if let Some(condvar) = &condvar {
827 let mut cmd = String::new();
828
829 // guaranteed to be a bare bool `true`/`false`, hence
830 // is_raw_string = false
831 let read_condvar = var_db_cmd(condvar, false, false, false);
832 writeln!(cmd, r#"rust_var=$({read_condvar})"#)?;
833 writeln!(
834 cmd,
835 r###"echo "##vso[task.setvariable variable=FLOWEY_CONDITION;issecret=false]$rust_var""###
836 )?;
837
838 let mut map = serde_yaml::Mapping::new();
839 map.insert("bash".into(), serde_yaml::Value::String(cmd));
840 map.insert(
841 "displayName".into(),
842 serde_yaml::Value::String("🌼❓ Write to 'FLOWEY_CONDITION'".into()),
843 );
844 output_steps.push(map.into());
845 }
846
847 for (rust_var, ado_var, is_secret) in rust_to_ado {
848 let mut cmd = String::new();
849
850 // flowey considers all ADO vars to be typed as raw strings
851 let read_rust_var = var_db_cmd(&rust_var, is_secret, false, true);
852 writeln!(cmd, r#"rust_var=$({read_rust_var})"#)?;
853 writeln!(
854 cmd,
855 r###"printf "##vso[task.setvariable variable={ado_var};issecret={is_secret}]%s\n" "$rust_var""###
856 )?;
857
858 let mut map = serde_yaml::Mapping::new();
859 map.insert("bash".into(), serde_yaml::Value::String(cmd));
860 map.insert(
861 "displayName".into(),
862 serde_yaml::Value::String(format!("🌼 Write to '{ado_var}'")),
863 );
864
865 if condvar.is_some() {
866 map.insert(
867 "condition".into(),
868 "and(eq(variables['FLOWEY_CONDITION'], true), succeeded(), not(canceled()))".into(),
869 );
870 }
871 output_steps.push(map.into());
872 }
873
874 if !raw_yaml.is_empty() {
875 let raw_yaml = if code.lock().is_some() {
876 let inline_snippet = crate::cli::exec_snippet::construct_exec_snippet_cli(
877 "$(FLOWEY_BIN)",
878 node_modpath,
879 code_idx,
880 job_idx,
881 );
882 let post_process =
883 raw_yaml.replace("{{FLOWEY_INLINE_SCRIPT}}", &inline_snippet);
884 if raw_yaml == post_process {
885 return Err(anyhow::anyhow!("if using inlins-enippet, YAML must include {{{{FLOWEY_INLINE_SCRIPT}}}}").context(format!(
886 "invalid yaml in node {node_modpath}: {raw_yaml}"
887 )));
888 }
889 post_process
890 } else {
891 raw_yaml
892 };
893
894 let step: serde_yaml::Value = serde_yaml::from_str(&raw_yaml)
895 .context(format!("invalid yaml in node {node_modpath}: {raw_yaml}"))?;
896 let step = {
897 let mut step = step;
898 let seq = step
899 .as_sequence_mut()
900 .context("yaml snippet did not parse as a sequence")?;
901
902 if seq.len() != 1 {
903 anyhow::bail!("yaml snippet contained more than one sequence element")
904 }
905
906 let map = seq
907 .first_mut()
908 .unwrap()
909 .as_mapping_mut()
910 .context("yaml snippet did not parse as a map")?;
911 let existing = map.insert("displayName".into(), label.into());
912 if existing.is_some() {
913 anyhow::bail!("yaml snippet included `displayName`")
914 }
915 if condvar.is_some() {
916 let existing = map.insert(
917 "condition".into(),
918 "and(eq(variables['FLOWEY_CONDITION'], true), succeeded(), not(canceled()))".into(),
919 );
920 if existing.is_some() {
921 anyhow::bail!("yaml snippet included `condition`")
922 }
923 }
924
925 step
926 };
927 output_steps.push(step.as_sequence().unwrap().first().unwrap().clone());
928 }
929
930 for (ado_var, rust_var, is_secret) in ado_to_rust {
931 // flowey considers all ADO vars to be typed as raw strings
932 let write_rust_var = var_db_cmd(&rust_var, is_secret, true, true);
933
934 let cmd = format!(
935 r#"
936{write_rust_var} <<'EOF'
937$({ado_var})
938EOF
939"#
940 )
941 .trim()
942 .to_string();
943
944 let mut map = serde_yaml::Mapping::new();
945 map.insert("bash".into(), serde_yaml::Value::String(cmd));
946 map.insert(
947 "displayName".into(),
948 serde_yaml::Value::String(format!("🌼 Read from '{ado_var}'")),
949 );
950 if condvar.is_some() {
951 map.insert(
952 "condition".into(),
953 "and(eq(variables['FLOWEY_CONDITION'], true), succeeded(), not(canceled()))".into(),
954 );
955 }
956 output_steps.push(map.into());
957 }
958 }
959 Step::GitHubYaml { .. } => anyhow::bail!("GitHub YAML not supported in ADO"),
960 }
961 }
962
963 if !rust_step_bash_buffer.is_empty() {
964 let rust_step_bash_buffer = std::mem::take(&mut rust_step_bash_buffer);
965 let mut map = serde_yaml::Mapping::new();
966 map.insert(
967 "bash".into(),
968 serde_yaml::Value::String(crate::cli::exec_snippet::construct_exec_snippet_cli_multi(
969 "$(FLOWEY_BIN)",
970 job_idx,
971 rust_step_bash_buffer,
972 )),
973 );
974 map.insert("displayName".into(), "🦀 flowey rust steps".into());
975 output_steps.push(map.into());
976 }
977
978 let request_db = request_db
979 .into_iter()
980 .map(|(node_handle, reqs)| {
981 (
982 node_handle.modpath().to_owned(),
983 reqs.into_iter()
984 .map(crate::cli::exec_snippet::SerializedRequest)
985 .collect(),
986 )
987 })
988 .collect();
989
990 Ok((output_steps, request_db))
991}
992