GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/openvmm

Public

mirrored fromhttps://github.com/microsoft/openvmmAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
release/2411-fork

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openvmm/flowey/flowey_cli/src/pipeline_resolver/github_yaml

flowey/flowey_cli/src/pipeline_resolver/github_yaml/mod.rs

903lines · modecode

RawDownload
Latest commit unavailable.
1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4//! Code for emitting a pipeline as a single self-contained GitHub Actions yaml file
5
6use super::common_yaml::check_generated_yaml_and_json;
7use super::common_yaml::write_generated_yaml_and_json;
8use super::generic::ResolvedJobArtifact;
9use super::generic::ResolvedJobUseParameter;
10use crate::cli::exec_snippet::FloweyPipelineStaticDb;
11use crate::cli::exec_snippet::VAR_DB_SEEDVAR_FLOWEY_WORKING_DIR;
12use crate::flow_resolver::stage1_dag::OutputGraphEntry;
13use crate::flow_resolver::stage1_dag::Step;
14use crate::pipeline_resolver::common_yaml::job_flowey_bootstrap_source;
15use crate::pipeline_resolver::common_yaml::FloweySource;
16use crate::pipeline_resolver::generic::ResolvedPipeline;
17use crate::pipeline_resolver::generic::ResolvedPipelineJob;
18use anyhow::Context;
19use flowey_core::node::user_facing::GhPermission;
20use flowey_core::node::user_facing::GhPermissionValue;
21use flowey_core::node::FlowArch;
22use flowey_core::node::FlowBackend;
23use flowey_core::node::FlowPlatform;
24use flowey_core::node::FlowPlatformKind;
25use flowey_core::node::NodeHandle;
26use flowey_core::pipeline::GhRunner;
27use flowey_core::pipeline::GhRunnerOsLabel;
28use std::collections::BTreeMap;
29use std::collections::BTreeSet;
30use std::fmt::Write;
31use std::path::Path;
32use std::path::PathBuf;
33mod github_yaml_defs;
34
35const RUNNER_TEMP: &str = "${{ runner.temp }}";
36
37/// Emit a pipeline as a single self-contained GitHub Actions yaml file
38pub fn github_yaml(
39 pipeline: ResolvedPipeline,
40 runtime_debug_log: bool,
41 repo_root: &Path,
42 pipeline_file: &Path,
43 flowey_crate: &str,
44 check: Option<PathBuf>,
45) -> anyhow::Result<()> {
46 if pipeline_file.extension().and_then(|s| s.to_str()) != Some("yaml") {
47 anyhow::bail!("pipeline name must end with .yaml")
48 }
49
50 let ResolvedPipeline {
51 graph,
52 order,
53 gh_name,
54 gh_schedule_triggers,
55 gh_ci_triggers,
56 gh_pr_triggers,
57 gh_bootstrap_template,
58 parameters,
59 ado_name: _,
60 ado_schedule_triggers: _,
61 ado_ci_triggers: _,
62 ado_pr_triggers: _,
63 ado_bootstrap_template: _,
64 ado_resources_repository: _,
65 ado_post_process_yaml_cb: _,
66 ado_variables: _,
67 } = pipeline;
68
69 let mut job_flowey_source: BTreeMap<petgraph::prelude::NodeIndex, FloweySource> =
70 job_flowey_bootstrap_source(&graph, &order);
71
72 let mut pipeline_static_db = FloweyPipelineStaticDb {
73 flow_backend: crate::cli::FlowBackendCli::Github,
74 var_db_backend_kind: crate::cli::exec_snippet::VarDbBackendKind::Json,
75 job_reqs: BTreeMap::new(),
76 };
77
78 let mut github_jobs = BTreeMap::new();
79
80 for job_idx in order {
81 let ResolvedPipelineJob {
82 ref root_nodes,
83 ref patches,
84 ref label,
85 platform,
86 arch,
87 ref external_read_vars,
88 ado_pool: _,
89 ref gh_override_if,
90 ref gh_global_env,
91 ref gh_pool,
92 ref gh_permissions,
93 cond_param_idx: _,
94 ref parameters_used,
95 ref artifacts_used,
96 ref artifacts_published,
97 ado_variables: _,
98 } = graph[job_idx];
99
100 let flowey_bin = platform.binary("flowey");
101 let (steps, req_db) = resolve_flow_as_github_yaml_steps(
102 root_nodes
103 .clone()
104 .into_iter()
105 .map(|(node, requests)| (node, (true, requests)))
106 .collect(),
107 patches.clone(),
108 external_read_vars.clone(),
109 platform,
110 arch,
111 job_idx.index(),
112 &flowey_bin,
113 gh_permissions,
114 )
115 .context(format!("in job '{label}'"))?;
116
117 {
118 let existing = pipeline_static_db.job_reqs.insert(job_idx.index(), req_db);
119 assert!(existing.is_none())
120 }
121
122 let mut gh_steps = Vec::new();
123
124 let flowey_source = job_flowey_source.remove(&job_idx).unwrap();
125
126 // actual artifact publish happens at the end of the job
127 if let FloweySource::Bootstrap(_artifact, _publish) = &flowey_source {
128 if gh_bootstrap_template.is_empty() {
129 anyhow::bail!("Did not specify flowey bootstrap template. Please provide one using `Pipeline::gh_set_flowey_bootstrap_template`")
130 }
131
132 let gh_bootstrap_template = gh_bootstrap_template
133 .replace("{{FLOWEY_BIN_EXTENSION}}", platform.exe_suffix())
134 .replace("{{FLOWEY_CRATE}}", flowey_crate)
135 .replace(
136 "{{FLOWEY_PIPELINE_PATH}}",
137 &pipeline_file.with_extension("").display().to_string(),
138 )
139 .replace(
140 "{{FLOWEY_TARGET}}",
141 match (platform, arch) {
142 (FlowPlatform::Windows, FlowArch::X86_64) => "x86_64-pc-windows-msvc",
143 (FlowPlatform::Windows, FlowArch::Aarch64) => "aarch64-pc-windows-msvc",
144 (FlowPlatform::Linux(_), FlowArch::X86_64) => "x86_64-unknown-linux-gnu",
145 (FlowPlatform::Linux(_), FlowArch::Aarch64) => "aarch64-unknown-linux-gnu",
146 (platform, arch) => {
147 anyhow::bail!("unsupported platform {platform} / arch {arch}")
148 }
149 },
150 )
151 .replace(
152 "{{FLOWEY_OUTDIR}}",
153 &format!("{RUNNER_TEMP}/bootstrapped-flowey"),
154 );
155
156 let bootstrap_steps: serde_yaml::Sequence =
157 serde_yaml::from_str(&gh_bootstrap_template)
158 .context("malformed flowey bootstrap template")?;
159
160 gh_steps.push({
161 let mut map = serde_yaml::Mapping::new();
162 map.insert("run".into(), "echo \"injected!\"".into());
163 map.insert("name".into(), "🌼🥾 Bootstrap flowey".into());
164 map.insert("shell".into(), "bash".into());
165 map.into()
166 });
167 gh_steps.extend(bootstrap_steps);
168 }
169
170 // the first few steps in any job are some "artisan" code, which
171 // downloads the previously bootstrapped flowey artifact and set up
172 // various vars that flowey will then rely on throughout the rest
173 // of the job
174
175 // download previously bootstrapped flowey
176 if let FloweySource::Consume(artifact) = &flowey_source {
177 gh_steps.push({
178 let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
179 r#"
180 name: 🌼🥾 Download bootstrapped flowey
181 uses: actions/download-artifact@v4
182 with:
183 name: {artifact}
184 path: {RUNNER_TEMP}/bootstrapped-flowey
185 "#
186 ))?;
187 map.into()
188 });
189 }
190
191 // also download any artifacts that'll be used
192 // TODO: Use a single download job to download all artifacts at once
193 // https://github.com/actions/download-artifact?tab=readme-ov-file#download-multiple-filtered-artifacts-to-the-same-directory
194 for ResolvedJobArtifact {
195 flowey_var: _,
196 name,
197 } in artifacts_used
198 {
199 gh_steps.push({
200 let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
201 r#"
202 name: '🌼📦 Download {name}'
203 uses: actions/download-artifact@v4
204 with:
205 name: {name}
206 path: {RUNNER_TEMP}/used_artifacts/{name}/
207 "#
208 ))
209 .unwrap();
210 map.into()
211 });
212 }
213
214 let mut flowey_bootstrap_bash = String::new();
215
216 {
217 let mut map = serde_yaml::Mapping::new();
218 map.insert(
219 "run".into(),
220 format!(r#"echo "{RUNNER_TEMP}/bootstrapped-flowey" >> $GITHUB_PATH"#).into(),
221 );
222 map.insert("shell".into(), "bash".into());
223 map.insert("name".into(), "🌼📦 Add flowey to PATH".into());
224 gh_steps.push(map.into());
225 }
226
227 let bootstrap_bash_var_db_inject = |var, is_raw_string| {
228 crate::cli::var_db::construct_var_db_cli(
229 &flowey_bin,
230 job_idx.index(),
231 var,
232 false,
233 true,
234 None,
235 is_raw_string,
236 None,
237 )
238 };
239
240 // and now use those vars to do some flowey bootstrap
241 writeln!(flowey_bootstrap_bash, "{}", {
242 let runtime_debug_level = if runtime_debug_log { "debug" } else { "info" };
243
244 let var_db_insert_runtime_debug_level =
245 bootstrap_bash_var_db_inject("FLOWEY_LOG", false);
246 let var_db_insert_working_dir =
247 bootstrap_bash_var_db_inject(VAR_DB_SEEDVAR_FLOWEY_WORKING_DIR, true);
248
249 // Need to use "normalized" path in cases where the path is being
250 // used directly from a bash context, as is the case when we are
251 // trying to invoke `flowey.exe` in argv0 position)
252 //
253 // https://github.com/microsoft/azure-pipelines-tasks/issues/10653#issuecomment-585669089
254 format!(
255 r###"
256AgentTempDirNormal="{RUNNER_TEMP}"
257AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|')
258echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV
259
260chmod +x $AgentTempDirNormal/bootstrapped-flowey/{flowey_bin}
261
262echo '"{runtime_debug_level}"' | {var_db_insert_runtime_debug_level}
263echo "{RUNNER_TEMP}/work" | {var_db_insert_working_dir}
264"###
265 )
266 .trim_start()
267 })?;
268
269 // import pipeline vars being used by the job into flowey
270 for ResolvedJobUseParameter {
271 flowey_var,
272 pipeline_param_idx,
273 } in parameters_used
274 {
275 let is_string = matches!(
276 parameters[*pipeline_param_idx],
277 flowey_core::pipeline::internal::Parameter::String { .. }
278 );
279
280 let default = match &parameters[*pipeline_param_idx] {
281 flowey_core::pipeline::internal::Parameter::Bool { default, .. } => {
282 default.map(|b| b.to_string())
283 }
284 flowey_core::pipeline::internal::Parameter::String { default, .. } => {
285 default.clone()
286 }
287 flowey_core::pipeline::internal::Parameter::Num { default, .. } => {
288 default.map(|n| n.to_string())
289 }
290 }
291 .expect("defaults are currently required for parameters in Github backend");
292
293 let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, is_string);
294
295 let cmd = format!(
296 r#"
297cat <<'EOF' | {var_db_inject_cmd}
298${{{{ inputs.param{pipeline_param_idx} != '' && inputs.param{pipeline_param_idx} || '{default}' }}}}
299EOF
300"#
301 )
302 .trim()
303 .to_string();
304 writeln!(flowey_bootstrap_bash, "{}", cmd)?;
305 }
306
307 // next, emit GitHub steps to create dirs for artifacts which will be
308 // published
309 for ResolvedJobArtifact { flowey_var, name } in artifacts_published {
310 writeln!(
311 flowey_bootstrap_bash,
312 r#"mkdir -p "$AgentTempDirNormal/publish_artifacts/{name}""#
313 )?;
314 let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, true);
315 match platform.kind() {
316 FlowPlatformKind::Windows => {
317 writeln!(
318 flowey_bootstrap_bash,
319 r#"echo "{RUNNER_TEMP}\\publish_artifacts\\{name}" | {var_db_inject_cmd}"#,
320 )?;
321 }
322 FlowPlatformKind::Unix => {
323 writeln!(
324 flowey_bootstrap_bash,
325 r#"echo "$AgentTempDirNormal/publish_artifacts/{name}" | {var_db_inject_cmd}"#,
326 )?;
327 }
328 }
329 }
330
331 // lastly, emit GitHub steps that report the dirs for any artifacts which
332 // are used by this job
333 for ResolvedJobArtifact { flowey_var, name } in artifacts_used {
334 let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, true);
335 match platform.kind() {
336 FlowPlatformKind::Windows => {
337 writeln!(
338 flowey_bootstrap_bash,
339 r#"echo "{RUNNER_TEMP}\\used_artifacts\\{name}" | {var_db_inject_cmd}"#,
340 )?;
341 }
342 FlowPlatformKind::Unix => {
343 writeln!(
344 flowey_bootstrap_bash,
345 r#"echo "$AgentTempDirNormal/used_artifacts/{name}" | {var_db_inject_cmd}"#,
346 )?;
347 }
348 }
349 }
350
351 gh_steps.push({
352 let mut map = serde_yaml::Mapping::new();
353 map.insert("name".into(), "🌼🛫 Initialize job".into());
354 map.insert(
355 "run".into(),
356 serde_yaml::Value::String(flowey_bootstrap_bash),
357 );
358 map.insert("shell".into(), "bash".into());
359 map.into()
360 });
361
362 // if this was a bootstrap job, also take a moment to run a "self check"
363 // to make sure that the current checked-in template matches the one it
364 // expected
365 if let FloweySource::Bootstrap(..) = &flowey_source {
366 let mut current_invocation = std::env::args().collect::<Vec<_>>();
367
368 current_invocation[0] = flowey_bin;
369
370 // if this code path is run while generating the YAML to compare the
371 // check against, we want to remove the --check param from the
372 // current call, or else there'll be a dupe
373 if let Some(i) = current_invocation
374 .iter()
375 .position(|s| s.starts_with("--check"))
376 {
377 // remove the --check param
378 let s = current_invocation.remove(i);
379 if !s.starts_with("--check=") {
380 // remove its freestanding argument
381 current_invocation.remove(i);
382 }
383 }
384
385 // insert the --check bit of the call alongside the --out param
386 {
387 let i = current_invocation
388 .iter()
389 .position(|s| s.starts_with("--out"))
390 .unwrap();
391
392 let current_yaml = match platform.kind() {
393 FlowPlatformKind::Windows => {
394 r#"$ESCAPED_AGENT_TEMPDIR\\bootstrapped-flowey\\pipeline.yaml"#
395 }
396 FlowPlatformKind::Unix => {
397 r#"$ESCAPED_AGENT_TEMPDIR/bootstrapped-flowey/pipeline.yaml"#
398 }
399 };
400
401 current_invocation.insert(i, current_yaml.into());
402 current_invocation.insert(i, "--check".into());
403 }
404
405 // Need to use an escaped version of the "true" windows/linux path
406 // here, or else the --check will fail.
407 let cmd = format!(
408 r###"
409ESCAPED_AGENT_TEMPDIR=$(
410cat <<'EOF' | sed 's/\\/\\\\/g'
411{RUNNER_TEMP}
412EOF
413)
414{}
415"###,
416 current_invocation.join(" ")
417 );
418
419 gh_steps.push({
420 let mut map = serde_yaml::Mapping::new();
421 map.insert("name".into(), "🌼🔎 Self-check YAML".into());
422 map.insert(
423 "run".into(),
424 serde_yaml::Value::String(cmd.trim().to_string()),
425 );
426 map.insert("shell".into(), "bash".into());
427 map.into()
428 })
429 }
430
431 // now that we've done all the job-level bootstrapping, we can emit all
432 // the actual steps the user cares about
433 gh_steps.extend(steps);
434
435 // ..and once that's done, the last order of business is to emit some
436 // GitHub steps to publish the various artifacts created by this job
437 for ResolvedJobArtifact {
438 flowey_var: _,
439 name,
440 } in artifacts_published
441 {
442 gh_steps.push({
443 let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
444 r#"
445 name: 🌼📦 Publish {name}
446 uses: actions/upload-artifact@v4
447 with:
448 name: {name}
449 path: {RUNNER_TEMP}/publish_artifacts/{name}/
450 "#
451 ))
452 .unwrap();
453 map.into()
454 });
455 }
456
457 // also, if this job also bootstrapped flowey that other nodes depend
458 // on, make sure to publish it!
459 if let FloweySource::Bootstrap(artifact, true) = flowey_source {
460 // don't leak the bootstrap job's runtime var db
461 gh_steps.push({
462 let mut map = serde_yaml::Mapping::new();
463 map.insert("name".into(), "🌼🧼 Redact bootstrap var db".into());
464 map.insert(
465 "run".into(),
466 serde_yaml::Value::String(format!(
467 "rm $AgentTempDirNormal/bootstrapped-flowey/job{}.json",
468 job_idx.index()
469 )),
470 );
471 map.insert("shell".into(), "bash".into());
472 map.into()
473 });
474
475 gh_steps.push({
476 let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
477 r#"
478 name: 🌼🥾 Publish bootstrapped flowey
479 uses: actions/upload-artifact@v4
480 with:
481 name: {artifact}
482 path: {RUNNER_TEMP}/bootstrapped-flowey
483 "#
484 ))
485 .unwrap();
486 map.into()
487 });
488 }
489
490 let runner_kind_to_yaml = |runner: &GhRunner| match runner {
491 GhRunner::GhHosted(s) => github_yaml_defs::Runner::GhHosted(match s {
492 GhRunnerOsLabel::UbuntuLatest => github_yaml_defs::RunnerOsLabel::UbuntuLatest,
493 GhRunnerOsLabel::Ubuntu2204 => github_yaml_defs::RunnerOsLabel::Ubuntu2204,
494 GhRunnerOsLabel::Ubuntu2004 => github_yaml_defs::RunnerOsLabel::Ubuntu2004,
495 GhRunnerOsLabel::WindowsLatest => github_yaml_defs::RunnerOsLabel::WindowsLatest,
496 GhRunnerOsLabel::Windows2022 => github_yaml_defs::RunnerOsLabel::Windows2022,
497 GhRunnerOsLabel::Windows2019 => github_yaml_defs::RunnerOsLabel::Windows2019,
498 GhRunnerOsLabel::MacOsLatest => github_yaml_defs::RunnerOsLabel::MacOsLatest,
499 GhRunnerOsLabel::MacOs14 => github_yaml_defs::RunnerOsLabel::MacOs14,
500 GhRunnerOsLabel::MacOs13 => github_yaml_defs::RunnerOsLabel::MacOs13,
501 GhRunnerOsLabel::MacOs12 => github_yaml_defs::RunnerOsLabel::MacOs12,
502 GhRunnerOsLabel::MacOs11 => github_yaml_defs::RunnerOsLabel::MacOs11,
503 GhRunnerOsLabel::Custom(s) => github_yaml_defs::RunnerOsLabel::Custom(s.into()),
504 }),
505 GhRunner::SelfHosted(v) => github_yaml_defs::Runner::SelfHosted(v.to_vec()),
506 GhRunner::RunnerGroup { group, labels } => github_yaml_defs::Runner::Group {
507 group: group.into(),
508 labels: labels.to_vec(),
509 },
510 };
511
512 let perm_val_to_yaml = |permission_value: &GhPermissionValue| match permission_value {
513 GhPermissionValue::Read => github_yaml_defs::PermissionValue::Read,
514 GhPermissionValue::Write => github_yaml_defs::PermissionValue::Write,
515 GhPermissionValue::None => github_yaml_defs::PermissionValue::None,
516 };
517
518 let perm_kind_to_yaml = |permission: &GhPermission| match permission {
519 GhPermission::Actions => github_yaml_defs::Permissions::Actions,
520 GhPermission::Attestations => github_yaml_defs::Permissions::Attestations,
521 GhPermission::Checks => github_yaml_defs::Permissions::Checks,
522 GhPermission::Contents => github_yaml_defs::Permissions::Contents,
523 GhPermission::Deployments => github_yaml_defs::Permissions::Deployments,
524 GhPermission::Discussions => github_yaml_defs::Permissions::Discussions,
525 GhPermission::IdToken => github_yaml_defs::Permissions::IdToken,
526 GhPermission::Issues => github_yaml_defs::Permissions::Issues,
527 GhPermission::Packages => github_yaml_defs::Permissions::Packages,
528 GhPermission::Pages => github_yaml_defs::Permissions::Pages,
529 GhPermission::PullRequests => github_yaml_defs::Permissions::PullRequests,
530 GhPermission::RepositoryProjects => github_yaml_defs::Permissions::RepositoryProjects,
531 GhPermission::SecurityEvents => github_yaml_defs::Permissions::SecurityEvents,
532 GhPermission::Statuses => github_yaml_defs::Permissions::Statuses,
533 };
534
535 let mut job_permissions = BTreeMap::new();
536 for permission_map in gh_permissions.values() {
537 for (permission, value) in permission_map {
538 if let Some(old_value) = job_permissions.insert(permission.clone(), value.clone()) {
539 if old_value != *value {
540 anyhow::bail!(
541 "permission {:?} was to conflicting values in job {:?}: {:?} and {:?}",
542 permission,
543 label,
544 old_value,
545 value
546 )
547 }
548 };
549 }
550 }
551
552 github_jobs.insert(
553 format!("job{}", job_idx.index()),
554 github_yaml_defs::Job {
555 name: label.clone(),
556 runs_on: gh_pool.clone().map(|runner| runner_kind_to_yaml(&runner)),
557 permissions: job_permissions
558 .iter()
559 .map(|k| (perm_kind_to_yaml(k.0), perm_val_to_yaml(k.1)))
560 .collect(),
561 needs: {
562 graph
563 .edges_directed(job_idx, petgraph::Direction::Incoming)
564 .map(|e| {
565 use petgraph::prelude::*;
566 format!("job{}", e.source().index())
567 })
568 .collect()
569 },
570 r#if: gh_override_if.clone(),
571 env: gh_global_env.clone(),
572 steps: gh_steps,
573 },
574 );
575 }
576
577 let mut concurrency = None;
578 let pipeline_trigger = github_yaml_defs::Triggers {
579 workflow_call: None,
580 workflow_dispatch: Some(github_yaml_defs::WorkflowDispatch {
581 inputs: github_yaml_defs::Inputs {
582 inputs: parameters
583 .into_iter()
584 .enumerate()
585 .map(|(idx, param)| {
586 (
587 format!("param{idx}"),
588 match param {
589 flowey_core::pipeline::internal::Parameter::Bool {
590 description,
591 default,
592 } => github_yaml_defs::Input {
593 description: Some(description.clone()),
594 default: default.map(github_yaml_defs::Default::Boolean),
595 required: default.is_none(),
596 ty: github_yaml_defs::InputType::Boolean,
597 },
598 flowey_core::pipeline::internal::Parameter::String {
599 description,
600 default,
601 possible_values: _,
602 } => github_yaml_defs::Input {
603 description: Some(description.clone()),
604 default: default
605 .as_ref()
606 .map(|s| github_yaml_defs::Default::String(s.to_string())),
607 required: default.is_none(),
608 ty: github_yaml_defs::InputType::String,
609 },
610 flowey_core::pipeline::internal::Parameter::Num {
611 description,
612 default,
613 possible_values: _,
614 } => github_yaml_defs::Input {
615 description: Some(description.clone()),
616 default: default.map(github_yaml_defs::Default::Number),
617 required: default.is_none(),
618 ty: github_yaml_defs::InputType::Number,
619 },
620 },
621 )
622 })
623 .collect::<BTreeMap<String, github_yaml_defs::Input>>(),
624 },
625 }),
626 pull_request: match gh_pr_triggers {
627 Some(gh_pr_triggers) => {
628 if gh_pr_triggers.auto_cancel {
629 concurrency = Some(github_yaml_defs::Concurrency {
630 // only cancel in-progress jobs or runs for the same branch
631 group: Some("${{ github.ref }}".to_string()),
632 cancel_in_progress: Some(true),
633 })
634 };
635 Some(github_yaml_defs::PrTrigger {
636 branches: gh_pr_triggers.branches.clone(),
637 branches_ignore: gh_pr_triggers.exclude_branches.clone(),
638 })
639 }
640 None => None,
641 },
642 push: match gh_ci_triggers {
643 Some(gh_ci_triggers) => Some(github_yaml_defs::CiTrigger {
644 branches: gh_ci_triggers.branches,
645 branches_ignore: gh_ci_triggers.exclude_branches,
646 tags: gh_ci_triggers.tags,
647 tags_ignore: gh_ci_triggers.exclude_tags,
648 }),
649 None => None,
650 },
651 schedule: gh_schedule_triggers
652 .iter()
653 .map(|s| github_yaml_defs::Cron {
654 cron: s.cron.clone(),
655 })
656 .collect(),
657 };
658
659 let github_pipeline = github_yaml_defs::Pipeline {
660 name: gh_name,
661 on: Some(pipeline_trigger),
662 concurrency,
663 jobs: Some(github_yaml_defs::Jobs { jobs: github_jobs }),
664 inputs: None,
665 };
666
667 if let Some(check) = check {
668 check_generated_yaml_and_json(
669 &github_pipeline,
670 &pipeline_static_db,
671 check,
672 repo_root,
673 pipeline_file,
674 None,
675 )
676 } else {
677 write_generated_yaml_and_json(
678 &github_pipeline,
679 &pipeline_static_db,
680 repo_root,
681 pipeline_file,
682 None,
683 )
684 }
685}
686
687/// Resolve a flow as a sequence of GitHub YAML steps.
688///
689/// These steps can then be marshalled into a well-formed GitHub pipeline yaml
690/// using a separate GitHub pipeline yaml builder
691// pub(crate) so that internal debug CLI tooling can use it
692fn resolve_flow_as_github_yaml_steps(
693 seed_nodes: BTreeMap<NodeHandle, (bool, Vec<Box<[u8]>>)>,
694 resolved_patches: flowey_core::patch::ResolvedPatches,
695 external_read_vars: BTreeSet<String>,
696 platform: FlowPlatform,
697 arch: FlowArch,
698 job_idx: usize,
699 flowey_bin: &str,
700 gh_permissions: &BTreeMap<NodeHandle, BTreeMap<GhPermission, GhPermissionValue>>,
701) -> anyhow::Result<(
702 Vec<serde_yaml::Value>,
703 BTreeMap<String, Vec<crate::cli::exec_snippet::SerializedRequest>>,
704)> {
705 let mut output_steps = Vec::new();
706
707 let (mut output_graph, request_db, err_unreachable_nodes) =
708 crate::flow_resolver::stage1_dag::stage1_dag(
709 FlowBackend::Github,
710 platform,
711 arch,
712 resolved_patches,
713 seed_nodes,
714 external_read_vars,
715 // TODO: support GitHub agents with persistent storage
716 None,
717 )?;
718
719 if err_unreachable_nodes.is_some() {
720 anyhow::bail!("detected unreachable nodes")
721 }
722
723 let output_order = petgraph::algo::toposort(&output_graph, None)
724 .expect("runtime variables cannot introduce a DAG cycle");
725
726 for node_idx in output_order.into_iter().rev() {
727 let OutputGraphEntry { node_handle, step } = output_graph[node_idx].1.take().unwrap();
728
729 let node_modpath = node_handle.modpath();
730
731 match step {
732 Step::Anchor { .. } => {}
733 Step::Rust {
734 idx,
735 label,
736 code: _,
737 } => {
738 let cmd = crate::cli::exec_snippet::construct_exec_snippet_cli(
739 flowey_bin,
740 node_modpath,
741 idx,
742 job_idx,
743 );
744
745 let mut map = serde_yaml::Mapping::new();
746 map.insert("name".into(), serde_yaml::Value::String(label));
747 map.insert("run".into(), serde_yaml::Value::String(cmd));
748 map.insert("shell".into(), "bash".into());
749 output_steps.push(map.into());
750 }
751 Step::AdoYaml {
752 ado_to_rust: _,
753 rust_to_ado: _,
754 label,
755 ..
756 } => anyhow::bail!("ADO YAML not supported in GitHub. In step '{}'", label),
757 Step::GitHubYaml {
758 gh_to_rust,
759 rust_to_gh,
760 label,
761 step_id,
762 uses,
763 with,
764 condvar,
765 permissions,
766 } => {
767 let var_db_cmd =
768 |var: &str, is_secret, update_from_file, is_raw_string, write_to_gh_env| {
769 crate::cli::var_db::construct_var_db_cli(
770 flowey_bin,
771 job_idx,
772 var,
773 is_secret,
774 false,
775 update_from_file,
776 is_raw_string,
777 write_to_gh_env,
778 )
779 };
780
781 for permission in permissions {
782 if let Some(permission_map) = gh_permissions.get(&node_handle) {
783 if let Some(permission_value) = permission_map.get(&permission.0) {
784 if *permission_value != permission.1 {
785 anyhow::bail!(
786 "permission mismatch for {:?}: expected {:?}, got {:?}",
787 permission.0,
788 permission.1,
789 permission_value
790 )
791 }
792 }
793 } else {
794 anyhow::bail!(
795 "permission missing for {:?}: expected {:?}",
796 permission.0,
797 permission.1
798 )
799 }
800 }
801
802 if let Some(condvar) = &condvar {
803 let mut cmd = String::new();
804
805 // guaranteed to be a bare bool `true`/`false`, hence
806 // is_raw_string = false
807 let set_condvar = var_db_cmd(
808 condvar,
809 false,
810 None,
811 false,
812 Some("FLOWEY_CONDITION".to_string()),
813 );
814 writeln!(cmd, r#"{set_condvar}"#)?;
815
816 let mut map = serde_yaml::Mapping::new();
817 map.insert("run".into(), serde_yaml::Value::String(cmd));
818 map.insert("shell".into(), "bash".into());
819 map.insert(
820 "name".into(),
821 serde_yaml::Value::String("🌼❓ Write to 'FLOWEY_CONDITION'".into()),
822 );
823 output_steps.push(map.into());
824 }
825
826 for (rust_var, gh_var, is_secret) in rust_to_gh {
827 let mut cmd = String::new();
828
829 // flowey considers all GitHub vars to be typed as raw strings
830 let set_gh_env_var =
831 var_db_cmd(&rust_var, is_secret, None, true, Some(gh_var.clone()));
832 writeln!(cmd, r#"{set_gh_env_var}"#)?;
833
834 let mut map = serde_yaml::Mapping::new();
835 map.insert("run".into(), serde_yaml::Value::String(cmd));
836 map.insert("shell".into(), "bash".into());
837 map.insert(
838 "name".into(),
839 serde_yaml::Value::String(format!("🌼 Write to '{gh_var}'")),
840 );
841
842 if condvar.is_some() {
843 map.insert("if".into(), "${{ fromJSON(env.FLOWEY_CONDITION) }}".into());
844 }
845 output_steps.push(map.into());
846 }
847
848 if !uses.is_empty() {
849 let mut map = serde_yaml::Mapping::new();
850 map.insert("id".into(), serde_yaml::Value::String(step_id.clone()));
851 map.insert("uses".into(), serde_yaml::Value::String(uses));
852 if !with.is_empty() {
853 let mut with_map = serde_yaml::Mapping::new();
854 for (k, v) in with {
855 with_map.insert(k.into(), v.into());
856 }
857 map.insert("with".into(), with_map.into());
858 }
859 map.insert("name".into(), label.into());
860 if condvar.is_some() {
861 map.insert("if".into(), "${{ fromJSON(env.FLOWEY_CONDITION) }}".into());
862 }
863
864 let step: serde_yaml::Value = map.into();
865 output_steps.push(step);
866 }
867
868 for (gh_var, rust_var, is_secret) in gh_to_rust {
869 // flowey considers all GitHub vars to be typed as raw strings
870 let write_rust_var = var_db_cmd(&rust_var, is_secret, Some("{0}"), true, None);
871 let cmd = format!(r#"${{{{ {gh_var} }}}}"#);
872
873 let mut map = serde_yaml::Mapping::new();
874 map.insert("run".into(), serde_yaml::Value::String(cmd));
875 map.insert("shell".into(), write_rust_var.into());
876 map.insert(
877 "name".into(),
878 serde_yaml::Value::String(format!("🌼 Read from '{gh_var}'")),
879 );
880 if condvar.is_some() {
881 map.insert("if".into(), "${{ fromJSON(env.FLOWEY_CONDITION) }}".into());
882 }
883
884 output_steps.push(map.into());
885 }
886 }
887 }
888 }
889
890 let request_db = request_db
891 .into_iter()
892 .map(|(node_handle, reqs)| {
893 (
894 node_handle.modpath().to_owned(),
895 reqs.into_iter()
896 .map(crate::cli::exec_snippet::SerializedRequest)
897 .collect(),
898 )
899 })
900 .collect();
901
902 Ok((output_steps, request_db))
903}