GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/openvmm

Public

mirrored from https://github.com/microsoft/openvmmAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
7d61702bf0cba28ca985f1a5ef27c4e1caec009c

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openvmm/flowey/flowey_cli/src/pipeline_resolver/github_yaml

flowey/flowey_cli/src/pipeline_resolver/github_yaml/mod.rs

861lines · modecode

RawDownload
Latest commit unavailable.
1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4//! Code for emitting a pipeline as a single self-contained GitHub Actions yaml file
5
6use super::common_yaml::BashCommands;
7use super::common_yaml::check_generated_yaml_and_json;
8use super::common_yaml::write_generated_yaml_and_json;
9use super::generic::ResolvedJobArtifact;
10use super::generic::ResolvedJobUseParameter;
11use crate::cli::exec_snippet::FloweyPipelineStaticDb;
12use crate::cli::exec_snippet::VAR_DB_SEEDVAR_FLOWEY_WORKING_DIR;
13use crate::cli::pipeline::CheckMode;
14use crate::cli::var_db::VarDbRequestBuilder;
15use crate::flow_resolver::stage1_dag::OutputGraphEntry;
16use crate::flow_resolver::stage1_dag::Step;
17use crate::pipeline_resolver::common_yaml::FloweySource;
18use crate::pipeline_resolver::common_yaml::job_flowey_bootstrap_source;
19use crate::pipeline_resolver::generic::ResolvedPipeline;
20use crate::pipeline_resolver::generic::ResolvedPipelineJob;
21use anyhow::Context;
22use flowey_core::node::FlowArch;
23use flowey_core::node::FlowBackend;
24use flowey_core::node::FlowPlatform;
25use flowey_core::node::FlowPlatformKind;
26use flowey_core::node::NodeHandle;
27use flowey_core::node::user_facing::GhPermission;
28use flowey_core::node::user_facing::GhPermissionValue;
29use flowey_core::pipeline::GhRunner;
30use flowey_core::pipeline::GhRunnerOsLabel;
31use std::collections::BTreeMap;
32use std::collections::BTreeSet;
33use std::fmt::Write;
34use std::path::Path;
35mod github_yaml_defs;
36
37const RUNNER_TEMP: &str = "${{ runner.temp }}";
38
39/// Emit a pipeline as a single self-contained GitHub Actions yaml file
40pub fn github_yaml(
41 pipeline: ResolvedPipeline,
42 runtime_debug_log: bool,
43 repo_root: &Path,
44 pipeline_file: &Path,
45 flowey_crate: &str,
46 check: CheckMode,
47) -> anyhow::Result<()> {
48 if pipeline_file.extension().and_then(|s| s.to_str()) != Some("yaml") {
49 anyhow::bail!("pipeline name must end with .yaml")
50 }
51
52 let ResolvedPipeline {
53 graph,
54 order,
55 gh_name,
56 gh_schedule_triggers,
57 gh_ci_triggers,
58 gh_pr_triggers,
59 gh_bootstrap_template,
60 parameters,
61 ado_name: _,
62 ado_schedule_triggers: _,
63 ado_ci_triggers: _,
64 ado_pr_triggers: _,
65 ado_bootstrap_template: _,
66 ado_resources_repository: _,
67 ado_post_process_yaml_cb: _,
68 ado_variables: _,
69 ado_job_id_overrides: _,
70 } = pipeline;
71
72 let mut job_flowey_source: BTreeMap<petgraph::prelude::NodeIndex, FloweySource> =
73 job_flowey_bootstrap_source(&graph, &order);
74
75 let mut pipeline_static_db = FloweyPipelineStaticDb {
76 flow_backend: crate::cli::FlowBackendCli::Github,
77 var_db_backend_kind: crate::cli::exec_snippet::VarDbBackendKind::Json,
78 job_reqs: BTreeMap::new(),
79 };
80
81 let mut github_jobs = BTreeMap::new();
82
83 for job_idx in order {
84 let ResolvedPipelineJob {
85 ref root_nodes,
86 ref patches,
87 ref label,
88 platform,
89 arch,
90 ref external_read_vars,
91 ado_pool: _,
92 ref gh_override_if,
93 ref gh_global_env,
94 ref gh_pool,
95 ref gh_permissions,
96 cond_param_idx: _,
97 ref parameters_used,
98 ref artifacts_used,
99 ref artifacts_published,
100 ado_variables: _,
101 } = graph[job_idx];
102
103 let flowey_bin = platform.binary("flowey");
104 let (steps, req_db) = resolve_flow_as_github_yaml_steps(
105 root_nodes
106 .clone()
107 .into_iter()
108 .map(|(node, requests)| (node, (true, requests)))
109 .collect(),
110 patches.clone(),
111 external_read_vars.clone(),
112 platform,
113 arch,
114 job_idx.index(),
115 &flowey_bin,
116 gh_permissions,
117 )
118 .context(format!("in job '{label}'"))?;
119
120 {
121 let existing = pipeline_static_db.job_reqs.insert(job_idx.index(), req_db);
122 assert!(existing.is_none())
123 }
124
125 let mut gh_steps = Vec::new();
126
127 let flowey_source = job_flowey_source.remove(&job_idx).unwrap();
128
129 let mut artifact_names = Vec::new();
130
131 let flowey_path = match &flowey_source {
132 FloweySource::Bootstrap { .. } => {
133 let flowey_path = "bootstrapped-flowey".to_string();
134
135 // actual artifact publish happens at the end of the job
136 if gh_bootstrap_template.is_empty() {
137 anyhow::bail!(
138 "Did not specify flowey bootstrap template. Please provide one using `Pipeline::gh_set_flowey_bootstrap_template`"
139 )
140 }
141
142 let gh_bootstrap_template = gh_bootstrap_template
143 .replace("{{FLOWEY_BIN_EXTENSION}}", platform.exe_suffix())
144 .replace("{{FLOWEY_CRATE}}", flowey_crate)
145 .replace(
146 "{{FLOWEY_PIPELINE_PATH}}",
147 &pipeline_file.with_extension("").display().to_string(),
148 )
149 .replace(
150 "{{FLOWEY_TARGET}}",
151 match (platform, arch) {
152 (FlowPlatform::Windows, FlowArch::X86_64) => "x86_64-pc-windows-msvc",
153 (FlowPlatform::Windows, FlowArch::Aarch64) => "aarch64-pc-windows-msvc",
154 (FlowPlatform::Linux(_), FlowArch::X86_64) => {
155 "x86_64-unknown-linux-gnu"
156 }
157 (FlowPlatform::Linux(_), FlowArch::Aarch64) => {
158 "aarch64-unknown-linux-gnu"
159 }
160 (platform, arch) => {
161 anyhow::bail!("unsupported platform {platform} / arch {arch}")
162 }
163 },
164 )
165 .replace("{{FLOWEY_OUTDIR}}", &format!("{RUNNER_TEMP}/{flowey_path}"));
166
167 let bootstrap_steps: serde_yaml::Sequence =
168 serde_yaml::from_str(&gh_bootstrap_template)
169 .context("malformed flowey bootstrap template")?;
170
171 gh_steps.extend(bootstrap_steps);
172 flowey_path
173 }
174 FloweySource::Consume(artifact) => {
175 // download previously bootstrapped flowey
176 artifact_names.push(artifact.as_str());
177 format!("used_artifacts/{artifact}")
178 }
179 };
180
181 // download any artifacts that'll be used
182 artifact_names.extend(artifacts_used.iter().map(|a| a.name.as_str()));
183 if !artifact_names.is_empty() {
184 let pattern = if let &[name] = artifact_names.as_slice() {
185 name.to_string()
186 } else {
187 format!("{{{}}}", artifact_names.join(","))
188 };
189 gh_steps.push({
190 let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
191 r#"
192 name: '🌼📦 Download artifacts'
193 uses: actions/download-artifact@v4
194 with:
195 pattern: '{pattern}'
196 path: {RUNNER_TEMP}/used_artifacts/
197 "#
198 ))
199 .unwrap();
200 map.into()
201 });
202 }
203
204 {
205 let mut map = serde_yaml::Mapping::new();
206 map.insert(
207 "run".into(),
208 format!(r#"echo "{RUNNER_TEMP}/{flowey_path}" >> $GITHUB_PATH"#).into(),
209 );
210 map.insert("shell".into(), "bash".into());
211 map.insert("name".into(), "🌼📦 Add flowey to PATH".into());
212 gh_steps.push(map.into());
213 }
214
215 let var_db = VarDbRequestBuilder::new(&flowey_bin, job_idx.index());
216
217 let bootstrap_bash_var_db_inject = |var, is_raw_string| {
218 var_db
219 .update_from_stdin(var, false)
220 .raw_string(is_raw_string)
221 .to_string()
222 };
223
224 // if this was a bootstrap job, also take a moment to run a "self check"
225 // to make sure that the current checked-in template matches the one it
226 // expected
227 if let FloweySource::Bootstrap(..) = &flowey_source {
228 let mut current_invocation = std::env::args().collect::<Vec<_>>();
229
230 current_invocation[0] = flowey_bin.clone();
231
232 // if this code path is run while generating the YAML to compare the
233 // check against, we want to remove the --runtime or --check param from the
234 // current call, or else there'll be a dupe
235 let mut strip_parameter = |prefix: &str| {
236 if let Some(i) = current_invocation
237 .iter()
238 .position(|s| s.starts_with(prefix))
239 {
240 current_invocation.remove(i);
241 if !current_invocation[i].starts_with(prefix) {
242 current_invocation.remove(i);
243 }
244 }
245 };
246
247 strip_parameter("--runtime");
248 strip_parameter("--check");
249
250 // insert the --check bit of the call alongside the --out param
251 {
252 let i = current_invocation
253 .iter()
254 .position(|s| s.starts_with("--out"))
255 .unwrap();
256
257 let current_yaml = match platform.kind() {
258 FlowPlatformKind::Windows => {
259 let win_path = flowey_path.replace('/', "\\");
260 format!(r#"$ESCAPED_AGENT_TEMPDIR\\{win_path}\\pipeline.yaml"#)
261 }
262 FlowPlatformKind::Unix => {
263 format!(r#"$ESCAPED_AGENT_TEMPDIR/{flowey_path}/pipeline.yaml"#)
264 }
265 };
266
267 current_invocation.insert(i, current_yaml);
268 current_invocation.insert(i, "--runtime".into());
269 }
270
271 // Need to use an escaped version of the "true" windows/linux path
272 // here, or else the --check will fail.
273 let cmd = format!(
274 r###"
275ESCAPED_AGENT_TEMPDIR=$(
276cat <<'EOF' | sed 's/\\/\\\\/g'
277{RUNNER_TEMP}
278EOF
279)
280{}
281"###,
282 current_invocation.join(" ")
283 );
284
285 gh_steps.push({
286 let mut map = serde_yaml::Mapping::new();
287 map.insert("name".into(), "🌼🔎 Self-check YAML".into());
288 map.insert(
289 "run".into(),
290 serde_yaml::Value::String(cmd.trim().to_string()),
291 );
292 map.insert("shell".into(), "bash".into());
293 map.into()
294 })
295 }
296
297 let mut flowey_bootstrap_bash = String::new();
298
299 // and now use those vars to do some flowey bootstrap
300 writeln!(flowey_bootstrap_bash, "{}", {
301 let runtime_debug_level = if runtime_debug_log { "debug" } else { "info" };
302
303 let var_db_insert_runtime_debug_level =
304 bootstrap_bash_var_db_inject("FLOWEY_LOG", false);
305 let var_db_insert_working_dir =
306 bootstrap_bash_var_db_inject(VAR_DB_SEEDVAR_FLOWEY_WORKING_DIR, true);
307
308 // Need to use "normalized" path in cases where the path is being
309 // used directly from a bash context, as is the case when we are
310 // trying to invoke `flowey.exe` in argv0 position)
311 //
312 // https://github.com/microsoft/azure-pipelines-tasks/issues/10653#issuecomment-585669089
313 format!(
314 r###"
315AgentTempDirNormal="{RUNNER_TEMP}"
316AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|')
317echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV
318
319chmod +x $AgentTempDirNormal/{flowey_path}/{flowey_bin}
320
321echo '"{runtime_debug_level}"' | {var_db_insert_runtime_debug_level}
322echo "{RUNNER_TEMP}/work" | {var_db_insert_working_dir}
323"###
324 )
325 .trim_start()
326 .to_owned()
327 })?;
328
329 // import pipeline vars being used by the job into flowey
330 for ResolvedJobUseParameter {
331 flowey_var,
332 pipeline_param_idx,
333 } in parameters_used
334 {
335 let is_string = matches!(
336 parameters[*pipeline_param_idx],
337 flowey_core::pipeline::internal::Parameter::String { .. }
338 );
339
340 let default = match &parameters[*pipeline_param_idx] {
341 flowey_core::pipeline::internal::Parameter::Bool { default, .. } => {
342 default.map(|b| b.to_string())
343 }
344 flowey_core::pipeline::internal::Parameter::String { default, .. } => {
345 default.clone()
346 }
347 flowey_core::pipeline::internal::Parameter::Num { default, .. } => {
348 default.map(|n| n.to_string())
349 }
350 }
351 .expect("defaults are currently required for parameters in Github backend");
352
353 let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, is_string);
354
355 let name = parameters[*pipeline_param_idx].name();
356
357 let cmd = format!(
358 r#"
359cat <<'EOF' | {var_db_inject_cmd}
360${{{{ inputs.{name} != '' && inputs.{name} || '{default}' }}}}
361EOF
362"#
363 )
364 .trim()
365 .to_string();
366 writeln!(flowey_bootstrap_bash, "{}", cmd)?;
367 }
368
369 // next, emit GitHub steps to create dirs for artifacts which will be
370 // published
371 for ResolvedJobArtifact { flowey_var, name } in artifacts_published {
372 writeln!(
373 flowey_bootstrap_bash,
374 r#"mkdir -p "$AgentTempDirNormal/publish_artifacts/{name}""#
375 )?;
376 let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, true);
377 match platform.kind() {
378 FlowPlatformKind::Windows => {
379 writeln!(
380 flowey_bootstrap_bash,
381 r#"echo "{RUNNER_TEMP}\\publish_artifacts\\{name}" | {var_db_inject_cmd}"#,
382 )?;
383 }
384 FlowPlatformKind::Unix => {
385 writeln!(
386 flowey_bootstrap_bash,
387 r#"echo "$AgentTempDirNormal/publish_artifacts/{name}" | {var_db_inject_cmd}"#,
388 )?;
389 }
390 }
391 }
392
393 // lastly, emit GitHub steps that report the dirs for any artifacts which
394 // are used by this job
395 for ResolvedJobArtifact { flowey_var, name } in artifacts_used {
396 let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, true);
397 match platform.kind() {
398 FlowPlatformKind::Windows => {
399 writeln!(
400 flowey_bootstrap_bash,
401 r#"echo "{RUNNER_TEMP}\\used_artifacts\\{name}" | {var_db_inject_cmd}"#,
402 )?;
403 }
404 FlowPlatformKind::Unix => {
405 writeln!(
406 flowey_bootstrap_bash,
407 r#"echo "$AgentTempDirNormal/used_artifacts/{name}" | {var_db_inject_cmd}"#,
408 )?;
409 }
410 }
411 }
412
413 gh_steps.push({
414 let mut map = serde_yaml::Mapping::new();
415 map.insert("name".into(), "🌼🛫 Initialize job".into());
416 map.insert(
417 "run".into(),
418 serde_yaml::Value::String(flowey_bootstrap_bash),
419 );
420 map.insert("shell".into(), "bash".into());
421 map.into()
422 });
423
424 // now that we've done all the job-level bootstrapping, we can emit all
425 // the actual steps the user cares about
426 gh_steps.extend(steps);
427
428 // ..and once that's done, the last order of business is to emit some
429 // GitHub steps to publish the various artifacts created by this job
430 for ResolvedJobArtifact {
431 flowey_var: _,
432 name,
433 } in artifacts_published
434 {
435 gh_steps.push({
436 let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
437 r#"
438 name: 🌼📦 Publish {name}
439 uses: actions/upload-artifact@v4
440 with:
441 name: {name}
442 path: {RUNNER_TEMP}/publish_artifacts/{name}/
443 include-hidden-files: true
444 "#
445 ))
446 .unwrap();
447 map.into()
448 });
449 }
450
451 // also, if this job also bootstrapped flowey that other nodes depend
452 // on, make sure to publish it!
453 if let FloweySource::Bootstrap(artifact, true) = flowey_source {
454 // don't leak the bootstrap job's runtime var db
455 gh_steps.push({
456 let mut map = serde_yaml::Mapping::new();
457 map.insert("name".into(), "🌼🧼 Redact bootstrap var db".into());
458 map.insert(
459 "run".into(),
460 serde_yaml::Value::String(format!(
461 "rm $AgentTempDirNormal/{flowey_path}/job{}.json",
462 job_idx.index()
463 )),
464 );
465 map.insert("shell".into(), "bash".into());
466 map.into()
467 });
468
469 gh_steps.push({
470 let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
471 r#"
472 name: 🌼🥾 Publish bootstrapped flowey
473 uses: actions/upload-artifact@v4
474 with:
475 name: {artifact}
476 path: {RUNNER_TEMP}/{flowey_path}
477 "#
478 ))
479 .unwrap();
480 map.into()
481 });
482 }
483
484 let runner_kind_to_yaml = |runner: &GhRunner| match runner {
485 GhRunner::GhHosted(s) => github_yaml_defs::Runner::GhHosted(match s {
486 GhRunnerOsLabel::UbuntuLatest => github_yaml_defs::RunnerOsLabel::UbuntuLatest,
487 GhRunnerOsLabel::Ubuntu2204 => github_yaml_defs::RunnerOsLabel::Ubuntu2204,
488 GhRunnerOsLabel::Ubuntu2004 => github_yaml_defs::RunnerOsLabel::Ubuntu2004,
489 GhRunnerOsLabel::WindowsLatest => github_yaml_defs::RunnerOsLabel::WindowsLatest,
490 GhRunnerOsLabel::Windows2022 => github_yaml_defs::RunnerOsLabel::Windows2022,
491 GhRunnerOsLabel::Windows2019 => github_yaml_defs::RunnerOsLabel::Windows2019,
492 GhRunnerOsLabel::MacOsLatest => github_yaml_defs::RunnerOsLabel::MacOsLatest,
493 GhRunnerOsLabel::MacOs14 => github_yaml_defs::RunnerOsLabel::MacOs14,
494 GhRunnerOsLabel::MacOs13 => github_yaml_defs::RunnerOsLabel::MacOs13,
495 GhRunnerOsLabel::MacOs12 => github_yaml_defs::RunnerOsLabel::MacOs12,
496 GhRunnerOsLabel::MacOs11 => github_yaml_defs::RunnerOsLabel::MacOs11,
497 GhRunnerOsLabel::Custom(s) => github_yaml_defs::RunnerOsLabel::Custom(s.into()),
498 }),
499 GhRunner::SelfHosted(v) => github_yaml_defs::Runner::SelfHosted(v.to_vec()),
500 GhRunner::RunnerGroup { group, labels } => github_yaml_defs::Runner::Group {
501 group: group.into(),
502 labels: labels.to_vec(),
503 },
504 };
505
506 let perm_val_to_yaml = |permission_value: &GhPermissionValue| match permission_value {
507 GhPermissionValue::Read => github_yaml_defs::PermissionValue::Read,
508 GhPermissionValue::Write => github_yaml_defs::PermissionValue::Write,
509 GhPermissionValue::None => github_yaml_defs::PermissionValue::None,
510 };
511
512 let perm_kind_to_yaml = |permission: &GhPermission| match permission {
513 GhPermission::Actions => github_yaml_defs::Permissions::Actions,
514 GhPermission::Attestations => github_yaml_defs::Permissions::Attestations,
515 GhPermission::Checks => github_yaml_defs::Permissions::Checks,
516 GhPermission::Contents => github_yaml_defs::Permissions::Contents,
517 GhPermission::Deployments => github_yaml_defs::Permissions::Deployments,
518 GhPermission::Discussions => github_yaml_defs::Permissions::Discussions,
519 GhPermission::IdToken => github_yaml_defs::Permissions::IdToken,
520 GhPermission::Issues => github_yaml_defs::Permissions::Issues,
521 GhPermission::Packages => github_yaml_defs::Permissions::Packages,
522 GhPermission::Pages => github_yaml_defs::Permissions::Pages,
523 GhPermission::PullRequests => github_yaml_defs::Permissions::PullRequests,
524 GhPermission::RepositoryProjects => github_yaml_defs::Permissions::RepositoryProjects,
525 GhPermission::SecurityEvents => github_yaml_defs::Permissions::SecurityEvents,
526 GhPermission::Statuses => github_yaml_defs::Permissions::Statuses,
527 };
528
529 let mut job_permissions = BTreeMap::new();
530 for permission_map in gh_permissions.values() {
531 for (permission, value) in permission_map {
532 if let Some(old_value) = job_permissions.insert(permission.clone(), value.clone()) {
533 if old_value != *value {
534 anyhow::bail!(
535 "permission {:?} was to conflicting values in job {:?}: {:?} and {:?}",
536 permission,
537 label,
538 old_value,
539 value
540 )
541 }
542 };
543 }
544 }
545
546 github_jobs.insert(
547 format!("job{}", job_idx.index()),
548 github_yaml_defs::Job {
549 name: label.clone(),
550 runs_on: gh_pool.clone().map(|runner| runner_kind_to_yaml(&runner)),
551 permissions: job_permissions
552 .iter()
553 .map(|k| (perm_kind_to_yaml(k.0), perm_val_to_yaml(k.1)))
554 .collect(),
555 needs: {
556 graph
557 .edges_directed(job_idx, petgraph::Direction::Incoming)
558 .map(|e| {
559 use petgraph::prelude::*;
560 format!("job{}", e.source().index())
561 })
562 .collect()
563 },
564 r#if: gh_override_if
565 .clone()
566 .or_else(|| Some("github.event.pull_request.draft == false".to_string())),
567 env: gh_global_env.clone(),
568 steps: gh_steps,
569 },
570 );
571 }
572
573 let mut concurrency = None;
574 let pipeline_trigger = github_yaml_defs::Triggers {
575 workflow_call: None,
576 workflow_dispatch: Some(github_yaml_defs::WorkflowDispatch {
577 inputs: github_yaml_defs::Inputs {
578 inputs: parameters
579 .into_iter()
580 .map(|param| {
581 (
582 param.name().to_string(),
583 match param {
584 flowey_core::pipeline::internal::Parameter::Bool {
585 name: _,
586 description,
587 kind: _,
588 default,
589 } => github_yaml_defs::Input {
590 description: Some(description.clone()),
591 default: default.map(github_yaml_defs::Default::Boolean),
592 required: default.is_none(),
593 ty: github_yaml_defs::InputType::Boolean,
594 },
595 flowey_core::pipeline::internal::Parameter::String {
596 name: _,
597 description,
598 kind: _,
599 default,
600 possible_values: _,
601 } => github_yaml_defs::Input {
602 description: Some(description.clone()),
603 default: default
604 .as_ref()
605 .map(|s| github_yaml_defs::Default::String(s.clone())),
606 required: default.is_none(),
607 ty: github_yaml_defs::InputType::String,
608 },
609 flowey_core::pipeline::internal::Parameter::Num {
610 name: _,
611 description,
612 kind: _,
613 default,
614 possible_values: _,
615 } => github_yaml_defs::Input {
616 description: Some(description.clone()),
617 default: default.map(github_yaml_defs::Default::Number),
618 required: default.is_none(),
619 ty: github_yaml_defs::InputType::Number,
620 },
621 },
622 )
623 })
624 .collect::<BTreeMap<String, github_yaml_defs::Input>>(),
625 },
626 }),
627 pull_request: match gh_pr_triggers {
628 Some(gh_pr_triggers) => {
629 if gh_pr_triggers.auto_cancel {
630 concurrency = Some(github_yaml_defs::Concurrency {
631 // only cancel in-progress jobs for the same workflow and branch
632 group: Some("${{ github.workflow }}-${{ github.ref }}".to_string()),
633 cancel_in_progress: Some(true),
634 })
635 };
636 Some(github_yaml_defs::PrTrigger {
637 branches: gh_pr_triggers.branches.clone(),
638 branches_ignore: gh_pr_triggers.exclude_branches.clone(),
639 types: gh_pr_triggers.types.clone(),
640 })
641 }
642 None => None,
643 },
644 push: match gh_ci_triggers {
645 Some(gh_ci_triggers) => Some(github_yaml_defs::CiTrigger {
646 branches: gh_ci_triggers.branches,
647 branches_ignore: gh_ci_triggers.exclude_branches,
648 tags: gh_ci_triggers.tags,
649 tags_ignore: gh_ci_triggers.exclude_tags,
650 }),
651 None => None,
652 },
653 schedule: gh_schedule_triggers
654 .iter()
655 .map(|s| github_yaml_defs::Cron {
656 cron: s.cron.clone(),
657 })
658 .collect(),
659 };
660
661 let github_pipeline = github_yaml_defs::Pipeline {
662 name: gh_name,
663 on: Some(pipeline_trigger),
664 concurrency,
665 jobs: Some(github_yaml_defs::Jobs { jobs: github_jobs }),
666 inputs: None,
667 };
668
669 match check {
670 CheckMode::Check(_) | CheckMode::Runtime(_) => check_generated_yaml_and_json(
671 &github_pipeline,
672 &pipeline_static_db,
673 check,
674 repo_root,
675 pipeline_file,
676 None,
677 ),
678 CheckMode::None => write_generated_yaml_and_json(
679 &github_pipeline,
680 &pipeline_static_db,
681 repo_root,
682 pipeline_file,
683 None,
684 ),
685 }
686}
687
688/// Resolve a flow as a sequence of GitHub YAML steps.
689///
690/// These steps can then be marshalled into a well-formed GitHub pipeline yaml
691/// using a separate GitHub pipeline yaml builder
692// pub(crate) so that internal debug CLI tooling can use it
693fn resolve_flow_as_github_yaml_steps(
694 seed_nodes: BTreeMap<NodeHandle, (bool, Vec<Box<[u8]>>)>,
695 resolved_patches: flowey_core::patch::ResolvedPatches,
696 external_read_vars: BTreeSet<String>,
697 platform: FlowPlatform,
698 arch: FlowArch,
699 job_idx: usize,
700 flowey_bin: &str,
701 gh_permissions: &BTreeMap<NodeHandle, BTreeMap<GhPermission, GhPermissionValue>>,
702) -> anyhow::Result<(
703 Vec<serde_yaml::Value>,
704 BTreeMap<String, Vec<crate::cli::exec_snippet::SerializedRequest>>,
705)> {
706 let mut output_steps = Vec::new();
707
708 let (mut output_graph, request_db, err_unreachable_nodes) =
709 crate::flow_resolver::stage1_dag::stage1_dag(
710 FlowBackend::Github,
711 platform,
712 arch,
713 resolved_patches,
714 seed_nodes,
715 external_read_vars,
716 // TODO: support GitHub agents with persistent storage
717 None,
718 )?;
719
720 if err_unreachable_nodes.is_some() {
721 anyhow::bail!("detected unreachable nodes")
722 }
723
724 let mut bash_commands = BashCommands::new_github();
725
726 let output_order = petgraph::algo::toposort(&output_graph, None)
727 .expect("runtime variables cannot introduce a DAG cycle");
728
729 let var_db = VarDbRequestBuilder::new(flowey_bin, job_idx);
730
731 for node_idx in output_order.into_iter().rev() {
732 let OutputGraphEntry { node_handle, step } = output_graph[node_idx].1.take().unwrap();
733
734 let node_modpath = node_handle.modpath();
735
736 match step {
737 Step::Anchor { .. } => {}
738 Step::Rust {
739 idx,
740 label,
741 can_merge,
742 code: _,
743 } => {
744 output_steps.extend(bash_commands.push(
745 Some(label),
746 can_merge,
747 crate::cli::exec_snippet::construct_exec_snippet_cli(
748 flowey_bin,
749 node_modpath,
750 idx,
751 job_idx,
752 ),
753 ));
754 }
755 Step::AdoYaml { label, .. } => {
756 anyhow::bail!("ADO YAML not supported in GitHub. In step '{}'", label)
757 }
758 Step::GitHubYaml {
759 gh_to_rust,
760 rust_to_gh,
761 label,
762 step_id,
763 uses,
764 with,
765 condvar,
766 permissions,
767 } => {
768 for permission in permissions {
769 if let Some(permission_map) = gh_permissions.get(&node_handle) {
770 if let Some(permission_value) = permission_map.get(&permission.0) {
771 if *permission_value != permission.1 {
772 anyhow::bail!(
773 "permission mismatch for {:?}: expected {:?}, got {:?}",
774 permission.0,
775 permission.1,
776 permission_value
777 )
778 }
779 }
780 } else {
781 anyhow::bail!(
782 "permission missing for {:?}: expected {:?}",
783 permission.0,
784 permission.1
785 )
786 }
787 }
788
789 for gh_var_state in rust_to_gh {
790 let set_gh_env_var = var_db
791 .write_to_gh_env(&gh_var_state.backing_var, &gh_var_state.raw_name)
792 .raw_string(!gh_var_state.is_object)
793 .condvar(condvar.as_deref());
794
795 bash_commands.push_minor(format!("{set_gh_env_var}\n"));
796 }
797
798 if !uses.is_empty() {
799 if let Some(condvar) = &condvar {
800 // guaranteed to be a bare bool `true`/`false`, hence
801 // is_raw_string = false
802 let set_condvar = var_db.write_to_gh_env(condvar, "FLOWEY_CONDITION");
803 bash_commands.push_minor(format!("{set_condvar}\n"));
804 }
805
806 let mut map = serde_yaml::Mapping::new();
807 map.insert("id".into(), serde_yaml::Value::String(step_id.clone()));
808 map.insert("uses".into(), serde_yaml::Value::String(uses));
809 if !with.is_empty() {
810 let mut with_map = serde_yaml::Mapping::new();
811 for (k, v) in with {
812 with_map.insert(k.into(), v.into());
813 }
814 map.insert("with".into(), with_map.into());
815 }
816 map.insert("name".into(), label.into());
817 if condvar.is_some() {
818 map.insert("if".into(), "${{ fromJSON(env.FLOWEY_CONDITION) }}".into());
819 }
820
821 let step: serde_yaml::Value = map.into();
822 output_steps.extend(bash_commands.flush());
823 output_steps.push(step);
824 }
825
826 for gh_var_state in gh_to_rust {
827 let value = if gh_var_state.is_object {
828 format!(r#"${{{{ toJSON({}) }}}}"#, gh_var_state.raw_name)
829 } else {
830 format!(r#"${{{{ {} }}}}"#, gh_var_state.raw_name)
831 };
832
833 let write_var = var_db
834 .update_from_stdin(&gh_var_state.backing_var, gh_var_state.is_secret)
835 .raw_string(!gh_var_state.is_object)
836 .condvar(condvar.as_deref())
837 .env_source(Some(&gh_var_state.raw_name));
838
839 let cmd = format!("{write_var} <<EOF\n{value}\nEOF",);
840 bash_commands.push_minor(cmd);
841 }
842 }
843 }
844 }
845
846 output_steps.extend(bash_commands.flush());
847
848 let request_db = request_db
849 .into_iter()
850 .map(|(node_handle, reqs)| {
851 (
852 node_handle.modpath().to_owned(),
853 reqs.into_iter()
854 .map(crate::cli::exec_snippet::SerializedRequest)
855 .collect(),
856 )
857 })
858 .collect();
859
860 Ok((output_steps, request_db))
861}
862