GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/openvmm

Public

mirrored from https://github.com/microsoft/openvmmAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
045b1b0dbef400c88d68fce3fbe84ee2cedb2fa2

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openvmm/flowey/flowey_cli/src/pipeline_resolver/github_yaml

flowey/flowey_cli/src/pipeline_resolver/github_yaml/mod.rs

867lines · modecode

RawDownload
Latest commit unavailable.
1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4//! Code for emitting a pipeline as a single self-contained GitHub Actions yaml file
5
6use super::common_yaml::BashCommands;
7use super::common_yaml::check_generated_yaml_and_json;
8use super::common_yaml::write_generated_yaml_and_json;
9use super::generic::ResolvedJobArtifact;
10use super::generic::ResolvedJobUseParameter;
11use crate::cli::exec_snippet::FloweyPipelineStaticDb;
12use crate::cli::exec_snippet::VAR_DB_SEEDVAR_FLOWEY_WORKING_DIR;
13use crate::cli::pipeline::CheckMode;
14use crate::cli::var_db::VarDbRequestBuilder;
15use crate::flow_resolver::stage1_dag::OutputGraphEntry;
16use crate::flow_resolver::stage1_dag::Step;
17use crate::pipeline_resolver::common_yaml::FloweySource;
18use crate::pipeline_resolver::common_yaml::job_flowey_bootstrap_source;
19use crate::pipeline_resolver::generic::ResolvedPipeline;
20use crate::pipeline_resolver::generic::ResolvedPipelineJob;
21use anyhow::Context;
22use flowey_core::node::FlowArch;
23use flowey_core::node::FlowBackend;
24use flowey_core::node::FlowPlatform;
25use flowey_core::node::FlowPlatformKind;
26use flowey_core::node::NodeHandle;
27use flowey_core::node::user_facing::GhPermission;
28use flowey_core::node::user_facing::GhPermissionValue;
29use flowey_core::pipeline::GhRunner;
30use flowey_core::pipeline::GhRunnerOsLabel;
31use std::collections::BTreeMap;
32use std::collections::BTreeSet;
33use std::fmt::Write;
34use std::path::Path;
35mod github_yaml_defs;
36
37const RUNNER_TEMP: &str = "${{ runner.temp }}";
38
39/// Emit a pipeline as a single self-contained GitHub Actions yaml file
40pub fn github_yaml(
41 pipeline: ResolvedPipeline,
42 runtime_debug_log: bool,
43 repo_root: &Path,
44 pipeline_file: &Path,
45 flowey_crate: &str,
46 check: CheckMode,
47) -> anyhow::Result<()> {
48 if pipeline_file.extension().and_then(|s| s.to_str()) != Some("yaml") {
49 anyhow::bail!("pipeline name must end with .yaml")
50 }
51
52 let ResolvedPipeline {
53 graph,
54 order,
55 gh_name,
56 gh_schedule_triggers,
57 gh_ci_triggers,
58 gh_pr_triggers,
59 gh_bootstrap_template,
60 parameters,
61 ado_name: _,
62 ado_schedule_triggers: _,
63 ado_ci_triggers: _,
64 ado_pr_triggers: _,
65 ado_bootstrap_template: _,
66 ado_resources_repository: _,
67 ado_post_process_yaml_cb: _,
68 ado_variables: _,
69 ado_job_id_overrides: _,
70 } = pipeline;
71
72 let mut job_flowey_source: BTreeMap<petgraph::prelude::NodeIndex, FloweySource> =
73 job_flowey_bootstrap_source(&graph, &order);
74
75 let mut pipeline_static_db = FloweyPipelineStaticDb {
76 flow_backend: crate::cli::FlowBackendCli::Github,
77 var_db_backend_kind: crate::cli::exec_snippet::VarDbBackendKind::Json,
78 job_reqs: BTreeMap::new(),
79 };
80
81 let mut github_jobs = BTreeMap::new();
82
83 for job_idx in order {
84 let ResolvedPipelineJob {
85 ref root_nodes,
86 ref patches,
87 ref label,
88 platform,
89 arch,
90 ref external_read_vars,
91 ado_pool: _,
92 timeout_minutes,
93 ref gh_override_if,
94 ref gh_global_env,
95 ref gh_pool,
96 ref gh_permissions,
97 cond_param_idx,
98 ref parameters_used,
99 ref artifacts_used,
100 ref artifacts_published,
101 ado_variables: _,
102 } = graph[job_idx];
103
104 if cond_param_idx.is_some() {
105 anyhow::bail!(
106 "conditional params are not supported in GitHub backend, use `gh_dangerous_override_if` instead"
107 );
108 }
109
110 let flowey_bin = platform.binary("flowey");
111 let (steps, req_db) = resolve_flow_as_github_yaml_steps(
112 root_nodes
113 .clone()
114 .into_iter()
115 .map(|(node, requests)| (node, (true, requests)))
116 .collect(),
117 patches.clone(),
118 external_read_vars.clone(),
119 platform,
120 arch,
121 job_idx.index(),
122 &flowey_bin,
123 gh_permissions,
124 )
125 .context(format!("in job '{label}'"))?;
126
127 {
128 let existing = pipeline_static_db.job_reqs.insert(job_idx.index(), req_db);
129 assert!(existing.is_none())
130 }
131
132 let mut gh_steps = Vec::new();
133
134 let flowey_source = job_flowey_source.remove(&job_idx).unwrap();
135
136 let mut artifact_names = Vec::new();
137
138 let flowey_path = match &flowey_source {
139 FloweySource::Bootstrap { .. } => {
140 let flowey_path = "bootstrapped-flowey".to_string();
141
142 // actual artifact publish happens at the end of the job
143 if gh_bootstrap_template.is_empty() {
144 anyhow::bail!(
145 "Did not specify flowey bootstrap template. Please provide one using `Pipeline::gh_set_flowey_bootstrap_template`"
146 )
147 }
148
149 let gh_bootstrap_template = gh_bootstrap_template
150 .replace("{{FLOWEY_BIN_EXTENSION}}", platform.exe_suffix())
151 .replace("{{FLOWEY_CRATE}}", flowey_crate)
152 .replace(
153 "{{FLOWEY_PIPELINE_PATH}}",
154 &pipeline_file.with_extension("").display().to_string(),
155 )
156 .replace(
157 "{{FLOWEY_TARGET}}",
158 match (platform, arch) {
159 (FlowPlatform::Windows, FlowArch::X86_64) => "x86_64-pc-windows-msvc",
160 (FlowPlatform::Windows, FlowArch::Aarch64) => "aarch64-pc-windows-msvc",
161 (FlowPlatform::Linux(_), FlowArch::X86_64) => {
162 "x86_64-unknown-linux-gnu"
163 }
164 (FlowPlatform::Linux(_), FlowArch::Aarch64) => {
165 "aarch64-unknown-linux-gnu"
166 }
167 (platform, arch) => {
168 anyhow::bail!("unsupported platform {platform} / arch {arch}")
169 }
170 },
171 )
172 .replace("{{FLOWEY_OUTDIR}}", &format!("{RUNNER_TEMP}/{flowey_path}"));
173
174 let bootstrap_steps: serde_yaml::Sequence =
175 serde_yaml::from_str(&gh_bootstrap_template)
176 .context("malformed flowey bootstrap template")?;
177
178 gh_steps.extend(bootstrap_steps);
179 flowey_path
180 }
181 FloweySource::Consume(artifact) => {
182 // download previously bootstrapped flowey
183 artifact_names.push(artifact.as_str());
184 format!("used_artifacts/{artifact}")
185 }
186 };
187
188 // download any artifacts that'll be used
189 artifact_names.extend(artifacts_used.iter().map(|a| a.name.as_str()));
190 if !artifact_names.is_empty() {
191 let pattern = if let &[name] = artifact_names.as_slice() {
192 name.to_string()
193 } else {
194 format!("{{{}}}", artifact_names.join(","))
195 };
196 gh_steps.push({
197 let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
198 r#"
199 name: '🌼📦 Download artifacts'
200 uses: actions/download-artifact@v4
201 with:
202 pattern: '{pattern}'
203 path: {RUNNER_TEMP}/used_artifacts/
204 "#
205 ))
206 .unwrap();
207 map.into()
208 });
209 }
210
211 {
212 let mut map = serde_yaml::Mapping::new();
213 map.insert(
214 "run".into(),
215 format!(r#"echo "{RUNNER_TEMP}/{flowey_path}" >> $GITHUB_PATH"#).into(),
216 );
217 map.insert("shell".into(), "bash".into());
218 map.insert("name".into(), "🌼📦 Add flowey to PATH".into());
219 gh_steps.push(map.into());
220 }
221
222 let var_db = VarDbRequestBuilder::new(&flowey_bin, job_idx.index());
223
224 let bootstrap_bash_var_db_inject = |var, is_raw_string| {
225 var_db
226 .update_from_stdin(var, false)
227 .raw_string(is_raw_string)
228 .to_string()
229 };
230
231 // if this was a bootstrap job, also take a moment to run a "self check"
232 // to make sure that the current checked-in template matches the one it
233 // expected
234 if let FloweySource::Bootstrap(..) = &flowey_source {
235 let mut current_invocation = std::env::args().collect::<Vec<_>>();
236
237 current_invocation[0] = flowey_bin.clone();
238
239 // if this code path is run while generating the YAML to compare the
240 // check against, we want to remove the --runtime or --check param from the
241 // current call, or else there'll be a dupe
242 let mut strip_parameter = |prefix: &str| {
243 if let Some(i) = current_invocation
244 .iter()
245 .position(|s| s.starts_with(prefix))
246 {
247 current_invocation.remove(i);
248 if !current_invocation[i].starts_with(prefix) {
249 current_invocation.remove(i);
250 }
251 }
252 };
253
254 strip_parameter("--runtime");
255 strip_parameter("--check");
256
257 // insert the --check bit of the call alongside the --out param
258 {
259 let i = current_invocation
260 .iter()
261 .position(|s| s.starts_with("--out"))
262 .unwrap();
263
264 let current_yaml = match platform.kind() {
265 FlowPlatformKind::Windows => {
266 let win_path = flowey_path.replace('/', "\\");
267 format!(r#"$ESCAPED_AGENT_TEMPDIR\\{win_path}\\pipeline.yaml"#)
268 }
269 FlowPlatformKind::Unix => {
270 format!(r#"$ESCAPED_AGENT_TEMPDIR/{flowey_path}/pipeline.yaml"#)
271 }
272 };
273
274 current_invocation.insert(i, current_yaml);
275 current_invocation.insert(i, "--runtime".into());
276 }
277
278 // Need to use an escaped version of the "true" windows/linux path
279 // here, or else the --check will fail.
280 let cmd = format!(
281 r###"
282ESCAPED_AGENT_TEMPDIR=$(
283cat <<'EOF' | sed 's/\\/\\\\/g'
284{RUNNER_TEMP}
285EOF
286)
287{}
288"###,
289 current_invocation.join(" ")
290 );
291
292 gh_steps.push({
293 let mut map = serde_yaml::Mapping::new();
294 map.insert("name".into(), "🌼🔎 Self-check YAML".into());
295 map.insert(
296 "run".into(),
297 serde_yaml::Value::String(cmd.trim().to_string()),
298 );
299 map.insert("shell".into(), "bash".into());
300 map.into()
301 })
302 }
303
304 let mut flowey_bootstrap_bash = String::new();
305
306 // and now use those vars to do some flowey bootstrap
307 writeln!(flowey_bootstrap_bash, "{}", {
308 let runtime_debug_level = if runtime_debug_log { "debug" } else { "info" };
309
310 let var_db_insert_runtime_debug_level =
311 bootstrap_bash_var_db_inject("FLOWEY_LOG", false);
312 let var_db_insert_working_dir =
313 bootstrap_bash_var_db_inject(VAR_DB_SEEDVAR_FLOWEY_WORKING_DIR, true);
314
315 // Need to use "normalized" path in cases where the path is being
316 // used directly from a bash context, as is the case when we are
317 // trying to invoke `flowey.exe` in argv0 position)
318 //
319 // https://github.com/microsoft/azure-pipelines-tasks/issues/10653#issuecomment-585669089
320 format!(
321 r###"
322AgentTempDirNormal="{RUNNER_TEMP}"
323AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|')
324echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV
325
326chmod +x $AgentTempDirNormal/{flowey_path}/{flowey_bin}
327
328echo '"{runtime_debug_level}"' | {var_db_insert_runtime_debug_level}
329echo "{RUNNER_TEMP}/work" | {var_db_insert_working_dir}
330"###
331 )
332 .trim_start()
333 .to_owned()
334 })?;
335
336 // import pipeline vars being used by the job into flowey
337 for ResolvedJobUseParameter {
338 flowey_var,
339 pipeline_param_idx,
340 } in parameters_used
341 {
342 let is_string = matches!(
343 parameters[*pipeline_param_idx],
344 flowey_core::pipeline::internal::Parameter::String { .. }
345 );
346
347 let default = match &parameters[*pipeline_param_idx] {
348 flowey_core::pipeline::internal::Parameter::Bool { default, .. } => {
349 default.map(|b| b.to_string())
350 }
351 flowey_core::pipeline::internal::Parameter::String { default, .. } => {
352 default.clone()
353 }
354 flowey_core::pipeline::internal::Parameter::Num { default, .. } => {
355 default.map(|n| n.to_string())
356 }
357 }
358 .expect("defaults are currently required for parameters in Github backend");
359
360 let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, is_string);
361
362 let name = parameters[*pipeline_param_idx].name();
363
364 let cmd = format!(
365 r#"
366cat <<'EOF' | {var_db_inject_cmd}
367${{{{ inputs.{name} != '' && inputs.{name} || '{default}' }}}}
368EOF
369"#
370 )
371 .trim()
372 .to_string();
373 writeln!(flowey_bootstrap_bash, "{}", cmd)?;
374 }
375
376 // next, emit GitHub steps to create dirs for artifacts which will be
377 // published
378 for ResolvedJobArtifact { flowey_var, name } in artifacts_published {
379 writeln!(
380 flowey_bootstrap_bash,
381 r#"mkdir -p "$AgentTempDirNormal/publish_artifacts/{name}""#
382 )?;
383 let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, true);
384 match platform.kind() {
385 FlowPlatformKind::Windows => {
386 writeln!(
387 flowey_bootstrap_bash,
388 r#"echo "{RUNNER_TEMP}\\publish_artifacts\\{name}" | {var_db_inject_cmd}"#,
389 )?;
390 }
391 FlowPlatformKind::Unix => {
392 writeln!(
393 flowey_bootstrap_bash,
394 r#"echo "$AgentTempDirNormal/publish_artifacts/{name}" | {var_db_inject_cmd}"#,
395 )?;
396 }
397 }
398 }
399
400 // lastly, emit GitHub steps that report the dirs for any artifacts which
401 // are used by this job
402 for ResolvedJobArtifact { flowey_var, name } in artifacts_used {
403 let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, true);
404 match platform.kind() {
405 FlowPlatformKind::Windows => {
406 writeln!(
407 flowey_bootstrap_bash,
408 r#"echo "{RUNNER_TEMP}\\used_artifacts\\{name}" | {var_db_inject_cmd}"#,
409 )?;
410 }
411 FlowPlatformKind::Unix => {
412 writeln!(
413 flowey_bootstrap_bash,
414 r#"echo "$AgentTempDirNormal/used_artifacts/{name}" | {var_db_inject_cmd}"#,
415 )?;
416 }
417 }
418 }
419
420 gh_steps.push({
421 let mut map = serde_yaml::Mapping::new();
422 map.insert("name".into(), "🌼🛫 Initialize job".into());
423 map.insert(
424 "run".into(),
425 serde_yaml::Value::String(flowey_bootstrap_bash),
426 );
427 map.insert("shell".into(), "bash".into());
428 map.into()
429 });
430
431 // now that we've done all the job-level bootstrapping, we can emit all
432 // the actual steps the user cares about
433 gh_steps.extend(steps);
434
435 // ..and once that's done, the last order of business is to emit some
436 // GitHub steps to publish the various artifacts created by this job
437 for ResolvedJobArtifact {
438 flowey_var: _,
439 name,
440 } in artifacts_published
441 {
442 gh_steps.push({
443 let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
444 r#"
445 name: 🌼📦 Publish {name}
446 uses: actions/upload-artifact@v4
447 with:
448 name: {name}
449 path: {RUNNER_TEMP}/publish_artifacts/{name}/
450 include-hidden-files: true
451 "#
452 ))
453 .unwrap();
454 map.into()
455 });
456 }
457
458 // also, if this job also bootstrapped flowey that other nodes depend
459 // on, make sure to publish it!
460 if let FloweySource::Bootstrap(artifact, true) = flowey_source {
461 // don't leak the bootstrap job's runtime var db
462 gh_steps.push({
463 let mut map = serde_yaml::Mapping::new();
464 map.insert("name".into(), "🌼🧼 Redact bootstrap var db".into());
465 map.insert(
466 "run".into(),
467 serde_yaml::Value::String(format!(
468 "rm $AgentTempDirNormal/{flowey_path}/job{}.json",
469 job_idx.index()
470 )),
471 );
472 map.insert("shell".into(), "bash".into());
473 map.into()
474 });
475
476 gh_steps.push({
477 let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
478 r#"
479 name: 🌼🥾 Publish bootstrapped flowey
480 uses: actions/upload-artifact@v4
481 with:
482 name: {artifact}
483 path: {RUNNER_TEMP}/{flowey_path}
484 "#
485 ))
486 .unwrap();
487 map.into()
488 });
489 }
490
491 let runner_kind_to_yaml = |runner: &GhRunner| match runner {
492 GhRunner::GhHosted(s) => github_yaml_defs::Runner::GhHosted(match s {
493 GhRunnerOsLabel::UbuntuLatest => github_yaml_defs::RunnerOsLabel::UbuntuLatest,
494 GhRunnerOsLabel::Ubuntu2404 => github_yaml_defs::RunnerOsLabel::Ubuntu2404,
495 GhRunnerOsLabel::Ubuntu2204 => github_yaml_defs::RunnerOsLabel::Ubuntu2204,
496 GhRunnerOsLabel::WindowsLatest => github_yaml_defs::RunnerOsLabel::WindowsLatest,
497 GhRunnerOsLabel::Windows2025 => github_yaml_defs::RunnerOsLabel::Windows2025,
498 GhRunnerOsLabel::Windows2022 => github_yaml_defs::RunnerOsLabel::Windows2022,
499 GhRunnerOsLabel::Ubuntu2404Arm => github_yaml_defs::RunnerOsLabel::Ubuntu2404Arm,
500 GhRunnerOsLabel::Ubuntu2204Arm => github_yaml_defs::RunnerOsLabel::Ubuntu2204Arm,
501 GhRunnerOsLabel::Windows11Arm => github_yaml_defs::RunnerOsLabel::Windows11Arm,
502 GhRunnerOsLabel::Custom(s) => github_yaml_defs::RunnerOsLabel::Custom(s.into()),
503 }),
504 GhRunner::SelfHosted(v) => github_yaml_defs::Runner::SelfHosted(v.clone()),
505 GhRunner::RunnerGroup { group, labels } => github_yaml_defs::Runner::Group {
506 group: group.into(),
507 labels: labels.clone(),
508 },
509 };
510
511 let perm_val_to_yaml = |permission_value: &GhPermissionValue| match permission_value {
512 GhPermissionValue::Read => github_yaml_defs::PermissionValue::Read,
513 GhPermissionValue::Write => github_yaml_defs::PermissionValue::Write,
514 GhPermissionValue::None => github_yaml_defs::PermissionValue::None,
515 };
516
517 let perm_kind_to_yaml = |permission: &GhPermission| match permission {
518 GhPermission::Actions => github_yaml_defs::Permissions::Actions,
519 GhPermission::Attestations => github_yaml_defs::Permissions::Attestations,
520 GhPermission::Checks => github_yaml_defs::Permissions::Checks,
521 GhPermission::Contents => github_yaml_defs::Permissions::Contents,
522 GhPermission::Deployments => github_yaml_defs::Permissions::Deployments,
523 GhPermission::Discussions => github_yaml_defs::Permissions::Discussions,
524 GhPermission::IdToken => github_yaml_defs::Permissions::IdToken,
525 GhPermission::Issues => github_yaml_defs::Permissions::Issues,
526 GhPermission::Packages => github_yaml_defs::Permissions::Packages,
527 GhPermission::Pages => github_yaml_defs::Permissions::Pages,
528 GhPermission::PullRequests => github_yaml_defs::Permissions::PullRequests,
529 GhPermission::RepositoryProjects => github_yaml_defs::Permissions::RepositoryProjects,
530 GhPermission::SecurityEvents => github_yaml_defs::Permissions::SecurityEvents,
531 GhPermission::Statuses => github_yaml_defs::Permissions::Statuses,
532 };
533
534 let mut job_permissions = BTreeMap::new();
535 for permission_map in gh_permissions.values() {
536 for (permission, value) in permission_map {
537 if let Some(old_value) = job_permissions.insert(permission.clone(), value.clone()) {
538 if old_value != *value {
539 anyhow::bail!(
540 "permission {:?} was to conflicting values in job {:?}: {:?} and {:?}",
541 permission,
542 label,
543 old_value,
544 value
545 )
546 }
547 };
548 }
549 }
550
551 github_jobs.insert(
552 format!("job{}", job_idx.index()),
553 github_yaml_defs::Job {
554 name: label.clone(),
555 timeout_minutes,
556 runs_on: gh_pool.clone().map(|runner| runner_kind_to_yaml(&runner)),
557 permissions: job_permissions
558 .iter()
559 .map(|k| (perm_kind_to_yaml(k.0), perm_val_to_yaml(k.1)))
560 .collect(),
561 needs: {
562 graph
563 .edges_directed(job_idx, petgraph::Direction::Incoming)
564 .map(|e| {
565 use petgraph::prelude::*;
566 format!("job{}", e.source().index())
567 })
568 .collect()
569 },
570 r#if: gh_override_if
571 .clone()
572 .or_else(|| Some("github.event.pull_request.draft == false".to_string())),
573 env: gh_global_env.clone(),
574 steps: gh_steps,
575 },
576 );
577 }
578
579 let mut concurrency = None;
580 let pipeline_trigger = github_yaml_defs::Triggers {
581 workflow_call: None,
582 workflow_dispatch: Some(github_yaml_defs::WorkflowDispatch {
583 inputs: github_yaml_defs::Inputs {
584 inputs: parameters
585 .into_iter()
586 .map(|param| {
587 (
588 param.name().to_string(),
589 match param {
590 flowey_core::pipeline::internal::Parameter::Bool {
591 name: _,
592 description,
593 kind: _,
594 default,
595 } => github_yaml_defs::Input {
596 description: Some(description.clone()),
597 default: default.map(github_yaml_defs::Default::Boolean),
598 required: default.is_none(),
599 ty: github_yaml_defs::InputType::Boolean,
600 },
601 flowey_core::pipeline::internal::Parameter::String {
602 name: _,
603 description,
604 kind: _,
605 default,
606 possible_values: _,
607 } => github_yaml_defs::Input {
608 description: Some(description.clone()),
609 default: default
610 .as_ref()
611 .map(|s| github_yaml_defs::Default::String(s.clone())),
612 required: default.is_none(),
613 ty: github_yaml_defs::InputType::String,
614 },
615 flowey_core::pipeline::internal::Parameter::Num {
616 name: _,
617 description,
618 kind: _,
619 default,
620 possible_values: _,
621 } => github_yaml_defs::Input {
622 description: Some(description.clone()),
623 default: default.map(github_yaml_defs::Default::Number),
624 required: default.is_none(),
625 ty: github_yaml_defs::InputType::Number,
626 },
627 },
628 )
629 })
630 .collect::<BTreeMap<String, github_yaml_defs::Input>>(),
631 },
632 }),
633 pull_request: match gh_pr_triggers {
634 Some(gh_pr_triggers) => {
635 if gh_pr_triggers.auto_cancel {
636 concurrency = Some(github_yaml_defs::Concurrency {
637 // only cancel in-progress jobs for the same workflow and branch
638 group: Some("${{ github.workflow }}-${{ github.ref }}".to_string()),
639 cancel_in_progress: Some(true),
640 })
641 };
642 Some(github_yaml_defs::PrTrigger {
643 branches: gh_pr_triggers.branches.clone(),
644 branches_ignore: gh_pr_triggers.exclude_branches.clone(),
645 types: gh_pr_triggers.types.clone(),
646 })
647 }
648 None => None,
649 },
650 push: match gh_ci_triggers {
651 Some(gh_ci_triggers) => Some(github_yaml_defs::CiTrigger {
652 branches: gh_ci_triggers.branches,
653 branches_ignore: gh_ci_triggers.exclude_branches,
654 tags: gh_ci_triggers.tags,
655 tags_ignore: gh_ci_triggers.exclude_tags,
656 }),
657 None => None,
658 },
659 schedule: gh_schedule_triggers
660 .iter()
661 .map(|s| github_yaml_defs::Cron {
662 cron: s.cron.clone(),
663 })
664 .collect(),
665 };
666
667 let github_pipeline = github_yaml_defs::Pipeline {
668 name: gh_name,
669 on: Some(pipeline_trigger),
670 concurrency,
671 jobs: Some(github_yaml_defs::Jobs { jobs: github_jobs }),
672 inputs: None,
673 };
674
675 match check {
676 CheckMode::Check(_) | CheckMode::Runtime(_) => check_generated_yaml_and_json(
677 &github_pipeline,
678 &pipeline_static_db,
679 check,
680 repo_root,
681 pipeline_file,
682 None,
683 ),
684 CheckMode::None => write_generated_yaml_and_json(
685 &github_pipeline,
686 &pipeline_static_db,
687 repo_root,
688 pipeline_file,
689 None,
690 ),
691 }
692}
693
694/// Resolve a flow as a sequence of GitHub YAML steps.
695///
696/// These steps can then be marshalled into a well-formed GitHub pipeline yaml
697/// using a separate GitHub pipeline yaml builder
698// pub(crate) so that internal debug CLI tooling can use it
699fn resolve_flow_as_github_yaml_steps(
700 seed_nodes: BTreeMap<NodeHandle, (bool, Vec<Box<[u8]>>)>,
701 resolved_patches: flowey_core::patch::ResolvedPatches,
702 external_read_vars: BTreeSet<String>,
703 platform: FlowPlatform,
704 arch: FlowArch,
705 job_idx: usize,
706 flowey_bin: &str,
707 gh_permissions: &BTreeMap<NodeHandle, BTreeMap<GhPermission, GhPermissionValue>>,
708) -> anyhow::Result<(
709 Vec<serde_yaml::Value>,
710 BTreeMap<String, Vec<crate::cli::exec_snippet::SerializedRequest>>,
711)> {
712 let mut output_steps = Vec::new();
713
714 let (mut output_graph, request_db, err_unreachable_nodes) =
715 crate::flow_resolver::stage1_dag::stage1_dag(
716 FlowBackend::Github,
717 platform,
718 arch,
719 resolved_patches,
720 seed_nodes,
721 external_read_vars,
722 // TODO: support GitHub agents with persistent storage
723 None,
724 )?;
725
726 if err_unreachable_nodes.is_some() {
727 anyhow::bail!("detected unreachable nodes")
728 }
729
730 let mut bash_commands = BashCommands::new_github();
731
732 let output_order = petgraph::algo::toposort(&output_graph, None)
733 .expect("runtime variables cannot introduce a DAG cycle");
734
735 let var_db = VarDbRequestBuilder::new(flowey_bin, job_idx);
736
737 for node_idx in output_order.into_iter().rev() {
738 let OutputGraphEntry { node_handle, step } = output_graph[node_idx].1.take().unwrap();
739
740 let node_modpath = node_handle.modpath();
741
742 match step {
743 Step::Anchor { .. } => {}
744 Step::Rust {
745 idx,
746 label,
747 can_merge,
748 code: _,
749 } => {
750 output_steps.extend(bash_commands.push(
751 Some(label),
752 can_merge,
753 crate::cli::exec_snippet::construct_exec_snippet_cli(
754 flowey_bin,
755 node_modpath,
756 idx,
757 job_idx,
758 ),
759 ));
760 }
761 Step::AdoYaml { label, .. } => {
762 anyhow::bail!("ADO YAML not supported in GitHub. In step '{}'", label)
763 }
764 Step::GitHubYaml {
765 gh_to_rust,
766 rust_to_gh,
767 label,
768 step_id,
769 uses,
770 with,
771 condvar,
772 permissions,
773 } => {
774 for permission in permissions {
775 if let Some(permission_map) = gh_permissions.get(&node_handle) {
776 if let Some(permission_value) = permission_map.get(&permission.0) {
777 if *permission_value != permission.1 {
778 anyhow::bail!(
779 "permission mismatch for {:?}: expected {:?}, got {:?}",
780 permission.0,
781 permission.1,
782 permission_value
783 )
784 }
785 }
786 } else {
787 anyhow::bail!(
788 "permission missing for {:?}: expected {:?}",
789 permission.0,
790 permission.1
791 )
792 }
793 }
794
795 for gh_var_state in rust_to_gh {
796 let set_gh_env_var = var_db
797 .write_to_gh_env(&gh_var_state.backing_var, &gh_var_state.raw_name)
798 .raw_string(!gh_var_state.is_object)
799 .condvar(condvar.as_deref());
800
801 bash_commands.push_minor(format!("{set_gh_env_var}\n"));
802 }
803
804 if !uses.is_empty() {
805 if let Some(condvar) = &condvar {
806 // guaranteed to be a bare bool `true`/`false`, hence
807 // is_raw_string = false
808 let set_condvar = var_db.write_to_gh_env(condvar, "FLOWEY_CONDITION");
809 bash_commands.push_minor(format!("{set_condvar}\n"));
810 }
811
812 let mut map = serde_yaml::Mapping::new();
813 map.insert("id".into(), serde_yaml::Value::String(step_id.clone()));
814 map.insert("uses".into(), serde_yaml::Value::String(uses));
815 if !with.is_empty() {
816 let mut with_map = serde_yaml::Mapping::new();
817 for (k, v) in with {
818 with_map.insert(k.into(), v.into());
819 }
820 map.insert("with".into(), with_map.into());
821 }
822 map.insert("name".into(), label.into());
823 if condvar.is_some() {
824 map.insert("if".into(), "${{ fromJSON(env.FLOWEY_CONDITION) }}".into());
825 }
826
827 let step: serde_yaml::Value = map.into();
828 output_steps.extend(bash_commands.flush());
829 output_steps.push(step);
830 }
831
832 for gh_var_state in gh_to_rust {
833 let value = if gh_var_state.is_object {
834 format!(r#"${{{{ toJSON({}) }}}}"#, gh_var_state.raw_name)
835 } else {
836 format!(r#"${{{{ {} }}}}"#, gh_var_state.raw_name)
837 };
838
839 let write_var = var_db
840 .update_from_stdin(&gh_var_state.backing_var, gh_var_state.is_secret)
841 .raw_string(!gh_var_state.is_object)
842 .condvar(condvar.as_deref())
843 .env_source(Some(&gh_var_state.raw_name));
844
845 let cmd = format!("{write_var} <<EOF\n{value}\nEOF",);
846 bash_commands.push_minor(cmd);
847 }
848 }
849 }
850 }
851
852 output_steps.extend(bash_commands.flush());
853
854 let request_db = request_db
855 .into_iter()
856 .map(|(node_handle, reqs)| {
857 (
858 node_handle.modpath().to_owned(),
859 reqs.into_iter()
860 .map(crate::cli::exec_snippet::SerializedRequest)
861 .collect(),
862 )
863 })
864 .collect();
865
866 Ok((output_steps, request_db))
867}
868