GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/openvmm

Public

mirrored from https://github.com/microsoft/openvmmAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
e0d27ae138ff5339c3640235f7b8564e3ff07c9d

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openvmm/flowey/flowey_cli/src/pipeline_resolver/github_yaml

flowey/flowey_cli/src/pipeline_resolver/github_yaml/mod.rs

872lines · modecode

RawDownload
flowey: implement nix shell command wrapper (#2825)5e39667
1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4//! Code for emitting a pipeline as a single self-contained GitHub Actions yaml file
5
6use super::common_yaml::BashCommands;
7use super::common_yaml::check_generated_yaml_and_json;
8use super::common_yaml::write_generated_yaml_and_json;
9use super::generic::ResolvedJobArtifact;
10use super::generic::ResolvedJobUseParameter;
11use crate::cli::exec_snippet::FloweyPipelineStaticDb;
12use crate::cli::exec_snippet::VAR_DB_SEEDVAR_FLOWEY_WORKING_DIR;
13use crate::cli::pipeline::CheckMode;
14use crate::cli::var_db::VarDbRequestBuilder;
15use crate::flow_resolver::stage1_dag::OutputGraphEntry;
16use crate::flow_resolver::stage1_dag::Step;
17use crate::pipeline_resolver::common_yaml::FloweySource;
18use crate::pipeline_resolver::common_yaml::job_flowey_bootstrap_source;
19use crate::pipeline_resolver::generic::ResolvedPipeline;
20use crate::pipeline_resolver::generic::ResolvedPipelineJob;
21use anyhow::Context;
22use flowey_core::node::FlowArch;
23use flowey_core::node::FlowBackend;
24use flowey_core::node::FlowPlatform;
25use flowey_core::node::FlowPlatformKind;
26use flowey_core::node::NodeHandle;
27use flowey_core::node::user_facing::GhPermission;
28use flowey_core::node::user_facing::GhPermissionValue;
29use flowey_core::pipeline::GhRunner;
30use flowey_core::pipeline::GhRunnerOsLabel;
31use std::collections::BTreeMap;
32use std::collections::BTreeSet;
33use std::fmt::Write;
34use std::path::Path;
35mod github_yaml_defs;
36
37const RUNNER_TEMP: &str = "${{ runner.temp }}";
38
39/// Emit a pipeline as a single self-contained GitHub Actions yaml file
40pub fn github_yaml(
41 pipeline: ResolvedPipeline,
42 runtime_debug_log: bool,
43 repo_root: &Path,
44 pipeline_file: &Path,
45 flowey_crate: &str,
46 check: CheckMode,
47) -> anyhow::Result<()> {
48 if pipeline_file.extension().and_then(|s| s.to_str()) != Some("yaml") {
49 anyhow::bail!("pipeline name must end with .yaml")
50 }
51
52 let ResolvedPipeline {
53 graph,
54 order,
55 gh_name,
56 gh_schedule_triggers,
57 gh_ci_triggers,
58 gh_pr_triggers,
59 gh_bootstrap_template,
60 parameters,
61 ado_name: _,
62 ado_schedule_triggers: _,
63 ado_ci_triggers: _,
64 ado_pr_triggers: _,
65 ado_bootstrap_template: _,
66 ado_resources_repository: _,
67 ado_post_process_yaml_cb: _,
68 ado_variables: _,
69 ado_job_id_overrides: _,
70 } = pipeline;
71
72 let mut job_flowey_source: BTreeMap<petgraph::prelude::NodeIndex, FloweySource> =
73 job_flowey_bootstrap_source(&graph, &order);
74
75 let mut pipeline_static_db = FloweyPipelineStaticDb {
76 flow_backend: crate::cli::FlowBackendCli::Github,
77 var_db_backend_kind: crate::cli::exec_snippet::VarDbBackendKind::Json,
78 job_reqs: BTreeMap::new(),
79 job_command_wrappers: BTreeMap::new(),
80 };
81
82 let mut github_jobs = BTreeMap::new();
83
84 for job_idx in order {
85 let ResolvedPipelineJob {
86 ref root_nodes,
87 ref patches,
88 ref label,
89 platform,
90 arch,
91 ref external_read_vars,
92 ado_pool: _,
93 timeout_minutes,
94 command_wrapper: ref command_wrapper_kind,
95 ref gh_override_if,
96 ref gh_global_env,
97 ref gh_pool,
98 ref gh_permissions,
99 cond_param_idx,
100 ref parameters_used,
101 ref artifacts_used,
102 ref artifacts_published,
103 ado_variables: _,
104 } = graph[job_idx];
105
106 if cond_param_idx.is_some() {
107 anyhow::bail!(
108 "conditional params are not supported in GitHub backend, use `gh_dangerous_override_if` instead"
109 );
110 }
111
112 let flowey_bin = platform.binary("flowey");
113 let (steps, req_db) = resolve_flow_as_github_yaml_steps(
114 root_nodes
115 .clone()
116 .into_iter()
117 .map(|(node, requests)| (node, (true, requests)))
118 .collect(),
119 patches.clone(),
120 external_read_vars.clone(),
121 platform,
122 arch,
123 job_idx.index(),
124 &flowey_bin,
125 gh_permissions,
126 )
127 .context(format!("in job '{label}'"))?;
128
129 {
130 let existing = pipeline_static_db.job_reqs.insert(job_idx.index(), req_db);
131 assert!(existing.is_none())
132 }
133
134 if let Some(wrapper_kind) = command_wrapper_kind {
135 pipeline_static_db
136 .job_command_wrappers
137 .insert(job_idx.index(), wrapper_kind.clone());
138 }
139
140 let mut gh_steps = Vec::new();
141
142 let flowey_source = job_flowey_source.remove(&job_idx).unwrap();
143
144 let mut artifact_names = Vec::new();
145
146 let flowey_path = match &flowey_source {
147 FloweySource::Bootstrap { .. } => {
148 let flowey_path = "bootstrapped-flowey".to_string();
149
150 // actual artifact publish happens at the end of the job
151 if gh_bootstrap_template.is_empty() {
152 anyhow::bail!(
153 "Did not specify flowey bootstrap template. Please provide one using `Pipeline::gh_set_flowey_bootstrap_template`"
154 )
155 }
156
157 let gh_bootstrap_template = gh_bootstrap_template
158 .replace("{{FLOWEY_BIN_EXTENSION}}", platform.exe_suffix())
159 .replace("{{FLOWEY_CRATE}}", flowey_crate)
160 .replace(
161 "{{FLOWEY_PIPELINE_PATH}}",
162 &pipeline_file.with_extension("").display().to_string(),
163 )
164 .replace(
165 "{{FLOWEY_TARGET}}",
166 match (platform, arch) {
167 (FlowPlatform::Windows, FlowArch::X86_64) => "x86_64-pc-windows-msvc",
168 (FlowPlatform::Windows, FlowArch::Aarch64) => "aarch64-pc-windows-msvc",
169 (FlowPlatform::Linux(_), FlowArch::X86_64) => {
170 "x86_64-unknown-linux-gnu"
171 }
172 (FlowPlatform::Linux(_), FlowArch::Aarch64) => {
173 "aarch64-unknown-linux-gnu"
174 }
175 (platform, arch) => {
176 anyhow::bail!("unsupported platform {platform} / arch {arch}")
177 }
178 },
179 )
180 .replace("{{FLOWEY_OUTDIR}}", &format!("{RUNNER_TEMP}/{flowey_path}"));
181
182 let bootstrap_steps: serde_yaml::Sequence =
183 serde_yaml::from_str(&gh_bootstrap_template)
184 .context("malformed flowey bootstrap template")?;
185
186 gh_steps.extend(bootstrap_steps);
187 flowey_path
188 }
189 FloweySource::Consume(artifact) => {
190 // download previously bootstrapped flowey
191 artifact_names.push(artifact.as_str());
192 format!("used_artifacts/{artifact}")
193 }
194 };
195
196 // download any artifacts that'll be used
197 artifact_names.extend(artifacts_used.iter().map(|a| a.name.as_str()));
198 if !artifact_names.is_empty() {
199 let pattern = if let &[name] = artifact_names.as_slice() {
200 name.to_string()
201 } else {
202 format!("{{{}}}", artifact_names.join(","))
203 };
204 gh_steps.push({
205 let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
206 r#"
207 name: '🌼📦 Download artifacts'
208 uses: actions/download-artifact@v4
209 with:
210 pattern: '{pattern}'
211 path: {RUNNER_TEMP}/used_artifacts/
212 "#
213 ))
214 .unwrap();
215 map.into()
216 });
217 }
218
219 {
220 let mut map = serde_yaml::Mapping::new();
221 map.insert(
222 "run".into(),
223 format!(r#"echo "{RUNNER_TEMP}/{flowey_path}" >> $GITHUB_PATH"#).into(),
224 );
225 map.insert("shell".into(), "bash".into());
226 map.insert("name".into(), "🌼📦 Add flowey to PATH".into());
227 gh_steps.push(map.into());
228 }
229
230 let var_db = VarDbRequestBuilder::new(&flowey_bin, job_idx.index());
231
232 let bootstrap_bash_var_db_inject = |var, is_raw_string| {
233 var_db
234 .update_from_stdin(var, false)
235 .raw_string(is_raw_string)
236 .to_string()
237 };
238
239 // if this was a bootstrap job, also take a moment to run a "self check"
240 // to make sure that the current checked-in template matches the one it
241 // expected
242 if let FloweySource::Bootstrap(..) = &flowey_source {
243 let mut current_invocation = std::env::args().collect::<Vec<_>>();
244
245 current_invocation[0] = flowey_bin.clone();
246
247 // if this code path is run while generating the YAML to compare the
248 // check against, we want to remove the --runtime or --check param from the
249 // current call, or else there'll be a dupe
250 let mut strip_parameter = |prefix: &str| {
251 if let Some(i) = current_invocation
252 .iter()
253 .position(|s| s.starts_with(prefix))
254 {
255 current_invocation.remove(i);
256 if !current_invocation[i].starts_with(prefix) {
257 current_invocation.remove(i);
258 }
259 }
260 };
261
262 strip_parameter("--runtime");
263 strip_parameter("--check");
264
265 // insert the --check bit of the call alongside the --out param
266 {
267 let i = current_invocation
268 .iter()
269 .position(|s| s.starts_with("--out"))
270 .unwrap();
271
272 let current_yaml = match platform.kind() {
273 FlowPlatformKind::Windows => {
274 let win_path = flowey_path.replace('/', "\\");
275 format!(r#"$ESCAPED_AGENT_TEMPDIR\\{win_path}\\pipeline.yaml"#)
276 }
277 FlowPlatformKind::Unix => {
278 format!(r#"$ESCAPED_AGENT_TEMPDIR/{flowey_path}/pipeline.yaml"#)
279 }
280 };
281
282 current_invocation.insert(i, current_yaml);
283 current_invocation.insert(i, "--runtime".into());
284 }
285
286 // Need to use an escaped version of the "true" windows/linux path
287 // here, or else the --check will fail.
288 let cmd = format!(
289 r###"
290ESCAPED_AGENT_TEMPDIR=$(
291cat <<'EOF' | sed 's/\\/\\\\/g'
292{RUNNER_TEMP}
293EOF
294)
295{}
296"###,
297 current_invocation.join(" ")
298 );
299
300 gh_steps.push({
301 let mut map = serde_yaml::Mapping::new();
302 map.insert("name".into(), "🌼🔎 Self-check YAML".into());
303 map.insert(
304 "run".into(),
305 serde_yaml::Value::String(cmd.trim().to_string()),
306 );
307 map.insert("shell".into(), "bash".into());
308 map.into()
309 })
310 }
311
312 let mut flowey_bootstrap_bash = String::new();
313
314 // and now use those vars to do some flowey bootstrap
315 writeln!(flowey_bootstrap_bash, "{}", {
316 let runtime_debug_level = if runtime_debug_log { "debug" } else { "info" };
317
318 let var_db_insert_runtime_debug_level =
319 bootstrap_bash_var_db_inject("FLOWEY_LOG", false);
320 let var_db_insert_working_dir =
321 bootstrap_bash_var_db_inject(VAR_DB_SEEDVAR_FLOWEY_WORKING_DIR, true);
322
323 // Need to use "normalized" path in cases where the path is being
324 // used directly from a bash context, as is the case when we are
325 // trying to invoke `flowey.exe` in argv0 position)
326 //
327 // https://github.com/microsoft/azure-pipelines-tasks/issues/10653#issuecomment-585669089
328 format!(
329 r###"
330AgentTempDirNormal="{RUNNER_TEMP}"
331AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|')
332echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV
333
334chmod +x $AgentTempDirNormal/{flowey_path}/{flowey_bin}
335
336echo '"{runtime_debug_level}"' | {var_db_insert_runtime_debug_level}
337echo "{RUNNER_TEMP}/work" | {var_db_insert_working_dir}
338"###
339 )
340 .trim_start()
341 .to_owned()
342 })?;
343
344 // import pipeline vars being used by the job into flowey
345 for ResolvedJobUseParameter {
346 flowey_var,
347 pipeline_param_idx,
348 } in parameters_used
349 {
350 let is_string = matches!(
351 parameters[*pipeline_param_idx],
352 flowey_core::pipeline::internal::Parameter::String { .. }
353 );
354
355 let default = match &parameters[*pipeline_param_idx] {
356 flowey_core::pipeline::internal::Parameter::Bool { default, .. } => {
357 default.map(|b| b.to_string())
358 }
359 flowey_core::pipeline::internal::Parameter::String { default, .. } => {
360 default.clone()
361 }
362 flowey_core::pipeline::internal::Parameter::Num { default, .. } => {
363 default.map(|n| n.to_string())
364 }
365 }
366 .expect("defaults are currently required for parameters in Github backend");
367
368 let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, is_string);
369
370 let name = parameters[*pipeline_param_idx].name();
371
372 let cmd = format!(
373 r#"
374cat <<'EOF' | {var_db_inject_cmd}
375${{{{ inputs.{name} != '' && inputs.{name} || '{default}' }}}}
376EOF
377"#
378 )
379 .trim()
380 .to_string();
381 writeln!(flowey_bootstrap_bash, "{}", cmd)?;
382 }
383
384 // next, emit GitHub steps to create dirs for artifacts which will be
385 // published
386 for ResolvedJobArtifact { flowey_var, name } in artifacts_published {
387 writeln!(
388 flowey_bootstrap_bash,
389 r#"mkdir -p "$AgentTempDirNormal/publish_artifacts/{name}""#
390 )?;
391 let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, true);
392 match platform.kind() {
393 FlowPlatformKind::Windows => {
394 writeln!(
395 flowey_bootstrap_bash,
396 r#"echo "{RUNNER_TEMP}\\publish_artifacts\\{name}" | {var_db_inject_cmd}"#,
397 )?;
398 }
399 FlowPlatformKind::Unix => {
400 writeln!(
401 flowey_bootstrap_bash,
402 r#"echo "$AgentTempDirNormal/publish_artifacts/{name}" | {var_db_inject_cmd}"#,
403 )?;
404 }
405 }
406 }
407
408 // lastly, emit GitHub steps that report the dirs for any artifacts which
409 // are used by this job
410 for ResolvedJobArtifact { flowey_var, name } in artifacts_used {
411 let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, true);
412 match platform.kind() {
413 FlowPlatformKind::Windows => {
414 writeln!(
415 flowey_bootstrap_bash,
416 r#"echo "{RUNNER_TEMP}\\used_artifacts\\{name}" | {var_db_inject_cmd}"#,
417 )?;
418 }
419 FlowPlatformKind::Unix => {
420 writeln!(
421 flowey_bootstrap_bash,
422 r#"echo "$AgentTempDirNormal/used_artifacts/{name}" | {var_db_inject_cmd}"#,
423 )?;
424 }
425 }
426 }
427
428 gh_steps.push({
429 let mut map = serde_yaml::Mapping::new();
430 map.insert("name".into(), "🌼🛫 Initialize job".into());
431 map.insert(
432 "run".into(),
433 serde_yaml::Value::String(flowey_bootstrap_bash),
434 );
435 map.insert("shell".into(), "bash".into());
436 map.into()
437 });
438
439 // now that we've done all the job-level bootstrapping, we can emit all
440 // the actual steps the user cares about
441 gh_steps.extend(steps);
442
443 // ..and once that's done, the last order of business is to emit some
444 // GitHub steps to publish the various artifacts created by this job
445 for ResolvedJobArtifact {
446 flowey_var: _,
447 name,
448 } in artifacts_published
449 {
450 gh_steps.push({
451 let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
452 r#"
453 name: 🌼📦 Publish {name}
454 uses: actions/upload-artifact@v4
455 with:
456 name: {name}
457 path: {RUNNER_TEMP}/publish_artifacts/{name}/
458 include-hidden-files: true
459 "#
460 ))
461 .unwrap();
462 map.into()
463 });
464 }
465
466 // also, if this job also bootstrapped flowey that other nodes depend
467 // on, make sure to publish it!
468 if let FloweySource::Bootstrap(artifact, true) = flowey_source {
469 // don't leak the bootstrap job's runtime var db
470 gh_steps.push({
471 let mut map = serde_yaml::Mapping::new();
472 map.insert("name".into(), "🌼🧼 Redact bootstrap var db".into());
473 map.insert(
474 "run".into(),
475 serde_yaml::Value::String(format!(
476 "rm $AgentTempDirNormal/{flowey_path}/job{}.json",
477 job_idx.index()
478 )),
479 );
480 map.insert("shell".into(), "bash".into());
481 map.into()
482 });
483
484 gh_steps.push({
485 let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
486 r#"
487 name: 🌼🥾 Publish bootstrapped flowey
488 uses: actions/upload-artifact@v4
489 with:
490 name: {artifact}
491 path: {RUNNER_TEMP}/{flowey_path}
492 "#
493 ))
494 .unwrap();
495 map.into()
496 });
497 }
498
499 let runner_kind_to_yaml = |runner: &GhRunner| match runner {
500 GhRunner::GhHosted(s) => github_yaml_defs::Runner::GhHosted(match s {
501 GhRunnerOsLabel::UbuntuLatest => github_yaml_defs::RunnerOsLabel::UbuntuLatest,
502 GhRunnerOsLabel::Ubuntu2404 => github_yaml_defs::RunnerOsLabel::Ubuntu2404,
503 GhRunnerOsLabel::Ubuntu2204 => github_yaml_defs::RunnerOsLabel::Ubuntu2204,
504 GhRunnerOsLabel::WindowsLatest => github_yaml_defs::RunnerOsLabel::WindowsLatest,
505 GhRunnerOsLabel::Windows2025 => github_yaml_defs::RunnerOsLabel::Windows2025,
506 GhRunnerOsLabel::Windows2022 => github_yaml_defs::RunnerOsLabel::Windows2022,
507 GhRunnerOsLabel::Ubuntu2404Arm => github_yaml_defs::RunnerOsLabel::Ubuntu2404Arm,
508 GhRunnerOsLabel::Ubuntu2204Arm => github_yaml_defs::RunnerOsLabel::Ubuntu2204Arm,
509 GhRunnerOsLabel::Windows11Arm => github_yaml_defs::RunnerOsLabel::Windows11Arm,
510 GhRunnerOsLabel::Custom(s) => github_yaml_defs::RunnerOsLabel::Custom(s.into()),
511 }),
512 GhRunner::SelfHosted(v) => github_yaml_defs::Runner::SelfHosted(v.clone()),
513 GhRunner::RunnerGroup { group, labels } => github_yaml_defs::Runner::Group {
514 group: group.into(),
515 labels: labels.clone(),
516 },
517 };
518
519 let perm_val_to_yaml = |permission_value: &GhPermissionValue| match permission_value {
520 GhPermissionValue::Read => github_yaml_defs::PermissionValue::Read,
521 GhPermissionValue::Write => github_yaml_defs::PermissionValue::Write,
522 GhPermissionValue::None => github_yaml_defs::PermissionValue::None,
523 };
524
525 let perm_kind_to_yaml = |permission: &GhPermission| match permission {
526 GhPermission::Actions => github_yaml_defs::Permissions::Actions,
527 GhPermission::Attestations => github_yaml_defs::Permissions::Attestations,
528 GhPermission::Checks => github_yaml_defs::Permissions::Checks,
529 GhPermission::Contents => github_yaml_defs::Permissions::Contents,
530 GhPermission::Deployments => github_yaml_defs::Permissions::Deployments,
531 GhPermission::Discussions => github_yaml_defs::Permissions::Discussions,
532 GhPermission::IdToken => github_yaml_defs::Permissions::IdToken,
533 GhPermission::Issues => github_yaml_defs::Permissions::Issues,
534 GhPermission::Packages => github_yaml_defs::Permissions::Packages,
535 GhPermission::Pages => github_yaml_defs::Permissions::Pages,
536 GhPermission::PullRequests => github_yaml_defs::Permissions::PullRequests,
537 GhPermission::RepositoryProjects => github_yaml_defs::Permissions::RepositoryProjects,
538 GhPermission::SecurityEvents => github_yaml_defs::Permissions::SecurityEvents,
539 GhPermission::Statuses => github_yaml_defs::Permissions::Statuses,
540 };
541
542 let mut job_permissions = BTreeMap::new();
543 for permission_map in gh_permissions.values() {
544 for (permission, value) in permission_map {
545 // Use the most permissible value set (this allows individual
546 // jobs to override the value set in inject_all_jobs_with)
547 if job_permissions
548 .get(permission)
549 .is_none_or(|old_value| *old_value < *value)
550 {
551 job_permissions.insert(permission.clone(), value.clone());
552 }
553 }
554 }
555
556 github_jobs.insert(
557 format!("job{}", job_idx.index()),
558 github_yaml_defs::Job {
559 name: label.clone(),
560 timeout_minutes,
561 runs_on: gh_pool.clone().map(|runner| runner_kind_to_yaml(&runner)),
562 permissions: job_permissions
563 .iter()
564 .map(|k| (perm_kind_to_yaml(k.0), perm_val_to_yaml(k.1)))
565 .collect(),
566 needs: {
567 graph
568 .edges_directed(job_idx, petgraph::Direction::Incoming)
569 .map(|e| {
570 use petgraph::prelude::*;
571 format!("job{}", e.source().index())
572 })
573 .collect()
574 },
575 r#if: gh_override_if
576 .clone()
577 .or_else(|| Some("github.event.pull_request.draft == false".to_string())),
578 env: gh_global_env.clone(),
579 steps: gh_steps,
580 },
581 );
582 }
583
584 let mut concurrency = None;
585 let pipeline_trigger = github_yaml_defs::Triggers {
586 workflow_call: None,
587 workflow_dispatch: Some(github_yaml_defs::WorkflowDispatch {
588 inputs: github_yaml_defs::Inputs {
589 inputs: parameters
590 .into_iter()
591 .map(|param| {
592 (
593 param.name().to_string(),
594 match param {
595 flowey_core::pipeline::internal::Parameter::Bool {
596 name: _,
597 description,
598 kind: _,
599 default,
600 } => github_yaml_defs::Input {
601 description: Some(description.clone()),
602 default: default.map(github_yaml_defs::Default::Boolean),
603 required: default.is_none(),
604 ty: github_yaml_defs::InputType::Boolean,
605 },
606 flowey_core::pipeline::internal::Parameter::String {
607 name: _,
608 description,
609 kind: _,
610 default,
611 possible_values: _,
612 } => github_yaml_defs::Input {
613 description: Some(description.clone()),
614 default: default
615 .as_ref()
616 .map(|s| github_yaml_defs::Default::String(s.clone())),
617 required: default.is_none(),
618 ty: github_yaml_defs::InputType::String,
619 },
620 flowey_core::pipeline::internal::Parameter::Num {
621 name: _,
622 description,
623 kind: _,
624 default,
625 possible_values: _,
626 } => github_yaml_defs::Input {
627 description: Some(description.clone()),
628 default: default.map(github_yaml_defs::Default::Number),
629 required: default.is_none(),
630 ty: github_yaml_defs::InputType::Number,
631 },
632 },
633 )
634 })
635 .collect::<BTreeMap<String, github_yaml_defs::Input>>(),
636 },
637 }),
638 pull_request: match gh_pr_triggers {
639 Some(gh_pr_triggers) => {
640 if gh_pr_triggers.auto_cancel {
641 concurrency = Some(github_yaml_defs::Concurrency {
642 // only cancel in-progress jobs for the same workflow and branch
643 group: Some("${{ github.workflow }}-${{ github.ref }}".to_string()),
644 cancel_in_progress: Some(true),
645 })
646 };
647 Some(github_yaml_defs::PrTrigger {
648 branches: gh_pr_triggers.branches.clone(),
649 branches_ignore: gh_pr_triggers.exclude_branches.clone(),
650 types: gh_pr_triggers.types.clone(),
651 })
652 }
653 None => None,
654 },
655 push: match gh_ci_triggers {
656 Some(gh_ci_triggers) => Some(github_yaml_defs::CiTrigger {
657 branches: gh_ci_triggers.branches,
658 branches_ignore: gh_ci_triggers.exclude_branches,
659 tags: gh_ci_triggers.tags,
660 tags_ignore: gh_ci_triggers.exclude_tags,
661 }),
662 None => None,
663 },
664 schedule: gh_schedule_triggers
665 .iter()
666 .map(|s| github_yaml_defs::Cron {
667 cron: s.cron.clone(),
668 })
669 .collect(),
670 };
671
672 let github_pipeline = github_yaml_defs::Pipeline {
673 name: gh_name,
674 on: Some(pipeline_trigger),
675 concurrency,
676 jobs: Some(github_yaml_defs::Jobs { jobs: github_jobs }),
677 inputs: None,
678 };
679
680 match check {
681 CheckMode::Check(_) | CheckMode::Runtime(_) => check_generated_yaml_and_json(
682 &github_pipeline,
683 &pipeline_static_db,
684 check,
685 repo_root,
686 pipeline_file,
687 None,
688 ),
689 CheckMode::None => write_generated_yaml_and_json(
690 &github_pipeline,
691 &pipeline_static_db,
692 repo_root,
693 pipeline_file,
694 None,
695 ),
696 }
697}
698
699/// Resolve a flow as a sequence of GitHub YAML steps.
700///
701/// These steps can then be marshalled into a well-formed GitHub pipeline yaml
702/// using a separate GitHub pipeline yaml builder
703// pub(crate) so that internal debug CLI tooling can use it
704fn resolve_flow_as_github_yaml_steps(
705 seed_nodes: BTreeMap<NodeHandle, (bool, Vec<Box<[u8]>>)>,
706 resolved_patches: flowey_core::patch::ResolvedPatches,
707 external_read_vars: BTreeSet<String>,
708 platform: FlowPlatform,
709 arch: FlowArch,
710 job_idx: usize,
711 flowey_bin: &str,
712 gh_permissions: &BTreeMap<NodeHandle, BTreeMap<GhPermission, GhPermissionValue>>,
713) -> anyhow::Result<(
714 Vec<serde_yaml::Value>,
715 BTreeMap<String, Vec<crate::cli::exec_snippet::SerializedRequest>>,
716)> {
717 let mut output_steps = Vec::new();
718
719 let (mut output_graph, request_db, err_unreachable_nodes) =
720 crate::flow_resolver::stage1_dag::stage1_dag(
721 FlowBackend::Github,
722 platform,
723 arch,
724 resolved_patches,
725 seed_nodes,
726 external_read_vars,
727 // TODO: support GitHub agents with persistent storage
728 None,
729 )?;
730
731 if err_unreachable_nodes.is_some() {
732 anyhow::bail!("detected unreachable nodes")
733 }
734
735 let mut bash_commands = BashCommands::new_github();
736
737 let output_order = petgraph::algo::toposort(&output_graph, None)
738 .expect("runtime variables cannot introduce a DAG cycle");
739
740 let var_db = VarDbRequestBuilder::new(flowey_bin, job_idx);
741
742 for node_idx in output_order.into_iter().rev() {
743 let OutputGraphEntry { node_handle, step } = output_graph[node_idx].1.take().unwrap();
744
745 let node_modpath = node_handle.modpath();
746
747 match step {
748 Step::Anchor { .. } => {}
749 Step::Rust {
750 idx,
751 label,
752 can_merge,
753 code: _,
754 } => {
755 output_steps.extend(bash_commands.push(
756 Some(label),
757 can_merge,
758 crate::cli::exec_snippet::construct_exec_snippet_cli(
759 flowey_bin,
760 node_modpath,
761 idx,
762 job_idx,
763 ),
764 ));
765 }
766 Step::AdoYaml { label, .. } => {
767 anyhow::bail!("ADO YAML not supported in GitHub. In step '{}'", label)
768 }
769 Step::GitHubYaml {
770 gh_to_rust,
771 rust_to_gh,
772 label,
773 step_id,
774 uses,
775 with,
776 condvar,
777 permissions,
778 } => {
779 for permission in permissions {
780 if let Some(permission_map) = gh_permissions.get(&node_handle) {
781 if let Some(permission_value) = permission_map.get(&permission.0) {
782 if *permission_value != permission.1 {
783 anyhow::bail!(
784 "permission mismatch for {:?}: expected {:?}, got {:?}",
785 permission.0,
786 permission.1,
787 permission_value
788 )
789 }
790 }
791 } else {
792 anyhow::bail!(
793 "permission missing for {:?}: expected {:?}",
794 permission.0,
795 permission.1
796 )
797 }
798 }
799
800 for gh_var_state in rust_to_gh {
801 let set_gh_env_var = var_db
802 .write_to_gh_env(&gh_var_state.backing_var, &gh_var_state.raw_name)
803 .raw_string(!gh_var_state.is_object)
804 .condvar(condvar.as_deref());
805
806 bash_commands.push_minor(format!("{set_gh_env_var}\n"));
807 }
808
809 if !uses.is_empty() {
810 if let Some(condvar) = &condvar {
811 // guaranteed to be a bare bool `true`/`false`, hence
812 // is_raw_string = false
813 let set_condvar = var_db.write_to_gh_env(condvar, "FLOWEY_CONDITION");
814 bash_commands.push_minor(format!("{set_condvar}\n"));
815 }
816
817 let mut map = serde_yaml::Mapping::new();
818 map.insert("id".into(), serde_yaml::Value::String(step_id.clone()));
819 map.insert("uses".into(), serde_yaml::Value::String(uses));
820 if !with.is_empty() {
821 let mut with_map = serde_yaml::Mapping::new();
822 for (k, v) in with {
823 with_map.insert(k.into(), v.into());
824 }
825 map.insert("with".into(), with_map.into());
826 }
827 map.insert("name".into(), label.into());
828 if condvar.is_some() {
829 map.insert("if".into(), "${{ fromJSON(env.FLOWEY_CONDITION) }}".into());
830 }
831
832 let step: serde_yaml::Value = map.into();
833 output_steps.extend(bash_commands.flush());
834 output_steps.push(step);
835 }
836
837 for gh_var_state in gh_to_rust {
838 let value = if gh_var_state.is_object {
839 format!(r#"${{{{ toJSON({}) }}}}"#, gh_var_state.raw_name)
840 } else {
841 format!(r#"${{{{ {} }}}}"#, gh_var_state.raw_name)
842 };
843
844 let write_var = var_db
845 .update_from_stdin(&gh_var_state.backing_var, gh_var_state.is_secret)
846 .raw_string(!gh_var_state.is_object)
847 .condvar(condvar.as_deref())
848 .env_source(Some(&gh_var_state.raw_name));
849
850 let cmd = format!("{write_var} <<EOF\n{value}\nEOF",);
851 bash_commands.push_minor(cmd);
852 }
853 }
854 }
855 }
856
857 output_steps.extend(bash_commands.flush());
858
859 let request_db = request_db
860 .into_iter()
861 .map(|(node_handle, reqs)| {
862 (
863 node_handle.modpath().to_owned(),
864 reqs.into_iter()
865 .map(crate::cli::exec_snippet::SerializedRequest)
866 .collect(),
867 )
868 })
869 .collect();
870
871 Ok((output_steps, request_db))
872}