GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/openvmm

Public

mirrored from https://github.com/microsoft/openvmmAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
copilot/investigate-firmware-packaging-issues

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openvmm/flowey/flowey_cli/src/pipeline_resolver/github_yaml

flowey/flowey_cli/src/pipeline_resolver/github_yaml/mod.rs

958lines · modecode

RawDownload
Latest commit unavailable.
1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4//! Code for emitting a pipeline as a single self-contained GitHub Actions yaml file
5
6use super::common_yaml::BashCommands;
7use super::common_yaml::check_generated_yaml_and_json;
8use super::common_yaml::write_generated_yaml_and_json;
9use super::generic::ResolvedJobArtifact;
10use super::generic::ResolvedJobUseParameter;
11use crate::cli::exec_snippet::FloweyPipelineStaticDb;
12use crate::cli::exec_snippet::VAR_DB_SEEDVAR_FLOWEY_WORKING_DIR;
13use crate::cli::pipeline::CheckMode;
14use crate::cli::var_db::VarDbRequestBuilder;
15use crate::flow_resolver::stage1_dag::OutputGraphEntry;
16use crate::flow_resolver::stage1_dag::Step;
17use crate::pipeline_resolver::common_yaml::FloweySource;
18use crate::pipeline_resolver::common_yaml::job_flowey_bootstrap_source;
19use crate::pipeline_resolver::generic::ResolvedPipeline;
20use crate::pipeline_resolver::generic::ResolvedPipelineJob;
21use anyhow::Context;
22use flowey_core::node::FlowArch;
23use flowey_core::node::FlowBackend;
24use flowey_core::node::FlowPlatform;
25use flowey_core::node::FlowPlatformKind;
26use flowey_core::node::FlowPlatformLinuxDistro;
27use flowey_core::node::NodeHandle;
28use flowey_core::node::user_facing::GhPermission;
29use flowey_core::node::user_facing::GhPermissionValue;
30use flowey_core::pipeline::GhRunner;
31use flowey_core::pipeline::GhRunnerOsLabel;
32use std::collections::BTreeMap;
33use std::collections::BTreeSet;
34use std::fmt::Write;
35use std::path::Path;
36mod github_yaml_defs;
37
38const RUNNER_TEMP: &str = "${{ runner.temp }}";
39
40/// Emit a pipeline as a single self-contained GitHub Actions yaml file
41pub fn github_yaml(
42 pipeline: ResolvedPipeline,
43 runtime_debug_log: bool,
44 repo_root: &Path,
45 pipeline_file: &Path,
46 flowey_crate: &str,
47 check: CheckMode,
48) -> anyhow::Result<()> {
49 if pipeline_file.extension().and_then(|s| s.to_str()) != Some("yaml") {
50 anyhow::bail!("pipeline name must end with .yaml")
51 }
52
53 let ResolvedPipeline {
54 graph,
55 order,
56 gh_name,
57 gh_schedule_triggers,
58 gh_ci_triggers,
59 gh_pr_triggers,
60 gh_bootstrap_template,
61 parameters,
62 ado_name: _,
63 ado_schedule_triggers: _,
64 ado_ci_triggers: _,
65 ado_pr_triggers: _,
66 ado_bootstrap_template: _,
67 ado_resources_repository: _,
68 ado_post_process_yaml_cb: _,
69 ado_variables: _,
70 ado_job_id_overrides: _,
71 } = pipeline;
72
73 let mut job_flowey_source: BTreeMap<petgraph::prelude::NodeIndex, FloweySource> =
74 job_flowey_bootstrap_source(&graph, &order);
75
76 let mut pipeline_static_db = FloweyPipelineStaticDb {
77 flow_backend: crate::cli::FlowBackendCli::Github,
78 var_db_backend_kind: crate::cli::exec_snippet::VarDbBackendKind::Json,
79 job_reqs: BTreeMap::new(),
80 job_configs: BTreeMap::new(),
81 job_command_wrappers: BTreeMap::new(),
82 job_platforms: BTreeMap::new(),
83 job_archs: BTreeMap::new(),
84 };
85
86 let mut github_jobs = BTreeMap::new();
87
88 for job_idx in order {
89 let ResolvedPipelineJob {
90 ref root_nodes,
91 ref root_configs,
92 ref patches,
93 ref label,
94 platform,
95 arch,
96 ref external_read_vars,
97 ado_pool: _,
98 timeout_minutes,
99 command_wrapper: ref command_wrapper_kind,
100 ref gh_override_if,
101 ref gh_global_env,
102 ref gh_pool,
103 ref gh_permissions,
104 cond_param_idx,
105 ref parameters_used,
106 ref artifacts_used,
107 ref artifacts_published,
108 ado_variables: _,
109 } = graph[job_idx];
110
111 if cond_param_idx.is_some() {
112 anyhow::bail!(
113 "conditional params are not supported in GitHub backend, use `gh_dangerous_override_if` instead"
114 );
115 }
116
117 let flowey_bin = platform.binary("flowey");
118 let super::common_yaml::ResolvedFlowSteps {
119 steps,
120 request_db: req_db,
121 config_db: cfg_db,
122 } = resolve_flow_as_github_yaml_steps(
123 root_nodes
124 .clone()
125 .into_iter()
126 .map(|(node, requests)| (node, (true, requests)))
127 .collect(),
128 root_configs.clone(),
129 patches.clone(),
130 external_read_vars.clone(),
131 platform,
132 arch,
133 job_idx.index(),
134 &flowey_bin,
135 gh_permissions,
136 )
137 .context(format!("in job '{label}'"))?;
138
139 {
140 let existing = pipeline_static_db.job_reqs.insert(job_idx.index(), req_db);
141 assert!(existing.is_none())
142 }
143
144 if !cfg_db.is_empty() {
145 pipeline_static_db
146 .job_configs
147 .insert(job_idx.index(), cfg_db);
148 }
149
150 if let Some(wrapper_kind) = command_wrapper_kind {
151 pipeline_static_db
152 .job_command_wrappers
153 .insert(job_idx.index(), wrapper_kind.clone());
154 }
155
156 pipeline_static_db
157 .job_platforms
158 .insert(job_idx.index(), platform);
159 pipeline_static_db.job_archs.insert(job_idx.index(), arch);
160
161 let mut gh_steps = Vec::new();
162
163 let flowey_source = job_flowey_source.remove(&job_idx).unwrap();
164
165 let mut artifact_names = Vec::new();
166
167 let flowey_path = match &flowey_source {
168 FloweySource::Bootstrap { .. } => {
169 let flowey_path = "bootstrapped-flowey".to_string();
170
171 // actual artifact publish happens at the end of the job
172 if gh_bootstrap_template.is_empty() {
173 anyhow::bail!(
174 "Did not specify flowey bootstrap template. Please provide one using `Pipeline::gh_set_flowey_bootstrap_template`"
175 )
176 }
177
178 let gh_bootstrap_template = gh_bootstrap_template
179 .replace("{{FLOWEY_BIN_EXTENSION}}", platform.exe_suffix())
180 .replace("{{FLOWEY_CRATE}}", flowey_crate)
181 .replace(
182 "{{FLOWEY_PIPELINE_PATH}}",
183 &pipeline_file.with_extension("").display().to_string(),
184 )
185 .replace(
186 "{{FLOWEY_TARGET}}",
187 match (platform, arch) {
188 (FlowPlatform::Windows, FlowArch::X86_64) => "x86_64-pc-windows-msvc",
189 (FlowPlatform::Windows, FlowArch::Aarch64) => "aarch64-pc-windows-msvc",
190 (FlowPlatform::Linux(_), FlowArch::X86_64) => {
191 "x86_64-unknown-linux-gnu"
192 }
193 (FlowPlatform::Linux(_), FlowArch::Aarch64) => {
194 "aarch64-unknown-linux-gnu"
195 }
196 (platform, arch) => {
197 anyhow::bail!("unsupported platform {platform} / arch {arch}")
198 }
199 },
200 )
201 .replace("{{FLOWEY_OUTDIR}}", &format!("{RUNNER_TEMP}/{flowey_path}"))
202 .replace(
203 "{{LINUX_INSTALL_DEPS}}",
204 match platform {
205 FlowPlatform::Linux(FlowPlatformLinuxDistro::AzureLinux) => {
206 "sudo tdnf install -y gcc glibc-devel"
207 }
208 FlowPlatform::Linux(FlowPlatformLinuxDistro::Fedora) => {
209 "sudo dnf install -y gcc"
210 }
211 _ => {
212 "i=0; while [ $i -lt 5 ] && ! sudo apt-get update; do let \"i=i+1\"; sleep 1; done;\n sudo apt-get -o DPkg::Lock::Timeout=60 install gcc -y"
213 }
214 },
215 );
216
217 let bootstrap_steps: serde_yaml::Sequence =
218 serde_yaml::from_str(&gh_bootstrap_template)
219 .context("malformed flowey bootstrap template")?;
220
221 gh_steps.extend(bootstrap_steps);
222 flowey_path
223 }
224 FloweySource::Consume(artifact) => {
225 // download previously bootstrapped flowey
226 artifact_names.push(artifact.as_str());
227 format!("used_artifacts/{artifact}")
228 }
229 };
230
231 // download any artifacts that'll be used
232 artifact_names.extend(artifacts_used.iter().map(|a| a.name.as_str()));
233 if !artifact_names.is_empty() {
234 // When downloading a single artifact by name, the contents are
235 // extracted directly to `path/` without a subdirectory. For
236 // multiple artifacts with `merge-multiple: false` (the default),
237 // named subdirectories are created automatically.
238 //
239 // Use `name:` with an explicit subdirectory path for single
240 // artifacts, and `pattern:` for multiple artifacts.
241 gh_steps.push({
242 let map: serde_yaml::Mapping = if let &[name] = artifact_names.as_slice() {
243 serde_yaml::from_str(&format!(
244 r#"
245 name: '🌼📦 Download artifacts'
246 uses: actions/download-artifact@v8
247 with:
248 name: '{name}'
249 path: {RUNNER_TEMP}/used_artifacts/{name}/
250 "#
251 ))
252 .unwrap()
253 } else {
254 let pattern = format!("{{{}}}", artifact_names.join(","));
255 serde_yaml::from_str(&format!(
256 r#"
257 name: '🌼📦 Download artifacts'
258 uses: actions/download-artifact@v8
259 with:
260 pattern: '{pattern}'
261 path: {RUNNER_TEMP}/used_artifacts/
262 "#
263 ))
264 .unwrap()
265 };
266 map.into()
267 });
268 }
269
270 {
271 let mut map = serde_yaml::Mapping::new();
272 map.insert(
273 "run".into(),
274 format!(r#"echo "{RUNNER_TEMP}/{flowey_path}" >> $GITHUB_PATH"#).into(),
275 );
276 map.insert("shell".into(), "bash".into());
277 map.insert("name".into(), "🌼📦 Add flowey to PATH".into());
278 gh_steps.push(map.into());
279 }
280
281 let var_db = VarDbRequestBuilder::new(&flowey_bin, job_idx.index());
282
283 let bootstrap_bash_var_db_inject = |var, is_raw_string| {
284 var_db
285 .update_from_stdin(var, false)
286 .raw_string(is_raw_string)
287 .to_string()
288 };
289
290 // if this was a bootstrap job, also take a moment to run a "self check"
291 // to make sure that the current checked-in template matches the one it
292 // expected
293 if let FloweySource::Bootstrap(..) = &flowey_source {
294 let mut current_invocation = std::env::args().collect::<Vec<_>>();
295
296 current_invocation[0] = flowey_bin.clone();
297
298 // if this code path is run while generating the YAML to compare the
299 // check against, we want to remove the --runtime or --check param from the
300 // current call, or else there'll be a dupe
301 let mut strip_parameter = |prefix: &str| {
302 if let Some(i) = current_invocation
303 .iter()
304 .position(|s| s.starts_with(prefix))
305 {
306 current_invocation.remove(i);
307 if !current_invocation[i].starts_with(prefix) {
308 current_invocation.remove(i);
309 }
310 }
311 };
312
313 strip_parameter("--runtime");
314 strip_parameter("--check");
315
316 // insert the --check bit of the call alongside the --out param
317 {
318 let i = current_invocation
319 .iter()
320 .position(|s| s.starts_with("--out"))
321 .unwrap();
322
323 let current_yaml = match platform.kind() {
324 FlowPlatformKind::Windows => {
325 let win_path = flowey_path.replace('/', "\\");
326 format!(r#"$ESCAPED_AGENT_TEMPDIR\\{win_path}\\pipeline.yaml"#)
327 }
328 FlowPlatformKind::Unix => {
329 format!(r#"$ESCAPED_AGENT_TEMPDIR/{flowey_path}/pipeline.yaml"#)
330 }
331 };
332
333 current_invocation.insert(i, current_yaml);
334 current_invocation.insert(i, "--runtime".into());
335 }
336
337 // Need to use an escaped version of the "true" windows/linux path
338 // here, or else the --check will fail.
339 let cmd = format!(
340 r###"
341ESCAPED_AGENT_TEMPDIR=$(
342cat <<'EOF' | sed 's/\\/\\\\/g'
343{RUNNER_TEMP}
344EOF
345)
346{}
347"###,
348 current_invocation.join(" ")
349 );
350
351 gh_steps.push({
352 let mut map = serde_yaml::Mapping::new();
353 map.insert("name".into(), "🌼🔎 Self-check YAML".into());
354 map.insert(
355 "run".into(),
356 serde_yaml::Value::String(cmd.trim().to_string()),
357 );
358 map.insert("shell".into(), "bash".into());
359 map.into()
360 })
361 }
362
363 let mut flowey_bootstrap_bash = String::new();
364
365 // and now use those vars to do some flowey bootstrap
366 writeln!(flowey_bootstrap_bash, "{}", {
367 let runtime_debug_level = if runtime_debug_log { "debug" } else { "info" };
368
369 let var_db_insert_runtime_debug_level =
370 bootstrap_bash_var_db_inject("FLOWEY_LOG", false);
371 let var_db_insert_working_dir =
372 bootstrap_bash_var_db_inject(VAR_DB_SEEDVAR_FLOWEY_WORKING_DIR, true);
373
374 // Need to use "normalized" path in cases where the path is being
375 // used directly from a bash context, as is the case when we are
376 // trying to invoke `flowey.exe` in argv0 position)
377 //
378 // https://github.com/microsoft/azure-pipelines-tasks/issues/10653#issuecomment-585669089
379 format!(
380 r###"
381AgentTempDirNormal="{RUNNER_TEMP}"
382AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|')
383echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV
384
385chmod +x $AgentTempDirNormal/{flowey_path}/{flowey_bin}
386
387echo '"{runtime_debug_level}"' | {var_db_insert_runtime_debug_level}
388echo "{RUNNER_TEMP}/work" | {var_db_insert_working_dir}
389"###
390 )
391 .trim_start()
392 .to_owned()
393 })?;
394
395 // import pipeline vars being used by the job into flowey
396 for ResolvedJobUseParameter {
397 flowey_var,
398 pipeline_param_idx,
399 } in parameters_used
400 {
401 let is_string = matches!(
402 parameters[*pipeline_param_idx],
403 flowey_core::pipeline::internal::Parameter::String { .. }
404 );
405
406 let default = match &parameters[*pipeline_param_idx] {
407 flowey_core::pipeline::internal::Parameter::Bool { default, .. } => {
408 default.map(|b| b.to_string())
409 }
410 flowey_core::pipeline::internal::Parameter::String { default, .. } => {
411 default.clone()
412 }
413 flowey_core::pipeline::internal::Parameter::Num { default, .. } => {
414 default.map(|n| n.to_string())
415 }
416 }
417 .expect("defaults are currently required for parameters in Github backend");
418
419 let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, is_string);
420
421 let name = parameters[*pipeline_param_idx].name();
422
423 let cmd = format!(
424 r#"
425cat <<'EOF' | {var_db_inject_cmd}
426${{{{ inputs.{name} != '' && inputs.{name} || '{default}' }}}}
427EOF
428"#
429 )
430 .trim()
431 .to_string();
432 writeln!(flowey_bootstrap_bash, "{}", cmd)?;
433 }
434
435 // next, emit GitHub steps to create dirs for artifacts which will be
436 // published
437 for ResolvedJobArtifact { flowey_var, name } in artifacts_published {
438 writeln!(
439 flowey_bootstrap_bash,
440 r#"mkdir -p "$AgentTempDirNormal/publish_artifacts/{name}""#
441 )?;
442 let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, true);
443 match platform.kind() {
444 FlowPlatformKind::Windows => {
445 writeln!(
446 flowey_bootstrap_bash,
447 r#"echo "{RUNNER_TEMP}\\publish_artifacts\\{name}" | {var_db_inject_cmd}"#,
448 )?;
449 }
450 FlowPlatformKind::Unix => {
451 writeln!(
452 flowey_bootstrap_bash,
453 r#"echo "$AgentTempDirNormal/publish_artifacts/{name}" | {var_db_inject_cmd}"#,
454 )?;
455 }
456 }
457 }
458
459 // lastly, emit GitHub steps that report the dirs for any artifacts which
460 // are used by this job
461 for ResolvedJobArtifact { flowey_var, name } in artifacts_used {
462 let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, true);
463 match platform.kind() {
464 FlowPlatformKind::Windows => {
465 writeln!(
466 flowey_bootstrap_bash,
467 r#"echo "{RUNNER_TEMP}\\used_artifacts\\{name}" | {var_db_inject_cmd}"#,
468 )?;
469 }
470 FlowPlatformKind::Unix => {
471 writeln!(
472 flowey_bootstrap_bash,
473 r#"echo "$AgentTempDirNormal/used_artifacts/{name}" | {var_db_inject_cmd}"#,
474 )?;
475 }
476 }
477 }
478
479 gh_steps.push({
480 let mut map = serde_yaml::Mapping::new();
481 map.insert("name".into(), "🌼🛫 Initialize job".into());
482 map.insert(
483 "run".into(),
484 serde_yaml::Value::String(flowey_bootstrap_bash),
485 );
486 map.insert("shell".into(), "bash".into());
487 map.into()
488 });
489
490 // now that we've done all the job-level bootstrapping, we can emit all
491 // the actual steps the user cares about
492 gh_steps.extend(steps);
493
494 // ..and once that's done, the last order of business is to emit some
495 // GitHub steps to publish the various artifacts created by this job
496 for ResolvedJobArtifact {
497 flowey_var: _,
498 name,
499 } in artifacts_published
500 {
501 gh_steps.push({
502 let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
503 r#"
504 name: 🌼📦 Publish {name}
505 uses: actions/upload-artifact@v7
506 with:
507 name: {name}
508 path: {RUNNER_TEMP}/publish_artifacts/{name}/
509 include-hidden-files: true
510 "#
511 ))
512 .unwrap();
513 map.into()
514 });
515 }
516
517 // also, if this job also bootstrapped flowey that other nodes depend
518 // on, make sure to publish it!
519 if let FloweySource::Bootstrap(artifact, true) = flowey_source {
520 // don't leak the bootstrap job's runtime var db
521 gh_steps.push({
522 let mut map = serde_yaml::Mapping::new();
523 map.insert("name".into(), "🌼🧼 Redact bootstrap var db".into());
524 map.insert(
525 "run".into(),
526 serde_yaml::Value::String(format!(
527 "rm $AgentTempDirNormal/{flowey_path}/job{}.json",
528 job_idx.index()
529 )),
530 );
531 map.insert("shell".into(), "bash".into());
532 map.into()
533 });
534
535 gh_steps.push({
536 let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
537 r#"
538 name: 🌼🥾 Publish bootstrapped flowey
539 uses: actions/upload-artifact@v7
540 with:
541 name: {artifact}
542 path: {RUNNER_TEMP}/{flowey_path}
543 "#
544 ))
545 .unwrap();
546 map.into()
547 });
548 }
549
550 let runner_kind_to_yaml = |runner: &GhRunner| match runner {
551 GhRunner::GhHosted(s) => github_yaml_defs::Runner::GhHosted(match s {
552 GhRunnerOsLabel::UbuntuLatest => github_yaml_defs::RunnerOsLabel::UbuntuLatest,
553 GhRunnerOsLabel::Ubuntu2404 => github_yaml_defs::RunnerOsLabel::Ubuntu2404,
554 GhRunnerOsLabel::Ubuntu2204 => github_yaml_defs::RunnerOsLabel::Ubuntu2204,
555 GhRunnerOsLabel::WindowsLatest => github_yaml_defs::RunnerOsLabel::WindowsLatest,
556 GhRunnerOsLabel::Windows2025 => github_yaml_defs::RunnerOsLabel::Windows2025,
557 GhRunnerOsLabel::Windows2022 => github_yaml_defs::RunnerOsLabel::Windows2022,
558 GhRunnerOsLabel::Ubuntu2404Arm => github_yaml_defs::RunnerOsLabel::Ubuntu2404Arm,
559 GhRunnerOsLabel::Ubuntu2204Arm => github_yaml_defs::RunnerOsLabel::Ubuntu2204Arm,
560 GhRunnerOsLabel::Windows11Arm => github_yaml_defs::RunnerOsLabel::Windows11Arm,
561 GhRunnerOsLabel::Custom(s) => github_yaml_defs::RunnerOsLabel::Custom(s.into()),
562 }),
563 GhRunner::SelfHosted(v) => github_yaml_defs::Runner::SelfHosted(v.clone()),
564 GhRunner::RunnerGroup { group, labels } => github_yaml_defs::Runner::Group {
565 group: group.into(),
566 labels: labels.clone(),
567 },
568 };
569
570 let perm_val_to_yaml = |permission_value: &GhPermissionValue| match permission_value {
571 GhPermissionValue::Read => github_yaml_defs::PermissionValue::Read,
572 GhPermissionValue::Write => github_yaml_defs::PermissionValue::Write,
573 GhPermissionValue::None => github_yaml_defs::PermissionValue::None,
574 };
575
576 let perm_kind_to_yaml = |permission: &GhPermission| match permission {
577 GhPermission::Actions => github_yaml_defs::Permissions::Actions,
578 GhPermission::Attestations => github_yaml_defs::Permissions::Attestations,
579 GhPermission::Checks => github_yaml_defs::Permissions::Checks,
580 GhPermission::Contents => github_yaml_defs::Permissions::Contents,
581 GhPermission::Deployments => github_yaml_defs::Permissions::Deployments,
582 GhPermission::Discussions => github_yaml_defs::Permissions::Discussions,
583 GhPermission::IdToken => github_yaml_defs::Permissions::IdToken,
584 GhPermission::Issues => github_yaml_defs::Permissions::Issues,
585 GhPermission::Packages => github_yaml_defs::Permissions::Packages,
586 GhPermission::Pages => github_yaml_defs::Permissions::Pages,
587 GhPermission::PullRequests => github_yaml_defs::Permissions::PullRequests,
588 GhPermission::RepositoryProjects => github_yaml_defs::Permissions::RepositoryProjects,
589 GhPermission::SecurityEvents => github_yaml_defs::Permissions::SecurityEvents,
590 GhPermission::Statuses => github_yaml_defs::Permissions::Statuses,
591 };
592
593 let mut job_permissions = BTreeMap::new();
594 for permission_map in gh_permissions.values() {
595 for (permission, value) in permission_map {
596 // Use the most permissible value set (this allows individual
597 // jobs to override the value set in inject_all_jobs_with)
598 if job_permissions
599 .get(permission)
600 .is_none_or(|old_value| *old_value < *value)
601 {
602 job_permissions.insert(permission.clone(), value.clone());
603 }
604 }
605 }
606
607 github_jobs.insert(
608 format!("job{}", job_idx.index()),
609 github_yaml_defs::Job {
610 name: label.clone(),
611 timeout_minutes,
612 runs_on: gh_pool.clone().map(|runner| {
613 let mut yaml_runner = runner_kind_to_yaml(&runner);
614 if let github_yaml_defs::Runner::SelfHosted(ref mut labels) = yaml_runner {
615 if labels.iter().any(|l| l.starts_with("1ES.Pool=")) {
616 labels.push(format!(
617 "JobId=job{}-${{{{ github.run_id }}}}-${{{{ github.run_number }}}}-${{{{ github.run_attempt }}}}",
618 job_idx.index()
619 ));
620 }
621 }
622 yaml_runner
623 }),
624 permissions: job_permissions
625 .iter()
626 .map(|k| (perm_kind_to_yaml(k.0), perm_val_to_yaml(k.1)))
627 .collect(),
628 needs: {
629 graph
630 .edges_directed(job_idx, petgraph::Direction::Incoming)
631 .map(|e| {
632 use petgraph::prelude::*;
633 format!("job{}", e.source().index())
634 })
635 .collect()
636 },
637 r#if: gh_override_if
638 .clone()
639 .or_else(|| Some("github.event.pull_request.draft == false".to_string())),
640 env: gh_global_env.clone(),
641 steps: gh_steps,
642 },
643 );
644 }
645
646 let mut concurrency = None;
647 let pipeline_trigger = github_yaml_defs::Triggers {
648 workflow_call: None,
649 workflow_dispatch: Some(github_yaml_defs::WorkflowDispatch {
650 inputs: github_yaml_defs::Inputs {
651 inputs: parameters
652 .into_iter()
653 .map(|param| {
654 (
655 param.name().to_string(),
656 match param {
657 flowey_core::pipeline::internal::Parameter::Bool {
658 name: _,
659 description,
660 kind: _,
661 default,
662 } => github_yaml_defs::Input {
663 description: Some(description.clone()),
664 default: default.map(github_yaml_defs::Default::Boolean),
665 required: default.is_none(),
666 ty: github_yaml_defs::InputType::Boolean,
667 },
668 flowey_core::pipeline::internal::Parameter::String {
669 name: _,
670 description,
671 kind: _,
672 default,
673 possible_values: _,
674 } => github_yaml_defs::Input {
675 description: Some(description.clone()),
676 default: default
677 .as_ref()
678 .map(|s| github_yaml_defs::Default::String(s.clone())),
679 required: default.is_none(),
680 ty: github_yaml_defs::InputType::String,
681 },
682 flowey_core::pipeline::internal::Parameter::Num {
683 name: _,
684 description,
685 kind: _,
686 default,
687 possible_values: _,
688 } => github_yaml_defs::Input {
689 description: Some(description.clone()),
690 default: default.map(github_yaml_defs::Default::Number),
691 required: default.is_none(),
692 ty: github_yaml_defs::InputType::Number,
693 },
694 },
695 )
696 })
697 .collect::<BTreeMap<String, github_yaml_defs::Input>>(),
698 },
699 }),
700 pull_request: match gh_pr_triggers {
701 Some(gh_pr_triggers) => {
702 if gh_pr_triggers.auto_cancel {
703 concurrency = Some(github_yaml_defs::Concurrency {
704 // only cancel in-progress jobs for the same workflow and branch
705 group: Some("${{ github.workflow }}-${{ github.ref }}".to_string()),
706 cancel_in_progress: Some(true),
707 })
708 };
709 Some(github_yaml_defs::PrTrigger {
710 branches: gh_pr_triggers.branches.clone(),
711 branches_ignore: gh_pr_triggers.exclude_branches.clone(),
712 types: gh_pr_triggers.types.clone(),
713 paths: gh_pr_triggers.paths.clone(),
714 paths_ignore: gh_pr_triggers.paths_ignore.clone(),
715 })
716 }
717 None => None,
718 },
719 push: match gh_ci_triggers {
720 Some(gh_ci_triggers) => Some(github_yaml_defs::CiTrigger {
721 branches: gh_ci_triggers.branches,
722 branches_ignore: gh_ci_triggers.exclude_branches,
723 tags: gh_ci_triggers.tags,
724 tags_ignore: gh_ci_triggers.exclude_tags,
725 paths: gh_ci_triggers.paths,
726 paths_ignore: gh_ci_triggers.paths_ignore,
727 }),
728 None => None,
729 },
730 schedule: gh_schedule_triggers
731 .iter()
732 .map(|s| github_yaml_defs::Cron {
733 cron: s.cron.clone(),
734 })
735 .collect(),
736 };
737
738 let github_pipeline = github_yaml_defs::Pipeline {
739 name: gh_name,
740 on: Some(pipeline_trigger),
741 concurrency,
742 jobs: Some(github_yaml_defs::Jobs { jobs: github_jobs }),
743 inputs: None,
744 };
745
746 match check {
747 CheckMode::Check(_) | CheckMode::Runtime(_) => check_generated_yaml_and_json(
748 &github_pipeline,
749 &pipeline_static_db,
750 check,
751 repo_root,
752 pipeline_file,
753 None,
754 ),
755 CheckMode::None => write_generated_yaml_and_json(
756 &github_pipeline,
757 &pipeline_static_db,
758 repo_root,
759 pipeline_file,
760 None,
761 ),
762 }
763}
764
765/// Resolve a flow as a sequence of GitHub YAML steps.
766///
767/// These steps can then be marshalled into a well-formed GitHub pipeline yaml
768/// using a separate GitHub pipeline yaml builder
769// pub(crate) so that internal debug CLI tooling can use it
770fn resolve_flow_as_github_yaml_steps(
771 seed_nodes: BTreeMap<NodeHandle, (bool, Vec<Box<[u8]>>)>,
772 seed_configs: BTreeMap<NodeHandle, Vec<Box<[u8]>>>,
773 resolved_patches: flowey_core::patch::ResolvedPatches,
774 external_read_vars: BTreeSet<String>,
775 platform: FlowPlatform,
776 arch: FlowArch,
777 job_idx: usize,
778 flowey_bin: &str,
779 gh_permissions: &BTreeMap<NodeHandle, BTreeMap<GhPermission, GhPermissionValue>>,
780) -> anyhow::Result<super::common_yaml::ResolvedFlowSteps> {
781 let mut output_steps = Vec::new();
782
783 let crate::flow_resolver::stage1_dag::Stage1DagOutput {
784 mut output_graph,
785 request_db,
786 config_db,
787 found_unreachable_nodes,
788 } = crate::flow_resolver::stage1_dag::stage1_dag(
789 FlowBackend::Github,
790 platform,
791 arch,
792 resolved_patches,
793 seed_nodes,
794 seed_configs,
795 external_read_vars,
796 // TODO: support GitHub agents with persistent storage
797 None,
798 )?;
799
800 if found_unreachable_nodes {
801 anyhow::bail!("detected unreachable nodes")
802 }
803
804 let mut bash_commands = BashCommands::new_github();
805
806 let output_order = petgraph::algo::toposort(&output_graph, None)
807 .expect("runtime variables cannot introduce a DAG cycle");
808
809 let var_db = VarDbRequestBuilder::new(flowey_bin, job_idx);
810
811 for node_idx in output_order.into_iter().rev() {
812 let OutputGraphEntry { node_handle, step } = output_graph[node_idx].1.take().unwrap();
813
814 let node_modpath = node_handle.modpath();
815
816 match step {
817 Step::Anchor { .. } => {}
818 Step::Rust {
819 idx,
820 label,
821 can_merge,
822 code: _,
823 } => {
824 output_steps.extend(bash_commands.push(
825 Some(label),
826 can_merge,
827 crate::cli::exec_snippet::construct_exec_snippet_cli(
828 flowey_bin,
829 node_modpath,
830 idx,
831 job_idx,
832 ),
833 ));
834 }
835 Step::AdoYaml { label, .. } => {
836 anyhow::bail!("ADO YAML not supported in GitHub. In step '{}'", label)
837 }
838 Step::GitHubYaml {
839 gh_to_rust,
840 rust_to_gh,
841 label,
842 step_id,
843 uses,
844 with,
845 condvar,
846 permissions,
847 } => {
848 for permission in permissions {
849 if let Some(permission_map) = gh_permissions.get(&node_handle) {
850 if let Some(permission_value) = permission_map.get(&permission.0) {
851 if *permission_value != permission.1 {
852 anyhow::bail!(
853 "permission mismatch for {:?}: expected {:?}, got {:?}",
854 permission.0,
855 permission.1,
856 permission_value
857 )
858 }
859 }
860 } else {
861 anyhow::bail!(
862 "permission missing for {:?}: expected {:?}",
863 permission.0,
864 permission.1
865 )
866 }
867 }
868
869 for gh_var_state in rust_to_gh {
870 let set_gh_env_var = var_db
871 .write_to_gh_env(&gh_var_state.backing_var, &gh_var_state.raw_name)
872 .raw_string(!gh_var_state.is_object)
873 .condvar(condvar.as_deref());
874
875 bash_commands.push_minor(format!("{set_gh_env_var}\n"));
876 }
877
878 if !uses.is_empty() {
879 if let Some(condvar) = &condvar {
880 // guaranteed to be a bare bool `true`/`false`, hence
881 // is_raw_string = false
882 let set_condvar = var_db.write_to_gh_env(condvar, "FLOWEY_CONDITION");
883 bash_commands.push_minor(format!("{set_condvar}\n"));
884 }
885
886 let mut map = serde_yaml::Mapping::new();
887 map.insert("id".into(), serde_yaml::Value::String(step_id.clone()));
888 map.insert("uses".into(), serde_yaml::Value::String(uses));
889 if !with.is_empty() {
890 let mut with_map = serde_yaml::Mapping::new();
891 for (k, v) in with {
892 with_map.insert(k.into(), v.into());
893 }
894 map.insert("with".into(), with_map.into());
895 }
896 map.insert("name".into(), label.into());
897 if condvar.is_some() {
898 map.insert("if".into(), "${{ fromJSON(env.FLOWEY_CONDITION) }}".into());
899 }
900
901 let step: serde_yaml::Value = map.into();
902 output_steps.extend(bash_commands.flush());
903 output_steps.push(step);
904 }
905
906 for gh_var_state in gh_to_rust {
907 let value = if gh_var_state.is_object {
908 format!(r#"${{{{ toJSON({}) }}}}"#, gh_var_state.raw_name)
909 } else {
910 format!(r#"${{{{ {} }}}}"#, gh_var_state.raw_name)
911 };
912
913 let write_var = var_db
914 .update_from_stdin(&gh_var_state.backing_var, gh_var_state.is_secret)
915 .raw_string(!gh_var_state.is_object)
916 .condvar(condvar.as_deref())
917 .env_source(Some(&gh_var_state.raw_name));
918
919 let cmd = format!("{write_var} <<EOF\n{value}\nEOF",);
920 bash_commands.push_minor(cmd);
921 }
922 }
923 }
924 }
925
926 output_steps.extend(bash_commands.flush());
927
928 let request_db = request_db
929 .into_iter()
930 .map(|(node_handle, reqs)| {
931 (
932 node_handle.modpath().to_owned(),
933 reqs.into_iter()
934 .map(crate::cli::exec_snippet::SerializedRequest)
935 .collect(),
936 )
937 })
938 .collect();
939
940 let config_db = config_db
941 .into_iter()
942 .map(|(node_handle, configs)| {
943 (
944 node_handle.modpath().to_owned(),
945 configs
946 .into_iter()
947 .map(crate::cli::exec_snippet::SerializedRequest)
948 .collect(),
949 )
950 })
951 .collect();
952
953 Ok(super::common_yaml::ResolvedFlowSteps {
954 steps: output_steps,
955 request_db,
956 config_db,
957 })
958}
959