GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/openvmm

Public

mirrored fromhttps://github.com/microsoft/openvmmAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
fe2bbc9829558a07c5e06d2b6ececc383b6593bf

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openvmm/flowey/flowey_cli/src/pipeline_resolver/github_yaml

flowey/flowey_cli/src/pipeline_resolver/github_yaml/mod.rs

943lines · modecode

RawDownload
Latest commit unavailable.
1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4//! Code for emitting a pipeline as a single self-contained GitHub Actions yaml file
5
6use super::common_yaml::BashCommands;
7use super::common_yaml::check_generated_yaml_and_json;
8use super::common_yaml::write_generated_yaml_and_json;
9use super::generic::ResolvedJobArtifact;
10use super::generic::ResolvedJobUseParameter;
11use crate::cli::exec_snippet::FloweyPipelineStaticDb;
12use crate::cli::exec_snippet::VAR_DB_SEEDVAR_FLOWEY_WORKING_DIR;
13use crate::cli::pipeline::CheckMode;
14use crate::cli::var_db::VarDbRequestBuilder;
15use crate::flow_resolver::stage1_dag::OutputGraphEntry;
16use crate::flow_resolver::stage1_dag::Step;
17use crate::pipeline_resolver::common_yaml::FloweySource;
18use crate::pipeline_resolver::common_yaml::job_flowey_bootstrap_source;
19use crate::pipeline_resolver::generic::ResolvedPipeline;
20use crate::pipeline_resolver::generic::ResolvedPipelineJob;
21use anyhow::Context;
22use flowey_core::node::FlowArch;
23use flowey_core::node::FlowBackend;
24use flowey_core::node::FlowPlatform;
25use flowey_core::node::FlowPlatformKind;
26use flowey_core::node::NodeHandle;
27use flowey_core::node::user_facing::GhPermission;
28use flowey_core::node::user_facing::GhPermissionValue;
29use flowey_core::pipeline::GhRunner;
30use flowey_core::pipeline::GhRunnerOsLabel;
31use std::collections::BTreeMap;
32use std::collections::BTreeSet;
33use std::fmt::Write;
34use std::path::Path;
35mod github_yaml_defs;
36
37const RUNNER_TEMP: &str = "${{ runner.temp }}";
38
39/// Emit a pipeline as a single self-contained GitHub Actions yaml file
40pub fn github_yaml(
41 pipeline: ResolvedPipeline,
42 runtime_debug_log: bool,
43 repo_root: &Path,
44 pipeline_file: &Path,
45 flowey_crate: &str,
46 check: CheckMode,
47) -> anyhow::Result<()> {
48 if pipeline_file.extension().and_then(|s| s.to_str()) != Some("yaml") {
49 anyhow::bail!("pipeline name must end with .yaml")
50 }
51
52 let ResolvedPipeline {
53 graph,
54 order,
55 gh_name,
56 gh_schedule_triggers,
57 gh_ci_triggers,
58 gh_pr_triggers,
59 gh_bootstrap_template,
60 parameters,
61 ado_name: _,
62 ado_schedule_triggers: _,
63 ado_ci_triggers: _,
64 ado_pr_triggers: _,
65 ado_bootstrap_template: _,
66 ado_resources_repository: _,
67 ado_post_process_yaml_cb: _,
68 ado_variables: _,
69 ado_job_id_overrides: _,
70 } = pipeline;
71
72 let mut job_flowey_source: BTreeMap<petgraph::prelude::NodeIndex, FloweySource> =
73 job_flowey_bootstrap_source(&graph, &order);
74
75 let mut pipeline_static_db = FloweyPipelineStaticDb {
76 flow_backend: crate::cli::FlowBackendCli::Github,
77 var_db_backend_kind: crate::cli::exec_snippet::VarDbBackendKind::Json,
78 job_reqs: BTreeMap::new(),
79 job_configs: BTreeMap::new(),
80 job_command_wrappers: BTreeMap::new(),
81 job_platforms: BTreeMap::new(),
82 job_archs: BTreeMap::new(),
83 };
84
85 let mut github_jobs = BTreeMap::new();
86
87 for job_idx in order {
88 let ResolvedPipelineJob {
89 ref root_nodes,
90 ref root_configs,
91 ref patches,
92 ref label,
93 platform,
94 arch,
95 ref external_read_vars,
96 ado_pool: _,
97 timeout_minutes,
98 command_wrapper: ref command_wrapper_kind,
99 ref gh_override_if,
100 ref gh_global_env,
101 ref gh_pool,
102 ref gh_permissions,
103 cond_param_idx,
104 ref parameters_used,
105 ref artifacts_used,
106 ref artifacts_published,
107 ado_variables: _,
108 } = graph[job_idx];
109
110 if cond_param_idx.is_some() {
111 anyhow::bail!(
112 "conditional params are not supported in GitHub backend, use `gh_dangerous_override_if` instead"
113 );
114 }
115
116 let flowey_bin = platform.binary("flowey");
117 let super::common_yaml::ResolvedFlowSteps {
118 steps,
119 request_db: req_db,
120 config_db: cfg_db,
121 } = resolve_flow_as_github_yaml_steps(
122 root_nodes
123 .clone()
124 .into_iter()
125 .map(|(node, requests)| (node, (true, requests)))
126 .collect(),
127 root_configs.clone(),
128 patches.clone(),
129 external_read_vars.clone(),
130 platform,
131 arch,
132 job_idx.index(),
133 &flowey_bin,
134 gh_permissions,
135 )
136 .context(format!("in job '{label}'"))?;
137
138 {
139 let existing = pipeline_static_db.job_reqs.insert(job_idx.index(), req_db);
140 assert!(existing.is_none())
141 }
142
143 if !cfg_db.is_empty() {
144 pipeline_static_db
145 .job_configs
146 .insert(job_idx.index(), cfg_db);
147 }
148
149 if let Some(wrapper_kind) = command_wrapper_kind {
150 pipeline_static_db
151 .job_command_wrappers
152 .insert(job_idx.index(), wrapper_kind.clone());
153 }
154
155 pipeline_static_db
156 .job_platforms
157 .insert(job_idx.index(), platform);
158 pipeline_static_db.job_archs.insert(job_idx.index(), arch);
159
160 let mut gh_steps = Vec::new();
161
162 let flowey_source = job_flowey_source.remove(&job_idx).unwrap();
163
164 let mut artifact_names = Vec::new();
165
166 let flowey_path = match &flowey_source {
167 FloweySource::Bootstrap { .. } => {
168 let flowey_path = "bootstrapped-flowey".to_string();
169
170 // actual artifact publish happens at the end of the job
171 if gh_bootstrap_template.is_empty() {
172 anyhow::bail!(
173 "Did not specify flowey bootstrap template. Please provide one using `Pipeline::gh_set_flowey_bootstrap_template`"
174 )
175 }
176
177 let gh_bootstrap_template = gh_bootstrap_template
178 .replace("{{FLOWEY_BIN_EXTENSION}}", platform.exe_suffix())
179 .replace("{{FLOWEY_CRATE}}", flowey_crate)
180 .replace(
181 "{{FLOWEY_PIPELINE_PATH}}",
182 &pipeline_file.with_extension("").display().to_string(),
183 )
184 .replace(
185 "{{FLOWEY_TARGET}}",
186 match (platform, arch) {
187 (FlowPlatform::Windows, FlowArch::X86_64) => "x86_64-pc-windows-msvc",
188 (FlowPlatform::Windows, FlowArch::Aarch64) => "aarch64-pc-windows-msvc",
189 (FlowPlatform::Linux(_), FlowArch::X86_64) => {
190 "x86_64-unknown-linux-gnu"
191 }
192 (FlowPlatform::Linux(_), FlowArch::Aarch64) => {
193 "aarch64-unknown-linux-gnu"
194 }
195 (platform, arch) => {
196 anyhow::bail!("unsupported platform {platform} / arch {arch}")
197 }
198 },
199 )
200 .replace("{{FLOWEY_OUTDIR}}", &format!("{RUNNER_TEMP}/{flowey_path}"));
201
202 let bootstrap_steps: serde_yaml::Sequence =
203 serde_yaml::from_str(&gh_bootstrap_template)
204 .context("malformed flowey bootstrap template")?;
205
206 gh_steps.extend(bootstrap_steps);
207 flowey_path
208 }
209 FloweySource::Consume(artifact) => {
210 // download previously bootstrapped flowey
211 artifact_names.push(artifact.as_str());
212 format!("used_artifacts/{artifact}")
213 }
214 };
215
216 // download any artifacts that'll be used
217 artifact_names.extend(artifacts_used.iter().map(|a| a.name.as_str()));
218 if !artifact_names.is_empty() {
219 // When downloading a single artifact by name, the contents are
220 // extracted directly to `path/` without a subdirectory. For
221 // multiple artifacts with `merge-multiple: false` (the default),
222 // named subdirectories are created automatically.
223 //
224 // Use `name:` with an explicit subdirectory path for single
225 // artifacts, and `pattern:` for multiple artifacts.
226 gh_steps.push({
227 let map: serde_yaml::Mapping = if let &[name] = artifact_names.as_slice() {
228 serde_yaml::from_str(&format!(
229 r#"
230 name: '🌼📦 Download artifacts'
231 uses: actions/download-artifact@v8
232 with:
233 name: '{name}'
234 path: {RUNNER_TEMP}/used_artifacts/{name}/
235 "#
236 ))
237 .unwrap()
238 } else {
239 let pattern = format!("{{{}}}", artifact_names.join(","));
240 serde_yaml::from_str(&format!(
241 r#"
242 name: '🌼📦 Download artifacts'
243 uses: actions/download-artifact@v8
244 with:
245 pattern: '{pattern}'
246 path: {RUNNER_TEMP}/used_artifacts/
247 "#
248 ))
249 .unwrap()
250 };
251 map.into()
252 });
253 }
254
255 {
256 let mut map = serde_yaml::Mapping::new();
257 map.insert(
258 "run".into(),
259 format!(r#"echo "{RUNNER_TEMP}/{flowey_path}" >> $GITHUB_PATH"#).into(),
260 );
261 map.insert("shell".into(), "bash".into());
262 map.insert("name".into(), "🌼📦 Add flowey to PATH".into());
263 gh_steps.push(map.into());
264 }
265
266 let var_db = VarDbRequestBuilder::new(&flowey_bin, job_idx.index());
267
268 let bootstrap_bash_var_db_inject = |var, is_raw_string| {
269 var_db
270 .update_from_stdin(var, false)
271 .raw_string(is_raw_string)
272 .to_string()
273 };
274
275 // if this was a bootstrap job, also take a moment to run a "self check"
276 // to make sure that the current checked-in template matches the one it
277 // expected
278 if let FloweySource::Bootstrap(..) = &flowey_source {
279 let mut current_invocation = std::env::args().collect::<Vec<_>>();
280
281 current_invocation[0] = flowey_bin.clone();
282
283 // if this code path is run while generating the YAML to compare the
284 // check against, we want to remove the --runtime or --check param from the
285 // current call, or else there'll be a dupe
286 let mut strip_parameter = |prefix: &str| {
287 if let Some(i) = current_invocation
288 .iter()
289 .position(|s| s.starts_with(prefix))
290 {
291 current_invocation.remove(i);
292 if !current_invocation[i].starts_with(prefix) {
293 current_invocation.remove(i);
294 }
295 }
296 };
297
298 strip_parameter("--runtime");
299 strip_parameter("--check");
300
301 // insert the --check bit of the call alongside the --out param
302 {
303 let i = current_invocation
304 .iter()
305 .position(|s| s.starts_with("--out"))
306 .unwrap();
307
308 let current_yaml = match platform.kind() {
309 FlowPlatformKind::Windows => {
310 let win_path = flowey_path.replace('/', "\\");
311 format!(r#"$ESCAPED_AGENT_TEMPDIR\\{win_path}\\pipeline.yaml"#)
312 }
313 FlowPlatformKind::Unix => {
314 format!(r#"$ESCAPED_AGENT_TEMPDIR/{flowey_path}/pipeline.yaml"#)
315 }
316 };
317
318 current_invocation.insert(i, current_yaml);
319 current_invocation.insert(i, "--runtime".into());
320 }
321
322 // Need to use an escaped version of the "true" windows/linux path
323 // here, or else the --check will fail.
324 let cmd = format!(
325 r###"
326ESCAPED_AGENT_TEMPDIR=$(
327cat <<'EOF' | sed 's/\\/\\\\/g'
328{RUNNER_TEMP}
329EOF
330)
331{}
332"###,
333 current_invocation.join(" ")
334 );
335
336 gh_steps.push({
337 let mut map = serde_yaml::Mapping::new();
338 map.insert("name".into(), "🌼🔎 Self-check YAML".into());
339 map.insert(
340 "run".into(),
341 serde_yaml::Value::String(cmd.trim().to_string()),
342 );
343 map.insert("shell".into(), "bash".into());
344 map.into()
345 })
346 }
347
348 let mut flowey_bootstrap_bash = String::new();
349
350 // and now use those vars to do some flowey bootstrap
351 writeln!(flowey_bootstrap_bash, "{}", {
352 let runtime_debug_level = if runtime_debug_log { "debug" } else { "info" };
353
354 let var_db_insert_runtime_debug_level =
355 bootstrap_bash_var_db_inject("FLOWEY_LOG", false);
356 let var_db_insert_working_dir =
357 bootstrap_bash_var_db_inject(VAR_DB_SEEDVAR_FLOWEY_WORKING_DIR, true);
358
359 // Need to use "normalized" path in cases where the path is being
360 // used directly from a bash context, as is the case when we are
361 // trying to invoke `flowey.exe` in argv0 position)
362 //
363 // https://github.com/microsoft/azure-pipelines-tasks/issues/10653#issuecomment-585669089
364 format!(
365 r###"
366AgentTempDirNormal="{RUNNER_TEMP}"
367AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|')
368echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV
369
370chmod +x $AgentTempDirNormal/{flowey_path}/{flowey_bin}
371
372echo '"{runtime_debug_level}"' | {var_db_insert_runtime_debug_level}
373echo "{RUNNER_TEMP}/work" | {var_db_insert_working_dir}
374"###
375 )
376 .trim_start()
377 .to_owned()
378 })?;
379
380 // import pipeline vars being used by the job into flowey
381 for ResolvedJobUseParameter {
382 flowey_var,
383 pipeline_param_idx,
384 } in parameters_used
385 {
386 let is_string = matches!(
387 parameters[*pipeline_param_idx],
388 flowey_core::pipeline::internal::Parameter::String { .. }
389 );
390
391 let default = match &parameters[*pipeline_param_idx] {
392 flowey_core::pipeline::internal::Parameter::Bool { default, .. } => {
393 default.map(|b| b.to_string())
394 }
395 flowey_core::pipeline::internal::Parameter::String { default, .. } => {
396 default.clone()
397 }
398 flowey_core::pipeline::internal::Parameter::Num { default, .. } => {
399 default.map(|n| n.to_string())
400 }
401 }
402 .expect("defaults are currently required for parameters in Github backend");
403
404 let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, is_string);
405
406 let name = parameters[*pipeline_param_idx].name();
407
408 let cmd = format!(
409 r#"
410cat <<'EOF' | {var_db_inject_cmd}
411${{{{ inputs.{name} != '' && inputs.{name} || '{default}' }}}}
412EOF
413"#
414 )
415 .trim()
416 .to_string();
417 writeln!(flowey_bootstrap_bash, "{}", cmd)?;
418 }
419
420 // next, emit GitHub steps to create dirs for artifacts which will be
421 // published
422 for ResolvedJobArtifact { flowey_var, name } in artifacts_published {
423 writeln!(
424 flowey_bootstrap_bash,
425 r#"mkdir -p "$AgentTempDirNormal/publish_artifacts/{name}""#
426 )?;
427 let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, true);
428 match platform.kind() {
429 FlowPlatformKind::Windows => {
430 writeln!(
431 flowey_bootstrap_bash,
432 r#"echo "{RUNNER_TEMP}\\publish_artifacts\\{name}" | {var_db_inject_cmd}"#,
433 )?;
434 }
435 FlowPlatformKind::Unix => {
436 writeln!(
437 flowey_bootstrap_bash,
438 r#"echo "$AgentTempDirNormal/publish_artifacts/{name}" | {var_db_inject_cmd}"#,
439 )?;
440 }
441 }
442 }
443
444 // lastly, emit GitHub steps that report the dirs for any artifacts which
445 // are used by this job
446 for ResolvedJobArtifact { flowey_var, name } in artifacts_used {
447 let var_db_inject_cmd = bootstrap_bash_var_db_inject(flowey_var, true);
448 match platform.kind() {
449 FlowPlatformKind::Windows => {
450 writeln!(
451 flowey_bootstrap_bash,
452 r#"echo "{RUNNER_TEMP}\\used_artifacts\\{name}" | {var_db_inject_cmd}"#,
453 )?;
454 }
455 FlowPlatformKind::Unix => {
456 writeln!(
457 flowey_bootstrap_bash,
458 r#"echo "$AgentTempDirNormal/used_artifacts/{name}" | {var_db_inject_cmd}"#,
459 )?;
460 }
461 }
462 }
463
464 gh_steps.push({
465 let mut map = serde_yaml::Mapping::new();
466 map.insert("name".into(), "🌼🛫 Initialize job".into());
467 map.insert(
468 "run".into(),
469 serde_yaml::Value::String(flowey_bootstrap_bash),
470 );
471 map.insert("shell".into(), "bash".into());
472 map.into()
473 });
474
475 // now that we've done all the job-level bootstrapping, we can emit all
476 // the actual steps the user cares about
477 gh_steps.extend(steps);
478
479 // ..and once that's done, the last order of business is to emit some
480 // GitHub steps to publish the various artifacts created by this job
481 for ResolvedJobArtifact {
482 flowey_var: _,
483 name,
484 } in artifacts_published
485 {
486 gh_steps.push({
487 let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
488 r#"
489 name: 🌼📦 Publish {name}
490 uses: actions/upload-artifact@v7
491 with:
492 name: {name}
493 path: {RUNNER_TEMP}/publish_artifacts/{name}/
494 include-hidden-files: true
495 "#
496 ))
497 .unwrap();
498 map.into()
499 });
500 }
501
502 // also, if this job also bootstrapped flowey that other nodes depend
503 // on, make sure to publish it!
504 if let FloweySource::Bootstrap(artifact, true) = flowey_source {
505 // don't leak the bootstrap job's runtime var db
506 gh_steps.push({
507 let mut map = serde_yaml::Mapping::new();
508 map.insert("name".into(), "🌼🧼 Redact bootstrap var db".into());
509 map.insert(
510 "run".into(),
511 serde_yaml::Value::String(format!(
512 "rm $AgentTempDirNormal/{flowey_path}/job{}.json",
513 job_idx.index()
514 )),
515 );
516 map.insert("shell".into(), "bash".into());
517 map.into()
518 });
519
520 gh_steps.push({
521 let map: serde_yaml::Mapping = serde_yaml::from_str(&format!(
522 r#"
523 name: 🌼🥾 Publish bootstrapped flowey
524 uses: actions/upload-artifact@v7
525 with:
526 name: {artifact}
527 path: {RUNNER_TEMP}/{flowey_path}
528 "#
529 ))
530 .unwrap();
531 map.into()
532 });
533 }
534
535 let runner_kind_to_yaml = |runner: &GhRunner| match runner {
536 GhRunner::GhHosted(s) => github_yaml_defs::Runner::GhHosted(match s {
537 GhRunnerOsLabel::UbuntuLatest => github_yaml_defs::RunnerOsLabel::UbuntuLatest,
538 GhRunnerOsLabel::Ubuntu2404 => github_yaml_defs::RunnerOsLabel::Ubuntu2404,
539 GhRunnerOsLabel::Ubuntu2204 => github_yaml_defs::RunnerOsLabel::Ubuntu2204,
540 GhRunnerOsLabel::WindowsLatest => github_yaml_defs::RunnerOsLabel::WindowsLatest,
541 GhRunnerOsLabel::Windows2025 => github_yaml_defs::RunnerOsLabel::Windows2025,
542 GhRunnerOsLabel::Windows2022 => github_yaml_defs::RunnerOsLabel::Windows2022,
543 GhRunnerOsLabel::Ubuntu2404Arm => github_yaml_defs::RunnerOsLabel::Ubuntu2404Arm,
544 GhRunnerOsLabel::Ubuntu2204Arm => github_yaml_defs::RunnerOsLabel::Ubuntu2204Arm,
545 GhRunnerOsLabel::Windows11Arm => github_yaml_defs::RunnerOsLabel::Windows11Arm,
546 GhRunnerOsLabel::Custom(s) => github_yaml_defs::RunnerOsLabel::Custom(s.into()),
547 }),
548 GhRunner::SelfHosted(v) => github_yaml_defs::Runner::SelfHosted(v.clone()),
549 GhRunner::RunnerGroup { group, labels } => github_yaml_defs::Runner::Group {
550 group: group.into(),
551 labels: labels.clone(),
552 },
553 };
554
555 let perm_val_to_yaml = |permission_value: &GhPermissionValue| match permission_value {
556 GhPermissionValue::Read => github_yaml_defs::PermissionValue::Read,
557 GhPermissionValue::Write => github_yaml_defs::PermissionValue::Write,
558 GhPermissionValue::None => github_yaml_defs::PermissionValue::None,
559 };
560
561 let perm_kind_to_yaml = |permission: &GhPermission| match permission {
562 GhPermission::Actions => github_yaml_defs::Permissions::Actions,
563 GhPermission::Attestations => github_yaml_defs::Permissions::Attestations,
564 GhPermission::Checks => github_yaml_defs::Permissions::Checks,
565 GhPermission::Contents => github_yaml_defs::Permissions::Contents,
566 GhPermission::Deployments => github_yaml_defs::Permissions::Deployments,
567 GhPermission::Discussions => github_yaml_defs::Permissions::Discussions,
568 GhPermission::IdToken => github_yaml_defs::Permissions::IdToken,
569 GhPermission::Issues => github_yaml_defs::Permissions::Issues,
570 GhPermission::Packages => github_yaml_defs::Permissions::Packages,
571 GhPermission::Pages => github_yaml_defs::Permissions::Pages,
572 GhPermission::PullRequests => github_yaml_defs::Permissions::PullRequests,
573 GhPermission::RepositoryProjects => github_yaml_defs::Permissions::RepositoryProjects,
574 GhPermission::SecurityEvents => github_yaml_defs::Permissions::SecurityEvents,
575 GhPermission::Statuses => github_yaml_defs::Permissions::Statuses,
576 };
577
578 let mut job_permissions = BTreeMap::new();
579 for permission_map in gh_permissions.values() {
580 for (permission, value) in permission_map {
581 // Use the most permissible value set (this allows individual
582 // jobs to override the value set in inject_all_jobs_with)
583 if job_permissions
584 .get(permission)
585 .is_none_or(|old_value| *old_value < *value)
586 {
587 job_permissions.insert(permission.clone(), value.clone());
588 }
589 }
590 }
591
592 github_jobs.insert(
593 format!("job{}", job_idx.index()),
594 github_yaml_defs::Job {
595 name: label.clone(),
596 timeout_minutes,
597 runs_on: gh_pool.clone().map(|runner| {
598 let mut yaml_runner = runner_kind_to_yaml(&runner);
599 if let github_yaml_defs::Runner::SelfHosted(ref mut labels) = yaml_runner {
600 if labels.iter().any(|l| l.starts_with("1ES.Pool=")) {
601 labels.push(format!(
602 "JobId=job{}-${{{{ github.run_id }}}}-${{{{ github.run_number }}}}-${{{{ github.run_attempt }}}}",
603 job_idx.index()
604 ));
605 }
606 }
607 yaml_runner
608 }),
609 permissions: job_permissions
610 .iter()
611 .map(|k| (perm_kind_to_yaml(k.0), perm_val_to_yaml(k.1)))
612 .collect(),
613 needs: {
614 graph
615 .edges_directed(job_idx, petgraph::Direction::Incoming)
616 .map(|e| {
617 use petgraph::prelude::*;
618 format!("job{}", e.source().index())
619 })
620 .collect()
621 },
622 r#if: gh_override_if
623 .clone()
624 .or_else(|| Some("github.event.pull_request.draft == false".to_string())),
625 env: gh_global_env.clone(),
626 steps: gh_steps,
627 },
628 );
629 }
630
631 let mut concurrency = None;
632 let pipeline_trigger = github_yaml_defs::Triggers {
633 workflow_call: None,
634 workflow_dispatch: Some(github_yaml_defs::WorkflowDispatch {
635 inputs: github_yaml_defs::Inputs {
636 inputs: parameters
637 .into_iter()
638 .map(|param| {
639 (
640 param.name().to_string(),
641 match param {
642 flowey_core::pipeline::internal::Parameter::Bool {
643 name: _,
644 description,
645 kind: _,
646 default,
647 } => github_yaml_defs::Input {
648 description: Some(description.clone()),
649 default: default.map(github_yaml_defs::Default::Boolean),
650 required: default.is_none(),
651 ty: github_yaml_defs::InputType::Boolean,
652 },
653 flowey_core::pipeline::internal::Parameter::String {
654 name: _,
655 description,
656 kind: _,
657 default,
658 possible_values: _,
659 } => github_yaml_defs::Input {
660 description: Some(description.clone()),
661 default: default
662 .as_ref()
663 .map(|s| github_yaml_defs::Default::String(s.clone())),
664 required: default.is_none(),
665 ty: github_yaml_defs::InputType::String,
666 },
667 flowey_core::pipeline::internal::Parameter::Num {
668 name: _,
669 description,
670 kind: _,
671 default,
672 possible_values: _,
673 } => github_yaml_defs::Input {
674 description: Some(description.clone()),
675 default: default.map(github_yaml_defs::Default::Number),
676 required: default.is_none(),
677 ty: github_yaml_defs::InputType::Number,
678 },
679 },
680 )
681 })
682 .collect::<BTreeMap<String, github_yaml_defs::Input>>(),
683 },
684 }),
685 pull_request: match gh_pr_triggers {
686 Some(gh_pr_triggers) => {
687 if gh_pr_triggers.auto_cancel {
688 concurrency = Some(github_yaml_defs::Concurrency {
689 // only cancel in-progress jobs for the same workflow and branch
690 group: Some("${{ github.workflow }}-${{ github.ref }}".to_string()),
691 cancel_in_progress: Some(true),
692 })
693 };
694 Some(github_yaml_defs::PrTrigger {
695 branches: gh_pr_triggers.branches.clone(),
696 branches_ignore: gh_pr_triggers.exclude_branches.clone(),
697 types: gh_pr_triggers.types.clone(),
698 paths: gh_pr_triggers.paths.clone(),
699 paths_ignore: gh_pr_triggers.paths_ignore.clone(),
700 })
701 }
702 None => None,
703 },
704 push: match gh_ci_triggers {
705 Some(gh_ci_triggers) => Some(github_yaml_defs::CiTrigger {
706 branches: gh_ci_triggers.branches,
707 branches_ignore: gh_ci_triggers.exclude_branches,
708 tags: gh_ci_triggers.tags,
709 tags_ignore: gh_ci_triggers.exclude_tags,
710 paths: gh_ci_triggers.paths,
711 paths_ignore: gh_ci_triggers.paths_ignore,
712 }),
713 None => None,
714 },
715 schedule: gh_schedule_triggers
716 .iter()
717 .map(|s| github_yaml_defs::Cron {
718 cron: s.cron.clone(),
719 })
720 .collect(),
721 };
722
723 let github_pipeline = github_yaml_defs::Pipeline {
724 name: gh_name,
725 on: Some(pipeline_trigger),
726 concurrency,
727 jobs: Some(github_yaml_defs::Jobs { jobs: github_jobs }),
728 inputs: None,
729 };
730
731 match check {
732 CheckMode::Check(_) | CheckMode::Runtime(_) => check_generated_yaml_and_json(
733 &github_pipeline,
734 &pipeline_static_db,
735 check,
736 repo_root,
737 pipeline_file,
738 None,
739 ),
740 CheckMode::None => write_generated_yaml_and_json(
741 &github_pipeline,
742 &pipeline_static_db,
743 repo_root,
744 pipeline_file,
745 None,
746 ),
747 }
748}
749
750/// Resolve a flow as a sequence of GitHub YAML steps.
751///
752/// These steps can then be marshalled into a well-formed GitHub pipeline yaml
753/// using a separate GitHub pipeline yaml builder
754// pub(crate) so that internal debug CLI tooling can use it
755fn resolve_flow_as_github_yaml_steps(
756 seed_nodes: BTreeMap<NodeHandle, (bool, Vec<Box<[u8]>>)>,
757 seed_configs: BTreeMap<NodeHandle, Vec<Box<[u8]>>>,
758 resolved_patches: flowey_core::patch::ResolvedPatches,
759 external_read_vars: BTreeSet<String>,
760 platform: FlowPlatform,
761 arch: FlowArch,
762 job_idx: usize,
763 flowey_bin: &str,
764 gh_permissions: &BTreeMap<NodeHandle, BTreeMap<GhPermission, GhPermissionValue>>,
765) -> anyhow::Result<super::common_yaml::ResolvedFlowSteps> {
766 let mut output_steps = Vec::new();
767
768 let crate::flow_resolver::stage1_dag::Stage1DagOutput {
769 mut output_graph,
770 request_db,
771 config_db,
772 found_unreachable_nodes,
773 } = crate::flow_resolver::stage1_dag::stage1_dag(
774 FlowBackend::Github,
775 platform,
776 arch,
777 resolved_patches,
778 seed_nodes,
779 seed_configs,
780 external_read_vars,
781 // TODO: support GitHub agents with persistent storage
782 None,
783 )?;
784
785 if found_unreachable_nodes {
786 anyhow::bail!("detected unreachable nodes")
787 }
788
789 let mut bash_commands = BashCommands::new_github();
790
791 let output_order = petgraph::algo::toposort(&output_graph, None)
792 .expect("runtime variables cannot introduce a DAG cycle");
793
794 let var_db = VarDbRequestBuilder::new(flowey_bin, job_idx);
795
796 for node_idx in output_order.into_iter().rev() {
797 let OutputGraphEntry { node_handle, step } = output_graph[node_idx].1.take().unwrap();
798
799 let node_modpath = node_handle.modpath();
800
801 match step {
802 Step::Anchor { .. } => {}
803 Step::Rust {
804 idx,
805 label,
806 can_merge,
807 code: _,
808 } => {
809 output_steps.extend(bash_commands.push(
810 Some(label),
811 can_merge,
812 crate::cli::exec_snippet::construct_exec_snippet_cli(
813 flowey_bin,
814 node_modpath,
815 idx,
816 job_idx,
817 ),
818 ));
819 }
820 Step::AdoYaml { label, .. } => {
821 anyhow::bail!("ADO YAML not supported in GitHub. In step '{}'", label)
822 }
823 Step::GitHubYaml {
824 gh_to_rust,
825 rust_to_gh,
826 label,
827 step_id,
828 uses,
829 with,
830 condvar,
831 permissions,
832 } => {
833 for permission in permissions {
834 if let Some(permission_map) = gh_permissions.get(&node_handle) {
835 if let Some(permission_value) = permission_map.get(&permission.0) {
836 if *permission_value != permission.1 {
837 anyhow::bail!(
838 "permission mismatch for {:?}: expected {:?}, got {:?}",
839 permission.0,
840 permission.1,
841 permission_value
842 )
843 }
844 }
845 } else {
846 anyhow::bail!(
847 "permission missing for {:?}: expected {:?}",
848 permission.0,
849 permission.1
850 )
851 }
852 }
853
854 for gh_var_state in rust_to_gh {
855 let set_gh_env_var = var_db
856 .write_to_gh_env(&gh_var_state.backing_var, &gh_var_state.raw_name)
857 .raw_string(!gh_var_state.is_object)
858 .condvar(condvar.as_deref());
859
860 bash_commands.push_minor(format!("{set_gh_env_var}\n"));
861 }
862
863 if !uses.is_empty() {
864 if let Some(condvar) = &condvar {
865 // guaranteed to be a bare bool `true`/`false`, hence
866 // is_raw_string = false
867 let set_condvar = var_db.write_to_gh_env(condvar, "FLOWEY_CONDITION");
868 bash_commands.push_minor(format!("{set_condvar}\n"));
869 }
870
871 let mut map = serde_yaml::Mapping::new();
872 map.insert("id".into(), serde_yaml::Value::String(step_id.clone()));
873 map.insert("uses".into(), serde_yaml::Value::String(uses));
874 if !with.is_empty() {
875 let mut with_map = serde_yaml::Mapping::new();
876 for (k, v) in with {
877 with_map.insert(k.into(), v.into());
878 }
879 map.insert("with".into(), with_map.into());
880 }
881 map.insert("name".into(), label.into());
882 if condvar.is_some() {
883 map.insert("if".into(), "${{ fromJSON(env.FLOWEY_CONDITION) }}".into());
884 }
885
886 let step: serde_yaml::Value = map.into();
887 output_steps.extend(bash_commands.flush());
888 output_steps.push(step);
889 }
890
891 for gh_var_state in gh_to_rust {
892 let value = if gh_var_state.is_object {
893 format!(r#"${{{{ toJSON({}) }}}}"#, gh_var_state.raw_name)
894 } else {
895 format!(r#"${{{{ {} }}}}"#, gh_var_state.raw_name)
896 };
897
898 let write_var = var_db
899 .update_from_stdin(&gh_var_state.backing_var, gh_var_state.is_secret)
900 .raw_string(!gh_var_state.is_object)
901 .condvar(condvar.as_deref())
902 .env_source(Some(&gh_var_state.raw_name));
903
904 let cmd = format!("{write_var} <<EOF\n{value}\nEOF",);
905 bash_commands.push_minor(cmd);
906 }
907 }
908 }
909 }
910
911 output_steps.extend(bash_commands.flush());
912
913 let request_db = request_db
914 .into_iter()
915 .map(|(node_handle, reqs)| {
916 (
917 node_handle.modpath().to_owned(),
918 reqs.into_iter()
919 .map(crate::cli::exec_snippet::SerializedRequest)
920 .collect(),
921 )
922 })
923 .collect();
924
925 let config_db = config_db
926 .into_iter()
927 .map(|(node_handle, configs)| {
928 (
929 node_handle.modpath().to_owned(),
930 configs
931 .into_iter()
932 .map(crate::cli::exec_snippet::SerializedRequest)
933 .collect(),
934 )
935 })
936 .collect();
937
938 Ok(super::common_yaml::ResolvedFlowSteps {
939 steps: output_steps,
940 request_db,
941 config_db,
942 })
943}
944