GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/openvmm

Public

mirrored fromhttps://github.com/microsoft/openvmmAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
e6a1eca0ac970ac1b75f1b4ea3ba48c126ddd5ea

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openvmm/flowey/flowey_core/src

flowey/flowey_core/src/pipeline.rs

1269lines · modecode

RawDownload
Latest commit unavailable.
1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4//! Core types and traits used to create and work with flowey pipelines.
5
6use self::internal::*;
7use crate::node::steps::ado::AdoResourcesRepositoryId;
8use crate::node::user_facing::AdoRuntimeVar;
9use crate::node::user_facing::GhContextVar;
10use crate::node::user_facing::GhPermission;
11use crate::node::user_facing::GhPermissionValue;
12use crate::node::FlowArch;
13use crate::node::FlowNodeBase;
14use crate::node::FlowPlatform;
15use crate::node::FlowPlatformLinuxDistro;
16use crate::node::IntoRequest;
17use crate::node::NodeHandle;
18use crate::node::ReadVar;
19use crate::node::WriteVar;
20use crate::patch::PatchResolver;
21use crate::patch::ResolvedPatches;
22use serde::de::DeserializeOwned;
23use serde::Serialize;
24use std::collections::BTreeMap;
25use std::collections::BTreeSet;
26use std::path::PathBuf;
27
28/// Pipeline types which are considered "user facing", and included in the
29/// `flowey` prelude.
30pub mod user_facing {
31 pub use super::AdoCiTriggers;
32 pub use super::AdoPrTriggers;
33 pub use super::AdoResourcesRepository;
34 pub use super::AdoResourcesRepositoryRef;
35 pub use super::AdoResourcesRepositoryType;
36 pub use super::AdoScheduleTriggers;
37 pub use super::GhCiTriggers;
38 pub use super::GhPrTriggers;
39 pub use super::GhRunner;
40 pub use super::GhRunnerOsLabel;
41 pub use super::GhScheduleTriggers;
42 pub use super::HostExt;
43 pub use super::IntoPipeline;
44 pub use super::Pipeline;
45 pub use super::PipelineBackendHint;
46 pub use super::PipelineJob;
47 pub use super::PipelineJobCtx;
48 pub use super::PipelineJobHandle;
49 pub use super::PublishArtifact;
50 pub use super::UseArtifact;
51 pub use super::UseParameter;
52 pub use crate::node::FlowArch;
53 pub use crate::node::FlowPlatform;
54}
55
56fn linux_distro() -> FlowPlatformLinuxDistro {
57 if let Ok(etc_os_release) = fs_err::read_to_string("/etc/os-release") {
58 if etc_os_release.contains("ID=ubuntu") {
59 FlowPlatformLinuxDistro::Ubuntu
60 } else if etc_os_release.contains("ID=fedora") {
61 FlowPlatformLinuxDistro::Fedora
62 } else {
63 FlowPlatformLinuxDistro::Unknown
64 }
65 } else {
66 FlowPlatformLinuxDistro::Unknown
67 }
68}
69
70pub trait HostExt: Sized {
71 /// Return the value for the current host machine.
72 ///
73 /// Will panic on non-local backends.
74 fn host(backend_hint: PipelineBackendHint) -> Self;
75}
76
77impl HostExt for FlowPlatform {
78 /// Return the platform of the current host machine.
79 ///
80 /// Will panic on non-local backends.
81 fn host(backend_hint: PipelineBackendHint) -> Self {
82 if !matches!(backend_hint, PipelineBackendHint::Local) {
83 panic!("can only use `FlowPlatform::host` when defining a local-only pipeline");
84 }
85
86 if cfg!(target_os = "windows") {
87 Self::Windows
88 } else if cfg!(target_os = "linux") {
89 Self::Linux(linux_distro())
90 } else if cfg!(target_os = "macos") {
91 Self::MacOs
92 } else {
93 panic!("no valid host-os")
94 }
95 }
96}
97
98impl HostExt for FlowArch {
99 /// Return the arch of the current host machine.
100 ///
101 /// Will panic on non-local backends.
102 fn host(backend_hint: PipelineBackendHint) -> Self {
103 if !matches!(backend_hint, PipelineBackendHint::Local) {
104 panic!("can only use `FlowArch::host` when defining a local-only pipeline");
105 }
106
107 // xtask-fmt allow-target-arch oneoff-flowey
108 if cfg!(target_arch = "x86_64") {
109 Self::X86_64
110 // xtask-fmt allow-target-arch oneoff-flowey
111 } else if cfg!(target_arch = "aarch64") {
112 Self::Aarch64
113 } else {
114 panic!("no valid host-arch")
115 }
116 }
117}
118
119/// Trigger ADO pipelines via Continuous Integration
120#[derive(Default, Debug)]
121pub struct AdoScheduleTriggers {
122 /// Friendly name for the scheduled run
123 pub display_name: String,
124 /// Run the pipeline whenever there is a commit on these specified branches
125 /// (supports glob syntax)
126 pub branches: Vec<String>,
127 /// Specify any branches which should be filtered out from the list of
128 /// `branches` (supports glob syntax)
129 pub exclude_branches: Vec<String>,
130 /// Run the pipeline in a schedule, as specified by a cron string
131 pub cron: String,
132}
133
134/// Trigger ADO pipelines per PR
135#[derive(Debug)]
136pub struct AdoPrTriggers {
137 /// Run the pipeline whenever there is a PR to these specified branches
138 /// (supports glob syntax)
139 pub branches: Vec<String>,
140 /// Specify any branches which should be filtered out from the list of
141 /// `branches` (supports glob syntax)
142 pub exclude_branches: Vec<String>,
143 /// Run the pipeline even if the PR is a draft PR. Defaults to `false`.
144 pub run_on_draft: bool,
145 /// Automatically cancel the pipeline run if a new commit lands in the
146 /// branch. Defaults to `true`.
147 pub auto_cancel: bool,
148}
149
150/// Trigger ADO pipelines per PR
151#[derive(Debug, Default)]
152pub struct AdoCiTriggers {
153 /// Run the pipeline whenever there is a PR to these specified branches
154 /// (supports glob syntax)
155 pub branches: Vec<String>,
156 /// Specify any branches which should be filtered out from the list of
157 /// `branches` (supports glob syntax)
158 pub exclude_branches: Vec<String>,
159 /// Whether to batch changes per branch.
160 pub batch: bool,
161}
162
163impl Default for AdoPrTriggers {
164 fn default() -> Self {
165 Self {
166 branches: Vec::new(),
167 exclude_branches: Vec::new(),
168 run_on_draft: false,
169 auto_cancel: true,
170 }
171 }
172}
173
174/// ADO repository resource.
175#[derive(Debug)]
176pub struct AdoResourcesRepository {
177 /// Type of repo that is being connected to.
178 pub repo_type: AdoResourcesRepositoryType,
179 /// Repository name. Format depends on `repo_type`.
180 pub name: String,
181 /// git ref to checkout.
182 pub git_ref: AdoResourcesRepositoryRef,
183 /// (optional) ID of the service endpoint connecting to this repository.
184 pub endpoint: Option<String>,
185}
186
187/// ADO repository resource type
188#[derive(Debug)]
189pub enum AdoResourcesRepositoryType {
190 /// Azure Repos Git repository
191 AzureReposGit,
192 /// Github repository
193 GitHub,
194}
195
196/// ADO repository ref
197#[derive(Debug)]
198pub enum AdoResourcesRepositoryRef<P = UseParameter<String>> {
199 /// Hard-coded ref (e.g: refs/heads/main)
200 Fixed(String),
201 /// Connected to pipeline-level parameter
202 Parameter(P),
203}
204
205/// Trigger Github Actions pipelines via Continuous Integration
206///
207/// NOTE: Github Actions doesn't support specifying the branch when triggered by `schedule`.
208/// To run on a specific branch, modify the branch checked out in the pipeline.
209#[derive(Default, Debug)]
210pub struct GhScheduleTriggers {
211 /// Run the pipeline in a schedule, as specified by a cron string
212 pub cron: String,
213}
214
215/// Trigger Github Actions pipelines per PR
216#[derive(Debug)]
217pub struct GhPrTriggers {
218 /// Run the pipeline whenever there is a PR to these specified branches
219 /// (supports glob syntax)
220 pub branches: Vec<String>,
221 /// Specify any branches which should be filtered out from the list of
222 /// `branches` (supports glob syntax)
223 pub exclude_branches: Vec<String>,
224 /// Run the pipeline even if the PR is a draft PR. Defaults to `false`.
225 pub run_on_draft: bool,
226 /// Automatically cancel the pipeline run if a new commit lands in the
227 /// branch. Defaults to `true`.
228 pub auto_cancel: bool,
229}
230
231/// Trigger Github Actions pipelines per PR
232#[derive(Debug, Default)]
233pub struct GhCiTriggers {
234 /// Run the pipeline whenever there is a PR to these specified branches
235 /// (supports glob syntax)
236 pub branches: Vec<String>,
237 /// Specify any branches which should be filtered out from the list of
238 /// `branches` (supports glob syntax)
239 pub exclude_branches: Vec<String>,
240 /// Run the pipeline whenever there is a PR to these specified tags
241 /// (supports glob syntax)
242 pub tags: Vec<String>,
243 /// Specify any tags which should be filtered out from the list of `tags`
244 /// (supports glob syntax)
245 pub exclude_tags: Vec<String>,
246}
247
248impl Default for GhPrTriggers {
249 fn default() -> Self {
250 Self {
251 branches: Vec::new(),
252 exclude_branches: Vec::new(),
253 run_on_draft: false,
254 auto_cancel: true,
255 }
256 }
257}
258
259#[derive(Debug, Clone, PartialEq)]
260pub enum GhRunnerOsLabel {
261 UbuntuLatest,
262 Ubuntu2204,
263 Ubuntu2004,
264 WindowsLatest,
265 Windows2022,
266 Windows2019,
267 MacOsLatest,
268 MacOs14,
269 MacOs13,
270 MacOs12,
271 MacOs11,
272 Custom(String),
273}
274
275/// GitHub runner type
276#[derive(Debug, Clone, PartialEq)]
277pub enum GhRunner {
278 // See <https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#choosing-github-hosted-runners>
279 // for more details.
280 GhHosted(GhRunnerOsLabel),
281 // Self hosted runners are selected by matching runner labels to <labels>.
282 // 'self-hosted' is a common label for self hosted runners, but is not required.
283 // Labels are case-insensitive and can take the form of arbitrary strings.
284 // See <https://docs.github.com/en/actions/hosting-your-own-runners> for more details.
285 SelfHosted(Vec<String>),
286 // This uses a runner belonging to <group> that matches all labels in <labels>.
287 // See <https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#choosing-github-hosted-runners>
288 // for more details.
289 RunnerGroup { group: String, labels: Vec<String> },
290}
291
292#[derive(Clone, Debug)]
293#[must_use]
294pub struct UseParameter<T> {
295 idx: usize,
296 _kind: std::marker::PhantomData<T>,
297}
298
299/// Opaque handle to an artifact which must be published by a single job.
300#[must_use]
301pub struct PublishArtifact {
302 idx: usize,
303}
304
305/// Opaque handle to an artifact which can be used by one or more jobs.
306#[derive(Clone)]
307#[must_use]
308pub struct UseArtifact {
309 idx: usize,
310}
311
312#[derive(Default)]
313pub struct Pipeline {
314 jobs: Vec<PipelineJobMetadata>,
315 artifacts: Vec<ArtifactMeta>,
316 parameters: Vec<ParameterMeta>,
317 extra_deps: BTreeSet<(usize, usize)>,
318 // builder internal
319 artifact_names: BTreeSet<String>,
320 dummy_done_idx: usize,
321 global_patchfns: Vec<crate::patch::PatchFn>,
322 inject_all_jobs_with: Option<Box<dyn for<'a> Fn(PipelineJob<'a>) -> PipelineJob<'a>>>,
323 // backend specific
324 ado_name: Option<String>,
325 ado_schedule_triggers: Vec<AdoScheduleTriggers>,
326 ado_ci_triggers: Option<AdoCiTriggers>,
327 ado_pr_triggers: Option<AdoPrTriggers>,
328 ado_resources_repository: Vec<InternalAdoResourcesRepository>,
329 ado_bootstrap_template: String,
330 ado_variables: BTreeMap<String, String>,
331 ado_post_process_yaml_cb: Option<Box<dyn FnOnce(serde_yaml::Value) -> serde_yaml::Value>>,
332 gh_name: Option<String>,
333 gh_schedule_triggers: Vec<GhScheduleTriggers>,
334 gh_ci_triggers: Option<GhCiTriggers>,
335 gh_pr_triggers: Option<GhPrTriggers>,
336 gh_bootstrap_template: String,
337}
338
339impl Pipeline {
340 pub fn new() -> Pipeline {
341 Pipeline::default()
342 }
343
344 /// Inject all pipeline jobs with some common logic. (e.g: to resolve common
345 /// configuration requirements shared by all jobs).
346 ///
347 /// Can only be invoked once per pipeline.
348 #[track_caller]
349 pub fn inject_all_jobs_with(
350 &mut self,
351 cb: impl for<'a> Fn(PipelineJob<'a>) -> PipelineJob<'a> + 'static,
352 ) -> &mut Self {
353 if self.inject_all_jobs_with.is_some() {
354 panic!("can only call inject_all_jobs_with once!")
355 }
356 self.inject_all_jobs_with = Some(Box::new(cb));
357 self
358 }
359
360 /// (ADO only) Provide a YAML template used to bootstrap flowey at the start
361 /// of an ADO pipeline.
362 ///
363 /// The template has access to the following vars, which will be statically
364 /// interpolated into the template's text:
365 ///
366 /// - `{{FLOWEY_OUTDIR}}`
367 /// - Directory to copy artifacts into.
368 /// - NOTE: this var will include `\` on Windows, and `/` on linux!
369 /// - `{{FLOWEY_BIN_EXTENSION}}`
370 /// - Extension of the expected flowey bin (either "", or ".exe")
371 /// - `{{FLOWEY_CRATE}}`
372 /// - Name of the project-specific flowey crate to be built
373 /// - `{{FLOWEY_TARGET}}`
374 /// - The target-triple flowey is being built for
375 /// - `{{FLOWEY_PIPELINE_PATH}}`
376 /// - Repo-root relative path to the pipeline (as provided when
377 /// generating the pipeline via the flowey CLI)
378 ///
379 /// The template's sole responsibility is to copy 3 files into the
380 /// `{{FLOWEY_OUTDIR}}`:
381 ///
382 /// 1. The bootstrapped flowey binary, with the file name
383 /// `flowey{{FLOWEY_BIN_EXTENSION}}`
384 /// 2. Two files called `pipeline.yaml` and `pipeline.json`, which are
385 /// copied of the pipeline YAML and pipeline JSON currently being run.
386 /// `{{FLOWEY_PIPELINE_PATH}}` is provided as a way to disambiguate in
387 /// cases where the same template is being for multiple pipelines (e.g: a
388 /// debug vs. release pipeline).
389 pub fn ado_set_flowey_bootstrap_template(&mut self, template: String) -> &mut Self {
390 self.ado_bootstrap_template = template;
391 self
392 }
393
394 /// (ADO only) Provide a callback function which will be used to
395 /// post-process any YAML flowey generates for the pipeline.
396 ///
397 /// Until flowey defines a stable API for maintaining out-of-tree backends,
398 /// this method can be used to integrate the output from the generic ADO
399 /// backend with any organization-specific templates that one may be
400 /// required to use (e.g: for compliance reasons).
401 pub fn ado_post_process_yaml(
402 &mut self,
403 cb: impl FnOnce(serde_yaml::Value) -> serde_yaml::Value + 'static,
404 ) -> &mut Self {
405 self.ado_post_process_yaml_cb = Some(Box::new(cb));
406 self
407 }
408
409 /// (ADO only) Add a new scheduled CI trigger. Can be called multiple times
410 /// to set up multiple schedules runs.
411 pub fn ado_add_schedule_trigger(&mut self, triggers: AdoScheduleTriggers) -> &mut Self {
412 self.ado_schedule_triggers.push(triggers);
413 self
414 }
415
416 /// (ADO only) Set a PR trigger. Calling this method multiple times will
417 /// overwrite any previously set triggers.
418 pub fn ado_set_pr_triggers(&mut self, triggers: AdoPrTriggers) -> &mut Self {
419 self.ado_pr_triggers = Some(triggers);
420 self
421 }
422
423 /// (ADO only) Set a CI trigger. Calling this method multiple times will
424 /// overwrite any previously set triggers.
425 pub fn ado_set_ci_triggers(&mut self, triggers: AdoCiTriggers) -> &mut Self {
426 self.ado_ci_triggers = Some(triggers);
427 self
428 }
429
430 /// (ADO only) Declare a new repository resource, returning a type-safe
431 /// handle which downstream ADO steps are able to consume via
432 /// [`AdoStepServices::resolve_repository_id`](crate::node::user_facing::AdoStepServices::resolve_repository_id).
433 pub fn ado_add_resources_repository(
434 &mut self,
435 repo: AdoResourcesRepository,
436 ) -> AdoResourcesRepositoryId {
437 let AdoResourcesRepository {
438 repo_type,
439 name,
440 git_ref,
441 endpoint,
442 } = repo;
443
444 let repo_id = format!("repo{}", self.ado_resources_repository.len());
445
446 self.ado_resources_repository
447 .push(InternalAdoResourcesRepository {
448 repo_id: repo_id.clone(),
449 repo_type,
450 name,
451 git_ref: match git_ref {
452 AdoResourcesRepositoryRef::Fixed(s) => AdoResourcesRepositoryRef::Fixed(s),
453 AdoResourcesRepositoryRef::Parameter(p) => {
454 AdoResourcesRepositoryRef::Parameter(p.idx)
455 }
456 },
457 endpoint,
458 });
459 AdoResourcesRepositoryId { repo_id }
460 }
461
462 /// (GitHub Actions only) Set the pipeline-level name.
463 ///
464 /// <https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#name>
465 pub fn gh_set_name(&mut self, name: impl AsRef<str>) -> &mut Self {
466 self.gh_name = Some(name.as_ref().into());
467 self
468 }
469
470 /// Provide a YAML template used to bootstrap flowey at the start of an GitHub
471 /// pipeline.
472 ///
473 /// The template has access to the following vars, which will be statically
474 /// interpolated into the template's text:
475 ///
476 /// - `{{FLOWEY_OUTDIR}}`
477 /// - Directory to copy artifacts into.
478 /// - NOTE: this var will include `\` on Windows, and `/` on linux!
479 /// - `{{FLOWEY_BIN_EXTENSION}}`
480 /// - Extension of the expected flowey bin (either "", or ".exe")
481 /// - `{{FLOWEY_CRATE}}`
482 /// - Name of the project-specific flowey crate to be built
483 /// - `{{FLOWEY_TARGET}}`
484 /// - The target-triple flowey is being built for
485 /// - `{{FLOWEY_PIPELINE_PATH}}`
486 /// - Repo-root relative path to the pipeline (as provided when
487 /// generating the pipeline via the flowey CLI)
488 ///
489 /// The template's sole responsibility is to copy 3 files into the
490 /// `{{FLOWEY_OUTDIR}}`:
491 ///
492 /// 1. The bootstrapped flowey binary, with the file name
493 /// `flowey{{FLOWEY_BIN_EXTENSION}}`
494 /// 2. Two files called `pipeline.yaml` and `pipeline.json`, which are
495 /// copied of the pipeline YAML and pipeline JSON currently being run.
496 /// `{{FLOWEY_PIPELINE_PATH}}` is provided as a way to disambiguate in
497 /// cases where the same template is being for multiple pipelines (e.g: a
498 /// debug vs. release pipeline).
499 pub fn gh_set_flowey_bootstrap_template(&mut self, template: String) -> &mut Self {
500 self.gh_bootstrap_template = template;
501 self
502 }
503
504 /// (GitHub Actions only) Add a new scheduled CI trigger. Can be called multiple times
505 /// to set up multiple schedules runs.
506 pub fn gh_add_schedule_trigger(&mut self, triggers: GhScheduleTriggers) -> &mut Self {
507 self.gh_schedule_triggers.push(triggers);
508 self
509 }
510
511 /// (GitHub Actions only) Set a PR trigger. Calling this method multiple times will
512 /// overwrite any previously set triggers.
513 pub fn gh_set_pr_triggers(&mut self, triggers: GhPrTriggers) -> &mut Self {
514 self.gh_pr_triggers = Some(triggers);
515 self
516 }
517
518 /// (GitHub Actions only) Set a CI trigger. Calling this method multiple times will
519 /// overwrite any previously set triggers.
520 pub fn gh_set_ci_triggers(&mut self, triggers: GhCiTriggers) -> &mut Self {
521 self.gh_ci_triggers = Some(triggers);
522 self
523 }
524
525 /// (GitHub Actions only) Use a pre-defined GitHub Actions secret variable.
526 ///
527 /// For more information on defining secrets for use in GitHub Actions, see
528 /// <https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions>
529 pub fn gh_use_secret(&mut self, secret_name: impl AsRef<str>) -> GhContextVar {
530 GhContextVar::from_secrets(secret_name)
531 }
532
533 pub fn new_job(
534 &mut self,
535 platform: FlowPlatform,
536 arch: FlowArch,
537 label: impl AsRef<str>,
538 ) -> PipelineJob<'_> {
539 let idx = self.jobs.len();
540 self.jobs.push(PipelineJobMetadata {
541 root_nodes: BTreeMap::new(),
542 patches: ResolvedPatches::build(),
543 label: label.as_ref().into(),
544 platform,
545 arch,
546 cond_param_idx: None,
547 ado_pool: None,
548 ado_variables: BTreeMap::new(),
549 gh_override_if: None,
550 gh_global_env: BTreeMap::new(),
551 gh_pool: None,
552 gh_permissions: BTreeMap::new(),
553 });
554
555 PipelineJob {
556 pipeline: self,
557 job_idx: idx,
558 }
559 }
560
561 /// Declare a dependency between two jobs that does is not a result of an
562 /// artifact.
563 pub fn non_artifact_dep(
564 &mut self,
565 job: &PipelineJobHandle,
566 depends_on_job: &PipelineJobHandle,
567 ) -> &mut Self {
568 self.extra_deps
569 .insert((depends_on_job.job_idx, job.job_idx));
570 self
571 }
572
573 #[track_caller]
574 pub fn new_artifact(&mut self, name: impl AsRef<str>) -> (PublishArtifact, UseArtifact) {
575 let name = name.as_ref();
576 let owned_name = name.to_string();
577
578 let not_exists = self.artifact_names.insert(owned_name.clone());
579 if !not_exists {
580 panic!("duplicate artifact name: {}", name)
581 }
582
583 let idx = self.artifacts.len();
584 self.artifacts.push(ArtifactMeta {
585 name: owned_name,
586 published_by_job: None,
587 used_by_jobs: BTreeSet::new(),
588 });
589
590 (PublishArtifact { idx }, UseArtifact { idx })
591 }
592
593 /// (ADO only) Set the pipeline-level name.
594 ///
595 /// <https://learn.microsoft.com/en-us/azure/devops/pipelines/process/run-number?view=azure-devops&tabs=yaml>
596 pub fn ado_add_name(&mut self, name: String) -> &mut Self {
597 self.ado_name = Some(name);
598 self
599 }
600
601 /// (ADO only) Declare a pipeline-level, named, read-only ADO variable.
602 ///
603 /// `name` and `value` are both arbitrary strings.
604 ///
605 /// Returns an instance of [`AdoRuntimeVar`], which, if need be, can be
606 /// converted into a [`ReadVar<String>`] using
607 /// [`NodeCtx::get_ado_variable`].
608 ///
609 /// NOTE: Unless required by some particular third-party task, it's strongly
610 /// recommended to _avoid_ using this method, and to simply use
611 /// [`ReadVar::from_static`] to get a obtain a static variable.
612 ///
613 /// [`NodeCtx::get_ado_variable`]: crate::node::NodeCtx::get_ado_variable
614 pub fn ado_new_named_variable(
615 &mut self,
616 name: impl AsRef<str>,
617 value: impl AsRef<str>,
618 ) -> AdoRuntimeVar {
619 let name = name.as_ref();
620 let value = value.as_ref();
621
622 self.ado_variables.insert(name.into(), value.into());
623
624 // safe, since we'll ensure that the global exists in the ADO backend
625 AdoRuntimeVar::dangerous_from_global(name, false)
626 }
627
628 /// (ADO only) Declare multiple pipeline-level, named, read-only ADO
629 /// variables at once.
630 ///
631 /// This is a convenience method to streamline invoking
632 /// [`Self::ado_new_named_variable`] multiple times.
633 ///
634 /// NOTE: Unless required by some particular third-party task, it's strongly
635 /// recommended to _avoid_ using this method, and to simply use
636 /// [`ReadVar::from_static`] to get a obtain a static variable.
637 ///
638 /// DEVNOTE: In the future, this API may be updated to return a handle that
639 /// will allow resolving the resulting `AdoRuntimeVar`, but for
640 /// implementation expediency, this API does not currently do this. If you
641 /// need to read the value of this variable at runtime, you may need to
642 /// invoke [`AdoRuntimeVar::dangerous_from_global`] manually.
643 ///
644 /// [`NodeCtx::get_ado_variable`]: crate::node::NodeCtx::get_ado_variable
645 pub fn ado_new_named_variables<K, V>(
646 &mut self,
647 vars: impl IntoIterator<Item = (K, V)>,
648 ) -> &mut Self
649 where
650 K: AsRef<str>,
651 V: AsRef<str>,
652 {
653 self.ado_variables.extend(
654 vars.into_iter()
655 .map(|(k, v)| (k.as_ref().into(), v.as_ref().into())),
656 );
657 self
658 }
659
660 /// Declare a pipeline-level runtime parameter with type `bool`.
661 ///
662 /// To obtain a [`ReadVar<bool>`] that can be used within a node, use the
663 /// [`PipelineJobCtx::use_parameter`] method.
664 ///
665 /// `description` is an arbitrary string, which will be be shown to users.
666 ///
667 /// `default` is the default value for the parameter. If none is provided,
668 /// the parameter _must_ be specified in order for the pipeline to run.
669 ///
670 /// `possible_values` can be used to limit the set of valid values the
671 /// parameter accepts.
672 pub fn new_parameter_bool(
673 &mut self,
674 description: impl AsRef<str>,
675 default: Option<bool>,
676 ) -> UseParameter<bool> {
677 let idx = self.parameters.len();
678 self.parameters.push(ParameterMeta {
679 parameter: Parameter::Bool {
680 description: description.as_ref().into(),
681 default,
682 },
683 used_by_jobs: BTreeSet::new(),
684 });
685
686 UseParameter {
687 idx,
688 _kind: std::marker::PhantomData,
689 }
690 }
691
692 /// Declare a pipeline-level runtime parameter with type `i64`.
693 ///
694 /// To obtain a [`ReadVar<i64>`] that can be used within a node, use the
695 /// [`PipelineJobCtx::use_parameter`] method.
696 ///
697 /// `description` is an arbitrary string, which will be be shown to users.
698 ///
699 /// `default` is the default value for the parameter. If none is provided,
700 /// the parameter _must_ be specified in order for the pipeline to run.
701 ///
702 /// `possible_values` can be used to limit the set of valid values the
703 /// parameter accepts.
704 pub fn new_parameter_num(
705 &mut self,
706 description: impl AsRef<str>,
707 default: Option<i64>,
708 possible_values: Option<Vec<i64>>,
709 ) -> UseParameter<i64> {
710 let idx = self.parameters.len();
711 self.parameters.push(ParameterMeta {
712 parameter: Parameter::Num {
713 description: description.as_ref().into(),
714 default,
715 possible_values,
716 },
717 used_by_jobs: BTreeSet::new(),
718 });
719
720 UseParameter {
721 idx,
722 _kind: std::marker::PhantomData,
723 }
724 }
725
726 /// Declare a pipeline-level runtime parameter with type `String`.
727 ///
728 /// To obtain a [`ReadVar<String>`] that can be used within a node, use the
729 /// [`PipelineJobCtx::use_parameter`] method.
730 ///
731 /// `description` is an arbitrary string, which will be be shown to users.
732 ///
733 /// `default` is the default value for the parameter. If none is provided,
734 /// the parameter _must_ be specified in order for the pipeline to run.
735 ///
736 /// `possible_values` allows restricting inputs to a set of possible values.
737 /// Depending on the backend, these options may be presented as a set of
738 /// radio buttons, a dropdown menu, or something in that vein. If `None`,
739 /// then any string is allowed.
740 pub fn new_parameter_string(
741 &mut self,
742 description: impl AsRef<str>,
743 default: Option<impl AsRef<str>>,
744 possible_values: Option<Vec<String>>,
745 ) -> UseParameter<String> {
746 let idx = self.parameters.len();
747 self.parameters.push(ParameterMeta {
748 parameter: Parameter::String {
749 description: description.as_ref().into(),
750 default: default.map(|x| x.as_ref().into()),
751 possible_values,
752 },
753 used_by_jobs: BTreeSet::new(),
754 });
755
756 UseParameter {
757 idx,
758 _kind: std::marker::PhantomData,
759 }
760 }
761}
762
763pub struct PipelineJobCtx<'a> {
764 pipeline: &'a mut Pipeline,
765 job_idx: usize,
766}
767
768impl PipelineJobCtx<'_> {
769 /// Create a new `WriteVar<SideEffect>` anchored to the pipeline job.
770 pub fn new_done_handle(&mut self) -> WriteVar<crate::node::SideEffect> {
771 self.pipeline.dummy_done_idx += 1;
772 crate::node::thin_air_write_runtime_var(
773 format!("start{}", self.pipeline.dummy_done_idx),
774 false,
775 )
776 }
777
778 /// Claim that this job will use this artifact, obtaining a path to a folder
779 /// with the artifact's contents.
780 pub fn use_artifact(&mut self, artifact: &UseArtifact) -> ReadVar<PathBuf> {
781 self.pipeline.artifacts[artifact.idx]
782 .used_by_jobs
783 .insert(self.job_idx);
784
785 crate::node::thin_air_read_runtime_var(
786 consistent_artifact_runtime_var_name(&self.pipeline.artifacts[artifact.idx].name, true),
787 false,
788 )
789 }
790
791 /// Claim that this job will publish this artifact, obtaining a path to a
792 /// fresh, empty folder which will be published as the specific artifact at
793 /// the end of the job.
794 pub fn publish_artifact(&mut self, artifact: PublishArtifact) -> ReadVar<PathBuf> {
795 let existing = self.pipeline.artifacts[artifact.idx]
796 .published_by_job
797 .replace(self.job_idx);
798 assert!(existing.is_none()); // PublishArtifact isn't cloneable
799
800 crate::node::thin_air_read_runtime_var(
801 consistent_artifact_runtime_var_name(
802 &self.pipeline.artifacts[artifact.idx].name,
803 false,
804 ),
805 false,
806 )
807 }
808
809 /// Obtain a `ReadVar<T>` corresponding to a pipeline parameter which is
810 /// specified at runtime.
811 pub fn use_parameter<T>(&mut self, param: UseParameter<T>) -> ReadVar<T>
812 where
813 T: Serialize + DeserializeOwned,
814 {
815 self.pipeline.parameters[param.idx]
816 .used_by_jobs
817 .insert(self.job_idx);
818
819 crate::node::thin_air_read_runtime_var(format!("param{}", param.idx), false)
820 }
821
822 /// Shortcut which allows defining a bool pipeline parameter within a Job.
823 ///
824 /// To share a single parameter between multiple jobs, don't use this method
825 /// - use [`Pipeline::new_parameter_bool`] + [`Self::use_parameter`] instead.
826 pub fn new_parameter_bool(
827 &mut self,
828 description: impl AsRef<str>,
829 default: Option<bool>,
830 ) -> ReadVar<bool> {
831 let param = self.pipeline.new_parameter_bool(description, default);
832 self.use_parameter(param)
833 }
834
835 /// Shortcut which allows defining a number pipeline parameter within a Job.
836 ///
837 /// To share a single parameter between multiple jobs, don't use this method
838 /// - use [`Pipeline::new_parameter_num`] + [`Self::use_parameter`] instead.
839 pub fn new_parameter_num(
840 &mut self,
841 description: impl AsRef<str>,
842 default: Option<i64>,
843 possible_values: Option<Vec<i64>>,
844 ) -> ReadVar<i64> {
845 let param = self
846 .pipeline
847 .new_parameter_num(description, default, possible_values);
848 self.use_parameter(param)
849 }
850
851 /// Shortcut which allows defining a string pipeline parameter within a Job.
852 ///
853 /// To share a single parameter between multiple jobs, don't use this method
854 /// - use [`Pipeline::new_parameter_string`] + [`Self::use_parameter`] instead.
855 pub fn new_parameter_string(
856 &mut self,
857 description: impl AsRef<str>,
858 default: Option<String>,
859 possible_values: Option<Vec<String>>,
860 ) -> ReadVar<String> {
861 let param = self
862 .pipeline
863 .new_parameter_string(description, default, possible_values);
864 self.use_parameter(param)
865 }
866}
867
868#[must_use]
869pub struct PipelineJob<'a> {
870 pipeline: &'a mut Pipeline,
871 job_idx: usize,
872}
873
874impl PipelineJob<'_> {
875 /// (ADO only) specify which agent pool this job will be run on.
876 pub fn ado_set_pool(self, pool: impl AsRef<str>) -> Self {
877 self.ado_set_pool_with_demands(pool, Vec::new())
878 }
879
880 /// (ADO only) specify which agent pool this job will be run on, with
881 /// additional special runner demands.
882 pub fn ado_set_pool_with_demands(self, pool: impl AsRef<str>, demands: Vec<String>) -> Self {
883 self.pipeline.jobs[self.job_idx].ado_pool = Some(AdoPool {
884 name: pool.as_ref().into(),
885 demands,
886 });
887 self
888 }
889
890 /// (ADO only) Declare a job-level, named, read-only ADO variable.
891 ///
892 /// `name` and `value` are both arbitrary strings, which may include ADO
893 /// template expressions.
894 ///
895 /// NOTE: Unless required by some particular third-party task, it's strongly
896 /// recommended to _avoid_ using this method, and to simply use
897 /// [`ReadVar::from_static`] to get a obtain a static variable.
898 ///
899 /// DEVNOTE: In the future, this API may be updated to return a handle that
900 /// will allow resolving the resulting `AdoRuntimeVar`, but for
901 /// implementation expediency, this API does not currently do this. If you
902 /// need to read the value of this variable at runtime, you may need to
903 /// invoke [`AdoRuntimeVar::dangerous_from_global`] manually.
904 ///
905 /// [`NodeCtx::get_ado_variable`]: crate::node::NodeCtx::get_ado_variable
906 pub fn ado_new_named_variable(self, name: impl AsRef<str>, value: impl AsRef<str>) -> Self {
907 let name = name.as_ref();
908 let value = value.as_ref();
909 self.pipeline.jobs[self.job_idx]
910 .ado_variables
911 .insert(name.into(), value.into());
912 self
913 }
914
915 /// (ADO only) Declare multiple job-level, named, read-only ADO variables at
916 /// once.
917 ///
918 /// This is a convenience method to streamline invoking
919 /// [`Self::ado_new_named_variable`] multiple times.
920 ///
921 /// NOTE: Unless required by some particular third-party task, it's strongly
922 /// recommended to _avoid_ using this method, and to simply use
923 /// [`ReadVar::from_static`] to get a obtain a static variable.
924 ///
925 /// DEVNOTE: In the future, this API may be updated to return a handle that
926 /// will allow resolving the resulting `AdoRuntimeVar`, but for
927 /// implementation expediency, this API does not currently do this. If you
928 /// need to read the value of this variable at runtime, you may need to
929 /// invoke [`AdoRuntimeVar::dangerous_from_global`] manually.
930 ///
931 /// [`NodeCtx::get_ado_variable`]: crate::node::NodeCtx::get_ado_variable
932 pub fn ado_new_named_variables<K, V>(self, vars: impl IntoIterator<Item = (K, V)>) -> Self
933 where
934 K: AsRef<str>,
935 V: AsRef<str>,
936 {
937 self.pipeline.jobs[self.job_idx].ado_variables.extend(
938 vars.into_iter()
939 .map(|(k, v)| (k.as_ref().into(), v.as_ref().into())),
940 );
941 self
942 }
943
944 /// (GitHub Actions only) specify which Github runner this job will be run on.
945 pub fn gh_set_pool(self, pool: GhRunner) -> Self {
946 self.pipeline.jobs[self.job_idx].gh_pool = Some(pool);
947 self
948 }
949
950 /// (GitHub Actions only) Manually override the `if:` condition for this
951 /// particular job.
952 ///
953 /// **This is dangerous**, as an improperly set `if` condition may break
954 /// downstream flowey jobs which assume flowey is in control of the job's
955 /// scheduling logic.
956 ///
957 /// See
958 /// <https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#jobsjob_idif>
959 /// for more info.
960 pub fn gh_dangerous_override_if(self, condition: impl AsRef<str>) -> Self {
961 self.pipeline.jobs[self.job_idx].gh_override_if = Some(condition.as_ref().into());
962 self
963 }
964
965 /// (GitHub Actions only) Declare a global job-level environment variable,
966 /// visible to all downstream steps.
967 ///
968 /// `name` and `value` are both arbitrary strings, which may include GitHub
969 /// Actions template expressions.
970 ///
971 /// **This is dangerous**, as it is easy to misuse this API in order to
972 /// write a node which takes an implicit dependency on there being a global
973 /// variable set on its behalf by the top-level pipeline code, making it
974 /// difficult to "locally reason" about the behavior of a node simply by
975 /// reading its code.
976 ///
977 /// Whenever possible, nodes should "late bind" environment variables:
978 /// accepting a compile-time / runtime flowey parameter, and then setting it
979 /// prior to executing a child command that requires it.
980 ///
981 /// Only use this API in exceptional cases, such as obtaining an environment
982 /// variable whose value is determined by a job-level GitHub Actions
983 /// expression evaluation.
984 pub fn gh_dangerous_global_env_var(
985 self,
986 name: impl AsRef<str>,
987 value: impl AsRef<str>,
988 ) -> Self {
989 let name = name.as_ref();
990 let value = value.as_ref();
991 self.pipeline.jobs[self.job_idx]
992 .gh_global_env
993 .insert(name.into(), value.into());
994 self
995 }
996
997 /// (GitHub Actions only) Grant permissions required by nodes in the job.
998 ///
999 /// For a given node handle, grant the specified permissions.
1000 /// The list provided must match the permissions specified within the node
1001 /// using `requires_permission`.
1002 ///
1003 /// NOTE: While this method is called at a node-level for auditability, the emitted
1004 /// yaml grants permissions at the job-level.
1005 ///
1006 /// This can lead to weird situations where node 1 might not specify a permission
1007 /// required according to Github Actions, but due to job-level granting of the permission
1008 /// by another node 2, the pipeline executes even though it wouldn't if node 2 was removed.
1009 ///
1010 /// For available permission scopes and their descriptions, see
1011 /// <https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions>.
1012 pub fn gh_grant_permissions<N: FlowNodeBase + 'static>(
1013 self,
1014 permissions: impl IntoIterator<Item = (GhPermission, GhPermissionValue)>,
1015 ) -> Self {
1016 let node_handle = NodeHandle::from_type::<N>();
1017 for (permission, value) in permissions {
1018 self.pipeline.jobs[self.job_idx]
1019 .gh_permissions
1020 .entry(node_handle)
1021 .or_default()
1022 .insert(permission, value);
1023 }
1024 self
1025 }
1026
1027 pub fn apply_patchfn(self, patchfn: crate::patch::PatchFn) -> Self {
1028 self.pipeline.jobs[self.job_idx]
1029 .patches
1030 .apply_patchfn(patchfn);
1031 self
1032 }
1033
1034 /// Only run the job if the specified condition is true.
1035 ///
1036 /// When running locally, the `cond`'s default value will be used to
1037 /// determine if the job will be run.
1038 pub fn with_condition(self, cond: UseParameter<bool>) -> Self {
1039 self.pipeline.jobs[self.job_idx].cond_param_idx = Some(cond.idx);
1040 self
1041 }
1042
1043 /// Add a flow node which will be run as part of the job.
1044 pub fn dep_on<R: IntoRequest + 'static>(
1045 self,
1046 f: impl FnOnce(&mut PipelineJobCtx<'_>) -> R,
1047 ) -> Self {
1048 // JobToNodeCtx will ensure artifact deps are taken care of
1049 let req = f(&mut PipelineJobCtx {
1050 pipeline: self.pipeline,
1051 job_idx: self.job_idx,
1052 });
1053
1054 self.pipeline.jobs[self.job_idx]
1055 .root_nodes
1056 .entry(NodeHandle::from_type::<R::Node>())
1057 .or_default()
1058 .push(serde_json::to_vec(&req.into_request()).unwrap().into());
1059
1060 self
1061 }
1062
1063 /// Finish describing the pipeline job.
1064 pub fn finish(self) -> PipelineJobHandle {
1065 PipelineJobHandle {
1066 job_idx: self.job_idx,
1067 }
1068 }
1069}
1070
1071#[derive(Clone)]
1072pub struct PipelineJobHandle {
1073 job_idx: usize,
1074}
1075
1076impl PipelineJobHandle {
1077 pub fn is_handle_for(&self, job: &PipelineJob<'_>) -> bool {
1078 self.job_idx == job.job_idx
1079 }
1080}
1081
1082#[derive(Clone, Copy)]
1083pub enum PipelineBackendHint {
1084 /// Pipeline is being run on the user's dev machine (via bash / direct run)
1085 Local,
1086 /// Pipeline is run on ADO
1087 Ado,
1088 /// Pipeline is run on GitHub Actions
1089 Github,
1090}
1091
1092pub trait IntoPipeline {
1093 fn into_pipeline(self, backend_hint: PipelineBackendHint) -> anyhow::Result<Pipeline>;
1094}
1095
1096/// Structs which should only be used by top-level flowey emitters. If you're a
1097/// pipeline author, these are not types you need to care about!
1098pub mod internal {
1099 use super::*;
1100 use std::collections::BTreeMap;
1101
1102 pub fn consistent_artifact_runtime_var_name(artifact: impl AsRef<str>, is_use: bool) -> String {
1103 format!(
1104 "artifact_{}_{}",
1105 if is_use { "use_from" } else { "publish_from" },
1106 artifact.as_ref()
1107 )
1108 }
1109
1110 pub fn consistent_param_runtime_var_name(idx: usize) -> String {
1111 format!("param{idx}")
1112 }
1113
1114 #[derive(Debug)]
1115 pub struct InternalAdoResourcesRepository {
1116 /// flowey-generated unique repo identifier
1117 pub repo_id: String,
1118 /// Type of repo that is being connected to.
1119 pub repo_type: AdoResourcesRepositoryType,
1120 /// Repository name. Format depends on `repo_type`.
1121 pub name: String,
1122 /// git ref to checkout.
1123 pub git_ref: AdoResourcesRepositoryRef<usize>,
1124 /// (optional) ID of the service endpoint connecting to this repository.
1125 pub endpoint: Option<String>,
1126 }
1127
1128 pub struct PipelineJobMetadata {
1129 pub root_nodes: BTreeMap<NodeHandle, Vec<Box<[u8]>>>,
1130 pub patches: PatchResolver,
1131 pub label: String,
1132 pub platform: FlowPlatform,
1133 pub arch: FlowArch,
1134 pub cond_param_idx: Option<usize>,
1135 // backend specific
1136 pub ado_pool: Option<AdoPool>,
1137 pub ado_variables: BTreeMap<String, String>,
1138 pub gh_override_if: Option<String>,
1139 pub gh_pool: Option<GhRunner>,
1140 pub gh_global_env: BTreeMap<String, String>,
1141 pub gh_permissions: BTreeMap<NodeHandle, BTreeMap<GhPermission, GhPermissionValue>>,
1142 }
1143
1144 // TODO: support a more structured format for demands
1145 // See https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/pool-demands
1146 #[derive(Debug, Clone)]
1147 pub struct AdoPool {
1148 pub name: String,
1149 pub demands: Vec<String>,
1150 }
1151
1152 #[derive(Debug)]
1153 pub struct ArtifactMeta {
1154 pub name: String,
1155 pub published_by_job: Option<usize>,
1156 pub used_by_jobs: BTreeSet<usize>,
1157 }
1158
1159 #[derive(Debug)]
1160 pub struct ParameterMeta {
1161 pub parameter: Parameter,
1162 pub used_by_jobs: BTreeSet<usize>,
1163 }
1164
1165 /// Mirror of [`Pipeline`], except with all field marked as `pub`.
1166 pub struct PipelineFinalized {
1167 pub jobs: Vec<PipelineJobMetadata>,
1168 pub artifacts: Vec<ArtifactMeta>,
1169 pub parameters: Vec<ParameterMeta>,
1170 pub extra_deps: BTreeSet<(usize, usize)>,
1171 // backend specific
1172 pub ado_name: Option<String>,
1173 pub ado_schedule_triggers: Vec<AdoScheduleTriggers>,
1174 pub ado_ci_triggers: Option<AdoCiTriggers>,
1175 pub ado_pr_triggers: Option<AdoPrTriggers>,
1176 pub ado_bootstrap_template: String,
1177 pub ado_resources_repository: Vec<InternalAdoResourcesRepository>,
1178 pub ado_post_process_yaml_cb:
1179 Option<Box<dyn FnOnce(serde_yaml::Value) -> serde_yaml::Value>>,
1180 pub ado_variables: BTreeMap<String, String>,
1181 pub gh_name: Option<String>,
1182 pub gh_schedule_triggers: Vec<GhScheduleTriggers>,
1183 pub gh_ci_triggers: Option<GhCiTriggers>,
1184 pub gh_pr_triggers: Option<GhPrTriggers>,
1185 pub gh_bootstrap_template: String,
1186 }
1187
1188 impl PipelineFinalized {
1189 pub fn from_pipeline(mut pipeline: Pipeline) -> Self {
1190 if let Some(cb) = pipeline.inject_all_jobs_with.take() {
1191 for job_idx in 0..pipeline.jobs.len() {
1192 let _ = cb(PipelineJob {
1193 pipeline: &mut pipeline,
1194 job_idx,
1195 });
1196 }
1197 }
1198
1199 let Pipeline {
1200 mut jobs,
1201 artifacts,
1202 parameters,
1203 extra_deps,
1204 ado_name,
1205 ado_bootstrap_template,
1206 ado_schedule_triggers,
1207 ado_ci_triggers,
1208 ado_pr_triggers,
1209 ado_resources_repository,
1210 ado_post_process_yaml_cb,
1211 ado_variables,
1212 gh_name,
1213 gh_schedule_triggers,
1214 gh_ci_triggers,
1215 gh_pr_triggers,
1216 gh_bootstrap_template,
1217 // not relevant to consumer code
1218 dummy_done_idx: _,
1219 artifact_names: _,
1220 global_patchfns,
1221 inject_all_jobs_with: _, // processed above
1222 } = pipeline;
1223
1224 for patchfn in global_patchfns {
1225 for job in &mut jobs {
1226 job.patches.apply_patchfn(patchfn)
1227 }
1228 }
1229
1230 Self {
1231 jobs,
1232 artifacts,
1233 parameters,
1234 extra_deps,
1235 ado_name,
1236 ado_schedule_triggers,
1237 ado_ci_triggers,
1238 ado_pr_triggers,
1239 ado_bootstrap_template,
1240 ado_resources_repository,
1241 ado_post_process_yaml_cb,
1242 ado_variables,
1243 gh_name,
1244 gh_schedule_triggers,
1245 gh_ci_triggers,
1246 gh_pr_triggers,
1247 gh_bootstrap_template,
1248 }
1249 }
1250 }
1251
1252 #[derive(Debug, Clone)]
1253 pub enum Parameter {
1254 Bool {
1255 description: String,
1256 default: Option<bool>,
1257 },
1258 String {
1259 description: String,
1260 default: Option<String>,
1261 possible_values: Option<Vec<String>>,
1262 },
1263 Num {
1264 description: String,
1265 default: Option<i64>,
1266 possible_values: Option<Vec<i64>>,
1267 },
1268 }
1269}
1270