// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
//! Core types and traits used to create and work with flowey pipelines.
use self::internal::*;
use crate::node::steps::ado::AdoResourcesRepositoryId;
use crate::node::user_facing::AdoRuntimeVar;
use crate::node::user_facing::GhContextVar;
use crate::node::user_facing::GhPermission;
use crate::node::user_facing::GhPermissionValue;
use crate::node::FlowArch;
use crate::node::FlowNodeBase;
use crate::node::FlowPlatform;
use crate::node::FlowPlatformLinuxDistro;
use crate::node::IntoRequest;
use crate::node::NodeHandle;
use crate::node::ReadVar;
use crate::node::WriteVar;
use crate::patch::PatchResolver;
use crate::patch::ResolvedPatches;
use serde::de::DeserializeOwned;
use serde::Serialize;
use std::collections::BTreeMap;
use std::collections::BTreeSet;
use std::path::PathBuf;
/// Pipeline types which are considered "user facing", and included in the
/// `flowey` prelude.
pub mod user_facing {
pub use super::AdoCiTriggers;
pub use super::AdoPrTriggers;
pub use super::AdoResourcesRepository;
pub use super::AdoResourcesRepositoryRef;
pub use super::AdoResourcesRepositoryType;
pub use super::AdoScheduleTriggers;
pub use super::GhCiTriggers;
pub use super::GhPrTriggers;
pub use super::GhRunner;
pub use super::GhRunnerOsLabel;
pub use super::GhScheduleTriggers;
pub use super::HostExt;
pub use super::IntoPipeline;
pub use super::Pipeline;
pub use super::PipelineBackendHint;
pub use super::PipelineJob;
pub use super::PipelineJobCtx;
pub use super::PipelineJobHandle;
pub use super::PublishArtifact;
pub use super::UseArtifact;
pub use super::UseParameter;
pub use crate::node::FlowArch;
pub use crate::node::FlowPlatform;
}
fn linux_distro() -> FlowPlatformLinuxDistro {
if let Ok(etc_os_release) = fs_err::read_to_string("/etc/os-release") {
if etc_os_release.contains("ID=ubuntu") {
FlowPlatformLinuxDistro::Ubuntu
} else if etc_os_release.contains("ID=fedora") {
FlowPlatformLinuxDistro::Fedora
} else {
FlowPlatformLinuxDistro::Unknown
}
} else {
FlowPlatformLinuxDistro::Unknown
}
}
pub trait HostExt: Sized {
/// Return the value for the current host machine.
///
/// Will panic on non-local backends.
fn host(backend_hint: PipelineBackendHint) -> Self;
}
impl HostExt for FlowPlatform {
/// Return the platform of the current host machine.
///
/// Will panic on non-local backends.
fn host(backend_hint: PipelineBackendHint) -> Self {
if !matches!(backend_hint, PipelineBackendHint::Local) {
panic!("can only use `FlowPlatform::host` when defining a local-only pipeline");
}
if cfg!(target_os = "windows") {
Self::Windows
} else if cfg!(target_os = "linux") {
Self::Linux(linux_distro())
} else if cfg!(target_os = "macos") {
Self::MacOs
} else {
panic!("no valid host-os")
}
}
}
impl HostExt for FlowArch {
/// Return the arch of the current host machine.
///
/// Will panic on non-local backends.
fn host(backend_hint: PipelineBackendHint) -> Self {
if !matches!(backend_hint, PipelineBackendHint::Local) {
panic!("can only use `FlowArch::host` when defining a local-only pipeline");
}
// xtask-fmt allow-target-arch oneoff-flowey
if cfg!(target_arch = "x86_64") {
Self::X86_64
// xtask-fmt allow-target-arch oneoff-flowey
} else if cfg!(target_arch = "aarch64") {
Self::Aarch64
} else {
panic!("no valid host-arch")
}
}
}
/// Trigger ADO pipelines via Continuous Integration
#[derive(Default, Debug)]
pub struct AdoScheduleTriggers {
/// Friendly name for the scheduled run
pub display_name: String,
/// Run the pipeline whenever there is a commit on these specified branches
/// (supports glob syntax)
pub branches: Vec<String>,
/// Specify any branches which should be filtered out from the list of
/// `branches` (supports glob syntax)
pub exclude_branches: Vec<String>,
/// Run the pipeline in a schedule, as specified by a cron string
pub cron: String,
}
/// Trigger ADO pipelines per PR
#[derive(Debug)]
pub struct AdoPrTriggers {
/// Run the pipeline whenever there is a PR to these specified branches
/// (supports glob syntax)
pub branches: Vec<String>,
/// Specify any branches which should be filtered out from the list of
/// `branches` (supports glob syntax)
pub exclude_branches: Vec<String>,
/// Run the pipeline even if the PR is a draft PR. Defaults to `false`.
pub run_on_draft: bool,
/// Automatically cancel the pipeline run if a new commit lands in the
/// branch. Defaults to `true`.
pub auto_cancel: bool,
}
/// Trigger ADO pipelines per PR
#[derive(Debug, Default)]
pub struct AdoCiTriggers {
/// Run the pipeline whenever there is a PR to these specified branches
/// (supports glob syntax)
pub branches: Vec<String>,
/// Specify any branches which should be filtered out from the list of
/// `branches` (supports glob syntax)
pub exclude_branches: Vec<String>,
/// Whether to batch changes per branch.
pub batch: bool,
}
impl Default for AdoPrTriggers {
fn default() -> Self {
Self {
branches: Vec::new(),
exclude_branches: Vec::new(),
run_on_draft: false,
auto_cancel: true,
}
}
}
/// ADO repository resource.
#[derive(Debug)]
pub struct AdoResourcesRepository {
/// Type of repo that is being connected to.
pub repo_type: AdoResourcesRepositoryType,
/// Repository name. Format depends on `repo_type`.
pub name: String,
/// git ref to checkout.
pub git_ref: AdoResourcesRepositoryRef,
/// (optional) ID of the service endpoint connecting to this repository.
pub endpoint: Option<String>,
}
/// ADO repository resource type
#[derive(Debug)]
pub enum AdoResourcesRepositoryType {
/// Azure Repos Git repository
AzureReposGit,
/// Github repository
GitHub,
}
/// ADO repository ref
#[derive(Debug)]
pub enum AdoResourcesRepositoryRef<P = UseParameter<String>> {
/// Hard-coded ref (e.g: refs/heads/main)
Fixed(String),
/// Connected to pipeline-level parameter
Parameter(P),
}
/// Trigger Github Actions pipelines via Continuous Integration
///
/// NOTE: Github Actions doesn't support specifying the branch when triggered by `schedule`.
/// To run on a specific branch, modify the branch checked out in the pipeline.
#[derive(Default, Debug)]
pub struct GhScheduleTriggers {
/// Run the pipeline in a schedule, as specified by a cron string
pub cron: String,
}
/// Trigger Github Actions pipelines per PR
#[derive(Debug)]
pub struct GhPrTriggers {
/// Run the pipeline whenever there is a PR to these specified branches
/// (supports glob syntax)
pub branches: Vec<String>,
/// Specify any branches which should be filtered out from the list of
/// `branches` (supports glob syntax)
pub exclude_branches: Vec<String>,
/// Run the pipeline even if the PR is a draft PR. Defaults to `false`.
pub run_on_draft: bool,
/// Automatically cancel the pipeline run if a new commit lands in the
/// branch. Defaults to `true`.
pub auto_cancel: bool,
}
/// Trigger Github Actions pipelines per PR
#[derive(Debug, Default)]
pub struct GhCiTriggers {
/// Run the pipeline whenever there is a PR to these specified branches
/// (supports glob syntax)
pub branches: Vec<String>,
/// Specify any branches which should be filtered out from the list of
/// `branches` (supports glob syntax)
pub exclude_branches: Vec<String>,
/// Run the pipeline whenever there is a PR to these specified tags
/// (supports glob syntax)
pub tags: Vec<String>,
/// Specify any tags which should be filtered out from the list of `tags`
/// (supports glob syntax)
pub exclude_tags: Vec<String>,
}
impl Default for GhPrTriggers {
fn default() -> Self {
Self {
branches: Vec::new(),
exclude_branches: Vec::new(),
run_on_draft: false,
auto_cancel: true,
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub enum GhRunnerOsLabel {
UbuntuLatest,
Ubuntu2204,
Ubuntu2004,
WindowsLatest,
Windows2022,
Windows2019,
MacOsLatest,
MacOs14,
MacOs13,
MacOs12,
MacOs11,
Custom(String),
}
/// GitHub runner type
#[derive(Debug, Clone, PartialEq)]
pub enum GhRunner {
// See <https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#choosing-github-hosted-runners>
// for more details.
GhHosted(GhRunnerOsLabel),
// Self hosted runners are selected by matching runner labels to <labels>.
// 'self-hosted' is a common label for self hosted runners, but is not required.
// Labels are case-insensitive and can take the form of arbitrary strings.
// See <https://docs.github.com/en/actions/hosting-your-own-runners> for more details.
SelfHosted(Vec<String>),
// This uses a runner belonging to <group> that matches all labels in <labels>.
// See <https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#choosing-github-hosted-runners>
// for more details.
RunnerGroup { group: String, labels: Vec<String> },
}
#[derive(Clone, Debug)]
#[must_use]
pub struct UseParameter<T> {
idx: usize,
_kind: std::marker::PhantomData<T>,
}
/// Opaque handle to an artifact which must be published by a single job.
#[must_use]
pub struct PublishArtifact {
idx: usize,
}
/// Opaque handle to an artifact which can be used by one or more jobs.
#[derive(Clone)]
#[must_use]
pub struct UseArtifact {
idx: usize,
}
#[derive(Default)]
pub struct Pipeline {
jobs: Vec<PipelineJobMetadata>,
artifacts: Vec<ArtifactMeta>,
parameters: Vec<ParameterMeta>,
extra_deps: BTreeSet<(usize, usize)>,
// builder internal
artifact_names: BTreeSet<String>,
dummy_done_idx: usize,
global_patchfns: Vec<crate::patch::PatchFn>,
inject_all_jobs_with: Option<Box<dyn for<'a> Fn(PipelineJob<'a>) -> PipelineJob<'a>>>,
// backend specific
ado_name: Option<String>,
ado_schedule_triggers: Vec<AdoScheduleTriggers>,
ado_ci_triggers: Option<AdoCiTriggers>,
ado_pr_triggers: Option<AdoPrTriggers>,
ado_resources_repository: Vec<InternalAdoResourcesRepository>,
ado_bootstrap_template: String,
ado_variables: BTreeMap<String, String>,
ado_post_process_yaml_cb: Option<Box<dyn FnOnce(serde_yaml::Value) -> serde_yaml::Value>>,
gh_name: Option<String>,
gh_schedule_triggers: Vec<GhScheduleTriggers>,
gh_ci_triggers: Option<GhCiTriggers>,
gh_pr_triggers: Option<GhPrTriggers>,
gh_bootstrap_template: String,
}
impl Pipeline {
pub fn new() -> Pipeline {
Pipeline::default()
}
/// Inject all pipeline jobs with some common logic. (e.g: to resolve common
/// configuration requirements shared by all jobs).
///
/// Can only be invoked once per pipeline.
#[track_caller]
pub fn inject_all_jobs_with(
&mut self,
cb: impl for<'a> Fn(PipelineJob<'a>) -> PipelineJob<'a> + 'static,
) -> &mut Self {
if self.inject_all_jobs_with.is_some() {
panic!("can only call inject_all_jobs_with once!")
}
self.inject_all_jobs_with = Some(Box::new(cb));
self
}
/// (ADO only) Provide a YAML template used to bootstrap flowey at the start
/// of an ADO pipeline.
///
/// The template has access to the following vars, which will be statically
/// interpolated into the template's text:
///
/// - `{{FLOWEY_OUTDIR}}`
/// - Directory to copy artifacts into.
/// - NOTE: this var will include `\` on Windows, and `/` on linux!
/// - `{{FLOWEY_BIN_EXTENSION}}`
/// - Extension of the expected flowey bin (either "", or ".exe")
/// - `{{FLOWEY_CRATE}}`
/// - Name of the project-specific flowey crate to be built
/// - `{{FLOWEY_TARGET}}`
/// - The target-triple flowey is being built for
/// - `{{FLOWEY_PIPELINE_PATH}}`
/// - Repo-root relative path to the pipeline (as provided when
/// generating the pipeline via the flowey CLI)
///
/// The template's sole responsibility is to copy 3 files into the
/// `{{FLOWEY_OUTDIR}}`:
///
/// 1. The bootstrapped flowey binary, with the file name
/// `flowey{{FLOWEY_BIN_EXTENSION}}`
/// 2. Two files called `pipeline.yaml` and `pipeline.json`, which are
/// copied of the pipeline YAML and pipeline JSON currently being run.
/// `{{FLOWEY_PIPELINE_PATH}}` is provided as a way to disambiguate in
/// cases where the same template is being for multiple pipelines (e.g: a
/// debug vs. release pipeline).
pub fn ado_set_flowey_bootstrap_template(&mut self, template: String) -> &mut Self {
self.ado_bootstrap_template = template;
self
}
/// (ADO only) Provide a callback function which will be used to
/// post-process any YAML flowey generates for the pipeline.
///
/// Until flowey defines a stable API for maintaining out-of-tree backends,
/// this method can be used to integrate the output from the generic ADO
/// backend with any organization-specific templates that one may be
/// required to use (e.g: for compliance reasons).
pub fn ado_post_process_yaml(
&mut self,
cb: impl FnOnce(serde_yaml::Value) -> serde_yaml::Value + 'static,
) -> &mut Self {
self.ado_post_process_yaml_cb = Some(Box::new(cb));
self
}
/// (ADO only) Add a new scheduled CI trigger. Can be called multiple times
/// to set up multiple schedules runs.
pub fn ado_add_schedule_trigger(&mut self, triggers: AdoScheduleTriggers) -> &mut Self {
self.ado_schedule_triggers.push(triggers);
self
}
/// (ADO only) Set a PR trigger. Calling this method multiple times will
/// overwrite any previously set triggers.
pub fn ado_set_pr_triggers(&mut self, triggers: AdoPrTriggers) -> &mut Self {
self.ado_pr_triggers = Some(triggers);
self
}
/// (ADO only) Set a CI trigger. Calling this method multiple times will
/// overwrite any previously set triggers.
pub fn ado_set_ci_triggers(&mut self, triggers: AdoCiTriggers) -> &mut Self {
self.ado_ci_triggers = Some(triggers);
self
}
/// (ADO only) Declare a new repository resource, returning a type-safe
/// handle which downstream ADO steps are able to consume via
/// [`AdoStepServices::resolve_repository_id`](crate::node::user_facing::AdoStepServices::resolve_repository_id).
pub fn ado_add_resources_repository(
&mut self,
repo: AdoResourcesRepository,
) -> AdoResourcesRepositoryId {
let AdoResourcesRepository {
repo_type,
name,
git_ref,
endpoint,
} = repo;
let repo_id = format!("repo{}", self.ado_resources_repository.len());
self.ado_resources_repository
.push(InternalAdoResourcesRepository {
repo_id: repo_id.clone(),
repo_type,
name,
git_ref: match git_ref {
AdoResourcesRepositoryRef::Fixed(s) => AdoResourcesRepositoryRef::Fixed(s),
AdoResourcesRepositoryRef::Parameter(p) => {
AdoResourcesRepositoryRef::Parameter(p.idx)
}
},
endpoint,
});
AdoResourcesRepositoryId { repo_id }
}
/// (GitHub Actions only) Set the pipeline-level name.
///
/// <https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#name>
pub fn gh_set_name(&mut self, name: impl AsRef<str>) -> &mut Self {
self.gh_name = Some(name.as_ref().into());
self
}
/// Provide a YAML template used to bootstrap flowey at the start of an GitHub
/// pipeline.
///
/// The template has access to the following vars, which will be statically
/// interpolated into the template's text:
///
/// - `{{FLOWEY_OUTDIR}}`
/// - Directory to copy artifacts into.
/// - NOTE: this var will include `\` on Windows, and `/` on linux!
/// - `{{FLOWEY_BIN_EXTENSION}}`
/// - Extension of the expected flowey bin (either "", or ".exe")
/// - `{{FLOWEY_CRATE}}`
/// - Name of the project-specific flowey crate to be built
/// - `{{FLOWEY_TARGET}}`
/// - The target-triple flowey is being built for
/// - `{{FLOWEY_PIPELINE_PATH}}`
/// - Repo-root relative path to the pipeline (as provided when
/// generating the pipeline via the flowey CLI)
///
/// The template's sole responsibility is to copy 3 files into the
/// `{{FLOWEY_OUTDIR}}`:
///
/// 1. The bootstrapped flowey binary, with the file name
/// `flowey{{FLOWEY_BIN_EXTENSION}}`
/// 2. Two files called `pipeline.yaml` and `pipeline.json`, which are
/// copied of the pipeline YAML and pipeline JSON currently being run.
/// `{{FLOWEY_PIPELINE_PATH}}` is provided as a way to disambiguate in
/// cases where the same template is being for multiple pipelines (e.g: a
/// debug vs. release pipeline).
pub fn gh_set_flowey_bootstrap_template(&mut self, template: String) -> &mut Self {
self.gh_bootstrap_template = template;
self
}
/// (GitHub Actions only) Add a new scheduled CI trigger. Can be called multiple times
/// to set up multiple schedules runs.
pub fn gh_add_schedule_trigger(&mut self, triggers: GhScheduleTriggers) -> &mut Self {
self.gh_schedule_triggers.push(triggers);
self
}
/// (GitHub Actions only) Set a PR trigger. Calling this method multiple times will
/// overwrite any previously set triggers.
pub fn gh_set_pr_triggers(&mut self, triggers: GhPrTriggers) -> &mut Self {
self.gh_pr_triggers = Some(triggers);
self
}
/// (GitHub Actions only) Set a CI trigger. Calling this method multiple times will
/// overwrite any previously set triggers.
pub fn gh_set_ci_triggers(&mut self, triggers: GhCiTriggers) -> &mut Self {
self.gh_ci_triggers = Some(triggers);
self
}
/// (GitHub Actions only) Use a pre-defined GitHub Actions secret variable.
///
/// For more information on defining secrets for use in GitHub Actions, see
/// <https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions>
pub fn gh_use_secret(&mut self, secret_name: impl AsRef<str>) -> GhContextVar {
GhContextVar::from_secrets(secret_name)
}
pub fn new_job(
&mut self,
platform: FlowPlatform,
arch: FlowArch,
label: impl AsRef<str>,
) -> PipelineJob<'_> {
let idx = self.jobs.len();
self.jobs.push(PipelineJobMetadata {
root_nodes: BTreeMap::new(),
patches: ResolvedPatches::build(),
label: label.as_ref().into(),
platform,
arch,
cond_param_idx: None,
ado_pool: None,
ado_variables: BTreeMap::new(),
gh_override_if: None,
gh_global_env: BTreeMap::new(),
gh_pool: None,
gh_permissions: BTreeMap::new(),
});
PipelineJob {
pipeline: self,
job_idx: idx,
}
}
/// Declare a dependency between two jobs that does is not a result of an
/// artifact.
pub fn non_artifact_dep(
&mut self,
job: &PipelineJobHandle,
depends_on_job: &PipelineJobHandle,
) -> &mut Self {
self.extra_deps
.insert((depends_on_job.job_idx, job.job_idx));
self
}
#[track_caller]
pub fn new_artifact(&mut self, name: impl AsRef<str>) -> (PublishArtifact, UseArtifact) {
let name = name.as_ref();
let owned_name = name.to_string();
let not_exists = self.artifact_names.insert(owned_name.clone());
if !not_exists {
panic!("duplicate artifact name: {}", name)
}
let idx = self.artifacts.len();
self.artifacts.push(ArtifactMeta {
name: owned_name,
published_by_job: None,
used_by_jobs: BTreeSet::new(),
});
(PublishArtifact { idx }, UseArtifact { idx })
}
/// (ADO only) Set the pipeline-level name.
///
/// <https://learn.microsoft.com/en-us/azure/devops/pipelines/process/run-number?view=azure-devops&tabs=yaml>
pub fn ado_add_name(&mut self, name: String) -> &mut Self {
self.ado_name = Some(name);
self
}
/// (ADO only) Declare a pipeline-level, named, read-only ADO variable.
///
/// `name` and `value` are both arbitrary strings.
///
/// Returns an instance of [`AdoRuntimeVar`], which, if need be, can be
/// converted into a [`ReadVar<String>`] using
/// [`NodeCtx::get_ado_variable`].
///
/// NOTE: Unless required by some particular third-party task, it's strongly
/// recommended to _avoid_ using this method, and to simply use
/// [`ReadVar::from_static`] to get a obtain a static variable.
///
/// [`NodeCtx::get_ado_variable`]: crate::node::NodeCtx::get_ado_variable
pub fn ado_new_named_variable(
&mut self,
name: impl AsRef<str>,
value: impl AsRef<str>,
) -> AdoRuntimeVar {
let name = name.as_ref();
let value = value.as_ref();
self.ado_variables.insert(name.into(), value.into());
// safe, since we'll ensure that the global exists in the ADO backend
AdoRuntimeVar::dangerous_from_global(name, false)
}
/// (ADO only) Declare multiple pipeline-level, named, read-only ADO
/// variables at once.
///
/// This is a convenience method to streamline invoking
/// [`Self::ado_new_named_variable`] multiple times.
///
/// NOTE: Unless required by some particular third-party task, it's strongly
/// recommended to _avoid_ using this method, and to simply use
/// [`ReadVar::from_static`] to get a obtain a static variable.
///
/// DEVNOTE: In the future, this API may be updated to return a handle that
/// will allow resolving the resulting `AdoRuntimeVar`, but for
/// implementation expediency, this API does not currently do this. If you
/// need to read the value of this variable at runtime, you may need to
/// invoke [`AdoRuntimeVar::dangerous_from_global`] manually.
///
/// [`NodeCtx::get_ado_variable`]: crate::node::NodeCtx::get_ado_variable
pub fn ado_new_named_variables<K, V>(
&mut self,
vars: impl IntoIterator<Item = (K, V)>,
) -> &mut Self
where
K: AsRef<str>,
V: AsRef<str>,
{
self.ado_variables.extend(
vars.into_iter()
.map(|(k, v)| (k.as_ref().into(), v.as_ref().into())),
);
self
}
/// Declare a pipeline-level runtime parameter with type `bool`.
///
/// To obtain a [`ReadVar<bool>`] that can be used within a node, use the
/// [`PipelineJobCtx::use_parameter`] method.
///
/// `description` is an arbitrary string, which will be be shown to users.
///
/// `default` is the default value for the parameter. If none is provided,
/// the parameter _must_ be specified in order for the pipeline to run.
///
/// `possible_values` can be used to limit the set of valid values the
/// parameter accepts.
pub fn new_parameter_bool(
&mut self,
description: impl AsRef<str>,
default: Option<bool>,
) -> UseParameter<bool> {
let idx = self.parameters.len();
self.parameters.push(ParameterMeta {
parameter: Parameter::Bool {
description: description.as_ref().into(),
default,
},
used_by_jobs: BTreeSet::new(),
});
UseParameter {
idx,
_kind: std::marker::PhantomData,
}
}
/// Declare a pipeline-level runtime parameter with type `i64`.
///
/// To obtain a [`ReadVar<i64>`] that can be used within a node, use the
/// [`PipelineJobCtx::use_parameter`] method.
///
/// `description` is an arbitrary string, which will be be shown to users.
///
/// `default` is the default value for the parameter. If none is provided,
/// the parameter _must_ be specified in order for the pipeline to run.
///
/// `possible_values` can be used to limit the set of valid values the
/// parameter accepts.
pub fn new_parameter_num(
&mut self,
description: impl AsRef<str>,
default: Option<i64>,
possible_values: Option<Vec<i64>>,
) -> UseParameter<i64> {
let idx = self.parameters.len();
self.parameters.push(ParameterMeta {
parameter: Parameter::Num {
description: description.as_ref().into(),
default,
possible_values,
},
used_by_jobs: BTreeSet::new(),
});
UseParameter {
idx,
_kind: std::marker::PhantomData,
}
}
/// Declare a pipeline-level runtime parameter with type `String`.
///
/// To obtain a [`ReadVar<String>`] that can be used within a node, use the
/// [`PipelineJobCtx::use_parameter`] method.
///
/// `description` is an arbitrary string, which will be be shown to users.
///
/// `default` is the default value for the parameter. If none is provided,
/// the parameter _must_ be specified in order for the pipeline to run.
///
/// `possible_values` allows restricting inputs to a set of possible values.
/// Depending on the backend, these options may be presented as a set of
/// radio buttons, a dropdown menu, or something in that vein. If `None`,
/// then any string is allowed.
pub fn new_parameter_string(
&mut self,
description: impl AsRef<str>,
default: Option<impl AsRef<str>>,
possible_values: Option<Vec<String>>,
) -> UseParameter<String> {
let idx = self.parameters.len();
self.parameters.push(ParameterMeta {
parameter: Parameter::String {
description: description.as_ref().into(),
default: default.map(|x| x.as_ref().into()),
possible_values,
},
used_by_jobs: BTreeSet::new(),
});
UseParameter {
idx,
_kind: std::marker::PhantomData,
}
}
}
pub struct PipelineJobCtx<'a> {
pipeline: &'a mut Pipeline,
job_idx: usize,
}
impl PipelineJobCtx<'_> {
/// Create a new `WriteVar<SideEffect>` anchored to the pipeline job.
pub fn new_done_handle(&mut self) -> WriteVar<crate::node::SideEffect> {
self.pipeline.dummy_done_idx += 1;
crate::node::thin_air_write_runtime_var(
format!("start{}", self.pipeline.dummy_done_idx),
false,
)
}
/// Claim that this job will use this artifact, obtaining a path to a folder
/// with the artifact's contents.
pub fn use_artifact(&mut self, artifact: &UseArtifact) -> ReadVar<PathBuf> {
self.pipeline.artifacts[artifact.idx]
.used_by_jobs
.insert(self.job_idx);
crate::node::thin_air_read_runtime_var(
consistent_artifact_runtime_var_name(&self.pipeline.artifacts[artifact.idx].name, true),
false,
)
}
/// Claim that this job will publish this artifact, obtaining a path to a
/// fresh, empty folder which will be published as the specific artifact at
/// the end of the job.
pub fn publish_artifact(&mut self, artifact: PublishArtifact) -> ReadVar<PathBuf> {
let existing = self.pipeline.artifacts[artifact.idx]
.published_by_job
.replace(self.job_idx);
assert!(existing.is_none()); // PublishArtifact isn't cloneable
crate::node::thin_air_read_runtime_var(
consistent_artifact_runtime_var_name(
&self.pipeline.artifacts[artifact.idx].name,
false,
),
false,
)
}
/// Obtain a `ReadVar<T>` corresponding to a pipeline parameter which is
/// specified at runtime.
pub fn use_parameter<T>(&mut self, param: UseParameter<T>) -> ReadVar<T>
where
T: Serialize + DeserializeOwned,
{
self.pipeline.parameters[param.idx]
.used_by_jobs
.insert(self.job_idx);
crate::node::thin_air_read_runtime_var(format!("param{}", param.idx), false)
}
/// Shortcut which allows defining a bool pipeline parameter within a Job.
///
/// To share a single parameter between multiple jobs, don't use this method
/// - use [`Pipeline::new_parameter_bool`] + [`Self::use_parameter`] instead.
pub fn new_parameter_bool(
&mut self,
description: impl AsRef<str>,
default: Option<bool>,
) -> ReadVar<bool> {
let param = self.pipeline.new_parameter_bool(description, default);
self.use_parameter(param)
}
/// Shortcut which allows defining a number pipeline parameter within a Job.
///
/// To share a single parameter between multiple jobs, don't use this method
/// - use [`Pipeline::new_parameter_num`] + [`Self::use_parameter`] instead.
pub fn new_parameter_num(
&mut self,
description: impl AsRef<str>,
default: Option<i64>,
possible_values: Option<Vec<i64>>,
) -> ReadVar<i64> {
let param = self
.pipeline
.new_parameter_num(description, default, possible_values);
self.use_parameter(param)
}
/// Shortcut which allows defining a string pipeline parameter within a Job.
///
/// To share a single parameter between multiple jobs, don't use this method
/// - use [`Pipeline::new_parameter_string`] + [`Self::use_parameter`] instead.
pub fn new_parameter_string(
&mut self,
description: impl AsRef<str>,
default: Option<String>,
possible_values: Option<Vec<String>>,
) -> ReadVar<String> {
let param = self
.pipeline
.new_parameter_string(description, default, possible_values);
self.use_parameter(param)
}
}
#[must_use]
pub struct PipelineJob<'a> {
pipeline: &'a mut Pipeline,
job_idx: usize,
}
impl PipelineJob<'_> {
/// (ADO only) specify which agent pool this job will be run on.
pub fn ado_set_pool(self, pool: impl AsRef<str>) -> Self {
self.ado_set_pool_with_demands(pool, Vec::new())
}
/// (ADO only) specify which agent pool this job will be run on, with
/// additional special runner demands.
pub fn ado_set_pool_with_demands(self, pool: impl AsRef<str>, demands: Vec<String>) -> Self {
self.pipeline.jobs[self.job_idx].ado_pool = Some(AdoPool {
name: pool.as_ref().into(),
demands,
});
self
}
/// (ADO only) Declare a job-level, named, read-only ADO variable.
///
/// `name` and `value` are both arbitrary strings, which may include ADO
/// template expressions.
///
/// NOTE: Unless required by some particular third-party task, it's strongly
/// recommended to _avoid_ using this method, and to simply use
/// [`ReadVar::from_static`] to get a obtain a static variable.
///
/// DEVNOTE: In the future, this API may be updated to return a handle that
/// will allow resolving the resulting `AdoRuntimeVar`, but for
/// implementation expediency, this API does not currently do this. If you
/// need to read the value of this variable at runtime, you may need to
/// invoke [`AdoRuntimeVar::dangerous_from_global`] manually.
///
/// [`NodeCtx::get_ado_variable`]: crate::node::NodeCtx::get_ado_variable
pub fn ado_new_named_variable(self, name: impl AsRef<str>, value: impl AsRef<str>) -> Self {
let name = name.as_ref();
let value = value.as_ref();
self.pipeline.jobs[self.job_idx]
.ado_variables
.insert(name.into(), value.into());
self
}
/// (ADO only) Declare multiple job-level, named, read-only ADO variables at
/// once.
///
/// This is a convenience method to streamline invoking
/// [`Self::ado_new_named_variable`] multiple times.
///
/// NOTE: Unless required by some particular third-party task, it's strongly
/// recommended to _avoid_ using this method, and to simply use
/// [`ReadVar::from_static`] to get a obtain a static variable.
///
/// DEVNOTE: In the future, this API may be updated to return a handle that
/// will allow resolving the resulting `AdoRuntimeVar`, but for
/// implementation expediency, this API does not currently do this. If you
/// need to read the value of this variable at runtime, you may need to
/// invoke [`AdoRuntimeVar::dangerous_from_global`] manually.
///
/// [`NodeCtx::get_ado_variable`]: crate::node::NodeCtx::get_ado_variable
pub fn ado_new_named_variables<K, V>(self, vars: impl IntoIterator<Item = (K, V)>) -> Self
where
K: AsRef<str>,
V: AsRef<str>,
{
self.pipeline.jobs[self.job_idx].ado_variables.extend(
vars.into_iter()
.map(|(k, v)| (k.as_ref().into(), v.as_ref().into())),
);
self
}
/// (GitHub Actions only) specify which Github runner this job will be run on.
pub fn gh_set_pool(self, pool: GhRunner) -> Self {
self.pipeline.jobs[self.job_idx].gh_pool = Some(pool);
self
}
/// (GitHub Actions only) Manually override the `if:` condition for this
/// particular job.
///
/// **This is dangerous**, as an improperly set `if` condition may break
/// downstream flowey jobs which assume flowey is in control of the job's
/// scheduling logic.
///
/// See
/// <https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#jobsjob_idif>
/// for more info.
pub fn gh_dangerous_override_if(self, condition: impl AsRef<str>) -> Self {
self.pipeline.jobs[self.job_idx].gh_override_if = Some(condition.as_ref().into());
self
}
/// (GitHub Actions only) Declare a global job-level environment variable,
/// visible to all downstream steps.
///
/// `name` and `value` are both arbitrary strings, which may include GitHub
/// Actions template expressions.
///
/// **This is dangerous**, as it is easy to misuse this API in order to
/// write a node which takes an implicit dependency on there being a global
/// variable set on its behalf by the top-level pipeline code, making it
/// difficult to "locally reason" about the behavior of a node simply by
/// reading its code.
///
/// Whenever possible, nodes should "late bind" environment variables:
/// accepting a compile-time / runtime flowey parameter, and then setting it
/// prior to executing a child command that requires it.
///
/// Only use this API in exceptional cases, such as obtaining an environment
/// variable whose value is determined by a job-level GitHub Actions
/// expression evaluation.
pub fn gh_dangerous_global_env_var(
self,
name: impl AsRef<str>,
value: impl AsRef<str>,
) -> Self {
let name = name.as_ref();
let value = value.as_ref();
self.pipeline.jobs[self.job_idx]
.gh_global_env
.insert(name.into(), value.into());
self
}
/// (GitHub Actions only) Grant permissions required by nodes in the job.
///
/// For a given node handle, grant the specified permissions.
/// The list provided must match the permissions specified within the node
/// using `requires_permission`.
///
/// NOTE: While this method is called at a node-level for auditability, the emitted
/// yaml grants permissions at the job-level.
///
/// This can lead to weird situations where node 1 might not specify a permission
/// required according to Github Actions, but due to job-level granting of the permission
/// by another node 2, the pipeline executes even though it wouldn't if node 2 was removed.
///
/// For available permission scopes and their descriptions, see
/// <https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions>.
pub fn gh_grant_permissions<N: FlowNodeBase + 'static>(
self,
permissions: impl IntoIterator<Item = (GhPermission, GhPermissionValue)>,
) -> Self {
let node_handle = NodeHandle::from_type::<N>();
for (permission, value) in permissions {
self.pipeline.jobs[self.job_idx]
.gh_permissions
.entry(node_handle)
.or_default()
.insert(permission, value);
}
self
}
pub fn apply_patchfn(self, patchfn: crate::patch::PatchFn) -> Self {
self.pipeline.jobs[self.job_idx]
.patches
.apply_patchfn(patchfn);
self
}
/// Only run the job if the specified condition is true.
///
/// When running locally, the `cond`'s default value will be used to
/// determine if the job will be run.
pub fn with_condition(self, cond: UseParameter<bool>) -> Self {
self.pipeline.jobs[self.job_idx].cond_param_idx = Some(cond.idx);
self
}
/// Add a flow node which will be run as part of the job.
pub fn dep_on<R: IntoRequest + 'static>(
self,
f: impl FnOnce(&mut PipelineJobCtx<'_>) -> R,
) -> Self {
// JobToNodeCtx will ensure artifact deps are taken care of
let req = f(&mut PipelineJobCtx {
pipeline: self.pipeline,
job_idx: self.job_idx,
});
self.pipeline.jobs[self.job_idx]
.root_nodes
.entry(NodeHandle::from_type::<R::Node>())
.or_default()
.push(serde_json::to_vec(&req.into_request()).unwrap().into());
self
}
/// Finish describing the pipeline job.
pub fn finish(self) -> PipelineJobHandle {
PipelineJobHandle {
job_idx: self.job_idx,
}
}
}
#[derive(Clone)]
pub struct PipelineJobHandle {
job_idx: usize,
}
impl PipelineJobHandle {
pub fn is_handle_for(&self, job: &PipelineJob<'_>) -> bool {
self.job_idx == job.job_idx
}
}
#[derive(Clone, Copy)]
pub enum PipelineBackendHint {
/// Pipeline is being run on the user's dev machine (via bash / direct run)
Local,
/// Pipeline is run on ADO
Ado,
/// Pipeline is run on GitHub Actions
Github,
}
pub trait IntoPipeline {
fn into_pipeline(self, backend_hint: PipelineBackendHint) -> anyhow::Result<Pipeline>;
}
/// Structs which should only be used by top-level flowey emitters. If you're a
/// pipeline author, these are not types you need to care about!
pub mod internal {
use super::*;
use std::collections::BTreeMap;
pub fn consistent_artifact_runtime_var_name(artifact: impl AsRef<str>, is_use: bool) -> String {
format!(
"artifact_{}_{}",
if is_use { "use_from" } else { "publish_from" },
artifact.as_ref()
)
}
pub fn consistent_param_runtime_var_name(idx: usize) -> String {
format!("param{idx}")
}
#[derive(Debug)]
pub struct InternalAdoResourcesRepository {
/// flowey-generated unique repo identifier
pub repo_id: String,
/// Type of repo that is being connected to.
pub repo_type: AdoResourcesRepositoryType,
/// Repository name. Format depends on `repo_type`.
pub name: String,
/// git ref to checkout.
pub git_ref: AdoResourcesRepositoryRef<usize>,
/// (optional) ID of the service endpoint connecting to this repository.
pub endpoint: Option<String>,
}
pub struct PipelineJobMetadata {
pub root_nodes: BTreeMap<NodeHandle, Vec<Box<[u8]>>>,
pub patches: PatchResolver,
pub label: String,
pub platform: FlowPlatform,
pub arch: FlowArch,
pub cond_param_idx: Option<usize>,
// backend specific
pub ado_pool: Option<AdoPool>,
pub ado_variables: BTreeMap<String, String>,
pub gh_override_if: Option<String>,
pub gh_pool: Option<GhRunner>,
pub gh_global_env: BTreeMap<String, String>,
pub gh_permissions: BTreeMap<NodeHandle, BTreeMap<GhPermission, GhPermissionValue>>,
}
// TODO: support a more structured format for demands
// See https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/pool-demands
#[derive(Debug, Clone)]
pub struct AdoPool {
pub name: String,
pub demands: Vec<String>,
}
#[derive(Debug)]
pub struct ArtifactMeta {
pub name: String,
pub published_by_job: Option<usize>,
pub used_by_jobs: BTreeSet<usize>,
}
#[derive(Debug)]
pub struct ParameterMeta {
pub parameter: Parameter,
pub used_by_jobs: BTreeSet<usize>,
}
/// Mirror of [`Pipeline`], except with all field marked as `pub`.
pub struct PipelineFinalized {
pub jobs: Vec<PipelineJobMetadata>,
pub artifacts: Vec<ArtifactMeta>,
pub parameters: Vec<ParameterMeta>,
pub extra_deps: BTreeSet<(usize, usize)>,
// backend specific
pub ado_name: Option<String>,
pub ado_schedule_triggers: Vec<AdoScheduleTriggers>,
pub ado_ci_triggers: Option<AdoCiTriggers>,
pub ado_pr_triggers: Option<AdoPrTriggers>,
pub ado_bootstrap_template: String,
pub ado_resources_repository: Vec<InternalAdoResourcesRepository>,
pub ado_post_process_yaml_cb:
Option<Box<dyn FnOnce(serde_yaml::Value) -> serde_yaml::Value>>,
pub ado_variables: BTreeMap<String, String>,
pub gh_name: Option<String>,
pub gh_schedule_triggers: Vec<GhScheduleTriggers>,
pub gh_ci_triggers: Option<GhCiTriggers>,
pub gh_pr_triggers: Option<GhPrTriggers>,
pub gh_bootstrap_template: String,
}
impl PipelineFinalized {
pub fn from_pipeline(mut pipeline: Pipeline) -> Self {
if let Some(cb) = pipeline.inject_all_jobs_with.take() {
for job_idx in 0..pipeline.jobs.len() {
let _ = cb(PipelineJob {
pipeline: &mut pipeline,
job_idx,
});
}
}
let Pipeline {
mut jobs,
artifacts,
parameters,
extra_deps,
ado_name,
ado_bootstrap_template,
ado_schedule_triggers,
ado_ci_triggers,
ado_pr_triggers,
ado_resources_repository,
ado_post_process_yaml_cb,
ado_variables,
gh_name,
gh_schedule_triggers,
gh_ci_triggers,
gh_pr_triggers,
gh_bootstrap_template,
// not relevant to consumer code
dummy_done_idx: _,
artifact_names: _,
global_patchfns,
inject_all_jobs_with: _, // processed above
} = pipeline;
for patchfn in global_patchfns {
for job in &mut jobs {
job.patches.apply_patchfn(patchfn)
}
}
Self {
jobs,
artifacts,
parameters,
extra_deps,
ado_name,
ado_schedule_triggers,
ado_ci_triggers,
ado_pr_triggers,
ado_bootstrap_template,
ado_resources_repository,
ado_post_process_yaml_cb,
ado_variables,
gh_name,
gh_schedule_triggers,
gh_ci_triggers,
gh_pr_triggers,
gh_bootstrap_template,
}
}
}
#[derive(Debug, Clone)]
pub enum Parameter {
Bool {
description: String,
default: Option<bool>,
},
String {
description: String,
default: Option<String>,
possible_values: Option<Vec<String>>,
},
Num {
description: String,
default: Option<i64>,
possible_values: Option<Vec<i64>>,
},
}
}microsoft/openvmm
Publicmirrored fromhttps://github.com/microsoft/openvmmAvailable
flowey/flowey_core/src/pipeline.rs
1269lines · modepreview