GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/openvmm

Public

mirrored fromhttps://github.com/microsoft/openvmmAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
9fa0c3ee87af75e07fa974b6005348ae6b9349ff

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openvmm/flowey/flowey_core/src

flowey/flowey_core/src/node.rs

2611lines · modecode

RawDownload
Latest commit unavailable.
1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4//! Core types and traits used to create and work with flowey nodes.
5
6mod github_context;
7mod spec;
8
9pub use github_context::GhVarState;
10
11use self::steps::ado::AdoRuntimeVar;
12use self::steps::ado::AdoStepServices;
13use self::steps::github::GhStepBuilder;
14use self::steps::rust::RustRuntimeServices;
15use self::user_facing::ClaimedGhParam;
16use self::user_facing::GhPermission;
17use self::user_facing::GhPermissionValue;
18use crate::node::github_context::GhContextVarReader;
19use github_context::state::Root;
20use serde::Deserialize;
21use serde::Serialize;
22use serde::de::DeserializeOwned;
23use std::cell::RefCell;
24use std::collections::BTreeMap;
25use std::path::PathBuf;
26use std::rc::Rc;
27use user_facing::GhParam;
28
29/// Node types which are considered "user facing", and re-exported in the
30/// `flowey` crate.
31pub mod user_facing {
32 pub use super::ClaimVar;
33 pub use super::ClaimedReadVar;
34 pub use super::ClaimedWriteVar;
35 pub use super::FlowArch;
36 pub use super::FlowBackend;
37 pub use super::FlowNode;
38 pub use super::FlowPlatform;
39 pub use super::FlowPlatformKind;
40 pub use super::GhUserSecretVar;
41 pub use super::ImportCtx;
42 pub use super::IntoRequest;
43 pub use super::NodeCtx;
44 pub use super::ReadVar;
45 pub use super::SideEffect;
46 pub use super::SimpleFlowNode;
47 pub use super::StepCtx;
48 pub use super::VarClaimed;
49 pub use super::VarEqBacking;
50 pub use super::VarNotClaimed;
51 pub use super::WriteVar;
52 pub use super::steps::ado::AdoResourcesRepositoryId;
53 pub use super::steps::ado::AdoRuntimeVar;
54 pub use super::steps::ado::AdoStepServices;
55 pub use super::steps::github::ClaimedGhParam;
56 pub use super::steps::github::GhParam;
57 pub use super::steps::github::GhPermission;
58 pub use super::steps::github::GhPermissionValue;
59 pub use super::steps::rust::RustRuntimeServices;
60 pub use crate::flowey_request;
61 pub use crate::new_flow_node;
62 pub use crate::new_simple_flow_node;
63 pub use crate::node::FlowPlatformLinuxDistro;
64
65 /// Helper method to streamline request validation in cases where a value is
66 /// expected to be identical across all incoming requests.
67 pub fn same_across_all_reqs<T: PartialEq>(
68 req_name: &str,
69 var: &mut Option<T>,
70 new: T,
71 ) -> anyhow::Result<()> {
72 match (var.as_ref(), new) {
73 (None, v) => *var = Some(v),
74 (Some(old), new) => {
75 if *old != new {
76 anyhow::bail!("`{}` must be consistent across requests", req_name);
77 }
78 }
79 }
80
81 Ok(())
82 }
83
84 /// Helper method to streamline request validation in cases where a value is
85 /// expected to be identical across all incoming requests, using a custom
86 /// comparison function.
87 pub fn same_across_all_reqs_backing_var<V: VarEqBacking>(
88 req_name: &str,
89 var: &mut Option<V>,
90 new: V,
91 ) -> anyhow::Result<()> {
92 match (var.as_ref(), new) {
93 (None, v) => *var = Some(v),
94 (Some(old), new) => {
95 if !old.eq(&new) {
96 anyhow::bail!("`{}` must be consistent across requests", req_name);
97 }
98 }
99 }
100
101 Ok(())
102 }
103}
104
105/// Check if `ReadVar` / `WriteVar` instances are backed by the same underlying
106/// flowey Var.
107///
108/// # Why not use `Eq`? Why have a whole separate trait?
109///
110/// `ReadVar` and `WriteVar` are, in some sense, flowey's analog to
111/// "pointers", insofar as these types primary purpose is to mediate access to
112/// some contained value, as opposed to being "values" themselves.
113///
114/// Assuming you agree with this analogy, then we can apply the same logic to
115/// `ReadVar` and `WriteVar` as Rust does to `Box<T>` wrt. what the `Eq`
116/// implementation should mean.
117///
118/// Namely: `Eq` should check the equality of the _contained objects_, as
119/// opposed to the pointers themselves.
120///
121/// Unfortunately, unlike `Box<T>`, it is _impossible_ to have an `Eq` impl for
122/// `ReadVar` / `WriteVar` that checks contents for equality, due to the fact
123/// that these types exist at flow resolution time, whereas the values they
124/// contain only exist at flow runtime.
125///
126/// As such, we have a separate trait to perform different kinds of equality
127/// checks on Vars.
128pub trait VarEqBacking {
129 /// Check if `self` is backed by the same variable as `other`.
130 fn eq(&self, other: &Self) -> bool;
131}
132
133impl<T> VarEqBacking for WriteVar<T>
134where
135 T: Serialize + DeserializeOwned,
136{
137 fn eq(&self, other: &Self) -> bool {
138 self.backing_var == other.backing_var && self.is_secret == other.is_secret
139 }
140}
141
142impl<T> VarEqBacking for ReadVar<T>
143where
144 T: Serialize + DeserializeOwned + PartialEq + Eq + Clone,
145{
146 fn eq(&self, other: &Self) -> bool {
147 self.backing_var == other.backing_var && self.is_secret == other.is_secret
148 }
149}
150
151// TODO: this should be generic across all tuple sizes
152impl<T, U> VarEqBacking for (T, U)
153where
154 T: VarEqBacking,
155 U: VarEqBacking,
156{
157 fn eq(&self, other: &Self) -> bool {
158 (self.0.eq(&other.0)) && (self.1.eq(&other.1))
159 }
160}
161
162/// Uninhabited type corresponding to a step which performs a side-effect,
163/// without returning a specific value.
164///
165/// e.g: A step responsible for installing a package from `apt` might claim a
166/// `WriteVar<SideEffect>`, with any step requiring the package to have been
167/// installed prior being able to claim the corresponding `ReadVar<SideEffect>.`
168#[derive(Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Copy)]
169pub enum SideEffect {}
170
171/// Uninhabited type used to denote that a particular [`WriteVar`] / [`ReadVar`]
172/// is not currently claimed by any step, and cannot be directly accessed.
173#[derive(Clone, Debug, Serialize, Deserialize)]
174pub enum VarNotClaimed {}
175
176/// Uninhabited type used to denote that a particular [`WriteVar`] / [`ReadVar`]
177/// is currently claimed by a step, and can be read/written to.
178#[derive(Clone, Debug, Serialize, Deserialize)]
179pub enum VarClaimed {}
180
181/// Write a value into a flowey Var at runtime, which can then be read via a
182/// corresponding [`ReadVar`].
183///
184/// Vars in flowey must be serde de/serializable, in order to be de/serialized
185/// between multiple steps/nodes.
186///
187/// In order to write a value into a `WriteVar`, it must first be _claimed_ by a
188/// particular step (using the [`ClaimVar::claim`] API). Once claimed, the Var
189/// can be written to using APIs such as [`RustRuntimeServices::write`], or
190/// [`AdoStepServices::set_var`]
191///
192/// Note that it is only possible to write a value into a `WriteVar` _once_.
193/// Once the value has been written, the `WriteVar` type is immediately
194/// consumed, making it impossible to overwrite the stored value at some later
195/// point in execution.
196///
197/// This "write-once" property is foundational to flowey's execution model, as
198/// by recoding what step wrote to a Var, and what step(s) read from the Var, it
199/// is possible to infer what order steps must be run in.
200#[derive(Debug, Serialize, Deserialize)]
201pub struct WriteVar<T: Serialize + DeserializeOwned, C = VarNotClaimed> {
202 backing_var: String,
203 is_secret: bool,
204
205 #[serde(skip)]
206 _kind: core::marker::PhantomData<(T, C)>,
207}
208
209/// A [`WriteVar`] which has been claimed by a particular step, allowing it
210/// to be written to at runtime.
211pub type ClaimedWriteVar<T> = WriteVar<T, VarClaimed>;
212
213impl<T: Serialize + DeserializeOwned> WriteVar<T, VarNotClaimed> {
214 /// (Internal API) Switch the claim marker to "claimed".
215 fn into_claimed(self) -> WriteVar<T, VarClaimed> {
216 let Self {
217 backing_var,
218 is_secret,
219 _kind,
220 } = self;
221
222 WriteVar {
223 backing_var,
224 is_secret,
225 _kind: std::marker::PhantomData,
226 }
227 }
228
229 /// Create a new [`ReadVar`] from this [`WriteVar`] handle.
230 #[must_use]
231 pub fn new_reader(&self) -> ReadVar<T> {
232 ReadVar {
233 backing_var: ReadVarBacking::RuntimeVar(self.backing_var.clone()),
234 is_secret: self.is_secret,
235 _kind: std::marker::PhantomData,
236 }
237 }
238
239 /// Write a static value into the Var.
240 #[track_caller]
241 pub fn write_static(self, ctx: &mut NodeCtx<'_>, val: T)
242 where
243 T: 'static,
244 {
245 let val = ReadVar::from_static(val);
246 val.write_into(ctx, self, |v| v);
247 }
248}
249
250impl<T: Serialize + DeserializeOwned, C> WriteVar<T, C> {
251 /// Return whether the WriteVar is a secret.
252 pub fn is_secret(&self) -> bool {
253 self.is_secret
254 }
255}
256
257/// Claim one or more flowey Vars for a particular step.
258///
259/// By having this be a trait, it is possible to `claim` both single instances
260/// of `ReadVar` / `WriteVar`, as well as whole _collections_ of Vars.
261//
262// FUTURE: flowey should include a derive macro for easily claiming read/write
263// vars in user-defined structs / enums.
264pub trait ClaimVar {
265 /// The claimed version of Self.
266 type Claimed;
267 /// Claim the Var for this step, allowing it to be accessed at runtime.
268 fn claim(self, ctx: &mut StepCtx<'_>) -> Self::Claimed;
269}
270
271impl<T: Serialize + DeserializeOwned> ClaimVar for ReadVar<T> {
272 type Claimed = ClaimedReadVar<T>;
273
274 fn claim(self, ctx: &mut StepCtx<'_>) -> ClaimedReadVar<T> {
275 if let ReadVarBacking::RuntimeVar(var) = &self.backing_var {
276 ctx.backend.borrow_mut().on_claimed_runtime_var(var, true);
277 }
278 self.into_claimed()
279 }
280}
281
282impl<T: Serialize + DeserializeOwned> ClaimVar for WriteVar<T> {
283 type Claimed = ClaimedWriteVar<T>;
284
285 fn claim(self, ctx: &mut StepCtx<'_>) -> ClaimedWriteVar<T> {
286 ctx.backend
287 .borrow_mut()
288 .on_claimed_runtime_var(&self.backing_var, false);
289 self.into_claimed()
290 }
291}
292
293impl<T: ClaimVar> ClaimVar for Vec<T> {
294 type Claimed = Vec<T::Claimed>;
295
296 fn claim(self, ctx: &mut StepCtx<'_>) -> Vec<T::Claimed> {
297 self.into_iter().map(|v| v.claim(ctx)).collect()
298 }
299}
300
301impl<T: ClaimVar> ClaimVar for Option<T> {
302 type Claimed = Option<T::Claimed>;
303
304 fn claim(self, ctx: &mut StepCtx<'_>) -> Option<T::Claimed> {
305 self.map(|x| x.claim(ctx))
306 }
307}
308
309impl<U: Ord, T: ClaimVar> ClaimVar for BTreeMap<U, T> {
310 type Claimed = BTreeMap<U, T::Claimed>;
311
312 fn claim(self, ctx: &mut StepCtx<'_>) -> BTreeMap<U, T::Claimed> {
313 self.into_iter().map(|(k, v)| (k, v.claim(ctx))).collect()
314 }
315}
316
317macro_rules! impl_tuple_claim {
318 ($($T:tt)*) => {
319 impl<$($T,)*> ClaimVar for ($($T,)*)
320 where
321 $($T: ClaimVar,)*
322 {
323 type Claimed = ($($T::Claimed,)*);
324
325 #[allow(non_snake_case)]
326 fn claim(self, ctx: &mut StepCtx<'_>) -> Self::Claimed {
327 let ($($T,)*) = self;
328 ($($T.claim(ctx),)*)
329 }
330 }
331 };
332}
333
334impl_tuple_claim!(A B C D E F G H I J);
335impl_tuple_claim!(A B C D E F G H I);
336impl_tuple_claim!(A B C D E F G H);
337impl_tuple_claim!(A B C D E F G);
338impl_tuple_claim!(A B C D E F);
339impl_tuple_claim!(A B C D E);
340impl_tuple_claim!(A B C D);
341impl_tuple_claim!(A B C);
342impl_tuple_claim!(A B);
343impl_tuple_claim!(A);
344
345/// Read a custom, user-defined secret by passing in the secret name.
346///
347/// Intended usage is to get a secret using the [`crate::pipeline::Pipeline::gh_use_secret`] API
348/// and to use the returned value through the [`NodeCtx::get_gh_context_var`] API.
349#[derive(Serialize, Deserialize, Clone)]
350pub struct GhUserSecretVar(pub(crate) String);
351
352/// Read a value from a flowey Var at runtime, returning the value written by
353/// the Var's corresponding [`WriteVar`].
354///
355/// Vars in flowey must be serde de/serializable, in order to be de/serialized
356/// between multiple steps/nodes.
357///
358/// In order to read the value contained within a `ReadVar`, it must first be
359/// _claimed_ by a particular step (using the [`ClaimVar::claim`] API). Once
360/// claimed, the Var can be read using APIs such as
361/// [`RustRuntimeServices::read`], or [`AdoStepServices::get_var`]
362///
363/// Note that all `ReadVar`s in flowey are _immutable_. In other words:
364/// reading the value of a `ReadVar` multiple times from multiple nodes will
365/// _always_ return the same value.
366///
367/// This is a natural consequence `ReadVar` obtaining its value from the result
368/// of a write into [`WriteVar`], whose API enforces that there can only ever be
369/// a single Write to a `WriteVar`.
370#[derive(Debug, Serialize, Deserialize)]
371pub struct ReadVar<T: Serialize + DeserializeOwned, C = VarNotClaimed> {
372 #[serde(bound = "")] // work around serde/issues/1296
373 backing_var: ReadVarBacking<T>,
374 is_secret: bool,
375 #[serde(skip)]
376 _kind: std::marker::PhantomData<C>,
377}
378
379/// A [`ReadVar`] which has been claimed by a particular step, allowing it to
380/// be read at runtime.
381pub type ClaimedReadVar<T> = ReadVar<T, VarClaimed>;
382
383// cloning is fine, since you can totally have multiple dependents
384impl<T: Serialize + DeserializeOwned, C> Clone for ReadVar<T, C> {
385 fn clone(&self) -> Self {
386 ReadVar {
387 backing_var: self.backing_var.clone(),
388 is_secret: self.is_secret,
389 _kind: std::marker::PhantomData,
390 }
391 }
392}
393
394#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
395enum ReadVarBacking<T: Serialize + DeserializeOwned> {
396 RuntimeVar(String),
397 #[serde(bound = "")] // work around serde/issues/1296
398 Inline(T),
399 InlineSideEffect,
400}
401
402// avoid requiring types to include an explicit clone bound
403impl<T: Serialize + DeserializeOwned> Clone for ReadVarBacking<T> {
404 fn clone(&self) -> Self {
405 match self {
406 Self::RuntimeVar(v) => Self::RuntimeVar(v.clone()),
407 Self::Inline(v) => {
408 Self::Inline(serde_json::from_value(serde_json::to_value(v).unwrap()).unwrap())
409 }
410 Self::InlineSideEffect => Self::InlineSideEffect,
411 }
412 }
413}
414
415impl<T: Serialize + DeserializeOwned> ReadVar<T> {
416 /// (Internal API) Switch the claim marker to "claimed".
417 fn into_claimed(self) -> ReadVar<T, VarClaimed> {
418 let Self {
419 backing_var,
420 is_secret,
421 _kind,
422 } = self;
423
424 ReadVar {
425 backing_var,
426 is_secret,
427 _kind: std::marker::PhantomData,
428 }
429 }
430
431 /// Discard any type information associated with the Var, and treat the Var
432 /// as through it was only a side effect.
433 ///
434 /// e.g: if a Node returns a `ReadVar<PathBuf>`, but you know that the mere
435 /// act of having _run_ the node has ensured the file is placed in a "magic
436 /// location" for some other node, then it may be useful to treat the
437 /// `ReadVar<PathBuf>` as a simple `ReadVar<SideEffect>`, which can be
438 /// passed along as part of a larger bundle of `Vec<ReadVar<SideEffect>>`.
439 #[must_use]
440 pub fn into_side_effect(self) -> ReadVar<SideEffect> {
441 ReadVar {
442 backing_var: match self.backing_var {
443 ReadVarBacking::RuntimeVar(var) => ReadVarBacking::RuntimeVar(var),
444 ReadVarBacking::Inline(_) => ReadVarBacking::InlineSideEffect,
445 ReadVarBacking::InlineSideEffect => ReadVarBacking::InlineSideEffect,
446 },
447 is_secret: self.is_secret,
448 _kind: std::marker::PhantomData,
449 }
450 }
451
452 /// Maps a `ReadVar<T>` to a new `ReadVar<U>`, by applying a function to the
453 /// Var at runtime.
454 #[track_caller]
455 #[must_use]
456 pub fn map<F, U>(&self, ctx: &mut NodeCtx<'_>, f: F) -> ReadVar<U>
457 where
458 T: 'static,
459 U: Serialize + DeserializeOwned + 'static,
460 F: FnOnce(T) -> U + 'static,
461 {
462 let (read_from, write_into) = ctx.new_maybe_secret_var(self.is_secret, "");
463 self.write_into(ctx, write_into, f);
464 read_from
465 }
466
467 /// Maps a `ReadVar<T>` into an existing `WriteVar<U>` by applying a
468 /// function to the Var at runtime.
469 #[track_caller]
470 pub fn write_into<F, U>(&self, ctx: &mut NodeCtx<'_>, write_into: WriteVar<U>, f: F)
471 where
472 T: 'static,
473 U: Serialize + DeserializeOwned + 'static,
474 F: FnOnce(T) -> U + 'static,
475 {
476 let this = self.clone();
477 ctx.emit_rust_step("🌼 write_into Var", move |ctx| {
478 let this = this.claim(ctx);
479 let write_into = write_into.claim(ctx);
480 move |rt| {
481 let this = rt.read(this);
482 rt.write(write_into, &f(this));
483 Ok(())
484 }
485 });
486 }
487
488 /// Zips self (`ReadVar<T>`) with another `ReadVar<U>`, returning a new
489 /// `ReadVar<(T, U)>`
490 #[track_caller]
491 #[must_use]
492 pub fn zip<U>(&self, ctx: &mut NodeCtx<'_>, other: ReadVar<U>) -> ReadVar<(T, U)>
493 where
494 T: 'static,
495 U: Serialize + DeserializeOwned + 'static,
496 {
497 let (read_from, write_into) =
498 ctx.new_maybe_secret_var(self.is_secret || other.is_secret, "");
499 let this = self.clone();
500 ctx.emit_rust_step("🌼 Zip Vars", move |ctx| {
501 let this = this.claim(ctx);
502 let other = other.claim(ctx);
503 let write_into = write_into.claim(ctx);
504 move |rt| {
505 let this = rt.read(this);
506 let other = rt.read(other);
507 rt.write(write_into, &(this, other));
508 Ok(())
509 }
510 });
511 read_from
512 }
513
514 /// Create a new `ReadVar` from a static value.
515 ///
516 /// **WARNING:** Static vars **CANNOT BE SECRETS**, as they are encoded as
517 /// plain-text in the output flow.
518 #[track_caller]
519 #[must_use]
520 pub fn from_static(val: T) -> ReadVar<T>
521 where
522 T: 'static,
523 {
524 ReadVar {
525 backing_var: ReadVarBacking::Inline(val),
526 is_secret: false,
527 _kind: std::marker::PhantomData,
528 }
529 }
530
531 /// If this [`ReadVar`] contains a static value, return it.
532 ///
533 /// Nodes can opt-in to using this method as a way to generate optimized
534 /// steps in cases where the value of a variable is known ahead of time.
535 ///
536 /// e.g: a node doing a git checkout could leverage this method to decide
537 /// whether its ADO backend should emit a conditional step for checking out
538 /// a repo, or if it can statically include / exclude the checkout request.
539 pub fn get_static(&self) -> Option<T> {
540 match self.clone().backing_var {
541 ReadVarBacking::Inline(v) => Some(v),
542 _ => None,
543 }
544 }
545
546 /// Transpose a `Vec<ReadVar<T>>` into a `ReadVar<Vec<T>>`
547 #[track_caller]
548 #[must_use]
549 pub fn transpose_vec(ctx: &mut NodeCtx<'_>, vec: Vec<ReadVar<T>>) -> ReadVar<Vec<T>>
550 where
551 T: 'static,
552 {
553 let (read_from, write_into) = ctx.new_maybe_secret_var(vec.iter().any(|v| v.is_secret), "");
554 ctx.emit_rust_step("🌼 Transpose Vec<ReadVar<T>>", move |ctx| {
555 let vec = vec.claim(ctx);
556 let write_into = write_into.claim(ctx);
557 move |rt| {
558 let mut v = Vec::new();
559 for var in vec {
560 v.push(rt.read(var));
561 }
562 rt.write(write_into, &v);
563 Ok(())
564 }
565 });
566 read_from
567 }
568
569 /// Consume this `ReadVar` outside the context of a step, signalling that it
570 /// won't be used.
571 pub fn claim_unused(self, ctx: &mut NodeCtx<'_>) {
572 match self.backing_var {
573 ReadVarBacking::RuntimeVar(s) => ctx.backend.borrow_mut().on_unused_read_var(&s),
574 ReadVarBacking::Inline(_) => {}
575 ReadVarBacking::InlineSideEffect => {}
576 }
577 }
578}
579
580/// DANGER: obtain a handle to a [`ReadVar`] "out of thin air".
581///
582/// This should NEVER be used from within a flowey node. This is a sharp tool,
583/// and should only be used by code implementing flow / pipeline resolution
584/// logic.
585#[must_use]
586pub fn thin_air_read_runtime_var<T>(backing_var: String, is_secret: bool) -> ReadVar<T>
587where
588 T: Serialize + DeserializeOwned,
589{
590 ReadVar {
591 backing_var: ReadVarBacking::RuntimeVar(backing_var),
592 is_secret,
593 _kind: std::marker::PhantomData,
594 }
595}
596
597/// DANGER: obtain a handle to a [`WriteVar`] "out of thin air".
598///
599/// This should NEVER be used from within a flowey node. This is a sharp tool,
600/// and should only be used by code implementing flow / pipeline resolution
601/// logic.
602#[must_use]
603pub fn thin_air_write_runtime_var<T>(backing_var: String, is_secret: bool) -> WriteVar<T>
604where
605 T: Serialize + DeserializeOwned,
606{
607 WriteVar {
608 backing_var,
609 is_secret,
610 _kind: std::marker::PhantomData,
611 }
612}
613
614/// DANGER: obtain a [`ReadVar`] backing variable and secret status.
615///
616/// This should NEVER be used from within a flowey node. This relies on
617/// flowey variable implementation details, and should only be used by code
618/// implementing flow / pipeline resolution logic.
619pub fn read_var_internals<T: Serialize + DeserializeOwned, C>(
620 var: &ReadVar<T, C>,
621) -> (Option<String>, bool) {
622 match &var.backing_var {
623 ReadVarBacking::RuntimeVar(s) => (Some(s.clone()), var.is_secret),
624 ReadVarBacking::Inline(_) => (None, var.is_secret),
625 ReadVarBacking::InlineSideEffect => (None, var.is_secret),
626 }
627}
628
629pub trait ImportCtxBackend {
630 fn on_possible_dep(&mut self, node_handle: NodeHandle);
631}
632
633/// Context passed to [`FlowNode::imports`].
634pub struct ImportCtx<'a> {
635 backend: &'a mut dyn ImportCtxBackend,
636}
637
638impl ImportCtx<'_> {
639 /// Declare that a Node can be referenced in [`FlowNode::emit`]
640 pub fn import<N: FlowNodeBase + 'static>(&mut self) {
641 self.backend.on_possible_dep(NodeHandle::from_type::<N>())
642 }
643}
644
645pub fn new_import_ctx(backend: &mut dyn ImportCtxBackend) -> ImportCtx<'_> {
646 ImportCtx { backend }
647}
648
649#[derive(Debug)]
650pub enum CtxAnchor {
651 PostJob,
652}
653
654pub trait NodeCtxBackend {
655 /// Handle to the current node this `ctx` corresponds to
656 fn current_node(&self) -> NodeHandle;
657
658 /// Return a string which uniquely identifies this particular Var
659 /// registration.
660 ///
661 /// Typically consists of `{current node handle}{ordinal}`
662 fn on_new_var(&mut self) -> String;
663
664 /// Invoked when a node claims a particular runtime variable
665 fn on_claimed_runtime_var(&mut self, var: &str, is_read: bool);
666
667 /// Invoked when a node marks a particular runtime variable as unused
668 fn on_unused_read_var(&mut self, var: &str);
669
670 /// Invoked when a node sets a request on a node.
671 ///
672 /// - `node_typeid` will always correspond to a node that was previously
673 /// passed to `on_register`.
674 /// - `req` may be an error, in the case where the NodeCtx failed to
675 /// serialize the provided request.
676 // FIXME: this should be using type-erased serde
677 fn on_request(&mut self, node_handle: NodeHandle, req: anyhow::Result<Box<[u8]>>);
678
679 fn on_emit_rust_step(
680 &mut self,
681 label: &str,
682 code: Box<dyn for<'a> FnOnce(&'a mut RustRuntimeServices<'_>) -> anyhow::Result<()>>,
683 );
684
685 fn on_emit_ado_step(
686 &mut self,
687 label: &str,
688 yaml_snippet: Box<dyn for<'a> FnOnce(&'a mut AdoStepServices<'_>) -> String>,
689 inline_script: Option<
690 Box<dyn for<'a> FnOnce(&'a mut RustRuntimeServices<'_>) -> anyhow::Result<()>>,
691 >,
692 condvar: Option<String>,
693 );
694
695 fn on_emit_gh_step(
696 &mut self,
697 label: &str,
698 uses: &str,
699 with: BTreeMap<String, ClaimedGhParam>,
700 condvar: Option<String>,
701 outputs: BTreeMap<String, Vec<GhVarState>>,
702 permissions: BTreeMap<GhPermission, GhPermissionValue>,
703 gh_to_rust: Vec<GhVarState>,
704 rust_to_gh: Vec<GhVarState>,
705 );
706
707 fn on_emit_side_effect_step(&mut self);
708
709 fn backend(&mut self) -> FlowBackend;
710 fn platform(&mut self) -> FlowPlatform;
711 fn arch(&mut self) -> FlowArch;
712
713 /// Return a node-specific persistent store path. The backend does not need
714 /// to ensure that the path exists - flowey will automatically emit a step
715 /// to construct the directory at runtime.
716 fn persistent_dir_path_var(&mut self) -> Option<String>;
717}
718
719pub fn new_node_ctx(backend: &mut dyn NodeCtxBackend) -> NodeCtx<'_> {
720 NodeCtx {
721 backend: Rc::new(RefCell::new(backend)),
722 }
723}
724
725/// What backend the flow is being running on.
726#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
727pub enum FlowBackend {
728 /// Running locally.
729 Local,
730 /// Running on ADO.
731 Ado,
732 /// Running on GitHub Actions
733 Github,
734}
735
736/// The kind platform the flow is being running on, Windows or Unix.
737#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
738pub enum FlowPlatformKind {
739 Windows,
740 Unix,
741}
742
743/// The kind platform the flow is being running on, Windows or Unix.
744#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
745pub enum FlowPlatformLinuxDistro {
746 /// Fedora (including WSL2)
747 Fedora,
748 /// Ubuntu (including WSL2)
749 Ubuntu,
750 /// An unknown distribution
751 Unknown,
752}
753
754/// What platform the flow is being running on.
755#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
756#[non_exhaustive]
757pub enum FlowPlatform {
758 /// Windows
759 Windows,
760 /// Linux (including WSL2)
761 Linux(FlowPlatformLinuxDistro),
762 /// macOS
763 MacOs,
764}
765
766impl FlowPlatform {
767 pub fn kind(&self) -> FlowPlatformKind {
768 match self {
769 Self::Windows => FlowPlatformKind::Windows,
770 Self::Linux(_) | Self::MacOs => FlowPlatformKind::Unix,
771 }
772 }
773
774 fn as_str(&self) -> &'static str {
775 match self {
776 Self::Windows => "windows",
777 Self::Linux(_) => "linux",
778 Self::MacOs => "macos",
779 }
780 }
781
782 /// The suffix to use for executables on this platform.
783 pub fn exe_suffix(&self) -> &'static str {
784 if self == &Self::Windows { ".exe" } else { "" }
785 }
786
787 /// The full name for a binary on this platform (i.e. `name + self.exe_suffix()`).
788 pub fn binary(&self, name: &str) -> String {
789 format!("{}{}", name, self.exe_suffix())
790 }
791}
792
793impl std::fmt::Display for FlowPlatform {
794 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
795 f.pad(self.as_str())
796 }
797}
798
799/// What architecture the flow is being running on.
800#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
801#[non_exhaustive]
802pub enum FlowArch {
803 X86_64,
804 Aarch64,
805}
806
807impl FlowArch {
808 fn as_str(&self) -> &'static str {
809 match self {
810 Self::X86_64 => "x86_64",
811 Self::Aarch64 => "aarch64",
812 }
813 }
814}
815
816impl std::fmt::Display for FlowArch {
817 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
818 f.pad(self.as_str())
819 }
820}
821
822/// Context object for an individual step.
823pub struct StepCtx<'a> {
824 backend: Rc<RefCell<&'a mut dyn NodeCtxBackend>>,
825}
826
827impl StepCtx<'_> {
828 /// What backend the flow is being running on (e.g: locally, ADO, GitHub,
829 /// etc...)
830 pub fn backend(&self) -> FlowBackend {
831 self.backend.borrow_mut().backend()
832 }
833
834 /// What platform the flow is being running on (e.g: windows, linux, wsl2,
835 /// etc...).
836 pub fn platform(&self) -> FlowPlatform {
837 self.backend.borrow_mut().platform()
838 }
839}
840
841const NO_ADO_INLINE_SCRIPT: Option<
842 for<'a> fn(&'a mut RustRuntimeServices<'_>) -> anyhow::Result<()>,
843> = None;
844
845/// Context object for a `FlowNode`.
846pub struct NodeCtx<'a> {
847 backend: Rc<RefCell<&'a mut dyn NodeCtxBackend>>,
848}
849
850impl<'ctx> NodeCtx<'ctx> {
851 /// Emit a Rust-based step.
852 ///
853 /// As a convenience feature, this function returns a special _optional_
854 /// [`ReadVar<SideEffect>`], which will not result in a "unused variable"
855 /// error if no subsequent step ends up claiming it.
856 pub fn emit_rust_step<F, G>(&mut self, label: impl AsRef<str>, code: F) -> ReadVar<SideEffect>
857 where
858 F: for<'a> FnOnce(&'a mut StepCtx<'_>) -> G,
859 G: for<'a> FnOnce(&'a mut RustRuntimeServices<'_>) -> anyhow::Result<()> + 'static,
860 {
861 let (read, write) = self.new_maybe_secret_var(false, "auto_se");
862
863 let ctx = &mut StepCtx {
864 backend: self.backend.clone(),
865 };
866 write.claim(ctx);
867
868 let code = code(ctx);
869 self.backend
870 .borrow_mut()
871 .on_emit_rust_step(label.as_ref(), Box::new(code));
872 read
873 }
874
875 /// Emit a Rust-based step, creating a new `ReadVar<T>` from the step's
876 /// return value.
877 ///
878 /// The var returned by this method is _not secret_. In order to create
879 /// secret variables, use the `ctx.new_var_secret()` method.
880 ///
881 /// This is a convenience function that streamlines the following common
882 /// flowey pattern:
883 ///
884 /// ```ignore
885 /// // creating a new Var explicitly
886 /// let (read_foo, write_foo) = ctx.new_var();
887 /// ctx.emit_rust_step("foo", |ctx| {
888 /// let write_foo = write_foo.claim(ctx);
889 /// |rt| {
890 /// rt.write(write_foo, &get_foo());
891 /// Ok(())
892 /// }
893 /// });
894 ///
895 /// // creating a new Var automatically
896 /// let read_foo = ctx.emit_rust_stepv("foo", |ctx| |rt| get_foo());
897 /// ```
898 #[must_use]
899 pub fn emit_rust_stepv<T, F, G>(&mut self, label: impl AsRef<str>, code: F) -> ReadVar<T>
900 where
901 T: Serialize + DeserializeOwned + 'static,
902 F: for<'a> FnOnce(&'a mut StepCtx<'_>) -> G,
903 G: for<'a> FnOnce(&'a mut RustRuntimeServices<'_>) -> anyhow::Result<T> + 'static,
904 {
905 let (read, write) = self.new_var();
906
907 let ctx = &mut StepCtx {
908 backend: self.backend.clone(),
909 };
910 let write = write.claim(ctx);
911
912 let code = code(ctx);
913 self.backend.borrow_mut().on_emit_rust_step(
914 label.as_ref(),
915 Box::new(|rt| {
916 let val = code(rt)?;
917 rt.write(write, &val);
918 Ok(())
919 }),
920 );
921 read
922 }
923
924 /// Load an ADO global runtime variable into a flowey [`ReadVar`].
925 #[track_caller]
926 #[must_use]
927 pub fn get_ado_variable(&mut self, ado_var: AdoRuntimeVar) -> ReadVar<String> {
928 let (var, write_var) = self.new_maybe_secret_var(ado_var.is_secret(), "");
929 self.emit_ado_step(format!("🌼 read {}", ado_var.as_raw_var_name()), |ctx| {
930 let write_var = write_var.claim(ctx);
931 |rt| {
932 rt.set_var(write_var, ado_var);
933 "".into()
934 }
935 });
936 var
937 }
938
939 /// Emit an ADO step.
940 pub fn emit_ado_step<F, G>(&mut self, display_name: impl AsRef<str>, yaml_snippet: F)
941 where
942 F: for<'a> FnOnce(&'a mut StepCtx<'_>) -> G,
943 G: for<'a> FnOnce(&'a mut AdoStepServices<'_>) -> String + 'static,
944 {
945 self.emit_ado_step_inner(display_name, None, |ctx| {
946 (yaml_snippet(ctx), NO_ADO_INLINE_SCRIPT)
947 })
948 }
949
950 /// Emit an ADO step, conditionally executed based on the value of `cond` at
951 /// runtime.
952 pub fn emit_ado_step_with_condition<F, G>(
953 &mut self,
954 display_name: impl AsRef<str>,
955 cond: ReadVar<bool>,
956 yaml_snippet: F,
957 ) where
958 F: for<'a> FnOnce(&'a mut StepCtx<'_>) -> G,
959 G: for<'a> FnOnce(&'a mut AdoStepServices<'_>) -> String + 'static,
960 {
961 self.emit_ado_step_inner(display_name, Some(cond), |ctx| {
962 (yaml_snippet(ctx), NO_ADO_INLINE_SCRIPT)
963 })
964 }
965
966 /// Emit an ADO step, conditionally executed based on the value of`cond` at
967 /// runtime.
968 pub fn emit_ado_step_with_condition_optional<F, G>(
969 &mut self,
970 display_name: impl AsRef<str>,
971 cond: Option<ReadVar<bool>>,
972 yaml_snippet: F,
973 ) where
974 F: for<'a> FnOnce(&'a mut StepCtx<'_>) -> G,
975 G: for<'a> FnOnce(&'a mut AdoStepServices<'_>) -> String + 'static,
976 {
977 self.emit_ado_step_inner(display_name, cond, |ctx| {
978 (yaml_snippet(ctx), NO_ADO_INLINE_SCRIPT)
979 })
980 }
981
982 /// Emit an ADO step which invokes a rust callback using an inline script.
983 ///
984 /// By using the `{{FLOWEY_INLINE_SCRIPT}}` template in the returned yaml
985 /// snippet, flowey will interpolate a command ~roughly akin to `flowey
986 /// exec-snippet <rust-snippet-id>` into the generated yaml.
987 ///
988 /// e.g: if we wanted to _manually_ wrap the bash ADO snippet for whatever
989 /// reason:
990 ///
991 /// ```text
992 /// - bash: |
993 /// echo "hello there!"
994 /// {{FLOWEY_INLINE_SCRIPT}}
995 /// echo echo "bye!"
996 /// ```
997 ///
998 /// # Limitations
999 ///
1000 /// At the moment, due to flowey API limitations, it is only possible to
1001 /// embed a single inline script into a YAML step.
1002 ///
1003 /// In the future, rather than having separate methods for "emit step with X
1004 /// inline scripts", flowey should support declaring "first-class" callbacks
1005 /// via a (hypothetical) `ctx.new_callback_var(|ctx| |rt, input: Input| ->
1006 /// Output { ... })` API, at which point.
1007 ///
1008 /// If such an API were to exist, one could simply use the "vanilla" emit
1009 /// yaml step functions with these first-class callbacks.
1010 pub fn emit_ado_step_with_inline_script<F, G, H>(
1011 &mut self,
1012 display_name: impl AsRef<str>,
1013 yaml_snippet: F,
1014 ) where
1015 F: for<'a> FnOnce(&'a mut StepCtx<'_>) -> (G, H),
1016 G: for<'a> FnOnce(&'a mut AdoStepServices<'_>) -> String + 'static,
1017 H: for<'a> FnOnce(&'a mut RustRuntimeServices<'_>) -> anyhow::Result<()> + 'static,
1018 {
1019 self.emit_ado_step_inner(display_name, None, |ctx| {
1020 let (f, g) = yaml_snippet(ctx);
1021 (f, Some(g))
1022 })
1023 }
1024
1025 fn emit_ado_step_inner<F, G, H>(
1026 &mut self,
1027 display_name: impl AsRef<str>,
1028 cond: Option<ReadVar<bool>>,
1029 yaml_snippet: F,
1030 ) where
1031 F: for<'a> FnOnce(&'a mut StepCtx<'_>) -> (G, Option<H>),
1032 G: for<'a> FnOnce(&'a mut AdoStepServices<'_>) -> String + 'static,
1033 H: for<'a> FnOnce(&'a mut RustRuntimeServices<'_>) -> anyhow::Result<()> + 'static,
1034 {
1035 let condvar = match cond.map(|c| c.backing_var) {
1036 // it seems silly to allow this... but it's not hard so why not?
1037 Some(ReadVarBacking::Inline(cond)) => {
1038 if !cond {
1039 return;
1040 } else {
1041 None
1042 }
1043 }
1044 Some(ReadVarBacking::RuntimeVar(var)) => {
1045 self.backend.borrow_mut().on_claimed_runtime_var(&var, true);
1046 Some(var)
1047 }
1048 Some(ReadVarBacking::InlineSideEffect) => unreachable!(),
1049 None => None,
1050 };
1051
1052 let (yaml_snippet, inline_script) = yaml_snippet(&mut StepCtx {
1053 backend: self.backend.clone(),
1054 });
1055 self.backend.borrow_mut().on_emit_ado_step(
1056 display_name.as_ref(),
1057 Box::new(yaml_snippet),
1058 if let Some(inline_script) = inline_script {
1059 Some(Box::new(inline_script))
1060 } else {
1061 None
1062 },
1063 condvar,
1064 );
1065 }
1066
1067 /// Load a GitHub context variable into a flowey [`ReadVar`].
1068 #[track_caller]
1069 #[must_use]
1070 pub fn get_gh_context_var(&mut self) -> GhContextVarReader<'ctx, Root> {
1071 GhContextVarReader {
1072 ctx: NodeCtx {
1073 backend: self.backend.clone(),
1074 },
1075 _state: std::marker::PhantomData,
1076 }
1077 }
1078
1079 /// Emit a GitHub Actions action step.
1080 pub fn emit_gh_step(
1081 &mut self,
1082 display_name: impl AsRef<str>,
1083 uses: impl AsRef<str>,
1084 ) -> GhStepBuilder {
1085 GhStepBuilder::new(display_name, uses)
1086 }
1087
1088 fn emit_gh_step_inner(
1089 &mut self,
1090 display_name: impl AsRef<str>,
1091 cond: Option<ReadVar<bool>>,
1092 uses: impl AsRef<str>,
1093 with: Option<BTreeMap<String, GhParam>>,
1094 outputs: BTreeMap<String, Vec<WriteVar<String>>>,
1095 run_after: Vec<ReadVar<SideEffect>>,
1096 permissions: BTreeMap<GhPermission, GhPermissionValue>,
1097 ) {
1098 let condvar = match cond.map(|c| c.backing_var) {
1099 // it seems silly to allow this... but it's not hard so why not?
1100 Some(ReadVarBacking::Inline(cond)) => {
1101 if !cond {
1102 return;
1103 } else {
1104 None
1105 }
1106 }
1107 Some(ReadVarBacking::RuntimeVar(var)) => {
1108 self.backend.borrow_mut().on_claimed_runtime_var(&var, true);
1109 Some(var)
1110 }
1111 Some(ReadVarBacking::InlineSideEffect) => unreachable!(),
1112 None => None,
1113 };
1114
1115 let with = with
1116 .unwrap_or_default()
1117 .into_iter()
1118 .map(|(k, v)| {
1119 (
1120 k.clone(),
1121 v.claim(&mut StepCtx {
1122 backend: self.backend.clone(),
1123 }),
1124 )
1125 })
1126 .collect();
1127
1128 for var in run_after {
1129 var.claim(&mut StepCtx {
1130 backend: self.backend.clone(),
1131 });
1132 }
1133
1134 let outputvars = outputs
1135 .into_iter()
1136 .map(|(name, vars)| {
1137 (
1138 name,
1139 vars.into_iter()
1140 .map(|var| {
1141 let var = var.claim(&mut StepCtx {
1142 backend: self.backend.clone(),
1143 });
1144 GhVarState {
1145 raw_name: None,
1146 backing_var: var.backing_var,
1147 is_secret: var.is_secret,
1148 is_object: false,
1149 }
1150 })
1151 .collect(),
1152 )
1153 })
1154 .collect();
1155
1156 self.backend.borrow_mut().on_emit_gh_step(
1157 display_name.as_ref(),
1158 uses.as_ref(),
1159 with,
1160 condvar,
1161 outputvars,
1162 permissions,
1163 Vec::new(),
1164 Vec::new(),
1165 );
1166 }
1167
1168 /// Emit a "side-effect" step, which simply claims a set of side-effects in
1169 /// order to resolve another set of side effects.
1170 ///
1171 /// The same functionality could be achieved (less efficiently) by emitting
1172 /// a Rust step (or ADO step, or github step, etc...) that claims both sets
1173 /// of side-effects, and then does nothing. By using this method - flowey is
1174 /// able to avoid emitting that additional noop step at runtime.
1175 pub fn emit_side_effect_step(
1176 &mut self,
1177 use_side_effects: impl IntoIterator<Item = ReadVar<SideEffect>>,
1178 resolve_side_effects: impl IntoIterator<Item = WriteVar<SideEffect>>,
1179 ) {
1180 let mut backend = self.backend.borrow_mut();
1181 for var in use_side_effects.into_iter() {
1182 if let ReadVarBacking::RuntimeVar(var) = &var.backing_var {
1183 backend.on_claimed_runtime_var(var, true);
1184 }
1185 }
1186
1187 for var in resolve_side_effects.into_iter() {
1188 backend.on_claimed_runtime_var(&var.backing_var, false);
1189 }
1190
1191 backend.on_emit_side_effect_step();
1192 }
1193
1194 /// What backend the flow is being running on (e.g: locally, ADO, GitHub,
1195 /// etc...)
1196 pub fn backend(&self) -> FlowBackend {
1197 self.backend.borrow_mut().backend()
1198 }
1199
1200 /// What platform the flow is being running on (e.g: windows, linux, wsl2,
1201 /// etc...).
1202 pub fn platform(&self) -> FlowPlatform {
1203 self.backend.borrow_mut().platform()
1204 }
1205
1206 /// What architecture the flow is being running on (x86_64 or Aarch64)
1207 pub fn arch(&self) -> FlowArch {
1208 self.backend.borrow_mut().arch()
1209 }
1210
1211 /// Set a request on a particular node.
1212 pub fn req<R>(&mut self, req: R)
1213 where
1214 R: IntoRequest + 'static,
1215 {
1216 let mut backend = self.backend.borrow_mut();
1217 backend.on_request(
1218 NodeHandle::from_type::<R::Node>(),
1219 serde_json::to_vec(&req.into_request())
1220 .map(Into::into)
1221 .map_err(Into::into),
1222 );
1223 }
1224
1225 /// Set a request on a particular node, simultaneously creating a new flowey
1226 /// Var in the process.
1227 #[track_caller]
1228 #[must_use]
1229 pub fn reqv<T, R>(&mut self, f: impl FnOnce(WriteVar<T>) -> R) -> ReadVar<T>
1230 where
1231 T: Serialize + DeserializeOwned,
1232 R: IntoRequest + 'static,
1233 {
1234 let (read, write) = self.new_var();
1235 self.req::<R>(f(write));
1236 read
1237 }
1238
1239 /// Set multiple requests on a particular node.
1240 pub fn requests<N>(&mut self, reqs: impl IntoIterator<Item = N::Request>)
1241 where
1242 N: FlowNodeBase + 'static,
1243 {
1244 let mut backend = self.backend.borrow_mut();
1245 for req in reqs.into_iter() {
1246 backend.on_request(
1247 NodeHandle::from_type::<N>(),
1248 serde_json::to_vec(&req).map(Into::into).map_err(Into::into),
1249 );
1250 }
1251 }
1252
1253 /// Allocate a new flowey Var, returning two handles: one for reading the
1254 /// value, and another for writing the value.
1255 ///
1256 /// This will return a non-secret Var, and its value may be displayed in
1257 /// logs and other output.
1258 #[track_caller]
1259 #[must_use]
1260 pub fn new_var<T>(&self) -> (ReadVar<T>, WriteVar<T>)
1261 where
1262 T: Serialize + DeserializeOwned,
1263 {
1264 self.new_maybe_secret_var(false, "")
1265 }
1266
1267 /// Allocate a new secret flowey Var, returning two handles: one for reading
1268 /// the value, and another for writing the value.
1269 ///
1270 /// A secret Var must not be displayed in logs or other output.
1271 #[track_caller]
1272 #[must_use]
1273 pub fn new_secret_var<T>(&self) -> (ReadVar<T>, WriteVar<T>)
1274 where
1275 T: Serialize + DeserializeOwned,
1276 {
1277 self.new_maybe_secret_var(true, "")
1278 }
1279
1280 #[track_caller]
1281 #[must_use]
1282 fn new_maybe_secret_var<T>(
1283 &self,
1284 is_secret: bool,
1285 prefix: &'static str,
1286 ) -> (ReadVar<T>, WriteVar<T>)
1287 where
1288 T: Serialize + DeserializeOwned,
1289 {
1290 // normalize call path to ensure determinism between windows and linux
1291 let caller = std::panic::Location::caller()
1292 .to_string()
1293 .replace('\\', "/");
1294
1295 // until we have a proper way to "split" debug info related to vars, we
1296 // kinda just lump it in with the var name itself.
1297 //
1298 // HACK: to work around cases where - depending on what the
1299 // current-working-dir is when incoking flowey - the returned
1300 // caller.file() path may leak the full path of the file (as opposed to
1301 // the relative path), resulting in inconsistencies between build
1302 // environments.
1303 //
1304 // For expediency, and to preserve some semblance of useful error
1305 // messages, we decided to play some sketchy games with the resulting
1306 // string to only preserve the _consistent_ bit of the path for a human
1307 // to use as reference.
1308 //
1309 // This is not ideal in the slightest, but it works OK for now
1310 let caller = caller
1311 .split_once("flowey/")
1312 .expect("due to a known limitation with flowey, all flowey code must have an ancestor dir called 'flowey/' somewhere in its full path")
1313 .1;
1314
1315 let colon = if prefix.is_empty() { "" } else { ":" };
1316 let ordinal = self.backend.borrow_mut().on_new_var();
1317 let backing_var = format!("{prefix}{colon}{ordinal}:{caller}");
1318
1319 (
1320 ReadVar {
1321 backing_var: ReadVarBacking::RuntimeVar(backing_var.clone()),
1322 is_secret,
1323 _kind: std::marker::PhantomData,
1324 },
1325 WriteVar {
1326 backing_var,
1327 is_secret,
1328 _kind: std::marker::PhantomData,
1329 },
1330 )
1331 }
1332
1333 /// Allocate special [`SideEffect`] var which can be used to schedule a
1334 /// "post-job" step associated with some existing step.
1335 ///
1336 /// This "post-job" step will then only run after all other regular steps
1337 /// have run (i.e: steps required to complete any top-level objectives
1338 /// passed in via [`crate::pipeline::PipelineJob::dep_on`]). This makes it
1339 /// useful for implementing various "cleanup" or "finalize" tasks.
1340 ///
1341 /// e.g: the Cache node uses this to upload the contents of a cache
1342 /// directory at the end of a Job.
1343 #[track_caller]
1344 #[must_use]
1345 pub fn new_post_job_side_effect(&self) -> (ReadVar<SideEffect>, WriteVar<SideEffect>) {
1346 self.new_maybe_secret_var(false, "post_job")
1347 }
1348
1349 /// Return a flowey Var pointing to a **node-specific** directory which
1350 /// will be persisted between runs, if such a directory is available.
1351 ///
1352 /// WARNING: this method is _very likely_ to return None when running on CI
1353 /// machines, as most CI agents are wiped between jobs!
1354 ///
1355 /// As such, it is NOT recommended that node authors reach for this method
1356 /// directly, and instead use abstractions such as the
1357 /// `flowey_lib_common::cache` Node, which implements node-level persistence
1358 /// in a way that works _regardless_ if a persistent_dir is available (e.g:
1359 /// by falling back to uploading / downloading artifacts to a "cache store"
1360 /// on platforms like ADO or Github Actions).
1361 #[track_caller]
1362 #[must_use]
1363 pub fn persistent_dir(&mut self) -> Option<ReadVar<PathBuf>> {
1364 let path: ReadVar<PathBuf> = ReadVar {
1365 backing_var: ReadVarBacking::RuntimeVar(
1366 self.backend.borrow_mut().persistent_dir_path_var()?,
1367 ),
1368 is_secret: false,
1369 _kind: std::marker::PhantomData,
1370 };
1371
1372 let folder_name = self
1373 .backend
1374 .borrow_mut()
1375 .current_node()
1376 .modpath()
1377 .replace("::", "__");
1378
1379 Some(
1380 self.emit_rust_stepv("🌼 Create persistent store dir", |ctx| {
1381 let path = path.claim(ctx);
1382 |rt| {
1383 let dir = rt.read(path).join(folder_name);
1384 fs_err::create_dir_all(&dir)?;
1385 Ok(dir)
1386 }
1387 }),
1388 )
1389 }
1390
1391 /// Check to see if a persistent dir is available, without yet creating it.
1392 pub fn supports_persistent_dir(&mut self) -> bool {
1393 self.backend
1394 .borrow_mut()
1395 .persistent_dir_path_var()
1396 .is_some()
1397 }
1398}
1399
1400// FUTURE: explore using type-erased serde here, instead of relying on
1401// `serde_json` in `flowey_core`.
1402pub trait RuntimeVarDb {
1403 fn get_var(&mut self, var_name: &str) -> Vec<u8> {
1404 self.try_get_var(var_name)
1405 .unwrap_or_else(|| panic!("db is missing var {}", var_name))
1406 }
1407
1408 fn try_get_var(&mut self, var_name: &str) -> Option<Vec<u8>>;
1409 fn set_var(&mut self, var_name: &str, is_secret: bool, value: Vec<u8>);
1410}
1411
1412impl RuntimeVarDb for Box<dyn RuntimeVarDb> {
1413 fn try_get_var(&mut self, var_name: &str) -> Option<Vec<u8>> {
1414 (**self).try_get_var(var_name)
1415 }
1416
1417 fn set_var(&mut self, var_name: &str, is_secret: bool, value: Vec<u8>) {
1418 (**self).set_var(var_name, is_secret, value)
1419 }
1420}
1421
1422pub mod steps {
1423 pub mod ado {
1424 use crate::node::ClaimedReadVar;
1425 use crate::node::ClaimedWriteVar;
1426 use crate::node::ReadVarBacking;
1427 use serde::Deserialize;
1428 use serde::Serialize;
1429 use std::borrow::Cow;
1430
1431 /// An ADO repository declared as a resource in the top-level pipeline.
1432 ///
1433 /// Created via [`crate::pipeline::Pipeline::ado_add_resources_repository`].
1434 ///
1435 /// Consumed via [`AdoStepServices::resolve_repository_id`].
1436 #[derive(Debug, Clone, Serialize, Deserialize)]
1437 pub struct AdoResourcesRepositoryId {
1438 pub(crate) repo_id: String,
1439 }
1440
1441 impl AdoResourcesRepositoryId {
1442 /// Create a `AdoResourcesRepositoryId` corresponding to `self`
1443 /// (i.e: the repo which stores the current pipeline).
1444 ///
1445 /// This is safe to do from any context, as the `self` resource will
1446 /// _always_ be available.
1447 pub fn new_self() -> Self {
1448 Self {
1449 repo_id: "self".into(),
1450 }
1451 }
1452
1453 /// (dangerous) get the raw ID associated with this resource.
1454 ///
1455 /// It is highly recommended to avoid losing type-safety, and
1456 /// sticking to [`AdoStepServices::resolve_repository_id`].in order
1457 /// to resolve this type to a String.
1458 pub fn dangerous_get_raw_id(&self) -> &str {
1459 &self.repo_id
1460 }
1461
1462 /// (dangerous) create a new ID out of thin air.
1463 ///
1464 /// It is highly recommended to avoid losing type-safety, and
1465 /// sticking to [`AdoStepServices::resolve_repository_id`].in order
1466 /// to resolve this type to a String.
1467 pub fn dangerous_new(repo_id: &str) -> Self {
1468 Self {
1469 repo_id: repo_id.into(),
1470 }
1471 }
1472 }
1473
1474 /// Handle to an ADO variable.
1475 ///
1476 /// Includes a (non-exhaustive) list of associated constants
1477 /// corresponding to global ADO vars which are _always_ available.
1478 #[derive(Clone, Debug, Serialize, Deserialize)]
1479 pub struct AdoRuntimeVar {
1480 is_secret: bool,
1481 ado_var: Cow<'static, str>,
1482 }
1483
1484 #[allow(non_upper_case_globals)]
1485 impl AdoRuntimeVar {
1486 /// `build.SourceBranch`
1487 ///
1488 /// NOTE: Includes the full branch ref (ex: `refs/heads/main`) so
1489 /// unlike `build.SourceBranchName`, a branch like `user/foo/bar`
1490 /// won't be stripped to just `bar`
1491 pub const BUILD__SOURCE_BRANCH: AdoRuntimeVar =
1492 AdoRuntimeVar::new("build.SourceBranch");
1493
1494 /// `build.BuildNumber`
1495 pub const BUILD__BUILD_NUMBER: AdoRuntimeVar = AdoRuntimeVar::new("build.BuildNumber");
1496
1497 /// `System.AccessToken`
1498 pub const SYSTEM__ACCESS_TOKEN: AdoRuntimeVar =
1499 AdoRuntimeVar::new_secret("System.AccessToken");
1500
1501 /// `System.System.JobAttempt`
1502 pub const SYSTEM__JOB_ATTEMPT: AdoRuntimeVar =
1503 AdoRuntimeVar::new_secret("System.JobAttempt");
1504 }
1505
1506 impl AdoRuntimeVar {
1507 const fn new(s: &'static str) -> Self {
1508 Self {
1509 is_secret: false,
1510 ado_var: Cow::Borrowed(s),
1511 }
1512 }
1513
1514 const fn new_secret(s: &'static str) -> Self {
1515 Self {
1516 is_secret: true,
1517 ado_var: Cow::Borrowed(s),
1518 }
1519 }
1520
1521 /// Check if the ADO var is tagged as being a secret
1522 pub fn is_secret(&self) -> bool {
1523 self.is_secret
1524 }
1525
1526 /// Get the raw underlying ADO variable name
1527 pub fn as_raw_var_name(&self) -> String {
1528 self.ado_var.as_ref().into()
1529 }
1530
1531 /// Get a handle to an ADO runtime variable corresponding to a
1532 /// global ADO variable with the given name.
1533 ///
1534 /// This method should be used rarely and with great care!
1535 ///
1536 /// ADO variables are global, and sidestep the type-safe data flow
1537 /// between flowey nodes entirely!
1538 pub fn dangerous_from_global(ado_var_name: impl AsRef<str>, is_secret: bool) -> Self {
1539 Self {
1540 is_secret,
1541 ado_var: ado_var_name.as_ref().to_owned().into(),
1542 }
1543 }
1544 }
1545
1546 pub fn new_ado_step_services(
1547 fresh_ado_var: &mut dyn FnMut() -> String,
1548 ) -> AdoStepServices<'_> {
1549 AdoStepServices {
1550 fresh_ado_var,
1551 ado_to_rust: Vec::new(),
1552 rust_to_ado: Vec::new(),
1553 }
1554 }
1555
1556 pub struct CompletedAdoStepServices {
1557 pub ado_to_rust: Vec<(String, String, bool)>,
1558 pub rust_to_ado: Vec<(String, String, bool)>,
1559 }
1560
1561 impl CompletedAdoStepServices {
1562 pub fn from_ado_step_services(access: AdoStepServices<'_>) -> Self {
1563 let AdoStepServices {
1564 fresh_ado_var: _,
1565 ado_to_rust,
1566 rust_to_ado,
1567 } = access;
1568
1569 Self {
1570 ado_to_rust,
1571 rust_to_ado,
1572 }
1573 }
1574 }
1575
1576 pub struct AdoStepServices<'a> {
1577 fresh_ado_var: &'a mut dyn FnMut() -> String,
1578 ado_to_rust: Vec<(String, String, bool)>,
1579 rust_to_ado: Vec<(String, String, bool)>,
1580 }
1581
1582 impl AdoStepServices<'_> {
1583 /// Return the raw string identifier for the given
1584 /// [`AdoResourcesRepositoryId`].
1585 pub fn resolve_repository_id(&self, repo_id: AdoResourcesRepositoryId) -> String {
1586 repo_id.repo_id
1587 }
1588
1589 /// Set the specified flowey Var using the value of the given ADO var.
1590 // TODO: is there a good way to allow auto-casting the ADO var back
1591 // to a WriteVar<T>, instead of just a String? It's complicated by
1592 // the fact that the ADO var to flowey bridge is handled by the ADO
1593 // backend, which itself needs to know type info...
1594 pub fn set_var(&mut self, var: ClaimedWriteVar<String>, from_ado_var: AdoRuntimeVar) {
1595 self.ado_to_rust
1596 .push((from_ado_var.ado_var.into(), var.backing_var, var.is_secret))
1597 }
1598
1599 /// Get the value of a flowey Var as a ADO runtime variable.
1600 pub fn get_var(&mut self, var: ClaimedReadVar<String>) -> AdoRuntimeVar {
1601 let backing_var = if let ReadVarBacking::RuntimeVar(var) = &var.backing_var {
1602 var
1603 } else {
1604 todo!("support inline ado read vars")
1605 };
1606
1607 let new_ado_var_name = (self.fresh_ado_var)();
1608
1609 self.rust_to_ado.push((
1610 backing_var.clone(),
1611 new_ado_var_name.clone(),
1612 var.is_secret,
1613 ));
1614 AdoRuntimeVar::dangerous_from_global(new_ado_var_name, var.is_secret)
1615 }
1616 }
1617 }
1618
1619 pub mod github {
1620 use crate::node::ClaimVar;
1621 use crate::node::NodeCtx;
1622 use crate::node::ReadVar;
1623 use crate::node::ReadVarBacking;
1624 use crate::node::SideEffect;
1625 use crate::node::StepCtx;
1626 use crate::node::VarClaimed;
1627 use crate::node::VarNotClaimed;
1628 use crate::node::WriteVar;
1629 use std::collections::BTreeMap;
1630
1631 pub struct GhStepBuilder {
1632 display_name: String,
1633 cond: Option<ReadVar<bool>>,
1634 uses: String,
1635 with: Option<BTreeMap<String, GhParam>>,
1636 outputs: BTreeMap<String, Vec<WriteVar<String>>>,
1637 run_after: Vec<ReadVar<SideEffect>>,
1638 permissions: BTreeMap<GhPermission, GhPermissionValue>,
1639 }
1640
1641 impl GhStepBuilder {
1642 /// Creates a new GitHub step builder, with the given display name and
1643 /// action to use. For example, the following code generates the following yaml:
1644 ///
1645 /// ```ignore
1646 /// GhStepBuilder::new("Check out repository code", "actions/checkout@v4").finish()
1647 /// ```
1648 ///
1649 /// ```ignore
1650 /// - name: Check out repository code
1651 /// uses: actions/checkout@v4
1652 /// ```
1653 ///
1654 /// For more information on the yaml syntax for the `name` and `uses` parameters,
1655 /// see <https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsname>
1656 pub fn new(display_name: impl AsRef<str>, uses: impl AsRef<str>) -> Self {
1657 Self {
1658 display_name: display_name.as_ref().into(),
1659 cond: None,
1660 uses: uses.as_ref().into(),
1661 with: None,
1662 outputs: BTreeMap::new(),
1663 run_after: Vec::new(),
1664 permissions: BTreeMap::new(),
1665 }
1666 }
1667
1668 /// Adds a condition [`ReadVar<bool>`] to the step,
1669 /// such that the step only executes if the condition is true.
1670 /// This is equivalent to using an `if` conditional in the yaml.
1671 ///
1672 /// For more information on the yaml syntax for `if` conditionals, see
1673 /// <https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsname>
1674 pub fn condition(mut self, cond: ReadVar<bool>) -> Self {
1675 self.cond = Some(cond);
1676 self
1677 }
1678
1679 /// Adds a parameter to the step, specified as a key-value pair corresponding
1680 /// to the param name and value. For example the following code generates the following yaml:
1681 ///
1682 /// ```rust,ignore
1683 /// let (client_id, write_client_id) = ctx.new_secret_var();
1684 /// let (tenant_id, write_tenant_id) = ctx.new_secret_var();
1685 /// let (subscription_id, write_subscription_id) = ctx.new_secret_var();
1686 /// // ... insert rust step writing to each of those secrets ...
1687 /// GhStepBuilder::new("Azure Login", "Azure/login@v2")
1688 /// .with("client-id", client_id)
1689 /// .with("tenant-id", tenant_id)
1690 /// .with("subscription-id", subscription_id)
1691 /// ```
1692 ///
1693 /// ```text
1694 /// - name: Azure Login
1695 /// uses: Azure/login@v2
1696 /// with:
1697 /// client-id: ${{ env.floweyvar1 }} // Assuming the backend wrote client_id to floweyvar1
1698 /// tenant-id: ${{ env.floweyvar2 }} // Assuming the backend wrote tenant-id to floweyvar2
1699 /// subscription-id: ${{ env.floweyvar3 }} // Assuming the backend wrote subscription-id to floweyvar3
1700 /// ```
1701 ///
1702 /// For more information on the yaml syntax for the `with` parameters,
1703 /// see <https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#jobsjob_idstepswith>
1704 pub fn with(mut self, k: impl AsRef<str>, v: impl Into<GhParam>) -> Self {
1705 self.with.get_or_insert_with(BTreeMap::new);
1706 if let Some(with) = &mut self.with {
1707 with.insert(k.as_ref().to_string(), v.into());
1708 }
1709 self
1710 }
1711
1712 /// Specifies an output to read from the step, specified as a key-value pair
1713 /// corresponding to the output name and the flowey var to write the output to.
1714 ///
1715 /// This is equivalent to writing into `v` the output of a step in the yaml using:
1716 /// `${{ steps.<backend-assigned-step-id>.outputs.<k> }}`
1717 ///
1718 /// For more information on step outputs, see
1719 /// <https://docs.github.com/en/actions/sharing-automations/creating-actions/metadata-syntax-for-github-actions#outputs-for-composite-actions>
1720 pub fn output(mut self, k: impl AsRef<str>, v: WriteVar<String>) -> Self {
1721 self.outputs
1722 .entry(k.as_ref().to_string())
1723 .or_default()
1724 .push(v);
1725 self
1726 }
1727
1728 /// Specifies a side-effect that must be resolved before this step can run.
1729 pub fn run_after(mut self, side_effect: ReadVar<SideEffect>) -> Self {
1730 self.run_after.push(side_effect);
1731 self
1732 }
1733
1734 /// Declare that this step requires a certain GITHUB_TOKEN permission in order to run.
1735 ///
1736 /// For more info about Github Actions permissions, see [`gh_grant_permissions`](crate::pipeline::PipelineJob::gh_grant_permissions) and
1737 /// <https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/assigning-permissions-to-jobs>
1738 pub fn requires_permission(
1739 mut self,
1740 perm: GhPermission,
1741 value: GhPermissionValue,
1742 ) -> Self {
1743 self.permissions.insert(perm, value);
1744 self
1745 }
1746
1747 /// Finish building the step, emitting it to the backend and returning a side-effect.
1748 #[track_caller]
1749 pub fn finish(self, ctx: &mut NodeCtx<'_>) -> ReadVar<SideEffect> {
1750 let (side_effect, claim_side_effect) = ctx.new_maybe_secret_var(false, "auto_se");
1751 ctx.backend
1752 .borrow_mut()
1753 .on_claimed_runtime_var(&claim_side_effect.backing_var, false);
1754
1755 ctx.emit_gh_step_inner(
1756 self.display_name,
1757 self.cond,
1758 self.uses,
1759 self.with,
1760 self.outputs,
1761 self.run_after,
1762 self.permissions,
1763 );
1764
1765 side_effect
1766 }
1767 }
1768
1769 #[derive(Clone, Debug)]
1770 pub enum GhParam<C = VarNotClaimed> {
1771 Static(String),
1772 FloweyVar(ReadVar<String, C>),
1773 }
1774
1775 impl From<String> for GhParam {
1776 fn from(param: String) -> GhParam {
1777 GhParam::Static(param)
1778 }
1779 }
1780
1781 impl From<&str> for GhParam {
1782 fn from(param: &str) -> GhParam {
1783 GhParam::Static(param.to_string())
1784 }
1785 }
1786
1787 impl From<ReadVar<String>> for GhParam {
1788 fn from(param: ReadVar<String>) -> GhParam {
1789 GhParam::FloweyVar(param)
1790 }
1791 }
1792
1793 pub type ClaimedGhParam = GhParam<VarClaimed>;
1794
1795 impl ClaimVar for GhParam {
1796 type Claimed = ClaimedGhParam;
1797
1798 fn claim(self, ctx: &mut StepCtx<'_>) -> ClaimedGhParam {
1799 match self {
1800 GhParam::Static(s) => ClaimedGhParam::Static(s),
1801 GhParam::FloweyVar(var) => match &var.backing_var {
1802 ReadVarBacking::RuntimeVar(_) => ClaimedGhParam::FloweyVar(var.claim(ctx)),
1803 ReadVarBacking::Inline(var) => ClaimedGhParam::Static(var.clone()),
1804 ReadVarBacking::InlineSideEffect => {
1805 panic!("inline side-effect vars are not supported")
1806 }
1807 },
1808 }
1809 }
1810 }
1811
1812 /// The assigned permission value for a scope.
1813 ///
1814 /// For more details on how these values affect a particular scope, refer to:
1815 /// <https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs>
1816 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
1817 pub enum GhPermissionValue {
1818 Read,
1819 Write,
1820 None,
1821 }
1822
1823 /// Refers to the scope of a permission granted to the GITHUB_TOKEN
1824 /// for a job.
1825 ///
1826 /// For more details on each scope, refer to:
1827 /// <https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs>
1828 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
1829 pub enum GhPermission {
1830 Actions,
1831 Attestations,
1832 Checks,
1833 Contents,
1834 Deployments,
1835 Discussions,
1836 IdToken,
1837 Issues,
1838 Packages,
1839 Pages,
1840 PullRequests,
1841 RepositoryProjects,
1842 SecurityEvents,
1843 Statuses,
1844 }
1845 }
1846
1847 pub mod rust {
1848 use crate::node::ClaimedReadVar;
1849 use crate::node::ClaimedWriteVar;
1850 use crate::node::FlowArch;
1851 use crate::node::FlowBackend;
1852 use crate::node::FlowPlatform;
1853 use crate::node::RuntimeVarDb;
1854 use serde::Serialize;
1855 use serde::de::DeserializeOwned;
1856
1857 pub fn new_rust_runtime_services(
1858 runtime_var_db: &mut dyn RuntimeVarDb,
1859 backend: FlowBackend,
1860 platform: FlowPlatform,
1861 arch: FlowArch,
1862 ) -> RustRuntimeServices<'_> {
1863 RustRuntimeServices {
1864 runtime_var_db,
1865 backend,
1866 platform,
1867 arch,
1868 }
1869 }
1870
1871 pub struct RustRuntimeServices<'a> {
1872 runtime_var_db: &'a mut dyn RuntimeVarDb,
1873 backend: FlowBackend,
1874 platform: FlowPlatform,
1875 arch: FlowArch,
1876 }
1877
1878 impl RustRuntimeServices<'_> {
1879 /// What backend the flow is being running on (e.g: locally, ADO,
1880 /// GitHub, etc...)
1881 pub fn backend(&self) -> FlowBackend {
1882 self.backend
1883 }
1884
1885 /// What platform the flow is being running on (e.g: windows, linux,
1886 /// etc...).
1887 pub fn platform(&self) -> FlowPlatform {
1888 self.platform
1889 }
1890
1891 /// What arch the flow is being running on (X86_64 or Aarch64)
1892 pub fn arch(&self) -> FlowArch {
1893 self.arch
1894 }
1895
1896 pub fn write<T>(&mut self, var: ClaimedWriteVar<T>, val: &T)
1897 where
1898 T: Serialize + DeserializeOwned,
1899 {
1900 self.runtime_var_db.set_var(
1901 &var.backing_var,
1902 var.is_secret,
1903 serde_json::to_vec(val).expect("improve this error path"),
1904 );
1905 }
1906
1907 pub fn write_all<T>(
1908 &mut self,
1909 vars: impl IntoIterator<Item = ClaimedWriteVar<T>>,
1910 val: &T,
1911 ) where
1912 T: Serialize + DeserializeOwned,
1913 {
1914 for var in vars {
1915 self.write(var, val)
1916 }
1917 }
1918
1919 pub fn read<T>(&mut self, var: ClaimedReadVar<T>) -> T
1920 where
1921 T: Serialize + DeserializeOwned,
1922 {
1923 match var.backing_var {
1924 crate::node::ReadVarBacking::RuntimeVar(var) => {
1925 let data = self.runtime_var_db.get_var(&var);
1926 serde_json::from_slice(&data).expect("improve this error path")
1927 }
1928 crate::node::ReadVarBacking::Inline(val) => val,
1929 crate::node::ReadVarBacking::InlineSideEffect => unreachable!(),
1930 }
1931 }
1932
1933 /// DANGEROUS: Set the value of _Global_ Environment Variable (GitHub Actions only).
1934 ///
1935 /// It is up to the caller to ensure that the variable does not get
1936 /// unintentionally overwritten or used.
1937 ///
1938 /// This method should be used rarely and with great care!
1939 pub fn dangerous_gh_set_global_env_var(
1940 &mut self,
1941 var: String,
1942 gh_env_var: String,
1943 ) -> anyhow::Result<()> {
1944 if !matches!(self.backend, FlowBackend::Github) {
1945 return Err(anyhow::anyhow!(
1946 "dangerous_set_gh_env_var can only be used on GitHub Actions"
1947 ));
1948 }
1949
1950 let gh_env_file_path = std::env::var("GITHUB_ENV")?;
1951 let mut gh_env_file = fs_err::OpenOptions::new()
1952 .append(true)
1953 .open(gh_env_file_path)?;
1954 let gh_env_var_assignment = format!(
1955 r#"{}<<EOF
1956{}
1957EOF
1958"#,
1959 gh_env_var, var
1960 );
1961 std::io::Write::write_all(&mut gh_env_file, gh_env_var_assignment.as_bytes())?;
1962
1963 Ok(())
1964 }
1965 }
1966 }
1967}
1968
1969/// The base underlying implementation of all FlowNode variants.
1970///
1971/// Do not implement this directly! Use the `new_flow_node!` family of macros
1972/// instead!
1973pub trait FlowNodeBase {
1974 type Request: Serialize + DeserializeOwned;
1975
1976 fn imports(&mut self, ctx: &mut ImportCtx<'_>);
1977 fn emit(&mut self, requests: Vec<Self::Request>, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()>;
1978
1979 /// A noop method that all human-written impls of `FlowNodeBase` are
1980 /// required to implement.
1981 ///
1982 /// By implementing this method, you're stating that you "know what you're
1983 /// doing" by having this manual impl.
1984 fn i_know_what_im_doing_with_this_manual_impl(&mut self);
1985}
1986
1987pub mod erased {
1988 use crate::node::FlowNodeBase;
1989 use crate::node::NodeCtx;
1990 use crate::node::user_facing::*;
1991
1992 pub struct ErasedNode<N: FlowNodeBase>(pub N);
1993
1994 impl<N: FlowNodeBase> ErasedNode<N> {
1995 pub fn from_node(node: N) -> Self {
1996 Self(node)
1997 }
1998 }
1999
2000 impl<N> FlowNodeBase for ErasedNode<N>
2001 where
2002 N: FlowNodeBase,
2003 {
2004 // FIXME: this should be using type-erased serde
2005 type Request = Box<[u8]>;
2006
2007 fn imports(&mut self, ctx: &mut ImportCtx<'_>) {
2008 self.0.imports(ctx)
2009 }
2010
2011 fn emit(&mut self, requests: Vec<Box<[u8]>>, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> {
2012 let mut converted_requests = Vec::new();
2013 for req in requests {
2014 converted_requests.push(serde_json::from_slice(&req)?)
2015 }
2016
2017 self.0.emit(converted_requests, ctx)
2018 }
2019
2020 fn i_know_what_im_doing_with_this_manual_impl(&mut self) {}
2021 }
2022}
2023
2024/// Cheap handle to a registered [`FlowNode`]
2025#[derive(Clone, Copy, PartialEq, Eq, Hash)]
2026pub struct NodeHandle(std::any::TypeId);
2027
2028impl Ord for NodeHandle {
2029 fn cmp(&self, other: &Self) -> std::cmp::Ordering {
2030 self.modpath().cmp(other.modpath())
2031 }
2032}
2033
2034impl PartialOrd for NodeHandle {
2035 fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
2036 Some(self.cmp(other))
2037 }
2038}
2039
2040impl std::fmt::Debug for NodeHandle {
2041 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2042 std::fmt::Debug::fmt(&self.try_modpath(), f)
2043 }
2044}
2045
2046impl NodeHandle {
2047 pub fn from_type<N: FlowNodeBase + 'static>() -> NodeHandle {
2048 NodeHandle(std::any::TypeId::of::<N>())
2049 }
2050
2051 pub fn from_modpath(modpath: &str) -> NodeHandle {
2052 node_luts::erased_node_by_modpath().get(modpath).unwrap().0
2053 }
2054
2055 pub fn try_from_modpath(modpath: &str) -> Option<NodeHandle> {
2056 node_luts::erased_node_by_modpath()
2057 .get(modpath)
2058 .map(|(s, _)| *s)
2059 }
2060
2061 pub fn new_erased_node(&self) -> Box<dyn FlowNodeBase<Request = Box<[u8]>>> {
2062 let ctor = node_luts::erased_node_by_typeid().get(self).unwrap();
2063 ctor()
2064 }
2065
2066 pub fn modpath(&self) -> &'static str {
2067 node_luts::modpath_by_node_typeid().get(self).unwrap()
2068 }
2069
2070 pub fn try_modpath(&self) -> Option<&'static str> {
2071 node_luts::modpath_by_node_typeid().get(self).cloned()
2072 }
2073
2074 /// Return a dummy NodeHandle, which will panic if `new_erased_node` is ever
2075 /// called on it.
2076 pub fn dummy() -> NodeHandle {
2077 NodeHandle(std::any::TypeId::of::<()>())
2078 }
2079}
2080
2081pub fn list_all_registered_nodes() -> impl Iterator<Item = NodeHandle> {
2082 node_luts::modpath_by_node_typeid().keys().cloned()
2083}
2084
2085// Encapsulate these look up tables in their own module to limit the scope of
2086// the HashMap import.
2087//
2088// In general, using HashMap in flowey is a recipe for disaster, given that
2089// iterating through the hash-map will result in non-deterministic orderings,
2090// which can cause annoying ordering churn.
2091//
2092// That said, in this case, it's OK since the code using these LUTs won't ever
2093// iterate through the map.
2094//
2095// Why is the HashMap even necessary vs. a BTreeMap?
2096//
2097// Well... NodeHandle's `Ord` impl does a `modpath` comparison instead of a
2098// TypeId comparison, since TypeId will vary between compilations.
2099mod node_luts {
2100 use super::FlowNodeBase;
2101 use super::NodeHandle;
2102 use std::collections::HashMap;
2103 use std::sync::OnceLock;
2104
2105 pub(super) fn modpath_by_node_typeid() -> &'static HashMap<NodeHandle, &'static str> {
2106 static TYPEID_TO_MODPATH: OnceLock<HashMap<NodeHandle, &'static str>> = OnceLock::new();
2107
2108 let lookup = TYPEID_TO_MODPATH.get_or_init(|| {
2109 let mut lookup = HashMap::new();
2110 for crate::node::private::FlowNodeMeta {
2111 module_path,
2112 ctor: _,
2113 get_typeid,
2114 } in crate::node::private::FLOW_NODES
2115 {
2116 let existing = lookup.insert(
2117 NodeHandle(get_typeid()),
2118 module_path
2119 .strip_suffix("::_only_one_call_to_flowey_node_per_module")
2120 .unwrap(),
2121 );
2122 // if this were to fire for an array where the key is a TypeId...
2123 // something has gone _terribly_ wrong
2124 assert!(existing.is_none())
2125 }
2126
2127 lookup
2128 });
2129
2130 lookup
2131 }
2132
2133 pub(super) fn erased_node_by_typeid()
2134 -> &'static HashMap<NodeHandle, fn() -> Box<dyn FlowNodeBase<Request = Box<[u8]>>>> {
2135 static LOOKUP: OnceLock<
2136 HashMap<NodeHandle, fn() -> Box<dyn FlowNodeBase<Request = Box<[u8]>>>>,
2137 > = OnceLock::new();
2138
2139 let lookup = LOOKUP.get_or_init(|| {
2140 let mut lookup = HashMap::new();
2141 for crate::node::private::FlowNodeMeta {
2142 module_path: _,
2143 ctor,
2144 get_typeid,
2145 } in crate::node::private::FLOW_NODES
2146 {
2147 let existing = lookup.insert(NodeHandle(get_typeid()), *ctor);
2148 // if this were to fire for an array where the key is a TypeId...
2149 // something has gone _terribly_ wrong
2150 assert!(existing.is_none())
2151 }
2152
2153 lookup
2154 });
2155
2156 lookup
2157 }
2158
2159 pub(super) fn erased_node_by_modpath() -> &'static HashMap<
2160 &'static str,
2161 (
2162 NodeHandle,
2163 fn() -> Box<dyn FlowNodeBase<Request = Box<[u8]>>>,
2164 ),
2165 > {
2166 static MODPATH_LOOKUP: OnceLock<
2167 HashMap<
2168 &'static str,
2169 (
2170 NodeHandle,
2171 fn() -> Box<dyn FlowNodeBase<Request = Box<[u8]>>>,
2172 ),
2173 >,
2174 > = OnceLock::new();
2175
2176 let lookup = MODPATH_LOOKUP.get_or_init(|| {
2177 let mut lookup = HashMap::new();
2178 for crate::node::private::FlowNodeMeta { module_path, ctor, get_typeid } in crate::node::private::FLOW_NODES {
2179 let existing = lookup.insert(module_path.strip_suffix("::_only_one_call_to_flowey_node_per_module").unwrap(), (NodeHandle(get_typeid()), *ctor));
2180 if existing.is_some() {
2181 panic!("conflicting node registrations at {module_path}! please ensure there is a single node per module!")
2182 }
2183 }
2184 lookup
2185 });
2186
2187 lookup
2188 }
2189}
2190
2191#[doc(hidden)]
2192pub mod private {
2193 pub use linkme;
2194
2195 pub struct FlowNodeMeta {
2196 pub module_path: &'static str,
2197 pub ctor: fn() -> Box<dyn super::FlowNodeBase<Request = Box<[u8]>>>,
2198 // FUTURE: there is a RFC to make this const
2199 pub get_typeid: fn() -> std::any::TypeId,
2200 }
2201
2202 #[linkme::distributed_slice]
2203 pub static FLOW_NODES: [FlowNodeMeta] = [..];
2204
2205 // UNSAFETY: linkme uses manual link sections, which are unsafe.
2206 #[expect(unsafe_code)]
2207 #[linkme::distributed_slice(FLOW_NODES)]
2208 static DUMMY_FLOW_NODE: FlowNodeMeta = FlowNodeMeta {
2209 module_path: "<dummy>::_only_one_call_to_flowey_node_per_module",
2210 ctor: || unreachable!(),
2211 get_typeid: std::any::TypeId::of::<()>,
2212 };
2213}
2214
2215#[doc(hidden)]
2216#[macro_export]
2217macro_rules! new_flow_node_base {
2218 (struct Node) => {
2219 /// (see module-level docs)
2220 #[non_exhaustive]
2221 pub struct Node;
2222
2223 mod _only_one_call_to_flowey_node_per_module {
2224 const _: () = {
2225 use $crate::node::private::linkme;
2226
2227 fn new_erased() -> Box<dyn $crate::node::FlowNodeBase<Request = Box<[u8]>>> {
2228 Box::new($crate::node::erased::ErasedNode(super::Node))
2229 }
2230
2231 #[linkme::distributed_slice($crate::node::private::FLOW_NODES)]
2232 #[linkme(crate = linkme)]
2233 static FLOW_NODE: $crate::node::private::FlowNodeMeta =
2234 $crate::node::private::FlowNodeMeta {
2235 module_path: module_path!(),
2236 ctor: new_erased,
2237 get_typeid: std::any::TypeId::of::<super::Node>,
2238 };
2239 };
2240 }
2241 };
2242}
2243
2244/// TODO: clearly verbalize what a `FlowNode` encompasses
2245pub trait FlowNode {
2246 /// TODO: clearly verbalize what a Request encompasses
2247 type Request: Serialize + DeserializeOwned;
2248
2249 /// A list of nodes that this node is capable of taking a dependency on.
2250 ///
2251 /// Attempting to take a dep on a node that wasn't imported via this method
2252 /// will result in an error during flow resolution time.
2253 ///
2254 /// * * *
2255 ///
2256 /// To put it bluntly: This is boilerplate.
2257 ///
2258 /// We (the flowey devs) are thinking about ways to avoid requiring this
2259 /// method, but do not have a good solution at this time.
2260 fn imports(ctx: &mut ImportCtx<'_>);
2261
2262 /// Given a set of incoming `requests`, emit various steps to run, set
2263 /// various dependencies, etc...
2264 fn emit(requests: Vec<Self::Request>, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()>;
2265}
2266
2267#[macro_export]
2268macro_rules! new_flow_node {
2269 (struct Node) => {
2270 $crate::new_flow_node_base!(struct Node);
2271
2272 impl $crate::node::FlowNodeBase for Node
2273 where
2274 Node: FlowNode,
2275 {
2276 type Request = <Node as FlowNode>::Request;
2277
2278 fn imports(&mut self, dep: &mut ImportCtx<'_>) {
2279 <Node as FlowNode>::imports(dep)
2280 }
2281
2282 fn emit(
2283 &mut self,
2284 requests: Vec<Self::Request>,
2285 ctx: &mut NodeCtx<'_>,
2286 ) -> anyhow::Result<()> {
2287 <Node as FlowNode>::emit(requests, ctx)
2288 }
2289
2290 fn i_know_what_im_doing_with_this_manual_impl(&mut self) {}
2291 }
2292 };
2293}
2294
2295/// A helper trait to streamline implementing [`FlowNode`] instances that only
2296/// ever operate on a single request at a time.
2297///
2298/// In essence, [`SimpleFlowNode`] handles the boilerplate (and rightward-drift)
2299/// of manually writing:
2300///
2301/// ```ignore
2302/// impl FlowNode for Node {
2303/// fn imports(dep: &mut ImportCtx<'_>) { ... }
2304/// fn emit(requests: Vec<Self::Request>, ctx: &mut NodeCtx<'_>) {
2305/// for req in requests {
2306/// Node::process_request(req, ctx)
2307/// }
2308/// }
2309/// }
2310/// ```
2311///
2312/// Nodes which accept a `struct Request` often fall into this pattern, whereas
2313/// nodes which accept a `enum Request` typically require additional logic to
2314/// aggregate / resolve incoming requests.
2315pub trait SimpleFlowNode {
2316 type Request: Serialize + DeserializeOwned;
2317
2318 /// A list of nodes that this node is capable of taking a dependency on.
2319 ///
2320 /// Attempting to take a dep on a node that wasn't imported via this method
2321 /// will result in an error during flow resolution time.
2322 ///
2323 /// * * *
2324 ///
2325 /// To put it bluntly: This is boilerplate.
2326 ///
2327 /// We (the flowey devs) are thinking about ways to avoid requiring this
2328 /// method, but do not have a good solution at this time.
2329 fn imports(ctx: &mut ImportCtx<'_>);
2330
2331 /// Process a single incoming `Self::Request`
2332 fn process_request(request: Self::Request, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()>;
2333}
2334
2335#[macro_export]
2336macro_rules! new_simple_flow_node {
2337 (struct Node) => {
2338 $crate::new_flow_node_base!(struct Node);
2339
2340 impl $crate::node::FlowNodeBase for Node
2341 where
2342 Node: SimpleFlowNode,
2343 {
2344 type Request = <Node as SimpleFlowNode>::Request;
2345
2346 fn imports(&mut self, dep: &mut ImportCtx<'_>) {
2347 <Node as SimpleFlowNode>::imports(dep)
2348 }
2349
2350 fn emit(&mut self, requests: Vec<Self::Request>, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> {
2351 for req in requests {
2352 <Node as SimpleFlowNode>::process_request(req, ctx)?
2353 }
2354
2355 Ok(())
2356 }
2357
2358 fn i_know_what_im_doing_with_this_manual_impl(&mut self) {}
2359 }
2360 };
2361}
2362
2363/// A "glue" trait which improves [`NodeCtx::req`] ergonomics, by tying a
2364/// particular `Request` type to its corresponding [`FlowNode`].
2365///
2366/// This trait should be autogenerated via [`flowey_request!`] - do not try to
2367/// implement it manually!
2368///
2369/// [`flowey_request!`]: crate::flowey_request
2370pub trait IntoRequest {
2371 type Node: FlowNodeBase;
2372 fn into_request(self) -> <Self::Node as FlowNodeBase>::Request;
2373
2374 /// By implementing this method manually, you're indicating that you know what you're
2375 /// doing,
2376 #[doc(hidden)]
2377 #[allow(nonstandard_style)]
2378 fn do_not_manually_impl_this_trait__use_the_flowey_request_macro_instead(&mut self);
2379}
2380
2381#[doc(hidden)]
2382#[macro_export]
2383macro_rules! __flowey_request_inner {
2384 //
2385 // @emit_struct: emit structs for each variant of the request enum
2386 //
2387 (@emit_struct [$req:ident]
2388 $(#[$a:meta])*
2389 $variant:ident($($tt:tt)*),
2390 $($rest:tt)*
2391 ) => {
2392 $(#[$a])*
2393 #[derive(Serialize, Deserialize)]
2394 pub struct $variant($($tt)*);
2395
2396 impl IntoRequest for $variant {
2397 type Node = Node;
2398 fn into_request(self) -> $req {
2399 $req::$variant(self)
2400 }
2401 fn do_not_manually_impl_this_trait__use_the_flowey_request_macro_instead(&mut self) {}
2402 }
2403
2404 $crate::__flowey_request_inner!(@emit_struct [$req] $($rest)*);
2405 };
2406 (@emit_struct [$req:ident]
2407 $(#[$a:meta])*
2408 $variant:ident { $($tt:tt)* },
2409 $($rest:tt)*
2410 ) => {
2411 $(#[$a])*
2412 #[derive(Serialize, Deserialize)]
2413 pub struct $variant {
2414 $($tt)*
2415 }
2416
2417 impl IntoRequest for $variant {
2418 type Node = Node;
2419 fn into_request(self) -> $req {
2420 $req::$variant(self)
2421 }
2422 fn do_not_manually_impl_this_trait__use_the_flowey_request_macro_instead(&mut self) {}
2423 }
2424
2425 $crate::__flowey_request_inner!(@emit_struct [$req] $($rest)*);
2426 };
2427 (@emit_struct [$req:ident]
2428 $(#[$a:meta])*
2429 $variant:ident,
2430 $($rest:tt)*
2431 ) => {
2432 $(#[$a])*
2433 #[derive(Serialize, Deserialize)]
2434 pub struct $variant;
2435
2436 impl IntoRequest for $variant {
2437 type Node = Node;
2438 fn into_request(self) -> $req {
2439 $req::$variant(self)
2440 }
2441 fn do_not_manually_impl_this_trait__use_the_flowey_request_macro_instead(&mut self) {}
2442 }
2443
2444 $crate::__flowey_request_inner!(@emit_struct [$req] $($rest)*);
2445 };
2446 (@emit_struct [$req:ident]
2447 ) => {};
2448
2449 //
2450 // @emit_req_enum: build up root request enum
2451 //
2452 (@emit_req_enum [$req:ident($($root_a:meta,)*), $($prev:ident[$($prev_a:meta,)*])*]
2453 $(#[$a:meta])*
2454 $variant:ident($($tt:tt)*),
2455 $($rest:tt)*
2456 ) => {
2457 $crate::__flowey_request_inner!(@emit_req_enum [$req($($root_a,)*), $($prev[$($prev_a,)*])* $variant[$($a,)*]] $($rest)*);
2458 };
2459 (@emit_req_enum [$req:ident($($root_a:meta,)*), $($prev:ident[$($prev_a:meta,)*])*]
2460 $(#[$a:meta])*
2461 $variant:ident { $($tt:tt)* },
2462 $($rest:tt)*
2463 ) => {
2464 $crate::__flowey_request_inner!(@emit_req_enum [$req($($root_a,)*), $($prev[$($prev_a,)*])* $variant[$($a,)*]] $($rest)*);
2465 };
2466 (@emit_req_enum [$req:ident($($root_a:meta,)*), $($prev:ident[$($prev_a:meta,)*])*]
2467 $(#[$a:meta])*
2468 $variant:ident,
2469 $($rest:tt)*
2470 ) => {
2471 $crate::__flowey_request_inner!(@emit_req_enum [$req($($root_a,)*), $($prev[$($prev_a,)*])* $variant[$($a,)*]] $($rest)*);
2472 };
2473 (@emit_req_enum [$req:ident($($root_a:meta,)*), $($prev:ident[$($prev_a:meta,)*])*]
2474 ) => {
2475 #[derive(Serialize, Deserialize)]
2476 pub enum $req {$(
2477 $(#[$prev_a])*
2478 $prev(self::req::$prev),
2479 )*}
2480
2481 impl IntoRequest for $req {
2482 type Node = Node;
2483 fn into_request(self) -> $req {
2484 self
2485 }
2486 fn do_not_manually_impl_this_trait__use_the_flowey_request_macro_instead(&mut self) {}
2487 }
2488 };
2489}
2490
2491/// Declare a new `Request` type for the current `Node`.
2492///
2493/// ## `struct` and `enum` Requests
2494///
2495/// When wrapping a vanilla Rust `struct` and `enum` declaration, this macro
2496/// simply derives [`Serialize`], [`Deserialize`], and [`IntoRequest`] for the
2497/// type, and does nothing else.
2498///
2499/// ## `enum_struct` Requests
2500///
2501/// This macro also supports a special kind of `enum_struct` derive, which
2502/// allows declaring a Request enum where each variant is split off into its own
2503/// separate (named) `struct`.
2504///
2505/// e.g:
2506///
2507/// ```ignore
2508/// flowey_request! {
2509/// pub enum_struct Foo {
2510/// Bar,
2511/// Baz(pub usize),
2512/// Qux(pub String),
2513/// }
2514/// }
2515/// ```
2516///
2517/// will be expanded into:
2518///
2519/// ```ignore
2520/// #[derive(Serialize, Deserialize)]
2521/// pub enum Foo {
2522/// Bar(req::Bar),
2523/// Baz(req::Baz),
2524/// Qux(req::Qux),
2525/// }
2526///
2527/// pud mod req {
2528/// #[derive(Serialize, Deserialize)]
2529/// pub struct Bar;
2530///
2531/// #[derive(Serialize, Deserialize)]
2532/// pub struct Baz(pub usize);
2533///
2534/// #[derive(Serialize, Deserialize)]
2535/// pub struct Qux(pub String);
2536/// }
2537/// ```
2538#[macro_export]
2539macro_rules! flowey_request {
2540 (
2541 $(#[$root_a:meta])*
2542 pub enum_struct $req:ident {
2543 $($tt:tt)*
2544 }
2545 ) => {
2546 $crate::__flowey_request_inner!(@emit_req_enum [$req($($root_a,)*),] $($tt)*);
2547 pub mod req {
2548 use super::*;
2549 $crate::__flowey_request_inner!(@emit_struct [$req] $($tt)*);
2550 }
2551 };
2552
2553 (
2554 $(#[$a:meta])*
2555 pub enum $req:ident {
2556 $($tt:tt)*
2557 }
2558 ) => {
2559 $(#[$a])*
2560 #[derive(Serialize, Deserialize)]
2561 pub enum $req {
2562 $($tt)*
2563 }
2564
2565 impl IntoRequest for $req {
2566 type Node = Node;
2567 fn into_request(self) -> $req {
2568 self
2569 }
2570 fn do_not_manually_impl_this_trait__use_the_flowey_request_macro_instead(&mut self) {}
2571 }
2572 };
2573
2574 (
2575 $(#[$a:meta])*
2576 pub struct $req:ident {
2577 $($tt:tt)*
2578 }
2579 ) => {
2580 $(#[$a])*
2581 #[derive(Serialize, Deserialize)]
2582 pub struct $req {
2583 $($tt)*
2584 }
2585
2586 impl IntoRequest for $req {
2587 type Node = Node;
2588 fn into_request(self) -> $req {
2589 self
2590 }
2591 fn do_not_manually_impl_this_trait__use_the_flowey_request_macro_instead(&mut self) {}
2592 }
2593 };
2594
2595 (
2596 $(#[$a:meta])*
2597 pub struct $req:ident($($tt:tt)*);
2598 ) => {
2599 $(#[$a])*
2600 #[derive(Serialize, Deserialize)]
2601 pub struct $req($($tt)*);
2602
2603 impl IntoRequest for $req {
2604 type Node = Node;
2605 fn into_request(self) -> $req {
2606 self
2607 }
2608 fn do_not_manually_impl_this_trait__use_the_flowey_request_macro_instead(&mut self) {}
2609 }
2610 };
2611}
2612