GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/openvmm

Public

mirrored from https://github.com/microsoft/openvmmAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
release/2411-fork

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openvmm/flowey/flowey_cli/src/pipeline_resolver

flowey/flowey_cli/src/pipeline_resolver/common_yaml.rs

276lines · modecode

RawDownload
Latest commit unavailable.
1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4//! Shared functionality for emitting a pipeline as ADO/GitHub YAML files
5
6use crate::cli::exec_snippet::FloweyPipelineStaticDb;
7use crate::pipeline_resolver::generic::ResolvedPipelineJob;
8use anyhow::Context;
9use flowey_core::node::FlowPlatform;
10use petgraph::visit::EdgeRef;
11use serde::Serialize;
12use serde_yaml::Value;
13use std::collections::BTreeMap;
14use std::collections::BTreeSet;
15use std::io::Write;
16use std::path::Path;
17use std::path::PathBuf;
18
19#[derive(Debug)]
20pub(crate) enum FloweySource {
21 // bool indicates if this node should publish the flowey it bootstraps for
22 // other nodes to consume
23 Bootstrap(String, bool),
24 Consume(String),
25}
26
27/// each job has one of three "roles" when it comes to bootstrapping flowey:
28///
29/// 1. Build flowey
30/// 2. Building _and_ publishing flowey
31/// 3. Consuming a pre-built flowey
32///
33/// We _could_ just have every bootstrap job also publish flowey, but this
34/// will spam the artifact feed with artifacts no one will consume, which is
35/// wasteful.
36///
37/// META: why go through all this hassle anyways? i.e: why not just do
38/// something dead simple like:
39///
40/// - discover which platforms exist in the graph
41/// - have the first jobs of every pipeline be standalone "bootstrap flowey"
42/// jobs, which all subsequent jobs of a certain platform can take a dep on
43///
44/// well... it turns out that provisioning job runners is _sloooooow_,
45/// and having every single pipeline run these "bootstrap flowey" steps
46/// gating the rest of the "interesting" stuff would really stink.
47///
48/// i.e: it's better to do redundant flowey bootstraps if it means that we
49/// can avoid the extra time it takes to tear down + re-provision a worker.
50pub(crate) fn job_flowey_bootstrap_source(
51 graph: &petgraph::Graph<ResolvedPipelineJob, ()>,
52 order: &Vec<petgraph::prelude::NodeIndex>,
53) -> BTreeMap<petgraph::prelude::NodeIndex, FloweySource> {
54 let mut bootstrapped_flowey = BTreeMap::new();
55
56 // the first traversal builds a list of all ancestors of a give node
57 let mut ancestors = BTreeMap::<
58 petgraph::prelude::NodeIndex,
59 BTreeSet<(petgraph::prelude::NodeIndex, FlowPlatform)>,
60 >::new();
61 for idx in order {
62 for ancestor_idx in graph
63 .edges_directed(*idx, petgraph::Direction::Incoming)
64 .map(|e| e.source())
65 {
66 ancestors
67 .entry(*idx)
68 .or_default()
69 .insert((ancestor_idx, graph[ancestor_idx].platform));
70
71 if let Some(set) = ancestors.get(&ancestor_idx).cloned() {
72 ancestors.get_mut(idx).unwrap().extend(&set);
73 }
74 }
75 }
76
77 // the second traversal assigns roles to each node
78 let mut floweyno = 0;
79 'outer: for idx in order {
80 let ancestors = ancestors.remove(idx).unwrap_or_default();
81
82 let mut elect_bootstrap = None;
83
84 for (ancestor_idx, platform) in ancestors {
85 if platform != graph[*idx].platform {
86 continue;
87 }
88
89 let role =
90 bootstrapped_flowey
91 .get_mut(&ancestor_idx)
92 .and_then(|existing| match existing {
93 FloweySource::Bootstrap(s, true) => Some(FloweySource::Consume(s.clone())),
94 FloweySource::Consume(s) => Some(FloweySource::Consume(s.clone())),
95 // there is an ancestor that is building, but not
96 // publishing. maybe they should get upgraded...
97 FloweySource::Bootstrap(_, false) => {
98 elect_bootstrap = Some(ancestor_idx);
99 None
100 }
101 });
102
103 if let Some(role) = role {
104 bootstrapped_flowey.insert(*idx, role);
105 continue 'outer;
106 }
107 }
108
109 // if we got here, that means we couldn't find a valid ancestor.
110 //
111 // check if we can upgrade an existing ancestor vs. bootstrapping
112 // things ourselves
113 if let Some(elect_bootstrap) = elect_bootstrap {
114 let FloweySource::Bootstrap(s, publish) =
115 bootstrapped_flowey.get_mut(&elect_bootstrap).unwrap()
116 else {
117 unreachable!()
118 };
119
120 *publish = true;
121 let s = s.clone();
122
123 bootstrapped_flowey.insert(*idx, FloweySource::Consume(s));
124 } else {
125 // Having this extra unique `floweyno` per bootstrap is
126 // necessary since GitHub doesn't let you double-publish an
127 // artifact with the same name
128 floweyno += 1;
129 let platform = graph[*idx].platform;
130 let arch = graph[*idx].arch;
131 bootstrapped_flowey.insert(
132 *idx,
133 FloweySource::Bootstrap(
134 format!("_internal-flowey-bootstrap-{arch}-{platform}-uid-{floweyno}"),
135 false,
136 ),
137 );
138 }
139 }
140
141 bootstrapped_flowey
142}
143
144/// convert `pipeline` to YAML and `pipeline_static_db` to JSON.
145/// if `check` is `Some`, then we will compare the generated YAML and JSON
146/// against the contents of `check` and error if they don't match.
147/// if `check` is `None`, then we will write the generated YAML and JSON to
148/// `repo_root/pipeline_file.yaml` and `repo_root/pipeline_file.json` respectively.
149fn check_or_write_generated_yaml_and_json<T>(
150 pipeline: &T,
151 pipeline_static_db: &FloweyPipelineStaticDb,
152 check: &Option<PathBuf>,
153 repo_root: &Path,
154 pipeline_file: &Path,
155 ado_post_process_yaml_cb: Option<Box<dyn FnOnce(Value) -> Value>>,
156) -> anyhow::Result<()>
157where
158 T: Serialize,
159{
160 let generated_yaml =
161 serde_yaml::to_value(pipeline).context("while serializing pipeline yaml")?;
162 let generated_yaml = if let Some(ado_post_process_yaml_cb) = ado_post_process_yaml_cb {
163 ado_post_process_yaml_cb(generated_yaml)
164 } else {
165 generated_yaml
166 };
167
168 let generated_yaml =
169 serde_yaml::to_string(&generated_yaml).context("while emitting pipeline yaml")?;
170 let generated_yaml = format!(
171 r#"
172##############################
173# THIS FILE IS AUTOGENERATED #
174# DO NOT MANUALLY EDIT #
175##############################
176{generated_yaml}"#
177 );
178 let generated_yaml = generated_yaml.trim_start();
179
180 let generated_json =
181 serde_json::to_string_pretty(pipeline_static_db).context("while emitting pipeline json")?;
182
183 if let Some(check_file) = check {
184 let existing_yaml = fs_err::read_to_string(check_file)
185 .context("cannot check pipeline that doesn't exist!")?;
186 let existing_json = fs_err::read_to_string(check_file.with_extension("json"))
187 .context("pipeline yaml doesn't have corresponding pipeline json")?;
188
189 let yaml_out_of_date = existing_yaml != generated_yaml;
190 let json_out_of_date = existing_json != generated_json;
191
192 if yaml_out_of_date {
193 println!(
194 "generated yaml {}:\n==========\n{generated_yaml}",
195 generated_yaml.len()
196 );
197 println!(
198 "existing yaml {}:\n==========\n{existing_yaml}",
199 existing_yaml.len()
200 );
201 }
202
203 if json_out_of_date {
204 println!(
205 "generated json {}:\n==========\n{generated_json}",
206 generated_json.len()
207 );
208 println!(
209 "existing json {}:\n==========\n{existing_json}",
210 existing_json.len()
211 );
212 }
213
214 if yaml_out_of_date || json_out_of_date {
215 anyhow::bail!("checked in pipeline YAML/JSON is out of date! run `cargo xflowey regen`")
216 }
217
218 Ok(())
219 } else {
220 let out_yaml_path = repo_root.join(pipeline_file);
221 let out_pipeline_db_path = repo_root.join(pipeline_file).with_extension("json");
222
223 let mut f = fs_err::File::create(out_yaml_path)?;
224 f.write_all(generated_yaml.as_bytes())
225 .context("while emitting pipeline yaml")?;
226
227 let mut f = fs_err::File::create(out_pipeline_db_path)?;
228 f.write_all(generated_json.as_bytes())
229 .context("while emitting pipeline database json")?;
230
231 Ok(())
232 }
233}
234
235/// See [`check_or_write_generated_yaml_and_json`]
236pub(crate) fn check_generated_yaml_and_json<T>(
237 pipeline: &T,
238 pipeline_static_db: &FloweyPipelineStaticDb,
239 check: PathBuf,
240 repo_root: &Path,
241 pipeline_file: &Path,
242 ado_post_process_yaml_cb: Option<Box<dyn FnOnce(Value) -> Value>>,
243) -> anyhow::Result<()>
244where
245 T: Serialize,
246{
247 check_or_write_generated_yaml_and_json(
248 pipeline,
249 pipeline_static_db,
250 &Some(check),
251 repo_root,
252 pipeline_file,
253 ado_post_process_yaml_cb,
254 )
255}
256
257/// See [`check_or_write_generated_yaml_and_json`]
258pub(crate) fn write_generated_yaml_and_json<T>(
259 pipeline: &T,
260 pipeline_static_db: &FloweyPipelineStaticDb,
261 repo_root: &Path,
262 pipeline_file: &Path,
263 ado_post_process_yaml_cb: Option<Box<dyn FnOnce(Value) -> Value>>,
264) -> anyhow::Result<()>
265where
266 T: Serialize,
267{
268 check_or_write_generated_yaml_and_json(
269 pipeline,
270 pipeline_static_db,
271 &None,
272 repo_root,
273 pipeline_file,
274 ado_post_process_yaml_cb,
275 )
276}
277