GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/openvmm

Public

mirrored fromhttps://github.com/microsoft/openvmmAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
fe2bbc9829558a07c5e06d2b6ececc383b6593bf

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openvmm/flowey/flowey_cli/src/pipeline_resolver

flowey/flowey_cli/src/pipeline_resolver/viz.rs

478lines · modecode

RawDownload
flowey: add first-class config support for flow nodes (#3160)5e694e9
1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4//! Debug backend that simply visualizes flows, instead of emitting them in any
5//! runnable format
6
7use crate::flow_resolver::stage1_dag::DepKind;
8use crate::flow_resolver::stage1_dag::OutputGraphEntry;
9use crate::flow_resolver::stage1_dag::StepId;
10use crate::pipeline_resolver::generic::ResolvedPipeline;
11use crate::pipeline_resolver::generic::ResolvedPipelineJob;
12use flowey_core::node::FlowArch;
13use flowey_core::node::FlowBackend;
14use flowey_core::node::FlowPlatform;
15use flowey_core::node::NodeHandle;
16use std::collections::BTreeMap;
17use std::collections::BTreeSet;
18
19pub fn viz_pipeline_toposort(
20 pipeline: ResolvedPipeline,
21 backend: FlowBackend,
22 with_persist_dir: bool,
23) -> anyhow::Result<()> {
24 viz_pipeline_generic(pipeline, backend, with_persist_dir, viz_flow_toposort)
25}
26
27pub fn viz_pipeline_flow_dot(
28 pipeline: ResolvedPipeline,
29 backend: FlowBackend,
30 with_persist_dir: bool,
31) -> anyhow::Result<()> {
32 viz_pipeline_generic(pipeline, backend, with_persist_dir, viz_flow_dot)
33}
34
35fn viz_pipeline_generic(
36 pipeline: ResolvedPipeline,
37 backend: FlowBackend,
38 with_persist_dir: bool,
39 f: fn(
40 seed_nodes: BTreeMap<NodeHandle, (bool, Vec<Box<[u8]>>)>,
41 seed_configs: BTreeMap<NodeHandle, Vec<Box<[u8]>>>,
42 resolved_patches: flowey_core::patch::ResolvedPatches,
43 external_read_vars: BTreeSet<String>,
44 backend: FlowBackend,
45 platform: FlowPlatform,
46 arch: FlowArch,
47 with_persist_dir: bool,
48 ) -> anyhow::Result<()>,
49) -> anyhow::Result<()> {
50 let ResolvedPipeline {
51 graph,
52 order,
53 parameters: _,
54 ado_name: _,
55 ado_schedule_triggers: _,
56 ado_ci_triggers: _,
57 ado_pr_triggers: _,
58 ado_bootstrap_template: _,
59 ado_resources_repository: _,
60 ado_post_process_yaml_cb: _,
61 ado_variables: _,
62 ado_job_id_overrides: _,
63 gh_name: _,
64 gh_schedule_triggers: _,
65 gh_ci_triggers: _,
66 gh_pr_triggers: _,
67 gh_bootstrap_template: _,
68 } = pipeline;
69
70 for idx in order {
71 let ResolvedPipelineJob {
72 ref root_nodes,
73 ref root_configs,
74 ref patches,
75 ref label,
76 platform,
77 arch,
78 cond_param_idx: _,
79 timeout_minutes: _,
80 command_wrapper: _,
81 ref ado_pool,
82 ado_variables: _,
83 gh_override_if: _,
84 gh_global_env: _,
85 ref gh_pool,
86 gh_permissions: _,
87 ref external_read_vars,
88 parameters_used: _,
89 ref artifacts_used,
90 ref artifacts_published,
91 } = graph[idx];
92
93 println!(
94 "== {}{}{} ==",
95 label,
96 ado_pool
97 .as_ref()
98 .map(|s| format!(" - {} ({})", s.name, s.demands.join(",")))
99 .unwrap_or_default(),
100 gh_pool
101 .as_ref()
102 .map(|s| format!(" - {:#?}", s))
103 .unwrap_or_default()
104 );
105 println!(
106 "artifacts used: {}",
107 artifacts_used
108 .iter()
109 .map(|a| a.name.clone())
110 .collect::<Vec<_>>()
111 .join(",\n ")
112 );
113 println!(
114 "artifacts published: {}",
115 artifacts_published
116 .iter()
117 .map(|a| a.name.clone())
118 .collect::<Vec<_>>()
119 .join(",\n ")
120 );
121 println!();
122
123 f(
124 root_nodes
125 .clone()
126 .into_iter()
127 .map(|(node, requests)| (node, (true, requests)))
128 .collect(),
129 root_configs.clone(),
130 patches.clone(),
131 external_read_vars.clone(),
132 backend,
133 platform,
134 arch,
135 with_persist_dir,
136 )?;
137
138 println!();
139 }
140
141 Ok(())
142}
143
144/// (debug) print the modpath of each node in topological sort of the flow
145pub fn viz_flow_toposort(
146 seed_nodes: BTreeMap<NodeHandle, (bool, Vec<Box<[u8]>>)>,
147 seed_configs: BTreeMap<NodeHandle, Vec<Box<[u8]>>>,
148 resolved_patches: flowey_core::patch::ResolvedPatches,
149 external_read_vars: BTreeSet<String>,
150 backend: FlowBackend,
151 platform: FlowPlatform,
152 arch: FlowArch,
153 with_persist_dir: bool,
154) -> anyhow::Result<()> {
155 // ignore the unreachable nodes error, since we want to allow debugging issues here
156 let crate::flow_resolver::stage1_dag::Stage1DagOutput {
157 mut output_graph, ..
158 } = crate::flow_resolver::stage1_dag::stage1_dag(
159 backend,
160 platform,
161 arch,
162 resolved_patches,
163 seed_nodes,
164 seed_configs,
165 external_read_vars,
166 with_persist_dir.then_some("<dummy>".into()),
167 )?;
168
169 let output_order = petgraph::algo::toposort(&output_graph, None)
170 .expect("runtime variables cannot introduce a DAG cycle");
171
172 let mut max_len = 0;
173 for &idx in output_order.iter().rev() {
174 max_len = max_len.max(
175 output_graph[idx]
176 .1
177 .as_ref()
178 .unwrap()
179 .node_handle
180 .modpath()
181 .len(),
182 )
183 }
184
185 for idx in output_order.into_iter().rev() {
186 let e = output_graph[idx].1.take().unwrap();
187 match &e.step {
188 crate::flow_resolver::stage1_dag::Step::Anchor { .. } => {}
189 crate::flow_resolver::stage1_dag::Step::Rust {
190 idx: _,
191 label,
192 can_merge: _,
193 code: _,
194 } => {
195 println!(
196 "{:width$} - rust - {}",
197 e.node_handle.modpath(),
198 label,
199 width = max_len
200 )
201 }
202 crate::flow_resolver::stage1_dag::Step::AdoYaml {
203 ado_to_rust: _,
204 rust_to_ado: _,
205 label,
206 raw_yaml: _,
207 condvar: _,
208 code_idx: _,
209 code: _,
210 } => println!(
211 "{:width$} - ado - {}",
212 e.node_handle.modpath(),
213 label,
214 width = max_len
215 ),
216 crate::flow_resolver::stage1_dag::Step::GitHubYaml { label, .. } => println!(
217 "{:width$} - github - {}",
218 e.node_handle.modpath(),
219 label,
220 width = max_len
221 ),
222 }
223 }
224
225 Ok(())
226}
227
228pub fn viz_pipeline_dot(pipeline: ResolvedPipeline, _backend: FlowBackend) -> anyhow::Result<()> {
229 let ResolvedPipeline {
230 graph,
231 order: _,
232 parameters: _,
233 ado_name: _,
234 ado_schedule_triggers: _,
235 ado_ci_triggers: _,
236 ado_pr_triggers: _,
237 ado_bootstrap_template: _,
238 ado_resources_repository: _,
239 ado_post_process_yaml_cb: _,
240 ado_variables: _,
241 ado_job_id_overrides: _,
242 gh_name: _,
243 gh_schedule_triggers: _,
244 gh_ci_triggers: _,
245 gh_pr_triggers: _,
246 gh_bootstrap_template: _,
247 } = pipeline;
248
249 #[derive(Clone)]
250 struct VizNode(ResolvedPipelineJob);
251
252 impl From<ResolvedPipelineJob> for VizNode {
253 fn from(value: ResolvedPipelineJob) -> Self {
254 Self(value)
255 }
256 }
257
258 impl std::fmt::Debug for VizNode {
259 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
260 let Self(ResolvedPipelineJob {
261 root_nodes: _,
262 root_configs: _,
263 patches: _,
264 label,
265 platform: _,
266 arch: _,
267 cond_param_idx: _,
268 timeout_minutes: _,
269 command_wrapper: _,
270 ado_pool,
271 ado_variables: _,
272 gh_override_if: _,
273 gh_global_env: _,
274 gh_pool,
275 gh_permissions: _,
276 external_read_vars: _,
277 parameters_used: _,
278 artifacts_used,
279 artifacts_published,
280 }) = self;
281
282 writeln!(
283 f,
284 "== {}{}{} ==",
285 label,
286 ado_pool
287 .as_ref()
288 .map(|s| format!(" - {} ({})", s.name, s.demands.join(",")))
289 .unwrap_or_default(),
290 gh_pool
291 .as_ref()
292 .map(|s| format!(" - {:#?}", s))
293 .unwrap_or_default()
294 )?;
295
296 writeln!(
297 f,
298 "artifacts used: {}",
299 artifacts_used
300 .iter()
301 .map(|a| a.name.clone())
302 .collect::<Vec<_>>()
303 .join(",\n ")
304 )?;
305 writeln!(
306 f,
307 "artifacts published: {}",
308 artifacts_published
309 .iter()
310 .map(|a| a.name.clone())
311 .collect::<Vec<_>>()
312 .join(",\n ")
313 )?;
314
315 Ok(())
316 }
317 }
318
319 println!(
320 "{:?}",
321 petgraph::dot::Dot::with_config(
322 &petgraph_viz_helper::clone_graph_with_wrappers::<_, _, VizNode, ()>(&graph),
323 &[petgraph::dot::Config::EdgeNoLabel]
324 )
325 );
326
327 Ok(())
328}
329
330/// (debug) emit a graph in the graphviz `.dot` format of the flow
331pub fn viz_flow_dot(
332 seed_nodes: BTreeMap<NodeHandle, (bool, Vec<Box<[u8]>>)>,
333 seed_configs: BTreeMap<NodeHandle, Vec<Box<[u8]>>>,
334 resolved_patches: flowey_core::patch::ResolvedPatches,
335 external_read_vars: BTreeSet<String>,
336 backend: FlowBackend,
337 platform: FlowPlatform,
338 arch: FlowArch,
339 with_persist_dir: bool,
340) -> anyhow::Result<()> {
341 // ignore the unreachable nodes error, since we want to allow debugging issues here
342 let crate::flow_resolver::stage1_dag::Stage1DagOutput { output_graph, .. } =
343 crate::flow_resolver::stage1_dag::stage1_dag(
344 backend,
345 platform,
346 arch,
347 resolved_patches,
348 seed_nodes,
349 seed_configs,
350 external_read_vars,
351 with_persist_dir.then_some("<dummy>".into()),
352 )?;
353
354 #[derive(Clone)]
355 struct VizNode((StepId, Option<OutputGraphEntry>));
356
357 impl From<(StepId, Option<OutputGraphEntry>)> for VizNode {
358 fn from(value: (StepId, Option<OutputGraphEntry>)) -> Self {
359 Self(value)
360 }
361 }
362
363 impl std::fmt::Debug for VizNode {
364 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
365 if self.0.1.is_none() {
366 return write!(f, "{:?} - ???", self.0.0);
367 }
368
369 let entry = &self.0.1.as_ref().unwrap();
370
371 write!(
372 f,
373 "{}:{}\n\n{}",
374 self.0.0.step_idx,
375 self.0.0.node.modpath(),
376 match &entry.step {
377 crate::flow_resolver::stage1_dag::Step::Anchor { label } => {
378 format!("<anchor:{label}>")
379 }
380 crate::flow_resolver::stage1_dag::Step::Rust {
381 idx,
382 label,
383 can_merge: _,
384 code: _,
385 } => format!("rust{idx}\n\n{}", label),
386 crate::flow_resolver::stage1_dag::Step::AdoYaml {
387 ado_to_rust: _,
388 rust_to_ado: _,
389 label,
390 raw_yaml: _,
391 condvar: _,
392 code_idx: _,
393 code: _,
394 } => format!("ado\n\n{}", label),
395 crate::flow_resolver::stage1_dag::Step::GitHubYaml { label, .. } => {
396 format!("github\n\n{}", label)
397 }
398 }
399 )
400 }
401 }
402
403 println!(
404 r#"
405digraph {{
406 #rankdir="LR"
407 edge [dir="back"];
408{:?}
409}}
410"#,
411 // petgraph::dot::Dot::with_config(
412 // &petgraph::visit::Reversed(&self::petgraph_viz_helper::clone_graph_with_wrappers::<
413 // _,
414 // _,
415 // VizNode,
416 // DepKind,
417 // >(&output_graph)),
418 // &[petgraph::dot::Config::GraphContentOnly]
419 // ),
420 petgraph::dot::Dot::with_config(
421 &petgraph_viz_helper::clone_graph_with_wrappers::<_, _, VizNode, DepKind>(
422 &output_graph
423 ),
424 &[petgraph::dot::Config::GraphContentOnly]
425 )
426 );
427
428 match petgraph::algo::toposort(&output_graph, None) {
429 Ok(order) => {
430 if order
431 .into_iter()
432 .filter(|idx| {
433 output_graph
434 .edges_directed(*idx, petgraph::Direction::Incoming)
435 .count()
436 == 0
437 })
438 .count()
439 != 1
440 {
441 println!("multiple root nodes detected!")
442 }
443 }
444 Err(_) => {
445 println!("Detected Cycle!")
446 }
447 }
448
449 Ok(())
450}
451
452// pub(crate) so dump_stage0_dag can use it
453pub(crate) mod petgraph_viz_helper {
454 use petgraph::visit::EdgeRef;
455 use std::collections::BTreeMap;
456
457 // thanks bing AI!
458 pub fn clone_graph_with_wrappers<N, E, NWrap, EWrap>(
459 graph: &petgraph::Graph<N, E>,
460 ) -> petgraph::Graph<NWrap, EWrap>
461 where
462 N: Clone,
463 E: Clone,
464 NWrap: From<N>,
465 EWrap: From<E>,
466 {
467 let mut new_graph = petgraph::Graph::new();
468 let node_map: BTreeMap<_, _> = graph
469 .node_indices()
470 .map(|i| (i, new_graph.add_node(NWrap::from(graph[i].clone()))))
471 .collect();
472 for edge in graph.edge_references() {
473 let (a, b) = (node_map[&edge.source()], node_map[&edge.target()]);
474 new_graph.add_edge(a, b, EWrap::from(edge.weight().clone()));
475 }
476 new_graph
477 }
478}