GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/hve-core

Public

mirrored fromhttps://github.com/microsoft/hve-coreAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
feat/1637-d-skill-paths

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

hve-core/scripts/evals

scripts/evals/Build-AgentInventory.ps1

262lines · modecode

RawDownload
Latest commit unavailable.
1#!/usr/bin/env pwsh
2# Copyright (c) Microsoft Corporation.
3# SPDX-License-Identifier: MIT
4#Requires -Version 7.0
5
6<#
7.SYNOPSIS
8 Generate the authoritative inventory of parent agents at evals/agent-behavior/AGENTS.yml.
9
10.DESCRIPTION
11 Scans `.github/agents/**/*.agent.md` and emits a deterministic YAML inventory of all
12 parent agents enrolled in the per-agent eval-behavior matrix. The inventory becomes the
13 single source of truth shared by `Build-AgentBehaviorSpec.ps1`, `Invoke-VallyEvals.ps1`,
14 `Test-AgentBehaviorCoverage.ps1`, and the dashboard.
15
16 Discovery rule:
17 1. Enumerate every `.agent.md` file under `.github/agents/`.
18 2. Drop any file whose YAML frontmatter sets `user-invocable: false`. This is the
19 canonical parent/subagent boundary marker; the `subagents/` folder convention is
20 informational only and is not consulted.
21 3. Files with no `user-invocable` key are treated as parent agents.
22
23 Frontmatter fields read per agent:
24 * `eval-class:` (Phase 2.2 populates) -> class slug; defaults to `unknown` when absent.
25 * `cost_tier:` (Phase 2.2 populates) -> light|medium|heavy; defaults to `light`.
26
27 Output shape (sorted by slug for determinism):
28 generated_at: <ISO-8601 UTC>
29 generator: scripts/evals/Build-AgentInventory.ps1
30 agents:
31 - slug: <slug>
32 path: <workspace-relative path>
33 class: <eval-class or unknown>
34 cost_tier: <light|medium|heavy>
35
36.PARAMETER RepoRoot
37 Repository root. Defaults to `git rev-parse --show-toplevel`.
38
39.PARAMETER OutputPath
40 YAML output path. Defaults to `<RepoRoot>/evals/agent-behavior/AGENTS.yml`.
41
42.PARAMETER Force
43 Overwrite an existing inventory file even when content matches.
44
45.PARAMETER GeneratedAt
46 Optional fixed ISO-8601 UTC timestamp for deterministic test fixtures.
47
48.EXAMPLE
49 pwsh scripts/evals/Build-AgentInventory.ps1
50 Regenerate the inventory in-place.
51
52.EXAMPLE
53 pwsh scripts/evals/Build-AgentInventory.ps1 -WhatIf
54 Report drift between the current inventory and what would be generated.
55#>
56[CmdletBinding(SupportsShouldProcess)]
57[OutputType([string])]
58param(
59 [string]$RepoRoot,
60 [string]$OutputPath,
61 [switch]$Force,
62 [string]$GeneratedAt
63)
64
65Set-StrictMode -Version Latest
66$ErrorActionPreference = 'Stop'
67
68function Import-YamlModule {
69 [CmdletBinding()]
70 param()
71
72 if (Get-Module -Name 'powershell-yaml') { return }
73 if (-not (Get-Module -ListAvailable -Name 'powershell-yaml')) {
74 throw "Required module 'powershell-yaml' is not installed. Run 'Install-Module powershell-yaml -Scope CurrentUser' before invoking this script."
75 }
76 Import-Module powershell-yaml -ErrorAction Stop | Out-Null
77}
78
79function Resolve-RepoRoot {
80 [CmdletBinding()]
81 [OutputType([string])]
82 param([string]$Override)
83
84 if ($Override) { return (Resolve-Path -LiteralPath $Override).Path }
85 try {
86 $root = (& git rev-parse --show-toplevel 2>$null).Trim()
87 if ($LASTEXITCODE -eq 0 -and $root) { return $root }
88 } catch {
89 Write-Verbose "git rev-parse failed: $($_.Exception.Message)"
90 }
91 return (Get-Location).Path
92}
93
94function ConvertTo-RelativePath {
95 [CmdletBinding()]
96 [OutputType([string])]
97 param(
98 [Parameter(Mandatory)] [string]$RepoRoot,
99 [Parameter(Mandatory)] [string]$Path
100 )
101
102 $rootFull = [System.IO.Path]::GetFullPath($RepoRoot)
103 $pathFull = [System.IO.Path]::GetFullPath($Path)
104 if ($pathFull.StartsWith($rootFull, [System.StringComparison]::OrdinalIgnoreCase)) {
105 $rel = $pathFull.Substring($rootFull.Length).TrimStart([char]'\', [char]'/')
106 return ($rel -replace '\\', '/')
107 }
108 return ($Path -replace '\\', '/')
109}
110
111function Read-AgentFrontmatter {
112 [CmdletBinding()]
113 [OutputType([hashtable])]
114 param([Parameter(Mandatory)] [string]$Path)
115
116 $raw = [System.IO.File]::ReadAllText($Path)
117 if ($raw -notmatch '(?s)^---\s*\r?\n(.*?)\r?\n---\s*(?:\r?\n|$)') {
118 return @{}
119 }
120
121 $yamlBlock = $matches[1]
122 try {
123 $parsed = ConvertFrom-Yaml -Yaml $yamlBlock
124 } catch {
125 throw "Failed to parse YAML frontmatter in '$Path': $($_.Exception.Message)"
126 }
127
128 $result = @{}
129 if ($parsed -is [System.Collections.IDictionary]) {
130 foreach ($key in $parsed.Keys) {
131 $result[[string]$key] = $parsed[$key]
132 }
133 }
134 return $result
135}
136
137function Get-AgentSlug {
138 [CmdletBinding()]
139 [OutputType([string])]
140 param([Parameter(Mandatory)] [string]$RelativePath)
141 return [System.IO.Path]::GetFileName($RelativePath) -replace '\.agent\.md$', ''
142}
143
144function Test-IsParentAgent {
145 [CmdletBinding()]
146 [OutputType([bool])]
147 param([Parameter(Mandatory)] [hashtable]$Frontmatter)
148
149 if (-not $Frontmatter.ContainsKey('user-invocable')) { return $true }
150 $value = $Frontmatter['user-invocable']
151 if ($value -is [bool]) { return $value }
152 # Defensive fallback: tolerate string forms that some authors may write.
153 if ($value -is [string]) { return ($value.Trim().ToLowerInvariant() -ne 'false') }
154 return $true
155}
156
157function Get-ParentAgentInventory {
158 [CmdletBinding()]
159 [OutputType([System.Collections.Generic.List[hashtable]])]
160 param([Parameter(Mandatory)] [string]$RepoRoot)
161
162 $agentsDir = Join-Path $RepoRoot '.github/agents'
163 if (-not (Test-Path -LiteralPath $agentsDir -PathType Container)) {
164 throw "Agents directory not found at '$agentsDir'."
165 }
166
167 $entries = [System.Collections.Generic.List[hashtable]]::new()
168 $files = @(Get-ChildItem -Path $agentsDir -Recurse -Filter '*.agent.md' -File -ErrorAction Stop)
169
170 foreach ($file in $files) {
171 $rel = ConvertTo-RelativePath -RepoRoot $RepoRoot -Path $file.FullName
172 $fm = Read-AgentFrontmatter -Path $file.FullName
173 if (-not (Test-IsParentAgent -Frontmatter $fm)) { continue }
174
175 $entries.Add([ordered]@{
176 slug = Get-AgentSlug -RelativePath $rel
177 path = $rel
178 class = if ($fm.ContainsKey('eval-class') -and $fm['eval-class']) { [string]$fm['eval-class'] } else { 'unknown' }
179 cost_tier = if ($fm.ContainsKey('cost_tier') -and $fm['cost_tier']) { [string]$fm['cost_tier'] } else { 'light' }
180 })
181 }
182
183 return [System.Collections.Generic.List[hashtable]]($entries | Sort-Object -Property { $_.slug })
184}
185
186function ConvertTo-YamlSingleQuoted {
187 [CmdletBinding()]
188 [OutputType([string])]
189 param([Parameter(Mandatory)] [string]$Value)
190 return "'" + ($Value -replace "'", "''") + "'"
191}
192
193function Format-InventoryYaml {
194 [CmdletBinding()]
195 [OutputType([string])]
196 param(
197 [Parameter(Mandatory)] [string]$GeneratedAt,
198 [Parameter(Mandatory)] [System.Collections.Generic.List[hashtable]]$Agents
199 )
200
201 $sb = [System.Text.StringBuilder]::new()
202 [void]$sb.AppendLine('# Generated by scripts/evals/Build-AgentInventory.ps1 - re-run with -Force to regenerate.')
203 [void]$sb.AppendLine('# Source of truth for the per-agent eval-behavior matrix.')
204 [void]$sb.AppendLine("generated_at: $GeneratedAt")
205 [void]$sb.AppendLine("generator: 'scripts/evals/Build-AgentInventory.ps1'")
206 [void]$sb.AppendLine('agents:')
207 foreach ($entry in $Agents) {
208 [void]$sb.AppendLine(" - slug: $($entry.slug)")
209 [void]$sb.AppendLine(" path: $(ConvertTo-YamlSingleQuoted -Value $entry.path)")
210 [void]$sb.AppendLine(" class: $($entry.class)")
211 [void]$sb.AppendLine(" cost_tier: $($entry.cost_tier)")
212 }
213 return $sb.ToString()
214}
215
216# --- Main ---
217Import-YamlModule
218
219$resolvedRoot = Resolve-RepoRoot -Override $RepoRoot
220if (-not $OutputPath) {
221 $OutputPath = Join-Path $resolvedRoot 'evals/agent-behavior/AGENTS.yml'
222}
223
224if (-not $GeneratedAt) {
225 $GeneratedAt = (Get-Date).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ")
226}
227
228$agents = Get-ParentAgentInventory -RepoRoot $resolvedRoot
229$rendered = Format-InventoryYaml -GeneratedAt $GeneratedAt -Agents $agents
230
231$outputDir = Split-Path -Parent $OutputPath
232if (-not (Test-Path -LiteralPath $outputDir -PathType Container)) {
233 if ($PSCmdlet.ShouldProcess($outputDir, 'Create directory')) {
234 New-Item -ItemType Directory -Path $outputDir -Force | Out-Null
235 }
236}
237
238$drift = $true
239if (Test-Path -LiteralPath $OutputPath -PathType Leaf) {
240 $existing = [System.IO.File]::ReadAllText($OutputPath)
241 # Compare ignoring the generated_at line (always changes when not pinned).
242 $existingNormalized = ($existing -split "`r?`n" | Where-Object { $_ -notmatch '^generated_at:' }) -join "`n"
243 $renderedNormalized = ($rendered -split "`r?`n" | Where-Object { $_ -notmatch '^generated_at:' }) -join "`n"
244 if ($existingNormalized -eq $renderedNormalized) { $drift = $false }
245}
246
247if ($PSCmdlet.ShouldProcess($OutputPath, 'Write agent inventory YAML')) {
248 if (-not $drift -and -not $Force) {
249 Write-Host "skipped (no drift): $OutputPath"
250 return $OutputPath
251 }
252 [System.IO.File]::WriteAllText($OutputPath, $rendered)
253 Write-Host "wrote: $OutputPath ($($agents.Count) agents)"
254} else {
255 if ($drift) {
256 Write-Host "drift detected: $OutputPath would change ($($agents.Count) agents)"
257 } else {
258 Write-Host "no drift: $OutputPath ($($agents.Count) agents)"
259 }
260}
261
262return $OutputPath
263