microsoft/hve-core
Publicmirrored from https://github.com/microsoft/hve-coreAvailable
scripts/tests/evals/Build-AgentInventory.Tests.ps1
146lines · modecode
| 1 | #Requires -Modules Pester |
| 2 | # Copyright (c) Microsoft Corporation. |
| 3 | # SPDX-License-Identifier: MIT |
| 4 | |
| 5 | BeforeAll { |
| 6 | $script:ScriptPath = (Resolve-Path (Join-Path $PSScriptRoot '../../evals/Build-AgentInventory.ps1')).Path |
| 7 | |
| 8 | function script:New-AgentFile { |
| 9 | param( |
| 10 | [Parameter(Mandatory)] [string]$Root, |
| 11 | [Parameter(Mandatory)] [string]$RelativePath, |
| 12 | [hashtable]$Frontmatter = @{} |
| 13 | ) |
| 14 | |
| 15 | $full = Join-Path $Root $RelativePath |
| 16 | $dir = Split-Path -Parent $full |
| 17 | if (-not (Test-Path -LiteralPath $dir -PathType Container)) { |
| 18 | New-Item -ItemType Directory -Path $dir -Force | Out-Null |
| 19 | } |
| 20 | $lines = @('---') |
| 21 | $lines += "name: $([System.IO.Path]::GetFileName($RelativePath) -replace '\.agent\.md$','')" |
| 22 | $lines += "description: Fixture agent for Build-AgentInventory tests." |
| 23 | foreach ($key in $Frontmatter.Keys) { $lines += "${key}: $($Frontmatter[$key])" } |
| 24 | $lines += '---' |
| 25 | $lines += '' |
| 26 | $lines += '# Body' |
| 27 | Set-Content -LiteralPath $full -Value ($lines -join "`n") -Encoding UTF8 |
| 28 | } |
| 29 | |
| 30 | function script:New-MinimalRepo { |
| 31 | param([Parameter(Mandatory)] [string]$Root) |
| 32 | |
| 33 | # 2 parent agents in standard locations |
| 34 | New-AgentFile -Root $Root -RelativePath '.github/agents/hve-core/task-planner.agent.md' -Frontmatter @{ |
| 35 | 'eval-class' = 'code-author' |
| 36 | 'cost_tier' = 'medium' |
| 37 | } |
| 38 | New-AgentFile -Root $Root -RelativePath '.github/agents/ado/ado-backlog-manager.agent.md' |
| 39 | |
| 40 | # Subagents marked with user-invocable: false MUST be excluded regardless of path. |
| 41 | New-AgentFile -Root $Root -RelativePath '.github/agents/hve-core/subagents/researcher-subagent.agent.md' -Frontmatter @{ |
| 42 | 'user-invocable' = 'false' |
| 43 | } |
| 44 | New-AgentFile -Root $Root -RelativePath '.github/agents/security/subagents/codebase-profiler.agent.md' -Frontmatter @{ |
| 45 | 'user-invocable' = 'false' |
| 46 | } |
| 47 | New-AgentFile -Root $Root -RelativePath '.github/agents/security/subagents/finding-deep-verifier.agent.md' -Frontmatter @{ |
| 48 | 'user-invocable' = 'false' |
| 49 | } |
| 50 | New-AgentFile -Root $Root -RelativePath '.github/agents/security/subagents/report-generator.agent.md' -Frontmatter @{ |
| 51 | 'user-invocable' = 'false' |
| 52 | } |
| 53 | New-AgentFile -Root $Root -RelativePath '.github/agents/security/subagents/skill-assessor.agent.md' -Frontmatter @{ |
| 54 | 'user-invocable' = 'false' |
| 55 | } |
| 56 | } |
| 57 | } |
| 58 | |
| 59 | Describe 'Build-AgentInventory.ps1' -Tag 'Unit' { |
| 60 | BeforeEach { |
| 61 | $script:TestRoot = Join-Path $TestDrive ([Guid]::NewGuid().ToString()) |
| 62 | New-Item -ItemType Directory -Path $script:TestRoot -Force | Out-Null |
| 63 | New-MinimalRepo -Root $script:TestRoot |
| 64 | $script:OutputPath = Join-Path $script:TestRoot 'evals/agent-behavior/AGENTS.yml' |
| 65 | } |
| 66 | |
| 67 | Context 'Discovery and frontmatter' { |
| 68 | BeforeEach { |
| 69 | & $script:ScriptPath -RepoRoot $script:TestRoot -OutputPath $script:OutputPath -GeneratedAt '2026-05-25T00:00:00Z' 6>$null | Out-Null |
| 70 | $script:Yaml = [System.IO.File]::ReadAllText($script:OutputPath) |
| 71 | } |
| 72 | |
| 73 | It 'Writes the inventory to the requested OutputPath' { |
| 74 | Test-Path -LiteralPath $script:OutputPath | Should -BeTrue |
| 75 | } |
| 76 | |
| 77 | It 'Emits the generator banner and pinned timestamp' { |
| 78 | $script:Yaml | Should -Match '# Generated by scripts/evals/Build-AgentInventory\.ps1' |
| 79 | $script:Yaml | Should -Match '(?m)^generated_at: 2026-05-25T00:00:00Z' |
| 80 | $script:Yaml | Should -Match "(?m)^generator: 'scripts/evals/Build-AgentInventory\.ps1'" |
| 81 | } |
| 82 | |
| 83 | It 'Includes both standard parent agents' { |
| 84 | $script:Yaml | Should -Match '(?m)^\s+- slug: task-planner\s*$' |
| 85 | $script:Yaml | Should -Match '(?m)^\s+- slug: ado-backlog-manager\s*$' |
| 86 | } |
| 87 | |
| 88 | It 'Excludes every agent with user-invocable: false regardless of path' { |
| 89 | $script:Yaml | Should -Not -Match '(?m)^\s+- slug: researcher-subagent\s*$' |
| 90 | $script:Yaml | Should -Not -Match '(?m)^\s+- slug: codebase-profiler\s*$' |
| 91 | $script:Yaml | Should -Not -Match '(?m)^\s+- slug: finding-deep-verifier\s*$' |
| 92 | $script:Yaml | Should -Not -Match '(?m)^\s+- slug: report-generator\s*$' |
| 93 | $script:Yaml | Should -Not -Match '(?m)^\s+- slug: skill-assessor\s*$' |
| 94 | } |
| 95 | |
| 96 | It 'Renders frontmatter eval-class and cost_tier when present' { |
| 97 | $script:Yaml | Should -Match "(?ms)^\s+- slug: task-planner\s*\n\s+path: '\.github/agents/hve-core/task-planner\.agent\.md'\s*\n\s+class: code-author\s*\n\s+cost_tier: medium" |
| 98 | } |
| 99 | |
| 100 | It 'Defaults class to unknown and cost_tier to light when frontmatter is silent' { |
| 101 | $script:Yaml | Should -Match "(?ms)^\s+- slug: ado-backlog-manager\s*\n\s+path: '\.github/agents/ado/ado-backlog-manager\.agent\.md'\s*\n\s+class: unknown\s*\n\s+cost_tier: light" |
| 102 | } |
| 103 | |
| 104 | It 'Sorts entries by slug for deterministic diffs' { |
| 105 | $slugs = [regex]::Matches($script:Yaml, '(?m)^\s+- slug:\s+(\S+)\s*$') | ForEach-Object { $_.Groups[1].Value } |
| 106 | $slugs | Should -Be ($slugs | Sort-Object) |
| 107 | } |
| 108 | |
| 109 | It 'Counts exactly the parent agents (those without user-invocable: false)' { |
| 110 | $count = ([regex]::Matches($script:Yaml, '(?m)^\s+- slug:\s+')).Count |
| 111 | $count | Should -Be 2 |
| 112 | } |
| 113 | } |
| 114 | |
| 115 | Context 'Drift detection' { |
| 116 | It 'Skips overwriting when the inventory is already up to date' { |
| 117 | & $script:ScriptPath -RepoRoot $script:TestRoot -OutputPath $script:OutputPath -GeneratedAt '2026-05-25T00:00:00Z' 6>$null | Out-Null |
| 118 | $firstWrite = (Get-Item -LiteralPath $script:OutputPath).LastWriteTimeUtc |
| 119 | Start-Sleep -Milliseconds 50 |
| 120 | & $script:ScriptPath -RepoRoot $script:TestRoot -OutputPath $script:OutputPath -GeneratedAt '2099-01-01T00:00:00Z' 6>$null | Out-Null |
| 121 | (Get-Item -LiteralPath $script:OutputPath).LastWriteTimeUtc | Should -Be $firstWrite |
| 122 | } |
| 123 | |
| 124 | It 'Overwrites when -Force is supplied even without content drift' { |
| 125 | & $script:ScriptPath -RepoRoot $script:TestRoot -OutputPath $script:OutputPath -GeneratedAt '2026-05-25T00:00:00Z' 6>$null | Out-Null |
| 126 | $firstWrite = (Get-Item -LiteralPath $script:OutputPath).LastWriteTimeUtc |
| 127 | Start-Sleep -Milliseconds 50 |
| 128 | & $script:ScriptPath -RepoRoot $script:TestRoot -OutputPath $script:OutputPath -GeneratedAt '2026-05-25T00:00:00Z' -Force 6>$null | Out-Null |
| 129 | (Get-Item -LiteralPath $script:OutputPath).LastWriteTimeUtc | Should -BeGreaterThan $firstWrite |
| 130 | } |
| 131 | |
| 132 | It 'Rewrites when frontmatter content changes' { |
| 133 | & $script:ScriptPath -RepoRoot $script:TestRoot -OutputPath $script:OutputPath -GeneratedAt '2026-05-25T00:00:00Z' 6>$null | Out-Null |
| 134 | $before = [System.IO.File]::ReadAllText($script:OutputPath) |
| 135 | New-AgentFile -Root $script:TestRoot -RelativePath '.github/agents/ado/ado-backlog-manager.agent.md' -Frontmatter @{ |
| 136 | 'eval-class' = 'workflow-router' |
| 137 | 'cost_tier' = 'heavy' |
| 138 | } |
| 139 | & $script:ScriptPath -RepoRoot $script:TestRoot -OutputPath $script:OutputPath -GeneratedAt '2026-05-25T00:00:01Z' 6>$null | Out-Null |
| 140 | $after = [System.IO.File]::ReadAllText($script:OutputPath) |
| 141 | $after | Should -Not -Be $before |
| 142 | $after | Should -Match 'class: workflow-router' |
| 143 | $after | Should -Match 'cost_tier: heavy' |
| 144 | } |
| 145 | } |
| 146 | } |
| 147 | |