microsoft/hve-core
Publicmirrored from https://github.com/microsoft/hve-coreAvailable
scripts/tests/evals/Invoke-AgentMatrix.Tests.ps1
361lines · modecode
| 1 | #Requires -Modules Pester |
| 2 | # Copyright (c) Microsoft Corporation. |
| 3 | # SPDX-License-Identifier: MIT |
| 4 | |
| 5 | BeforeAll { |
| 6 | $script:ScriptPath = Join-Path $PSScriptRoot '../../evals/Invoke-AgentMatrix.ps1' |
| 7 | $script:RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '../../..') | Select-Object -ExpandProperty Path |
| 8 | $script:InventoryPath = Join-Path $script:RepoRoot 'evals/agent-behavior/AGENTS.yml' |
| 9 | |
| 10 | # Dot-source the script (guarded main is skipped) so the inventory readers are |
| 11 | # available, then derive the expected slug set from the live inventory. Tests |
| 12 | # assert conformance against this set rather than a hard-coded agent count. |
| 13 | . $script:ScriptPath |
| 14 | $script:Inventory = Read-AgentInventory -RepoRoot $script:RepoRoot |
| 15 | $script:InventorySlugs = @($script:Inventory | ForEach-Object { $_['slug'] } | Sort-Object -Unique) |
| 16 | } |
| 17 | |
| 18 | Describe 'Invoke-AgentMatrix.ps1 (dry-run)' -Tag 'Unit' { |
| 19 | |
| 20 | BeforeEach { |
| 21 | $script:OutputDir = Join-Path $TestDrive ("am-" + [Guid]::NewGuid().ToString('N')) |
| 22 | New-Item -ItemType Directory -Path $script:OutputDir -Force | Out-Null |
| 23 | $script:SummaryPath = Join-Path $script:OutputDir 'agent-matrix-summary.json' |
| 24 | } |
| 25 | |
| 26 | Context 'All mode' { |
| 27 | BeforeEach { |
| 28 | & $script:ScriptPath ` |
| 29 | -All ` |
| 30 | -Tier pr ` |
| 31 | -RepoRoot $script:RepoRoot ` |
| 32 | -OutputDir $script:OutputDir ` |
| 33 | -WhatIf *> $null |
| 34 | $script:Summary = Get-Content -LiteralPath $script:SummaryPath -Raw | ConvertFrom-Json |
| 35 | } |
| 36 | |
| 37 | It 'Exits with code 0' { |
| 38 | $LASTEXITCODE | Should -Be 0 |
| 39 | } |
| 40 | |
| 41 | It 'Writes the aggregate summary JSON' { |
| 42 | Test-Path -LiteralPath $script:SummaryPath | Should -BeTrue |
| 43 | } |
| 44 | |
| 45 | It 'Records tier=pr and mode=all' { |
| 46 | $script:Summary.tier | Should -Be 'pr' |
| 47 | $script:Summary.mode | Should -Be 'all' |
| 48 | } |
| 49 | |
| 50 | It 'Reports verdict=dry-run' { |
| 51 | $script:Summary.overall | Should -Be 'dry-run' |
| 52 | } |
| 53 | |
| 54 | It 'Enumerates every inventory agent exactly once (DD-09)' { |
| 55 | $resultSlugs = @($script:Summary.results | ForEach-Object { $_.slug } | Sort-Object -Unique) |
| 56 | $script:Summary.agentCount | Should -Be $script:InventorySlugs.Count |
| 57 | $script:Summary.results.Count | Should -Be $script:InventorySlugs.Count |
| 58 | $script:Summary.plannedCommands.Count | Should -Be $script:InventorySlugs.Count |
| 59 | $resultSlugs | Should -Be $script:InventorySlugs |
| 60 | } |
| 61 | |
| 62 | It 'Records a class and cost_tier for every result row' { |
| 63 | foreach ($row in $script:Summary.results) { |
| 64 | $row.slug | Should -Not -BeNullOrEmpty |
| 65 | $row.class | Should -Not -BeNullOrEmpty |
| 66 | $row.cost_tier | Should -Not -BeNullOrEmpty |
| 67 | $row.overall | Should -Be 'dry-run' |
| 68 | } |
| 69 | } |
| 70 | |
| 71 | It 'Plans a vally command per slug using --eval-spec eval.yaml with an agent tag' { |
| 72 | $first = $script:Summary.plannedCommands[0] |
| 73 | $first | Should -Match '^npx vally eval --eval-spec evals/agent-behavior/eval\.yaml --tag agent=[^ ]+ --model \S+$' |
| 74 | } |
| 75 | } |
| 76 | |
| 77 | Context 'Changed mode with explicit slugs' { |
| 78 | BeforeEach { |
| 79 | & $script:ScriptPath ` |
| 80 | -Changed @('task-researcher', 'task-planner') ` |
| 81 | -Tier pr ` |
| 82 | -RepoRoot $script:RepoRoot ` |
| 83 | -OutputDir $script:OutputDir ` |
| 84 | -WhatIf *> $null |
| 85 | $script:Summary = Get-Content -LiteralPath $script:SummaryPath -Raw | ConvertFrom-Json |
| 86 | } |
| 87 | |
| 88 | It 'Exits with code 0' { |
| 89 | $LASTEXITCODE | Should -Be 0 |
| 90 | } |
| 91 | |
| 92 | It 'Records mode=changed' { |
| 93 | $script:Summary.mode | Should -Be 'changed' |
| 94 | } |
| 95 | |
| 96 | It 'Enumerates only the requested known slugs' { |
| 97 | $script:Summary.agentCount | Should -Be 2 |
| 98 | $slugs = @($script:Summary.results | ForEach-Object { $_.slug }) |
| 99 | $slugs | Should -Contain 'task-researcher' |
| 100 | $slugs | Should -Contain 'task-planner' |
| 101 | } |
| 102 | } |
| 103 | |
| 104 | Context 'Changed mode with no slugs' { |
| 105 | BeforeEach { |
| 106 | & $script:ScriptPath ` |
| 107 | -Changed @() ` |
| 108 | -Tier pr ` |
| 109 | -RepoRoot $script:RepoRoot ` |
| 110 | -OutputDir $script:OutputDir ` |
| 111 | -WhatIf *> $null |
| 112 | $script:Summary = Get-Content -LiteralPath $script:SummaryPath -Raw | ConvertFrom-Json |
| 113 | } |
| 114 | |
| 115 | It 'Exits with code 0' { |
| 116 | $LASTEXITCODE | Should -Be 0 |
| 117 | } |
| 118 | |
| 119 | It 'Writes an empty summary' { |
| 120 | $script:Summary.agentCount | Should -Be 0 |
| 121 | $script:Summary.results.Count | Should -Be 0 |
| 122 | } |
| 123 | } |
| 124 | |
| 125 | Context 'Nightly tier metadata' { |
| 126 | BeforeEach { |
| 127 | & $script:ScriptPath ` |
| 128 | -All ` |
| 129 | -Tier nightly ` |
| 130 | -RepoRoot $script:RepoRoot ` |
| 131 | -OutputDir $script:OutputDir ` |
| 132 | -WhatIf *> $null |
| 133 | $script:Summary = Get-Content -LiteralPath $script:SummaryPath -Raw | ConvertFrom-Json |
| 134 | } |
| 135 | |
| 136 | It 'Records tier=nightly' { |
| 137 | $script:Summary.tier | Should -Be 'nightly' |
| 138 | } |
| 139 | |
| 140 | It 'Exits 0 in dry-run even at nightly tier' { |
| 141 | $LASTEXITCODE | Should -Be 0 |
| 142 | } |
| 143 | } |
| 144 | |
| 145 | Context 'Parameter validation' { |
| 146 | It 'Rejects an unknown tier' { |
| 147 | { & $script:ScriptPath -All -Tier 'weekly' -RepoRoot $script:RepoRoot -OutputDir $script:OutputDir -WhatIf } | |
| 148 | Should -Throw |
| 149 | } |
| 150 | |
| 151 | It 'Rejects combining -All and -Changed' { |
| 152 | { & $script:ScriptPath -All -Changed @('task-researcher') -RepoRoot $script:RepoRoot -OutputDir $script:OutputDir -WhatIf } | |
| 153 | Should -Throw |
| 154 | } |
| 155 | } |
| 156 | } |
| 157 | |
| 158 | Describe 'Invoke-AgentMatrix helper functions' -Tag 'Unit' { |
| 159 | |
| 160 | BeforeAll { |
| 161 | . $script:ScriptPath |
| 162 | } |
| 163 | |
| 164 | Context 'Get-GraderStatusesFromLog' { |
| 165 | It 'Parses pass/fail grader lines' { |
| 166 | $lines = @( |
| 167 | 'grader "header-present": pass', |
| 168 | 'grader "scope-adherence": fail', |
| 169 | 'grader "no-source-edit": pass' |
| 170 | ) |
| 171 | $result = @(Get-GraderStatusesFromLog -Lines $lines) |
| 172 | $result.Count | Should -Be 3 |
| 173 | ($result | Where-Object { $_['name'] -eq 'header-present' }).status | Should -Be 'pass' |
| 174 | ($result | Where-Object { $_['name'] -eq 'scope-adherence' }).status | Should -Be 'fail' |
| 175 | } |
| 176 | |
| 177 | It 'Deduplicates repeated grader names' { |
| 178 | $lines = @( |
| 179 | 'grader "header-present": pass', |
| 180 | 'grader "header-present": fail' |
| 181 | ) |
| 182 | $result = @(Get-GraderStatusesFromLog -Lines $lines) |
| 183 | $result.Count | Should -Be 1 |
| 184 | $result[0]['status'] | Should -Be 'pass' |
| 185 | } |
| 186 | |
| 187 | It 'Returns an empty collection on empty input' { |
| 188 | $result = @(Get-GraderStatusesFromLog -Lines @()) |
| 189 | $result.Count | Should -Be 0 |
| 190 | } |
| 191 | |
| 192 | It 'Ignores lines that do not match the grader pattern' { |
| 193 | $result = @(Get-GraderStatusesFromLog -Lines @('random log line', 'no grader here')) |
| 194 | $result.Count | Should -Be 0 |
| 195 | } |
| 196 | } |
| 197 | |
| 198 | Context 'New-AgentSummary' { |
| 199 | BeforeEach { |
| 200 | $script:Entry = @{ slug = 'task-researcher'; class = 'research-writer'; cost_tier = 'light' } |
| 201 | $script:Graders = [System.Collections.Generic.List[hashtable]]::new() |
| 202 | $script:Graders.Add(@{ name = 'header-present'; status = 'pass' }) |
| 203 | } |
| 204 | |
| 205 | It 'Reports overall=pass when ExitCode=0 and no failing graders' { |
| 206 | $summary = New-AgentSummary -AgentEntry $script:Entry -ExitCode 0 -Graders $script:Graders -LogPath 'x.log' |
| 207 | $summary.overall | Should -Be 'pass' |
| 208 | $summary.slug | Should -Be 'task-researcher' |
| 209 | $summary.class | Should -Be 'research-writer' |
| 210 | $summary.cost_tier | Should -Be 'light' |
| 211 | $summary.logPath | Should -Be 'x.log' |
| 212 | $summary.exitCode | Should -Be 0 |
| 213 | } |
| 214 | |
| 215 | It 'Reports overall=fail when ExitCode is non-zero' { |
| 216 | $summary = New-AgentSummary -AgentEntry $script:Entry -ExitCode 2 -Graders $script:Graders -LogPath 'x.log' |
| 217 | $summary.overall | Should -Be 'fail' |
| 218 | $summary.exitCode | Should -Be 2 |
| 219 | } |
| 220 | |
| 221 | It 'Reports overall=fail when a grader status is fail even with exit 0' { |
| 222 | $script:Graders.Add(@{ name = 'scope'; status = 'fail' }) |
| 223 | $summary = New-AgentSummary -AgentEntry $script:Entry -ExitCode 0 -Graders $script:Graders -LogPath 'x.log' |
| 224 | $summary.overall | Should -Be 'fail' |
| 225 | } |
| 226 | } |
| 227 | |
| 228 | Context 'New-MatrixSummary' { |
| 229 | It 'Collects failure slugs and sets overall=fail' { |
| 230 | $results = [System.Collections.Generic.List[hashtable]]::new() |
| 231 | $results.Add(@{ slug = 'a'; overall = 'pass' }) |
| 232 | $results.Add(@{ slug = 'b'; overall = 'fail' }) |
| 233 | $summary = New-MatrixSummary -Tier 'nightly' -Mode 'all' -Results $results -PlannedCommands @('cmd-a','cmd-b') |
| 234 | $summary.overall | Should -Be 'fail' |
| 235 | $summary.failures | Should -Contain 'b' |
| 236 | $summary.agentCount | Should -Be 2 |
| 237 | $summary.tier | Should -Be 'nightly' |
| 238 | $summary.mode | Should -Be 'all' |
| 239 | $summary.plannedCommands.Count | Should -Be 2 |
| 240 | } |
| 241 | |
| 242 | It 'Sets overall=pass when all results pass' { |
| 243 | $results = [System.Collections.Generic.List[hashtable]]::new() |
| 244 | $results.Add(@{ slug = 'a'; overall = 'pass' }) |
| 245 | $results.Add(@{ slug = 'b'; overall = 'pass' }) |
| 246 | $summary = New-MatrixSummary -Tier 'pr' -Mode 'changed' -Results $results -PlannedCommands @() |
| 247 | $summary.overall | Should -Be 'pass' |
| 248 | $summary.failures.Count | Should -Be 0 |
| 249 | } |
| 250 | |
| 251 | It 'Honors an explicit verdict override' { |
| 252 | $results = [System.Collections.Generic.List[hashtable]]::new() |
| 253 | $summary = New-MatrixSummary -Tier 'pr' -Mode 'all' -Results $results -PlannedCommands @() -Verdict 'dry-run' |
| 254 | $summary.overall | Should -Be 'dry-run' |
| 255 | } |
| 256 | } |
| 257 | |
| 258 | Context 'Resolve-SlugSet' { |
| 259 | BeforeAll { |
| 260 | $script:Inventory = Read-AgentInventory -RepoRoot $script:RepoRoot |
| 261 | } |
| 262 | |
| 263 | It 'Returns every inventory slug in All mode' { |
| 264 | $slugs = Resolve-SlugSet -RepoRoot $script:RepoRoot -Inventory $script:Inventory -ParameterSet 'All' |
| 265 | $expected = @($script:Inventory | ForEach-Object { $_['slug'] } | Sort-Object -Unique) |
| 266 | $slugs.Count | Should -Be $script:Inventory.Count |
| 267 | $slugs | Should -Be $expected |
| 268 | } |
| 269 | |
| 270 | It 'Filters Changed inputs to known slugs' { |
| 271 | $slugs = Resolve-SlugSet -RepoRoot $script:RepoRoot -Inventory $script:Inventory -ParameterSet 'Changed' -Changed @('task-researcher', 'definitely-not-an-agent') |
| 272 | $slugs | Should -Contain 'task-researcher' |
| 273 | $slugs | Should -Not -Contain 'definitely-not-an-agent' |
| 274 | } |
| 275 | |
| 276 | It 'Returns an empty array when Changed is empty' { |
| 277 | $slugs = Resolve-SlugSet -RepoRoot $script:RepoRoot -Inventory $script:Inventory -ParameterSet 'Changed' -Changed @() |
| 278 | $slugs.Count | Should -Be 0 |
| 279 | } |
| 280 | } |
| 281 | |
| 282 | Context 'Get-GraderStatusesFromLog pattern extraction' { |
| 283 | It 'Extracts pattern from positive-match glyph line ("matches pattern ...")' { |
| 284 | $checkGlyph = [string][char]0x2714 |
| 285 | $lines = @( |
| 286 | 'Graders (1/1)', |
| 287 | " $checkGlyph field-vocab-present Output matches pattern /(?i)(title|description)/", |
| 288 | '' |
| 289 | ) |
| 290 | $result = @(Get-GraderStatusesFromLog -Lines $lines) |
| 291 | $result.Count | Should -Be 1 |
| 292 | $result[0]['name'] | Should -Be 'field-vocab-present' |
| 293 | $result[0]['status'] | Should -Be 'pass' |
| 294 | $result[0]['pattern'] | Should -Be '/(?i)(title|description)/' |
| 295 | } |
| 296 | |
| 297 | It 'Extracts pattern from negative-match glyph line ("does not match pattern ...")' { |
| 298 | $crossGlyph = [string][char]0x2718 |
| 299 | $lines = @( |
| 300 | 'Graders (0/1)', |
| 301 | " $crossGlyph tracking-file-write Output does not match pattern /(?i)\.copilot-tracking/workitems/", |
| 302 | '' |
| 303 | ) |
| 304 | $result = @(Get-GraderStatusesFromLog -Lines $lines) |
| 305 | $result.Count | Should -Be 1 |
| 306 | $result[0]['name'] | Should -Be 'tracking-file-write' |
| 307 | $result[0]['status'] | Should -Be 'fail' |
| 308 | $result[0]['pattern'] | Should -Be '/(?i)\.copilot-tracking/workitems/' |
| 309 | } |
| 310 | } |
| 311 | |
| 312 | Context 'Merge-GraderDetails' { |
| 313 | It 'Preserves log message when rich grader provides only evidence' { |
| 314 | $logGrader = @{ |
| 315 | name = 'field-vocab-present' |
| 316 | status = 'pass' |
| 317 | message = 'Output matches pattern /(?i)(title)/' |
| 318 | pattern = '/(?i)(title)/' |
| 319 | } |
| 320 | $logList = [System.Collections.Generic.List[hashtable]]::new() |
| 321 | $logList.Add($logGrader) |
| 322 | $richGrader = @{ |
| 323 | name = 'field-vocab-present' |
| 324 | status = 'pass' |
| 325 | evidence = 'Output matches pattern /(?i)(title)/' |
| 326 | pattern = '/(?i)(title)/' |
| 327 | label = 'vocab' |
| 328 | kind = 'regex' |
| 329 | } |
| 330 | $merged = @(Merge-GraderDetails -LogGraders $logList -RichGraders @($richGrader)) |
| 331 | $merged.Count | Should -Be 1 |
| 332 | $merged[0]['message'] | Should -Be 'Output matches pattern /(?i)(title)/' |
| 333 | $merged[0]['pattern'] | Should -Be '/(?i)(title)/' |
| 334 | $merged[0]['evidence'] | Should -Be 'Output matches pattern /(?i)(title)/' |
| 335 | $merged[0]['label'] | Should -Be 'vocab' |
| 336 | $merged[0]['kind'] | Should -Be 'regex' |
| 337 | } |
| 338 | |
| 339 | It 'Backfills pattern from rich grader when log pattern is empty' { |
| 340 | $logGrader = @{ |
| 341 | name = 'no-source-edit' |
| 342 | status = 'pass' |
| 343 | message = 'Output does not match pattern /\.cs/' |
| 344 | pattern = '' |
| 345 | } |
| 346 | $logList = [System.Collections.Generic.List[hashtable]]::new() |
| 347 | $logList.Add($logGrader) |
| 348 | $richGrader = @{ |
| 349 | name = 'no-source-edit' |
| 350 | status = 'pass' |
| 351 | evidence = 'Output does not match pattern /\.cs/' |
| 352 | pattern = '/\.cs/' |
| 353 | label = '' |
| 354 | kind = '' |
| 355 | } |
| 356 | $merged = @(Merge-GraderDetails -LogGraders $logList -RichGraders @($richGrader)) |
| 357 | $merged[0]['pattern'] | Should -Be '/\.cs/' |
| 358 | $merged[0]['message'] | Should -Be 'Output does not match pattern /\.cs/' |
| 359 | } |
| 360 | } |
| 361 | } |
| 362 | |