microsoft/hve-core
Publicmirrored from https://github.com/microsoft/hve-coreAvailable
scripts/tests/evals/_AgentMatrixFixtures.psm1
124lines · modecode
| 1 | # Copyright (c) Microsoft Corporation. |
| 2 | # SPDX-License-Identifier: MIT |
| 3 | |
| 4 | # _AgentMatrixFixtures.psm1 |
| 5 | # |
| 6 | # Purpose: Shared fixture builders and regex helpers for |
| 7 | # `New-AgentMatrixDashboard.ps1` Pester tests. Leading underscore keeps the |
| 8 | # filename out of Pester's default `*.Tests.ps1` discovery. |
| 9 | # |
| 10 | |
| 11 | function New-FixtureRoot { |
| 12 | [CmdletBinding()] |
| 13 | [OutputType([pscustomobject])] |
| 14 | param([Parameter(Mandatory)] [string]$Base) |
| 15 | |
| 16 | $root = Join-Path $Base ("amd-" + [Guid]::NewGuid().ToString('N')) |
| 17 | $matrixRoot = Join-Path $root 'evals/results/agent-matrix' |
| 18 | $surfaceRoot = Join-Path $root 'evals/baseline-equivalence/surface-signatures' |
| 19 | $inventoryPath = Join-Path $root 'evals/agent-behavior/AGENTS.yml' |
| 20 | New-Item -ItemType Directory -Path $matrixRoot, $surfaceRoot -Force | Out-Null |
| 21 | New-Item -ItemType Directory -Path (Split-Path -Parent $inventoryPath) -Force | Out-Null |
| 22 | return [pscustomobject]@{ |
| 23 | Root = $root |
| 24 | MatrixRoot = $matrixRoot |
| 25 | SurfaceRoot = $surfaceRoot |
| 26 | InventoryPath = $inventoryPath |
| 27 | } |
| 28 | } |
| 29 | |
| 30 | function New-FixtureInventory { |
| 31 | [CmdletBinding()] |
| 32 | param( |
| 33 | [Parameter(Mandatory)] [string]$Path, |
| 34 | [Parameter(Mandatory)] [hashtable[]]$Agents |
| 35 | ) |
| 36 | |
| 37 | $lines = @('agents:') |
| 38 | foreach ($a in $Agents) { |
| 39 | $lines += " - slug: $($a.slug)" |
| 40 | $lines += " class: $($a.class)" |
| 41 | $lines += " cost_tier: $($a.cost_tier)" |
| 42 | $lines += " path: .github/agents/$($a.slug).agent.md" |
| 43 | } |
| 44 | Set-Content -LiteralPath $Path -Value ($lines -join "`n") -Encoding utf8NoBOM |
| 45 | } |
| 46 | |
| 47 | function New-FixtureDatedRun { |
| 48 | [CmdletBinding()] |
| 49 | [OutputType([string])] |
| 50 | param( |
| 51 | [Parameter(Mandatory)] [string]$MatrixRoot, |
| 52 | [Parameter(Mandatory)] [string]$Date, |
| 53 | [Parameter(Mandatory)] [hashtable[]]$Results, |
| 54 | [string]$Tier = 'nightly', |
| 55 | [string]$Mode = 'all', |
| 56 | [string]$Overall = 'pass' |
| 57 | ) |
| 58 | |
| 59 | $runDir = Join-Path $MatrixRoot $Date |
| 60 | New-Item -ItemType Directory -Path $runDir -Force | Out-Null |
| 61 | |
| 62 | foreach ($r in $Results) { |
| 63 | $perAgent = [ordered]@{ |
| 64 | slug = $r.slug |
| 65 | class = $r.class |
| 66 | cost_tier = $r.cost_tier |
| 67 | graders = if ($r.ContainsKey('graders')) { $r.graders } else { @() } |
| 68 | overall = $r.overall |
| 69 | exitCode = if ($r.ContainsKey('exitCode')) { $r.exitCode } else { 0 } |
| 70 | logPath = "logs/agent-matrix/$($r.slug)-fake.log" |
| 71 | } |
| 72 | Set-Content ` |
| 73 | -LiteralPath (Join-Path $runDir "$($r.slug).json") ` |
| 74 | -Value ($perAgent | ConvertTo-Json -Depth 4) ` |
| 75 | -Encoding utf8NoBOM |
| 76 | } |
| 77 | |
| 78 | $summary = [ordered]@{ |
| 79 | generatedAt = "$($Date)T12:00:00Z" |
| 80 | tier = $Tier |
| 81 | mode = $Mode |
| 82 | agentCount = $Results.Count |
| 83 | overall = $Overall |
| 84 | failures = @($Results | Where-Object { $_.overall -eq 'fail' } | ForEach-Object { $_.slug }) |
| 85 | results = @($Results | ForEach-Object { |
| 86 | [ordered]@{ |
| 87 | slug = $_.slug |
| 88 | class = $_.class |
| 89 | cost_tier = $_.cost_tier |
| 90 | graders = if ($_.ContainsKey('graders')) { $_.graders } else { @() } |
| 91 | overall = $_.overall |
| 92 | exitCode = if ($_.ContainsKey('exitCode')) { $_.exitCode } else { 0 } |
| 93 | logPath = "logs/agent-matrix/$($_.slug)-fake.log" |
| 94 | } |
| 95 | }) |
| 96 | plannedCommands = @($Results | ForEach-Object { "npx vally eval --eval-spec evals/agent-behavior/stimuli/$($_.slug).yml" }) |
| 97 | } |
| 98 | $summaryPath = Join-Path $runDir 'agent-matrix-summary.json' |
| 99 | Set-Content -LiteralPath $summaryPath -Value ($summary | ConvertTo-Json -Depth 5) -Encoding utf8NoBOM |
| 100 | return $summaryPath |
| 101 | } |
| 102 | |
| 103 | function Get-DrillRowRegex { |
| 104 | <# |
| 105 | .SYNOPSIS |
| 106 | Builds a regex string that anchors to a dashboard drill row for a given agent slug. |
| 107 | |
| 108 | .DESCRIPTION |
| 109 | Returns a `(?s)`-prefixed pattern that locates the `<tr class="drill" |
| 110 | data-drill-for="<slug>">` row in rendered dashboard HTML, then matches the |
| 111 | caller-supplied `Inner` fragment anywhere inside that row. The slug is |
| 112 | regex-escaped; `Inner` is appended as-is so callers can compose subpatterns. |
| 113 | #> |
| 114 | [CmdletBinding()] |
| 115 | [OutputType([string])] |
| 116 | param( |
| 117 | [Parameter(Mandatory)] [string]$Slug, |
| 118 | [Parameter(Mandatory)] [string]$Inner |
| 119 | ) |
| 120 | |
| 121 | return '(?s)data-drill-for="' + [regex]::Escape($Slug) + '"[^>]*>.*?' + $Inner |
| 122 | } |
| 123 | |
| 124 | Export-ModuleMember -Function New-FixtureRoot, New-FixtureInventory, New-FixtureDatedRun, Get-DrillRowRegex |
| 125 | |