microsoft/hve-core
Publicmirrored from https://github.com/microsoft/hve-coreAvailable
scripts/tests/evals/EquivalenceWrappers.Tests.ps1
155lines · modecode
| 1 | #Requires -Modules Pester |
| 2 | # Copyright (c) Microsoft Corporation. |
| 3 | # SPDX-License-Identifier: MIT |
| 4 | |
| 5 | BeforeAll { |
| 6 | $script:DashboardScript = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot '../../evals/New-EquivalenceDashboard.ps1')).Path |
| 7 | $script:DriverScript = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot '../../evals/Invoke-BaselineEquivalence.ps1')).Path |
| 8 | $script:FixturesRoot = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot 'fixtures/equivalence')).Path |
| 9 | |
| 10 | function script:Initialize-DashboardFixture { |
| 11 | param( |
| 12 | [Parameter(Mandatory)][string]$RepoRoot, |
| 13 | [Parameter(Mandatory)][string]$Model, |
| 14 | [Parameter(Mandatory)][string]$RunId |
| 15 | ) |
| 16 | |
| 17 | $resultsRoot = Join-Path $RepoRoot 'evals/results/baseline-equivalence' |
| 18 | $runRoot = Join-Path $resultsRoot "$Model/$RunId" |
| 19 | $baselineDir = Join-Path $runRoot 'baseline' |
| 20 | $customizedDir = Join-Path $runRoot 'customized' |
| 21 | New-Item -ItemType Directory -Path $baselineDir -Force | Out-Null |
| 22 | New-Item -ItemType Directory -Path $customizedDir -Force | Out-Null |
| 23 | |
| 24 | Copy-Item -LiteralPath (Join-Path $script:FixturesRoot 'baseline/20260101T000000000Z/results.jsonl') ` |
| 25 | -Destination (Join-Path $baselineDir 'results.jsonl') -Force |
| 26 | Copy-Item -LiteralPath (Join-Path $script:FixturesRoot 'customized/20260101T000000000Z/results.jsonl') ` |
| 27 | -Destination (Join-Path $customizedDir 'results.jsonl') -Force |
| 28 | |
| 29 | return $resultsRoot |
| 30 | } |
| 31 | } |
| 32 | |
| 33 | Describe 'New-EquivalenceDashboard.ps1' -Tag 'Unit' { |
| 34 | It 'Declares -Agent as a mandatory string parameter' { |
| 35 | $cmd = Get-Command -Name $script:DashboardScript |
| 36 | $param = $cmd.Parameters['Agent'] |
| 37 | $param | Should -Not -BeNullOrEmpty |
| 38 | $param.ParameterType | Should -Be ([string]) |
| 39 | $param.Attributes.Where({ $_ -is [System.Management.Automation.ParameterAttribute] }).Mandatory | Should -Contain $true |
| 40 | } |
| 41 | |
| 42 | Context 'Applied artifact wiring' { |
| 43 | BeforeAll { |
| 44 | $script:RepoRoot = Join-Path $TestDrive 'dashboard-repo' |
| 45 | $script:Model = 'unit-model' |
| 46 | $script:RunId = 'unit-run' |
| 47 | $script:ResultsRoot = Initialize-DashboardFixture -RepoRoot $script:RepoRoot -Model $script:Model -RunId $script:RunId |
| 48 | |
| 49 | $script:Workspace = Join-Path $script:RepoRoot 'evals/baseline-equivalence/customized/workspace' |
| 50 | New-Item -ItemType Directory -Path (Join-Path $script:Workspace '.github/agents') -Force | Out-Null |
| 51 | Set-Content -LiteralPath (Join-Path $script:Workspace '.github/agents/foo.agent.md') -Value 'seed' -Encoding utf8NoBOM |
| 52 | |
| 53 | $script:OutPath = Join-Path $TestDrive 'dashboard.html' |
| 54 | & $script:DashboardScript ` |
| 55 | -RunId $script:RunId ` |
| 56 | -Model $script:Model ` |
| 57 | -Agent 'task-researcher' ` |
| 58 | -RepoRoot $script:RepoRoot ` |
| 59 | -ResultsRoot $script:ResultsRoot ` |
| 60 | -OutPath $script:OutPath *> $null |
| 61 | $script:Html = Get-Content -LiteralPath $script:OutPath -Raw |
| 62 | } |
| 63 | |
| 64 | It 'Writes the HTML output to the requested path' { |
| 65 | Test-Path -LiteralPath $script:OutPath | Should -BeTrue |
| 66 | } |
| 67 | |
| 68 | It 'Renders the seeded workspace artifact in the applied list' { |
| 69 | $script:Html | Should -Match '<li>\.github/agents/foo\.agent\.md</li>' |
| 70 | } |
| 71 | } |
| 72 | |
| 73 | Context 'Empty workspace falls back to no applied artifacts' { |
| 74 | BeforeAll { |
| 75 | $script:RepoRoot2 = Join-Path $TestDrive 'dashboard-repo-empty' |
| 76 | $script:ResultsRoot2 = Initialize-DashboardFixture -RepoRoot $script:RepoRoot2 -Model 'unit-model' -RunId 'unit-run' |
| 77 | |
| 78 | # No workspace directory at all. |
| 79 | $script:OutPath2 = Join-Path $TestDrive 'dashboard-empty.html' |
| 80 | & $script:DashboardScript ` |
| 81 | -RunId 'unit-run' ` |
| 82 | -Model 'unit-model' ` |
| 83 | -Agent 'task-researcher' ` |
| 84 | -RepoRoot $script:RepoRoot2 ` |
| 85 | -ResultsRoot $script:ResultsRoot2 ` |
| 86 | -OutPath $script:OutPath2 *> $null |
| 87 | $script:Html2 = Get-Content -LiteralPath $script:OutPath2 -Raw |
| 88 | } |
| 89 | |
| 90 | It 'Renders the applied list with a (none) placeholder when no artifacts exist' { |
| 91 | $script:Html2 | Should -Match '<li><em>\(none\)</em></li>' |
| 92 | } |
| 93 | } |
| 94 | } |
| 95 | |
| 96 | Describe 'Invoke-BaselineEquivalence.ps1' -Tag 'Unit' { |
| 97 | Context 'Applied artifact wiring (workspace present)' { |
| 98 | BeforeAll { |
| 99 | $script:DriverRepo = Join-Path $TestDrive 'driver-repo' |
| 100 | $script:Workspace = Join-Path $script:DriverRepo 'evals/baseline-equivalence/customized/workspace' |
| 101 | New-Item -ItemType Directory -Path (Join-Path $script:Workspace '.github/agents') -Force | Out-Null |
| 102 | Set-Content -LiteralPath (Join-Path $script:Workspace '.github/agents/bar.agent.md') -Value 'seed' -Encoding utf8NoBOM |
| 103 | |
| 104 | $signatureDir = Join-Path $script:DriverRepo 'evals/baseline-equivalence/surface-signatures' |
| 105 | New-Item -ItemType Directory -Path $signatureDir -Force | Out-Null |
| 106 | Set-Content -LiteralPath (Join-Path $signatureDir 'task-researcher.yml') -Value "description: stub`n" -Encoding utf8NoBOM |
| 107 | $compareSpecPath = Join-Path $script:DriverRepo 'evals/baseline-equivalence/compare.eval.yml' |
| 108 | Set-Content -LiteralPath $compareSpecPath -Value "surface_signatures: {}`n" -Encoding utf8NoBOM |
| 109 | |
| 110 | $script:SummaryPath = Join-Path $TestDrive 'driver-summary.json' |
| 111 | & $script:DriverScript ` |
| 112 | -Agent 'task-researcher' ` |
| 113 | -Tier 'pr' ` |
| 114 | -RepoRoot $script:DriverRepo ` |
| 115 | -OutputPath $script:SummaryPath ` |
| 116 | -WhatIf *> $null |
| 117 | $script:Summary = Get-Content -LiteralPath $script:SummaryPath -Raw | ConvertFrom-Json |
| 118 | } |
| 119 | |
| 120 | It 'Writes the summary JSON' { |
| 121 | Test-Path -LiteralPath $script:SummaryPath | Should -BeTrue |
| 122 | } |
| 123 | |
| 124 | It 'Populates variants.b.applied with the seeded artifact' { |
| 125 | $script:Summary.variants.b.applied | Should -Contain '.github/agents/bar.agent.md' |
| 126 | } |
| 127 | } |
| 128 | |
| 129 | Context 'Applied artifact wiring (workspace absent)' { |
| 130 | BeforeAll { |
| 131 | $script:DriverRepoEmpty = Join-Path $TestDrive 'driver-repo-empty' |
| 132 | New-Item -ItemType Directory -Path $script:DriverRepoEmpty -Force | Out-Null |
| 133 | |
| 134 | $signatureDirEmpty = Join-Path $script:DriverRepoEmpty 'evals/baseline-equivalence/surface-signatures' |
| 135 | New-Item -ItemType Directory -Path $signatureDirEmpty -Force | Out-Null |
| 136 | Set-Content -LiteralPath (Join-Path $signatureDirEmpty 'task-researcher.yml') -Value "description: stub`n" -Encoding utf8NoBOM |
| 137 | $compareSpecPathEmpty = Join-Path $script:DriverRepoEmpty 'evals/baseline-equivalence/compare.eval.yml' |
| 138 | New-Item -ItemType Directory -Path (Split-Path -Parent $compareSpecPathEmpty) -Force | Out-Null |
| 139 | Set-Content -LiteralPath $compareSpecPathEmpty -Value "surface_signatures: {}`n" -Encoding utf8NoBOM |
| 140 | |
| 141 | $script:SummaryPathEmpty = Join-Path $TestDrive 'driver-summary-empty.json' |
| 142 | & $script:DriverScript ` |
| 143 | -Agent 'task-researcher' ` |
| 144 | -Tier 'pr' ` |
| 145 | -RepoRoot $script:DriverRepoEmpty ` |
| 146 | -OutputPath $script:SummaryPathEmpty ` |
| 147 | -WhatIf *> $null |
| 148 | $script:SummaryEmpty = Get-Content -LiteralPath $script:SummaryPathEmpty -Raw | ConvertFrom-Json |
| 149 | } |
| 150 | |
| 151 | It 'Emits an empty variants.b.applied array' { |
| 152 | @($script:SummaryEmpty.variants.b.applied).Count | Should -Be 0 |
| 153 | } |
| 154 | } |
| 155 | } |
| 156 | |