#!/usr/bin/env pwsh
# Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: MIT
#Requires -Version 7.0

<#
.SYNOPSIS
    Executes vally evals for the AI artifacts changed in a pull request.

.DESCRIPTION
    Reads the changed-artifact manifest produced by `Get-ChangedAIArtifact.ps1`,
    resolves each artifact to its matching eval spec(s) via the same
    `StimulusIndex` used by `Test-StimulusPresence.ps1`, and invokes
    `vally eval` exactly once per unique spec path. Per-spec results are
    aggregated into:

      logs/eval-results-<kind>-<artifactId>.json - one file per artifact, with
                                                   its associated spec results
      logs/eval-summary.json                     - roll-up totals + perArtifact
                                                   + perSpec arrays for CI

    Exit codes:
      0 = all changed artifacts passed (or manifest is empty / only deletions).
      1 = at least one spec failed (non-zero vally exit code or failed trials).
      2 = invalid input: missing manifest, missing eval root, or missing
          coverage for any non-deleted artifact (should have been caught by
          `Test-StimulusPresence.ps1` upstream).

.PARAMETER ManifestPath
    Path to the changed-artifact manifest. Defaults to
    `logs/changed-ai-artifacts.json`. Resolved relative to the repository root
    when not absolute.

.PARAMETER EvalRoot
    Filesystem path to the eval spec root. Defaults to `evals/`. Resolved
    relative to the repository root when not absolute.

.PARAMETER LogsDir
    Directory where per-artifact JSON files and the eval-summary are written.
    Defaults to `logs/`. Created if it does not exist.

.PARAMETER Model
    Model passed to `vally eval --model`. Defaults to `claude-opus-4.7` to
    match the PR-tier model used by `Invoke-BaselineEquivalence.ps1`.

.PARAMETER VallyCommand
    Path or name of the vally executable. Defaults to `vally`. Tests pass the
    absolute path to `scripts/tests/evals/fixtures/stub-vally.ps1`.

.PARAMETER EquivalenceDriverPath
    Path to the baseline-equivalence driver script. Defaults to
    `<RepoRoot>/scripts/evals/Invoke-BaselineEquivalence.ps1`. Tests override this
    to point at a stub script. Invoked once per `kind = 'agent'` artifact via
    `pwsh -NoProfile -File <path> -Agent <slug> -Tier <tier> -RepoRoot <root> -OutputPath <json>`.

.PARAMETER EquivalenceTier
    Tier passed to the equivalence driver (`pr` or `nightly`). Defaults to `pr`.
    Per DD-01, PR-tier equivalence dispatch is advisory: failures surface in
    summary JSON but do not increment `failedSpecs` or change exit code.

.PARAMETER FailFast
    Stop after the first spec invocation that returns a non-zero exit code or
    any failed trial. Default: process every spec, then exit non-zero if any
    failed.

.PARAMETER SkipInputModeration
    Skip pre-eval content moderation of stimulus prompts. Default: $false.

.PARAMETER SkipOutputModeration
    Skip post-eval content moderation of model outputs. Default: $false.

.PARAMETER ModerationThreshold
    Toxicity threshold (0.0-1.0) for content moderation. Defaults to 0.5.
    Individual specs may override this via the optional top-level
    `moderation.threshold` field.

.PARAMETER RepoRoot
    Repository root. Defaults to `git rev-parse --show-toplevel`.

.EXAMPLE
    pwsh scripts/evals/Invoke-VallyEvals.ps1

    Reads `logs/changed-ai-artifacts.json`, runs the matched specs against
    `vally`, and writes per-artifact + summary JSON under `logs/`.

.NOTES
    Runs from the PR-time `eval-execute` job once coverage and lint pass.
#>
[CmdletBinding()]
param(
    [string]$ManifestPath,
    [string]$EvalRoot,
    [string]$LogsDir,
    [string]$Model = 'claude-opus-4.7',
    [string]$VallyCommand = 'vally',
    [string]$EquivalenceDriverPath,
    [ValidateSet('pr','nightly')]
    [string]$EquivalenceTier = 'pr',
    [switch]$FailFast,
    [switch]$SkipInputModeration,
    [switch]$SkipOutputModeration,
    [double]$ModerationThreshold = 0.5,
    [string]$RepoRoot
)

$ErrorActionPreference = 'Stop'

Import-Module (Join-Path $PSScriptRoot 'Modules/StimulusIndex.psm1') -Force
Import-Module (Join-Path $PSScriptRoot 'Modules/VallyRunner.psm1') -Force

if (-not (Get-Module -Name powershell-yaml)) {
    Import-Module powershell-yaml -ErrorAction Stop
}

function Get-SpecModerationThreshold {
    [CmdletBinding()]
    [OutputType([Nullable[double]])]
    param(
        [Parameter(Mandatory)][string]$SpecPath
    )

    if (-not (Test-Path -LiteralPath $SpecPath -PathType Leaf)) { return $null }

    try {
        $raw = Get-Content -LiteralPath $SpecPath -Raw -Encoding utf8
        if ([string]::IsNullOrWhiteSpace($raw)) { return $null }
        $parsed = $raw | ConvertFrom-Yaml
    }
    catch {
        Write-Verbose "Get-SpecModerationThreshold: failed to parse '$SpecPath': $_"
        return $null
    }

    if ($null -eq $parsed -or -not ($parsed -is [System.Collections.IDictionary])) { return $null }
    if (-not $parsed.ContainsKey('moderation')) { return $null }
    $moderation = $parsed['moderation']
    if (-not ($moderation -is [System.Collections.IDictionary]) -or -not $moderation.ContainsKey('threshold')) { return $null }

    $value = $moderation['threshold']
    try { return [double]$value } catch { return $null }
}

function Test-SpecIsAdvisory {
    # Returns $true when every stimulus in the spec carries `tags.advisory: true`.
    # Per DD-05, advisory specs surface failures in the run summary but never
    # promote the dispatcher's overall exit code to non-zero.
    [CmdletBinding()]
    [OutputType([bool])]
    param(
        [Parameter(Mandatory)][string]$SpecPath
    )

    if (-not (Test-Path -LiteralPath $SpecPath -PathType Leaf)) { return $false }

    try {
        $raw = Get-Content -LiteralPath $SpecPath -Raw -Encoding utf8
        if ([string]::IsNullOrWhiteSpace($raw)) { return $false }
        $parsed = $raw | ConvertFrom-Yaml
    }
    catch {
        Write-Verbose "Test-SpecIsAdvisory: failed to parse '$SpecPath': $_"
        return $false
    }

    if ($null -eq $parsed -or -not ($parsed -is [System.Collections.IDictionary])) { return $false }
    if (-not $parsed.ContainsKey('stimuli')) { return $false }
    $stimuli = $parsed['stimuli']
    if ($null -eq $stimuli -or -not ($stimuli -is [System.Collections.IEnumerable]) -or $stimuli -is [string]) { return $false }

    $any = $false
    foreach ($stimulus in $stimuli) {
        $any = $true
        if (-not ($stimulus -is [System.Collections.IDictionary])) { return $false }
        if (-not $stimulus.Contains('tags')) { return $false }
        $tags = $stimulus['tags']
        if (-not ($tags -is [System.Collections.IDictionary]) -or -not $tags.Contains('advisory')) { return $false }
        if (-not [bool]$tags['advisory']) { return $false }
    }

    return $any
}

function Get-SpecStimulusAdvisoryMap {
    # Returns @{<stimulus-name> = [bool]} when at least one stimulus carries
    # `tags.advisory`, supporting per-stimulus graduation from advisory to
    # authoritative within a single spec. Returns $null when no stimulus
    # declares an advisory tag; callers then fall back to Test-SpecIsAdvisory.
    [CmdletBinding()]
    [OutputType([hashtable])]
    param(
        [Parameter(Mandatory)][string]$SpecPath
    )

    if (-not (Test-Path -LiteralPath $SpecPath -PathType Leaf)) { return $null }

    try {
        $raw = Get-Content -LiteralPath $SpecPath -Raw -Encoding utf8
        if ([string]::IsNullOrWhiteSpace($raw)) { return $null }
        $parsed = $raw | ConvertFrom-Yaml
    }
    catch {
        Write-Verbose "Get-SpecStimulusAdvisoryMap: failed to parse '$SpecPath': $_"
        return $null
    }

    if ($null -eq $parsed -or -not ($parsed -is [System.Collections.IDictionary])) { return $null }
    if (-not $parsed.ContainsKey('stimuli')) { return $null }
    $stimuli = $parsed['stimuli']
    if ($null -eq $stimuli -or -not ($stimuli -is [System.Collections.IEnumerable]) -or $stimuli -is [string]) { return $null }

    $map = @{}
    $sawAdvisoryTag = $false
    foreach ($stimulus in $stimuli) {
        if (-not ($stimulus -is [System.Collections.IDictionary])) { continue }
        if (-not $stimulus.Contains('name')) { continue }
        $name = [string]$stimulus['name']
        if ([string]::IsNullOrWhiteSpace($name)) { continue }

        $advisory = $false
        if ($stimulus.Contains('tags') -and $stimulus['tags'] -is [System.Collections.IDictionary] -and $stimulus['tags'].Contains('advisory')) {
            $sawAdvisoryTag = $true
            $advisory = [bool]$stimulus['tags']['advisory']
        }
        $map[$name] = $advisory
    }

    if (-not $sawAdvisoryTag) { return $null }
    return $map
}

function Resolve-RepoRoot {
    [CmdletBinding()]
    [OutputType([string])]
    param([string]$Hint)

    if (-not [string]::IsNullOrWhiteSpace($Hint)) {
        return (Resolve-Path -LiteralPath $Hint).ProviderPath
    }

    try {
        $gitRoot = git rev-parse --show-toplevel 2>$null
        if ($LASTEXITCODE -eq 0 -and -not [string]::IsNullOrWhiteSpace($gitRoot)) {
            return (Resolve-Path -LiteralPath $gitRoot.Trim()).ProviderPath
        }
    }
    catch { $null = $_ }

    return (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot '../..')).ProviderPath
}

function Resolve-PathFromRoot {
    [CmdletBinding()]
    [OutputType([string])]
    param(
        [Parameter(Mandatory = $true)][string]$Path,
        [Parameter(Mandatory = $true)][string]$RepoRoot
    )

    if ([System.IO.Path]::IsPathRooted($Path)) { return $Path }
    return (Join-Path -Path $RepoRoot -ChildPath $Path)
}

function ConvertTo-SafeKey {
    [CmdletBinding()]
    [OutputType([string])]
    param([Parameter(Mandatory = $true)][string]$Value)

    return ($Value -replace '[^A-Za-z0-9\-_.]', '_')
}

function Get-ArtifactFileKey {
    [CmdletBinding()]
    [OutputType([string])]
    param(
        [Parameter(Mandatory = $true)][string]$Kind,
        [Parameter(Mandatory = $true)][string]$ArtifactId
    )

    # kind prefix prevents collisions when a skill and a prompt share a slug.
    return "$Kind-$(ConvertTo-SafeKey -Value $ArtifactId)"
}

function Write-JsonFile {
    [CmdletBinding()]
    param(
        [Parameter(Mandatory = $true)]$Value,
        [Parameter(Mandatory = $true)][string]$Path
    )

    $dir = Split-Path -Parent $Path
    if ($dir -and -not (Test-Path -LiteralPath $dir)) {
        New-Item -ItemType Directory -Path $dir -Force | Out-Null
    }
    $json = $Value | ConvertTo-Json -Depth 12
    Set-Content -LiteralPath $Path -Value $json -Encoding utf8NoBOM
}

if ($MyInvocation.InvocationName -eq '.') { return }

$resolvedRoot = Resolve-RepoRoot -Hint $RepoRoot

if ([string]::IsNullOrWhiteSpace($ManifestPath)) { $ManifestPath = 'logs/changed-ai-artifacts.json' }
if ([string]::IsNullOrWhiteSpace($EvalRoot))     { $EvalRoot     = 'evals' }
if ([string]::IsNullOrWhiteSpace($LogsDir))      { $LogsDir      = 'logs' }

$resolvedManifest = Resolve-PathFromRoot -Path $ManifestPath -RepoRoot $resolvedRoot
$resolvedEvalRoot = Resolve-PathFromRoot -Path $EvalRoot     -RepoRoot $resolvedRoot
$resolvedLogsDir  = Resolve-PathFromRoot -Path $LogsDir      -RepoRoot $resolvedRoot

if (-not (Test-Path -LiteralPath $resolvedManifest -PathType Leaf)) {
    Write-Host "::error file=$ManifestPath::Manifest not found: $resolvedManifest"
    exit 2
}
if (-not (Test-Path -LiteralPath $resolvedEvalRoot -PathType Container)) {
    Write-Host "::error::Eval root not found: $resolvedEvalRoot"
    exit 2
}
if (-not (Test-Path -LiteralPath $resolvedLogsDir -PathType Container)) {
    New-Item -ItemType Directory -Path $resolvedLogsDir -Force | Out-Null
}

$manifest = Get-Content -LiteralPath $resolvedManifest -Raw | ConvertFrom-Json
$artifacts = @()
if ($null -ne $manifest -and $null -ne $manifest.artifacts) {
    $artifacts = @($manifest.artifacts | Where-Object { [string]$_.status -ne 'D' })
}

$summaryPath = Join-Path -Path $resolvedLogsDir -ChildPath 'eval-summary.json'

if ($artifacts.Count -eq 0) {
    $emptySummary = [ordered]@{
        manifestPath = $resolvedManifest
        evalRoot     = $resolvedEvalRoot
        model        = $Model
        totals       = [ordered]@{
            artifacts        = 0
            specs            = 0
            assertionsPassed = 0
            assertionsFailed = 0
            durationMs       = 0
            failedSpecs      = 0
        }
        perArtifact  = @()
        perSpec      = @()
        equivalence  = @()
    }
    Write-JsonFile -Value $emptySummary -Path $summaryPath
    Write-Host "No changed AI artifacts to evaluate. Summary written to $summaryPath"
    exit 0
}

$index = New-StimulusIndex -EvalRoot $resolvedEvalRoot

if (-not $EquivalenceDriverPath) {
    $EquivalenceDriverPath = Join-Path -Path $resolvedRoot -ChildPath 'scripts/evals/Invoke-BaselineEquivalence.ps1'
}

$artifactPlan   = [System.Collections.Generic.List[hashtable]]::new()
$uniqueSpecs    = @{}
$equivalenceSpecs = @{}
$missingSpecs   = [System.Collections.Generic.List[hashtable]]::new()

foreach ($artifact in $artifacts) {
    $kind       = [string]$artifact.kind
    $artifactId = [string]$artifact.artifactId
    $specs      = Test-StimulusCoverage -Index $index -Kind $kind -ArtifactId $artifactId

    if ($specs.Count -eq 0) {
        $missingSpecs.Add(@{ kind = $kind; artifactId = $artifactId; path = [string]$artifact.path })
        continue
    }

    foreach ($specRel in $specs) {
        if (-not $uniqueSpecs.ContainsKey($specRel)) {
            $uniqueSpecs[$specRel] = Join-Path -Path $index.root -ChildPath $specRel
        }
    }

    if ($kind -eq 'agent') {
        $equivKey = "equivalence:$artifactId"
        if (-not $equivalenceSpecs.ContainsKey($equivKey)) {
            $equivalenceSpecs[$equivKey] = $artifactId
        }
    }

    $artifactPlan.Add(@{
        kind        = $kind
        artifactId  = $artifactId
        path        = [string]$artifact.path
        status      = [string]$artifact.status
        specs       = @($specs)
    })
}

if ($missingSpecs.Count -gt 0) {
    foreach ($m in $missingSpecs) {
        Write-Host "::error file=$($m.path)::No eval spec resolves $($m.kind):$($m.artifactId); run Test-StimulusPresence first."
    }
    Write-Host "::error::Cannot execute evals: $($missingSpecs.Count) artifact(s) have no covering spec."
    exit 2
}

$runsRoot = Join-Path -Path $resolvedLogsDir -ChildPath 'eval-runs'
if (-not (Test-Path -LiteralPath $runsRoot)) {
    New-Item -ItemType Directory -Path $runsRoot -Force | Out-Null
}
$moderationScript = Join-Path -Path $resolvedRoot -ChildPath 'scripts/evals/Invoke-ContentModeration.ps1'

$specResults = @{}
$failedSpecs = 0

foreach ($specRel in $uniqueSpecs.Keys) {
    $specAbs = $uniqueSpecs[$specRel]
    $specKey = ConvertTo-SafeKey -Value $specRel
    $specOut = Join-Path -Path $runsRoot -ChildPath $specKey
    $specLog = Join-Path -Path $resolvedLogsDir -ChildPath "vally-eval-$specKey.log"

    # Pre-eval content moderation (input)
    $inputModeration = @{ flagged = $false; flaggedCount = 0; outputPath = $null }
    $specThreshold = Get-SpecModerationThreshold -SpecPath $specAbs
    $effectiveThreshold = if ($null -ne $specThreshold) { $specThreshold } else { $ModerationThreshold }
    if ($null -ne $specThreshold) {
        Write-Verbose "Per-spec moderation.threshold=$specThreshold overrides default ($ModerationThreshold) for $specRel"
    }
    if (-not $SkipInputModeration) {
        Write-Verbose "Pre-eval content moderation for spec: $specRel"
        $inputModeration = Test-SpecInputModeration `
            -SpecPath $specAbs `
            -ArtifactId $specKey `
            -ModerationScript $moderationScript `
            -Threshold $effectiveThreshold `
            -RepoRoot $resolvedRoot

        if ($inputModeration.flagged) {
            Write-Host "::warning file=$specRel::Content moderation flagged $($inputModeration.flaggedCount) input prompt(s); skipping eval"
            $specResults[$specRel] = @{
                specPath         = $specAbs
                exitCode         = 0
                runDir           = $null
                assertionsPassed = 0
                assertionsFailed = 0
                durationMs       = 0
                trials           = 0
                resultsPath      = $null
                moderationInput  = $inputModeration
                moderationOutput = $null
                status           = 'content-moderation-input'
            }
            $failedSpecs++
            if ($FailFast) {
                Write-Host "::warning::FailFast set; skipping remaining specs after input moderation failure in $specRel"
                break
            }
            continue
        }
    }

    Write-Host "Running: vally eval --eval-spec $specRel --model $Model" -ForegroundColor Cyan
    $result = Invoke-VallySpec `
        -SpecPath $specAbs `
        -OutputDir $specOut `
        -Model $Model `
        -VallyCommand $VallyCommand `
        -LogPath $specLog

    # Post-eval content moderation (output)
    $outputModeration = @{ flagged = $false; flaggedCount = 0; outputPath = $null }
    if (-not $SkipOutputModeration -and $result.runDir) {
        Write-Verbose "Post-eval content moderation for spec: $specRel"
        $outputModeration = Test-SpecOutputModeration `
            -RunDir $result.runDir `
            -ArtifactId $specKey `
            -ModerationScript $moderationScript `
            -Threshold $effectiveThreshold `
            -RepoRoot $resolvedRoot

        if ($outputModeration.flagged) {
            Write-Host "::warning file=$specRel::Content moderation flagged $($outputModeration.flaggedCount) model output(s)"
            $result.status = 'content-moderation-output'
            $result.assertionsFailed = [Math]::Max($result.assertionsFailed, $outputModeration.flaggedCount)
        }
    }

    $result['moderationInput'] = $inputModeration
    $result['moderationOutput'] = $outputModeration

    $advisoryMap = Get-SpecStimulusAdvisoryMap -SpecPath $specAbs
    $result['perStimulusAdvisory'] = $advisoryMap

    if ($null -ne $advisoryMap) {
        $advisoryPassed = 0
        $advisoryFailed = 0
        $authoritativePassed = 0
        $authoritativeFailed = 0
        if ($result.ContainsKey('perStimulus') -and $result.perStimulus) {
            foreach ($stimulusName in $result.perStimulus.Keys) {
                $bucket = $result.perStimulus[$stimulusName]
                $stimAdvisory = $false
                if ($advisoryMap.ContainsKey($stimulusName)) {
                    $stimAdvisory = [bool]$advisoryMap[$stimulusName]
                }
                if ($stimAdvisory) {
                    $advisoryPassed += [int]$bucket.assertionsPassed
                    $advisoryFailed += [int]$bucket.assertionsFailed
                }
                else {
                    $authoritativePassed += [int]$bucket.assertionsPassed
                    $authoritativeFailed += [int]$bucket.assertionsFailed
                }
            }
        }
        $result['advisoryPassed'] = $advisoryPassed
        $result['advisoryFailed'] = $advisoryFailed
        $result['authoritativePassed'] = $authoritativePassed
        $result['authoritativeFailed'] = $authoritativeFailed
        $result['isAdvisory'] = ($authoritativeFailed -eq 0 -and $advisoryFailed -gt 0)

        if (-not $result.ContainsKey('status')) {
            if ($authoritativeFailed -gt 0 -or $outputModeration.flagged) {
                $result['status'] = 'fail'
            }
            elseif ($advisoryFailed -gt 0) {
                $result['status'] = 'advisory-fail'
            }
            elseif ($result.exitCode -ne 0) {
                $result['status'] = 'fail'
            }
            else {
                $result['status'] = 'pass'
            }
        }

        $specResults[$specRel] = $result

        $promote = $authoritativeFailed -gt 0 -or $outputModeration.flagged
        if (-not $promote -and $result.exitCode -ne 0 -and $advisoryFailed -eq 0 -and $authoritativeFailed -eq 0) {
            $promote = $true
        }

        if ($promote) {
            $failedSpecs++
            if ($advisoryFailed -gt 0) {
                Write-Host "::warning file=$specRel::Per-stimulus advisory failures coexist with authoritative failures; promoting to CI failure"
            }
            if ($FailFast) {
                Write-Host "::warning::FailFast set; skipping remaining specs after failure in $specRel"
                break
            }
        }
        elseif ($advisoryFailed -gt 0) {
            Write-Host "::warning file=$specRel::Per-stimulus advisory failures: $advisoryFailed assertion(s) across advisory stimuli; not promoting to CI failure"
        }
    }
    else {
        $isAdvisory = Test-SpecIsAdvisory -SpecPath $specAbs
        $result['isAdvisory'] = $isAdvisory
        if (-not $result.ContainsKey('status')) {
            $result['status'] = if ($result.exitCode -ne 0 -or $result.assertionsFailed -gt 0) { 'fail' } else { 'pass' }
        }

        $specResults[$specRel] = $result

        if ($result.exitCode -ne 0 -or $result.assertionsFailed -gt 0 -or $outputModeration.flagged) {
            if ($isAdvisory) {
                $result['status'] = 'advisory-fail'
                Write-Host "::warning file=$specRel::Advisory spec failed (exit=$($result.exitCode), assertionsFailed=$($result.assertionsFailed)); not promoting to CI failure"
            }
            else {
                $failedSpecs++
                if ($FailFast) {
                    Write-Host "::warning::FailFast set; skipping remaining specs after failure in $specRel"
                    break
                }
            }
        }
    }
}

# Dep-map reverse-lookup expansion: instruction/skill/subagent changes promote
# parent agents into the equivalence dispatch set. The manifest's
# `affectedAgents` field is precomputed by Get-ChangedAIArtifact.ps1 via the
# AffectedAgents module, which also refreshes logs/agent-dependency-map.json
# when stale.
$manifestAffected = @()
if ($null -ne $manifest -and $manifest.PSObject.Properties.Name -contains 'affectedAgents' -and $null -ne $manifest.affectedAgents) {
    $manifestAffected = @($manifest.affectedAgents)
}
foreach ($slug in $manifestAffected) {
    if ([string]::IsNullOrWhiteSpace($slug)) { continue }
    $equivKey = "equivalence:$slug"
    if (-not $equivalenceSpecs.ContainsKey($equivKey)) {
        $equivalenceSpecs[$equivKey] = $slug
    }
}

# Equivalence dispatch (Tier 2 baseline-equivalence). Per DD-01, PR-tier failures
# are advisory: they surface in summary but do not increment $failedSpecs.
$equivalenceResults = [System.Collections.Generic.List[object]]::new()
foreach ($equivKey in $equivalenceSpecs.Keys) {
    $agentSlug = $equivalenceSpecs[$equivKey]
    $equivOutPath = Join-Path -Path $resolvedLogsDir -ChildPath "baseline-equivalence-$agentSlug.json"
    $equivArgs = @(
        '-NoProfile', '-File', $EquivalenceDriverPath,
        '-Agent', $agentSlug,
        '-Tier', $EquivalenceTier,
        '-RepoRoot', $resolvedRoot,
        '-OutputPath', $equivOutPath
    )

    Write-Host "Running: pwsh $($equivArgs -join ' ')" -ForegroundColor Cyan
    & pwsh @equivArgs
    $equivExit = $LASTEXITCODE

    $runs = 0; $invFail = 0; $divFail = 0; $verdict = 'unknown'
    if (Test-Path -LiteralPath $equivOutPath) {
        try {
            $equivSummary = Get-Content -LiteralPath $equivOutPath -Raw | ConvertFrom-Json
            if ($null -ne $equivSummary.runs)               { $runs    = [int]$equivSummary.runs }
            if ($null -ne $equivSummary.invariantFailures)  { $invFail = [int]$equivSummary.invariantFailures }
            if ($null -ne $equivSummary.divergenceFailures) { $divFail = [int]$equivSummary.divergenceFailures }
            if ($null -ne $equivSummary.verdict)            { $verdict = [string]$equivSummary.verdict }
        }
        catch {
            Write-Host "::warning::Failed to parse equivalence summary $equivOutPath" -ForegroundColor Yellow
        }
    }

    $assertionsFailed = $invFail + $divFail
    $assertionsPassed = [Math]::Max(0, $runs - $assertionsFailed)

    $equivalenceResults.Add([ordered]@{
        agent              = $agentSlug
        tier               = $EquivalenceTier
        verdict            = $verdict
        exitCode           = $equivExit
        trials             = $runs
        assertionsPassed   = $assertionsPassed
        assertionsFailed   = $assertionsFailed
        invariantFailures  = $invFail
        divergenceFailures = $divFail
        resultsPath        = "logs/baseline-equivalence-$agentSlug.json"
    }) | Out-Null

    if ($EquivalenceTier -ne 'pr' -and ($equivExit -ne 0 -or $assertionsFailed -gt 0)) {
        $failedSpecs++
    }
}

$perArtifact = [System.Collections.Generic.List[object]]::new()
foreach ($plan in $artifactPlan) {
    $artifactPassed    = 0
    $artifactFailed    = 0
    $artifactDurationMs = 0
    $artifactExitCode  = 0
    $specBreakdown     = [System.Collections.Generic.List[object]]::new()
    $allSpecsRan       = $true

    foreach ($specRel in $plan.specs) {
        if (-not $specResults.ContainsKey($specRel)) {
            $allSpecsRan = $false
            continue
        }
        $r = $specResults[$specRel]
        $artifactPassed     += [int]$r.assertionsPassed
        $artifactFailed     += [int]$r.assertionsFailed
        $artifactDurationMs += [int]$r.durationMs
        if ($r.exitCode -ne 0 -and $artifactExitCode -eq 0) { $artifactExitCode = $r.exitCode }

        $specBreakdown.Add([ordered]@{
            specPath         = $specRel
            exitCode         = $r.exitCode
            assertionsPassed = $r.assertionsPassed
            assertionsFailed = $r.assertionsFailed
            durationMs       = $r.durationMs
            trials           = $r.trials
            runDir           = $r.runDir
            resultsPath      = $r.resultsPath
        })
    }

    $status = if (-not $allSpecsRan) { 'skipped' }
              elseif ($artifactFailed -gt 0 -or $artifactExitCode -ne 0) { 'fail' }
              else { 'pass' }

    $artifactKey  = Get-ArtifactFileKey -Kind $plan.kind -ArtifactId $plan.artifactId
    $artifactFile = Join-Path -Path $resolvedLogsDir -ChildPath "eval-results-$artifactKey.json"
    $artifactRecord = [ordered]@{
        kind             = $plan.kind
        artifactId       = $plan.artifactId
        path             = $plan.path
        changeStatus     = $plan.status
        status           = $status
        durationMs       = $artifactDurationMs
        assertionsPassed = $artifactPassed
        assertionsFailed = $artifactFailed
        specs            = @($specBreakdown)
    }
    Write-JsonFile -Value $artifactRecord -Path $artifactFile

    $perArtifact.Add([ordered]@{
        kind             = $plan.kind
        artifactId       = $plan.artifactId
        path             = $plan.path
        changeStatus     = $plan.status
        status           = $status
        durationMs       = $artifactDurationMs
        assertionsPassed = $artifactPassed
        assertionsFailed = $artifactFailed
        specCount        = $specBreakdown.Count
        resultsFile      = "logs/eval-results-$artifactKey.json"
    }) | Out-Null
}

$perSpec = [System.Collections.Generic.List[object]]::new()
foreach ($specRel in $specResults.Keys) {
    $r = $specResults[$specRel]
    $record = [ordered]@{
        specPath         = $specRel
        exitCode         = $r.exitCode
        assertionsPassed = $r.assertionsPassed
        assertionsFailed = $r.assertionsFailed
        durationMs       = $r.durationMs
        trials           = $r.trials
    }
    if ($r.ContainsKey('status')) { $record['status'] = $r.status }
    if ($r.ContainsKey('isAdvisory')) { $record['isAdvisory'] = [bool]$r.isAdvisory }
    if ($r.ContainsKey('perStimulusAdvisory') -and $null -ne $r.perStimulusAdvisory) {
        $record['advisoryPassed'] = [int]$r.advisoryPassed
        $record['advisoryFailed'] = [int]$r.advisoryFailed
        $record['authoritativePassed'] = [int]$r.authoritativePassed
        $record['authoritativeFailed'] = [int]$r.authoritativeFailed
        $record['perStimulusAdvisory'] = $r.perStimulusAdvisory
    }
    $perSpec.Add($record) | Out-Null
}

$totalPassed   = 0
$totalFailed   = 0
$totalDuration = 0
foreach ($a in $perArtifact) {
    $totalPassed   += [int]$a.assertionsPassed
    $totalFailed   += [int]$a.assertionsFailed
    $totalDuration += [int]$a.durationMs
}

$summary = [ordered]@{
    manifestPath = $resolvedManifest
    evalRoot     = $resolvedEvalRoot
    model        = $Model
    totals       = [ordered]@{
        artifacts        = $perArtifact.Count
        specs            = $perSpec.Count
        assertionsPassed = $totalPassed
        assertionsFailed = $totalFailed
        durationMs       = $totalDuration
        failedSpecs      = $failedSpecs
    }
    perArtifact  = @($perArtifact)
    perSpec      = @($perSpec)
    equivalence  = @($equivalenceResults)
}

Write-JsonFile -Value $summary -Path $summaryPath
Write-Host "Eval summary: $summaryPath ($($perArtifact.Count) artifact(s), $($perSpec.Count) spec(s); $failedSpecs failed spec(s))"

if ($failedSpecs -gt 0) { exit 1 }
exit 0
