name: Evals - Per-Agent Matrix (On-Demand)

# Manual full-matrix dispatch for the agent-behavior suite. Honors the
# 2026-05-24 cross-plan rule that prohibits scheduled eval jobs: this workflow
# is invoked on demand via the GitHub UI / API and uses Tier nightly exit
# policy (exit 1 on any per-agent overall: fail).

on:
  workflow_dispatch:
    inputs:
      tier:
        description: "Exit policy tier (pr=advisory, nightly=strict)."
        required: false
        default: "nightly"
        type: choice
        options:
          - nightly
          - pr

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  agent-matrix:
    name: Per-Agent Matrix
    runs-on: ubuntu-latest
    permissions:
      contents: read
    env:
      COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          persist-credentials: false
          fetch-depth: 0

      - name: Setup Node.js
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
        with:
          node-version: "24"
          cache: "npm"

      - name: Install npm dependencies
        run: npm ci

      - name: Install PowerShell-Yaml
        shell: pwsh
        run: |
          Install-Module -Name PowerShell-Yaml -RequiredVersion 0.4.7 -Force -Scope CurrentUser

      - name: Set up Python
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "3.11"

      - name: Install uv
        uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
        with:
          version: "0.10.9"

      - name: Create logs directory
        shell: pwsh
        run: New-Item -ItemType Directory -Force -Path logs | Out-Null

      - name: Configure Copilot home
        shell: pwsh
        run: |
          $copilotHome = Join-Path $env:RUNNER_TEMP 'copilot-home'
          New-Item -ItemType Directory -Force -Path $copilotHome | Out-Null
          "COPILOT_HOME=$copilotHome" | Out-File -FilePath $env:GITHUB_ENV -Append

      - name: Verify Copilot token
        shell: pwsh
        run: pwsh -NoProfile -File scripts/evals/Test-CopilotToken.ps1

      - name: Run per-agent agent-behavior matrix (all)
        shell: pwsh
        env:
          MATRIX_TIER: ${{ inputs.tier }}
        run: |
          pwsh -NoProfile -File scripts/evals/Invoke-AgentMatrix.ps1 `
            -All `
            -Tier $env:MATRIX_TIER

      - name: Render per-agent matrix dashboard
        if: always()
        shell: pwsh
        run: |
          pwsh -NoProfile -File scripts/evals/New-AgentMatrixDashboard.ps1

      - name: Upload matrix artifacts
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: agent-matrix-results
          path: |
            logs/agent-matrix/**/*.log
            logs/agent-matrix-dashboard.html
            evals/results/agent-matrix/**/*.json
          if-no-files-found: ignore
          retention-days: 30
