microsoft/hve-core

Public

mirrored fromhttps://github.com/microsoft/hve-coreAvailable

CodeCommitsIssuesPull requestsActionsInsightsSecurity
feat/context-working

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

scripts/collections/Validate-Collections.ps1

541lines · modecode

1#!/usr/bin/env pwsh
2# Copyright (c) Microsoft Corporation.
3# SPDX-License-Identifier: MIT
4#Requires -Version 7.0
5
6<#
7.SYNOPSIS
8 Validates collection manifests for Copilot CLI plugin generation.
9
10.DESCRIPTION
11 Reads all .collection.yml files from collections/ and validates structure,
12 required fields, artifact path existence, and kind-suffix consistency.
13
14.EXAMPLE
15 ./Validate-Collections.ps1
16#>
17
18[CmdletBinding()]
19param(
20 [Parameter()]
21 [string]$OutputPath = (Join-Path $PSScriptRoot '../../logs/collection-validation-results.json')
22)
23
24$ErrorActionPreference = 'Stop'
25
26Import-Module (Join-Path $PSScriptRoot 'Modules/CollectionHelpers.psm1') -Force
27Import-Module (Join-Path $PSScriptRoot '../lib/Modules/CIHelpers.psm1') -Force
28
29#region Validation Helpers
30
31function Test-KindSuffix {
32 <#
33 .SYNOPSIS
34 Validates that an item path matches its declared kind suffix.
35
36 .DESCRIPTION
37 Checks kind-suffix consistency: agent files end with .agent.md,
38 prompt files with .prompt.md, instruction files with .instructions.md,
39 and skill items are directories containing a SKILL.md file.
40
41 .PARAMETER Kind
42 The declared artifact kind (agent, prompt, instruction, skill).
43
44 .PARAMETER ItemPath
45 The relative path from the collection manifest.
46
47 .PARAMETER RepoRoot
48 Absolute path to the repository root for skill directory checks.
49
50 .OUTPUTS
51 [string] Error message if validation fails, empty string if valid.
52 #>
53 [CmdletBinding()]
54 [OutputType([string])]
55 param(
56 [Parameter(Mandatory = $true)]
57 [string]$Kind,
58
59 [Parameter(Mandatory = $true)]
60 [string]$ItemPath,
61
62 [Parameter(Mandatory = $true)]
63 [string]$RepoRoot
64 )
65
66 switch ($Kind) {
67 'agent' {
68 if ($ItemPath -notmatch '\.agent\.md$') {
69 return "kind 'agent' expects *.agent.md but got '$ItemPath'"
70 }
71 }
72 'prompt' {
73 if ($ItemPath -notmatch '\.prompt\.md$') {
74 return "kind 'prompt' expects *.prompt.md but got '$ItemPath'"
75 }
76 }
77 'instruction' {
78 if ($ItemPath -notmatch '\.instructions\.md$') {
79 return "kind 'instruction' expects *.instructions.md but got '$ItemPath'"
80 }
81 }
82 'skill' {
83 $skillDir = Join-Path -Path $RepoRoot -ChildPath $ItemPath
84 $skillFile = Join-Path -Path $skillDir -ChildPath 'SKILL.md'
85 if (-not (Test-Path -Path $skillFile)) {
86 return "kind 'skill' expects SKILL.md inside '$ItemPath'"
87 }
88 }
89 }
90
91 return ''
92}
93
94function Get-CollectionItemKey {
95 <#
96 .SYNOPSIS
97 Builds a stable uniqueness key for collection items.
98
99 .DESCRIPTION
100 Uses kind and path to identify the same artifact across collections.
101
102 .PARAMETER Kind
103 Artifact kind.
104
105 .PARAMETER ItemPath
106 Artifact path.
107
108 .OUTPUTS
109 [string] Composite key.
110 #>
111 [CmdletBinding()]
112 [OutputType([string])]
113 param(
114 [Parameter(Mandatory = $true)]
115 [string]$Kind,
116
117 [Parameter(Mandatory = $true)]
118 [string]$ItemPath
119 )
120
121 return "$Kind|$ItemPath"
122}
123
124#endregion Validation Helpers
125
126#region Orchestration
127
128function Invoke-CollectionValidation {
129 <#
130 .SYNOPSIS
131 Validates all collection manifests for correctness.
132
133 .DESCRIPTION
134 Scans the collections/ directory for .collection.yml files and validates
135 each manifest for required fields (id, name, description, items), id
136 format, artifact path existence, kind-suffix consistency, and duplicate
137 ids across collections.
138
139 .PARAMETER RepoRoot
140 Absolute path to the repository root directory.
141
142 .OUTPUTS
143 Hashtable with Success bool, ErrorCount int, CollectionCount int, and Results array.
144 #>
145 [CmdletBinding()]
146 [OutputType([hashtable])]
147 param(
148 [Parameter(Mandatory = $true)]
149 [ValidateNotNullOrEmpty()]
150 [string]$RepoRoot
151 )
152
153 $validationResults = [System.Collections.Generic.List[hashtable]]::new()
154
155 function Add-ValidationResult {
156 param(
157 [Parameter(Mandatory = $true)]
158 [string]$Collection,
159
160 [Parameter(Mandatory = $true)]
161 [string]$ErrorType,
162
163 [Parameter(Mandatory = $true)]
164 [string]$Message,
165
166 [Parameter()]
167 [ValidateSet('Error', 'Warning')]
168 [string]$Severity = 'Error'
169 )
170
171 $validationResults.Add(@{
172 Collection = $Collection
173 Severity = $Severity
174 ErrorType = $ErrorType
175 Message = $Message
176 })
177 }
178
179 $collectionsDir = Join-Path -Path $RepoRoot -ChildPath 'collections'
180 $collectionFiles = Get-ChildItem -Path $collectionsDir -Filter '*.collection.yml' -File
181
182 if ($collectionFiles.Count -eq 0) {
183 Write-Host ' WARN No collection manifests found in collections/' -ForegroundColor Yellow
184 Add-ValidationResult -Collection 'collections' -ErrorType 'NoCollectionManifests' -Message 'No collection manifests found in collections/' -Severity 'Warning'
185 return @{ Success = $true; ErrorCount = 0; CollectionCount = 0; Results = @($validationResults) }
186 }
187
188 Write-Host 'Validating collections...'
189
190 $errorCount = 0
191 $seenIds = @{}
192 $validatedCount = 0
193 $allowedMaturities = @('stable', 'preview', 'experimental', 'deprecated', 'removed')
194 $canonicalCollectionId = 'hve-core-all'
195 $itemOccurrences = @{}
196
197 $knownCollectionIds = @{}
198 foreach ($cf in $collectionFiles) {
199 $cfId = $cf.Name -replace '\.collection\.yml$', ''
200 $knownCollectionIds[$cfId] = $true
201 }
202
203 # Sub-domain folders that group artifacts shared across multiple themed collections
204 # but are intentionally not collections themselves.
205 $sharedSubdomainFolders = @{
206 'shared' = $true
207 'rai-planning' = $true
208 }
209
210 foreach ($file in $collectionFiles) {
211 $baseName = $file.Name -replace '\.collection\.yml$', ''
212 $collectionLabel = $baseName
213 $companionPath = Join-Path -Path $collectionsDir -ChildPath "$baseName.collection.md"
214 if (-not (Test-Path -Path $companionPath)) {
215 Write-Host " WARN $($file.Name): missing companion '$baseName.collection.md'" -ForegroundColor Yellow
216 Add-ValidationResult -Collection $collectionLabel -ErrorType 'MissingCompanionCollectionMd' -Message "missing companion '$baseName.collection.md'" -Severity 'Warning'
217 }
218
219 if (Test-Path -Path $companionPath) {
220 $mdContent = Get-Content -Path $companionPath -Raw
221 $hasBegin = $mdContent.Contains($CollectionMdBeginMarker)
222 $hasEnd = $mdContent.Contains($CollectionMdEndMarker)
223
224 if ($hasBegin -xor $hasEnd) {
225 Write-Host " WARN $($file.Name): $baseName.collection.md has mismatched auto-generation markers" -ForegroundColor Yellow
226 Add-ValidationResult -Collection $collectionLabel -ErrorType 'MismatchedAutoGenerationMarkers' -Message "$baseName.collection.md has mismatched auto-generation markers" -Severity 'Warning'
227 }
228
229 if ($hasBegin -and $hasEnd) {
230 $beginIdx = $mdContent.IndexOf($CollectionMdBeginMarker)
231 $endIdx = $mdContent.IndexOf($CollectionMdEndMarker)
232 if ($endIdx -le $beginIdx) {
233 Write-Host " WARN $($file.Name): $baseName.collection.md has markers in wrong order" -ForegroundColor Yellow
234 Add-ValidationResult -Collection $collectionLabel -ErrorType 'CollectionMarkersWrongOrder' -Message "$baseName.collection.md has markers in wrong order" -Severity 'Warning'
235 }
236 }
237 }
238
239 $manifest = Get-CollectionManifest -CollectionPath $file.FullName
240 $fileErrors = @()
241 $seenItemKeys = @{}
242
243 # Required fields
244 $requiredFields = @('id', 'name', 'description', 'items')
245 foreach ($field in $requiredFields) {
246 if (-not $manifest.ContainsKey($field) -or $null -eq $manifest[$field]) {
247 $fileErrors += @{ ErrorType = 'MissingRequiredField'; Message = "missing required field '$field'" }
248 }
249 }
250
251 # Skip further checks if required fields are absent
252 if ($fileErrors.Count -gt 0) {
253 foreach ($err in $fileErrors) {
254 Write-Host " x $($file.Name): $($err.Message)" -ForegroundColor Red
255 Add-ValidationResult -Collection $collectionLabel -ErrorType $err.ErrorType -Message $err.Message
256 }
257 $errorCount += $fileErrors.Count
258 continue
259 }
260
261 $id = $manifest.id
262 $collectionLabel = $id
263
264 # Id format
265 if ($id -notmatch '^[a-z0-9-]+$') {
266 $fileErrors += @{ ErrorType = 'InvalidIdFormat'; Message = "id '$id' must match ^[a-z0-9-]+$" }
267 }
268
269 # Duplicate id check
270 if ($seenIds.ContainsKey($id)) {
271 $fileErrors += @{ ErrorType = 'DuplicateCollectionId'; Message = "duplicate id '$id' (also in $($seenIds[$id]))" }
272 }
273 else {
274 $seenIds[$id] = $file.Name
275 }
276
277 # Validate collection-level maturity if present
278 if ($manifest.ContainsKey('maturity') -and -not [string]::IsNullOrWhiteSpace([string]$manifest.maturity)) {
279 $collMaturity = [string]$manifest.maturity
280 if ($allowedMaturities -notcontains $collMaturity) {
281 $fileErrors += @{ ErrorType = 'InvalidCollectionMaturity'; Message = "invalid collection maturity '$collMaturity' (allowed: $($allowedMaturities -join ', '))" }
282 }
283 }
284
285 # Validate each item
286 $itemCount = $manifest.items.Count
287 foreach ($item in $manifest.items) {
288 $itemPath = $item.path
289 $kind = $item.kind
290 $absolutePath = Join-Path -Path $RepoRoot -ChildPath $itemPath
291 $itemMaturity = $null
292 if ($item.ContainsKey('maturity')) {
293 $itemMaturity = [string]$item.maturity
294 }
295 $effectiveMaturity = Resolve-CollectionItemMaturity -Maturity $itemMaturity
296
297 # Repo-specific path exclusion
298 if (Test-HveCoreRepoRelativePath -Path $itemPath) {
299 $fileErrors += @{ ErrorType = 'RepoSpecificPath'; Message = "repo-specific path not allowed in collections: $itemPath (root-level artifacts under .github/{type}/ are excluded from distribution)" }
300 }
301
302 # Path existence
303 if (-not (Test-Path -Path $absolutePath)) {
304 $fileErrors += @{ ErrorType = 'PathNotFound'; Message = "path not found: $itemPath" }
305 }
306
307 # Kind-suffix consistency
308 if ($kind) {
309 $suffixError = Test-KindSuffix -Kind $kind -ItemPath $itemPath -RepoRoot $RepoRoot
310 if ($suffixError) {
311 $fileErrors += @{ ErrorType = 'MissingSuffix'; Message = $suffixError }
312 }
313 }
314 else {
315 $fileErrors += @{ ErrorType = 'MissingItemKind'; Message = "item missing 'kind': $itemPath" }
316 }
317
318 if (-not [string]::IsNullOrWhiteSpace($itemMaturity) -and ($allowedMaturities -notcontains $itemMaturity)) {
319 $fileErrors += @{ ErrorType = 'InvalidMaturity'; Message = "invalid maturity '$itemMaturity' for item '$itemPath' (allowed: $($allowedMaturities -join ', '))" }
320 }
321
322 # Check 2: intra-collection duplicate detection
323 if (-not [string]::IsNullOrWhiteSpace($itemPath) -and -not [string]::IsNullOrWhiteSpace($kind)) {
324 $dupKey = Get-CollectionItemKey -Kind $kind -ItemPath $itemPath
325 if ($seenItemKeys.ContainsKey($dupKey)) {
326 $fileErrors += @{ ErrorType = 'IntraCollectionDuplicate'; Message = "duplicate item '$dupKey' appears more than once in collection '$id'" }
327 } else {
328 $seenItemKeys[$dupKey] = $true
329 }
330 }
331
332 # Check 3: collection-id to folder name consistency
333 if ($id -ne 'hve-core-all') {
334 $pathSegments = $itemPath -split '[/\\]'
335 # Expected pattern: .github/{type}/{collection-id}/{file-or-deeper}
336 if ($pathSegments.Count -ge 4 -and $pathSegments[0] -eq '.github') {
337 $folderName = $pathSegments[2]
338 if (-not $sharedSubdomainFolders.ContainsKey($folderName) -and -not $knownCollectionIds.ContainsKey($folderName)) {
339 Write-Host " WARN collection '$id': item folder '$folderName' does not match any known collection ID: $itemPath" -ForegroundColor Yellow
340 Add-ValidationResult -Collection $collectionLabel -ErrorType 'UnknownCollectionFolderReference' -Message "item folder '$folderName' does not match any known collection ID: $itemPath" -Severity 'Warning'
341 }
342 }
343 }
344
345 if (-not [string]::IsNullOrWhiteSpace($itemPath) -and -not [string]::IsNullOrWhiteSpace($kind)) {
346 $itemKey = Get-CollectionItemKey -Kind $kind -ItemPath $itemPath
347 if (-not $itemOccurrences.ContainsKey($itemKey)) {
348 $itemOccurrences[$itemKey] = @()
349 }
350
351 $itemOccurrences[$itemKey] += @{
352 CollectionId = $id
353 CollectionFile = $file.Name
354 Kind = $kind
355 Path = $itemPath
356 Maturity = $effectiveMaturity
357 }
358 }
359
360 # Informational log for instruction items
361 if ($kind -eq 'instruction') {
362 Write-Verbose " instruction: $itemPath"
363 }
364 }
365
366 if ($fileErrors.Count -gt 0) {
367 Write-Host " FAIL $id ($itemCount items) - $($fileErrors.Count) error(s)" -ForegroundColor Red
368 foreach ($err in $fileErrors) {
369 Write-Host " $($err.Message)" -ForegroundColor Red
370 Add-ValidationResult -Collection $collectionLabel -ErrorType $err.ErrorType -Message $err.Message
371 }
372 $errorCount += $fileErrors.Count
373 }
374 else {
375 Write-Host " OK $id ($itemCount items)"
376 }
377
378 $validatedCount++
379 }
380
381 $canonicalManifestFound = ($collectionFiles | Where-Object {
382 ($_.Name -replace '\.collection\.yml$', '') -eq $canonicalCollectionId
383 }).Count -gt 0
384 if (-not $canonicalManifestFound) {
385 Write-Host " WARN '$canonicalCollectionId.collection.yml' not found; skipping orphan and cross-collection coverage checks" -ForegroundColor Yellow
386 Add-ValidationResult -Collection $canonicalCollectionId -ErrorType 'CanonicalCollectionMissing' -Message "'$canonicalCollectionId.collection.yml' not found; skipping orphan and cross-collection coverage checks" -Severity 'Warning'
387 }
388
389 # Duplicate artifact key detection across all collections
390 $artifactKeyMap = @{}
391 foreach ($itemKey in $itemOccurrences.Keys) {
392 $occurrences = $itemOccurrences[$itemKey]
393 $first = $occurrences[0]
394 $artifactKey = Get-CollectionArtifactKey -Kind $first.Kind -Path $first.Path
395 $compositeKey = "$($first.Kind)|$artifactKey"
396
397 if (-not $artifactKeyMap.ContainsKey($compositeKey)) {
398 $artifactKeyMap[$compositeKey] = @()
399 }
400 if ($artifactKeyMap[$compositeKey] -notcontains $first.Path) {
401 $artifactKeyMap[$compositeKey] += $first.Path
402 }
403 }
404
405 foreach ($compositeKey in $artifactKeyMap.Keys) {
406 $paths = $artifactKeyMap[$compositeKey]
407 if ($paths.Count -gt 1) {
408 $kindLabel = ($compositeKey -split '\|')[0]
409 $nameLabel = ($compositeKey -split '\|')[1]
410 $pathList = ($paths | Sort-Object) -join ', '
411 Write-Host " FAIL duplicate $kindLabel artifact key '$nameLabel' found at distinct paths: $pathList" -ForegroundColor Red
412 Add-ValidationResult -Collection 'all-collections' -ErrorType 'DuplicateArtifactKey' -Message "duplicate $kindLabel artifact key '$nameLabel' found at distinct paths: $pathList"
413 $errorCount++
414 }
415 }
416
417 foreach ($itemKey in $itemOccurrences.Keys) {
418 $occurrences = $itemOccurrences[$itemKey]
419 $canonicalMatches = @($occurrences | Where-Object { $_.CollectionId -eq $canonicalCollectionId })
420 $themedMatches = @($occurrences | Where-Object { $_.CollectionId -ne $canonicalCollectionId })
421
422 # Check 4: item in one or more themed collections but absent from hve-core-all
423 # Skip when all themed occurrences are marked maturity:'removed' (intentional tombstone
424 # excluded from hve-core-all by Update-HveCoreAllCollection).
425 $activeThemedMatches = @($themedMatches | Where-Object { $_.Maturity -ne 'removed' })
426 if ($canonicalManifestFound -and $activeThemedMatches.Count -gt 0 -and $canonicalMatches.Count -eq 0) {
427 $themedCollections = ($activeThemedMatches | ForEach-Object { $_.CollectionId } | Sort-Object -Unique) -join ', '
428 Write-Host " FAIL item '$itemKey' exists in themed collection(s) [$themedCollections] but is absent from '$canonicalCollectionId'" -ForegroundColor Red
429 Add-ValidationResult -Collection $canonicalCollectionId -ErrorType 'ThemedItemMissingFromCanonical' -Message "item '$itemKey' exists in themed collection(s) [$themedCollections] but is absent from '$canonicalCollectionId'"
430 $errorCount++
431 continue
432 }
433
434 # Maturity conflict: only when item appears in canonical AND at least one themed
435 if ($canonicalMatches.Count -gt 0 -and $themedMatches.Count -gt 0) {
436 $canonical = $canonicalMatches[0]
437 foreach ($occurrence in $themedMatches) {
438 if ($occurrence.Maturity -ne $canonical.Maturity) {
439 Write-Host " FAIL maturity conflict for '$itemKey': canonical '$canonicalCollectionId'='$($canonical.Maturity)', '$($occurrence.CollectionId)'='$($occurrence.Maturity)'" -ForegroundColor Red
440 Add-ValidationResult -Collection $occurrence.CollectionId -ErrorType 'MaturityConflict' -Message "maturity conflict for '$itemKey': canonical '$canonicalCollectionId'='$($canonical.Maturity)', '$($occurrence.CollectionId)'='$($occurrence.Maturity)'"
441 $errorCount++
442 }
443 }
444 }
445 }
446
447 if ($canonicalManifestFound) {
448 # Check 1: Orphan artifact detection
449 $onDiskArtifacts = Get-ArtifactFiles -RepoRoot $RepoRoot
450 foreach ($artifact in $onDiskArtifacts) {
451 $diskKey = Get-CollectionItemKey -Kind $artifact.kind -ItemPath $artifact.path
452 $occurrences = if ($itemOccurrences.ContainsKey($diskKey)) { $itemOccurrences[$diskKey] } else { @() }
453
454 $inCanonical = @($occurrences | Where-Object { $_.CollectionId -eq $canonicalCollectionId }).Count -gt 0
455 $inThemed = @($occurrences | Where-Object { $_.CollectionId -ne $canonicalCollectionId }).Count -gt 0
456
457 if (-not $inCanonical) {
458 # Skip orphan failure when all themed occurrences are tombstoned (maturity:'removed').
459 $themedActive = @($occurrences | Where-Object { $_.CollectionId -ne $canonicalCollectionId -and $_.Maturity -ne 'removed' }).Count -gt 0
460 $themedRemoved = @($occurrences | Where-Object { $_.CollectionId -ne $canonicalCollectionId -and $_.Maturity -eq 'removed' }).Count -gt 0
461 if ($themedRemoved -and -not $themedActive) {
462 Write-Verbose "Skipping orphan check for tombstoned item '$diskKey'"
463 } else {
464 Write-Host " FAIL orphan: '$diskKey' is on disk but absent from '$canonicalCollectionId'" -ForegroundColor Red
465 Add-ValidationResult -Collection $canonicalCollectionId -ErrorType 'OrphanArtifact' -Message "'$diskKey' is on disk but absent from '$canonicalCollectionId'"
466 $errorCount++
467 }
468 } elseif (-not $inThemed) {
469 Write-Host " WARN '$diskKey' exists in '$canonicalCollectionId' but is not in any themed collection" -ForegroundColor Yellow
470 Add-ValidationResult -Collection $canonicalCollectionId -ErrorType 'CanonicalOnlyArtifact' -Message "'$diskKey' exists in '$canonicalCollectionId' but is not in any themed collection" -Severity 'Warning'
471 }
472 }
473 }
474
475 Write-Host ''
476 Write-Host "$validatedCount collections validated, $errorCount errors"
477
478 return @{
479 Success = ($errorCount -eq 0)
480 ErrorCount = $errorCount
481 CollectionCount = $validatedCount
482 Results = @($validationResults)
483 }
484}
485
486function Export-CollectionValidationReport {
487 [CmdletBinding()]
488 param(
489 [Parameter(Mandatory = $true)]
490 [hashtable]$ValidationResult,
491
492 [Parameter(Mandatory = $true)]
493 [string]$OutputPath
494 )
495
496 $logsDir = Split-Path -Path $OutputPath -Parent
497 if (-not [string]::IsNullOrWhiteSpace($logsDir) -and -not (Test-Path -Path $logsDir)) {
498 New-Item -ItemType Directory -Path $logsDir -Force | Out-Null
499 }
500
501 $report = @{
502 Timestamp = (Get-Date).ToUniversalTime().ToString('o')
503 TotalCollections = $ValidationResult.CollectionCount
504 ErrorCount = $ValidationResult.ErrorCount
505 Results = @($ValidationResult.Results)
506 }
507
508 $report | ConvertTo-Json -Depth 10 | Set-Content -Path $OutputPath -Encoding utf8
509}
510
511#endregion Orchestration
512
513#region Main Execution
514if ($MyInvocation.InvocationName -ne '.') {
515 try {
516 # Verify PowerShell-Yaml module
517 if (-not (Get-Module -ListAvailable -Name PowerShell-Yaml)) {
518 throw "Required module 'PowerShell-Yaml' is not installed."
519 }
520 Import-Module PowerShell-Yaml -ErrorAction Stop
521
522 # Resolve paths
523 $ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
524 $RepoRoot = (Get-Item "$ScriptDir/../..").FullName
525
526 $result = Invoke-CollectionValidation -RepoRoot $RepoRoot
527 Export-CollectionValidationReport -ValidationResult $result -OutputPath $OutputPath
528
529 if (-not $result.Success) {
530 throw "Validation failed with $($result.ErrorCount) error(s)."
531 }
532
533 exit 0
534 }
535 catch {
536 Write-Error "Collection validation failed: $($_.Exception.Message)"
537 Write-CIAnnotation -Message $_.Exception.Message -Level Error
538 exit 1
539 }
540}
541#endregion
542