microsoft/hve-core

Public

mirrored fromhttps://github.com/microsoft/hve-coreAvailable

CodeCommitsIssuesPull requestsActionsInsightsSecurity
feat/a11y-pr1-scripts-validators

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

scripts/collections/Validate-Collections.ps1

542lines · modecode

1#!/usr/bin/env pwsh
2# Copyright (c) Microsoft Corporation.
3# SPDX-License-Identifier: MIT
4#Requires -Version 7.0
5
6<#
7.SYNOPSIS
8 Validates collection manifests for Copilot CLI plugin generation.
9
10.DESCRIPTION
11 Reads all .collection.yml files from collections/ and validates structure,
12 required fields, artifact path existence, and kind-suffix consistency.
13
14.EXAMPLE
15 ./Validate-Collections.ps1
16#>
17
18[CmdletBinding()]
19param(
20 [Parameter()]
21 [string]$OutputPath = (Join-Path $PSScriptRoot '../../logs/collection-validation-results.json')
22)
23
24$ErrorActionPreference = 'Stop'
25
26Import-Module (Join-Path $PSScriptRoot 'Modules/CollectionHelpers.psm1') -Force
27Import-Module (Join-Path $PSScriptRoot '../lib/Modules/CIHelpers.psm1') -Force
28
29#region Validation Helpers
30
31function Test-KindSuffix {
32 <#
33 .SYNOPSIS
34 Validates that an item path matches its declared kind suffix.
35
36 .DESCRIPTION
37 Checks kind-suffix consistency: agent files end with .agent.md,
38 prompt files with .prompt.md, instruction files with .instructions.md,
39 and skill items are directories containing a SKILL.md file.
40
41 .PARAMETER Kind
42 The declared artifact kind (agent, prompt, instruction, skill).
43
44 .PARAMETER ItemPath
45 The relative path from the collection manifest.
46
47 .PARAMETER RepoRoot
48 Absolute path to the repository root for skill directory checks.
49
50 .OUTPUTS
51 [string] Error message if validation fails, empty string if valid.
52 #>
53 [CmdletBinding()]
54 [OutputType([string])]
55 param(
56 [Parameter(Mandatory = $true)]
57 [string]$Kind,
58
59 [Parameter(Mandatory = $true)]
60 [string]$ItemPath,
61
62 [Parameter(Mandatory = $true)]
63 [string]$RepoRoot
64 )
65
66 switch ($Kind) {
67 'agent' {
68 if ($ItemPath -notmatch '\.agent\.md$') {
69 return "kind 'agent' expects *.agent.md but got '$ItemPath'"
70 }
71 }
72 'prompt' {
73 if ($ItemPath -notmatch '\.prompt\.md$') {
74 return "kind 'prompt' expects *.prompt.md but got '$ItemPath'"
75 }
76 }
77 'instruction' {
78 if ($ItemPath -notmatch '\.instructions\.md$') {
79 return "kind 'instruction' expects *.instructions.md but got '$ItemPath'"
80 }
81 }
82 'skill' {
83 $skillDir = Join-Path -Path $RepoRoot -ChildPath $ItemPath
84 $skillFile = Join-Path -Path $skillDir -ChildPath 'SKILL.md'
85 if (-not (Test-Path -Path $skillFile)) {
86 return "kind 'skill' expects SKILL.md inside '$ItemPath'"
87 }
88 }
89 }
90
91 return ''
92}
93
94function Get-CollectionItemKey {
95 <#
96 .SYNOPSIS
97 Builds a stable uniqueness key for collection items.
98
99 .DESCRIPTION
100 Uses kind and path to identify the same artifact across collections.
101
102 .PARAMETER Kind
103 Artifact kind.
104
105 .PARAMETER ItemPath
106 Artifact path.
107
108 .OUTPUTS
109 [string] Composite key.
110 #>
111 [CmdletBinding()]
112 [OutputType([string])]
113 param(
114 [Parameter(Mandatory = $true)]
115 [string]$Kind,
116
117 [Parameter(Mandatory = $true)]
118 [string]$ItemPath
119 )
120
121 return "$Kind|$ItemPath"
122}
123
124#endregion Validation Helpers
125
126#region Orchestration
127
128function Invoke-CollectionValidation {
129 <#
130 .SYNOPSIS
131 Validates all collection manifests for correctness.
132
133 .DESCRIPTION
134 Scans the collections/ directory for .collection.yml files and validates
135 each manifest for required fields (id, name, description, items), id
136 format, artifact path existence, kind-suffix consistency, and duplicate
137 ids across collections.
138
139 .PARAMETER RepoRoot
140 Absolute path to the repository root directory.
141
142 .OUTPUTS
143 Hashtable with Success bool, ErrorCount int, CollectionCount int, and Results array.
144 #>
145 [CmdletBinding()]
146 [OutputType([hashtable])]
147 param(
148 [Parameter(Mandatory = $true)]
149 [ValidateNotNullOrEmpty()]
150 [string]$RepoRoot
151 )
152
153 $validationResults = [System.Collections.Generic.List[hashtable]]::new()
154
155 function Add-ValidationResult {
156 param(
157 [Parameter(Mandatory = $true)]
158 [string]$Collection,
159
160 [Parameter(Mandatory = $true)]
161 [string]$ErrorType,
162
163 [Parameter(Mandatory = $true)]
164 [string]$Message,
165
166 [Parameter()]
167 [ValidateSet('Error', 'Warning')]
168 [string]$Severity = 'Error'
169 )
170
171 $validationResults.Add(@{
172 Collection = $Collection
173 Severity = $Severity
174 ErrorType = $ErrorType
175 Message = $Message
176 })
177 }
178
179 $collectionsDir = Join-Path -Path $RepoRoot -ChildPath 'collections'
180 $collectionFiles = Get-ChildItem -Path $collectionsDir -Filter '*.collection.yml' -File
181
182 if ($collectionFiles.Count -eq 0) {
183 Write-Host ' WARN No collection manifests found in collections/' -ForegroundColor Yellow
184 Add-ValidationResult -Collection 'collections' -ErrorType 'NoCollectionManifests' -Message 'No collection manifests found in collections/' -Severity 'Warning'
185 return @{ Success = $true; ErrorCount = 0; CollectionCount = 0; Results = @($validationResults) }
186 }
187
188 Write-Host 'Validating collections...'
189
190 $errorCount = 0
191 $seenIds = @{}
192 $validatedCount = 0
193 $allowedMaturities = @('stable', 'preview', 'experimental', 'deprecated', 'removed')
194 $canonicalCollectionId = 'hve-core-all'
195 $itemOccurrences = @{}
196
197 $knownCollectionIds = @{}
198 foreach ($cf in $collectionFiles) {
199 $cfId = $cf.Name -replace '\.collection\.yml$', ''
200 $knownCollectionIds[$cfId] = $true
201 }
202
203 # Sub-domain folders that group artifacts shared across multiple themed collections
204 # but are intentionally not collections themselves.
205 $sharedSubdomainFolders = @{
206 'shared' = $true
207 'rai-planning' = $true
208 'accessibility' = $true
209 }
210
211 foreach ($file in $collectionFiles) {
212 $baseName = $file.Name -replace '\.collection\.yml$', ''
213 $collectionLabel = $baseName
214 $companionPath = Join-Path -Path $collectionsDir -ChildPath "$baseName.collection.md"
215 if (-not (Test-Path -Path $companionPath)) {
216 Write-Host " WARN $($file.Name): missing companion '$baseName.collection.md'" -ForegroundColor Yellow
217 Add-ValidationResult -Collection $collectionLabel -ErrorType 'MissingCompanionCollectionMd' -Message "missing companion '$baseName.collection.md'" -Severity 'Warning'
218 }
219
220 if (Test-Path -Path $companionPath) {
221 $mdContent = Get-Content -Path $companionPath -Raw
222 $hasBegin = $mdContent.Contains($CollectionMdBeginMarker)
223 $hasEnd = $mdContent.Contains($CollectionMdEndMarker)
224
225 if ($hasBegin -xor $hasEnd) {
226 Write-Host " WARN $($file.Name): $baseName.collection.md has mismatched auto-generation markers" -ForegroundColor Yellow
227 Add-ValidationResult -Collection $collectionLabel -ErrorType 'MismatchedAutoGenerationMarkers' -Message "$baseName.collection.md has mismatched auto-generation markers" -Severity 'Warning'
228 }
229
230 if ($hasBegin -and $hasEnd) {
231 $beginIdx = $mdContent.IndexOf($CollectionMdBeginMarker)
232 $endIdx = $mdContent.IndexOf($CollectionMdEndMarker)
233 if ($endIdx -le $beginIdx) {
234 Write-Host " WARN $($file.Name): $baseName.collection.md has markers in wrong order" -ForegroundColor Yellow
235 Add-ValidationResult -Collection $collectionLabel -ErrorType 'CollectionMarkersWrongOrder' -Message "$baseName.collection.md has markers in wrong order" -Severity 'Warning'
236 }
237 }
238 }
239
240 $manifest = Get-CollectionManifest -CollectionPath $file.FullName
241 $fileErrors = @()
242 $seenItemKeys = @{}
243
244 # Required fields
245 $requiredFields = @('id', 'name', 'description', 'items')
246 foreach ($field in $requiredFields) {
247 if (-not $manifest.ContainsKey($field) -or $null -eq $manifest[$field]) {
248 $fileErrors += @{ ErrorType = 'MissingRequiredField'; Message = "missing required field '$field'" }
249 }
250 }
251
252 # Skip further checks if required fields are absent
253 if ($fileErrors.Count -gt 0) {
254 foreach ($err in $fileErrors) {
255 Write-Host " x $($file.Name): $($err.Message)" -ForegroundColor Red
256 Add-ValidationResult -Collection $collectionLabel -ErrorType $err.ErrorType -Message $err.Message
257 }
258 $errorCount += $fileErrors.Count
259 continue
260 }
261
262 $id = $manifest.id
263 $collectionLabel = $id
264
265 # Id format
266 if ($id -notmatch '^[a-z0-9-]+$') {
267 $fileErrors += @{ ErrorType = 'InvalidIdFormat'; Message = "id '$id' must match ^[a-z0-9-]+$" }
268 }
269
270 # Duplicate id check
271 if ($seenIds.ContainsKey($id)) {
272 $fileErrors += @{ ErrorType = 'DuplicateCollectionId'; Message = "duplicate id '$id' (also in $($seenIds[$id]))" }
273 }
274 else {
275 $seenIds[$id] = $file.Name
276 }
277
278 # Validate collection-level maturity if present
279 if ($manifest.ContainsKey('maturity') -and -not [string]::IsNullOrWhiteSpace([string]$manifest.maturity)) {
280 $collMaturity = [string]$manifest.maturity
281 if ($allowedMaturities -notcontains $collMaturity) {
282 $fileErrors += @{ ErrorType = 'InvalidCollectionMaturity'; Message = "invalid collection maturity '$collMaturity' (allowed: $($allowedMaturities -join ', '))" }
283 }
284 }
285
286 # Validate each item
287 $itemCount = $manifest.items.Count
288 foreach ($item in $manifest.items) {
289 $itemPath = $item.path
290 $kind = $item.kind
291 $absolutePath = Join-Path -Path $RepoRoot -ChildPath $itemPath
292 $itemMaturity = $null
293 if ($item.ContainsKey('maturity')) {
294 $itemMaturity = [string]$item.maturity
295 }
296 $effectiveMaturity = Resolve-CollectionItemMaturity -Maturity $itemMaturity
297
298 # Repo-specific path exclusion
299 if (Test-HveCoreRepoRelativePath -Path $itemPath) {
300 $fileErrors += @{ ErrorType = 'RepoSpecificPath'; Message = "repo-specific path not allowed in collections: $itemPath (root-level artifacts under .github/{type}/ are excluded from distribution)" }
301 }
302
303 # Path existence
304 if (-not (Test-Path -Path $absolutePath)) {
305 $fileErrors += @{ ErrorType = 'PathNotFound'; Message = "path not found: $itemPath" }
306 }
307
308 # Kind-suffix consistency
309 if ($kind) {
310 $suffixError = Test-KindSuffix -Kind $kind -ItemPath $itemPath -RepoRoot $RepoRoot
311 if ($suffixError) {
312 $fileErrors += @{ ErrorType = 'MissingSuffix'; Message = $suffixError }
313 }
314 }
315 else {
316 $fileErrors += @{ ErrorType = 'MissingItemKind'; Message = "item missing 'kind': $itemPath" }
317 }
318
319 if (-not [string]::IsNullOrWhiteSpace($itemMaturity) -and ($allowedMaturities -notcontains $itemMaturity)) {
320 $fileErrors += @{ ErrorType = 'InvalidMaturity'; Message = "invalid maturity '$itemMaturity' for item '$itemPath' (allowed: $($allowedMaturities -join ', '))" }
321 }
322
323 # Check 2: intra-collection duplicate detection
324 if (-not [string]::IsNullOrWhiteSpace($itemPath) -and -not [string]::IsNullOrWhiteSpace($kind)) {
325 $dupKey = Get-CollectionItemKey -Kind $kind -ItemPath $itemPath
326 if ($seenItemKeys.ContainsKey($dupKey)) {
327 $fileErrors += @{ ErrorType = 'IntraCollectionDuplicate'; Message = "duplicate item '$dupKey' appears more than once in collection '$id'" }
328 } else {
329 $seenItemKeys[$dupKey] = $true
330 }
331 }
332
333 # Check 3: collection-id to folder name consistency
334 if ($id -ne 'hve-core-all') {
335 $pathSegments = $itemPath -split '[/\\]'
336 # Expected pattern: .github/{type}/{collection-id}/{file-or-deeper}
337 if ($pathSegments.Count -ge 4 -and $pathSegments[0] -eq '.github') {
338 $folderName = $pathSegments[2]
339 if (-not $sharedSubdomainFolders.ContainsKey($folderName) -and -not $knownCollectionIds.ContainsKey($folderName)) {
340 Write-Host " WARN collection '$id': item folder '$folderName' does not match any known collection ID: $itemPath" -ForegroundColor Yellow
341 Add-ValidationResult -Collection $collectionLabel -ErrorType 'UnknownCollectionFolderReference' -Message "item folder '$folderName' does not match any known collection ID: $itemPath" -Severity 'Warning'
342 }
343 }
344 }
345
346 if (-not [string]::IsNullOrWhiteSpace($itemPath) -and -not [string]::IsNullOrWhiteSpace($kind)) {
347 $itemKey = Get-CollectionItemKey -Kind $kind -ItemPath $itemPath
348 if (-not $itemOccurrences.ContainsKey($itemKey)) {
349 $itemOccurrences[$itemKey] = @()
350 }
351
352 $itemOccurrences[$itemKey] += @{
353 CollectionId = $id
354 CollectionFile = $file.Name
355 Kind = $kind
356 Path = $itemPath
357 Maturity = $effectiveMaturity
358 }
359 }
360
361 # Informational log for instruction items
362 if ($kind -eq 'instruction') {
363 Write-Verbose " instruction: $itemPath"
364 }
365 }
366
367 if ($fileErrors.Count -gt 0) {
368 Write-Host " FAIL $id ($itemCount items) - $($fileErrors.Count) error(s)" -ForegroundColor Red
369 foreach ($err in $fileErrors) {
370 Write-Host " $($err.Message)" -ForegroundColor Red
371 Add-ValidationResult -Collection $collectionLabel -ErrorType $err.ErrorType -Message $err.Message
372 }
373 $errorCount += $fileErrors.Count
374 }
375 else {
376 Write-Host " OK $id ($itemCount items)"
377 }
378
379 $validatedCount++
380 }
381
382 $canonicalManifestFound = ($collectionFiles | Where-Object {
383 ($_.Name -replace '\.collection\.yml$', '') -eq $canonicalCollectionId
384 }).Count -gt 0
385 if (-not $canonicalManifestFound) {
386 Write-Host " WARN '$canonicalCollectionId.collection.yml' not found; skipping orphan and cross-collection coverage checks" -ForegroundColor Yellow
387 Add-ValidationResult -Collection $canonicalCollectionId -ErrorType 'CanonicalCollectionMissing' -Message "'$canonicalCollectionId.collection.yml' not found; skipping orphan and cross-collection coverage checks" -Severity 'Warning'
388 }
389
390 # Duplicate artifact key detection across all collections
391 $artifactKeyMap = @{}
392 foreach ($itemKey in $itemOccurrences.Keys) {
393 $occurrences = $itemOccurrences[$itemKey]
394 $first = $occurrences[0]
395 $artifactKey = Get-CollectionArtifactKey -Kind $first.Kind -Path $first.Path
396 $compositeKey = "$($first.Kind)|$artifactKey"
397
398 if (-not $artifactKeyMap.ContainsKey($compositeKey)) {
399 $artifactKeyMap[$compositeKey] = @()
400 }
401 if ($artifactKeyMap[$compositeKey] -notcontains $first.Path) {
402 $artifactKeyMap[$compositeKey] += $first.Path
403 }
404 }
405
406 foreach ($compositeKey in $artifactKeyMap.Keys) {
407 $paths = $artifactKeyMap[$compositeKey]
408 if ($paths.Count -gt 1) {
409 $kindLabel = ($compositeKey -split '\|')[0]
410 $nameLabel = ($compositeKey -split '\|')[1]
411 $pathList = ($paths | Sort-Object) -join ', '
412 Write-Host " FAIL duplicate $kindLabel artifact key '$nameLabel' found at distinct paths: $pathList" -ForegroundColor Red
413 Add-ValidationResult -Collection 'all-collections' -ErrorType 'DuplicateArtifactKey' -Message "duplicate $kindLabel artifact key '$nameLabel' found at distinct paths: $pathList"
414 $errorCount++
415 }
416 }
417
418 foreach ($itemKey in $itemOccurrences.Keys) {
419 $occurrences = $itemOccurrences[$itemKey]
420 $canonicalMatches = @($occurrences | Where-Object { $_.CollectionId -eq $canonicalCollectionId })
421 $themedMatches = @($occurrences | Where-Object { $_.CollectionId -ne $canonicalCollectionId })
422
423 # Check 4: item in one or more themed collections but absent from hve-core-all
424 # Skip when all themed occurrences are marked maturity:'removed' (intentional tombstone
425 # excluded from hve-core-all by Update-HveCoreAllCollection).
426 $activeThemedMatches = @($themedMatches | Where-Object { $_.Maturity -ne 'removed' })
427 if ($canonicalManifestFound -and $activeThemedMatches.Count -gt 0 -and $canonicalMatches.Count -eq 0) {
428 $themedCollections = ($activeThemedMatches | ForEach-Object { $_.CollectionId } | Sort-Object -Unique) -join ', '
429 Write-Host " FAIL item '$itemKey' exists in themed collection(s) [$themedCollections] but is absent from '$canonicalCollectionId'" -ForegroundColor Red
430 Add-ValidationResult -Collection $canonicalCollectionId -ErrorType 'ThemedItemMissingFromCanonical' -Message "item '$itemKey' exists in themed collection(s) [$themedCollections] but is absent from '$canonicalCollectionId'"
431 $errorCount++
432 continue
433 }
434
435 # Maturity conflict: only when item appears in canonical AND at least one themed
436 if ($canonicalMatches.Count -gt 0 -and $themedMatches.Count -gt 0) {
437 $canonical = $canonicalMatches[0]
438 foreach ($occurrence in $themedMatches) {
439 if ($occurrence.Maturity -ne $canonical.Maturity) {
440 Write-Host " FAIL maturity conflict for '$itemKey': canonical '$canonicalCollectionId'='$($canonical.Maturity)', '$($occurrence.CollectionId)'='$($occurrence.Maturity)'" -ForegroundColor Red
441 Add-ValidationResult -Collection $occurrence.CollectionId -ErrorType 'MaturityConflict' -Message "maturity conflict for '$itemKey': canonical '$canonicalCollectionId'='$($canonical.Maturity)', '$($occurrence.CollectionId)'='$($occurrence.Maturity)'"
442 $errorCount++
443 }
444 }
445 }
446 }
447
448 if ($canonicalManifestFound) {
449 # Check 1: Orphan artifact detection
450 $onDiskArtifacts = Get-ArtifactFiles -RepoRoot $RepoRoot
451 foreach ($artifact in $onDiskArtifacts) {
452 $diskKey = Get-CollectionItemKey -Kind $artifact.kind -ItemPath $artifact.path
453 $occurrences = if ($itemOccurrences.ContainsKey($diskKey)) { $itemOccurrences[$diskKey] } else { @() }
454
455 $inCanonical = @($occurrences | Where-Object { $_.CollectionId -eq $canonicalCollectionId }).Count -gt 0
456 $inThemed = @($occurrences | Where-Object { $_.CollectionId -ne $canonicalCollectionId }).Count -gt 0
457
458 if (-not $inCanonical) {
459 # Skip orphan failure when all themed occurrences are tombstoned (maturity:'removed').
460 $themedActive = @($occurrences | Where-Object { $_.CollectionId -ne $canonicalCollectionId -and $_.Maturity -ne 'removed' }).Count -gt 0
461 $themedRemoved = @($occurrences | Where-Object { $_.CollectionId -ne $canonicalCollectionId -and $_.Maturity -eq 'removed' }).Count -gt 0
462 if ($themedRemoved -and -not $themedActive) {
463 Write-Verbose "Skipping orphan check for tombstoned item '$diskKey'"
464 } else {
465 Write-Host " FAIL orphan: '$diskKey' is on disk but absent from '$canonicalCollectionId'" -ForegroundColor Red
466 Add-ValidationResult -Collection $canonicalCollectionId -ErrorType 'OrphanArtifact' -Message "'$diskKey' is on disk but absent from '$canonicalCollectionId'"
467 $errorCount++
468 }
469 } elseif (-not $inThemed) {
470 Write-Host " WARN '$diskKey' exists in '$canonicalCollectionId' but is not in any themed collection" -ForegroundColor Yellow
471 Add-ValidationResult -Collection $canonicalCollectionId -ErrorType 'CanonicalOnlyArtifact' -Message "'$diskKey' exists in '$canonicalCollectionId' but is not in any themed collection" -Severity 'Warning'
472 }
473 }
474 }
475
476 Write-Host ''
477 Write-Host "$validatedCount collections validated, $errorCount errors"
478
479 return @{
480 Success = ($errorCount -eq 0)
481 ErrorCount = $errorCount
482 CollectionCount = $validatedCount
483 Results = @($validationResults)
484 }
485}
486
487function Export-CollectionValidationReport {
488 [CmdletBinding()]
489 param(
490 [Parameter(Mandatory = $true)]
491 [hashtable]$ValidationResult,
492
493 [Parameter(Mandatory = $true)]
494 [string]$OutputPath
495 )
496
497 $logsDir = Split-Path -Path $OutputPath -Parent
498 if (-not [string]::IsNullOrWhiteSpace($logsDir) -and -not (Test-Path -Path $logsDir)) {
499 New-Item -ItemType Directory -Path $logsDir -Force | Out-Null
500 }
501
502 $report = @{
503 Timestamp = (Get-Date).ToUniversalTime().ToString('o')
504 TotalCollections = $ValidationResult.CollectionCount
505 ErrorCount = $ValidationResult.ErrorCount
506 Results = @($ValidationResult.Results)
507 }
508
509 $report | ConvertTo-Json -Depth 10 | Set-Content -Path $OutputPath -Encoding utf8
510}
511
512#endregion Orchestration
513
514#region Main Execution
515if ($MyInvocation.InvocationName -ne '.') {
516 try {
517 # Verify PowerShell-Yaml module
518 if (-not (Get-Module -ListAvailable -Name PowerShell-Yaml)) {
519 throw "Required module 'PowerShell-Yaml' is not installed."
520 }
521 Import-Module PowerShell-Yaml -ErrorAction Stop
522
523 # Resolve paths
524 $ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
525 $RepoRoot = (Get-Item "$ScriptDir/../..").FullName
526
527 $result = Invoke-CollectionValidation -RepoRoot $RepoRoot
528 Export-CollectionValidationReport -ValidationResult $result -OutputPath $OutputPath
529
530 if (-not $result.Success) {
531 throw "Validation failed with $($result.ErrorCount) error(s)."
532 }
533
534 exit 0
535 }
536 catch {
537 Write-Error "Collection validation failed: $($_.Exception.Message)"
538 Write-CIAnnotation -Message $_.Exception.Message -Level Error
539 exit 1
540 }
541}
542#endregion
543