GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

microsoft/TypeAgent

Public

mirrored fromhttps://github.com/microsoft/TypeAgentAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
copilot/fix-shell-and-cli-windows-job

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

TypeAgent/ts/packages/agentSdkWrapper

ts/packages/agentSdkWrapper/testGrammar.js

182lines · modecode

RawDownload
Latest commit unavailable.
1#!/usr/bin/env node
2// Copyright (c) Microsoft Corporation.
3// Licensed under the MIT License.
4
5// Flexible test script for grammar generation
6// Usage:
7// node testGrammar.js <test-file> [test-index or test-range]
8// Examples:
9// node testGrammar.js calendarTests.jsonl # Run all tests
10// node testGrammar.js calendarTests.jsonl 5 # Run test 5 only
11// node testGrammar.js calendarTests.jsonl 3-7 # Run tests 3-7
12// node testGrammar.js playerTests.jsonl 1,3,5 # Run tests 1, 3, and 5
13
14import { ClaudeGrammarGenerator } from "./dist/grammarGenerator.js";
15import { loadTestCases, loadSchemasForTests } from "./dist/testRunner.js";
16
17function parseTestSelection(selectionStr, totalTests) {
18 if (!selectionStr) {
19 // Run all tests
20 return Array.from({ length: totalTests }, (_, i) => i);
21 }
22
23 const indices = [];
24
25 // Handle comma-separated indices: "1,3,5"
26 if (selectionStr.includes(",")) {
27 const parts = selectionStr.split(",");
28 for (const part of parts) {
29 const index = parseInt(part.trim()) - 1; // Convert to 0-based
30 if (index >= 0 && index < totalTests) {
31 indices.push(index);
32 }
33 }
34 return indices;
35 }
36
37 // Handle range: "3-7"
38 if (selectionStr.includes("-")) {
39 const [start, end] = selectionStr
40 .split("-")
41 .map((s) => parseInt(s.trim()) - 1);
42 if (start >= 0 && end < totalTests && start <= end) {
43 for (let i = start; i <= end; i++) {
44 indices.push(i);
45 }
46 }
47 return indices;
48 }
49
50 // Handle single index: "5"
51 const index = parseInt(selectionStr) - 1;
52 if (index >= 0 && index < totalTests) {
53 return [index];
54 }
55
56 return [];
57}
58
59async function main() {
60 const args = process.argv.slice(2);
61 if (args.length < 1) {
62 console.error(
63 "Usage: node testGrammar.js <test-file> [test-index or test-range]",
64 );
65 console.error("Examples:");
66 console.error(
67 " node testGrammar.js calendarTests.jsonl # Run all tests",
68 );
69 console.error(
70 " node testGrammar.js calendarTests.jsonl 5 # Run test 5 only",
71 );
72 console.error(
73 " node testGrammar.js calendarTests.jsonl 3-7 # Run tests 3-7",
74 );
75 console.error(
76 " node testGrammar.js calendarTests.jsonl 1,3,5 # Run tests 1, 3, and 5",
77 );
78 process.exit(1);
79 }
80
81 const testFile = args[0];
82 const testSelection = args[1];
83 const testPath =
84 testFile.includes("/") || testFile.includes("\\")
85 ? testFile
86 : `./tests/${testFile}`;
87
88 console.log(`Loading test cases from ${testPath}...`);
89 const allTestCases = loadTestCases(testPath);
90 console.log(`Loaded ${allTestCases.length} test cases\n`);
91
92 const selectedIndices = parseTestSelection(
93 testSelection,
94 allTestCases.length,
95 );
96 if (selectedIndices.length === 0) {
97 console.error("No valid tests selected");
98 process.exit(1);
99 }
100
101 const testCases = selectedIndices.map((i) => allTestCases[i]);
102 console.log(
103 `Running ${testCases.length} of ${allTestCases.length} tests\n`,
104 );
105
106 console.log("Loading schemas...");
107 const schemas = loadSchemasForTests(testCases);
108 console.log(`Loaded ${schemas.size} schemas\n`);
109
110 const generator = new ClaudeGrammarGenerator();
111
112 for (const [arrayIndex, testCase] of testCases.entries()) {
113 const originalIndex = selectedIndices[arrayIndex];
114 console.log(`\n${"=".repeat(80)}`);
115 console.log(
116 `Test ${originalIndex + 1}/${allTestCases.length}: "${testCase.request}"`,
117 );
118 console.log(`Action: ${testCase.action.actionName}`);
119 console.log(
120 `Parameters: ${JSON.stringify(testCase.action.parameters)}`,
121 );
122 console.log(`Note: ${testCase.note || "N/A"}`);
123 console.log(`${"=".repeat(80)}`);
124
125 try {
126 const schemaInfo = schemas.get(testCase.schemaName);
127 if (!schemaInfo) {
128 console.error(`❌ Schema not found: ${testCase.schemaName}`);
129 continue;
130 }
131
132 const analysis = await generator.generateGrammar(
133 testCase,
134 schemaInfo,
135 );
136
137 if (analysis.shouldGenerateGrammar) {
138 console.log("\n✅ ACCEPTED - Grammar should be generated");
139 console.log("\nGrammar Pattern:");
140 console.log(` ${analysis.grammarPattern}`);
141
142 console.log("\nParameter Mappings:");
143 for (const mapping of analysis.parameterMappings) {
144 console.log(` ${mapping.parameterName}:`);
145 console.log(` Source: "${mapping.sourceText}"`);
146 console.log(
147 ` Value: ${JSON.stringify(mapping.targetValue)}`,
148 );
149 console.log(` Wildcard: ${mapping.isWildcard}`);
150 if (mapping.conversion) {
151 console.log(
152 ` Conversion: ${mapping.conversion.type} - ${mapping.conversion.description}`,
153 );
154 }
155 }
156
157 console.log("\nFixed Phrases:");
158 console.log(` ${analysis.fixedPhrases.join(", ")}`);
159
160 const grammarRule = generator.formatAsGrammarRule(
161 testCase,
162 analysis,
163 );
164 console.log("\nFull Grammar Rule:");
165 console.log(grammarRule);
166 } else {
167 console.log("\n❌ REJECTED - Grammar should NOT be generated");
168 console.log(`Reason: ${analysis.rejectionReason}`);
169 }
170
171 console.log("\nReasoning:");
172 console.log(` ${analysis.reasoning}`);
173 } catch (error) {
174 console.error(`\n❌ ERROR: ${error.message}`);
175 if (error.stack) {
176 console.error(error.stack);
177 }
178 }
179 }
180}
181
182main().catch(console.error);
183