GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

openai/openai-dotnet

Public

mirrored from https://github.com/openai/openai-dotnetAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
OpenAI_2.6.0

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openai-dotnet/tests/Chat

tests/Chat/ChatToolTests.cs

388lines · modecode

RawDownload
Latest commit unavailable.
1using Microsoft.ClientModel.TestFramework;
2using NUnit.Framework;
3using OpenAI.Chat;
4using OpenAI.Tests.Utility;
5using System;
6using System.ClientModel;
7using System.Collections.Generic;
8using System.Linq;
9using System.Text.Json;
10using System.Text.Json.Nodes;
11using System.Threading.Tasks;
12using static OpenAI.Tests.TestHelpers;
13
14namespace OpenAI.Tests.Chat;
15
16[Category("Chat")]
17[Category("ChatTools")]
18public class ChatToolTests : OpenAIRecordedTestBase
19{
20 public enum SchemaPresence { WithSchema, WithoutSchema }
21 public enum StrictnessPresence { Unspecified, Strict, NotStrict }
22 public enum FailureExpectation { FailureExpected, FailureNotExpected }
23
24 public ChatToolTests(bool isAsync) : base(isAsync)
25 {
26 }
27
28 [RecordedTest]
29 public async Task ConstraintsWork()
30 {
31 ChatClient client = GetProxiedOpenAIClient<ChatClient>(TestScenario.Chat);
32 IEnumerable<ChatMessage> messages = [new UserChatMessage("What's the number for the word 'banana'?")];
33
34 foreach (var (choice, reason) in new (ChatToolChoice, ChatFinishReason)[]
35 {
36 (null, ChatFinishReason.ToolCalls),
37 (ChatToolChoice.CreateNoneChoice(), ChatFinishReason.Stop),
38 (ChatToolChoice.CreateFunctionChoice(GetNumberForWordToolName), ChatFinishReason.Stop),
39 (ChatToolChoice.CreateAutoChoice(), ChatFinishReason.ToolCalls),
40 // TODO: Add test for ChatToolChoice.Required
41 })
42 {
43 ChatCompletionOptions options = new()
44 {
45 Tools = { s_numberForWordTool },
46 ToolChoice = choice,
47 };
48 ClientResult<ChatCompletion> result = await client.CompleteChatAsync(messages, options);
49 Assert.That(result.Value.FinishReason, Is.EqualTo(reason));
50 }
51 }
52
53 [RecordedTest]
54 public async Task NoParameterToolWorks()
55 {
56 ChatClient client = GetProxiedOpenAIClient<ChatClient>(TestScenario.Chat);
57 ICollection<ChatMessage> messages = [new UserChatMessage("What's my favorite color?")];
58 ChatCompletionOptions options = new()
59 {
60 Tools = { s_getFavoriteColorTool },
61 };
62 ClientResult<ChatCompletion> result = await client.CompleteChatAsync(messages, options);
63
64 Assert.That(result.Value.FinishReason, Is.EqualTo(ChatFinishReason.ToolCalls));
65 Assert.That(result.Value.ToolCalls.Count, Is.EqualTo(1));
66 var toolCall = result.Value.ToolCalls[0];
67 var toolCallArguments = toolCall.FunctionArguments.ToObjectFromJson<Dictionary<string, object>>();
68 Assert.That(toolCall.FunctionName, Is.EqualTo(GetFavoriteColorToolName));
69 Assert.That(toolCall.Id, Is.Not.Null.And.Not.Empty);
70 Assert.That(toolCallArguments.Count, Is.EqualTo(0));
71
72 messages.Add(new AssistantChatMessage(result.Value));
73 messages.Add(new ToolChatMessage(toolCall.Id, "green"));
74 result = await client.CompleteChatAsync(messages, options);
75
76 Assert.That(result.Value.FinishReason, Is.EqualTo(ChatFinishReason.Stop));
77 Assert.That(result.Value.Content[0].Text.ToLowerInvariant(), Contains.Substring("green"));
78 }
79
80 [RecordedTest]
81 public async Task ParametersWork()
82 {
83 ChatClient client = GetProxiedOpenAIClient<ChatClient>(TestScenario.Chat);
84 ChatCompletionOptions options = new()
85 {
86 Tools = { s_getFavoriteColorForMonthTool },
87 };
88 List<ChatMessage> messages =
89 [
90 new UserChatMessage("What's my favorite color in February?"),
91 ];
92
93 ClientResult<ChatCompletion> result = await client.CompleteChatAsync(messages, options);
94 Assert.That(result.Value.FinishReason, Is.EqualTo(ChatFinishReason.ToolCalls));
95 Assert.That(result.Value.ToolCalls?.Count, Is.EqualTo(1));
96 var toolCall = result.Value.ToolCalls[0];
97 Assert.That(toolCall.FunctionName, Is.EqualTo(GetFavoriteColorForMonthToolName));
98 JsonObject argumentsJson = JsonSerializer.Deserialize<JsonObject>(toolCall.FunctionArguments);
99 Assert.That(argumentsJson.Count, Is.EqualTo(1));
100 Assert.That(argumentsJson.ContainsKey("month_name"));
101 Assert.That(argumentsJson["month_name"].ToString().ToLowerInvariant(), Is.EqualTo("february"));
102 messages.Add(new AssistantChatMessage(result.Value));
103 messages.Add(new ToolChatMessage(toolCall.Id, "chartreuse"));
104
105 result = await client.CompleteChatAsync(messages, options);
106 Assert.That(result.Value.Content[0].Text.ToLowerInvariant(), Contains.Substring("chartreuse"));
107 }
108
109#pragma warning disable CS0618
110 [RecordedTest]
111 public async Task FunctionsWork()
112 {
113 ChatClient client = GetProxiedOpenAIClient<ChatClient>(TestScenario.Chat);
114 ChatCompletionOptions options = new()
115 {
116 Functions = { s_getFavoriteColorForMonthFunction },
117 };
118 List<ChatMessage> messages =
119 [
120 new UserChatMessage("What's my favorite color in February?"),
121 ];
122 ClientResult<ChatCompletion> result = await client.CompleteChatAsync(messages, options);
123 Assert.That(result.Value.FinishReason, Is.EqualTo(ChatFinishReason.FunctionCall));
124 var functionCall = result.Value.FunctionCall;
125 Assert.That(functionCall, Is.Not.Null);
126 Assert.That(functionCall.FunctionName, Is.EqualTo(GetFavoriteColorForMonthFunctionName));
127 JsonObject argumentsJson = JsonSerializer.Deserialize<JsonObject>(functionCall.FunctionArguments);
128 Assert.That(argumentsJson.Count, Is.EqualTo(1));
129 Assert.That(argumentsJson.ContainsKey("month_name"));
130 Assert.That(argumentsJson["month_name"].ToString().ToLowerInvariant(), Is.EqualTo("february"));
131 messages.Add(new AssistantChatMessage(result.Value));
132 messages.Add(new FunctionChatMessage(GetFavoriteColorForMonthFunctionName, "chartreuse"));
133 result = await client.CompleteChatAsync(messages, options);
134 Assert.That(result.Value.Content[0].Text.ToLowerInvariant(), Contains.Substring("chartreuse"));
135 }
136#pragma warning restore CS0618
137
138 [RecordedTest]
139 public async Task ParallelToolCalls()
140 {
141 ChatClient client = GetProxiedOpenAIClient<ChatClient>(TestScenario.Chat);
142 ChatCompletionOptions options = new()
143 {
144 Tools = { s_getWeatherForCityTool },
145 };
146 List<ChatMessage> messages = [
147 new UserChatMessage("Tell me what's the current weather in the following cities: Santiago and Karachi."),
148 ];
149 ClientResult<ChatCompletion> result = await client.CompleteChatAsync(messages, options);
150
151 Assert.That(result.Value.FinishReason, Is.EqualTo(ChatFinishReason.ToolCalls));
152 Assert.That(result.Value.ToolCalls.Count, Is.EqualTo(2));
153
154 var santiagoToolCall = result.Value.ToolCalls.Single(call => call.FunctionArguments.ToString().ToLowerInvariant().Contains("santiago"));
155 var karachiToolCall = result.Value.ToolCalls.Single(call => call.FunctionArguments.ToString().ToLowerInvariant().Contains("karachi"));
156
157 JsonObject argumentsJson = JsonSerializer.Deserialize<JsonObject>(santiagoToolCall.FunctionArguments);
158 Assert.That(argumentsJson.Count, Is.EqualTo(1));
159 Assert.That(argumentsJson.ContainsKey("city_name"));
160 Assert.That(argumentsJson["city_name"].ToString().ToLowerInvariant(), Is.EqualTo("santiago"));
161
162 argumentsJson = JsonSerializer.Deserialize<JsonObject>(karachiToolCall.FunctionArguments);
163 Assert.That(argumentsJson.Count, Is.EqualTo(1));
164 Assert.That(argumentsJson.ContainsKey("city_name"));
165 Assert.That(argumentsJson["city_name"].ToString().ToLowerInvariant(), Is.EqualTo("karachi"));
166
167 messages.Add(new AssistantChatMessage(result.Value));
168 messages.Add(new ToolChatMessage(santiagoToolCall.Id, "rainy"));
169 messages.Add(new ToolChatMessage(karachiToolCall.Id, "sunny"));
170
171 result = await client.CompleteChatAsync(messages, options);
172
173 Assert.That(result.Value.FinishReason, Is.EqualTo(ChatFinishReason.Stop));
174 Assert.That(result.Value.Content[0].Text.ToLowerInvariant(), Contains.Substring("rainy"));
175 Assert.That(result.Value.Content[0].Text.ToLowerInvariant(), Contains.Substring("sunny"));
176 }
177
178 [RecordedTest]
179 public async Task ConsecutiveToolCalls()
180 {
181 ChatClient client = GetProxiedOpenAIClient<ChatClient>(TestScenario.Chat);
182 ChatCompletionOptions options = new()
183 {
184 Tools = { s_getWeatherForCityTool, s_getMoodForWeatherTool },
185 };
186 List<ChatMessage> messages = [
187 new UserChatMessage("Can you guess my mood given that I'm currently located in Osaka?"),
188 ];
189 ClientResult<ChatCompletion> result = await client.CompleteChatAsync(messages, options);
190
191 Assert.That(result.Value.ToolCalls?.Count, Is.EqualTo(1));
192 var toolCall = result.Value.ToolCalls[0];
193 Assert.That(toolCall.FunctionName, Is.EqualTo(GetWeatherForCityToolName));
194
195 JsonObject argumentsJson = JsonSerializer.Deserialize<JsonObject>(toolCall.FunctionArguments);
196 Assert.That(argumentsJson.Count, Is.EqualTo(1));
197 Assert.That(argumentsJson.ContainsKey("city_name"));
198 Assert.That(argumentsJson["city_name"].ToString().ToLowerInvariant(), Is.EqualTo("osaka"));
199
200 messages.Add(new AssistantChatMessage(result.Value));
201 messages.Add(new ToolChatMessage(toolCall.Id, "rainy"));
202 result = await client.CompleteChatAsync(messages, options);
203
204 Assert.That(result.Value.ToolCalls?.Count, Is.EqualTo(1));
205 toolCall = result.Value.ToolCalls[0];
206 Assert.That(toolCall.FunctionName, Is.EqualTo(GetMoodForWeatherToolName));
207
208 argumentsJson = JsonSerializer.Deserialize<JsonObject>(toolCall.FunctionArguments);
209 Assert.That(argumentsJson.Count, Is.EqualTo(1));
210 Assert.That(argumentsJson.ContainsKey("weather"));
211 Assert.That(argumentsJson["weather"].ToString().ToLowerInvariant(), Is.EqualTo("rainy"));
212
213 messages.Add(new AssistantChatMessage(result.Value));
214 messages.Add(new ToolChatMessage(toolCall.Id, "bored"));
215 result = await client.CompleteChatAsync(messages, options);
216
217 Assert.That(result.Value.Content[0].Text.ToLowerInvariant(), Contains.Substring("bored"));
218 }
219
220 [RecordedTest]
221 [TestCase(SchemaPresence.WithoutSchema, StrictnessPresence.Unspecified)]
222 [TestCase(SchemaPresence.WithoutSchema, StrictnessPresence.NotStrict)]
223 [TestCase(SchemaPresence.WithoutSchema, StrictnessPresence.Strict, FailureExpectation.FailureExpected)]
224 [TestCase(SchemaPresence.WithSchema, StrictnessPresence.Unspecified)]
225 [TestCase(SchemaPresence.WithSchema, StrictnessPresence.NotStrict)]
226 [TestCase(SchemaPresence.WithSchema, StrictnessPresence.Strict)]
227 public async Task StructuredOutputs(
228 SchemaPresence schemaPresence,
229 StrictnessPresence strictnessPresence,
230 FailureExpectation failureExpectation = FailureExpectation.FailureNotExpected)
231 {
232 // Note: proper output requires 2024-08-06 or later models
233 ChatClient client = GetProxiedOpenAIClient<ChatClient>(TestScenario.Chat, "gpt-4o-2024-08-06");
234
235 const string toolName = "get_favorite_color_for_day_of_week";
236 const string toolDescription = "Given a weekday name like Tuesday, gets the favorite color of the user on that day.";
237 BinaryData toolSchema = schemaPresence == SchemaPresence.WithSchema
238 ? BinaryData.FromObjectAsJson(new
239 {
240 type = "object",
241 properties = new
242 {
243 the_day_of_the_week = new
244 {
245 type = "string"
246 }
247 },
248 required = new[] { "the_day_of_the_week" },
249 additionalProperties = !(strictnessPresence == StrictnessPresence.Strict),
250 })
251 : null;
252 bool? useStrictSchema = strictnessPresence switch
253 {
254 StrictnessPresence.Strict => true,
255 StrictnessPresence.NotStrict => false,
256 _ => null,
257 };
258
259 ChatCompletionOptions options = new()
260 {
261 Tools = { ChatTool.CreateFunctionTool(toolName, toolDescription, toolSchema, useStrictSchema) },
262 };
263
264 List<ChatMessage> messages = [
265 new SystemChatMessage("Call applicable tools when the user asks a question. Prefer JSON output when possible."),
266 new UserChatMessage("What's my favorite color on Tuesday?"),
267 ];
268
269 if (failureExpectation == FailureExpectation.FailureExpected)
270 {
271 ClientResultException thrownException = Assert.ThrowsAsync<ClientResultException>(async () =>
272 {
273 ChatCompletion completion = await client.CompleteChatAsync(messages, options);
274 });
275 Assert.That(thrownException.Message, Does.Contain("function.parameters"));
276 }
277 else
278 {
279 ChatCompletion completion = await client.CompleteChatAsync(messages, options);
280 Assert.That(completion.FinishReason, Is.EqualTo(ChatFinishReason.ToolCalls));
281 Assert.That(completion.ToolCalls, Has.Count.EqualTo(1));
282 Assert.That(completion.ToolCalls[0].FunctionArguments, Is.Not.Null);
283
284 if (schemaPresence == SchemaPresence.WithSchema && strictnessPresence == StrictnessPresence.Strict)
285 {
286 using JsonDocument argumentsDocument = JsonDocument.Parse(completion.ToolCalls[0].FunctionArguments);
287 Assert.That(argumentsDocument.RootElement.GetProperty("the_day_of_the_week").GetString(), Is.EqualTo("Tuesday"));
288 }
289 }
290 }
291
292 private const string GetNumberForWordToolName = "get_number_for_word";
293 private static ChatTool s_numberForWordTool = ChatTool.CreateFunctionTool(
294 GetNumberForWordToolName,
295 "gets an arbitrary number assigned to a given word",
296 BinaryData.FromString("""
297 {
298 "type": "object",
299 "properties": {
300 "word": {
301 "type": "string"
302 }
303 }
304 }
305 """)
306 );
307
308 private const string GetFavoriteColorToolName = "get_favorite_color";
309 private static ChatTool s_getFavoriteColorTool = ChatTool.CreateFunctionTool(
310 GetFavoriteColorToolName,
311 "gets the favorite color of the caller"
312 );
313
314 private const string GetFavoriteColorForMonthToolName = "get_favorite_color_for_month";
315 private static ChatTool s_getFavoriteColorForMonthTool = ChatTool.CreateFunctionTool(
316 GetFavoriteColorForMonthToolName,
317 "gets the caller's favorite color for a given month",
318 BinaryData.FromString("""
319 {
320 "type": "object",
321 "properties": {
322 "month_name": {
323 "type": "string",
324 "description": "the name of a calendar month, e.g. February or October."
325 }
326 },
327 "required": [ "month_name" ]
328 }
329 """)
330 );
331
332#pragma warning disable CS0618
333 private const string GetFavoriteColorForMonthFunctionName = "get_favorite_color_for_month";
334 private static ChatFunction s_getFavoriteColorForMonthFunction = new ChatFunction(GetFavoriteColorForMonthFunctionName)
335 {
336 FunctionDescription = "gets the caller's favorite color for a given month",
337 FunctionParameters = BinaryData.FromString("""
338 {
339 "type": "object",
340 "properties": {
341 "month_name": {
342 "type": "string",
343 "description": "the name of a calendar month, e.g. February or October."
344 }
345 },
346 "required": [ "month_name" ]
347 }
348 """)
349 };
350#pragma warning restore CS0618
351
352 private const string GetWeatherForCityToolName = "get_weather_for_city";
353 private static ChatTool s_getWeatherForCityTool = ChatTool.CreateFunctionTool(
354 GetWeatherForCityToolName,
355 "gets the current weather for a given city",
356 BinaryData.FromString("""
357 {
358 "type": "object",
359 "properties": {
360 "city_name": {
361 "type": "string",
362 "description": "the name of a city, e.g. Johannesburg or Ho Chi Minh City."
363 }
364 },
365 "required": [ "city_name" ]
366 }
367 """)
368 );
369
370 private const string GetMoodForWeatherToolName = "get_mood_for_weather";
371 private static ChatTool s_getMoodForWeatherTool = ChatTool.CreateFunctionTool(
372 GetMoodForWeatherToolName,
373 "gets the caller's mood for a given weather",
374 BinaryData.FromString("""
375 {
376 "type": "object",
377 "properties": {
378 "weather": {
379 "type": "string",
380 "description": "the current weather of where the caller is located, e.g. sunny or cloudy."
381 }
382 },
383 "required": [ "weather" ]
384 }
385 """)
386 );
387
388}
389