GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

openai/openai-dotnet

Public

mirrored from https://github.com/openai/openai-dotnetAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
OpenAI_2.3.0

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openai-dotnet/tests/Chat

tests/Chat/ChatToolTests.cs

414lines · modecode

RawDownload
Latest commit unavailable.
1using NUnit.Framework;
2using OpenAI.Chat;
3using OpenAI.Tests.Utility;
4using System;
5using System.ClientModel;
6using System.Collections.Generic;
7using System.Linq;
8using System.Text.Json;
9using System.Text.Json.Nodes;
10using System.Threading.Tasks;
11using static OpenAI.Tests.TestHelpers;
12
13namespace OpenAI.Tests.Chat;
14
15[TestFixture(true)]
16[TestFixture(false)]
17[Parallelizable(ParallelScope.All)]
18[Category("Chat")]
19public class ChatToolTests : SyncAsyncTestBase
20{
21 public ChatToolTests(bool isAsync) : base(isAsync)
22 {
23 }
24
25 private const string GetNumberForWordToolName = "get_number_for_word";
26 private static ChatTool s_numberForWordTool = ChatTool.CreateFunctionTool(
27 GetNumberForWordToolName,
28 "gets an arbitrary number assigned to a given word",
29 BinaryData.FromString("""
30 {
31 "type": "object",
32 "properties": {
33 "word": {
34 "type": "string"
35 }
36 }
37 }
38 """)
39 );
40
41 private const string GetFavoriteColorToolName = "get_favorite_color";
42 private static ChatTool s_getFavoriteColorTool = ChatTool.CreateFunctionTool(
43 GetFavoriteColorToolName,
44 "gets the favorite color of the caller"
45 );
46
47 private const string GetFavoriteColorForMonthToolName = "get_favorite_color_for_month";
48 private static ChatTool s_getFavoriteColorForMonthTool = ChatTool.CreateFunctionTool(
49 GetFavoriteColorForMonthToolName,
50 "gets the caller's favorite color for a given month",
51 BinaryData.FromString("""
52 {
53 "type": "object",
54 "properties": {
55 "month_name": {
56 "type": "string",
57 "description": "the name of a calendar month, e.g. February or October."
58 }
59 },
60 "required": [ "month_name" ]
61 }
62 """)
63 );
64
65#pragma warning disable CS0618
66 private const string GetFavoriteColorForMonthFunctionName = "get_favorite_color_for_month";
67 private static ChatFunction s_getFavoriteColorForMonthFunction = new ChatFunction(GetFavoriteColorForMonthFunctionName)
68 {
69 FunctionDescription = "gets the caller's favorite color for a given month",
70 FunctionParameters = BinaryData.FromString("""
71 {
72 "type": "object",
73 "properties": {
74 "month_name": {
75 "type": "string",
76 "description": "the name of a calendar month, e.g. February or October."
77 }
78 },
79 "required": [ "month_name" ]
80 }
81 """)
82 };
83#pragma warning restore CS0618
84
85 private const string GetWeatherForCityToolName = "get_weather_for_city";
86 private static ChatTool s_getWeatherForCityTool = ChatTool.CreateFunctionTool(
87 GetWeatherForCityToolName,
88 "gets the current weather for a given city",
89 BinaryData.FromString("""
90 {
91 "type": "object",
92 "properties": {
93 "city_name": {
94 "type": "string",
95 "description": "the name of a city, e.g. Johannesburg or Ho Chi Minh City."
96 }
97 },
98 "required": [ "city_name" ]
99 }
100 """)
101 );
102
103 private const string GetMoodForWeatherToolName = "get_mood_for_weather";
104 private static ChatTool s_getMoodForWeatherTool = ChatTool.CreateFunctionTool(
105 GetMoodForWeatherToolName,
106 "gets the caller's mood for a given weather",
107 BinaryData.FromString("""
108 {
109 "type": "object",
110 "properties": {
111 "weather": {
112 "type": "string",
113 "description": "the current weather of where the caller is located, e.g. sunny or cloudy."
114 }
115 },
116 "required": [ "weather" ]
117 }
118 """)
119 );
120
121 [Test]
122 public async Task ConstraintsWork()
123 {
124 ChatClient client = GetTestClient<ChatClient>(TestScenario.Chat);
125 IEnumerable<ChatMessage> messages = [new UserChatMessage("What's the number for the word 'banana'?")];
126
127 foreach (var (choice, reason) in new (ChatToolChoice, ChatFinishReason)[]
128 {
129 (null, ChatFinishReason.ToolCalls),
130 (ChatToolChoice.CreateNoneChoice(), ChatFinishReason.Stop),
131 (ChatToolChoice.CreateFunctionChoice(GetNumberForWordToolName), ChatFinishReason.Stop),
132 (ChatToolChoice.CreateAutoChoice(), ChatFinishReason.ToolCalls),
133 // TODO: Add test for ChatToolChoice.Required
134 })
135 {
136 ChatCompletionOptions options = new()
137 {
138 Tools = { s_numberForWordTool },
139 ToolChoice = choice,
140 };
141 ClientResult<ChatCompletion> result = IsAsync
142 ? await client.CompleteChatAsync(messages, options)
143 : client.CompleteChat(messages, options);
144 Assert.That(result.Value.FinishReason, Is.EqualTo(reason));
145 }
146 }
147
148 [Test]
149 public async Task NoParameterToolWorks()
150 {
151 ChatClient client = GetTestClient<ChatClient>(TestScenario.Chat);
152 ICollection<ChatMessage> messages = [new UserChatMessage("What's my favorite color?")];
153 ChatCompletionOptions options = new()
154 {
155 Tools = { s_getFavoriteColorTool },
156 };
157 ClientResult<ChatCompletion> result = IsAsync
158 ? await client.CompleteChatAsync(messages, options)
159 : client.CompleteChat(messages, options);
160
161 Assert.That(result.Value.FinishReason, Is.EqualTo(ChatFinishReason.ToolCalls));
162 Assert.That(result.Value.ToolCalls.Count, Is.EqualTo(1));
163 var toolCall = result.Value.ToolCalls[0];
164 var toolCallArguments = toolCall.FunctionArguments.ToObjectFromJson<Dictionary<string, object>>();
165 Assert.That(toolCall.FunctionName, Is.EqualTo(GetFavoriteColorToolName));
166 Assert.That(toolCall.Id, Is.Not.Null.And.Not.Empty);
167 Assert.That(toolCallArguments.Count, Is.EqualTo(0));
168
169 messages.Add(new AssistantChatMessage(result.Value));
170 messages.Add(new ToolChatMessage(toolCall.Id, "green"));
171 result = IsAsync
172 ? await client.CompleteChatAsync(messages, options)
173 : client.CompleteChat(messages, options);
174
175 Assert.That(result.Value.FinishReason, Is.EqualTo(ChatFinishReason.Stop));
176 Assert.That(result.Value.Content[0].Text.ToLowerInvariant(), Contains.Substring("green"));
177 }
178
179 [Test]
180 public async Task ParametersWork()
181 {
182 ChatClient client = GetTestClient<ChatClient>(TestScenario.Chat);
183 ChatCompletionOptions options = new()
184 {
185 Tools = { s_getFavoriteColorForMonthTool },
186 };
187 List<ChatMessage> messages =
188 [
189 new UserChatMessage("What's my favorite color in February?"),
190 ];
191 ClientResult<ChatCompletion> result = IsAsync
192 ? await client.CompleteChatAsync(messages, options)
193 : client.CompleteChat(messages, options);
194 Assert.That(result.Value.FinishReason, Is.EqualTo(ChatFinishReason.ToolCalls));
195 Assert.That(result.Value.ToolCalls?.Count, Is.EqualTo(1));
196 var toolCall = result.Value.ToolCalls[0];
197 Assert.That(toolCall.FunctionName, Is.EqualTo(GetFavoriteColorForMonthToolName));
198 JsonObject argumentsJson = JsonSerializer.Deserialize<JsonObject>(toolCall.FunctionArguments);
199 Assert.That(argumentsJson.Count, Is.EqualTo(1));
200 Assert.That(argumentsJson.ContainsKey("month_name"));
201 Assert.That(argumentsJson["month_name"].ToString().ToLowerInvariant(), Is.EqualTo("february"));
202 messages.Add(new AssistantChatMessage(result.Value));
203 messages.Add(new ToolChatMessage(toolCall.Id, "chartreuse"));
204 result = IsAsync
205 ? await client.CompleteChatAsync(messages, options)
206 : client.CompleteChat(messages, options);
207 Assert.That(result.Value.Content[0].Text.ToLowerInvariant(), Contains.Substring("chartreuse"));
208 }
209
210#pragma warning disable CS0618
211 [Test]
212 public async Task FunctionsWork()
213 {
214 ChatClient client = GetTestClient<ChatClient>(TestScenario.Chat);
215 ChatCompletionOptions options = new()
216 {
217 Functions = { s_getFavoriteColorForMonthFunction },
218 };
219 List<ChatMessage> messages =
220 [
221 new UserChatMessage("What's my favorite color in February?"),
222 ];
223 ClientResult<ChatCompletion> result = IsAsync
224 ? await client.CompleteChatAsync(messages, options)
225 : client.CompleteChat(messages, options);
226 Assert.That(result.Value.FinishReason, Is.EqualTo(ChatFinishReason.FunctionCall));
227 var functionCall = result.Value.FunctionCall;
228 Assert.That(functionCall, Is.Not.Null);
229 Assert.That(functionCall.FunctionName, Is.EqualTo(GetFavoriteColorForMonthFunctionName));
230 JsonObject argumentsJson = JsonSerializer.Deserialize<JsonObject>(functionCall.FunctionArguments);
231 Assert.That(argumentsJson.Count, Is.EqualTo(1));
232 Assert.That(argumentsJson.ContainsKey("month_name"));
233 Assert.That(argumentsJson["month_name"].ToString().ToLowerInvariant(), Is.EqualTo("february"));
234 messages.Add(new AssistantChatMessage(result.Value));
235 messages.Add(new FunctionChatMessage(GetFavoriteColorForMonthFunctionName, "chartreuse"));
236 result = IsAsync
237 ? await client.CompleteChatAsync(messages, options)
238 : client.CompleteChat(messages, options);
239 Assert.That(result.Value.Content[0].Text.ToLowerInvariant(), Contains.Substring("chartreuse"));
240 }
241#pragma warning restore CS0618
242
243 [Test]
244 public async Task ParallelToolCalls()
245 {
246 ChatClient client = GetTestClient<ChatClient>(TestScenario.Chat);
247 ChatCompletionOptions options = new()
248 {
249 Tools = { s_getWeatherForCityTool },
250 };
251 List<ChatMessage> messages = [
252 new UserChatMessage("Tell me what's the current weather in the following cities: Santiago and Karachi."),
253 ];
254 ClientResult<ChatCompletion> result = IsAsync
255 ? await client.CompleteChatAsync(messages, options)
256 : client.CompleteChat(messages, options);
257
258 Assert.That(result.Value.FinishReason, Is.EqualTo(ChatFinishReason.ToolCalls));
259 Assert.That(result.Value.ToolCalls.Count, Is.EqualTo(2));
260
261 var santiagoToolCall = result.Value.ToolCalls.Single(call => call.FunctionArguments.ToString().ToLowerInvariant().Contains("santiago"));
262 var karachiToolCall = result.Value.ToolCalls.Single(call => call.FunctionArguments.ToString().ToLowerInvariant().Contains("karachi"));
263
264 JsonObject argumentsJson = JsonSerializer.Deserialize<JsonObject>(santiagoToolCall.FunctionArguments);
265 Assert.That(argumentsJson.Count, Is.EqualTo(1));
266 Assert.That(argumentsJson.ContainsKey("city_name"));
267 Assert.That(argumentsJson["city_name"].ToString().ToLowerInvariant(), Is.EqualTo("santiago"));
268
269 argumentsJson = JsonSerializer.Deserialize<JsonObject>(karachiToolCall.FunctionArguments);
270 Assert.That(argumentsJson.Count, Is.EqualTo(1));
271 Assert.That(argumentsJson.ContainsKey("city_name"));
272 Assert.That(argumentsJson["city_name"].ToString().ToLowerInvariant(), Is.EqualTo("karachi"));
273
274 messages.Add(new AssistantChatMessage(result.Value));
275 messages.Add(new ToolChatMessage(santiagoToolCall.Id, "rainy"));
276 messages.Add(new ToolChatMessage(karachiToolCall.Id, "sunny"));
277
278 result = IsAsync
279 ? await client.CompleteChatAsync(messages, options)
280 : client.CompleteChat(messages, options);
281
282 Assert.That(result.Value.FinishReason, Is.EqualTo(ChatFinishReason.Stop));
283 Assert.That(result.Value.Content[0].Text.ToLowerInvariant(), Contains.Substring("rainy"));
284 Assert.That(result.Value.Content[0].Text.ToLowerInvariant(), Contains.Substring("sunny"));
285 }
286
287 [Test]
288 public async Task ConsecutiveToolCalls()
289 {
290 ChatClient client = GetTestClient<ChatClient>(TestScenario.Chat);
291 ChatCompletionOptions options = new()
292 {
293 Tools = { s_getWeatherForCityTool, s_getMoodForWeatherTool },
294 };
295 List<ChatMessage> messages = [
296 new UserChatMessage("Can you guess my mood given that I'm currently located in Osaka?"),
297 ];
298 ClientResult<ChatCompletion> result = IsAsync
299 ? await client.CompleteChatAsync(messages, options)
300 : client.CompleteChat(messages, options);
301
302 Assert.That(result.Value.ToolCalls?.Count, Is.EqualTo(1));
303 var toolCall = result.Value.ToolCalls[0];
304 Assert.That(toolCall.FunctionName, Is.EqualTo(GetWeatherForCityToolName));
305
306 JsonObject argumentsJson = JsonSerializer.Deserialize<JsonObject>(toolCall.FunctionArguments);
307 Assert.That(argumentsJson.Count, Is.EqualTo(1));
308 Assert.That(argumentsJson.ContainsKey("city_name"));
309 Assert.That(argumentsJson["city_name"].ToString().ToLowerInvariant(), Is.EqualTo("osaka"));
310
311 messages.Add(new AssistantChatMessage(result.Value));
312 messages.Add(new ToolChatMessage(toolCall.Id, "rainy"));
313 result = IsAsync
314 ? await client.CompleteChatAsync(messages, options)
315 : client.CompleteChat(messages, options);
316
317 Assert.That(result.Value.ToolCalls?.Count, Is.EqualTo(1));
318 toolCall = result.Value.ToolCalls[0];
319 Assert.That(toolCall.FunctionName, Is.EqualTo(GetMoodForWeatherToolName));
320
321 argumentsJson = JsonSerializer.Deserialize<JsonObject>(toolCall.FunctionArguments);
322 Assert.That(argumentsJson.Count, Is.EqualTo(1));
323 Assert.That(argumentsJson.ContainsKey("weather"));
324 Assert.That(argumentsJson["weather"].ToString().ToLowerInvariant(), Is.EqualTo("rainy"));
325
326 messages.Add(new AssistantChatMessage(result.Value));
327 messages.Add(new ToolChatMessage(toolCall.Id, "bored"));
328 result = IsAsync
329 ? await client.CompleteChatAsync(messages, options)
330 : client.CompleteChat(messages, options);
331
332 Assert.That(result.Value.Content[0].Text.ToLowerInvariant(), Contains.Substring("bored"));
333 }
334
335 public enum SchemaPresence { WithSchema, WithoutSchema }
336 public enum StrictnessPresence { Unspecified, Strict, NotStrict }
337 public enum FailureExpectation { FailureExpected, FailureNotExpected }
338
339 [Test]
340 [TestCase(SchemaPresence.WithoutSchema, StrictnessPresence.Unspecified)]
341 [TestCase(SchemaPresence.WithoutSchema, StrictnessPresence.NotStrict)]
342 [TestCase(SchemaPresence.WithoutSchema, StrictnessPresence.Strict, FailureExpectation.FailureExpected)]
343 [TestCase(SchemaPresence.WithSchema, StrictnessPresence.Unspecified)]
344 [TestCase(SchemaPresence.WithSchema, StrictnessPresence.NotStrict)]
345 [TestCase(SchemaPresence.WithSchema, StrictnessPresence.Strict)]
346 public async Task StructuredOutputs(
347 SchemaPresence schemaPresence,
348 StrictnessPresence strictnessPresence,
349 FailureExpectation failureExpectation = FailureExpectation.FailureNotExpected)
350 {
351 // Note: proper output requires 2024-08-06 or later models
352 ChatClient client = GetTestClient<ChatClient>(TestScenario.Chat, "gpt-4o-2024-08-06");
353
354 const string toolName = "get_favorite_color_for_day_of_week";
355 const string toolDescription = "Given a weekday name like Tuesday, gets the favorite color of the user on that day.";
356 BinaryData toolSchema = schemaPresence == SchemaPresence.WithSchema
357 ? BinaryData.FromObjectAsJson(new
358 {
359 type = "object",
360 properties = new
361 {
362 the_day_of_the_week = new
363 {
364 type = "string"
365 }
366 },
367 required = new[] { "the_day_of_the_week" },
368 additionalProperties = !(strictnessPresence == StrictnessPresence.Strict),
369 })
370 : null;
371 bool? useStrictSchema = strictnessPresence switch
372 {
373 StrictnessPresence.Strict => true,
374 StrictnessPresence.NotStrict => false,
375 _ => null,
376 };
377
378 ChatCompletionOptions options = new()
379 {
380 Tools = { ChatTool.CreateFunctionTool(toolName, toolDescription, toolSchema, useStrictSchema) },
381 };
382
383 List<ChatMessage> messages = [
384 new SystemChatMessage("Call applicable tools when the user asks a question. Prefer JSON output when possible."),
385 new UserChatMessage("What's my favorite color on Tuesday?"),
386 ];
387
388 if (failureExpectation == FailureExpectation.FailureExpected)
389 {
390 ClientResultException thrownException = Assert.ThrowsAsync<ClientResultException>(async () =>
391 {
392 ChatCompletion completion = IsAsync
393 ? await client.CompleteChatAsync(messages, options)
394 : client.CompleteChat(messages, options);
395 });
396 Assert.That(thrownException.Message, Does.Contain("function.parameters"));
397 }
398 else
399 {
400 ChatCompletion completion = IsAsync
401 ? await client.CompleteChatAsync(messages, options)
402 : client.CompleteChat(messages, options);
403 Assert.That(completion.FinishReason, Is.EqualTo(ChatFinishReason.ToolCalls));
404 Assert.That(completion.ToolCalls, Has.Count.EqualTo(1));
405 Assert.That(completion.ToolCalls[0].FunctionArguments, Is.Not.Null);
406
407 if (schemaPresence == SchemaPresence.WithSchema && strictnessPresence == StrictnessPresence.Strict)
408 {
409 using JsonDocument argumentsDocument = JsonDocument.Parse(completion.ToolCalls[0].FunctionArguments);
410 Assert.That(argumentsDocument.RootElement.GetProperty("the_day_of_the_week").GetString(), Is.EqualTo("Tuesday"));
411 }
412 }
413 }
414}
415