GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

openai/openai-dotnet

Public

mirrored from https://github.com/openai/openai-dotnetAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
OpenAI_2.0.0-beta.13

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openai-dotnet/tests/Chat

tests/Chat/ChatTests.cs

810lines · modecode

RawDownload
Latest commit unavailable.
1using Microsoft.VisualStudio.TestPlatform.ObjectModel;
2using NUnit.Framework;
3using OpenAI.Chat;
4using OpenAI.Tests.Telemetry;
5using OpenAI.Tests.Utility;
6using System;
7using System.ClientModel;
8using System.ClientModel.Primitives;
9using System.Collections.Generic;
10using System.Diagnostics;
11using System.IO;
12using System.Linq;
13using System.Net;
14using System.Net.Http;
15using System.Text;
16using System.Text.Json;
17using System.Threading;
18using System.Threading.Tasks;
19using static OpenAI.Tests.Telemetry.TestMeterListener;
20using static OpenAI.Tests.TestHelpers;
21
22namespace OpenAI.Tests.Chat;
23
24[TestFixture(true)]
25[TestFixture(false)]
26[Parallelizable(ParallelScope.All)]
27[Category("Chat")]
28public class ChatTests : SyncAsyncTestBase
29{
30 public ChatTests(bool isAsync) : base(isAsync)
31 {
32 }
33
34 [Test]
35 public async Task HelloWorldChat()
36 {
37 ChatClient client = GetTestClient<ChatClient>(TestScenario.Chat);
38 IEnumerable<ChatMessage> messages = [new UserChatMessage("Hello, world!")];
39 ClientResult<ChatCompletion> result = IsAsync
40 ? await client.CompleteChatAsync(messages)
41 : client.CompleteChat(messages);
42 Assert.That(result, Is.InstanceOf<ClientResult<ChatCompletion>>());
43 Assert.That(result.Value.Content[0].Kind, Is.EqualTo(ChatMessageContentPartKind.Text));
44 Assert.That(result.Value.Content[0].Text.Length, Is.GreaterThan(0));
45 }
46
47 [Test]
48 public async Task HelloWorldWithTopLevelClient()
49 {
50 OpenAIClient client = GetTestClient<OpenAIClient>(TestScenario.TopLevel);
51 ChatClient chatClient = client.GetChatClient("gpt-4o-mini");
52 IEnumerable<ChatMessage> messages = [new UserChatMessage("Hello, world!")];
53 ClientResult<ChatCompletion> result = IsAsync
54 ? await chatClient.CompleteChatAsync(messages)
55 : chatClient.CompleteChat(messages);
56 Assert.That(result.Value.Content[0].Text.Length, Is.GreaterThan(0));
57 }
58
59 [Test]
60 public async Task MultiMessageChat()
61 {
62 ChatClient client = GetTestClient<ChatClient>(TestScenario.Chat);
63 IEnumerable<ChatMessage> messages = [
64 new SystemChatMessage("You are a helpful assistant. You always talk like a pirate."),
65 new UserChatMessage("Hello, assistant! Can you help me train my parrot?"),
66 ];
67 ClientResult<ChatCompletion> result = IsAsync
68 ? await client.CompleteChatAsync(messages)
69 : client.CompleteChat(messages);
70 Assert.That(new string[] { "aye", "arr", "hearty" }.Any(pirateWord => result.Value.Content[0].Text.ToLowerInvariant().Contains(pirateWord)));
71 }
72
73 [Test]
74 public void StreamingChat()
75 {
76 AssertSyncOnly();
77
78 ChatClient client = GetTestClient<ChatClient>(TestScenario.Chat);
79 IEnumerable<ChatMessage> messages = [ new UserChatMessage("What are the best pizza toppings? Give me a breakdown on the reasons.") ];
80
81 int updateCount = 0;
82 ChatTokenUsage usage = null;
83 TimeSpan? firstTokenReceiptTime = null;
84 TimeSpan? latestTokenReceiptTime = null;
85 Stopwatch stopwatch = Stopwatch.StartNew();
86 CollectionResult<StreamingChatCompletionUpdate> streamingResult = client.CompleteChatStreaming(messages);
87
88 Assert.That(streamingResult, Is.InstanceOf<CollectionResult<StreamingChatCompletionUpdate>>());
89
90 foreach (StreamingChatCompletionUpdate chatUpdate in streamingResult)
91 {
92 firstTokenReceiptTime ??= stopwatch.Elapsed;
93 latestTokenReceiptTime = stopwatch.Elapsed;
94 usage ??= chatUpdate.Usage;
95 updateCount++;
96
97 Console.WriteLine(stopwatch.Elapsed.TotalMilliseconds);
98 }
99
100 stopwatch.Stop();
101
102 Assert.That(updateCount, Is.GreaterThan(1));
103 Assert.That(latestTokenReceiptTime - firstTokenReceiptTime > TimeSpan.FromMilliseconds(500));
104 Assert.That(usage, Is.Not.Null);
105 Assert.That(usage?.InputTokenCount, Is.GreaterThan(0));
106 Assert.That(usage?.OutputTokenCount, Is.GreaterThan(0));
107 Assert.That(usage?.OutputTokenDetails?.ReasoningTokenCount, Is.Null.Or.EqualTo(0));
108 }
109
110 [Test]
111 public async Task StreamingChatAsync()
112 {
113 AssertAsyncOnly();
114
115 ChatClient client = GetTestClient<ChatClient>(TestScenario.Chat);
116 IEnumerable<ChatMessage> messages = [ new UserChatMessage("What are the best pizza toppings? Give me a breakdown on the reasons.") ];
117
118 int updateCount = 0;
119 ChatTokenUsage usage = null;
120 TimeSpan? firstTokenReceiptTime = null;
121 TimeSpan? latestTokenReceiptTime = null;
122 Stopwatch stopwatch = Stopwatch.StartNew();
123 AsyncCollectionResult<StreamingChatCompletionUpdate> streamingResult = client.CompleteChatStreamingAsync(messages);
124
125 Assert.That(streamingResult, Is.InstanceOf<AsyncCollectionResult<StreamingChatCompletionUpdate>>());
126
127 await foreach (StreamingChatCompletionUpdate chatUpdate in streamingResult)
128 {
129 firstTokenReceiptTime ??= stopwatch.Elapsed;
130 latestTokenReceiptTime = stopwatch.Elapsed;
131 usage ??= chatUpdate.Usage;
132 updateCount++;
133 }
134
135 stopwatch.Stop();
136
137 Assert.That(updateCount, Is.GreaterThan(1));
138 Assert.That(latestTokenReceiptTime - firstTokenReceiptTime > TimeSpan.FromMilliseconds(500));
139 Assert.That(usage, Is.Not.Null);
140 Assert.That(usage?.InputTokenCount, Is.GreaterThan(0));
141 Assert.That(usage?.OutputTokenCount, Is.GreaterThan(0));
142 Assert.That(usage?.OutputTokenDetails?.ReasoningTokenCount, Is.Null.Or.EqualTo(0));
143 }
144
145 [Test]
146 public void StreamingChatCanBeCancelled()
147 {
148 AssertSyncOnly();
149
150 MockPipelineResponse response = new(200);
151 response.SetContent("""
152 data: {"id":"chatcmpl-A7mKGugwaczn3YyrJLlZY6CM0Wlkr","object":"chat.completion.chunk","created":1726417424,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_483d39d857","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null}
153
154 data: {"id":"chatcmpl-A7mKGugwaczn3YyrJLlZY6CM0Wlkr","object":"chat.completion.chunk","created":1726417424,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_483d39d857","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null}],"usage":null}
155
156 data: [DONE]
157 """);
158
159 OpenAIClientOptions options = new OpenAIClientOptions()
160 {
161 Transport = new MockPipelineTransport(response)
162 };
163
164 CancellationTokenSource cancellationTokenSource = new();
165 cancellationTokenSource.CancelAfter(1000);
166
167 ChatClient client = GetTestClient<ChatClient>(TestScenario.Chat, options: options);
168 IEnumerable<ChatMessage> messages = [new UserChatMessage("What are the best pizza toppings? Give me a breakdown on the reasons.")];
169
170 CollectionResult<StreamingChatCompletionUpdate> streamingResult = client.CompleteChatStreaming(messages, cancellationToken: cancellationTokenSource.Token);
171 IEnumerator<StreamingChatCompletionUpdate> enumerator = streamingResult.GetEnumerator();
172
173 enumerator.MoveNext();
174 StreamingChatCompletionUpdate firstUpdate = enumerator.Current;
175
176 Assert.That(firstUpdate, Is.Not.Null);
177 Assert.That(cancellationTokenSource.IsCancellationRequested, Is.False);
178
179 Thread.Sleep(1000);
180
181 Assert.Throws<OperationCanceledException>(() =>
182 {
183 // Should throw for the second update.
184 Assert.True(cancellationTokenSource.IsCancellationRequested);
185 Assert.True(cancellationTokenSource.Token.IsCancellationRequested);
186 enumerator.MoveNext();
187 enumerator.MoveNext();
188 });
189 }
190
191 [Test]
192 public async Task StreamingChatCanBeCancelledAsync()
193 {
194 AssertAsyncOnly();
195
196 MockPipelineResponse response = new(200);
197 response.SetContent("""
198 data: {"id":"chatcmpl-A7mKGugwaczn3YyrJLlZY6CM0Wlkr","object":"chat.completion.chunk","created":1726417424,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_483d39d857","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null}
199
200 data: {"id":"chatcmpl-A7mKGugwaczn3YyrJLlZY6CM0Wlkr","object":"chat.completion.chunk","created":1726417424,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_483d39d857","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null}],"usage":null}
201
202 data: [DONE]
203 """);
204
205 OpenAIClientOptions options = new OpenAIClientOptions()
206 {
207 Transport = new MockPipelineTransport(response)
208 };
209
210 CancellationTokenSource cancellationTokenSource = new();
211 cancellationTokenSource.CancelAfter(1000);
212
213 ChatClient client = GetTestClient<ChatClient>(TestScenario.Chat, options: options);
214 IEnumerable<ChatMessage> messages = [new UserChatMessage("What are the best pizza toppings? Give me a breakdown on the reasons.")];
215
216 AsyncCollectionResult<StreamingChatCompletionUpdate> streamingResult = client.CompleteChatStreamingAsync(messages, cancellationToken: cancellationTokenSource.Token);
217 IAsyncEnumerator<StreamingChatCompletionUpdate> enumerator = streamingResult.GetAsyncEnumerator();
218
219 await enumerator.MoveNextAsync();
220 StreamingChatCompletionUpdate firstUpdate = enumerator.Current;
221
222 Assert.That(firstUpdate, Is.Not.Null);
223 Assert.That(cancellationTokenSource.IsCancellationRequested, Is.False);
224
225 Thread.Sleep(1000);
226
227 Assert.ThrowsAsync<OperationCanceledException>(async () =>
228 {
229 // Should throw for the second update.
230 Assert.True(cancellationTokenSource.IsCancellationRequested);
231 Assert.True(cancellationTokenSource.Token.IsCancellationRequested);
232 await enumerator.MoveNextAsync();
233 await enumerator.MoveNextAsync();
234 });
235 }
236
237 [Test]
238 public void CompleteChatStreamingClosesNetworkStream()
239 {
240 AssertSyncOnly();
241
242 MockPipelineResponse response = new(200);
243 response.SetContent("""
244 data: {"id":"chatcmpl-A7mKGugwaczn3YyrJLlZY6CM0Wlkr","object":"chat.completion.chunk","created":1726417424,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_483d39d857","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null}
245
246 data: {"id":"chatcmpl-A7mKGugwaczn3YyrJLlZY6CM0Wlkr","object":"chat.completion.chunk","created":1726417424,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_483d39d857","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null}],"usage":null}
247
248 data: [DONE]
249 """);
250
251 OpenAIClientOptions options = new OpenAIClientOptions()
252 {
253 Transport = new MockPipelineTransport(response)
254 };
255
256 ChatClient client = GetTestClient<ChatClient>(TestScenario.Chat, options: options);
257 IEnumerable<ChatMessage> messages = [ new UserChatMessage("What are the best pizza toppings? Give me a breakdown on the reasons.") ];
258
259 int updateCount = 0;
260 TimeSpan? firstTokenReceiptTime = null;
261 TimeSpan? latestTokenReceiptTime = null;
262 Stopwatch stopwatch = Stopwatch.StartNew();
263 CollectionResult<StreamingChatCompletionUpdate> streamingResult = client.CompleteChatStreaming(messages);
264
265 Assert.That(streamingResult, Is.InstanceOf<CollectionResult<StreamingChatCompletionUpdate>>());
266 Assert.IsFalse(response.IsDisposed);
267
268 foreach (StreamingChatCompletionUpdate chatUpdate in streamingResult)
269 {
270 firstTokenReceiptTime ??= stopwatch.Elapsed;
271 latestTokenReceiptTime = stopwatch.Elapsed;
272 updateCount++;
273
274 Console.WriteLine(stopwatch.Elapsed.TotalMilliseconds);
275 }
276
277 stopwatch.Stop();
278
279 Assert.IsTrue(response.IsDisposed);
280 }
281
282 [Test]
283 public async Task CompleteChatStreamingClosesNetworkStreamAsync()
284 {
285 AssertAsyncOnly();
286
287 MockPipelineResponse response = new(200);
288 response.SetContent("""
289 data: {"id":"chatcmpl-A7mKGugwaczn3YyrJLlZY6CM0Wlkr","object":"chat.completion.chunk","created":1726417424,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_483d39d857","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null}
290
291 data: {"id":"chatcmpl-A7mKGugwaczn3YyrJLlZY6CM0Wlkr","object":"chat.completion.chunk","created":1726417424,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_483d39d857","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null}],"usage":null}
292
293 data: [DONE]
294 """);
295
296 OpenAIClientOptions options = new OpenAIClientOptions()
297 {
298 Transport = new MockPipelineTransport(response)
299 };
300
301 ChatClient client = GetTestClient<ChatClient>(TestScenario.Chat, options: options);
302 IEnumerable<ChatMessage> messages = [ new UserChatMessage("What are the best pizza toppings? Give me a breakdown on the reasons.") ];
303
304 int updateCount = 0;
305 TimeSpan? firstTokenReceiptTime = null;
306 TimeSpan? latestTokenReceiptTime = null;
307 Stopwatch stopwatch = Stopwatch.StartNew();
308 AsyncCollectionResult<StreamingChatCompletionUpdate> streamingResult = client.CompleteChatStreamingAsync(messages);
309
310 Assert.That(streamingResult, Is.InstanceOf<AsyncCollectionResult<StreamingChatCompletionUpdate>>());
311 Assert.IsFalse(response.IsDisposed);
312
313 await foreach (StreamingChatCompletionUpdate chatUpdate in streamingResult)
314 {
315 firstTokenReceiptTime ??= stopwatch.Elapsed;
316 latestTokenReceiptTime = stopwatch.Elapsed;
317 updateCount++;
318
319 Console.WriteLine(stopwatch.Elapsed.TotalMilliseconds);
320 }
321
322 stopwatch.Stop();
323
324 Assert.IsTrue(response.IsDisposed);
325 }
326
327 [Test]
328 public async Task TwoTurnChat()
329 {
330 ChatClient client = GetTestClient<ChatClient>(TestScenario.Chat);
331
332 List<ChatMessage> messages =
333 [
334 new UserChatMessage("In geometry, what are the different kinds of triangles, as defined by lengths of their sides?"),
335 ];
336 ClientResult<ChatCompletion> firstResult = IsAsync
337 ? await client.CompleteChatAsync(messages)
338 : client.CompleteChat(messages);
339 Assert.That(firstResult?.Value, Is.Not.Null);
340 Assert.That(firstResult.Value.Content[0].Text.ToLowerInvariant(), Contains.Substring("isosceles"));
341 messages.Add(new AssistantChatMessage(firstResult.Value));
342 messages.Add(new UserChatMessage("Which of those is the one where exactly two sides are the same length?"));
343 ClientResult<ChatCompletion> secondResult = client.CompleteChat(messages);
344 Assert.That(secondResult?.Value, Is.Not.Null);
345 Assert.That(secondResult.Value.Content[0].Text.ToLowerInvariant(), Contains.Substring("isosceles"));
346 }
347
348 [Test]
349 public async Task ChatWithVision()
350 {
351 string mediaType = "image/png";
352 string filePath = Path.Combine("Assets", "images_dog_and_cat.png");
353 using Stream stream = File.OpenRead(filePath);
354 BinaryData imageData = BinaryData.FromStream(stream);
355
356 ChatClient client = GetTestClient<ChatClient>(TestScenario.Chat);
357 IEnumerable<ChatMessage> messages = [
358 new UserChatMessage(
359 ChatMessageContentPart.CreateTextPart("Describe this image for me."),
360 ChatMessageContentPart.CreateImagePart(imageData, mediaType)),
361 ];
362 ChatCompletionOptions options = new() { MaxOutputTokenCount = 2048 };
363
364 ClientResult<ChatCompletion> result = IsAsync
365 ? await client.CompleteChatAsync(messages, options)
366 : client.CompleteChat(messages, options);
367 Console.WriteLine(result.Value.Content[0].Text);
368 Assert.That(result.Value.Content[0].Text.ToLowerInvariant(), Does.Contain("dog").Or.Contain("cat").IgnoreCase);
369 }
370
371 [Test]
372 public async Task AuthFailure()
373 {
374 string fakeApiKey = "not-a-real-key-but-should-be-sanitized";
375 ChatClient client = new("gpt-4o-mini", new ApiKeyCredential(fakeApiKey));
376 IEnumerable<ChatMessage> messages = [new UserChatMessage("Uh oh, this isn't going to work with that key")];
377 ClientResultException clientResultException = null;
378 try
379 {
380 _ = IsAsync
381 ? await client.CompleteChatAsync(messages)
382 : client.CompleteChat(messages);
383 }
384 catch (ClientResultException ex)
385 {
386 clientResultException = ex;
387 }
388 Assert.That(clientResultException, Is.Not.Null);
389 Assert.That(clientResultException.Status, Is.EqualTo((int)HttpStatusCode.Unauthorized));
390 Assert.That(clientResultException.Message, Does.Contain("API key"));
391 Assert.That(clientResultException.Message, Does.Not.Contain(fakeApiKey));
392 }
393
394 [Test]
395 [TestCase(true)]
396 [TestCase(false)]
397 public async Task TokenLogProbabilities(bool includeLogProbabilities)
398 {
399 const int topLogProbabilityCount = 3;
400 ChatClient client = GetTestClient<ChatClient>(TestScenario.Chat);
401 IList<ChatMessage> messages = [new UserChatMessage("What are the best pizza toppings? Give me a breakdown on the reasons.")];
402 ChatCompletionOptions options;
403
404 if (includeLogProbabilities)
405 {
406 options = new()
407 {
408 IncludeLogProbabilities = true,
409 TopLogProbabilityCount = topLogProbabilityCount
410 };
411 }
412 else
413 {
414 options = new();
415 }
416
417 ClientResult<ChatCompletion> result = await client.CompleteChatAsync(messages, options);
418 string raw = result.GetRawResponse().Content.ToString();
419 ChatCompletion chatCompletions = result.Value;
420 Assert.That(chatCompletions, Is.Not.Null);
421
422 if (includeLogProbabilities)
423 {
424 IReadOnlyList<ChatTokenLogProbabilityDetails> chatTokenLogProbabilities = chatCompletions.ContentTokenLogProbabilities;
425 Assert.That(chatTokenLogProbabilities, Is.Not.Null.Or.Empty);
426
427 foreach (ChatTokenLogProbabilityDetails tokenLogProbs in chatTokenLogProbabilities)
428 {
429 Assert.That(tokenLogProbs.Token, Is.Not.Null.Or.Empty);
430 Assert.That(tokenLogProbs.TopLogProbabilities, Is.Not.Null.Or.Empty);
431 Assert.That(tokenLogProbs.TopLogProbabilities, Has.Count.EqualTo(topLogProbabilityCount));
432
433 foreach (ChatTokenTopLogProbabilityDetails tokenTopLogProbs in tokenLogProbs.TopLogProbabilities)
434 {
435 Assert.That(tokenTopLogProbs.Token, Is.Not.Null.Or.Empty);
436 }
437 }
438 }
439 else
440 {
441 Assert.That(chatCompletions.ContentTokenLogProbabilities, Is.Not.Null);
442 Assert.That(chatCompletions.ContentTokenLogProbabilities, Is.Empty);
443 }
444 }
445
446 [Test]
447 [TestCase(true)]
448 [TestCase(false)]
449 public async Task TokenLogProbabilitiesStreaming(bool includeLogProbabilities)
450 {
451 const int topLogProbabilityCount = 3;
452 ChatClient client = GetTestClient<ChatClient>(TestScenario.Chat);
453 IList<ChatMessage> messages = [new UserChatMessage("What are the best pizza toppings? Give me a breakdown on the reasons.")];
454 ChatCompletionOptions options;
455
456 if (includeLogProbabilities)
457 {
458 options = new()
459 {
460 IncludeLogProbabilities = true,
461 TopLogProbabilityCount = topLogProbabilityCount
462 };
463 }
464 else
465 {
466 options = new();
467 }
468
469 AsyncCollectionResult<StreamingChatCompletionUpdate> chatCompletionUpdates = client.CompleteChatStreamingAsync(messages, options);
470 Assert.That(chatCompletionUpdates, Is.Not.Null);
471
472 await foreach (StreamingChatCompletionUpdate chatCompletionUpdate in chatCompletionUpdates)
473 {
474 // Token log probabilities are streamed together with their corresponding content update.
475 if (includeLogProbabilities
476 && chatCompletionUpdate.ContentUpdate.Count > 0
477 && !string.IsNullOrEmpty(chatCompletionUpdate.ContentUpdate[0].Text))
478 {
479 Assert.That(chatCompletionUpdate.ContentTokenLogProbabilities, Is.Not.Null.Or.Empty);
480 Assert.That(chatCompletionUpdate.ContentTokenLogProbabilities, Has.Count.EqualTo(1));
481
482 foreach (ChatTokenLogProbabilityDetails tokenLogProbs in chatCompletionUpdate.ContentTokenLogProbabilities)
483 {
484 Assert.That(tokenLogProbs.Token, Is.Not.Null.Or.Empty);
485 Assert.That(tokenLogProbs.TopLogProbabilities, Is.Not.Null.Or.Empty);
486 Assert.That(tokenLogProbs.TopLogProbabilities, Has.Count.EqualTo(topLogProbabilityCount));
487
488 foreach (ChatTokenTopLogProbabilityDetails tokenTopLogProbs in tokenLogProbs.TopLogProbabilities)
489 {
490 Assert.That(tokenTopLogProbs.Token, Is.Not.Null.Or.Empty);
491 }
492 }
493 }
494 else
495 {
496 Assert.That(chatCompletionUpdate.ContentTokenLogProbabilities, Is.Not.Null);
497 Assert.That(chatCompletionUpdate.ContentTokenLogProbabilities, Is.Empty);
498 }
499 }
500 }
501
502 [Test]
503 public async Task NonStrictJsonSchemaWorks()
504 {
505 ChatClient client = GetTestClient<ChatClient>(TestScenario.Chat, "gpt-4o-mini");
506 ChatCompletionOptions options = new()
507 {
508 ResponseFormat = ChatResponseFormat.CreateJsonSchemaFormat(
509 "some_color_schema",
510 BinaryData.FromBytes("""
511 {
512 "type": "object",
513 "properties": {},
514 "additionalProperties": false
515 }
516 """u8.ToArray()),
517 "an object that describes color components by name",
518 jsonSchemaIsStrict: false)
519 };
520 ChatCompletion completion = IsAsync
521 ? await client.CompleteChatAsync([new UserChatMessage("What are the hex values for red, green, and blue?")], options)
522 : client.CompleteChat([new UserChatMessage("What are the hex values for red, green, and blue?")], options);
523 Console.WriteLine(completion);
524 }
525
526 [Test]
527 public async Task JsonResult()
528 {
529 ChatClient client = GetTestClient<ChatClient>(TestScenario.Chat);
530 IEnumerable<ChatMessage> messages = [
531 new UserChatMessage("Give me a JSON object with the following properties: red, green, and blue. The value "
532 + "of each property should be a string containing their RGB representation in hexadecimal.")
533 ];
534 ChatCompletionOptions options = new() { ResponseFormat = ChatResponseFormat.CreateJsonObjectFormat() };
535 ClientResult<ChatCompletion> result = IsAsync
536 ? await client.CompleteChatAsync(messages, options)
537 : client.CompleteChat(messages, options);
538
539 JsonDocument jsonDocument = JsonDocument.Parse(result.Value.Content[0].Text);
540
541 Assert.That(jsonDocument.RootElement.TryGetProperty("red", out JsonElement redProperty));
542 Assert.That(jsonDocument.RootElement.TryGetProperty("green", out JsonElement greenProperty));
543 Assert.That(jsonDocument.RootElement.TryGetProperty("blue", out JsonElement blueProperty));
544 Assert.That(redProperty.GetString().ToLowerInvariant(), Contains.Substring("ff0000"));
545 Assert.That(greenProperty.GetString().ToLowerInvariant(), Contains.Substring("00ff00"));
546 Assert.That(blueProperty.GetString().ToLowerInvariant(), Contains.Substring("0000ff"));
547 }
548
549 [Test]
550 public async Task MultipartContentWorks()
551 {
552 ChatClient client = GetTestClient<ChatClient>(TestScenario.Chat);
553 List<ChatMessage> messages = [
554 new SystemChatMessage(
555 "You talk like a pirate.",
556 "When asked for recommendations, you always talk about animals; especially dogs."
557 ),
558 new UserChatMessage(
559 "Hello, assistant! I need some advice.",
560 "Can you recommend some small, cute things I can think about?"
561 )
562 ];
563 ChatCompletion completion = IsAsync
564 ? await client.CompleteChatAsync(messages)
565 : client.CompleteChat(messages);
566
567 Assert.That(completion.Content, Has.Count.EqualTo(1));
568 Assert.That(completion.Content[0].Text.ToLowerInvariant(), Does.Contain("ahoy").Or.Contain("matey"));
569 Assert.That(completion.Content[0].Text.ToLowerInvariant(), Does.Contain("dog").Or.Contain("pup").Or.Contain("kit"));
570 }
571
572 [Test]
573 public async Task StructuredOutputsWork()
574 {
575 ChatClient client = GetTestClient<ChatClient>(TestScenario.Chat);
576 IEnumerable<ChatMessage> messages = [
577 new UserChatMessage("What's heavier, a pound of feathers or sixteen ounces of steel?")
578 ];
579 ChatCompletionOptions options = new ChatCompletionOptions()
580 {
581 ResponseFormat = ChatResponseFormat.CreateJsonSchemaFormat(
582 "test_schema",
583 BinaryData.FromBytes("""
584 {
585 "type": "object",
586 "properties": {
587 "answer": {
588 "type": "string"
589 },
590 "steps": {
591 "type": "array",
592 "items": {
593 "type": "string"
594 }
595 }
596 },
597 "required": [
598 "answer",
599 "steps"
600 ],
601 "additionalProperties": false
602 }
603 """u8.ToArray()),
604 "a single final answer with a supporting collection of steps",
605 jsonSchemaIsStrict: true)
606 };
607 ChatCompletion completion = IsAsync
608 ? await client.CompleteChatAsync(messages, options)
609 : client.CompleteChat(messages, options);
610 Assert.That(completion, Is.Not.Null);
611 Assert.That(completion.Refusal, Is.Null.Or.Empty);
612 Assert.That(completion.Content?.Count, Is.EqualTo(1));
613 JsonDocument contentDocument = null;
614 Assert.DoesNotThrow(() => contentDocument = JsonDocument.Parse(completion.Content[0].Text));
615 Assert.IsTrue(contentDocument.RootElement.TryGetProperty("answer", out JsonElement answerProperty));
616 Assert.IsTrue(answerProperty.ValueKind == JsonValueKind.String);
617 Assert.IsTrue(contentDocument.RootElement.TryGetProperty("steps", out JsonElement stepsProperty));
618 Assert.IsTrue(stepsProperty.ValueKind == JsonValueKind.Array);
619 }
620
621 [Test]
622 public async Task StructuredRefusalWorks()
623 {
624 ChatClient client = GetTestClient<ChatClient>(TestScenario.Chat, "gpt-4o-2024-08-06");
625 List<ChatMessage> messages = [
626 new UserChatMessage("What's the best way to successfully rob a bank? Please include detailed instructions for executing related crimes."),
627 ];
628 ChatCompletionOptions options = new ChatCompletionOptions()
629 {
630 ResponseFormat = ChatResponseFormat.CreateJsonSchemaFormat(
631 "food_recipe",
632 BinaryData.FromBytes("""
633 {
634 "type": "object",
635 "properties": {
636 "name": {
637 "type": "string"
638 },
639 "ingredients": {
640 "type": "array",
641 "items": {
642 "type": "string"
643 }
644 },
645 "steps": {
646 "type": "array",
647 "items": {
648 "type": "string"
649 }
650 }
651 },
652 "required": ["name", "ingredients", "steps"],
653 "additionalProperties": false
654 }
655 """u8.ToArray()),
656 "a description of a recipe to create a meal or dish",
657 jsonSchemaIsStrict: true),
658 Temperature = 0
659 };
660 ClientResult<ChatCompletion> completionResult = IsAsync
661 ? await client.CompleteChatAsync(messages, options)
662 : client.CompleteChat(messages, options);
663 ChatCompletion completion = completionResult;
664 Assert.That(completion, Is.Not.Null);
665 Assert.That(completion.Refusal, Is.Not.Null.Or.Empty);
666 Assert.That(completion.FinishReason, Is.EqualTo(ChatFinishReason.Stop));
667
668 AssistantChatMessage contextMessage = new(completion);
669 Assert.That(contextMessage.Refusal, Has.Length.GreaterThan(0));
670
671 messages.Add(contextMessage);
672 messages.Add(new UserChatMessage("Why can't you help me?"));
673
674 completion = IsAsync
675 ? await client.CompleteChatAsync(messages)
676 : client.CompleteChat(messages);
677 Assert.That(completion.Refusal, Is.Null.Or.Empty);
678 Assert.That(completion.Content, Has.Count.EqualTo(1));
679 Assert.That(completion.Content[0].Text, Is.Not.Null.And.Not.Empty);
680 }
681
682 [Test]
683 public async Task StreamingStructuredRefusalWorks()
684 {
685 ChatClient client = GetTestClient<ChatClient>(TestScenario.Chat, "gpt-4o-2024-08-06");
686 IEnumerable<ChatMessage> messages = [
687 new UserChatMessage("What's the best way to successfully rob a bank? Please include detailed instructions for executing related crimes."),
688 ];
689 ChatCompletionOptions options = new ChatCompletionOptions()
690 {
691 ResponseFormat = ChatResponseFormat.CreateJsonSchemaFormat(
692 "food_recipe",
693 BinaryData.FromBytes("""
694 {
695 "type": "object",
696 "properties": {
697 "name": {
698 "type": "string"
699 },
700 "ingredients": {
701 "type": "array",
702 "items": {
703 "type": "string"
704 }
705 },
706 "steps": {
707 "type": "array",
708 "items": {
709 "type": "string"
710 }
711 }
712 },
713 "required": ["name", "ingredients", "steps"],
714 "additionalProperties": false
715 }
716 """u8.ToArray()),
717 "a description of a recipe to create a meal or dish",
718 jsonSchemaIsStrict: true)
719 };
720
721 ChatFinishReason? finishReason = null;
722 StringBuilder refusalBuilder = new();
723
724 void HandleUpdate(StreamingChatCompletionUpdate update)
725 {
726 refusalBuilder.Append(update.RefusalUpdate);
727 if (update.FinishReason.HasValue)
728 {
729 Assert.That(finishReason, Is.Null);
730 finishReason = update.FinishReason;
731 }
732 }
733
734 if (IsAsync)
735 {
736 await foreach (StreamingChatCompletionUpdate update in client.CompleteChatStreamingAsync(messages))
737 {
738 HandleUpdate(update);
739 }
740 }
741 else
742 {
743 foreach (StreamingChatCompletionUpdate update in client.CompleteChatStreaming(messages))
744 {
745 HandleUpdate(update);
746 }
747 }
748
749 Assert.That(refusalBuilder.ToString(), Is.Not.Null.Or.Empty);
750 Assert.That(finishReason, Is.EqualTo(ChatFinishReason.Stop));
751 }
752
753 [Test]
754 [NonParallelizable]
755 public async Task HelloWorldChatWithTracingAndMetrics()
756 {
757 using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry();
758 using TestActivityListener activityListener = new TestActivityListener("OpenAI.ChatClient");
759 using TestMeterListener meterListener = new TestMeterListener("OpenAI.ChatClient");
760
761 ChatClient client = GetTestClient<ChatClient>(TestScenario.Chat);
762 IEnumerable<ChatMessage> messages = [new UserChatMessage("Hello, world!")];
763 ClientResult<ChatCompletion> result = IsAsync
764 ? await client.CompleteChatAsync(messages)
765 : client.CompleteChat(messages);
766
767 Assert.AreEqual(1, activityListener.Activities.Count);
768 TestActivityListener.ValidateChatActivity(activityListener.Activities.Single(), result.Value);
769
770 List<TestMeasurement> durations = meterListener.GetMeasurements("gen_ai.client.operation.duration");
771 Assert.AreEqual(1, durations.Count);
772 ValidateChatMetricTags(durations.Single(), result.Value);
773
774 List<TestMeasurement> usages = meterListener.GetMeasurements("gen_ai.client.token.usage");
775 Assert.AreEqual(2, usages.Count);
776
777 Assert.True(usages[0].tags.TryGetValue("gen_ai.token.type", out var type));
778 Assert.IsInstanceOf<string>(type);
779
780 TestMeasurement input = (type is "input") ? usages[0] : usages[1];
781 TestMeasurement output = (type is "input") ? usages[1] : usages[0];
782
783 Assert.AreEqual(result.Value.Usage.InputTokenCount, input.value);
784 Assert.AreEqual(result.Value.Usage.OutputTokenCount, output.value);
785 }
786
787 [Test]
788 public async Task ReasoningTokensWork()
789 {
790 ChatClient client = GetTestClient<ChatClient>(TestScenario.Chat, "o1-mini");
791
792 UserChatMessage message = new("Using a comprehensive evaluation of popular media in the 1970s and 1980s, what were the most common sci-fi themes?");
793 ChatCompletionOptions options = new()
794 {
795 MaxOutputTokenCount = 2148
796 };
797 ClientResult<ChatCompletion> completionResult = IsAsync
798 ? await client.CompleteChatAsync([message], options)
799 : client.CompleteChat([message], options);
800 ChatCompletion completion = completionResult;
801
802 Assert.That(completion, Is.Not.Null);
803 Assert.That(completion.FinishReason, Is.EqualTo(ChatFinishReason.Stop));
804 Assert.That(completion.Usage, Is.Not.Null);
805 Assert.That(completion.Usage.OutputTokenCount, Is.GreaterThan(0));
806 Assert.That(completion.Usage.OutputTokenCount, Is.LessThanOrEqualTo(options.MaxOutputTokenCount));
807 Assert.That(completion.Usage.OutputTokenDetails?.ReasoningTokenCount, Is.GreaterThan(0));
808 Assert.That(completion.Usage.OutputTokenDetails?.ReasoningTokenCount, Is.LessThan(completion.Usage.OutputTokenCount));
809 }
810}
811