GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

openai/openai-dotnet

Public

mirrored from https://github.com/openai/openai-dotnetAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
OpenAI_2.3.0

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openai-dotnet/examples/Realtime

examples/Realtime/Example01_AudioFromFileWithToolsAsync.cs

216lines · modeblame

RawDownload
Latest commit unavailable.
b0f9e5c3Jose Arriaga Maldonado1 years ago1using NUnit.Framework;
2using OpenAI.Images;
5dce104aJose Arriaga Maldonado1 years ago3using OpenAI.Realtime;
b0f9e5c3Jose Arriaga Maldonado1 years ago4using System;
5using System.ClientModel;
6using System.Collections.Generic;
7using System.IO;
8using System.Linq;
9using System.Threading.Tasks;
10
11namespace OpenAI.Examples;
12
13#pragma warning disable OPENAI002
14
15public partial class RealtimeExamples
16{
17[Test]
18public async Task Example01_AudioFromFileWithToolsAsync()
19{
5dce104aJose Arriaga Maldonado1 years ago20RealtimeClient client = new(
b0f9e5c3Jose Arriaga Maldonado1 years ago21credential: new ApiKeyCredential(Environment.GetEnvironmentVariable("OPENAI_API_KEY")));
5dce104aJose Arriaga Maldonado1 years ago22using RealtimeSession session = await client.StartConversationSessionAsync(
23model: "gpt-4o-realtime-preview");
b0f9e5c3Jose Arriaga Maldonado1 years ago24
25// Session options control connection-wide behavior shared across all conversations,
26// including audio input format and voice activity detection settings.
27ConversationSessionOptions sessionOptions = new()
28{
29Instructions = "You are a cheerful assistant that talks like a pirate. "
30+ "Always inform the user when you are about to call a tool. "
31+ "Prefer to call tools whenever applicable.",
32Voice = ConversationVoice.Alloy,
33Tools = { CreateSampleWeatherTool() },
5dce104aJose Arriaga Maldonado1 years ago34InputAudioFormat = RealtimeAudioFormat.Pcm16,
35OutputAudioFormat = RealtimeAudioFormat.Pcm16,
b0f9e5c3Jose Arriaga Maldonado1 years ago36// Input transcription options must be provided to enable transcribed feedback for input audio
37InputTranscriptionOptions = new()
38{
39Model = "whisper-1",
40},
41};
42
5dce104aJose Arriaga Maldonado1 years ago43await session.ConfigureConversationSessionAsync(sessionOptions);
b0f9e5c3Jose Arriaga Maldonado1 years ago44
45// Conversation history or text input are provided by adding messages to the conversation.
46// Adding a message will not automatically begin a response turn.
47await session.AddItemAsync(
5dce104aJose Arriaga Maldonado1 years ago48RealtimeItem.CreateUserMessage(["I'm trying to decide what to wear on my trip."]));
b0f9e5c3Jose Arriaga Maldonado1 years ago49
50string inputAudioPath = FindFile("Assets\\realtime_whats_the_weather_pcm16_24khz_mono.wav");
51using Stream inputAudioStream = File.OpenRead(inputAudioPath);
52_ = session.SendInputAudioAsync(inputAudioStream);
53
54Dictionary<string, Stream> outputAudioStreamsById = [];
55
5dce104aJose Arriaga Maldonado1 years ago56await foreach (RealtimeUpdate update in session.ReceiveUpdatesAsync())
b0f9e5c3Jose Arriaga Maldonado1 years ago57{
58if (update is ConversationSessionStartedUpdate sessionStartedUpdate)
59{
60Console.WriteLine($"<<< Session started. ID: {sessionStartedUpdate.SessionId}");
61Console.WriteLine();
62}
63
5dce104aJose Arriaga Maldonado1 years ago64if (update is InputAudioSpeechStartedUpdate speechStartedUpdate)
b0f9e5c3Jose Arriaga Maldonado1 years ago65{
66Console.WriteLine(
67$" -- Voice activity detection started at {speechStartedUpdate.AudioStartTime}");
68}
69
5dce104aJose Arriaga Maldonado1 years ago70if (update is InputAudioSpeechFinishedUpdate speechFinishedUpdate)
b0f9e5c3Jose Arriaga Maldonado1 years ago71{
72Console.WriteLine(
73$" -- Voice activity detection ended at {speechFinishedUpdate.AudioEndTime}");
74}
75
76// Item started updates notify that the model generation process will insert a new item into
77// the conversation and begin streaming its content via content updates.
5dce104aJose Arriaga Maldonado1 years ago78if (update is OutputStreamingStartedUpdate itemStreamingStartedUpdate)
b0f9e5c3Jose Arriaga Maldonado1 years ago79{
80Console.WriteLine($" -- Begin streaming of new item");
81if (!string.IsNullOrEmpty(itemStreamingStartedUpdate.FunctionName))
82{
83Console.Write($" {itemStreamingStartedUpdate.FunctionName}: ");
84}
85}
86
5dce104aJose Arriaga Maldonado1 years ago87if (update is OutputDeltaUpdate deltaUpdate)
b0f9e5c3Jose Arriaga Maldonado1 years ago88{
89// With audio output enabled, the audio transcript of the delta update contains an approximation of
90// the words spoken by the model. Without audio output, the text of the delta update will contain
91// the segments making up the text content of a message.
92Console.Write(deltaUpdate.AudioTranscript);
93Console.Write(deltaUpdate.Text);
94Console.Write(deltaUpdate.FunctionArguments);
95if (deltaUpdate.AudioBytes is not null)
96{
97if (!outputAudioStreamsById.TryGetValue(deltaUpdate.ItemId, out Stream value))
98{
99string filename = $"output_{sessionOptions.OutputAudioFormat}_{deltaUpdate.ItemId}.raw";
100value = File.OpenWrite(filename);
101outputAudioStreamsById[deltaUpdate.ItemId] = value;
102}
103
104value.Write(deltaUpdate.AudioBytes);
105}
106}
107
108// Item finished updates arrive when all streamed data for an item has arrived and the
109// accumulated results are available. In the case of function calls, this is the point
110// where all arguments are expected to be present.
5dce104aJose Arriaga Maldonado1 years ago111if (update is OutputStreamingFinishedUpdate itemStreamingFinishedUpdate)
b0f9e5c3Jose Arriaga Maldonado1 years ago112{
113Console.WriteLine();
114Console.WriteLine($" -- Item streaming finished, item_id={itemStreamingFinishedUpdate.ItemId}");
115
116if (itemStreamingFinishedUpdate.FunctionCallId is not null)
117{
118Console.WriteLine($" + Responding to tool invoked by item: {itemStreamingFinishedUpdate.FunctionName}");
5dce104aJose Arriaga Maldonado1 years ago119RealtimeItem functionOutputItem = RealtimeItem.CreateFunctionCallOutput(
b0f9e5c3Jose Arriaga Maldonado1 years ago120callId: itemStreamingFinishedUpdate.FunctionCallId,
121output: "70 degrees Fahrenheit and sunny");
122await session.AddItemAsync(functionOutputItem);
123}
124else if (itemStreamingFinishedUpdate.MessageContentParts?.Count > 0)
125{
126Console.Write($" + [{itemStreamingFinishedUpdate.MessageRole}]: ");
127foreach (ConversationContentPart contentPart in itemStreamingFinishedUpdate.MessageContentParts)
128{
129Console.Write(contentPart.AudioTranscript);
130}
131Console.WriteLine();
132}
133}
134
5dce104aJose Arriaga Maldonado1 years ago135if (update is InputAudioTranscriptionFinishedUpdate transcriptionCompletedUpdate)
b0f9e5c3Jose Arriaga Maldonado1 years ago136{
137Console.WriteLine();
138Console.WriteLine($" -- User audio transcript: {transcriptionCompletedUpdate.Transcript}");
139Console.WriteLine();
140}
141
5dce104aJose Arriaga Maldonado1 years ago142if (update is ResponseFinishedUpdate turnFinishedUpdate)
b0f9e5c3Jose Arriaga Maldonado1 years ago143{
144Console.WriteLine($" -- Model turn generation finished. Status: {turnFinishedUpdate.Status}");
145
146// Here, if we processed tool calls in the course of the model turn, we finish the
147// client turn to resume model generation. The next model turn will reflect the tool
148// responses that were already provided.
149if (turnFinishedUpdate.CreatedItems.Any(item => item.FunctionName?.Length > 0))
150{
151Console.WriteLine($" -- Ending client turn for pending tool responses");
152await session.StartResponseAsync();
153}
154else
155{
156break;
157}
158}
159
5dce104aJose Arriaga Maldonado1 years ago160if (update is RealtimeErrorUpdate errorUpdate)
b0f9e5c3Jose Arriaga Maldonado1 years ago161{
162Console.WriteLine();
163Console.WriteLine($"ERROR: {errorUpdate.Message}");
164break;
165}
166}
167
168foreach ((string itemId, Stream outputAudioStream) in outputAudioStreamsById)
169{
170Console.WriteLine($"Raw audio output for {itemId}: {outputAudioStream.Length} bytes");
171outputAudioStream.Dispose();
172}
173}
174
175private static ConversationFunctionTool CreateSampleWeatherTool()
176{
86407c80Jose Arriaga Maldonado1 years ago177return new ConversationFunctionTool("get_weather_for_location")
b0f9e5c3Jose Arriaga Maldonado1 years ago178{
179Description = "gets the weather for a location",
180Parameters = BinaryData.FromString("""
181{
182"type": "object",
183"properties": {
184"location": {
185"type": "string",
186"description": "The city and state, e.g. San Francisco, CA"
187},
188"unit": {
189"type": "string",
190"enum": ["c","f"]
191}
192},
193"required": ["location","unit"]
194}
195""")
196};
197}
198
199private static string FindFile(string fileName)
200{
201for (string currentDirectory = Directory.GetCurrentDirectory();
202currentDirectory != null && currentDirectory != Path.GetPathRoot(currentDirectory);
203currentDirectory = Directory.GetParent(currentDirectory)?.FullName!)
204{
205string filePath = Path.Combine(currentDirectory, fileName);
206if (File.Exists(filePath))
207{
208return filePath;
209}
210}
211
212throw new FileNotFoundException($"File '{fileName}' not found.");
213}
214}
5dce104aJose Arriaga Maldonado1 years ago215
216#pragma warning restore OPENAI002