GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

openai/openai-dotnet

Public

mirrored from https://github.com/openai/openai-dotnetAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
joseharriaga/MessageResponseItem

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openai-dotnet/tests/FineTuning

tests/FineTuning/FineTuningClientTests.cs

305lines · modecode

RawDownload
Latest commit unavailable.
1using Microsoft.ClientModel.TestFramework;
2using NUnit.Framework;
3using NUnit.Framework.Internal;
4using OpenAI.Files;
5using OpenAI.FineTuning;
6using OpenAI.Tests.Utility;
7using System;
8using System.ClientModel;
9using System.IO;
10using System.Linq;
11using System.Threading.Tasks;
12using static OpenAI.Tests.TestHelpers;
13
14namespace OpenAI.Tests.FineTuning;
15
16[Category("FineTuning")]
17public class FineTuningClientTests : OpenAIRecordedTestBase
18{
19 OpenAIFileClient fileClient;
20
21 string samplePath;
22 string validationPath;
23
24 OpenAIFile sampleFile;
25 OpenAIFile validationFile;
26
27 string sampleFileId;
28 string validationFileId;
29
30 public FineTuningClientTests(bool isAsync) : base(isAsync)
31 {
32 }
33
34 [OneTimeSetUp]
35 public void Setup()
36 {
37 if (Mode == RecordedTestMode.Playback)
38 {
39 return;
40 }
41 fileClient = GetTestClient<OpenAIFileClient>(TestScenario.Files);
42
43 samplePath = Path.Combine("Assets", "fine_tuning_sample.jsonl");
44 validationPath = Path.Combine("Assets", "fine_tuning_sample_validation.jsonl");
45
46 sampleFile = fileClient.UploadFile(samplePath, FileUploadPurpose.FineTune);
47 sampleFileId = sampleFile.Id;
48 validationFile = fileClient.UploadFile(validationPath, FileUploadPurpose.FineTune);
49 validationFileId = validationFile.Id;
50
51 }
52
53 [SetUp]
54 public void PerTestSetUp()
55 {
56 if (Mode == RecordedTestMode.Record)
57 {
58 Recording.SetVariable("SAMPLE_FILE_ID", sampleFileId);
59 Recording.SetVariable("VALIDATION_FILE_ID", validationFileId);
60 _ = Recording.Now; // To save the date time to the recording file
61 }
62 if (Mode == RecordedTestMode.Playback)
63 {
64 var sampleFileId = Recording.GetVariable("SAMPLE_FILE_ID", null);
65 var validationFileId = Recording.GetVariable("VALIDATION_FILE_ID", null);
66
67 sampleFile = OpenAIFilesModelFactory.OpenAIFileInfo(
68 id: sampleFileId,
69 sizeInBytes: 123,
70 createdAt: Recording.Now,
71 filename: samplePath,
72 purpose: FilePurpose.FineTune);
73
74 validationFile = OpenAIFilesModelFactory.OpenAIFileInfo(
75 id: validationFileId,
76 sizeInBytes: 123,
77 createdAt: Recording.Now,
78 filename: validationPath,
79 purpose: FilePurpose.FineTune);
80 }
81 }
82
83 [OneTimeTearDown]
84 public void TearDown()
85 {
86 if (Mode == RecordedTestMode.Playback)
87 {
88 return;
89 }
90 fileClient.DeleteFile(sampleFile.Id);
91 fileClient.DeleteFile(validationFile.Id);
92 }
93
94 [RecordedTest]
95 public async Task MinimalRequiredParams()
96 {
97 FineTuningClient client = GetTestClient();
98 FineTuningJob ft = await client.FineTuneAsync("gpt-3.5-turbo", sampleFile.Id, false);
99
100 // Assert.AreEqual(0, ft.Hyperparameters.CycleCount);
101 Assert.That(ft.Status.InProgress);
102 Assert.That(ft.HasCompleted, Is.False);
103
104 await ft.CancelAndUpdateAsync();
105
106 Assert.That(ft.Status, Is.EqualTo(FineTuningStatus.Cancelled));
107 Assert.That(ft.Status.InProgress, Is.False);
108 Assert.That(ft.HasCompleted);
109 }
110
111 [RecordedTest]
112 public async Task AllParameters()
113 {
114 // This test does not check for Integrations because it requires a valid API key
115
116 var options = new FineTuningOptions()
117 {
118 TrainingMethod = FineTuningTrainingMethod.CreateSupervised(
119 epochCount: 1,
120 batchSize: 2,
121 learningRate: 3),
122 Suffix = "TestFTJob",
123 ValidationFile = validationFile.Id,
124 Seed = 1234567
125 };
126
127 FineTuningClient client = GetTestClient();
128 FineTuningJob ft = await client.FineTuneAsync("gpt-3.5-turbo", sampleFile.Id, false, options);
129
130 ft.CancelAndUpdate();
131
132#pragma warning disable CS0618
133 Assert.That(ft.Hyperparameters.EpochCount, Is.EqualTo(1));
134 Assert.That(ft.Hyperparameters.BatchSize, Is.EqualTo(2));
135 Assert.That(ft.Hyperparameters.LearningRateMultiplier, Is.EqualTo(3));
136#pragma warning restore
137
138 if (ft.MethodHyperparameters is HyperparametersForSupervised hp)
139 {
140 Assert.That(hp.EpochCount, Is.EqualTo(1));
141 Assert.That(hp.BatchSize, Is.EqualTo(2));
142 Assert.That(hp.LearningRateMultiplier, Is.EqualTo(3));
143 }
144 else
145 {
146 Assert.Fail($"Expected HyperparametersForSupervised, got {ft.MethodHyperparameters?.GetType().ToString() ?? "null"}");
147 }
148
149 Assert.That(ft.UserProvidedSuffix, Is.EqualTo("TestFTJob"));
150 Assert.That(ft.Seed, Is.EqualTo(1234567));
151 Assert.That(ft.ValidationFileId, Is.EqualTo(validationFile.Id));
152 }
153
154 [RecordedTest]
155 [Explicit("This test requires wandb.ai account and api key integration.")]
156 public void WandBIntegrations()
157 {
158 FineTuningClient client = GetTestClient();
159 FineTuningJob job = client.FineTune(
160 "gpt-3.5-turbo",
161 sampleFile.Id,
162 false, options: new()
163 {
164 Integrations = { new WeightsAndBiasesIntegration("ft-tests") },
165 }
166 );
167 job.CancelAndUpdate();
168 }
169
170 [RecordedTest]
171 public void ExceptionThrownOnInvalidFileName()
172 {
173 FineTuningClient client = GetTestClient();
174 Assert.ThrowsAsync<ClientResultException>(async () =>
175 await client.FineTuneAsync(baseModel: "gpt-3.5-turbo", trainingFileId: "Invalid File Name", waitUntilCompleted: false)
176 );
177 }
178
179 [RecordedTest]
180 public void ExceptionThrownOnInvalidModelName()
181 {
182 FineTuningClient client = GetTestClient();
183 Assert.ThrowsAsync<ClientResultException>(async () =>
184 await client.FineTuneAsync(baseModel: "gpt-nonexistent", trainingFileId: sampleFile.Id, waitUntilCompleted: false)
185 );
186 }
187
188 [RecordedTest]
189 public void ExceptionThrownOnInvalidValidationIdAsync()
190 {
191 FineTuningClient client = GetTestClient();
192 Assert.ThrowsAsync<ClientResultException>(async () =>
193 {
194 await client.FineTuneAsync(
195 "gpt-3.5-turbo",
196 sampleFile.Id,
197 false, new() { ValidationFile = "7" }
198 );
199 });
200 }
201
202 [RecordedTest]
203 public void GetJobs()
204 {
205 FineTuningClient client = GetTestClient();
206
207 // Arrange
208 Console.WriteLine("Getting jobs");
209 var jobs = client.GetJobsAsync().Take(10).ToBlockingEnumerable();
210
211 Console.WriteLine("Got jobs");
212
213 // Act
214 var counter = 0;
215 foreach (var job in jobs) // Network call will happen here on first iteration.
216 {
217 Console.WriteLine($"{counter} jobs");
218 Console.WriteLine($"Job: {job.JobId}");
219 Assert.That(job.JobId.StartsWith("ftjob"));
220 counter++;
221 }
222 Console.WriteLine($"Got {counter} jobs");
223
224 // Assert
225 Assert.That(counter, Is.GreaterThan(0));
226 Assert.That(counter, Is.LessThanOrEqualTo(10));
227 }
228
229 [RecordedTest]
230 public async Task GetJobsWithAfter()
231 {
232 FineTuningClient client = GetTestClient();
233 var firstJob = await client.GetJobsAsync().FirstAsync();
234
235 if (firstJob is null)
236 {
237 Assert.Fail("No jobs found. At least 2 jobs have to be found to run this test.");
238 }
239 var secondJob = await client.GetJobsAsync(new() { AfterJobId = firstJob.JobId }).FirstAsync();
240
241 Assert.That(secondJob.JobId, Is.Not.EqualTo(firstJob.JobId));
242 // Can't assert that one was created after the next because they might be created at the same second.
243 // Assert.Greater(secondJob.CreatedAt, firstJob.CreatedAt, $"{firstJob}, {secondJob}");
244 }
245
246 /// Manual experiments show that there are always at least 2 events:
247 /// First one is that the job is created
248 /// Second one is "validating training file"
249 /// If this test starts failing because of the wrong count, please first check if the above is still true
250 [RecordedTest]
251 public async Task GetJobEvents()
252 {
253 FineTuningClient client = GetTestClient();
254 // Arrange
255 FineTuningJob job = await client.FineTuneAsync("gpt-3.5-turbo", sampleFile.Id, false);
256
257 GetEventsOptions options = new()
258 {
259 PageSize = 1
260 };
261 job.CancelAndUpdate();
262
263 // Act
264 var events = IsAsync
265 ? job.GetEventsAsync(options).ToBlockingEnumerable()
266 : job.GetEvents(options);
267
268 var first = events.FirstOrDefault();
269
270 // Assert
271 if (first is null)
272 {
273 Assert.Fail("No events found.");
274 }
275 }
276
277 [RecordedTest]
278 public async Task GetCheckpoints()
279 {
280 FineTuningClient client = GetTestClient();
281 // Arrange
282 // TODO: When `status` option becomes available, use it to get a succeeded job
283 FineTuningJob job = await client.GetJobsAsync(new() { PageSize = 100 })
284 .Where((job) => job.Status == "succeeded").FirstAsync();
285
286
287 // Act
288 var checkpoints = IsAsync
289 ? job.GetCheckpointsAsync().ToBlockingEnumerable()
290 : job.GetCheckpoints();
291 FineTuningCheckpoint first = checkpoints.FirstOrDefault();
292
293 // Assert
294 if (first is null)
295 {
296 Assert.Fail("No checkpoints found.");
297 }
298
299 FineTuningCheckpointMetrics metrics = first.Metrics;
300 Assert.That(metrics, Is.Not.Null);
301 Assert.That(metrics.StepNumber, Is.GreaterThan(0));
302 }
303
304 private FineTuningClient GetTestClient() => GetProxiedOpenAIClient<FineTuningClient>(TestScenario.FineTuning);
305}