GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

openai/openai-dotnet

Public

mirrored from https://github.com/openai/openai-dotnetAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
achandmsft-patch-2

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openai-dotnet/tests/FineTuning

tests/FineTuning/FineTuningClientTests.cs

301lines · modecode

RawDownload
Latest commit unavailable.
1using NUnit.Framework;
2using NUnit.Framework.Internal;
3using OpenAI.Files;
4using OpenAI.FineTuning;
5using System;
6using System.ClientModel;
7using System.IO;
8using System.Linq;
9using System.Threading.Tasks;
10using static OpenAI.Tests.TestHelpers;
11
12namespace OpenAI.Tests.FineTuning;
13
14public static class Extensions
15{
16 public static bool IsAsync(this Method method) => method == Method.Async;
17}
18
19public enum Method
20{
21 Sync,
22 Async
23}
24
25
26[TestFixture]
27[Parallelizable(ParallelScope.Fixtures)]
28[Category("FineTuning")]
29public class FineTuningClientTests
30{
31
32
33 FineTuningClient client;
34 OpenAIFileClient fileClient;
35
36 string samplePath;
37 string validationPath;
38
39 OpenAIFile sampleFile;
40 OpenAIFile validationFile;
41
42 [OneTimeSetUp]
43 public void Setup()
44 {
45 client = GetTestClient();
46 fileClient = GetTestClient<OpenAIFileClient>(TestScenario.Files);
47
48 samplePath = Path.Combine("Assets", "fine_tuning_sample.jsonl");
49 validationPath = Path.Combine("Assets", "fine_tuning_sample_validation.jsonl");
50
51 sampleFile = fileClient.UploadFile(samplePath, FileUploadPurpose.FineTune);
52 validationFile = fileClient.UploadFile(validationPath, FileUploadPurpose.FineTune);
53
54 }
55
56 [OneTimeTearDown]
57 public void TearDown()
58 {
59 fileClient.DeleteFile(sampleFile.Id);
60 fileClient.DeleteFile(validationFile.Id);
61 }
62
63
64
65 [Test]
66 [Parallelizable(ParallelScope.All)]
67 public async Task MinimalRequiredParams([Values(Method.Sync, Method.Async)] Method method)
68 {
69 FineTuningJob ft = method.IsAsync()
70 ? await client.FineTuneAsync("gpt-3.5-turbo", sampleFile.Id, false)
71 : client.FineTune("gpt-3.5-turbo", sampleFile.Id, false);
72
73 // Assert.AreEqual(0, ft.Hyperparameters.CycleCount);
74 Assert.True(ft.Status.InProgress);
75 Assert.False(ft.HasCompleted);
76
77 _ = method.IsAsync()
78 ? await ft.CancelAndUpdateAsync()
79 : ft.CancelAndUpdate();
80
81 Assert.AreEqual(FineTuningStatus.Cancelled, ft.Status);
82 Assert.False(ft.Status.InProgress);
83 Assert.True(ft.HasCompleted);
84 }
85
86
87
88 [Test]
89 [Parallelizable(ParallelScope.All)]
90 public async Task AllParameters([Values(Method.Sync, Method.Async)] Method method)
91 {
92 // This test does not check for Integrations because it requires a valid API key
93
94 var options = new FineTuningOptions()
95 {
96 TrainingMethod = FineTuningTrainingMethod.CreateSupervised(
97 epochCount: 1,
98 batchSize: 2,
99 learningRate: 3),
100 Suffix = "TestFTJob",
101 ValidationFile = validationFile.Id,
102 Seed = 1234567
103 };
104
105
106 FineTuningJob ft = method.IsAsync()
107 ? await client.FineTuneAsync("gpt-3.5-turbo", sampleFile.Id, false, options)
108 : client.FineTune("gpt-3.5-turbo", sampleFile.Id, false, options);
109
110 ft.CancelAndUpdate();
111
112#pragma warning disable CS0618
113 Assert.AreEqual(1, ft.Hyperparameters.EpochCount);
114 Assert.AreEqual(2, ft.Hyperparameters.BatchSize);
115 Assert.AreEqual(3, ft.Hyperparameters.LearningRateMultiplier);
116#pragma warning restore
117
118 if (ft.MethodHyperparameters is HyperparametersForSupervised hp)
119 {
120 Assert.AreEqual(1, hp.EpochCount);
121 Assert.AreEqual(2, hp.BatchSize);
122 Assert.AreEqual(3, hp.LearningRateMultiplier);
123 }
124 else
125 {
126 Assert.Fail($"Expected HyperparametersForSupervised, got {ft.MethodHyperparameters?.GetType().ToString() ?? "null"}");
127 }
128
129 Assert.AreEqual(ft.UserProvidedSuffix, "TestFTJob");
130 Assert.AreEqual(1234567, ft.Seed);
131 Assert.AreEqual(validationFile.Id, ft.ValidationFileId);
132 }
133
134 [Test]
135 [Parallelizable]
136 [Explicit("This test requires wandb.ai account and api key integration.")]
137 public void WandBIntegrations()
138 {
139 FineTuningJob job = client.FineTune(
140 "gpt-3.5-turbo",
141 sampleFile.Id,
142 false, options: new()
143 {
144 Integrations = { new WeightsAndBiasesIntegration("ft-tests") },
145 }
146 );
147 job.CancelAndUpdate();
148 }
149
150 [Test]
151 [Parallelizable]
152 public void ExceptionThrownOnInvalidFileName()
153 {
154 Assert.Throws<ClientResultException>(() =>
155 client.FineTune(baseModel: "gpt-3.5-turbo", trainingFileId: "Invalid File Name", waitUntilCompleted: false)
156 );
157 }
158
159 [Test]
160 [Parallelizable]
161 public void ExceptionThrownOnInvalidModelName()
162 {
163 Assert.Throws<ClientResultException>(() =>
164 client.FineTune(baseModel: "gpt-nonexistent", trainingFileId: sampleFile.Id, waitUntilCompleted: false)
165 );
166 }
167
168 [Test]
169 [Parallelizable]
170 public void ExceptionThrownOnInvalidValidationId()
171 {
172 Assert.Throws<ClientResultException>(() =>
173 {
174 client.FineTune(
175 "gpt-3.5-turbo",
176 sampleFile.Id,
177 false, new() { ValidationFile = "7" }
178 );
179 });
180 }
181
182 [Test]
183 [Parallelizable]
184 public void ExceptionThrownOnInvalidValidationIdAsync()
185 {
186 Assert.ThrowsAsync<ClientResultException>(async () =>
187 {
188 await client.FineTuneAsync(
189 "gpt-3.5-turbo",
190 sampleFile.Id,
191 false, new() { ValidationFile = "7" }
192 );
193 });
194 }
195
196 [Test]
197 [Parallelizable(ParallelScope.All)]
198 public void GetJobs([Values(Method.Sync, Method.Async)] Method method)
199 {
200 // Arrange
201 Console.WriteLine("Getting jobs");
202 var jobs = method.IsAsync()
203 ? client.GetJobsAsync().Take(10).ToBlockingEnumerable()
204 : client.GetJobs().Take(10);
205
206 Console.WriteLine("Got jobs");
207
208 // Act
209 var counter = 0;
210 foreach (var job in jobs) // Network call will happen here on first iteration.
211 {
212 Console.WriteLine($"{counter} jobs");
213 Console.WriteLine($"Job: {job.JobId}");
214 Assert.IsTrue(job.JobId.StartsWith("ftjob"));
215 counter++;
216 }
217 Console.WriteLine($"Got {counter} jobs");
218
219 // Assert
220 Assert.Greater(counter, 0);
221 Assert.LessOrEqual(counter, 10);
222 }
223
224 [Test]
225 [Parallelizable]
226 public void GetJobsWithAfter()
227 {
228 var firstJob = client.GetJobs().First();
229
230 if (firstJob is null)
231 {
232 Assert.Fail("No jobs found. At least 2 jobs have to be found to run this test.");
233 }
234 var secondJob = client.GetJobs(new() { AfterJobId = firstJob.JobId }).First();
235
236 Assert.AreNotEqual(firstJob.JobId, secondJob.JobId);
237 // Can't assert that one was created after the next because they might be created at the same second.
238 // Assert.Greater(secondJob.CreatedAt, firstJob.CreatedAt, $"{firstJob}, {secondJob}");
239 }
240
241 /// Manual experiments show that there are always at least 2 events:
242 /// First one is that the job is created
243 /// Second one is "validating training file"
244 /// If this test starts failing because of the wrong count, please first check if the above is still true
245 [Test]
246 [Parallelizable(ParallelScope.All)]
247 public void GetJobEvents([Values(Method.Sync, Method.Async)] Method method)
248 {
249 // Arrange
250 FineTuningJob job = client.FineTune("gpt-3.5-turbo", sampleFile.Id, false);
251
252 GetEventsOptions options = new()
253 {
254 PageSize = 1
255 };
256 job.CancelAndUpdate();
257
258 // Act
259 var events = method.IsAsync()
260 ? job.GetEventsAsync(options).ToBlockingEnumerable()
261 : job.GetEvents(options);
262
263 var first = events.FirstOrDefault();
264
265 // Assert
266 if (first is null)
267 {
268 Assert.Fail("No events found.");
269 }
270 }
271
272 [Test]
273 [Parallelizable(ParallelScope.All)]
274 public void GetCheckpoints([Values(Method.Sync, Method.Async)] Method method)
275 {
276 // Arrange
277 // TODO: When `status` option becomes available, use it to get a succeeded job
278 FineTuningJob job = client.GetJobs(new() { PageSize = 100 })
279 .Where((job) => job.Status == "succeeded")
280 .First();
281
282
283 // Act
284 var checkpoints = method.IsAsync()
285 ? job.GetCheckpointsAsync().ToBlockingEnumerable()
286 : job.GetCheckpoints();
287 FineTuningCheckpoint first = checkpoints.FirstOrDefault();
288
289 // Assert
290 if (first is null)
291 {
292 Assert.Fail("No checkpoints found.");
293 }
294
295 FineTuningCheckpointMetrics metrics = first.Metrics;
296 Assert.NotNull(metrics);
297 Assert.Greater(metrics.StepNumber, 0);
298 }
299
300 private static FineTuningClient GetTestClient() => GetTestClient<FineTuningClient>(TestScenario.FineTuning);
301}