GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

openai/openai-python

Public

mirrored from https://github.com/openai/openai-pythonAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
codex/gate-pypi-publish

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openai-python/tests/lib/chat

tests/lib/chat/test_completions.py

1001lines · modecode

RawDownload
Latest commit unavailable.
1from __future__ import annotations
2
3from enum import Enum
4from typing import List, Optional
5from typing_extensions import Literal, TypeVar
6
7import pytest
8from respx import MockRouter
9from pydantic import Field, BaseModel
10from inline_snapshot import snapshot
11
12import openai
13from openai import OpenAI, AsyncOpenAI
14from openai._utils import assert_signatures_in_sync
15from openai._compat import PYDANTIC_V1
16
17from ..utils import print_obj
18from ...conftest import base_url
19from ..snapshots import make_snapshot_request, make_async_snapshot_request
20from ..schema_types.query import Query
21
22_T = TypeVar("_T")
23
24# all the snapshots in this file are auto-generated from the live API
25#
26# you can update them with
27#
28# `OPENAI_LIVE=1 pytest --inline-snapshot=fix -p no:xdist -o addopts=""`
29
30
31@pytest.mark.respx(base_url=base_url)
32def test_parse_nothing(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
33 completion = make_snapshot_request(
34 lambda c: c.chat.completions.parse(
35 model="gpt-4o-2024-08-06",
36 messages=[
37 {
38 "role": "user",
39 "content": "What's the weather like in SF?",
40 },
41 ],
42 ),
43 content_snapshot=snapshot(
44 '{"id": "chatcmpl-ABfvaueLEMLNYbT8YzpJxsmiQ6HSY", "object": "chat.completion", "created": 1727346142, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "I\'m unable to provide real-time weather updates. To get the current weather in San Francisco, I recommend checking a reliable weather website or app like the Weather Channel or a local news station.", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 14, "completion_tokens": 37, "total_tokens": 51, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_b40fb1c6fb"}'
45 ),
46 path="/chat/completions",
47 mock_client=client,
48 respx_mock=respx_mock,
49 )
50
51 assert print_obj(completion, monkeypatch) == snapshot(
52 """\
53ParsedChatCompletion(
54 choices=[
55 ParsedChoice(
56 finish_reason='stop',
57 index=0,
58 logprobs=None,
59 message=ParsedChatCompletionMessage(
60 annotations=None,
61 audio=None,
62 content="I'm unable to provide real-time weather updates. To get the current weather in San Francisco, I
63recommend checking a reliable weather website or app like the Weather Channel or a local news station.",
64 function_call=None,
65 parsed=None,
66 refusal=None,
67 role='assistant',
68 tool_calls=None
69 )
70 )
71 ],
72 created=1727346142,
73 id='chatcmpl-ABfvaueLEMLNYbT8YzpJxsmiQ6HSY',
74 model='gpt-4o-2024-08-06',
75 moderation=None,
76 object='chat.completion',
77 service_tier=None,
78 system_fingerprint='fp_b40fb1c6fb',
79 usage=CompletionUsage(
80 completion_tokens=37,
81 completion_tokens_details=CompletionTokensDetails(
82 accepted_prediction_tokens=None,
83 audio_tokens=None,
84 reasoning_tokens=0,
85 rejected_prediction_tokens=None
86 ),
87 prompt_tokens=14,
88 prompt_tokens_details=None,
89 total_tokens=51
90 )
91)
92"""
93 )
94
95
96@pytest.mark.respx(base_url=base_url)
97def test_parse_pydantic_model(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
98 class Location(BaseModel):
99 city: str
100 temperature: float
101 units: Literal["c", "f"]
102
103 completion = make_snapshot_request(
104 lambda c: c.chat.completions.parse(
105 model="gpt-4o-2024-08-06",
106 messages=[
107 {
108 "role": "user",
109 "content": "What's the weather like in SF?",
110 },
111 ],
112 response_format=Location,
113 ),
114 content_snapshot=snapshot(
115 '{"id": "chatcmpl-ABfvbtVnTu5DeC4EFnRYj8mtfOM99", "object": "chat.completion", "created": 1727346143, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":65,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 79, "completion_tokens": 14, "total_tokens": 93, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
116 ),
117 path="/chat/completions",
118 mock_client=client,
119 respx_mock=respx_mock,
120 )
121
122 assert print_obj(completion, monkeypatch) == snapshot(
123 """\
124ParsedChatCompletion(
125 choices=[
126 ParsedChoice(
127 finish_reason='stop',
128 index=0,
129 logprobs=None,
130 message=ParsedChatCompletionMessage(
131 annotations=None,
132 audio=None,
133 content='{"city":"San Francisco","temperature":65,"units":"f"}',
134 function_call=None,
135 parsed=Location(city='San Francisco', temperature=65.0, units='f'),
136 refusal=None,
137 role='assistant',
138 tool_calls=None
139 )
140 )
141 ],
142 created=1727346143,
143 id='chatcmpl-ABfvbtVnTu5DeC4EFnRYj8mtfOM99',
144 model='gpt-4o-2024-08-06',
145 moderation=None,
146 object='chat.completion',
147 service_tier=None,
148 system_fingerprint='fp_5050236cbd',
149 usage=CompletionUsage(
150 completion_tokens=14,
151 completion_tokens_details=CompletionTokensDetails(
152 accepted_prediction_tokens=None,
153 audio_tokens=None,
154 reasoning_tokens=0,
155 rejected_prediction_tokens=None
156 ),
157 prompt_tokens=79,
158 prompt_tokens_details=None,
159 total_tokens=93
160 )
161)
162"""
163 )
164
165
166@pytest.mark.respx(base_url=base_url)
167def test_parse_pydantic_model_optional_default(
168 client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch
169) -> None:
170 class Location(BaseModel):
171 city: str
172 temperature: float
173 units: Optional[Literal["c", "f"]] = None
174
175 completion = make_snapshot_request(
176 lambda c: c.chat.completions.parse(
177 model="gpt-4o-2024-08-06",
178 messages=[
179 {
180 "role": "user",
181 "content": "What's the weather like in SF?",
182 },
183 ],
184 response_format=Location,
185 ),
186 content_snapshot=snapshot(
187 '{"id": "chatcmpl-ABfvcC8grKYsRkSoMp9CCAhbXAd0b", "object": "chat.completion", "created": 1727346144, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":65,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 88, "completion_tokens": 14, "total_tokens": 102, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_b40fb1c6fb"}'
188 ),
189 path="/chat/completions",
190 mock_client=client,
191 respx_mock=respx_mock,
192 )
193
194 assert print_obj(completion, monkeypatch) == snapshot(
195 """\
196ParsedChatCompletion(
197 choices=[
198 ParsedChoice(
199 finish_reason='stop',
200 index=0,
201 logprobs=None,
202 message=ParsedChatCompletionMessage(
203 annotations=None,
204 audio=None,
205 content='{"city":"San Francisco","temperature":65,"units":"f"}',
206 function_call=None,
207 parsed=Location(city='San Francisco', temperature=65.0, units='f'),
208 refusal=None,
209 role='assistant',
210 tool_calls=None
211 )
212 )
213 ],
214 created=1727346144,
215 id='chatcmpl-ABfvcC8grKYsRkSoMp9CCAhbXAd0b',
216 model='gpt-4o-2024-08-06',
217 moderation=None,
218 object='chat.completion',
219 service_tier=None,
220 system_fingerprint='fp_b40fb1c6fb',
221 usage=CompletionUsage(
222 completion_tokens=14,
223 completion_tokens_details=CompletionTokensDetails(
224 accepted_prediction_tokens=None,
225 audio_tokens=None,
226 reasoning_tokens=0,
227 rejected_prediction_tokens=None
228 ),
229 prompt_tokens=88,
230 prompt_tokens_details=None,
231 total_tokens=102
232 )
233)
234"""
235 )
236
237
238@pytest.mark.respx(base_url=base_url)
239def test_parse_pydantic_model_enum(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
240 class Color(Enum):
241 """The detected color"""
242
243 RED = "red"
244 BLUE = "blue"
245 GREEN = "green"
246
247 class ColorDetection(BaseModel):
248 color: Color
249 hex_color_code: str = Field(description="The hex color code of the detected color")
250
251 if PYDANTIC_V1:
252 ColorDetection.update_forward_refs(**locals()) # type: ignore
253
254 completion = make_snapshot_request(
255 lambda c: c.chat.completions.parse(
256 model="gpt-4o-2024-08-06",
257 messages=[
258 {"role": "user", "content": "What color is a Coke can?"},
259 ],
260 response_format=ColorDetection,
261 ),
262 content_snapshot=snapshot(
263 '{"id": "chatcmpl-ABfvjIatz0zrZu50gRbMtlp0asZpz", "object": "chat.completion", "created": 1727346151, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"color\\":\\"red\\",\\"hex_color_code\\":\\"#FF0000\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 109, "completion_tokens": 14, "total_tokens": 123, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
264 ),
265 path="/chat/completions",
266 mock_client=client,
267 respx_mock=respx_mock,
268 )
269
270 assert print_obj(completion.choices[0], monkeypatch) == snapshot(
271 """\
272ParsedChoice(
273 finish_reason='stop',
274 index=0,
275 logprobs=None,
276 message=ParsedChatCompletionMessage(
277 annotations=None,
278 audio=None,
279 content='{"color":"red","hex_color_code":"#FF0000"}',
280 function_call=None,
281 parsed=ColorDetection(color=<Color.RED: 'red'>, hex_color_code='#FF0000'),
282 refusal=None,
283 role='assistant',
284 tool_calls=None
285 )
286)
287"""
288 )
289
290
291@pytest.mark.respx(base_url=base_url)
292def test_parse_pydantic_model_multiple_choices(
293 client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch
294) -> None:
295 class Location(BaseModel):
296 city: str
297 temperature: float
298 units: Literal["c", "f"]
299
300 completion = make_snapshot_request(
301 lambda c: c.chat.completions.parse(
302 model="gpt-4o-2024-08-06",
303 messages=[
304 {
305 "role": "user",
306 "content": "What's the weather like in SF?",
307 },
308 ],
309 n=3,
310 response_format=Location,
311 ),
312 content_snapshot=snapshot(
313 '{"id": "chatcmpl-ABfvp8qzboW92q8ONDF4DPHlI7ckC", "object": "chat.completion", "created": 1727346157, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":64,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}, {"index": 1, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":65,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}, {"index": 2, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":63.0,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 79, "completion_tokens": 44, "total_tokens": 123, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_b40fb1c6fb"}'
314 ),
315 path="/chat/completions",
316 mock_client=client,
317 respx_mock=respx_mock,
318 )
319
320 assert print_obj(completion.choices, monkeypatch) == snapshot(
321 """\
322[
323 ParsedChoice(
324 finish_reason='stop',
325 index=0,
326 logprobs=None,
327 message=ParsedChatCompletionMessage(
328 annotations=None,
329 audio=None,
330 content='{"city":"San Francisco","temperature":64,"units":"f"}',
331 function_call=None,
332 parsed=Location(city='San Francisco', temperature=64.0, units='f'),
333 refusal=None,
334 role='assistant',
335 tool_calls=None
336 )
337 ),
338 ParsedChoice(
339 finish_reason='stop',
340 index=1,
341 logprobs=None,
342 message=ParsedChatCompletionMessage(
343 annotations=None,
344 audio=None,
345 content='{"city":"San Francisco","temperature":65,"units":"f"}',
346 function_call=None,
347 parsed=Location(city='San Francisco', temperature=65.0, units='f'),
348 refusal=None,
349 role='assistant',
350 tool_calls=None
351 )
352 ),
353 ParsedChoice(
354 finish_reason='stop',
355 index=2,
356 logprobs=None,
357 message=ParsedChatCompletionMessage(
358 annotations=None,
359 audio=None,
360 content='{"city":"San Francisco","temperature":63.0,"units":"f"}',
361 function_call=None,
362 parsed=Location(city='San Francisco', temperature=63.0, units='f'),
363 refusal=None,
364 role='assistant',
365 tool_calls=None
366 )
367 )
368]
369"""
370 )
371
372
373@pytest.mark.respx(base_url=base_url)
374@pytest.mark.skipif(PYDANTIC_V1, reason="dataclasses only supported in v2")
375def test_parse_pydantic_dataclass(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
376 from pydantic.dataclasses import dataclass
377
378 @dataclass
379 class CalendarEvent:
380 name: str
381 date: str
382 participants: List[str]
383
384 completion = make_snapshot_request(
385 lambda c: c.chat.completions.parse(
386 model="gpt-4o-2024-08-06",
387 messages=[
388 {"role": "system", "content": "Extract the event information."},
389 {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."},
390 ],
391 response_format=CalendarEvent,
392 ),
393 content_snapshot=snapshot(
394 '{"id": "chatcmpl-ABfvqhz4uUUWsw8Ohw2Mp9B4sKKV8", "object": "chat.completion", "created": 1727346158, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"name\\":\\"Science Fair\\",\\"date\\":\\"Friday\\",\\"participants\\":[\\"Alice\\",\\"Bob\\"]}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 92, "completion_tokens": 17, "total_tokens": 109, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_7568d46099"}'
395 ),
396 path="/chat/completions",
397 mock_client=client,
398 respx_mock=respx_mock,
399 )
400
401 assert print_obj(completion, monkeypatch) == snapshot(
402 """\
403ParsedChatCompletion(
404 choices=[
405 ParsedChoice(
406 finish_reason='stop',
407 index=0,
408 logprobs=None,
409 message=ParsedChatCompletionMessage(
410 annotations=None,
411 audio=None,
412 content='{"name":"Science Fair","date":"Friday","participants":["Alice","Bob"]}',
413 function_call=None,
414 parsed=CalendarEvent(name='Science Fair', date='Friday', participants=['Alice', 'Bob']),
415 refusal=None,
416 role='assistant',
417 tool_calls=None
418 )
419 )
420 ],
421 created=1727346158,
422 id='chatcmpl-ABfvqhz4uUUWsw8Ohw2Mp9B4sKKV8',
423 model='gpt-4o-2024-08-06',
424 moderation=None,
425 object='chat.completion',
426 service_tier=None,
427 system_fingerprint='fp_7568d46099',
428 usage=CompletionUsage(
429 completion_tokens=17,
430 completion_tokens_details=CompletionTokensDetails(
431 accepted_prediction_tokens=None,
432 audio_tokens=None,
433 reasoning_tokens=0,
434 rejected_prediction_tokens=None
435 ),
436 prompt_tokens=92,
437 prompt_tokens_details=None,
438 total_tokens=109
439 )
440)
441"""
442 )
443
444
445@pytest.mark.respx(base_url=base_url)
446def test_pydantic_tool_model_all_types(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
447 completion = make_snapshot_request(
448 lambda c: c.chat.completions.parse(
449 model="gpt-4o-2024-08-06",
450 messages=[
451 {
452 "role": "user",
453 "content": "look up all my orders in may of last year that were fulfilled but not delivered on time",
454 },
455 ],
456 tools=[openai.pydantic_function_tool(Query)],
457 response_format=Query,
458 ),
459 content_snapshot=snapshot(
460 '{"id": "chatcmpl-ABfvtNiaTNUF6OymZUnEFc9lPq9p1", "object": "chat.completion", "created": 1727346161, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_NKpApJybW1MzOjZO2FzwYw0d", "type": "function", "function": {"name": "Query", "arguments": "{\\"name\\":\\"May 2022 Fulfilled Orders Not Delivered on Time\\",\\"table_name\\":\\"orders\\",\\"columns\\":[\\"id\\",\\"status\\",\\"expected_delivery_date\\",\\"delivered_at\\",\\"shipped_at\\",\\"ordered_at\\",\\"canceled_at\\"],\\"conditions\\":[{\\"column\\":\\"ordered_at\\",\\"operator\\":\\">=\\",\\"value\\":\\"2022-05-01\\"},{\\"column\\":\\"ordered_at\\",\\"operator\\":\\"<=\\",\\"value\\":\\"2022-05-31\\"},{\\"column\\":\\"status\\",\\"operator\\":\\"=\\",\\"value\\":\\"fulfilled\\"},{\\"column\\":\\"delivered_at\\",\\"operator\\":\\">\\",\\"value\\":{\\"column_name\\":\\"expected_delivery_date\\"}}],\\"order_by\\":\\"asc\\"}"}}], "refusal": null}, "logprobs": null, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 512, "completion_tokens": 132, "total_tokens": 644, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_7568d46099"}'
461 ),
462 path="/chat/completions",
463 mock_client=client,
464 respx_mock=respx_mock,
465 )
466
467 assert print_obj(completion.choices[0], monkeypatch) == snapshot(
468 """\
469ParsedChoice(
470 finish_reason='tool_calls',
471 index=0,
472 logprobs=None,
473 message=ParsedChatCompletionMessage(
474 annotations=None,
475 audio=None,
476 content=None,
477 function_call=None,
478 parsed=None,
479 refusal=None,
480 role='assistant',
481 tool_calls=[
482 ParsedFunctionToolCall(
483 function=ParsedFunction(
484 arguments='{"name":"May 2022 Fulfilled Orders Not Delivered on
485Time","table_name":"orders","columns":["id","status","expected_delivery_date","delivered_at","shipped_at","ordered_at","
486canceled_at"],"conditions":[{"column":"ordered_at","operator":">=","value":"2022-05-01"},{"column":"ordered_at","operato
487r":"<=","value":"2022-05-31"},{"column":"status","operator":"=","value":"fulfilled"},{"column":"delivered_at","operator"
488:">","value":{"column_name":"expected_delivery_date"}}],"order_by":"asc"}',
489 name='Query',
490 parsed_arguments=Query(
491 columns=[
492 <Column.id: 'id'>,
493 <Column.status: 'status'>,
494 <Column.expected_delivery_date: 'expected_delivery_date'>,
495 <Column.delivered_at: 'delivered_at'>,
496 <Column.shipped_at: 'shipped_at'>,
497 <Column.ordered_at: 'ordered_at'>,
498 <Column.canceled_at: 'canceled_at'>
499 ],
500 conditions=[
501 Condition(column='ordered_at', operator=<Operator.ge: '>='>, value='2022-05-01'),
502 Condition(column='ordered_at', operator=<Operator.le: '<='>, value='2022-05-31'),
503 Condition(column='status', operator=<Operator.eq: '='>, value='fulfilled'),
504 Condition(
505 column='delivered_at',
506 operator=<Operator.gt: '>'>,
507 value=DynamicValue(column_name='expected_delivery_date')
508 )
509 ],
510 name='May 2022 Fulfilled Orders Not Delivered on Time',
511 order_by=<OrderBy.asc: 'asc'>,
512 table_name=<Table.orders: 'orders'>
513 )
514 ),
515 id='call_NKpApJybW1MzOjZO2FzwYw0d',
516 type='function'
517 )
518 ]
519 )
520)
521"""
522 )
523
524
525@pytest.mark.respx(base_url=base_url)
526def test_parse_max_tokens_reached(client: OpenAI, respx_mock: MockRouter) -> None:
527 class Location(BaseModel):
528 city: str
529 temperature: float
530 units: Literal["c", "f"]
531
532 with pytest.raises(openai.LengthFinishReasonError):
533 make_snapshot_request(
534 lambda c: c.chat.completions.parse(
535 model="gpt-4o-2024-08-06",
536 messages=[
537 {
538 "role": "user",
539 "content": "What's the weather like in SF?",
540 },
541 ],
542 max_tokens=1,
543 response_format=Location,
544 ),
545 content_snapshot=snapshot(
546 '{"id": "chatcmpl-ABfvvX7eB1KsfeZj8VcF3z7G7SbaA", "object": "chat.completion", "created": 1727346163, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"", "refusal": null}, "logprobs": null, "finish_reason": "length"}], "usage": {"prompt_tokens": 79, "completion_tokens": 1, "total_tokens": 80, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_7568d46099"}'
547 ),
548 path="/chat/completions",
549 mock_client=client,
550 respx_mock=respx_mock,
551 )
552
553
554@pytest.mark.respx(base_url=base_url)
555def test_parse_pydantic_model_refusal(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
556 class Location(BaseModel):
557 city: str
558 temperature: float
559 units: Literal["c", "f"]
560
561 completion = make_snapshot_request(
562 lambda c: c.chat.completions.parse(
563 model="gpt-4o-2024-08-06",
564 messages=[
565 {
566 "role": "user",
567 "content": "How do I make anthrax?",
568 },
569 ],
570 response_format=Location,
571 ),
572 content_snapshot=snapshot(
573 '{"id": "chatcmpl-ABfvwoKVWPQj2UPlAcAKM7s40GsRx", "object": "chat.completion", "created": 1727346164, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "refusal": "I\'m very sorry, but I can\'t assist with that."}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 79, "completion_tokens": 12, "total_tokens": 91, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
574 ),
575 path="/chat/completions",
576 mock_client=client,
577 respx_mock=respx_mock,
578 )
579
580 assert print_obj(completion.choices, monkeypatch) == snapshot(
581 """\
582[
583 ParsedChoice(
584 finish_reason='stop',
585 index=0,
586 logprobs=None,
587 message=ParsedChatCompletionMessage(
588 annotations=None,
589 audio=None,
590 content=None,
591 function_call=None,
592 parsed=None,
593 refusal="I'm very sorry, but I can't assist with that.",
594 role='assistant',
595 tool_calls=None
596 )
597 )
598]
599"""
600 )
601
602
603@pytest.mark.respx(base_url=base_url)
604def test_parse_pydantic_tool(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
605 class GetWeatherArgs(BaseModel):
606 city: str
607 country: str
608 units: Literal["c", "f"] = "c"
609
610 completion = make_snapshot_request(
611 lambda c: c.chat.completions.parse(
612 model="gpt-4o-2024-08-06",
613 messages=[
614 {
615 "role": "user",
616 "content": "What's the weather like in Edinburgh?",
617 },
618 ],
619 tools=[
620 openai.pydantic_function_tool(GetWeatherArgs),
621 ],
622 ),
623 content_snapshot=snapshot(
624 '{"id": "chatcmpl-ABfvx6Z4dchiW2nya1N8KMsHFrQRE", "object": "chat.completion", "created": 1727346165, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_Y6qJ7ofLgOrBnMD5WbVAeiRV", "type": "function", "function": {"name": "GetWeatherArgs", "arguments": "{\\"city\\":\\"Edinburgh\\",\\"country\\":\\"UK\\",\\"units\\":\\"c\\"}"}}], "refusal": null}, "logprobs": null, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 76, "completion_tokens": 24, "total_tokens": 100, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_e45dabd248"}'
625 ),
626 path="/chat/completions",
627 mock_client=client,
628 respx_mock=respx_mock,
629 )
630
631 assert print_obj(completion.choices, monkeypatch) == snapshot(
632 """\
633[
634 ParsedChoice(
635 finish_reason='tool_calls',
636 index=0,
637 logprobs=None,
638 message=ParsedChatCompletionMessage(
639 annotations=None,
640 audio=None,
641 content=None,
642 function_call=None,
643 parsed=None,
644 refusal=None,
645 role='assistant',
646 tool_calls=[
647 ParsedFunctionToolCall(
648 function=ParsedFunction(
649 arguments='{"city":"Edinburgh","country":"UK","units":"c"}',
650 name='GetWeatherArgs',
651 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='UK', units='c')
652 ),
653 id='call_Y6qJ7ofLgOrBnMD5WbVAeiRV',
654 type='function'
655 )
656 ]
657 )
658 )
659]
660"""
661 )
662
663
664@pytest.mark.respx(base_url=base_url)
665def test_parse_multiple_pydantic_tools(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
666 class GetWeatherArgs(BaseModel):
667 """Get the temperature for the given country/city combo"""
668
669 city: str
670 country: str
671 units: Literal["c", "f"] = "c"
672
673 class GetStockPrice(BaseModel):
674 ticker: str
675 exchange: str
676
677 completion = make_snapshot_request(
678 lambda c: c.chat.completions.parse(
679 model="gpt-4o-2024-08-06",
680 messages=[
681 {
682 "role": "user",
683 "content": "What's the weather like in Edinburgh?",
684 },
685 {
686 "role": "user",
687 "content": "What's the price of AAPL?",
688 },
689 ],
690 tools=[
691 openai.pydantic_function_tool(GetWeatherArgs),
692 openai.pydantic_function_tool(
693 GetStockPrice, name="get_stock_price", description="Fetch the latest price for a given ticker"
694 ),
695 ],
696 ),
697 content_snapshot=snapshot(
698 '{"id": "chatcmpl-ABfvyvfNWKcl7Ohqos4UFrmMs1v4C", "object": "chat.completion", "created": 1727346166, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_fdNz3vOBKYgOIpMdWotB9MjY", "type": "function", "function": {"name": "GetWeatherArgs", "arguments": "{\\"city\\": \\"Edinburgh\\", \\"country\\": \\"GB\\", \\"units\\": \\"c\\"}"}}, {"id": "call_h1DWI1POMJLb0KwIyQHWXD4p", "type": "function", "function": {"name": "get_stock_price", "arguments": "{\\"ticker\\": \\"AAPL\\", \\"exchange\\": \\"NASDAQ\\"}"}}], "refusal": null}, "logprobs": null, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 149, "completion_tokens": 60, "total_tokens": 209, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_b40fb1c6fb"}'
699 ),
700 path="/chat/completions",
701 mock_client=client,
702 respx_mock=respx_mock,
703 )
704
705 assert print_obj(completion.choices, monkeypatch) == snapshot(
706 """\
707[
708 ParsedChoice(
709 finish_reason='tool_calls',
710 index=0,
711 logprobs=None,
712 message=ParsedChatCompletionMessage(
713 annotations=None,
714 audio=None,
715 content=None,
716 function_call=None,
717 parsed=None,
718 refusal=None,
719 role='assistant',
720 tool_calls=[
721 ParsedFunctionToolCall(
722 function=ParsedFunction(
723 arguments='{"city": "Edinburgh", "country": "GB", "units": "c"}',
724 name='GetWeatherArgs',
725 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='GB', units='c')
726 ),
727 id='call_fdNz3vOBKYgOIpMdWotB9MjY',
728 type='function'
729 ),
730 ParsedFunctionToolCall(
731 function=ParsedFunction(
732 arguments='{"ticker": "AAPL", "exchange": "NASDAQ"}',
733 name='get_stock_price',
734 parsed_arguments=GetStockPrice(exchange='NASDAQ', ticker='AAPL')
735 ),
736 id='call_h1DWI1POMJLb0KwIyQHWXD4p',
737 type='function'
738 )
739 ]
740 )
741 )
742]
743"""
744 )
745
746
747@pytest.mark.respx(base_url=base_url)
748def test_parse_strict_tools(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
749 completion = make_snapshot_request(
750 lambda c: c.chat.completions.parse(
751 model="gpt-4o-2024-08-06",
752 messages=[
753 {
754 "role": "user",
755 "content": "What's the weather like in SF?",
756 },
757 ],
758 tools=[
759 {
760 "type": "function",
761 "function": {
762 "name": "get_weather",
763 "parameters": {
764 "type": "object",
765 "properties": {
766 "city": {"type": "string"},
767 "state": {"type": "string"},
768 },
769 "required": [
770 "city",
771 "state",
772 ],
773 "additionalProperties": False,
774 },
775 "strict": True,
776 },
777 }
778 ],
779 ),
780 content_snapshot=snapshot(
781 '{"id": "chatcmpl-ABfvzdvCI6RaIkiEFNjqGXCSYnlzf", "object": "chat.completion", "created": 1727346167, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_CUdUoJpsWWVdxXntucvnol1M", "type": "function", "function": {"name": "get_weather", "arguments": "{\\"city\\":\\"San Francisco\\",\\"state\\":\\"CA\\"}"}}], "refusal": null}, "logprobs": null, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 48, "completion_tokens": 19, "total_tokens": 67, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
782 ),
783 path="/chat/completions",
784 mock_client=client,
785 respx_mock=respx_mock,
786 )
787
788 assert print_obj(completion.choices, monkeypatch) == snapshot(
789 """\
790[
791 ParsedChoice(
792 finish_reason='tool_calls',
793 index=0,
794 logprobs=None,
795 message=ParsedChatCompletionMessage(
796 annotations=None,
797 audio=None,
798 content=None,
799 function_call=None,
800 parsed=None,
801 refusal=None,
802 role='assistant',
803 tool_calls=[
804 ParsedFunctionToolCall(
805 function=ParsedFunction(
806 arguments='{"city":"San Francisco","state":"CA"}',
807 name='get_weather',
808 parsed_arguments={'city': 'San Francisco', 'state': 'CA'}
809 ),
810 id='call_CUdUoJpsWWVdxXntucvnol1M',
811 type='function'
812 )
813 ]
814 )
815 )
816]
817"""
818 )
819
820
821def test_parse_non_strict_tools(client: OpenAI) -> None:
822 with pytest.raises(
823 ValueError, match="`get_weather` is not strict. Only `strict` function tools can be auto-parsed"
824 ):
825 client.chat.completions.parse(
826 model="gpt-4o-2024-08-06",
827 messages=[],
828 tools=[
829 {
830 "type": "function",
831 "function": {
832 "name": "get_weather",
833 "parameters": {},
834 },
835 }
836 ],
837 )
838
839
840@pytest.mark.respx(base_url=base_url)
841def test_parse_pydantic_raw_response(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
842 class Location(BaseModel):
843 city: str
844 temperature: float
845 units: Literal["c", "f"]
846
847 response = make_snapshot_request(
848 lambda c: c.chat.completions.with_raw_response.parse(
849 model="gpt-4o-2024-08-06",
850 messages=[
851 {
852 "role": "user",
853 "content": "What's the weather like in SF?",
854 },
855 ],
856 response_format=Location,
857 ),
858 content_snapshot=snapshot(
859 '{"id": "chatcmpl-ABrDYCa8W1w66eUxKDO8TQF1m6trT", "object": "chat.completion", "created": 1727389540, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":58,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 79, "completion_tokens": 14, "total_tokens": 93, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
860 ),
861 path="/chat/completions",
862 mock_client=client,
863 respx_mock=respx_mock,
864 )
865 assert response.http_request.headers.get("x-stainless-helper-method") == "chat.completions.parse"
866
867 completion = response.parse()
868 message = completion.choices[0].message
869 assert message.parsed is not None
870 assert isinstance(message.parsed.city, str)
871 assert print_obj(completion, monkeypatch) == snapshot(
872 """\
873ParsedChatCompletion(
874 choices=[
875 ParsedChoice(
876 finish_reason='stop',
877 index=0,
878 logprobs=None,
879 message=ParsedChatCompletionMessage(
880 annotations=None,
881 audio=None,
882 content='{"city":"San Francisco","temperature":58,"units":"f"}',
883 function_call=None,
884 parsed=Location(city='San Francisco', temperature=58.0, units='f'),
885 refusal=None,
886 role='assistant',
887 tool_calls=None
888 )
889 )
890 ],
891 created=1727389540,
892 id='chatcmpl-ABrDYCa8W1w66eUxKDO8TQF1m6trT',
893 model='gpt-4o-2024-08-06',
894 moderation=None,
895 object='chat.completion',
896 service_tier=None,
897 system_fingerprint='fp_5050236cbd',
898 usage=CompletionUsage(
899 completion_tokens=14,
900 completion_tokens_details=CompletionTokensDetails(
901 accepted_prediction_tokens=None,
902 audio_tokens=None,
903 reasoning_tokens=0,
904 rejected_prediction_tokens=None
905 ),
906 prompt_tokens=79,
907 prompt_tokens_details=None,
908 total_tokens=93
909 )
910)
911"""
912 )
913
914
915@pytest.mark.respx(base_url=base_url)
916@pytest.mark.asyncio
917async def test_async_parse_pydantic_raw_response(
918 async_client: AsyncOpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch
919) -> None:
920 class Location(BaseModel):
921 city: str
922 temperature: float
923 units: Literal["c", "f"]
924
925 response = await make_async_snapshot_request(
926 lambda c: c.chat.completions.with_raw_response.parse(
927 model="gpt-4o-2024-08-06",
928 messages=[
929 {
930 "role": "user",
931 "content": "What's the weather like in SF?",
932 },
933 ],
934 response_format=Location,
935 ),
936 content_snapshot=snapshot(
937 '{"id": "chatcmpl-ABrDQWOiw0PK5JOsxl1D9ooeQgznq", "object": "chat.completion", "created": 1727389532, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":65,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 79, "completion_tokens": 14, "total_tokens": 93, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
938 ),
939 path="/chat/completions",
940 mock_client=async_client,
941 respx_mock=respx_mock,
942 )
943 assert response.http_request.headers.get("x-stainless-helper-method") == "chat.completions.parse"
944
945 completion = response.parse()
946 message = completion.choices[0].message
947 assert message.parsed is not None
948 assert isinstance(message.parsed.city, str)
949 assert print_obj(completion, monkeypatch) == snapshot(
950 """\
951ParsedChatCompletion(
952 choices=[
953 ParsedChoice(
954 finish_reason='stop',
955 index=0,
956 logprobs=None,
957 message=ParsedChatCompletionMessage(
958 annotations=None,
959 audio=None,
960 content='{"city":"San Francisco","temperature":65,"units":"f"}',
961 function_call=None,
962 parsed=Location(city='San Francisco', temperature=65.0, units='f'),
963 refusal=None,
964 role='assistant',
965 tool_calls=None
966 )
967 )
968 ],
969 created=1727389532,
970 id='chatcmpl-ABrDQWOiw0PK5JOsxl1D9ooeQgznq',
971 model='gpt-4o-2024-08-06',
972 moderation=None,
973 object='chat.completion',
974 service_tier=None,
975 system_fingerprint='fp_5050236cbd',
976 usage=CompletionUsage(
977 completion_tokens=14,
978 completion_tokens_details=CompletionTokensDetails(
979 accepted_prediction_tokens=None,
980 audio_tokens=None,
981 reasoning_tokens=0,
982 rejected_prediction_tokens=None
983 ),
984 prompt_tokens=79,
985 prompt_tokens_details=None,
986 total_tokens=93
987 )
988)
989"""
990 )
991
992
993@pytest.mark.parametrize("sync", [True, False], ids=["sync", "async"])
994def test_parse_method_in_sync(sync: bool, client: OpenAI, async_client: AsyncOpenAI) -> None:
995 checking_client: OpenAI | AsyncOpenAI = client if sync else async_client
996
997 assert_signatures_in_sync(
998 checking_client.chat.completions.create,
999 checking_client.chat.completions.parse,
1000 exclude_params={"response_format", "stream"},
1001 )
1002