GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

openai/openai-python

Public

mirrored from https://github.com/openai/openai-pythonAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
50de514b910aced32104833acd892bebfb2cf123

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openai-python/tests/lib/chat

tests/lib/chat/test_completions.py

1007lines · modecode

RawDownload
Latest commit unavailable.
1from __future__ import annotations
2
3import os
4import json
5from enum import Enum
6from typing import Any, List, Callable, Optional, Awaitable
7from typing_extensions import Literal, TypeVar
8
9import httpx
10import pytest
11from respx import MockRouter
12from pydantic import Field, BaseModel
13from inline_snapshot import snapshot
14
15import openai
16from openai import OpenAI, AsyncOpenAI
17from openai._utils import assert_signatures_in_sync
18from openai._compat import PYDANTIC_V2
19
20from ._utils import print_obj
21from ...conftest import base_url
22from ..schema_types.query import Query
23
24_T = TypeVar("_T")
25
26# all the snapshots in this file are auto-generated from the live API
27#
28# you can update them with
29#
30# `OPENAI_LIVE=1 pytest --inline-snapshot=fix`
31
32
33@pytest.mark.respx(base_url=base_url)
34def test_parse_nothing(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
35 completion = _make_snapshot_request(
36 lambda c: c.beta.chat.completions.parse(
37 model="gpt-4o-2024-08-06",
38 messages=[
39 {
40 "role": "user",
41 "content": "What's the weather like in SF?",
42 },
43 ],
44 ),
45 content_snapshot=snapshot(
46 '{"id": "chatcmpl-ABfvaueLEMLNYbT8YzpJxsmiQ6HSY", "object": "chat.completion", "created": 1727346142, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "I\'m unable to provide real-time weather updates. To get the current weather in San Francisco, I recommend checking a reliable weather website or app like the Weather Channel or a local news station.", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 14, "completion_tokens": 37, "total_tokens": 51, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_b40fb1c6fb"}'
47 ),
48 mock_client=client,
49 respx_mock=respx_mock,
50 )
51
52 assert print_obj(completion, monkeypatch) == snapshot(
53 """\
54ParsedChatCompletion[NoneType](
55 choices=[
56 ParsedChoice[NoneType](
57 finish_reason='stop',
58 index=0,
59 logprobs=None,
60 message=ParsedChatCompletionMessage[NoneType](
61 content="I'm unable to provide real-time weather updates. To get the current weather in San Francisco, I
62recommend checking a reliable weather website or app like the Weather Channel or a local news station.",
63 function_call=None,
64 parsed=None,
65 refusal=None,
66 role='assistant',
67 tool_calls=[]
68 )
69 )
70 ],
71 created=1727346142,
72 id='chatcmpl-ABfvaueLEMLNYbT8YzpJxsmiQ6HSY',
73 model='gpt-4o-2024-08-06',
74 object='chat.completion',
75 service_tier=None,
76 system_fingerprint='fp_b40fb1c6fb',
77 usage=CompletionUsage(
78 completion_tokens=37,
79 completion_tokens_details=CompletionTokensDetails(audio_tokens=None, reasoning_tokens=0),
80 prompt_tokens=14,
81 prompt_tokens_details=None,
82 total_tokens=51
83 )
84)
85"""
86 )
87
88
89@pytest.mark.respx(base_url=base_url)
90def test_parse_pydantic_model(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
91 class Location(BaseModel):
92 city: str
93 temperature: float
94 units: Literal["c", "f"]
95
96 completion = _make_snapshot_request(
97 lambda c: c.beta.chat.completions.parse(
98 model="gpt-4o-2024-08-06",
99 messages=[
100 {
101 "role": "user",
102 "content": "What's the weather like in SF?",
103 },
104 ],
105 response_format=Location,
106 ),
107 content_snapshot=snapshot(
108 '{"id": "chatcmpl-ABfvbtVnTu5DeC4EFnRYj8mtfOM99", "object": "chat.completion", "created": 1727346143, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":65,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 79, "completion_tokens": 14, "total_tokens": 93, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
109 ),
110 mock_client=client,
111 respx_mock=respx_mock,
112 )
113
114 assert print_obj(completion, monkeypatch) == snapshot(
115 """\
116ParsedChatCompletion[Location](
117 choices=[
118 ParsedChoice[Location](
119 finish_reason='stop',
120 index=0,
121 logprobs=None,
122 message=ParsedChatCompletionMessage[Location](
123 content='{"city":"San Francisco","temperature":65,"units":"f"}',
124 function_call=None,
125 parsed=Location(city='San Francisco', temperature=65.0, units='f'),
126 refusal=None,
127 role='assistant',
128 tool_calls=[]
129 )
130 )
131 ],
132 created=1727346143,
133 id='chatcmpl-ABfvbtVnTu5DeC4EFnRYj8mtfOM99',
134 model='gpt-4o-2024-08-06',
135 object='chat.completion',
136 service_tier=None,
137 system_fingerprint='fp_5050236cbd',
138 usage=CompletionUsage(
139 completion_tokens=14,
140 completion_tokens_details=CompletionTokensDetails(audio_tokens=None, reasoning_tokens=0),
141 prompt_tokens=79,
142 prompt_tokens_details=None,
143 total_tokens=93
144 )
145)
146"""
147 )
148
149
150@pytest.mark.respx(base_url=base_url)
151def test_parse_pydantic_model_optional_default(
152 client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch
153) -> None:
154 class Location(BaseModel):
155 city: str
156 temperature: float
157 units: Optional[Literal["c", "f"]] = None
158
159 completion = _make_snapshot_request(
160 lambda c: c.beta.chat.completions.parse(
161 model="gpt-4o-2024-08-06",
162 messages=[
163 {
164 "role": "user",
165 "content": "What's the weather like in SF?",
166 },
167 ],
168 response_format=Location,
169 ),
170 content_snapshot=snapshot(
171 '{"id": "chatcmpl-ABfvcC8grKYsRkSoMp9CCAhbXAd0b", "object": "chat.completion", "created": 1727346144, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":65,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 88, "completion_tokens": 14, "total_tokens": 102, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_b40fb1c6fb"}'
172 ),
173 mock_client=client,
174 respx_mock=respx_mock,
175 )
176
177 assert print_obj(completion, monkeypatch) == snapshot(
178 """\
179ParsedChatCompletion[Location](
180 choices=[
181 ParsedChoice[Location](
182 finish_reason='stop',
183 index=0,
184 logprobs=None,
185 message=ParsedChatCompletionMessage[Location](
186 content='{"city":"San Francisco","temperature":65,"units":"f"}',
187 function_call=None,
188 parsed=Location(city='San Francisco', temperature=65.0, units='f'),
189 refusal=None,
190 role='assistant',
191 tool_calls=[]
192 )
193 )
194 ],
195 created=1727346144,
196 id='chatcmpl-ABfvcC8grKYsRkSoMp9CCAhbXAd0b',
197 model='gpt-4o-2024-08-06',
198 object='chat.completion',
199 service_tier=None,
200 system_fingerprint='fp_b40fb1c6fb',
201 usage=CompletionUsage(
202 completion_tokens=14,
203 completion_tokens_details=CompletionTokensDetails(audio_tokens=None, reasoning_tokens=0),
204 prompt_tokens=88,
205 prompt_tokens_details=None,
206 total_tokens=102
207 )
208)
209"""
210 )
211
212
213@pytest.mark.respx(base_url=base_url)
214def test_parse_pydantic_model_enum(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
215 class Color(Enum):
216 """The detected color"""
217
218 RED = "red"
219 BLUE = "blue"
220 GREEN = "green"
221
222 class ColorDetection(BaseModel):
223 color: Color
224 hex_color_code: str = Field(description="The hex color code of the detected color")
225
226 if not PYDANTIC_V2:
227 ColorDetection.update_forward_refs(**locals()) # type: ignore
228
229 completion = _make_snapshot_request(
230 lambda c: c.beta.chat.completions.parse(
231 model="gpt-4o-2024-08-06",
232 messages=[
233 {"role": "user", "content": "What color is a Coke can?"},
234 ],
235 response_format=ColorDetection,
236 ),
237 content_snapshot=snapshot(
238 '{"id": "chatcmpl-ABfvjIatz0zrZu50gRbMtlp0asZpz", "object": "chat.completion", "created": 1727346151, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"color\\":\\"red\\",\\"hex_color_code\\":\\"#FF0000\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 109, "completion_tokens": 14, "total_tokens": 123, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
239 ),
240 mock_client=client,
241 respx_mock=respx_mock,
242 )
243
244 assert print_obj(completion.choices[0], monkeypatch) == snapshot(
245 """\
246ParsedChoice[ColorDetection](
247 finish_reason='stop',
248 index=0,
249 logprobs=None,
250 message=ParsedChatCompletionMessage[ColorDetection](
251 content='{"color":"red","hex_color_code":"#FF0000"}',
252 function_call=None,
253 parsed=ColorDetection(color=<Color.RED: 'red'>, hex_color_code='#FF0000'),
254 refusal=None,
255 role='assistant',
256 tool_calls=[]
257 )
258)
259"""
260 )
261
262
263@pytest.mark.respx(base_url=base_url)
264def test_parse_pydantic_model_multiple_choices(
265 client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch
266) -> None:
267 class Location(BaseModel):
268 city: str
269 temperature: float
270 units: Literal["c", "f"]
271
272 completion = _make_snapshot_request(
273 lambda c: c.beta.chat.completions.parse(
274 model="gpt-4o-2024-08-06",
275 messages=[
276 {
277 "role": "user",
278 "content": "What's the weather like in SF?",
279 },
280 ],
281 n=3,
282 response_format=Location,
283 ),
284 content_snapshot=snapshot(
285 '{"id": "chatcmpl-ABfvp8qzboW92q8ONDF4DPHlI7ckC", "object": "chat.completion", "created": 1727346157, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":64,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}, {"index": 1, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":65,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}, {"index": 2, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":63.0,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 79, "completion_tokens": 44, "total_tokens": 123, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_b40fb1c6fb"}'
286 ),
287 mock_client=client,
288 respx_mock=respx_mock,
289 )
290
291 assert print_obj(completion.choices, monkeypatch) == snapshot(
292 """\
293[
294 ParsedChoice[Location](
295 finish_reason='stop',
296 index=0,
297 logprobs=None,
298 message=ParsedChatCompletionMessage[Location](
299 content='{"city":"San Francisco","temperature":64,"units":"f"}',
300 function_call=None,
301 parsed=Location(city='San Francisco', temperature=64.0, units='f'),
302 refusal=None,
303 role='assistant',
304 tool_calls=[]
305 )
306 ),
307 ParsedChoice[Location](
308 finish_reason='stop',
309 index=1,
310 logprobs=None,
311 message=ParsedChatCompletionMessage[Location](
312 content='{"city":"San Francisco","temperature":65,"units":"f"}',
313 function_call=None,
314 parsed=Location(city='San Francisco', temperature=65.0, units='f'),
315 refusal=None,
316 role='assistant',
317 tool_calls=[]
318 )
319 ),
320 ParsedChoice[Location](
321 finish_reason='stop',
322 index=2,
323 logprobs=None,
324 message=ParsedChatCompletionMessage[Location](
325 content='{"city":"San Francisco","temperature":63.0,"units":"f"}',
326 function_call=None,
327 parsed=Location(city='San Francisco', temperature=63.0, units='f'),
328 refusal=None,
329 role='assistant',
330 tool_calls=[]
331 )
332 )
333]
334"""
335 )
336
337
338@pytest.mark.respx(base_url=base_url)
339@pytest.mark.skipif(not PYDANTIC_V2, reason="dataclasses only supported in v2")
340def test_parse_pydantic_dataclass(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
341 from pydantic.dataclasses import dataclass
342
343 @dataclass
344 class CalendarEvent:
345 name: str
346 date: str
347 participants: List[str]
348
349 completion = _make_snapshot_request(
350 lambda c: c.beta.chat.completions.parse(
351 model="gpt-4o-2024-08-06",
352 messages=[
353 {"role": "system", "content": "Extract the event information."},
354 {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."},
355 ],
356 response_format=CalendarEvent,
357 ),
358 content_snapshot=snapshot(
359 '{"id": "chatcmpl-ABfvqhz4uUUWsw8Ohw2Mp9B4sKKV8", "object": "chat.completion", "created": 1727346158, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"name\\":\\"Science Fair\\",\\"date\\":\\"Friday\\",\\"participants\\":[\\"Alice\\",\\"Bob\\"]}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 92, "completion_tokens": 17, "total_tokens": 109, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_7568d46099"}'
360 ),
361 mock_client=client,
362 respx_mock=respx_mock,
363 )
364
365 assert print_obj(completion, monkeypatch) == snapshot(
366 """\
367ParsedChatCompletion[CalendarEvent](
368 choices=[
369 ParsedChoice[CalendarEvent](
370 finish_reason='stop',
371 index=0,
372 logprobs=None,
373 message=ParsedChatCompletionMessage[CalendarEvent](
374 content='{"name":"Science Fair","date":"Friday","participants":["Alice","Bob"]}',
375 function_call=None,
376 parsed=CalendarEvent(name='Science Fair', date='Friday', participants=['Alice', 'Bob']),
377 refusal=None,
378 role='assistant',
379 tool_calls=[]
380 )
381 )
382 ],
383 created=1727346158,
384 id='chatcmpl-ABfvqhz4uUUWsw8Ohw2Mp9B4sKKV8',
385 model='gpt-4o-2024-08-06',
386 object='chat.completion',
387 service_tier=None,
388 system_fingerprint='fp_7568d46099',
389 usage=CompletionUsage(
390 completion_tokens=17,
391 completion_tokens_details=CompletionTokensDetails(audio_tokens=None, reasoning_tokens=0),
392 prompt_tokens=92,
393 prompt_tokens_details=None,
394 total_tokens=109
395 )
396)
397"""
398 )
399
400
401@pytest.mark.respx(base_url=base_url)
402def test_pydantic_tool_model_all_types(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
403 completion = _make_snapshot_request(
404 lambda c: c.beta.chat.completions.parse(
405 model="gpt-4o-2024-08-06",
406 messages=[
407 {
408 "role": "user",
409 "content": "look up all my orders in may of last year that were fulfilled but not delivered on time",
410 },
411 ],
412 tools=[openai.pydantic_function_tool(Query)],
413 response_format=Query,
414 ),
415 content_snapshot=snapshot(
416 '{"id": "chatcmpl-ABfvtNiaTNUF6OymZUnEFc9lPq9p1", "object": "chat.completion", "created": 1727346161, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_NKpApJybW1MzOjZO2FzwYw0d", "type": "function", "function": {"name": "Query", "arguments": "{\\"name\\":\\"May 2022 Fulfilled Orders Not Delivered on Time\\",\\"table_name\\":\\"orders\\",\\"columns\\":[\\"id\\",\\"status\\",\\"expected_delivery_date\\",\\"delivered_at\\",\\"shipped_at\\",\\"ordered_at\\",\\"canceled_at\\"],\\"conditions\\":[{\\"column\\":\\"ordered_at\\",\\"operator\\":\\">=\\",\\"value\\":\\"2022-05-01\\"},{\\"column\\":\\"ordered_at\\",\\"operator\\":\\"<=\\",\\"value\\":\\"2022-05-31\\"},{\\"column\\":\\"status\\",\\"operator\\":\\"=\\",\\"value\\":\\"fulfilled\\"},{\\"column\\":\\"delivered_at\\",\\"operator\\":\\">\\",\\"value\\":{\\"column_name\\":\\"expected_delivery_date\\"}}],\\"order_by\\":\\"asc\\"}"}}], "refusal": null}, "logprobs": null, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 512, "completion_tokens": 132, "total_tokens": 644, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_7568d46099"}'
417 ),
418 mock_client=client,
419 respx_mock=respx_mock,
420 )
421
422 assert print_obj(completion.choices[0], monkeypatch) == snapshot(
423 """\
424ParsedChoice[Query](
425 finish_reason='tool_calls',
426 index=0,
427 logprobs=None,
428 message=ParsedChatCompletionMessage[Query](
429 content=None,
430 function_call=None,
431 parsed=None,
432 refusal=None,
433 role='assistant',
434 tool_calls=[
435 ParsedFunctionToolCall(
436 function=ParsedFunction(
437 arguments='{"name":"May 2022 Fulfilled Orders Not Delivered on
438Time","table_name":"orders","columns":["id","status","expected_delivery_date","delivered_at","shipped_at","ordered_at","
439canceled_at"],"conditions":[{"column":"ordered_at","operator":">=","value":"2022-05-01"},{"column":"ordered_at","operato
440r":"<=","value":"2022-05-31"},{"column":"status","operator":"=","value":"fulfilled"},{"column":"delivered_at","operator"
441:">","value":{"column_name":"expected_delivery_date"}}],"order_by":"asc"}',
442 name='Query',
443 parsed_arguments=Query(
444 columns=[
445 <Column.id: 'id'>,
446 <Column.status: 'status'>,
447 <Column.expected_delivery_date: 'expected_delivery_date'>,
448 <Column.delivered_at: 'delivered_at'>,
449 <Column.shipped_at: 'shipped_at'>,
450 <Column.ordered_at: 'ordered_at'>,
451 <Column.canceled_at: 'canceled_at'>
452 ],
453 conditions=[
454 Condition(column='ordered_at', operator=<Operator.ge: '>='>, value='2022-05-01'),
455 Condition(column='ordered_at', operator=<Operator.le: '<='>, value='2022-05-31'),
456 Condition(column='status', operator=<Operator.eq: '='>, value='fulfilled'),
457 Condition(
458 column='delivered_at',
459 operator=<Operator.gt: '>'>,
460 value=DynamicValue(column_name='expected_delivery_date')
461 )
462 ],
463 name='May 2022 Fulfilled Orders Not Delivered on Time',
464 order_by=<OrderBy.asc: 'asc'>,
465 table_name=<Table.orders: 'orders'>
466 )
467 ),
468 id='call_NKpApJybW1MzOjZO2FzwYw0d',
469 type='function'
470 )
471 ]
472 )
473)
474"""
475 )
476
477
478@pytest.mark.respx(base_url=base_url)
479def test_parse_max_tokens_reached(client: OpenAI, respx_mock: MockRouter) -> None:
480 class Location(BaseModel):
481 city: str
482 temperature: float
483 units: Literal["c", "f"]
484
485 with pytest.raises(openai.LengthFinishReasonError):
486 _make_snapshot_request(
487 lambda c: c.beta.chat.completions.parse(
488 model="gpt-4o-2024-08-06",
489 messages=[
490 {
491 "role": "user",
492 "content": "What's the weather like in SF?",
493 },
494 ],
495 max_tokens=1,
496 response_format=Location,
497 ),
498 content_snapshot=snapshot(
499 '{"id": "chatcmpl-ABfvvX7eB1KsfeZj8VcF3z7G7SbaA", "object": "chat.completion", "created": 1727346163, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"", "refusal": null}, "logprobs": null, "finish_reason": "length"}], "usage": {"prompt_tokens": 79, "completion_tokens": 1, "total_tokens": 80, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_7568d46099"}'
500 ),
501 mock_client=client,
502 respx_mock=respx_mock,
503 )
504
505
506@pytest.mark.respx(base_url=base_url)
507def test_parse_pydantic_model_refusal(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
508 class Location(BaseModel):
509 city: str
510 temperature: float
511 units: Literal["c", "f"]
512
513 completion = _make_snapshot_request(
514 lambda c: c.beta.chat.completions.parse(
515 model="gpt-4o-2024-08-06",
516 messages=[
517 {
518 "role": "user",
519 "content": "How do I make anthrax?",
520 },
521 ],
522 response_format=Location,
523 ),
524 content_snapshot=snapshot(
525 '{"id": "chatcmpl-ABfvwoKVWPQj2UPlAcAKM7s40GsRx", "object": "chat.completion", "created": 1727346164, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "refusal": "I\'m very sorry, but I can\'t assist with that."}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 79, "completion_tokens": 12, "total_tokens": 91, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
526 ),
527 mock_client=client,
528 respx_mock=respx_mock,
529 )
530
531 assert print_obj(completion.choices, monkeypatch) == snapshot(
532 """\
533[
534 ParsedChoice[Location](
535 finish_reason='stop',
536 index=0,
537 logprobs=None,
538 message=ParsedChatCompletionMessage[Location](
539 content=None,
540 function_call=None,
541 parsed=None,
542 refusal="I'm very sorry, but I can't assist with that.",
543 role='assistant',
544 tool_calls=[]
545 )
546 )
547]
548"""
549 )
550
551
552@pytest.mark.respx(base_url=base_url)
553def test_parse_pydantic_tool(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
554 class GetWeatherArgs(BaseModel):
555 city: str
556 country: str
557 units: Literal["c", "f"] = "c"
558
559 completion = _make_snapshot_request(
560 lambda c: c.beta.chat.completions.parse(
561 model="gpt-4o-2024-08-06",
562 messages=[
563 {
564 "role": "user",
565 "content": "What's the weather like in Edinburgh?",
566 },
567 ],
568 tools=[
569 openai.pydantic_function_tool(GetWeatherArgs),
570 ],
571 ),
572 content_snapshot=snapshot(
573 '{"id": "chatcmpl-ABfvx6Z4dchiW2nya1N8KMsHFrQRE", "object": "chat.completion", "created": 1727346165, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_Y6qJ7ofLgOrBnMD5WbVAeiRV", "type": "function", "function": {"name": "GetWeatherArgs", "arguments": "{\\"city\\":\\"Edinburgh\\",\\"country\\":\\"UK\\",\\"units\\":\\"c\\"}"}}], "refusal": null}, "logprobs": null, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 76, "completion_tokens": 24, "total_tokens": 100, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_e45dabd248"}'
574 ),
575 mock_client=client,
576 respx_mock=respx_mock,
577 )
578
579 assert print_obj(completion.choices, monkeypatch) == snapshot(
580 """\
581[
582 ParsedChoice[NoneType](
583 finish_reason='tool_calls',
584 index=0,
585 logprobs=None,
586 message=ParsedChatCompletionMessage[NoneType](
587 content=None,
588 function_call=None,
589 parsed=None,
590 refusal=None,
591 role='assistant',
592 tool_calls=[
593 ParsedFunctionToolCall(
594 function=ParsedFunction(
595 arguments='{"city":"Edinburgh","country":"UK","units":"c"}',
596 name='GetWeatherArgs',
597 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='UK', units='c')
598 ),
599 id='call_Y6qJ7ofLgOrBnMD5WbVAeiRV',
600 type='function'
601 )
602 ]
603 )
604 )
605]
606"""
607 )
608
609
610@pytest.mark.respx(base_url=base_url)
611def test_parse_multiple_pydantic_tools(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
612 class GetWeatherArgs(BaseModel):
613 """Get the temperature for the given country/city combo"""
614
615 city: str
616 country: str
617 units: Literal["c", "f"] = "c"
618
619 class GetStockPrice(BaseModel):
620 ticker: str
621 exchange: str
622
623 completion = _make_snapshot_request(
624 lambda c: c.beta.chat.completions.parse(
625 model="gpt-4o-2024-08-06",
626 messages=[
627 {
628 "role": "user",
629 "content": "What's the weather like in Edinburgh?",
630 },
631 {
632 "role": "user",
633 "content": "What's the price of AAPL?",
634 },
635 ],
636 tools=[
637 openai.pydantic_function_tool(GetWeatherArgs),
638 openai.pydantic_function_tool(
639 GetStockPrice, name="get_stock_price", description="Fetch the latest price for a given ticker"
640 ),
641 ],
642 ),
643 content_snapshot=snapshot(
644 '{"id": "chatcmpl-ABfvyvfNWKcl7Ohqos4UFrmMs1v4C", "object": "chat.completion", "created": 1727346166, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_fdNz3vOBKYgOIpMdWotB9MjY", "type": "function", "function": {"name": "GetWeatherArgs", "arguments": "{\\"city\\": \\"Edinburgh\\", \\"country\\": \\"GB\\", \\"units\\": \\"c\\"}"}}, {"id": "call_h1DWI1POMJLb0KwIyQHWXD4p", "type": "function", "function": {"name": "get_stock_price", "arguments": "{\\"ticker\\": \\"AAPL\\", \\"exchange\\": \\"NASDAQ\\"}"}}], "refusal": null}, "logprobs": null, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 149, "completion_tokens": 60, "total_tokens": 209, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_b40fb1c6fb"}'
645 ),
646 mock_client=client,
647 respx_mock=respx_mock,
648 )
649
650 assert print_obj(completion.choices, monkeypatch) == snapshot(
651 """\
652[
653 ParsedChoice[NoneType](
654 finish_reason='tool_calls',
655 index=0,
656 logprobs=None,
657 message=ParsedChatCompletionMessage[NoneType](
658 content=None,
659 function_call=None,
660 parsed=None,
661 refusal=None,
662 role='assistant',
663 tool_calls=[
664 ParsedFunctionToolCall(
665 function=ParsedFunction(
666 arguments='{"city": "Edinburgh", "country": "GB", "units": "c"}',
667 name='GetWeatherArgs',
668 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='GB', units='c')
669 ),
670 id='call_fdNz3vOBKYgOIpMdWotB9MjY',
671 type='function'
672 ),
673 ParsedFunctionToolCall(
674 function=ParsedFunction(
675 arguments='{"ticker": "AAPL", "exchange": "NASDAQ"}',
676 name='get_stock_price',
677 parsed_arguments=GetStockPrice(exchange='NASDAQ', ticker='AAPL')
678 ),
679 id='call_h1DWI1POMJLb0KwIyQHWXD4p',
680 type='function'
681 )
682 ]
683 )
684 )
685]
686"""
687 )
688
689
690@pytest.mark.respx(base_url=base_url)
691def test_parse_strict_tools(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
692 completion = _make_snapshot_request(
693 lambda c: c.beta.chat.completions.parse(
694 model="gpt-4o-2024-08-06",
695 messages=[
696 {
697 "role": "user",
698 "content": "What's the weather like in SF?",
699 },
700 ],
701 tools=[
702 {
703 "type": "function",
704 "function": {
705 "name": "get_weather",
706 "parameters": {
707 "type": "object",
708 "properties": {
709 "city": {"type": "string"},
710 "state": {"type": "string"},
711 },
712 "required": [
713 "city",
714 "state",
715 ],
716 "additionalProperties": False,
717 },
718 "strict": True,
719 },
720 }
721 ],
722 ),
723 content_snapshot=snapshot(
724 '{"id": "chatcmpl-ABfvzdvCI6RaIkiEFNjqGXCSYnlzf", "object": "chat.completion", "created": 1727346167, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_CUdUoJpsWWVdxXntucvnol1M", "type": "function", "function": {"name": "get_weather", "arguments": "{\\"city\\":\\"San Francisco\\",\\"state\\":\\"CA\\"}"}}], "refusal": null}, "logprobs": null, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 48, "completion_tokens": 19, "total_tokens": 67, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
725 ),
726 mock_client=client,
727 respx_mock=respx_mock,
728 )
729
730 assert print_obj(completion.choices, monkeypatch) == snapshot(
731 """\
732[
733 ParsedChoice[NoneType](
734 finish_reason='tool_calls',
735 index=0,
736 logprobs=None,
737 message=ParsedChatCompletionMessage[NoneType](
738 content=None,
739 function_call=None,
740 parsed=None,
741 refusal=None,
742 role='assistant',
743 tool_calls=[
744 ParsedFunctionToolCall(
745 function=ParsedFunction(
746 arguments='{"city":"San Francisco","state":"CA"}',
747 name='get_weather',
748 parsed_arguments={'city': 'San Francisco', 'state': 'CA'}
749 ),
750 id='call_CUdUoJpsWWVdxXntucvnol1M',
751 type='function'
752 )
753 ]
754 )
755 )
756]
757"""
758 )
759
760
761def test_parse_non_strict_tools(client: OpenAI) -> None:
762 with pytest.raises(
763 ValueError, match="`get_weather` is not strict. Only `strict` function tools can be auto-parsed"
764 ):
765 client.beta.chat.completions.parse(
766 model="gpt-4o-2024-08-06",
767 messages=[],
768 tools=[
769 {
770 "type": "function",
771 "function": {
772 "name": "get_weather",
773 "parameters": {},
774 },
775 }
776 ],
777 )
778
779
780@pytest.mark.respx(base_url=base_url)
781def test_parse_pydantic_raw_response(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
782 class Location(BaseModel):
783 city: str
784 temperature: float
785 units: Literal["c", "f"]
786
787 response = _make_snapshot_request(
788 lambda c: c.beta.chat.completions.with_raw_response.parse(
789 model="gpt-4o-2024-08-06",
790 messages=[
791 {
792 "role": "user",
793 "content": "What's the weather like in SF?",
794 },
795 ],
796 response_format=Location,
797 ),
798 content_snapshot=snapshot(
799 '{"id": "chatcmpl-ABrDYCa8W1w66eUxKDO8TQF1m6trT", "object": "chat.completion", "created": 1727389540, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":58,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 79, "completion_tokens": 14, "total_tokens": 93, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
800 ),
801 mock_client=client,
802 respx_mock=respx_mock,
803 )
804 assert response.http_request.headers.get("x-stainless-helper-method") == "beta.chat.completions.parse"
805
806 completion = response.parse()
807 message = completion.choices[0].message
808 assert message.parsed is not None
809 assert isinstance(message.parsed.city, str)
810 assert print_obj(completion, monkeypatch) == snapshot(
811 """\
812ParsedChatCompletion[Location](
813 choices=[
814 ParsedChoice[Location](
815 finish_reason='stop',
816 index=0,
817 logprobs=None,
818 message=ParsedChatCompletionMessage[Location](
819 content='{"city":"San Francisco","temperature":58,"units":"f"}',
820 function_call=None,
821 parsed=Location(city='San Francisco', temperature=58.0, units='f'),
822 refusal=None,
823 role='assistant',
824 tool_calls=[]
825 )
826 )
827 ],
828 created=1727389540,
829 id='chatcmpl-ABrDYCa8W1w66eUxKDO8TQF1m6trT',
830 model='gpt-4o-2024-08-06',
831 object='chat.completion',
832 service_tier=None,
833 system_fingerprint='fp_5050236cbd',
834 usage=CompletionUsage(
835 completion_tokens=14,
836 completion_tokens_details=CompletionTokensDetails(audio_tokens=None, reasoning_tokens=0),
837 prompt_tokens=79,
838 prompt_tokens_details=None,
839 total_tokens=93
840 )
841)
842"""
843 )
844
845
846@pytest.mark.respx(base_url=base_url)
847@pytest.mark.asyncio
848async def test_async_parse_pydantic_raw_response(
849 async_client: AsyncOpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch
850) -> None:
851 class Location(BaseModel):
852 city: str
853 temperature: float
854 units: Literal["c", "f"]
855
856 response = await _make_async_snapshot_request(
857 lambda c: c.beta.chat.completions.with_raw_response.parse(
858 model="gpt-4o-2024-08-06",
859 messages=[
860 {
861 "role": "user",
862 "content": "What's the weather like in SF?",
863 },
864 ],
865 response_format=Location,
866 ),
867 content_snapshot=snapshot(
868 '{"id": "chatcmpl-ABrDQWOiw0PK5JOsxl1D9ooeQgznq", "object": "chat.completion", "created": 1727389532, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":65,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 79, "completion_tokens": 14, "total_tokens": 93, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
869 ),
870 mock_client=async_client,
871 respx_mock=respx_mock,
872 )
873 assert response.http_request.headers.get("x-stainless-helper-method") == "beta.chat.completions.parse"
874
875 completion = response.parse()
876 message = completion.choices[0].message
877 assert message.parsed is not None
878 assert isinstance(message.parsed.city, str)
879 assert print_obj(completion, monkeypatch) == snapshot(
880 """\
881ParsedChatCompletion[Location](
882 choices=[
883 ParsedChoice[Location](
884 finish_reason='stop',
885 index=0,
886 logprobs=None,
887 message=ParsedChatCompletionMessage[Location](
888 content='{"city":"San Francisco","temperature":65,"units":"f"}',
889 function_call=None,
890 parsed=Location(city='San Francisco', temperature=65.0, units='f'),
891 refusal=None,
892 role='assistant',
893 tool_calls=[]
894 )
895 )
896 ],
897 created=1727389532,
898 id='chatcmpl-ABrDQWOiw0PK5JOsxl1D9ooeQgznq',
899 model='gpt-4o-2024-08-06',
900 object='chat.completion',
901 service_tier=None,
902 system_fingerprint='fp_5050236cbd',
903 usage=CompletionUsage(
904 completion_tokens=14,
905 completion_tokens_details=CompletionTokensDetails(audio_tokens=None, reasoning_tokens=0),
906 prompt_tokens=79,
907 prompt_tokens_details=None,
908 total_tokens=93
909 )
910)
911"""
912 )
913
914
915@pytest.mark.parametrize("sync", [True, False], ids=["sync", "async"])
916def test_parse_method_in_sync(sync: bool, client: OpenAI, async_client: AsyncOpenAI) -> None:
917 checking_client: OpenAI | AsyncOpenAI = client if sync else async_client
918
919 assert_signatures_in_sync(
920 checking_client.chat.completions.create,
921 checking_client.beta.chat.completions.parse,
922 exclude_params={"response_format", "stream"},
923 )
924
925
926def _make_snapshot_request(
927 func: Callable[[OpenAI], _T],
928 *,
929 content_snapshot: Any,
930 respx_mock: MockRouter,
931 mock_client: OpenAI,
932) -> _T:
933 live = os.environ.get("OPENAI_LIVE") == "1"
934 if live:
935
936 def _on_response(response: httpx.Response) -> None:
937 # update the content snapshot
938 assert json.dumps(json.loads(response.read())) == content_snapshot
939
940 respx_mock.stop()
941
942 client = OpenAI(
943 http_client=httpx.Client(
944 event_hooks={
945 "response": [_on_response],
946 }
947 )
948 )
949 else:
950 respx_mock.post("/chat/completions").mock(
951 return_value=httpx.Response(
952 200,
953 content=content_snapshot._old_value,
954 headers={"content-type": "application/json"},
955 )
956 )
957
958 client = mock_client
959
960 result = func(client)
961
962 if live:
963 client.close()
964
965 return result
966
967
968async def _make_async_snapshot_request(
969 func: Callable[[AsyncOpenAI], Awaitable[_T]],
970 *,
971 content_snapshot: Any,
972 respx_mock: MockRouter,
973 mock_client: AsyncOpenAI,
974) -> _T:
975 live = os.environ.get("OPENAI_LIVE") == "1"
976 if live:
977
978 async def _on_response(response: httpx.Response) -> None:
979 # update the content snapshot
980 assert json.dumps(json.loads(await response.aread())) == content_snapshot
981
982 respx_mock.stop()
983
984 client = AsyncOpenAI(
985 http_client=httpx.AsyncClient(
986 event_hooks={
987 "response": [_on_response],
988 }
989 )
990 )
991 else:
992 respx_mock.post("/chat/completions").mock(
993 return_value=httpx.Response(
994 200,
995 content=content_snapshot._old_value,
996 headers={"content-type": "application/json"},
997 )
998 )
999
1000 client = mock_client
1001
1002 result = await func(client)
1003
1004 if live:
1005 await client.close()
1006
1007 return result
1008