GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

openai/openai-python

Public

mirrored from https://github.com/openai/openai-pythonAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
v1.99.7

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openai-python/tests/lib/chat

tests/lib/chat/test_completions.py

1067lines · modecode

RawDownload
Latest commit unavailable.
1from __future__ import annotations
2
3import os
4import json
5from enum import Enum
6from typing import Any, List, Callable, Optional, Awaitable
7from typing_extensions import Literal, TypeVar
8
9import httpx
10import pytest
11from respx import MockRouter
12from pydantic import Field, BaseModel
13from inline_snapshot import snapshot
14
15import openai
16from openai import OpenAI, AsyncOpenAI
17from openai._utils import assert_signatures_in_sync
18from openai._compat import PYDANTIC_V2
19
20from ._utils import print_obj, get_snapshot_value
21from ...conftest import base_url
22from ..schema_types.query import Query
23
24_T = TypeVar("_T")
25
26# all the snapshots in this file are auto-generated from the live API
27#
28# you can update them with
29#
30# `OPENAI_LIVE=1 pytest --inline-snapshot=fix`
31
32
33@pytest.mark.respx(base_url=base_url)
34def test_parse_nothing(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
35 completion = _make_snapshot_request(
36 lambda c: c.chat.completions.parse(
37 model="gpt-4o-2024-08-06",
38 messages=[
39 {
40 "role": "user",
41 "content": "What's the weather like in SF?",
42 },
43 ],
44 ),
45 content_snapshot=snapshot(
46 '{"id": "chatcmpl-ABfvaueLEMLNYbT8YzpJxsmiQ6HSY", "object": "chat.completion", "created": 1727346142, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "I\'m unable to provide real-time weather updates. To get the current weather in San Francisco, I recommend checking a reliable weather website or app like the Weather Channel or a local news station.", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 14, "completion_tokens": 37, "total_tokens": 51, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_b40fb1c6fb"}'
47 ),
48 mock_client=client,
49 respx_mock=respx_mock,
50 )
51
52 assert print_obj(completion, monkeypatch) == snapshot(
53 """\
54ParsedChatCompletion[NoneType](
55 choices=[
56 ParsedChoice[NoneType](
57 finish_reason='stop',
58 index=0,
59 logprobs=None,
60 message=ParsedChatCompletionMessage[NoneType](
61 annotations=None,
62 audio=None,
63 content="I'm unable to provide real-time weather updates. To get the current weather in San Francisco, I
64recommend checking a reliable weather website or app like the Weather Channel or a local news station.",
65 function_call=None,
66 parsed=None,
67 refusal=None,
68 role='assistant',
69 tool_calls=None
70 )
71 )
72 ],
73 created=1727346142,
74 id='chatcmpl-ABfvaueLEMLNYbT8YzpJxsmiQ6HSY',
75 model='gpt-4o-2024-08-06',
76 object='chat.completion',
77 service_tier=None,
78 system_fingerprint='fp_b40fb1c6fb',
79 usage=CompletionUsage(
80 completion_tokens=37,
81 completion_tokens_details=CompletionTokensDetails(
82 accepted_prediction_tokens=None,
83 audio_tokens=None,
84 reasoning_tokens=0,
85 rejected_prediction_tokens=None
86 ),
87 prompt_tokens=14,
88 prompt_tokens_details=None,
89 total_tokens=51
90 )
91)
92"""
93 )
94
95
96@pytest.mark.respx(base_url=base_url)
97def test_parse_pydantic_model(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
98 class Location(BaseModel):
99 city: str
100 temperature: float
101 units: Literal["c", "f"]
102
103 completion = _make_snapshot_request(
104 lambda c: c.chat.completions.parse(
105 model="gpt-4o-2024-08-06",
106 messages=[
107 {
108 "role": "user",
109 "content": "What's the weather like in SF?",
110 },
111 ],
112 response_format=Location,
113 ),
114 content_snapshot=snapshot(
115 '{"id": "chatcmpl-ABfvbtVnTu5DeC4EFnRYj8mtfOM99", "object": "chat.completion", "created": 1727346143, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":65,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 79, "completion_tokens": 14, "total_tokens": 93, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
116 ),
117 mock_client=client,
118 respx_mock=respx_mock,
119 )
120
121 assert print_obj(completion, monkeypatch) == snapshot(
122 """\
123ParsedChatCompletion[Location](
124 choices=[
125 ParsedChoice[Location](
126 finish_reason='stop',
127 index=0,
128 logprobs=None,
129 message=ParsedChatCompletionMessage[Location](
130 annotations=None,
131 audio=None,
132 content='{"city":"San Francisco","temperature":65,"units":"f"}',
133 function_call=None,
134 parsed=Location(city='San Francisco', temperature=65.0, units='f'),
135 refusal=None,
136 role='assistant',
137 tool_calls=None
138 )
139 )
140 ],
141 created=1727346143,
142 id='chatcmpl-ABfvbtVnTu5DeC4EFnRYj8mtfOM99',
143 model='gpt-4o-2024-08-06',
144 object='chat.completion',
145 service_tier=None,
146 system_fingerprint='fp_5050236cbd',
147 usage=CompletionUsage(
148 completion_tokens=14,
149 completion_tokens_details=CompletionTokensDetails(
150 accepted_prediction_tokens=None,
151 audio_tokens=None,
152 reasoning_tokens=0,
153 rejected_prediction_tokens=None
154 ),
155 prompt_tokens=79,
156 prompt_tokens_details=None,
157 total_tokens=93
158 )
159)
160"""
161 )
162
163
164@pytest.mark.respx(base_url=base_url)
165def test_parse_pydantic_model_optional_default(
166 client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch
167) -> None:
168 class Location(BaseModel):
169 city: str
170 temperature: float
171 units: Optional[Literal["c", "f"]] = None
172
173 completion = _make_snapshot_request(
174 lambda c: c.chat.completions.parse(
175 model="gpt-4o-2024-08-06",
176 messages=[
177 {
178 "role": "user",
179 "content": "What's the weather like in SF?",
180 },
181 ],
182 response_format=Location,
183 ),
184 content_snapshot=snapshot(
185 '{"id": "chatcmpl-ABfvcC8grKYsRkSoMp9CCAhbXAd0b", "object": "chat.completion", "created": 1727346144, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":65,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 88, "completion_tokens": 14, "total_tokens": 102, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_b40fb1c6fb"}'
186 ),
187 mock_client=client,
188 respx_mock=respx_mock,
189 )
190
191 assert print_obj(completion, monkeypatch) == snapshot(
192 """\
193ParsedChatCompletion[Location](
194 choices=[
195 ParsedChoice[Location](
196 finish_reason='stop',
197 index=0,
198 logprobs=None,
199 message=ParsedChatCompletionMessage[Location](
200 annotations=None,
201 audio=None,
202 content='{"city":"San Francisco","temperature":65,"units":"f"}',
203 function_call=None,
204 parsed=Location(city='San Francisco', temperature=65.0, units='f'),
205 refusal=None,
206 role='assistant',
207 tool_calls=None
208 )
209 )
210 ],
211 created=1727346144,
212 id='chatcmpl-ABfvcC8grKYsRkSoMp9CCAhbXAd0b',
213 model='gpt-4o-2024-08-06',
214 object='chat.completion',
215 service_tier=None,
216 system_fingerprint='fp_b40fb1c6fb',
217 usage=CompletionUsage(
218 completion_tokens=14,
219 completion_tokens_details=CompletionTokensDetails(
220 accepted_prediction_tokens=None,
221 audio_tokens=None,
222 reasoning_tokens=0,
223 rejected_prediction_tokens=None
224 ),
225 prompt_tokens=88,
226 prompt_tokens_details=None,
227 total_tokens=102
228 )
229)
230"""
231 )
232
233
234@pytest.mark.respx(base_url=base_url)
235def test_parse_pydantic_model_enum(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
236 class Color(Enum):
237 """The detected color"""
238
239 RED = "red"
240 BLUE = "blue"
241 GREEN = "green"
242
243 class ColorDetection(BaseModel):
244 color: Color
245 hex_color_code: str = Field(description="The hex color code of the detected color")
246
247 if not PYDANTIC_V2:
248 ColorDetection.update_forward_refs(**locals()) # type: ignore
249
250 completion = _make_snapshot_request(
251 lambda c: c.chat.completions.parse(
252 model="gpt-4o-2024-08-06",
253 messages=[
254 {"role": "user", "content": "What color is a Coke can?"},
255 ],
256 response_format=ColorDetection,
257 ),
258 content_snapshot=snapshot(
259 '{"id": "chatcmpl-ABfvjIatz0zrZu50gRbMtlp0asZpz", "object": "chat.completion", "created": 1727346151, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"color\\":\\"red\\",\\"hex_color_code\\":\\"#FF0000\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 109, "completion_tokens": 14, "total_tokens": 123, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
260 ),
261 mock_client=client,
262 respx_mock=respx_mock,
263 )
264
265 assert print_obj(completion.choices[0], monkeypatch) == snapshot(
266 """\
267ParsedChoice[ColorDetection](
268 finish_reason='stop',
269 index=0,
270 logprobs=None,
271 message=ParsedChatCompletionMessage[ColorDetection](
272 annotations=None,
273 audio=None,
274 content='{"color":"red","hex_color_code":"#FF0000"}',
275 function_call=None,
276 parsed=ColorDetection(color=<Color.RED: 'red'>, hex_color_code='#FF0000'),
277 refusal=None,
278 role='assistant',
279 tool_calls=None
280 )
281)
282"""
283 )
284
285
286@pytest.mark.respx(base_url=base_url)
287def test_parse_pydantic_model_multiple_choices(
288 client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch
289) -> None:
290 class Location(BaseModel):
291 city: str
292 temperature: float
293 units: Literal["c", "f"]
294
295 completion = _make_snapshot_request(
296 lambda c: c.chat.completions.parse(
297 model="gpt-4o-2024-08-06",
298 messages=[
299 {
300 "role": "user",
301 "content": "What's the weather like in SF?",
302 },
303 ],
304 n=3,
305 response_format=Location,
306 ),
307 content_snapshot=snapshot(
308 '{"id": "chatcmpl-ABfvp8qzboW92q8ONDF4DPHlI7ckC", "object": "chat.completion", "created": 1727346157, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":64,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}, {"index": 1, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":65,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}, {"index": 2, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":63.0,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 79, "completion_tokens": 44, "total_tokens": 123, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_b40fb1c6fb"}'
309 ),
310 mock_client=client,
311 respx_mock=respx_mock,
312 )
313
314 assert print_obj(completion.choices, monkeypatch) == snapshot(
315 """\
316[
317 ParsedChoice[Location](
318 finish_reason='stop',
319 index=0,
320 logprobs=None,
321 message=ParsedChatCompletionMessage[Location](
322 annotations=None,
323 audio=None,
324 content='{"city":"San Francisco","temperature":64,"units":"f"}',
325 function_call=None,
326 parsed=Location(city='San Francisco', temperature=64.0, units='f'),
327 refusal=None,
328 role='assistant',
329 tool_calls=None
330 )
331 ),
332 ParsedChoice[Location](
333 finish_reason='stop',
334 index=1,
335 logprobs=None,
336 message=ParsedChatCompletionMessage[Location](
337 annotations=None,
338 audio=None,
339 content='{"city":"San Francisco","temperature":65,"units":"f"}',
340 function_call=None,
341 parsed=Location(city='San Francisco', temperature=65.0, units='f'),
342 refusal=None,
343 role='assistant',
344 tool_calls=None
345 )
346 ),
347 ParsedChoice[Location](
348 finish_reason='stop',
349 index=2,
350 logprobs=None,
351 message=ParsedChatCompletionMessage[Location](
352 annotations=None,
353 audio=None,
354 content='{"city":"San Francisco","temperature":63.0,"units":"f"}',
355 function_call=None,
356 parsed=Location(city='San Francisco', temperature=63.0, units='f'),
357 refusal=None,
358 role='assistant',
359 tool_calls=None
360 )
361 )
362]
363"""
364 )
365
366
367@pytest.mark.respx(base_url=base_url)
368@pytest.mark.skipif(not PYDANTIC_V2, reason="dataclasses only supported in v2")
369def test_parse_pydantic_dataclass(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
370 from pydantic.dataclasses import dataclass
371
372 @dataclass
373 class CalendarEvent:
374 name: str
375 date: str
376 participants: List[str]
377
378 completion = _make_snapshot_request(
379 lambda c: c.chat.completions.parse(
380 model="gpt-4o-2024-08-06",
381 messages=[
382 {"role": "system", "content": "Extract the event information."},
383 {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."},
384 ],
385 response_format=CalendarEvent,
386 ),
387 content_snapshot=snapshot(
388 '{"id": "chatcmpl-ABfvqhz4uUUWsw8Ohw2Mp9B4sKKV8", "object": "chat.completion", "created": 1727346158, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"name\\":\\"Science Fair\\",\\"date\\":\\"Friday\\",\\"participants\\":[\\"Alice\\",\\"Bob\\"]}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 92, "completion_tokens": 17, "total_tokens": 109, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_7568d46099"}'
389 ),
390 mock_client=client,
391 respx_mock=respx_mock,
392 )
393
394 assert print_obj(completion, monkeypatch) == snapshot(
395 """\
396ParsedChatCompletion[CalendarEvent](
397 choices=[
398 ParsedChoice[CalendarEvent](
399 finish_reason='stop',
400 index=0,
401 logprobs=None,
402 message=ParsedChatCompletionMessage[CalendarEvent](
403 annotations=None,
404 audio=None,
405 content='{"name":"Science Fair","date":"Friday","participants":["Alice","Bob"]}',
406 function_call=None,
407 parsed=CalendarEvent(name='Science Fair', date='Friday', participants=['Alice', 'Bob']),
408 refusal=None,
409 role='assistant',
410 tool_calls=None
411 )
412 )
413 ],
414 created=1727346158,
415 id='chatcmpl-ABfvqhz4uUUWsw8Ohw2Mp9B4sKKV8',
416 model='gpt-4o-2024-08-06',
417 object='chat.completion',
418 service_tier=None,
419 system_fingerprint='fp_7568d46099',
420 usage=CompletionUsage(
421 completion_tokens=17,
422 completion_tokens_details=CompletionTokensDetails(
423 accepted_prediction_tokens=None,
424 audio_tokens=None,
425 reasoning_tokens=0,
426 rejected_prediction_tokens=None
427 ),
428 prompt_tokens=92,
429 prompt_tokens_details=None,
430 total_tokens=109
431 )
432)
433"""
434 )
435
436
437@pytest.mark.respx(base_url=base_url)
438def test_pydantic_tool_model_all_types(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
439 completion = _make_snapshot_request(
440 lambda c: c.chat.completions.parse(
441 model="gpt-4o-2024-08-06",
442 messages=[
443 {
444 "role": "user",
445 "content": "look up all my orders in may of last year that were fulfilled but not delivered on time",
446 },
447 ],
448 tools=[openai.pydantic_function_tool(Query)],
449 response_format=Query,
450 ),
451 content_snapshot=snapshot(
452 '{"id": "chatcmpl-ABfvtNiaTNUF6OymZUnEFc9lPq9p1", "object": "chat.completion", "created": 1727346161, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_NKpApJybW1MzOjZO2FzwYw0d", "type": "function", "function": {"name": "Query", "arguments": "{\\"name\\":\\"May 2022 Fulfilled Orders Not Delivered on Time\\",\\"table_name\\":\\"orders\\",\\"columns\\":[\\"id\\",\\"status\\",\\"expected_delivery_date\\",\\"delivered_at\\",\\"shipped_at\\",\\"ordered_at\\",\\"canceled_at\\"],\\"conditions\\":[{\\"column\\":\\"ordered_at\\",\\"operator\\":\\">=\\",\\"value\\":\\"2022-05-01\\"},{\\"column\\":\\"ordered_at\\",\\"operator\\":\\"<=\\",\\"value\\":\\"2022-05-31\\"},{\\"column\\":\\"status\\",\\"operator\\":\\"=\\",\\"value\\":\\"fulfilled\\"},{\\"column\\":\\"delivered_at\\",\\"operator\\":\\">\\",\\"value\\":{\\"column_name\\":\\"expected_delivery_date\\"}}],\\"order_by\\":\\"asc\\"}"}}], "refusal": null}, "logprobs": null, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 512, "completion_tokens": 132, "total_tokens": 644, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_7568d46099"}'
453 ),
454 mock_client=client,
455 respx_mock=respx_mock,
456 )
457
458 assert print_obj(completion.choices[0], monkeypatch) == snapshot(
459 """\
460ParsedChoice[Query](
461 finish_reason='tool_calls',
462 index=0,
463 logprobs=None,
464 message=ParsedChatCompletionMessage[Query](
465 annotations=None,
466 audio=None,
467 content=None,
468 function_call=None,
469 parsed=None,
470 refusal=None,
471 role='assistant',
472 tool_calls=[
473 ParsedFunctionToolCall(
474 function=ParsedFunction(
475 arguments='{"name":"May 2022 Fulfilled Orders Not Delivered on
476Time","table_name":"orders","columns":["id","status","expected_delivery_date","delivered_at","shipped_at","ordered_at","
477canceled_at"],"conditions":[{"column":"ordered_at","operator":">=","value":"2022-05-01"},{"column":"ordered_at","operato
478r":"<=","value":"2022-05-31"},{"column":"status","operator":"=","value":"fulfilled"},{"column":"delivered_at","operator"
479:">","value":{"column_name":"expected_delivery_date"}}],"order_by":"asc"}',
480 name='Query',
481 parsed_arguments=Query(
482 columns=[
483 <Column.id: 'id'>,
484 <Column.status: 'status'>,
485 <Column.expected_delivery_date: 'expected_delivery_date'>,
486 <Column.delivered_at: 'delivered_at'>,
487 <Column.shipped_at: 'shipped_at'>,
488 <Column.ordered_at: 'ordered_at'>,
489 <Column.canceled_at: 'canceled_at'>
490 ],
491 conditions=[
492 Condition(column='ordered_at', operator=<Operator.ge: '>='>, value='2022-05-01'),
493 Condition(column='ordered_at', operator=<Operator.le: '<='>, value='2022-05-31'),
494 Condition(column='status', operator=<Operator.eq: '='>, value='fulfilled'),
495 Condition(
496 column='delivered_at',
497 operator=<Operator.gt: '>'>,
498 value=DynamicValue(column_name='expected_delivery_date')
499 )
500 ],
501 name='May 2022 Fulfilled Orders Not Delivered on Time',
502 order_by=<OrderBy.asc: 'asc'>,
503 table_name=<Table.orders: 'orders'>
504 )
505 ),
506 id='call_NKpApJybW1MzOjZO2FzwYw0d',
507 type='function'
508 )
509 ]
510 )
511)
512"""
513 )
514
515
516@pytest.mark.respx(base_url=base_url)
517def test_parse_max_tokens_reached(client: OpenAI, respx_mock: MockRouter) -> None:
518 class Location(BaseModel):
519 city: str
520 temperature: float
521 units: Literal["c", "f"]
522
523 with pytest.raises(openai.LengthFinishReasonError):
524 _make_snapshot_request(
525 lambda c: c.chat.completions.parse(
526 model="gpt-4o-2024-08-06",
527 messages=[
528 {
529 "role": "user",
530 "content": "What's the weather like in SF?",
531 },
532 ],
533 max_tokens=1,
534 response_format=Location,
535 ),
536 content_snapshot=snapshot(
537 '{"id": "chatcmpl-ABfvvX7eB1KsfeZj8VcF3z7G7SbaA", "object": "chat.completion", "created": 1727346163, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"", "refusal": null}, "logprobs": null, "finish_reason": "length"}], "usage": {"prompt_tokens": 79, "completion_tokens": 1, "total_tokens": 80, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_7568d46099"}'
538 ),
539 mock_client=client,
540 respx_mock=respx_mock,
541 )
542
543
544@pytest.mark.respx(base_url=base_url)
545def test_parse_pydantic_model_refusal(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
546 class Location(BaseModel):
547 city: str
548 temperature: float
549 units: Literal["c", "f"]
550
551 completion = _make_snapshot_request(
552 lambda c: c.chat.completions.parse(
553 model="gpt-4o-2024-08-06",
554 messages=[
555 {
556 "role": "user",
557 "content": "How do I make anthrax?",
558 },
559 ],
560 response_format=Location,
561 ),
562 content_snapshot=snapshot(
563 '{"id": "chatcmpl-ABfvwoKVWPQj2UPlAcAKM7s40GsRx", "object": "chat.completion", "created": 1727346164, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "refusal": "I\'m very sorry, but I can\'t assist with that."}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 79, "completion_tokens": 12, "total_tokens": 91, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
564 ),
565 mock_client=client,
566 respx_mock=respx_mock,
567 )
568
569 assert print_obj(completion.choices, monkeypatch) == snapshot(
570 """\
571[
572 ParsedChoice[Location](
573 finish_reason='stop',
574 index=0,
575 logprobs=None,
576 message=ParsedChatCompletionMessage[Location](
577 annotations=None,
578 audio=None,
579 content=None,
580 function_call=None,
581 parsed=None,
582 refusal="I'm very sorry, but I can't assist with that.",
583 role='assistant',
584 tool_calls=None
585 )
586 )
587]
588"""
589 )
590
591
592@pytest.mark.respx(base_url=base_url)
593def test_parse_pydantic_tool(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
594 class GetWeatherArgs(BaseModel):
595 city: str
596 country: str
597 units: Literal["c", "f"] = "c"
598
599 completion = _make_snapshot_request(
600 lambda c: c.chat.completions.parse(
601 model="gpt-4o-2024-08-06",
602 messages=[
603 {
604 "role": "user",
605 "content": "What's the weather like in Edinburgh?",
606 },
607 ],
608 tools=[
609 openai.pydantic_function_tool(GetWeatherArgs),
610 ],
611 ),
612 content_snapshot=snapshot(
613 '{"id": "chatcmpl-ABfvx6Z4dchiW2nya1N8KMsHFrQRE", "object": "chat.completion", "created": 1727346165, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_Y6qJ7ofLgOrBnMD5WbVAeiRV", "type": "function", "function": {"name": "GetWeatherArgs", "arguments": "{\\"city\\":\\"Edinburgh\\",\\"country\\":\\"UK\\",\\"units\\":\\"c\\"}"}}], "refusal": null}, "logprobs": null, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 76, "completion_tokens": 24, "total_tokens": 100, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_e45dabd248"}'
614 ),
615 mock_client=client,
616 respx_mock=respx_mock,
617 )
618
619 assert print_obj(completion.choices, monkeypatch) == snapshot(
620 """\
621[
622 ParsedChoice[NoneType](
623 finish_reason='tool_calls',
624 index=0,
625 logprobs=None,
626 message=ParsedChatCompletionMessage[NoneType](
627 annotations=None,
628 audio=None,
629 content=None,
630 function_call=None,
631 parsed=None,
632 refusal=None,
633 role='assistant',
634 tool_calls=[
635 ParsedFunctionToolCall(
636 function=ParsedFunction(
637 arguments='{"city":"Edinburgh","country":"UK","units":"c"}',
638 name='GetWeatherArgs',
639 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='UK', units='c')
640 ),
641 id='call_Y6qJ7ofLgOrBnMD5WbVAeiRV',
642 type='function'
643 )
644 ]
645 )
646 )
647]
648"""
649 )
650
651
652@pytest.mark.respx(base_url=base_url)
653def test_parse_multiple_pydantic_tools(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
654 class GetWeatherArgs(BaseModel):
655 """Get the temperature for the given country/city combo"""
656
657 city: str
658 country: str
659 units: Literal["c", "f"] = "c"
660
661 class GetStockPrice(BaseModel):
662 ticker: str
663 exchange: str
664
665 completion = _make_snapshot_request(
666 lambda c: c.chat.completions.parse(
667 model="gpt-4o-2024-08-06",
668 messages=[
669 {
670 "role": "user",
671 "content": "What's the weather like in Edinburgh?",
672 },
673 {
674 "role": "user",
675 "content": "What's the price of AAPL?",
676 },
677 ],
678 tools=[
679 openai.pydantic_function_tool(GetWeatherArgs),
680 openai.pydantic_function_tool(
681 GetStockPrice, name="get_stock_price", description="Fetch the latest price for a given ticker"
682 ),
683 ],
684 ),
685 content_snapshot=snapshot(
686 '{"id": "chatcmpl-ABfvyvfNWKcl7Ohqos4UFrmMs1v4C", "object": "chat.completion", "created": 1727346166, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_fdNz3vOBKYgOIpMdWotB9MjY", "type": "function", "function": {"name": "GetWeatherArgs", "arguments": "{\\"city\\": \\"Edinburgh\\", \\"country\\": \\"GB\\", \\"units\\": \\"c\\"}"}}, {"id": "call_h1DWI1POMJLb0KwIyQHWXD4p", "type": "function", "function": {"name": "get_stock_price", "arguments": "{\\"ticker\\": \\"AAPL\\", \\"exchange\\": \\"NASDAQ\\"}"}}], "refusal": null}, "logprobs": null, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 149, "completion_tokens": 60, "total_tokens": 209, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_b40fb1c6fb"}'
687 ),
688 mock_client=client,
689 respx_mock=respx_mock,
690 )
691
692 assert print_obj(completion.choices, monkeypatch) == snapshot(
693 """\
694[
695 ParsedChoice[NoneType](
696 finish_reason='tool_calls',
697 index=0,
698 logprobs=None,
699 message=ParsedChatCompletionMessage[NoneType](
700 annotations=None,
701 audio=None,
702 content=None,
703 function_call=None,
704 parsed=None,
705 refusal=None,
706 role='assistant',
707 tool_calls=[
708 ParsedFunctionToolCall(
709 function=ParsedFunction(
710 arguments='{"city": "Edinburgh", "country": "GB", "units": "c"}',
711 name='GetWeatherArgs',
712 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='GB', units='c')
713 ),
714 id='call_fdNz3vOBKYgOIpMdWotB9MjY',
715 type='function'
716 ),
717 ParsedFunctionToolCall(
718 function=ParsedFunction(
719 arguments='{"ticker": "AAPL", "exchange": "NASDAQ"}',
720 name='get_stock_price',
721 parsed_arguments=GetStockPrice(exchange='NASDAQ', ticker='AAPL')
722 ),
723 id='call_h1DWI1POMJLb0KwIyQHWXD4p',
724 type='function'
725 )
726 ]
727 )
728 )
729]
730"""
731 )
732
733
734@pytest.mark.respx(base_url=base_url)
735def test_parse_strict_tools(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
736 completion = _make_snapshot_request(
737 lambda c: c.chat.completions.parse(
738 model="gpt-4o-2024-08-06",
739 messages=[
740 {
741 "role": "user",
742 "content": "What's the weather like in SF?",
743 },
744 ],
745 tools=[
746 {
747 "type": "function",
748 "function": {
749 "name": "get_weather",
750 "parameters": {
751 "type": "object",
752 "properties": {
753 "city": {"type": "string"},
754 "state": {"type": "string"},
755 },
756 "required": [
757 "city",
758 "state",
759 ],
760 "additionalProperties": False,
761 },
762 "strict": True,
763 },
764 }
765 ],
766 ),
767 content_snapshot=snapshot(
768 '{"id": "chatcmpl-ABfvzdvCI6RaIkiEFNjqGXCSYnlzf", "object": "chat.completion", "created": 1727346167, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_CUdUoJpsWWVdxXntucvnol1M", "type": "function", "function": {"name": "get_weather", "arguments": "{\\"city\\":\\"San Francisco\\",\\"state\\":\\"CA\\"}"}}], "refusal": null}, "logprobs": null, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 48, "completion_tokens": 19, "total_tokens": 67, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
769 ),
770 mock_client=client,
771 respx_mock=respx_mock,
772 )
773
774 assert print_obj(completion.choices, monkeypatch) == snapshot(
775 """\
776[
777 ParsedChoice[NoneType](
778 finish_reason='tool_calls',
779 index=0,
780 logprobs=None,
781 message=ParsedChatCompletionMessage[NoneType](
782 annotations=None,
783 audio=None,
784 content=None,
785 function_call=None,
786 parsed=None,
787 refusal=None,
788 role='assistant',
789 tool_calls=[
790 ParsedFunctionToolCall(
791 function=ParsedFunction(
792 arguments='{"city":"San Francisco","state":"CA"}',
793 name='get_weather',
794 parsed_arguments={'city': 'San Francisco', 'state': 'CA'}
795 ),
796 id='call_CUdUoJpsWWVdxXntucvnol1M',
797 type='function'
798 )
799 ]
800 )
801 )
802]
803"""
804 )
805
806
807def test_parse_non_strict_tools(client: OpenAI) -> None:
808 with pytest.raises(
809 ValueError, match="`get_weather` is not strict. Only `strict` function tools can be auto-parsed"
810 ):
811 client.chat.completions.parse(
812 model="gpt-4o-2024-08-06",
813 messages=[],
814 tools=[
815 {
816 "type": "function",
817 "function": {
818 "name": "get_weather",
819 "parameters": {},
820 },
821 }
822 ],
823 )
824
825
826@pytest.mark.respx(base_url=base_url)
827def test_parse_pydantic_raw_response(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
828 class Location(BaseModel):
829 city: str
830 temperature: float
831 units: Literal["c", "f"]
832
833 response = _make_snapshot_request(
834 lambda c: c.chat.completions.with_raw_response.parse(
835 model="gpt-4o-2024-08-06",
836 messages=[
837 {
838 "role": "user",
839 "content": "What's the weather like in SF?",
840 },
841 ],
842 response_format=Location,
843 ),
844 content_snapshot=snapshot(
845 '{"id": "chatcmpl-ABrDYCa8W1w66eUxKDO8TQF1m6trT", "object": "chat.completion", "created": 1727389540, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":58,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 79, "completion_tokens": 14, "total_tokens": 93, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
846 ),
847 mock_client=client,
848 respx_mock=respx_mock,
849 )
850 assert response.http_request.headers.get("x-stainless-helper-method") == "chat.completions.parse"
851
852 completion = response.parse()
853 message = completion.choices[0].message
854 assert message.parsed is not None
855 assert isinstance(message.parsed.city, str)
856 assert print_obj(completion, monkeypatch) == snapshot(
857 """\
858ParsedChatCompletion[Location](
859 choices=[
860 ParsedChoice[Location](
861 finish_reason='stop',
862 index=0,
863 logprobs=None,
864 message=ParsedChatCompletionMessage[Location](
865 annotations=None,
866 audio=None,
867 content='{"city":"San Francisco","temperature":58,"units":"f"}',
868 function_call=None,
869 parsed=Location(city='San Francisco', temperature=58.0, units='f'),
870 refusal=None,
871 role='assistant',
872 tool_calls=None
873 )
874 )
875 ],
876 created=1727389540,
877 id='chatcmpl-ABrDYCa8W1w66eUxKDO8TQF1m6trT',
878 model='gpt-4o-2024-08-06',
879 object='chat.completion',
880 service_tier=None,
881 system_fingerprint='fp_5050236cbd',
882 usage=CompletionUsage(
883 completion_tokens=14,
884 completion_tokens_details=CompletionTokensDetails(
885 accepted_prediction_tokens=None,
886 audio_tokens=None,
887 reasoning_tokens=0,
888 rejected_prediction_tokens=None
889 ),
890 prompt_tokens=79,
891 prompt_tokens_details=None,
892 total_tokens=93
893 )
894)
895"""
896 )
897
898
899@pytest.mark.respx(base_url=base_url)
900@pytest.mark.asyncio
901async def test_async_parse_pydantic_raw_response(
902 async_client: AsyncOpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch
903) -> None:
904 class Location(BaseModel):
905 city: str
906 temperature: float
907 units: Literal["c", "f"]
908
909 response = await _make_async_snapshot_request(
910 lambda c: c.chat.completions.with_raw_response.parse(
911 model="gpt-4o-2024-08-06",
912 messages=[
913 {
914 "role": "user",
915 "content": "What's the weather like in SF?",
916 },
917 ],
918 response_format=Location,
919 ),
920 content_snapshot=snapshot(
921 '{"id": "chatcmpl-ABrDQWOiw0PK5JOsxl1D9ooeQgznq", "object": "chat.completion", "created": 1727389532, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":65,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 79, "completion_tokens": 14, "total_tokens": 93, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
922 ),
923 mock_client=async_client,
924 respx_mock=respx_mock,
925 )
926 assert response.http_request.headers.get("x-stainless-helper-method") == "chat.completions.parse"
927
928 completion = response.parse()
929 message = completion.choices[0].message
930 assert message.parsed is not None
931 assert isinstance(message.parsed.city, str)
932 assert print_obj(completion, monkeypatch) == snapshot(
933 """\
934ParsedChatCompletion[Location](
935 choices=[
936 ParsedChoice[Location](
937 finish_reason='stop',
938 index=0,
939 logprobs=None,
940 message=ParsedChatCompletionMessage[Location](
941 annotations=None,
942 audio=None,
943 content='{"city":"San Francisco","temperature":65,"units":"f"}',
944 function_call=None,
945 parsed=Location(city='San Francisco', temperature=65.0, units='f'),
946 refusal=None,
947 role='assistant',
948 tool_calls=None
949 )
950 )
951 ],
952 created=1727389532,
953 id='chatcmpl-ABrDQWOiw0PK5JOsxl1D9ooeQgznq',
954 model='gpt-4o-2024-08-06',
955 object='chat.completion',
956 service_tier=None,
957 system_fingerprint='fp_5050236cbd',
958 usage=CompletionUsage(
959 completion_tokens=14,
960 completion_tokens_details=CompletionTokensDetails(
961 accepted_prediction_tokens=None,
962 audio_tokens=None,
963 reasoning_tokens=0,
964 rejected_prediction_tokens=None
965 ),
966 prompt_tokens=79,
967 prompt_tokens_details=None,
968 total_tokens=93
969 )
970)
971"""
972 )
973
974
975@pytest.mark.parametrize("sync", [True, False], ids=["sync", "async"])
976def test_parse_method_in_sync(sync: bool, client: OpenAI, async_client: AsyncOpenAI) -> None:
977 checking_client: OpenAI | AsyncOpenAI = client if sync else async_client
978
979 assert_signatures_in_sync(
980 checking_client.chat.completions.create,
981 checking_client.chat.completions.parse,
982 exclude_params={"response_format", "stream"},
983 )
984
985
986def _make_snapshot_request(
987 func: Callable[[OpenAI], _T],
988 *,
989 content_snapshot: Any,
990 respx_mock: MockRouter,
991 mock_client: OpenAI,
992) -> _T:
993 live = os.environ.get("OPENAI_LIVE") == "1"
994 if live:
995
996 def _on_response(response: httpx.Response) -> None:
997 # update the content snapshot
998 assert json.dumps(json.loads(response.read())) == content_snapshot
999
1000 respx_mock.stop()
1001
1002 client = OpenAI(
1003 http_client=httpx.Client(
1004 event_hooks={
1005 "response": [_on_response],
1006 }
1007 )
1008 )
1009 else:
1010 respx_mock.post("/chat/completions").mock(
1011 return_value=httpx.Response(
1012 200,
1013 content=get_snapshot_value(content_snapshot),
1014 headers={"content-type": "application/json"},
1015 )
1016 )
1017
1018 client = mock_client
1019
1020 result = func(client)
1021
1022 if live:
1023 client.close()
1024
1025 return result
1026
1027
1028async def _make_async_snapshot_request(
1029 func: Callable[[AsyncOpenAI], Awaitable[_T]],
1030 *,
1031 content_snapshot: Any,
1032 respx_mock: MockRouter,
1033 mock_client: AsyncOpenAI,
1034) -> _T:
1035 live = os.environ.get("OPENAI_LIVE") == "1"
1036 if live:
1037
1038 async def _on_response(response: httpx.Response) -> None:
1039 # update the content snapshot
1040 assert json.dumps(json.loads(await response.aread())) == content_snapshot
1041
1042 respx_mock.stop()
1043
1044 client = AsyncOpenAI(
1045 http_client=httpx.AsyncClient(
1046 event_hooks={
1047 "response": [_on_response],
1048 }
1049 )
1050 )
1051 else:
1052 respx_mock.post("/chat/completions").mock(
1053 return_value=httpx.Response(
1054 200,
1055 content=get_snapshot_value(content_snapshot),
1056 headers={"content-type": "application/json"},
1057 )
1058 )
1059
1060 client = mock_client
1061
1062 result = await func(client)
1063
1064 if live:
1065 await client.close()
1066
1067 return result
1068