GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

openai/openai-python

Public

mirrored fromhttps://github.com/openai/openai-pythonAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
v1.61.0

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openai-python/tests/lib/chat

tests/lib/chat/test_completions.py

1052lines · modecode

RawDownload
Latest commit unavailable.
1from __future__ import annotations
2
3import os
4import json
5from enum import Enum
6from typing import Any, List, Callable, Optional, Awaitable
7from typing_extensions import Literal, TypeVar
8
9import httpx
10import pytest
11from respx import MockRouter
12from pydantic import Field, BaseModel
13from inline_snapshot import snapshot
14
15import openai
16from openai import OpenAI, AsyncOpenAI
17from openai._utils import assert_signatures_in_sync
18from openai._compat import PYDANTIC_V2
19
20from ._utils import print_obj
21from ...conftest import base_url
22from ..schema_types.query import Query
23
24_T = TypeVar("_T")
25
26# all the snapshots in this file are auto-generated from the live API
27#
28# you can update them with
29#
30# `OPENAI_LIVE=1 pytest --inline-snapshot=fix`
31
32
33@pytest.mark.respx(base_url=base_url)
34def test_parse_nothing(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
35 completion = _make_snapshot_request(
36 lambda c: c.beta.chat.completions.parse(
37 model="gpt-4o-2024-08-06",
38 messages=[
39 {
40 "role": "user",
41 "content": "What's the weather like in SF?",
42 },
43 ],
44 ),
45 content_snapshot=snapshot(
46 '{"id": "chatcmpl-ABfvaueLEMLNYbT8YzpJxsmiQ6HSY", "object": "chat.completion", "created": 1727346142, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "I\'m unable to provide real-time weather updates. To get the current weather in San Francisco, I recommend checking a reliable weather website or app like the Weather Channel or a local news station.", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 14, "completion_tokens": 37, "total_tokens": 51, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_b40fb1c6fb"}'
47 ),
48 mock_client=client,
49 respx_mock=respx_mock,
50 )
51
52 assert print_obj(completion, monkeypatch) == snapshot(
53 """\
54ParsedChatCompletion[NoneType](
55 choices=[
56 ParsedChoice[NoneType](
57 finish_reason='stop',
58 index=0,
59 logprobs=None,
60 message=ParsedChatCompletionMessage[NoneType](
61 audio=None,
62 content="I'm unable to provide real-time weather updates. To get the current weather in San Francisco, I
63recommend checking a reliable weather website or app like the Weather Channel or a local news station.",
64 function_call=None,
65 parsed=None,
66 refusal=None,
67 role='assistant',
68 tool_calls=[]
69 )
70 )
71 ],
72 created=1727346142,
73 id='chatcmpl-ABfvaueLEMLNYbT8YzpJxsmiQ6HSY',
74 model='gpt-4o-2024-08-06',
75 object='chat.completion',
76 service_tier=None,
77 system_fingerprint='fp_b40fb1c6fb',
78 usage=CompletionUsage(
79 completion_tokens=37,
80 completion_tokens_details=CompletionTokensDetails(
81 accepted_prediction_tokens=None,
82 audio_tokens=None,
83 reasoning_tokens=0,
84 rejected_prediction_tokens=None
85 ),
86 prompt_tokens=14,
87 prompt_tokens_details=None,
88 total_tokens=51
89 )
90)
91"""
92 )
93
94
95@pytest.mark.respx(base_url=base_url)
96def test_parse_pydantic_model(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
97 class Location(BaseModel):
98 city: str
99 temperature: float
100 units: Literal["c", "f"]
101
102 completion = _make_snapshot_request(
103 lambda c: c.beta.chat.completions.parse(
104 model="gpt-4o-2024-08-06",
105 messages=[
106 {
107 "role": "user",
108 "content": "What's the weather like in SF?",
109 },
110 ],
111 response_format=Location,
112 ),
113 content_snapshot=snapshot(
114 '{"id": "chatcmpl-ABfvbtVnTu5DeC4EFnRYj8mtfOM99", "object": "chat.completion", "created": 1727346143, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":65,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 79, "completion_tokens": 14, "total_tokens": 93, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
115 ),
116 mock_client=client,
117 respx_mock=respx_mock,
118 )
119
120 assert print_obj(completion, monkeypatch) == snapshot(
121 """\
122ParsedChatCompletion[Location](
123 choices=[
124 ParsedChoice[Location](
125 finish_reason='stop',
126 index=0,
127 logprobs=None,
128 message=ParsedChatCompletionMessage[Location](
129 audio=None,
130 content='{"city":"San Francisco","temperature":65,"units":"f"}',
131 function_call=None,
132 parsed=Location(city='San Francisco', temperature=65.0, units='f'),
133 refusal=None,
134 role='assistant',
135 tool_calls=[]
136 )
137 )
138 ],
139 created=1727346143,
140 id='chatcmpl-ABfvbtVnTu5DeC4EFnRYj8mtfOM99',
141 model='gpt-4o-2024-08-06',
142 object='chat.completion',
143 service_tier=None,
144 system_fingerprint='fp_5050236cbd',
145 usage=CompletionUsage(
146 completion_tokens=14,
147 completion_tokens_details=CompletionTokensDetails(
148 accepted_prediction_tokens=None,
149 audio_tokens=None,
150 reasoning_tokens=0,
151 rejected_prediction_tokens=None
152 ),
153 prompt_tokens=79,
154 prompt_tokens_details=None,
155 total_tokens=93
156 )
157)
158"""
159 )
160
161
162@pytest.mark.respx(base_url=base_url)
163def test_parse_pydantic_model_optional_default(
164 client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch
165) -> None:
166 class Location(BaseModel):
167 city: str
168 temperature: float
169 units: Optional[Literal["c", "f"]] = None
170
171 completion = _make_snapshot_request(
172 lambda c: c.beta.chat.completions.parse(
173 model="gpt-4o-2024-08-06",
174 messages=[
175 {
176 "role": "user",
177 "content": "What's the weather like in SF?",
178 },
179 ],
180 response_format=Location,
181 ),
182 content_snapshot=snapshot(
183 '{"id": "chatcmpl-ABfvcC8grKYsRkSoMp9CCAhbXAd0b", "object": "chat.completion", "created": 1727346144, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":65,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 88, "completion_tokens": 14, "total_tokens": 102, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_b40fb1c6fb"}'
184 ),
185 mock_client=client,
186 respx_mock=respx_mock,
187 )
188
189 assert print_obj(completion, monkeypatch) == snapshot(
190 """\
191ParsedChatCompletion[Location](
192 choices=[
193 ParsedChoice[Location](
194 finish_reason='stop',
195 index=0,
196 logprobs=None,
197 message=ParsedChatCompletionMessage[Location](
198 audio=None,
199 content='{"city":"San Francisco","temperature":65,"units":"f"}',
200 function_call=None,
201 parsed=Location(city='San Francisco', temperature=65.0, units='f'),
202 refusal=None,
203 role='assistant',
204 tool_calls=[]
205 )
206 )
207 ],
208 created=1727346144,
209 id='chatcmpl-ABfvcC8grKYsRkSoMp9CCAhbXAd0b',
210 model='gpt-4o-2024-08-06',
211 object='chat.completion',
212 service_tier=None,
213 system_fingerprint='fp_b40fb1c6fb',
214 usage=CompletionUsage(
215 completion_tokens=14,
216 completion_tokens_details=CompletionTokensDetails(
217 accepted_prediction_tokens=None,
218 audio_tokens=None,
219 reasoning_tokens=0,
220 rejected_prediction_tokens=None
221 ),
222 prompt_tokens=88,
223 prompt_tokens_details=None,
224 total_tokens=102
225 )
226)
227"""
228 )
229
230
231@pytest.mark.respx(base_url=base_url)
232def test_parse_pydantic_model_enum(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
233 class Color(Enum):
234 """The detected color"""
235
236 RED = "red"
237 BLUE = "blue"
238 GREEN = "green"
239
240 class ColorDetection(BaseModel):
241 color: Color
242 hex_color_code: str = Field(description="The hex color code of the detected color")
243
244 if not PYDANTIC_V2:
245 ColorDetection.update_forward_refs(**locals()) # type: ignore
246
247 completion = _make_snapshot_request(
248 lambda c: c.beta.chat.completions.parse(
249 model="gpt-4o-2024-08-06",
250 messages=[
251 {"role": "user", "content": "What color is a Coke can?"},
252 ],
253 response_format=ColorDetection,
254 ),
255 content_snapshot=snapshot(
256 '{"id": "chatcmpl-ABfvjIatz0zrZu50gRbMtlp0asZpz", "object": "chat.completion", "created": 1727346151, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"color\\":\\"red\\",\\"hex_color_code\\":\\"#FF0000\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 109, "completion_tokens": 14, "total_tokens": 123, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
257 ),
258 mock_client=client,
259 respx_mock=respx_mock,
260 )
261
262 assert print_obj(completion.choices[0], monkeypatch) == snapshot(
263 """\
264ParsedChoice[ColorDetection](
265 finish_reason='stop',
266 index=0,
267 logprobs=None,
268 message=ParsedChatCompletionMessage[ColorDetection](
269 audio=None,
270 content='{"color":"red","hex_color_code":"#FF0000"}',
271 function_call=None,
272 parsed=ColorDetection(color=<Color.RED: 'red'>, hex_color_code='#FF0000'),
273 refusal=None,
274 role='assistant',
275 tool_calls=[]
276 )
277)
278"""
279 )
280
281
282@pytest.mark.respx(base_url=base_url)
283def test_parse_pydantic_model_multiple_choices(
284 client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch
285) -> None:
286 class Location(BaseModel):
287 city: str
288 temperature: float
289 units: Literal["c", "f"]
290
291 completion = _make_snapshot_request(
292 lambda c: c.beta.chat.completions.parse(
293 model="gpt-4o-2024-08-06",
294 messages=[
295 {
296 "role": "user",
297 "content": "What's the weather like in SF?",
298 },
299 ],
300 n=3,
301 response_format=Location,
302 ),
303 content_snapshot=snapshot(
304 '{"id": "chatcmpl-ABfvp8qzboW92q8ONDF4DPHlI7ckC", "object": "chat.completion", "created": 1727346157, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":64,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}, {"index": 1, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":65,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}, {"index": 2, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":63.0,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 79, "completion_tokens": 44, "total_tokens": 123, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_b40fb1c6fb"}'
305 ),
306 mock_client=client,
307 respx_mock=respx_mock,
308 )
309
310 assert print_obj(completion.choices, monkeypatch) == snapshot(
311 """\
312[
313 ParsedChoice[Location](
314 finish_reason='stop',
315 index=0,
316 logprobs=None,
317 message=ParsedChatCompletionMessage[Location](
318 audio=None,
319 content='{"city":"San Francisco","temperature":64,"units":"f"}',
320 function_call=None,
321 parsed=Location(city='San Francisco', temperature=64.0, units='f'),
322 refusal=None,
323 role='assistant',
324 tool_calls=[]
325 )
326 ),
327 ParsedChoice[Location](
328 finish_reason='stop',
329 index=1,
330 logprobs=None,
331 message=ParsedChatCompletionMessage[Location](
332 audio=None,
333 content='{"city":"San Francisco","temperature":65,"units":"f"}',
334 function_call=None,
335 parsed=Location(city='San Francisco', temperature=65.0, units='f'),
336 refusal=None,
337 role='assistant',
338 tool_calls=[]
339 )
340 ),
341 ParsedChoice[Location](
342 finish_reason='stop',
343 index=2,
344 logprobs=None,
345 message=ParsedChatCompletionMessage[Location](
346 audio=None,
347 content='{"city":"San Francisco","temperature":63.0,"units":"f"}',
348 function_call=None,
349 parsed=Location(city='San Francisco', temperature=63.0, units='f'),
350 refusal=None,
351 role='assistant',
352 tool_calls=[]
353 )
354 )
355]
356"""
357 )
358
359
360@pytest.mark.respx(base_url=base_url)
361@pytest.mark.skipif(not PYDANTIC_V2, reason="dataclasses only supported in v2")
362def test_parse_pydantic_dataclass(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
363 from pydantic.dataclasses import dataclass
364
365 @dataclass
366 class CalendarEvent:
367 name: str
368 date: str
369 participants: List[str]
370
371 completion = _make_snapshot_request(
372 lambda c: c.beta.chat.completions.parse(
373 model="gpt-4o-2024-08-06",
374 messages=[
375 {"role": "system", "content": "Extract the event information."},
376 {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."},
377 ],
378 response_format=CalendarEvent,
379 ),
380 content_snapshot=snapshot(
381 '{"id": "chatcmpl-ABfvqhz4uUUWsw8Ohw2Mp9B4sKKV8", "object": "chat.completion", "created": 1727346158, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"name\\":\\"Science Fair\\",\\"date\\":\\"Friday\\",\\"participants\\":[\\"Alice\\",\\"Bob\\"]}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 92, "completion_tokens": 17, "total_tokens": 109, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_7568d46099"}'
382 ),
383 mock_client=client,
384 respx_mock=respx_mock,
385 )
386
387 assert print_obj(completion, monkeypatch) == snapshot(
388 """\
389ParsedChatCompletion[CalendarEvent](
390 choices=[
391 ParsedChoice[CalendarEvent](
392 finish_reason='stop',
393 index=0,
394 logprobs=None,
395 message=ParsedChatCompletionMessage[CalendarEvent](
396 audio=None,
397 content='{"name":"Science Fair","date":"Friday","participants":["Alice","Bob"]}',
398 function_call=None,
399 parsed=CalendarEvent(name='Science Fair', date='Friday', participants=['Alice', 'Bob']),
400 refusal=None,
401 role='assistant',
402 tool_calls=[]
403 )
404 )
405 ],
406 created=1727346158,
407 id='chatcmpl-ABfvqhz4uUUWsw8Ohw2Mp9B4sKKV8',
408 model='gpt-4o-2024-08-06',
409 object='chat.completion',
410 service_tier=None,
411 system_fingerprint='fp_7568d46099',
412 usage=CompletionUsage(
413 completion_tokens=17,
414 completion_tokens_details=CompletionTokensDetails(
415 accepted_prediction_tokens=None,
416 audio_tokens=None,
417 reasoning_tokens=0,
418 rejected_prediction_tokens=None
419 ),
420 prompt_tokens=92,
421 prompt_tokens_details=None,
422 total_tokens=109
423 )
424)
425"""
426 )
427
428
429@pytest.mark.respx(base_url=base_url)
430def test_pydantic_tool_model_all_types(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
431 completion = _make_snapshot_request(
432 lambda c: c.beta.chat.completions.parse(
433 model="gpt-4o-2024-08-06",
434 messages=[
435 {
436 "role": "user",
437 "content": "look up all my orders in may of last year that were fulfilled but not delivered on time",
438 },
439 ],
440 tools=[openai.pydantic_function_tool(Query)],
441 response_format=Query,
442 ),
443 content_snapshot=snapshot(
444 '{"id": "chatcmpl-ABfvtNiaTNUF6OymZUnEFc9lPq9p1", "object": "chat.completion", "created": 1727346161, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_NKpApJybW1MzOjZO2FzwYw0d", "type": "function", "function": {"name": "Query", "arguments": "{\\"name\\":\\"May 2022 Fulfilled Orders Not Delivered on Time\\",\\"table_name\\":\\"orders\\",\\"columns\\":[\\"id\\",\\"status\\",\\"expected_delivery_date\\",\\"delivered_at\\",\\"shipped_at\\",\\"ordered_at\\",\\"canceled_at\\"],\\"conditions\\":[{\\"column\\":\\"ordered_at\\",\\"operator\\":\\">=\\",\\"value\\":\\"2022-05-01\\"},{\\"column\\":\\"ordered_at\\",\\"operator\\":\\"<=\\",\\"value\\":\\"2022-05-31\\"},{\\"column\\":\\"status\\",\\"operator\\":\\"=\\",\\"value\\":\\"fulfilled\\"},{\\"column\\":\\"delivered_at\\",\\"operator\\":\\">\\",\\"value\\":{\\"column_name\\":\\"expected_delivery_date\\"}}],\\"order_by\\":\\"asc\\"}"}}], "refusal": null}, "logprobs": null, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 512, "completion_tokens": 132, "total_tokens": 644, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_7568d46099"}'
445 ),
446 mock_client=client,
447 respx_mock=respx_mock,
448 )
449
450 assert print_obj(completion.choices[0], monkeypatch) == snapshot(
451 """\
452ParsedChoice[Query](
453 finish_reason='tool_calls',
454 index=0,
455 logprobs=None,
456 message=ParsedChatCompletionMessage[Query](
457 audio=None,
458 content=None,
459 function_call=None,
460 parsed=None,
461 refusal=None,
462 role='assistant',
463 tool_calls=[
464 ParsedFunctionToolCall(
465 function=ParsedFunction(
466 arguments='{"name":"May 2022 Fulfilled Orders Not Delivered on
467Time","table_name":"orders","columns":["id","status","expected_delivery_date","delivered_at","shipped_at","ordered_at","
468canceled_at"],"conditions":[{"column":"ordered_at","operator":">=","value":"2022-05-01"},{"column":"ordered_at","operato
469r":"<=","value":"2022-05-31"},{"column":"status","operator":"=","value":"fulfilled"},{"column":"delivered_at","operator"
470:">","value":{"column_name":"expected_delivery_date"}}],"order_by":"asc"}',
471 name='Query',
472 parsed_arguments=Query(
473 columns=[
474 <Column.id: 'id'>,
475 <Column.status: 'status'>,
476 <Column.expected_delivery_date: 'expected_delivery_date'>,
477 <Column.delivered_at: 'delivered_at'>,
478 <Column.shipped_at: 'shipped_at'>,
479 <Column.ordered_at: 'ordered_at'>,
480 <Column.canceled_at: 'canceled_at'>
481 ],
482 conditions=[
483 Condition(column='ordered_at', operator=<Operator.ge: '>='>, value='2022-05-01'),
484 Condition(column='ordered_at', operator=<Operator.le: '<='>, value='2022-05-31'),
485 Condition(column='status', operator=<Operator.eq: '='>, value='fulfilled'),
486 Condition(
487 column='delivered_at',
488 operator=<Operator.gt: '>'>,
489 value=DynamicValue(column_name='expected_delivery_date')
490 )
491 ],
492 name='May 2022 Fulfilled Orders Not Delivered on Time',
493 order_by=<OrderBy.asc: 'asc'>,
494 table_name=<Table.orders: 'orders'>
495 )
496 ),
497 id='call_NKpApJybW1MzOjZO2FzwYw0d',
498 type='function'
499 )
500 ]
501 )
502)
503"""
504 )
505
506
507@pytest.mark.respx(base_url=base_url)
508def test_parse_max_tokens_reached(client: OpenAI, respx_mock: MockRouter) -> None:
509 class Location(BaseModel):
510 city: str
511 temperature: float
512 units: Literal["c", "f"]
513
514 with pytest.raises(openai.LengthFinishReasonError):
515 _make_snapshot_request(
516 lambda c: c.beta.chat.completions.parse(
517 model="gpt-4o-2024-08-06",
518 messages=[
519 {
520 "role": "user",
521 "content": "What's the weather like in SF?",
522 },
523 ],
524 max_tokens=1,
525 response_format=Location,
526 ),
527 content_snapshot=snapshot(
528 '{"id": "chatcmpl-ABfvvX7eB1KsfeZj8VcF3z7G7SbaA", "object": "chat.completion", "created": 1727346163, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"", "refusal": null}, "logprobs": null, "finish_reason": "length"}], "usage": {"prompt_tokens": 79, "completion_tokens": 1, "total_tokens": 80, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_7568d46099"}'
529 ),
530 mock_client=client,
531 respx_mock=respx_mock,
532 )
533
534
535@pytest.mark.respx(base_url=base_url)
536def test_parse_pydantic_model_refusal(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
537 class Location(BaseModel):
538 city: str
539 temperature: float
540 units: Literal["c", "f"]
541
542 completion = _make_snapshot_request(
543 lambda c: c.beta.chat.completions.parse(
544 model="gpt-4o-2024-08-06",
545 messages=[
546 {
547 "role": "user",
548 "content": "How do I make anthrax?",
549 },
550 ],
551 response_format=Location,
552 ),
553 content_snapshot=snapshot(
554 '{"id": "chatcmpl-ABfvwoKVWPQj2UPlAcAKM7s40GsRx", "object": "chat.completion", "created": 1727346164, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "refusal": "I\'m very sorry, but I can\'t assist with that."}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 79, "completion_tokens": 12, "total_tokens": 91, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
555 ),
556 mock_client=client,
557 respx_mock=respx_mock,
558 )
559
560 assert print_obj(completion.choices, monkeypatch) == snapshot(
561 """\
562[
563 ParsedChoice[Location](
564 finish_reason='stop',
565 index=0,
566 logprobs=None,
567 message=ParsedChatCompletionMessage[Location](
568 audio=None,
569 content=None,
570 function_call=None,
571 parsed=None,
572 refusal="I'm very sorry, but I can't assist with that.",
573 role='assistant',
574 tool_calls=[]
575 )
576 )
577]
578"""
579 )
580
581
582@pytest.mark.respx(base_url=base_url)
583def test_parse_pydantic_tool(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
584 class GetWeatherArgs(BaseModel):
585 city: str
586 country: str
587 units: Literal["c", "f"] = "c"
588
589 completion = _make_snapshot_request(
590 lambda c: c.beta.chat.completions.parse(
591 model="gpt-4o-2024-08-06",
592 messages=[
593 {
594 "role": "user",
595 "content": "What's the weather like in Edinburgh?",
596 },
597 ],
598 tools=[
599 openai.pydantic_function_tool(GetWeatherArgs),
600 ],
601 ),
602 content_snapshot=snapshot(
603 '{"id": "chatcmpl-ABfvx6Z4dchiW2nya1N8KMsHFrQRE", "object": "chat.completion", "created": 1727346165, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_Y6qJ7ofLgOrBnMD5WbVAeiRV", "type": "function", "function": {"name": "GetWeatherArgs", "arguments": "{\\"city\\":\\"Edinburgh\\",\\"country\\":\\"UK\\",\\"units\\":\\"c\\"}"}}], "refusal": null}, "logprobs": null, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 76, "completion_tokens": 24, "total_tokens": 100, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_e45dabd248"}'
604 ),
605 mock_client=client,
606 respx_mock=respx_mock,
607 )
608
609 assert print_obj(completion.choices, monkeypatch) == snapshot(
610 """\
611[
612 ParsedChoice[NoneType](
613 finish_reason='tool_calls',
614 index=0,
615 logprobs=None,
616 message=ParsedChatCompletionMessage[NoneType](
617 audio=None,
618 content=None,
619 function_call=None,
620 parsed=None,
621 refusal=None,
622 role='assistant',
623 tool_calls=[
624 ParsedFunctionToolCall(
625 function=ParsedFunction(
626 arguments='{"city":"Edinburgh","country":"UK","units":"c"}',
627 name='GetWeatherArgs',
628 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='UK', units='c')
629 ),
630 id='call_Y6qJ7ofLgOrBnMD5WbVAeiRV',
631 type='function'
632 )
633 ]
634 )
635 )
636]
637"""
638 )
639
640
641@pytest.mark.respx(base_url=base_url)
642def test_parse_multiple_pydantic_tools(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
643 class GetWeatherArgs(BaseModel):
644 """Get the temperature for the given country/city combo"""
645
646 city: str
647 country: str
648 units: Literal["c", "f"] = "c"
649
650 class GetStockPrice(BaseModel):
651 ticker: str
652 exchange: str
653
654 completion = _make_snapshot_request(
655 lambda c: c.beta.chat.completions.parse(
656 model="gpt-4o-2024-08-06",
657 messages=[
658 {
659 "role": "user",
660 "content": "What's the weather like in Edinburgh?",
661 },
662 {
663 "role": "user",
664 "content": "What's the price of AAPL?",
665 },
666 ],
667 tools=[
668 openai.pydantic_function_tool(GetWeatherArgs),
669 openai.pydantic_function_tool(
670 GetStockPrice, name="get_stock_price", description="Fetch the latest price for a given ticker"
671 ),
672 ],
673 ),
674 content_snapshot=snapshot(
675 '{"id": "chatcmpl-ABfvyvfNWKcl7Ohqos4UFrmMs1v4C", "object": "chat.completion", "created": 1727346166, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_fdNz3vOBKYgOIpMdWotB9MjY", "type": "function", "function": {"name": "GetWeatherArgs", "arguments": "{\\"city\\": \\"Edinburgh\\", \\"country\\": \\"GB\\", \\"units\\": \\"c\\"}"}}, {"id": "call_h1DWI1POMJLb0KwIyQHWXD4p", "type": "function", "function": {"name": "get_stock_price", "arguments": "{\\"ticker\\": \\"AAPL\\", \\"exchange\\": \\"NASDAQ\\"}"}}], "refusal": null}, "logprobs": null, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 149, "completion_tokens": 60, "total_tokens": 209, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_b40fb1c6fb"}'
676 ),
677 mock_client=client,
678 respx_mock=respx_mock,
679 )
680
681 assert print_obj(completion.choices, monkeypatch) == snapshot(
682 """\
683[
684 ParsedChoice[NoneType](
685 finish_reason='tool_calls',
686 index=0,
687 logprobs=None,
688 message=ParsedChatCompletionMessage[NoneType](
689 audio=None,
690 content=None,
691 function_call=None,
692 parsed=None,
693 refusal=None,
694 role='assistant',
695 tool_calls=[
696 ParsedFunctionToolCall(
697 function=ParsedFunction(
698 arguments='{"city": "Edinburgh", "country": "GB", "units": "c"}',
699 name='GetWeatherArgs',
700 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='GB', units='c')
701 ),
702 id='call_fdNz3vOBKYgOIpMdWotB9MjY',
703 type='function'
704 ),
705 ParsedFunctionToolCall(
706 function=ParsedFunction(
707 arguments='{"ticker": "AAPL", "exchange": "NASDAQ"}',
708 name='get_stock_price',
709 parsed_arguments=GetStockPrice(exchange='NASDAQ', ticker='AAPL')
710 ),
711 id='call_h1DWI1POMJLb0KwIyQHWXD4p',
712 type='function'
713 )
714 ]
715 )
716 )
717]
718"""
719 )
720
721
722@pytest.mark.respx(base_url=base_url)
723def test_parse_strict_tools(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
724 completion = _make_snapshot_request(
725 lambda c: c.beta.chat.completions.parse(
726 model="gpt-4o-2024-08-06",
727 messages=[
728 {
729 "role": "user",
730 "content": "What's the weather like in SF?",
731 },
732 ],
733 tools=[
734 {
735 "type": "function",
736 "function": {
737 "name": "get_weather",
738 "parameters": {
739 "type": "object",
740 "properties": {
741 "city": {"type": "string"},
742 "state": {"type": "string"},
743 },
744 "required": [
745 "city",
746 "state",
747 ],
748 "additionalProperties": False,
749 },
750 "strict": True,
751 },
752 }
753 ],
754 ),
755 content_snapshot=snapshot(
756 '{"id": "chatcmpl-ABfvzdvCI6RaIkiEFNjqGXCSYnlzf", "object": "chat.completion", "created": 1727346167, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_CUdUoJpsWWVdxXntucvnol1M", "type": "function", "function": {"name": "get_weather", "arguments": "{\\"city\\":\\"San Francisco\\",\\"state\\":\\"CA\\"}"}}], "refusal": null}, "logprobs": null, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 48, "completion_tokens": 19, "total_tokens": 67, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
757 ),
758 mock_client=client,
759 respx_mock=respx_mock,
760 )
761
762 assert print_obj(completion.choices, monkeypatch) == snapshot(
763 """\
764[
765 ParsedChoice[NoneType](
766 finish_reason='tool_calls',
767 index=0,
768 logprobs=None,
769 message=ParsedChatCompletionMessage[NoneType](
770 audio=None,
771 content=None,
772 function_call=None,
773 parsed=None,
774 refusal=None,
775 role='assistant',
776 tool_calls=[
777 ParsedFunctionToolCall(
778 function=ParsedFunction(
779 arguments='{"city":"San Francisco","state":"CA"}',
780 name='get_weather',
781 parsed_arguments={'city': 'San Francisco', 'state': 'CA'}
782 ),
783 id='call_CUdUoJpsWWVdxXntucvnol1M',
784 type='function'
785 )
786 ]
787 )
788 )
789]
790"""
791 )
792
793
794def test_parse_non_strict_tools(client: OpenAI) -> None:
795 with pytest.raises(
796 ValueError, match="`get_weather` is not strict. Only `strict` function tools can be auto-parsed"
797 ):
798 client.beta.chat.completions.parse(
799 model="gpt-4o-2024-08-06",
800 messages=[],
801 tools=[
802 {
803 "type": "function",
804 "function": {
805 "name": "get_weather",
806 "parameters": {},
807 },
808 }
809 ],
810 )
811
812
813@pytest.mark.respx(base_url=base_url)
814def test_parse_pydantic_raw_response(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
815 class Location(BaseModel):
816 city: str
817 temperature: float
818 units: Literal["c", "f"]
819
820 response = _make_snapshot_request(
821 lambda c: c.beta.chat.completions.with_raw_response.parse(
822 model="gpt-4o-2024-08-06",
823 messages=[
824 {
825 "role": "user",
826 "content": "What's the weather like in SF?",
827 },
828 ],
829 response_format=Location,
830 ),
831 content_snapshot=snapshot(
832 '{"id": "chatcmpl-ABrDYCa8W1w66eUxKDO8TQF1m6trT", "object": "chat.completion", "created": 1727389540, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":58,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 79, "completion_tokens": 14, "total_tokens": 93, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
833 ),
834 mock_client=client,
835 respx_mock=respx_mock,
836 )
837 assert response.http_request.headers.get("x-stainless-helper-method") == "beta.chat.completions.parse"
838
839 completion = response.parse()
840 message = completion.choices[0].message
841 assert message.parsed is not None
842 assert isinstance(message.parsed.city, str)
843 assert print_obj(completion, monkeypatch) == snapshot(
844 """\
845ParsedChatCompletion[Location](
846 choices=[
847 ParsedChoice[Location](
848 finish_reason='stop',
849 index=0,
850 logprobs=None,
851 message=ParsedChatCompletionMessage[Location](
852 audio=None,
853 content='{"city":"San Francisco","temperature":58,"units":"f"}',
854 function_call=None,
855 parsed=Location(city='San Francisco', temperature=58.0, units='f'),
856 refusal=None,
857 role='assistant',
858 tool_calls=[]
859 )
860 )
861 ],
862 created=1727389540,
863 id='chatcmpl-ABrDYCa8W1w66eUxKDO8TQF1m6trT',
864 model='gpt-4o-2024-08-06',
865 object='chat.completion',
866 service_tier=None,
867 system_fingerprint='fp_5050236cbd',
868 usage=CompletionUsage(
869 completion_tokens=14,
870 completion_tokens_details=CompletionTokensDetails(
871 accepted_prediction_tokens=None,
872 audio_tokens=None,
873 reasoning_tokens=0,
874 rejected_prediction_tokens=None
875 ),
876 prompt_tokens=79,
877 prompt_tokens_details=None,
878 total_tokens=93
879 )
880)
881"""
882 )
883
884
885@pytest.mark.respx(base_url=base_url)
886@pytest.mark.asyncio
887async def test_async_parse_pydantic_raw_response(
888 async_client: AsyncOpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch
889) -> None:
890 class Location(BaseModel):
891 city: str
892 temperature: float
893 units: Literal["c", "f"]
894
895 response = await _make_async_snapshot_request(
896 lambda c: c.beta.chat.completions.with_raw_response.parse(
897 model="gpt-4o-2024-08-06",
898 messages=[
899 {
900 "role": "user",
901 "content": "What's the weather like in SF?",
902 },
903 ],
904 response_format=Location,
905 ),
906 content_snapshot=snapshot(
907 '{"id": "chatcmpl-ABrDQWOiw0PK5JOsxl1D9ooeQgznq", "object": "chat.completion", "created": 1727389532, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":65,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 79, "completion_tokens": 14, "total_tokens": 93, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
908 ),
909 mock_client=async_client,
910 respx_mock=respx_mock,
911 )
912 assert response.http_request.headers.get("x-stainless-helper-method") == "beta.chat.completions.parse"
913
914 completion = response.parse()
915 message = completion.choices[0].message
916 assert message.parsed is not None
917 assert isinstance(message.parsed.city, str)
918 assert print_obj(completion, monkeypatch) == snapshot(
919 """\
920ParsedChatCompletion[Location](
921 choices=[
922 ParsedChoice[Location](
923 finish_reason='stop',
924 index=0,
925 logprobs=None,
926 message=ParsedChatCompletionMessage[Location](
927 audio=None,
928 content='{"city":"San Francisco","temperature":65,"units":"f"}',
929 function_call=None,
930 parsed=Location(city='San Francisco', temperature=65.0, units='f'),
931 refusal=None,
932 role='assistant',
933 tool_calls=[]
934 )
935 )
936 ],
937 created=1727389532,
938 id='chatcmpl-ABrDQWOiw0PK5JOsxl1D9ooeQgznq',
939 model='gpt-4o-2024-08-06',
940 object='chat.completion',
941 service_tier=None,
942 system_fingerprint='fp_5050236cbd',
943 usage=CompletionUsage(
944 completion_tokens=14,
945 completion_tokens_details=CompletionTokensDetails(
946 accepted_prediction_tokens=None,
947 audio_tokens=None,
948 reasoning_tokens=0,
949 rejected_prediction_tokens=None
950 ),
951 prompt_tokens=79,
952 prompt_tokens_details=None,
953 total_tokens=93
954 )
955)
956"""
957 )
958
959
960@pytest.mark.parametrize("sync", [True, False], ids=["sync", "async"])
961def test_parse_method_in_sync(sync: bool, client: OpenAI, async_client: AsyncOpenAI) -> None:
962 checking_client: OpenAI | AsyncOpenAI = client if sync else async_client
963
964 assert_signatures_in_sync(
965 checking_client.chat.completions.create,
966 checking_client.beta.chat.completions.parse,
967 exclude_params={"response_format", "stream"},
968 )
969
970
971def _make_snapshot_request(
972 func: Callable[[OpenAI], _T],
973 *,
974 content_snapshot: Any,
975 respx_mock: MockRouter,
976 mock_client: OpenAI,
977) -> _T:
978 live = os.environ.get("OPENAI_LIVE") == "1"
979 if live:
980
981 def _on_response(response: httpx.Response) -> None:
982 # update the content snapshot
983 assert json.dumps(json.loads(response.read())) == content_snapshot
984
985 respx_mock.stop()
986
987 client = OpenAI(
988 http_client=httpx.Client(
989 event_hooks={
990 "response": [_on_response],
991 }
992 )
993 )
994 else:
995 respx_mock.post("/chat/completions").mock(
996 return_value=httpx.Response(
997 200,
998 content=content_snapshot._old_value,
999 headers={"content-type": "application/json"},
1000 )
1001 )
1002
1003 client = mock_client
1004
1005 result = func(client)
1006
1007 if live:
1008 client.close()
1009
1010 return result
1011
1012
1013async def _make_async_snapshot_request(
1014 func: Callable[[AsyncOpenAI], Awaitable[_T]],
1015 *,
1016 content_snapshot: Any,
1017 respx_mock: MockRouter,
1018 mock_client: AsyncOpenAI,
1019) -> _T:
1020 live = os.environ.get("OPENAI_LIVE") == "1"
1021 if live:
1022
1023 async def _on_response(response: httpx.Response) -> None:
1024 # update the content snapshot
1025 assert json.dumps(json.loads(await response.aread())) == content_snapshot
1026
1027 respx_mock.stop()
1028
1029 client = AsyncOpenAI(
1030 http_client=httpx.AsyncClient(
1031 event_hooks={
1032 "response": [_on_response],
1033 }
1034 )
1035 )
1036 else:
1037 respx_mock.post("/chat/completions").mock(
1038 return_value=httpx.Response(
1039 200,
1040 content=content_snapshot._old_value,
1041 headers={"content-type": "application/json"},
1042 )
1043 )
1044
1045 client = mock_client
1046
1047 result = await func(client)
1048
1049 if live:
1050 await client.close()
1051
1052 return result
1053