GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

openai/openai-python

Public

mirrored fromhttps://github.com/openai/openai-pythonAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
v1.49.0

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openai-python/tests/lib/chat

tests/lib/chat/test_completions.py

826lines · modecode

RawDownload
Latest commit unavailable.
1from __future__ import annotations
2
3import os
4import json
5from enum import Enum
6from typing import Any, List, Callable, Optional
7from typing_extensions import Literal, TypeVar
8
9import httpx
10import pytest
11from respx import MockRouter
12from pydantic import Field, BaseModel
13from inline_snapshot import snapshot
14
15import openai
16from openai import OpenAI, AsyncOpenAI
17from openai._utils import assert_signatures_in_sync
18from openai._compat import PYDANTIC_V2
19
20from ._utils import print_obj
21from ...conftest import base_url
22from ..schema_types.query import Query
23
24_T = TypeVar("_T")
25
26# all the snapshots in this file are auto-generated from the live API
27#
28# you can update them with
29#
30# `OPENAI_LIVE=1 pytest --inline-snapshot=fix`
31
32
33@pytest.mark.respx(base_url=base_url)
34def test_parse_nothing(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
35 completion = _make_snapshot_request(
36 lambda c: c.beta.chat.completions.parse(
37 model="gpt-4o-2024-08-06",
38 messages=[
39 {
40 "role": "user",
41 "content": "What's the weather like in SF?",
42 },
43 ],
44 ),
45 content_snapshot=snapshot(
46 '{"id": "chatcmpl-ABfvaueLEMLNYbT8YzpJxsmiQ6HSY", "object": "chat.completion", "created": 1727346142, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "I\'m unable to provide real-time weather updates. To get the current weather in San Francisco, I recommend checking a reliable weather website or app like the Weather Channel or a local news station.", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 14, "completion_tokens": 37, "total_tokens": 51, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_b40fb1c6fb"}'
47 ),
48 mock_client=client,
49 respx_mock=respx_mock,
50 )
51
52 assert print_obj(completion, monkeypatch) == snapshot(
53 """\
54ParsedChatCompletion[NoneType](
55 choices=[
56 ParsedChoice[NoneType](
57 finish_reason='stop',
58 index=0,
59 logprobs=None,
60 message=ParsedChatCompletionMessage[NoneType](
61 content="I'm unable to provide real-time weather updates. To get the current weather in San Francisco, I
62recommend checking a reliable weather website or app like the Weather Channel or a local news station.",
63 function_call=None,
64 parsed=None,
65 refusal=None,
66 role='assistant',
67 tool_calls=[]
68 )
69 )
70 ],
71 created=1727346142,
72 id='chatcmpl-ABfvaueLEMLNYbT8YzpJxsmiQ6HSY',
73 model='gpt-4o-2024-08-06',
74 object='chat.completion',
75 service_tier=None,
76 system_fingerprint='fp_b40fb1c6fb',
77 usage=CompletionUsage(
78 completion_tokens=37,
79 completion_tokens_details=CompletionTokensDetails(reasoning_tokens=0),
80 prompt_tokens=14,
81 total_tokens=51
82 )
83)
84"""
85 )
86
87
88@pytest.mark.respx(base_url=base_url)
89def test_parse_pydantic_model(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
90 class Location(BaseModel):
91 city: str
92 temperature: float
93 units: Literal["c", "f"]
94
95 completion = _make_snapshot_request(
96 lambda c: c.beta.chat.completions.parse(
97 model="gpt-4o-2024-08-06",
98 messages=[
99 {
100 "role": "user",
101 "content": "What's the weather like in SF?",
102 },
103 ],
104 response_format=Location,
105 ),
106 content_snapshot=snapshot(
107 '{"id": "chatcmpl-ABfvbtVnTu5DeC4EFnRYj8mtfOM99", "object": "chat.completion", "created": 1727346143, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":65,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 79, "completion_tokens": 14, "total_tokens": 93, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
108 ),
109 mock_client=client,
110 respx_mock=respx_mock,
111 )
112
113 assert print_obj(completion, monkeypatch) == snapshot(
114 """\
115ParsedChatCompletion[Location](
116 choices=[
117 ParsedChoice[Location](
118 finish_reason='stop',
119 index=0,
120 logprobs=None,
121 message=ParsedChatCompletionMessage[Location](
122 content='{"city":"San Francisco","temperature":65,"units":"f"}',
123 function_call=None,
124 parsed=Location(city='San Francisco', temperature=65.0, units='f'),
125 refusal=None,
126 role='assistant',
127 tool_calls=[]
128 )
129 )
130 ],
131 created=1727346143,
132 id='chatcmpl-ABfvbtVnTu5DeC4EFnRYj8mtfOM99',
133 model='gpt-4o-2024-08-06',
134 object='chat.completion',
135 service_tier=None,
136 system_fingerprint='fp_5050236cbd',
137 usage=CompletionUsage(
138 completion_tokens=14,
139 completion_tokens_details=CompletionTokensDetails(reasoning_tokens=0),
140 prompt_tokens=79,
141 total_tokens=93
142 )
143)
144"""
145 )
146
147
148@pytest.mark.respx(base_url=base_url)
149def test_parse_pydantic_model_optional_default(
150 client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch
151) -> None:
152 class Location(BaseModel):
153 city: str
154 temperature: float
155 units: Optional[Literal["c", "f"]] = None
156
157 completion = _make_snapshot_request(
158 lambda c: c.beta.chat.completions.parse(
159 model="gpt-4o-2024-08-06",
160 messages=[
161 {
162 "role": "user",
163 "content": "What's the weather like in SF?",
164 },
165 ],
166 response_format=Location,
167 ),
168 content_snapshot=snapshot(
169 '{"id": "chatcmpl-ABfvcC8grKYsRkSoMp9CCAhbXAd0b", "object": "chat.completion", "created": 1727346144, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":65,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 88, "completion_tokens": 14, "total_tokens": 102, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_b40fb1c6fb"}'
170 ),
171 mock_client=client,
172 respx_mock=respx_mock,
173 )
174
175 assert print_obj(completion, monkeypatch) == snapshot(
176 """\
177ParsedChatCompletion[Location](
178 choices=[
179 ParsedChoice[Location](
180 finish_reason='stop',
181 index=0,
182 logprobs=None,
183 message=ParsedChatCompletionMessage[Location](
184 content='{"city":"San Francisco","temperature":65,"units":"f"}',
185 function_call=None,
186 parsed=Location(city='San Francisco', temperature=65.0, units='f'),
187 refusal=None,
188 role='assistant',
189 tool_calls=[]
190 )
191 )
192 ],
193 created=1727346144,
194 id='chatcmpl-ABfvcC8grKYsRkSoMp9CCAhbXAd0b',
195 model='gpt-4o-2024-08-06',
196 object='chat.completion',
197 service_tier=None,
198 system_fingerprint='fp_b40fb1c6fb',
199 usage=CompletionUsage(
200 completion_tokens=14,
201 completion_tokens_details=CompletionTokensDetails(reasoning_tokens=0),
202 prompt_tokens=88,
203 total_tokens=102
204 )
205)
206"""
207 )
208
209
210@pytest.mark.respx(base_url=base_url)
211def test_parse_pydantic_model_enum(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
212 class Color(Enum):
213 """The detected color"""
214
215 RED = "red"
216 BLUE = "blue"
217 GREEN = "green"
218
219 class ColorDetection(BaseModel):
220 color: Color
221 hex_color_code: str = Field(description="The hex color code of the detected color")
222
223 if not PYDANTIC_V2:
224 ColorDetection.update_forward_refs(**locals()) # type: ignore
225
226 completion = _make_snapshot_request(
227 lambda c: c.beta.chat.completions.parse(
228 model="gpt-4o-2024-08-06",
229 messages=[
230 {"role": "user", "content": "What color is a Coke can?"},
231 ],
232 response_format=ColorDetection,
233 ),
234 content_snapshot=snapshot(
235 '{"id": "chatcmpl-ABfvjIatz0zrZu50gRbMtlp0asZpz", "object": "chat.completion", "created": 1727346151, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"color\\":\\"red\\",\\"hex_color_code\\":\\"#FF0000\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 109, "completion_tokens": 14, "total_tokens": 123, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
236 ),
237 mock_client=client,
238 respx_mock=respx_mock,
239 )
240
241 assert print_obj(completion.choices[0], monkeypatch) == snapshot(
242 """\
243ParsedChoice[ColorDetection](
244 finish_reason='stop',
245 index=0,
246 logprobs=None,
247 message=ParsedChatCompletionMessage[ColorDetection](
248 content='{"color":"red","hex_color_code":"#FF0000"}',
249 function_call=None,
250 parsed=ColorDetection(color=<Color.RED: 'red'>, hex_color_code='#FF0000'),
251 refusal=None,
252 role='assistant',
253 tool_calls=[]
254 )
255)
256"""
257 )
258
259
260@pytest.mark.respx(base_url=base_url)
261def test_parse_pydantic_model_multiple_choices(
262 client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch
263) -> None:
264 class Location(BaseModel):
265 city: str
266 temperature: float
267 units: Literal["c", "f"]
268
269 completion = _make_snapshot_request(
270 lambda c: c.beta.chat.completions.parse(
271 model="gpt-4o-2024-08-06",
272 messages=[
273 {
274 "role": "user",
275 "content": "What's the weather like in SF?",
276 },
277 ],
278 n=3,
279 response_format=Location,
280 ),
281 content_snapshot=snapshot(
282 '{"id": "chatcmpl-ABfvp8qzboW92q8ONDF4DPHlI7ckC", "object": "chat.completion", "created": 1727346157, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":64,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}, {"index": 1, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":65,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}, {"index": 2, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":63.0,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 79, "completion_tokens": 44, "total_tokens": 123, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_b40fb1c6fb"}'
283 ),
284 mock_client=client,
285 respx_mock=respx_mock,
286 )
287
288 assert print_obj(completion.choices, monkeypatch) == snapshot(
289 """\
290[
291 ParsedChoice[Location](
292 finish_reason='stop',
293 index=0,
294 logprobs=None,
295 message=ParsedChatCompletionMessage[Location](
296 content='{"city":"San Francisco","temperature":64,"units":"f"}',
297 function_call=None,
298 parsed=Location(city='San Francisco', temperature=64.0, units='f'),
299 refusal=None,
300 role='assistant',
301 tool_calls=[]
302 )
303 ),
304 ParsedChoice[Location](
305 finish_reason='stop',
306 index=1,
307 logprobs=None,
308 message=ParsedChatCompletionMessage[Location](
309 content='{"city":"San Francisco","temperature":65,"units":"f"}',
310 function_call=None,
311 parsed=Location(city='San Francisco', temperature=65.0, units='f'),
312 refusal=None,
313 role='assistant',
314 tool_calls=[]
315 )
316 ),
317 ParsedChoice[Location](
318 finish_reason='stop',
319 index=2,
320 logprobs=None,
321 message=ParsedChatCompletionMessage[Location](
322 content='{"city":"San Francisco","temperature":63.0,"units":"f"}',
323 function_call=None,
324 parsed=Location(city='San Francisco', temperature=63.0, units='f'),
325 refusal=None,
326 role='assistant',
327 tool_calls=[]
328 )
329 )
330]
331"""
332 )
333
334
335@pytest.mark.respx(base_url=base_url)
336@pytest.mark.skipif(not PYDANTIC_V2, reason="dataclasses only supported in v2")
337def test_parse_pydantic_dataclass(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
338 from pydantic.dataclasses import dataclass
339
340 @dataclass
341 class CalendarEvent:
342 name: str
343 date: str
344 participants: List[str]
345
346 completion = _make_snapshot_request(
347 lambda c: c.beta.chat.completions.parse(
348 model="gpt-4o-2024-08-06",
349 messages=[
350 {"role": "system", "content": "Extract the event information."},
351 {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."},
352 ],
353 response_format=CalendarEvent,
354 ),
355 content_snapshot=snapshot(
356 '{"id": "chatcmpl-ABfvqhz4uUUWsw8Ohw2Mp9B4sKKV8", "object": "chat.completion", "created": 1727346158, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"name\\":\\"Science Fair\\",\\"date\\":\\"Friday\\",\\"participants\\":[\\"Alice\\",\\"Bob\\"]}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 92, "completion_tokens": 17, "total_tokens": 109, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_7568d46099"}'
357 ),
358 mock_client=client,
359 respx_mock=respx_mock,
360 )
361
362 assert print_obj(completion, monkeypatch) == snapshot(
363 """\
364ParsedChatCompletion[CalendarEvent](
365 choices=[
366 ParsedChoice[CalendarEvent](
367 finish_reason='stop',
368 index=0,
369 logprobs=None,
370 message=ParsedChatCompletionMessage[CalendarEvent](
371 content='{"name":"Science Fair","date":"Friday","participants":["Alice","Bob"]}',
372 function_call=None,
373 parsed=CalendarEvent(name='Science Fair', date='Friday', participants=['Alice', 'Bob']),
374 refusal=None,
375 role='assistant',
376 tool_calls=[]
377 )
378 )
379 ],
380 created=1727346158,
381 id='chatcmpl-ABfvqhz4uUUWsw8Ohw2Mp9B4sKKV8',
382 model='gpt-4o-2024-08-06',
383 object='chat.completion',
384 service_tier=None,
385 system_fingerprint='fp_7568d46099',
386 usage=CompletionUsage(
387 completion_tokens=17,
388 completion_tokens_details=CompletionTokensDetails(reasoning_tokens=0),
389 prompt_tokens=92,
390 total_tokens=109
391 )
392)
393"""
394 )
395
396
397@pytest.mark.respx(base_url=base_url)
398def test_pydantic_tool_model_all_types(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
399 completion = _make_snapshot_request(
400 lambda c: c.beta.chat.completions.parse(
401 model="gpt-4o-2024-08-06",
402 messages=[
403 {
404 "role": "user",
405 "content": "look up all my orders in may of last year that were fulfilled but not delivered on time",
406 },
407 ],
408 tools=[openai.pydantic_function_tool(Query)],
409 response_format=Query,
410 ),
411 content_snapshot=snapshot(
412 '{"id": "chatcmpl-ABfvtNiaTNUF6OymZUnEFc9lPq9p1", "object": "chat.completion", "created": 1727346161, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_NKpApJybW1MzOjZO2FzwYw0d", "type": "function", "function": {"name": "Query", "arguments": "{\\"name\\":\\"May 2022 Fulfilled Orders Not Delivered on Time\\",\\"table_name\\":\\"orders\\",\\"columns\\":[\\"id\\",\\"status\\",\\"expected_delivery_date\\",\\"delivered_at\\",\\"shipped_at\\",\\"ordered_at\\",\\"canceled_at\\"],\\"conditions\\":[{\\"column\\":\\"ordered_at\\",\\"operator\\":\\">=\\",\\"value\\":\\"2022-05-01\\"},{\\"column\\":\\"ordered_at\\",\\"operator\\":\\"<=\\",\\"value\\":\\"2022-05-31\\"},{\\"column\\":\\"status\\",\\"operator\\":\\"=\\",\\"value\\":\\"fulfilled\\"},{\\"column\\":\\"delivered_at\\",\\"operator\\":\\">\\",\\"value\\":{\\"column_name\\":\\"expected_delivery_date\\"}}],\\"order_by\\":\\"asc\\"}"}}], "refusal": null}, "logprobs": null, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 512, "completion_tokens": 132, "total_tokens": 644, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_7568d46099"}'
413 ),
414 mock_client=client,
415 respx_mock=respx_mock,
416 )
417
418 assert print_obj(completion.choices[0], monkeypatch) == snapshot(
419 """\
420ParsedChoice[Query](
421 finish_reason='tool_calls',
422 index=0,
423 logprobs=None,
424 message=ParsedChatCompletionMessage[Query](
425 content=None,
426 function_call=None,
427 parsed=None,
428 refusal=None,
429 role='assistant',
430 tool_calls=[
431 ParsedFunctionToolCall(
432 function=ParsedFunction(
433 arguments='{"name":"May 2022 Fulfilled Orders Not Delivered on
434Time","table_name":"orders","columns":["id","status","expected_delivery_date","delivered_at","shipped_at","ordered_at","
435canceled_at"],"conditions":[{"column":"ordered_at","operator":">=","value":"2022-05-01"},{"column":"ordered_at","operato
436r":"<=","value":"2022-05-31"},{"column":"status","operator":"=","value":"fulfilled"},{"column":"delivered_at","operator"
437:">","value":{"column_name":"expected_delivery_date"}}],"order_by":"asc"}',
438 name='Query',
439 parsed_arguments=Query(
440 columns=[
441 <Column.id: 'id'>,
442 <Column.status: 'status'>,
443 <Column.expected_delivery_date: 'expected_delivery_date'>,
444 <Column.delivered_at: 'delivered_at'>,
445 <Column.shipped_at: 'shipped_at'>,
446 <Column.ordered_at: 'ordered_at'>,
447 <Column.canceled_at: 'canceled_at'>
448 ],
449 conditions=[
450 Condition(column='ordered_at', operator=<Operator.ge: '>='>, value='2022-05-01'),
451 Condition(column='ordered_at', operator=<Operator.le: '<='>, value='2022-05-31'),
452 Condition(column='status', operator=<Operator.eq: '='>, value='fulfilled'),
453 Condition(
454 column='delivered_at',
455 operator=<Operator.gt: '>'>,
456 value=DynamicValue(column_name='expected_delivery_date')
457 )
458 ],
459 name='May 2022 Fulfilled Orders Not Delivered on Time',
460 order_by=<OrderBy.asc: 'asc'>,
461 table_name=<Table.orders: 'orders'>
462 )
463 ),
464 id='call_NKpApJybW1MzOjZO2FzwYw0d',
465 type='function'
466 )
467 ]
468 )
469)
470"""
471 )
472
473
474@pytest.mark.respx(base_url=base_url)
475def test_parse_max_tokens_reached(client: OpenAI, respx_mock: MockRouter) -> None:
476 class Location(BaseModel):
477 city: str
478 temperature: float
479 units: Literal["c", "f"]
480
481 with pytest.raises(openai.LengthFinishReasonError):
482 _make_snapshot_request(
483 lambda c: c.beta.chat.completions.parse(
484 model="gpt-4o-2024-08-06",
485 messages=[
486 {
487 "role": "user",
488 "content": "What's the weather like in SF?",
489 },
490 ],
491 max_tokens=1,
492 response_format=Location,
493 ),
494 content_snapshot=snapshot(
495 '{"id": "chatcmpl-ABfvvX7eB1KsfeZj8VcF3z7G7SbaA", "object": "chat.completion", "created": 1727346163, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"", "refusal": null}, "logprobs": null, "finish_reason": "length"}], "usage": {"prompt_tokens": 79, "completion_tokens": 1, "total_tokens": 80, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_7568d46099"}'
496 ),
497 mock_client=client,
498 respx_mock=respx_mock,
499 )
500
501
502@pytest.mark.respx(base_url=base_url)
503def test_parse_pydantic_model_refusal(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
504 class Location(BaseModel):
505 city: str
506 temperature: float
507 units: Literal["c", "f"]
508
509 completion = _make_snapshot_request(
510 lambda c: c.beta.chat.completions.parse(
511 model="gpt-4o-2024-08-06",
512 messages=[
513 {
514 "role": "user",
515 "content": "How do I make anthrax?",
516 },
517 ],
518 response_format=Location,
519 ),
520 content_snapshot=snapshot(
521 '{"id": "chatcmpl-ABfvwoKVWPQj2UPlAcAKM7s40GsRx", "object": "chat.completion", "created": 1727346164, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "refusal": "I\'m very sorry, but I can\'t assist with that."}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 79, "completion_tokens": 12, "total_tokens": 91, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
522 ),
523 mock_client=client,
524 respx_mock=respx_mock,
525 )
526
527 assert print_obj(completion.choices, monkeypatch) == snapshot(
528 """\
529[
530 ParsedChoice[Location](
531 finish_reason='stop',
532 index=0,
533 logprobs=None,
534 message=ParsedChatCompletionMessage[Location](
535 content=None,
536 function_call=None,
537 parsed=None,
538 refusal="I'm very sorry, but I can't assist with that.",
539 role='assistant',
540 tool_calls=[]
541 )
542 )
543]
544"""
545 )
546
547
548@pytest.mark.respx(base_url=base_url)
549def test_parse_pydantic_tool(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
550 class GetWeatherArgs(BaseModel):
551 city: str
552 country: str
553 units: Literal["c", "f"] = "c"
554
555 completion = _make_snapshot_request(
556 lambda c: c.beta.chat.completions.parse(
557 model="gpt-4o-2024-08-06",
558 messages=[
559 {
560 "role": "user",
561 "content": "What's the weather like in Edinburgh?",
562 },
563 ],
564 tools=[
565 openai.pydantic_function_tool(GetWeatherArgs),
566 ],
567 ),
568 content_snapshot=snapshot(
569 '{"id": "chatcmpl-ABfvx6Z4dchiW2nya1N8KMsHFrQRE", "object": "chat.completion", "created": 1727346165, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_Y6qJ7ofLgOrBnMD5WbVAeiRV", "type": "function", "function": {"name": "GetWeatherArgs", "arguments": "{\\"city\\":\\"Edinburgh\\",\\"country\\":\\"UK\\",\\"units\\":\\"c\\"}"}}], "refusal": null}, "logprobs": null, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 76, "completion_tokens": 24, "total_tokens": 100, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_e45dabd248"}'
570 ),
571 mock_client=client,
572 respx_mock=respx_mock,
573 )
574
575 assert print_obj(completion.choices, monkeypatch) == snapshot(
576 """\
577[
578 ParsedChoice[NoneType](
579 finish_reason='tool_calls',
580 index=0,
581 logprobs=None,
582 message=ParsedChatCompletionMessage[NoneType](
583 content=None,
584 function_call=None,
585 parsed=None,
586 refusal=None,
587 role='assistant',
588 tool_calls=[
589 ParsedFunctionToolCall(
590 function=ParsedFunction(
591 arguments='{"city":"Edinburgh","country":"UK","units":"c"}',
592 name='GetWeatherArgs',
593 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='UK', units='c')
594 ),
595 id='call_Y6qJ7ofLgOrBnMD5WbVAeiRV',
596 type='function'
597 )
598 ]
599 )
600 )
601]
602"""
603 )
604
605
606@pytest.mark.respx(base_url=base_url)
607def test_parse_multiple_pydantic_tools(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
608 class GetWeatherArgs(BaseModel):
609 """Get the temperature for the given country/city combo"""
610
611 city: str
612 country: str
613 units: Literal["c", "f"] = "c"
614
615 class GetStockPrice(BaseModel):
616 ticker: str
617 exchange: str
618
619 completion = _make_snapshot_request(
620 lambda c: c.beta.chat.completions.parse(
621 model="gpt-4o-2024-08-06",
622 messages=[
623 {
624 "role": "user",
625 "content": "What's the weather like in Edinburgh?",
626 },
627 {
628 "role": "user",
629 "content": "What's the price of AAPL?",
630 },
631 ],
632 tools=[
633 openai.pydantic_function_tool(GetWeatherArgs),
634 openai.pydantic_function_tool(
635 GetStockPrice, name="get_stock_price", description="Fetch the latest price for a given ticker"
636 ),
637 ],
638 ),
639 content_snapshot=snapshot(
640 '{"id": "chatcmpl-ABfvyvfNWKcl7Ohqos4UFrmMs1v4C", "object": "chat.completion", "created": 1727346166, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_fdNz3vOBKYgOIpMdWotB9MjY", "type": "function", "function": {"name": "GetWeatherArgs", "arguments": "{\\"city\\": \\"Edinburgh\\", \\"country\\": \\"GB\\", \\"units\\": \\"c\\"}"}}, {"id": "call_h1DWI1POMJLb0KwIyQHWXD4p", "type": "function", "function": {"name": "get_stock_price", "arguments": "{\\"ticker\\": \\"AAPL\\", \\"exchange\\": \\"NASDAQ\\"}"}}], "refusal": null}, "logprobs": null, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 149, "completion_tokens": 60, "total_tokens": 209, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_b40fb1c6fb"}'
641 ),
642 mock_client=client,
643 respx_mock=respx_mock,
644 )
645
646 assert print_obj(completion.choices, monkeypatch) == snapshot(
647 """\
648[
649 ParsedChoice[NoneType](
650 finish_reason='tool_calls',
651 index=0,
652 logprobs=None,
653 message=ParsedChatCompletionMessage[NoneType](
654 content=None,
655 function_call=None,
656 parsed=None,
657 refusal=None,
658 role='assistant',
659 tool_calls=[
660 ParsedFunctionToolCall(
661 function=ParsedFunction(
662 arguments='{"city": "Edinburgh", "country": "GB", "units": "c"}',
663 name='GetWeatherArgs',
664 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='GB', units='c')
665 ),
666 id='call_fdNz3vOBKYgOIpMdWotB9MjY',
667 type='function'
668 ),
669 ParsedFunctionToolCall(
670 function=ParsedFunction(
671 arguments='{"ticker": "AAPL", "exchange": "NASDAQ"}',
672 name='get_stock_price',
673 parsed_arguments=GetStockPrice(exchange='NASDAQ', ticker='AAPL')
674 ),
675 id='call_h1DWI1POMJLb0KwIyQHWXD4p',
676 type='function'
677 )
678 ]
679 )
680 )
681]
682"""
683 )
684
685
686@pytest.mark.respx(base_url=base_url)
687def test_parse_strict_tools(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
688 completion = _make_snapshot_request(
689 lambda c: c.beta.chat.completions.parse(
690 model="gpt-4o-2024-08-06",
691 messages=[
692 {
693 "role": "user",
694 "content": "What's the weather like in SF?",
695 },
696 ],
697 tools=[
698 {
699 "type": "function",
700 "function": {
701 "name": "get_weather",
702 "parameters": {
703 "type": "object",
704 "properties": {
705 "city": {"type": "string"},
706 "state": {"type": "string"},
707 },
708 "required": [
709 "city",
710 "state",
711 ],
712 "additionalProperties": False,
713 },
714 "strict": True,
715 },
716 }
717 ],
718 ),
719 content_snapshot=snapshot(
720 '{"id": "chatcmpl-ABfvzdvCI6RaIkiEFNjqGXCSYnlzf", "object": "chat.completion", "created": 1727346167, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_CUdUoJpsWWVdxXntucvnol1M", "type": "function", "function": {"name": "get_weather", "arguments": "{\\"city\\":\\"San Francisco\\",\\"state\\":\\"CA\\"}"}}], "refusal": null}, "logprobs": null, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 48, "completion_tokens": 19, "total_tokens": 67, "completion_tokens_details": {"reasoning_tokens": 0}}, "system_fingerprint": "fp_5050236cbd"}'
721 ),
722 mock_client=client,
723 respx_mock=respx_mock,
724 )
725
726 assert print_obj(completion.choices, monkeypatch) == snapshot(
727 """\
728[
729 ParsedChoice[NoneType](
730 finish_reason='tool_calls',
731 index=0,
732 logprobs=None,
733 message=ParsedChatCompletionMessage[NoneType](
734 content=None,
735 function_call=None,
736 parsed=None,
737 refusal=None,
738 role='assistant',
739 tool_calls=[
740 ParsedFunctionToolCall(
741 function=ParsedFunction(
742 arguments='{"city":"San Francisco","state":"CA"}',
743 name='get_weather',
744 parsed_arguments={'city': 'San Francisco', 'state': 'CA'}
745 ),
746 id='call_CUdUoJpsWWVdxXntucvnol1M',
747 type='function'
748 )
749 ]
750 )
751 )
752]
753"""
754 )
755
756
757def test_parse_non_strict_tools(client: OpenAI) -> None:
758 with pytest.raises(
759 ValueError, match="`get_weather` is not strict. Only `strict` function tools can be auto-parsed"
760 ):
761 client.beta.chat.completions.parse(
762 model="gpt-4o-2024-08-06",
763 messages=[],
764 tools=[
765 {
766 "type": "function",
767 "function": {
768 "name": "get_weather",
769 "parameters": {},
770 },
771 }
772 ],
773 )
774
775
776@pytest.mark.parametrize("sync", [True, False], ids=["sync", "async"])
777def test_parse_method_in_sync(sync: bool, client: OpenAI, async_client: AsyncOpenAI) -> None:
778 checking_client: OpenAI | AsyncOpenAI = client if sync else async_client
779
780 assert_signatures_in_sync(
781 checking_client.chat.completions.create,
782 checking_client.beta.chat.completions.parse,
783 exclude_params={"response_format", "stream"},
784 )
785
786
787def _make_snapshot_request(
788 func: Callable[[OpenAI], _T],
789 *,
790 content_snapshot: Any,
791 respx_mock: MockRouter,
792 mock_client: OpenAI,
793) -> _T:
794 live = os.environ.get("OPENAI_LIVE") == "1"
795 if live:
796
797 def _on_response(response: httpx.Response) -> None:
798 # update the content snapshot
799 assert json.dumps(json.loads(response.read())) == content_snapshot
800
801 respx_mock.stop()
802
803 client = OpenAI(
804 http_client=httpx.Client(
805 event_hooks={
806 "response": [_on_response],
807 }
808 )
809 )
810 else:
811 respx_mock.post("/chat/completions").mock(
812 return_value=httpx.Response(
813 200,
814 content=content_snapshot._old_value,
815 headers={"content-type": "application/json"},
816 )
817 )
818
819 client = mock_client
820
821 result = func(client)
822
823 if live:
824 client.close()
825
826 return result
827