GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

openai/openai-python

Public

mirrored fromhttps://github.com/openai/openai-pythonAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
v1.45.0

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openai-python/tests/lib/chat

tests/lib/chat/test_completions.py

804lines · modecode

RawDownload
Latest commit unavailable.
1from __future__ import annotations
2
3import os
4import json
5from enum import Enum
6from typing import Any, List, Callable, Optional
7from typing_extensions import Literal, TypeVar
8
9import httpx
10import pytest
11from respx import MockRouter
12from pydantic import Field, BaseModel
13from inline_snapshot import snapshot
14
15import openai
16from openai import OpenAI, AsyncOpenAI
17from openai._utils import assert_signatures_in_sync
18from openai._compat import PYDANTIC_V2
19
20from ._utils import print_obj
21from ...conftest import base_url
22from ..schema_types.query import Query
23
24_T = TypeVar("_T")
25
26# all the snapshots in this file are auto-generated from the live API
27#
28# you can update them with
29#
30# `OPENAI_LIVE=1 pytest --inline-snapshot=fix`
31
32
33@pytest.mark.respx(base_url=base_url)
34def test_parse_nothing(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
35 completion = _make_snapshot_request(
36 lambda c: c.beta.chat.completions.parse(
37 model="gpt-4o-2024-08-06",
38 messages=[
39 {
40 "role": "user",
41 "content": "What's the weather like in SF?",
42 },
43 ],
44 ),
45 content_snapshot=snapshot(
46 '{"id": "chatcmpl-9tXjSozlYq8oGdlRH3vgLsiUNRg8c", "object": "chat.completion", "created": 1723024734, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "I\'m unable to provide real-time weather updates. To find out the current weather in San Francisco, please check a reliable weather website or app.", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 14, "completion_tokens": 28, "total_tokens": 42}, "system_fingerprint": "fp_845eaabc1f"}'
47 ),
48 mock_client=client,
49 respx_mock=respx_mock,
50 )
51
52 assert print_obj(completion, monkeypatch) == snapshot(
53 """\
54ParsedChatCompletion[NoneType](
55 choices=[
56 ParsedChoice[NoneType](
57 finish_reason='stop',
58 index=0,
59 logprobs=None,
60 message=ParsedChatCompletionMessage[NoneType](
61 content="I'm unable to provide real-time weather updates. To find out the current weather in San
62Francisco, please check a reliable weather website or app.",
63 function_call=None,
64 parsed=None,
65 refusal=None,
66 role='assistant',
67 tool_calls=[]
68 )
69 )
70 ],
71 created=1723024734,
72 id='chatcmpl-9tXjSozlYq8oGdlRH3vgLsiUNRg8c',
73 model='gpt-4o-2024-08-06',
74 object='chat.completion',
75 service_tier=None,
76 system_fingerprint='fp_845eaabc1f',
77 usage=CompletionUsage(completion_tokens=28, completion_tokens_details=None, prompt_tokens=14, total_tokens=42)
78)
79"""
80 )
81
82
83@pytest.mark.respx(base_url=base_url)
84def test_parse_pydantic_model(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
85 class Location(BaseModel):
86 city: str
87 temperature: float
88 units: Literal["c", "f"]
89
90 completion = _make_snapshot_request(
91 lambda c: c.beta.chat.completions.parse(
92 model="gpt-4o-2024-08-06",
93 messages=[
94 {
95 "role": "user",
96 "content": "What's the weather like in SF?",
97 },
98 ],
99 response_format=Location,
100 ),
101 content_snapshot=snapshot(
102 '{"id": "chatcmpl-9tXjTNupyDe7nL1Z8eOO6BdSyrHAD", "object": "chat.completion", "created": 1723024735, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":56,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 17, "completion_tokens": 14, "total_tokens": 31}, "system_fingerprint": "fp_2a322c9ffc"}'
103 ),
104 mock_client=client,
105 respx_mock=respx_mock,
106 )
107
108 assert print_obj(completion, monkeypatch) == snapshot(
109 """\
110ParsedChatCompletion[Location](
111 choices=[
112 ParsedChoice[Location](
113 finish_reason='stop',
114 index=0,
115 logprobs=None,
116 message=ParsedChatCompletionMessage[Location](
117 content='{"city":"San Francisco","temperature":56,"units":"f"}',
118 function_call=None,
119 parsed=Location(city='San Francisco', temperature=56.0, units='f'),
120 refusal=None,
121 role='assistant',
122 tool_calls=[]
123 )
124 )
125 ],
126 created=1723024735,
127 id='chatcmpl-9tXjTNupyDe7nL1Z8eOO6BdSyrHAD',
128 model='gpt-4o-2024-08-06',
129 object='chat.completion',
130 service_tier=None,
131 system_fingerprint='fp_2a322c9ffc',
132 usage=CompletionUsage(completion_tokens=14, completion_tokens_details=None, prompt_tokens=17, total_tokens=31)
133)
134"""
135 )
136
137
138@pytest.mark.respx(base_url=base_url)
139def test_parse_pydantic_model_optional_default(
140 client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch
141) -> None:
142 class Location(BaseModel):
143 city: str
144 temperature: float
145 units: Optional[Literal["c", "f"]] = None
146
147 completion = _make_snapshot_request(
148 lambda c: c.beta.chat.completions.parse(
149 model="gpt-4o-2024-08-06",
150 messages=[
151 {
152 "role": "user",
153 "content": "What's the weather like in SF?",
154 },
155 ],
156 response_format=Location,
157 ),
158 content_snapshot=snapshot(
159 '{"id": "chatcmpl-9y39Q2jGzWmeEZlm5CoNVOuQzcxP4", "object": "chat.completion", "created": 1724098820, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":62,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 17, "completion_tokens": 14, "total_tokens": 31}, "system_fingerprint": "fp_2a322c9ffc"}'
160 ),
161 mock_client=client,
162 respx_mock=respx_mock,
163 )
164
165 assert print_obj(completion, monkeypatch) == snapshot(
166 """\
167ParsedChatCompletion[Location](
168 choices=[
169 ParsedChoice[Location](
170 finish_reason='stop',
171 index=0,
172 logprobs=None,
173 message=ParsedChatCompletionMessage[Location](
174 content='{"city":"San Francisco","temperature":62,"units":"f"}',
175 function_call=None,
176 parsed=Location(city='San Francisco', temperature=62.0, units='f'),
177 refusal=None,
178 role='assistant',
179 tool_calls=[]
180 )
181 )
182 ],
183 created=1724098820,
184 id='chatcmpl-9y39Q2jGzWmeEZlm5CoNVOuQzcxP4',
185 model='gpt-4o-2024-08-06',
186 object='chat.completion',
187 service_tier=None,
188 system_fingerprint='fp_2a322c9ffc',
189 usage=CompletionUsage(completion_tokens=14, completion_tokens_details=None, prompt_tokens=17, total_tokens=31)
190)
191"""
192 )
193
194
195@pytest.mark.respx(base_url=base_url)
196def test_parse_pydantic_model_enum(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
197 class Color(Enum):
198 """The detected color"""
199
200 RED = "red"
201 BLUE = "blue"
202 GREEN = "green"
203
204 class ColorDetection(BaseModel):
205 color: Color
206 hex_color_code: str = Field(description="The hex color code of the detected color")
207
208 if not PYDANTIC_V2:
209 ColorDetection.update_forward_refs(**locals()) # type: ignore
210
211 completion = _make_snapshot_request(
212 lambda c: c.beta.chat.completions.parse(
213 model="gpt-4o-2024-08-06",
214 messages=[
215 {"role": "user", "content": "What color is a Coke can?"},
216 ],
217 response_format=ColorDetection,
218 ),
219 content_snapshot=snapshot(
220 '{"id": "chatcmpl-9vK4UZVr385F2UgZlP1ShwPn2nFxG", "object": "chat.completion", "created": 1723448878, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"color\\":\\"red\\",\\"hex_color_code\\":\\"#FF0000\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 18, "completion_tokens": 14, "total_tokens": 32}, "system_fingerprint": "fp_845eaabc1f"}'
221 ),
222 mock_client=client,
223 respx_mock=respx_mock,
224 )
225
226 assert print_obj(completion.choices[0], monkeypatch) == snapshot(
227 """\
228ParsedChoice[ColorDetection](
229 finish_reason='stop',
230 index=0,
231 logprobs=None,
232 message=ParsedChatCompletionMessage[ColorDetection](
233 content='{"color":"red","hex_color_code":"#FF0000"}',
234 function_call=None,
235 parsed=ColorDetection(color=<Color.RED: 'red'>, hex_color_code='#FF0000'),
236 refusal=None,
237 role='assistant',
238 tool_calls=[]
239 )
240)
241"""
242 )
243
244
245@pytest.mark.respx(base_url=base_url)
246def test_parse_pydantic_model_multiple_choices(
247 client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch
248) -> None:
249 class Location(BaseModel):
250 city: str
251 temperature: float
252 units: Literal["c", "f"]
253
254 completion = _make_snapshot_request(
255 lambda c: c.beta.chat.completions.parse(
256 model="gpt-4o-2024-08-06",
257 messages=[
258 {
259 "role": "user",
260 "content": "What's the weather like in SF?",
261 },
262 ],
263 n=3,
264 response_format=Location,
265 ),
266 content_snapshot=snapshot(
267 '{"id": "chatcmpl-9tXjUrNFyyjSB2FJ842TMDNRM6Gen", "object": "chat.completion", "created": 1723024736, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":58,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}, {"index": 1, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":58,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}, {"index": 2, "message": {"role": "assistant", "content": "{\\"city\\":\\"San Francisco\\",\\"temperature\\":63,\\"units\\":\\"f\\"}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 17, "completion_tokens": 42, "total_tokens": 59}, "system_fingerprint": "fp_845eaabc1f"}'
268 ),
269 mock_client=client,
270 respx_mock=respx_mock,
271 )
272
273 assert print_obj(completion.choices, monkeypatch) == snapshot(
274 """\
275[
276 ParsedChoice[Location](
277 finish_reason='stop',
278 index=0,
279 logprobs=None,
280 message=ParsedChatCompletionMessage[Location](
281 content='{"city":"San Francisco","temperature":58,"units":"f"}',
282 function_call=None,
283 parsed=Location(city='San Francisco', temperature=58.0, units='f'),
284 refusal=None,
285 role='assistant',
286 tool_calls=[]
287 )
288 ),
289 ParsedChoice[Location](
290 finish_reason='stop',
291 index=1,
292 logprobs=None,
293 message=ParsedChatCompletionMessage[Location](
294 content='{"city":"San Francisco","temperature":58,"units":"f"}',
295 function_call=None,
296 parsed=Location(city='San Francisco', temperature=58.0, units='f'),
297 refusal=None,
298 role='assistant',
299 tool_calls=[]
300 )
301 ),
302 ParsedChoice[Location](
303 finish_reason='stop',
304 index=2,
305 logprobs=None,
306 message=ParsedChatCompletionMessage[Location](
307 content='{"city":"San Francisco","temperature":63,"units":"f"}',
308 function_call=None,
309 parsed=Location(city='San Francisco', temperature=63.0, units='f'),
310 refusal=None,
311 role='assistant',
312 tool_calls=[]
313 )
314 )
315]
316"""
317 )
318
319
320@pytest.mark.respx(base_url=base_url)
321@pytest.mark.skipif(not PYDANTIC_V2, reason="dataclasses only supported in v2")
322def test_parse_pydantic_dataclass(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
323 from pydantic.dataclasses import dataclass
324
325 @dataclass
326 class CalendarEvent:
327 name: str
328 date: str
329 participants: List[str]
330
331 completion = _make_snapshot_request(
332 lambda c: c.beta.chat.completions.parse(
333 model="gpt-4o-2024-08-06",
334 messages=[
335 {"role": "system", "content": "Extract the event information."},
336 {"role": "user", "content": "Alice and Bob are going to a science fair on Friday."},
337 ],
338 response_format=CalendarEvent,
339 ),
340 content_snapshot=snapshot(
341 '{"id": "chatcmpl-9wdGqXkJJARAz7rOrLH5u5FBwLjF3", "object": "chat.completion", "created": 1723761008, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"name\\":\\"Science Fair\\",\\"date\\":\\"Friday\\",\\"participants\\":[\\"Alice\\",\\"Bob\\"]}", "refusal": null}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 32, "completion_tokens": 17, "total_tokens": 49}, "system_fingerprint": "fp_2a322c9ffc"}'
342 ),
343 mock_client=client,
344 respx_mock=respx_mock,
345 )
346
347 assert print_obj(completion, monkeypatch) == snapshot(
348 """\
349ParsedChatCompletion[CalendarEvent](
350 choices=[
351 ParsedChoice[CalendarEvent](
352 finish_reason='stop',
353 index=0,
354 logprobs=None,
355 message=ParsedChatCompletionMessage[CalendarEvent](
356 content='{"name":"Science Fair","date":"Friday","participants":["Alice","Bob"]}',
357 function_call=None,
358 parsed=CalendarEvent(name='Science Fair', date='Friday', participants=['Alice', 'Bob']),
359 refusal=None,
360 role='assistant',
361 tool_calls=[]
362 )
363 )
364 ],
365 created=1723761008,
366 id='chatcmpl-9wdGqXkJJARAz7rOrLH5u5FBwLjF3',
367 model='gpt-4o-2024-08-06',
368 object='chat.completion',
369 service_tier=None,
370 system_fingerprint='fp_2a322c9ffc',
371 usage=CompletionUsage(completion_tokens=17, completion_tokens_details=None, prompt_tokens=32, total_tokens=49)
372)
373"""
374 )
375
376
377@pytest.mark.respx(base_url=base_url)
378def test_pydantic_tool_model_all_types(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
379 completion = _make_snapshot_request(
380 lambda c: c.beta.chat.completions.parse(
381 model="gpt-4o-2024-08-06",
382 messages=[
383 {
384 "role": "user",
385 "content": "look up all my orders in may of last year that were fulfilled but not delivered on time",
386 },
387 ],
388 tools=[openai.pydantic_function_tool(Query)],
389 response_format=Query,
390 ),
391 content_snapshot=snapshot(
392 '{"id": "chatcmpl-9tXjVJVCLTn7CWFhpjETixvvApCk3", "object": "chat.completion", "created": 1723024737, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_Un4g0IXeQGOyqKBS3zhqNCox", "type": "function", "function": {"name": "Query", "arguments": "{\\"table_name\\":\\"orders\\",\\"columns\\":[\\"id\\",\\"status\\",\\"expected_delivery_date\\",\\"delivered_at\\",\\"shipped_at\\",\\"ordered_at\\"],\\"conditions\\":[{\\"column\\":\\"ordered_at\\",\\"operator\\":\\">=\\",\\"value\\":\\"2022-05-01\\"},{\\"column\\":\\"ordered_at\\",\\"operator\\":\\"<=\\",\\"value\\":\\"2022-05-31\\"},{\\"column\\":\\"status\\",\\"operator\\":\\"=\\",\\"value\\":\\"fulfilled\\"},{\\"column\\":\\"delivered_at\\",\\"operator\\":\\">\\",\\"value\\":{\\"column_name\\":\\"expected_delivery_date\\"}}],\\"order_by\\":\\"asc\\"}"}}], "refusal": null}, "logprobs": null, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 195, "completion_tokens": 114, "total_tokens": 309}, "system_fingerprint": "fp_845eaabc1f"}'
393 ),
394 mock_client=client,
395 respx_mock=respx_mock,
396 )
397
398 assert print_obj(completion.choices[0], monkeypatch) == snapshot(
399 """\
400ParsedChoice[Query](
401 finish_reason='tool_calls',
402 index=0,
403 logprobs=None,
404 message=ParsedChatCompletionMessage[Query](
405 content=None,
406 function_call=None,
407 parsed=None,
408 refusal=None,
409 role='assistant',
410 tool_calls=[
411 ParsedFunctionToolCall(
412 function=ParsedFunction(
413 arguments='{"table_name":"orders","columns":["id","status","expected_delivery_date","delivered_at","
414shipped_at","ordered_at"],"conditions":[{"column":"ordered_at","operator":">=","value":"2022-05-01"},{"column":"ordered_
415at","operator":"<=","value":"2022-05-31"},{"column":"status","operator":"=","value":"fulfilled"},{"column":"delivered_at
416","operator":">","value":{"column_name":"expected_delivery_date"}}],"order_by":"asc"}',
417 name='Query',
418 parsed_arguments=Query(
419 columns=[
420 <Column.id: 'id'>,
421 <Column.status: 'status'>,
422 <Column.expected_delivery_date: 'expected_delivery_date'>,
423 <Column.delivered_at: 'delivered_at'>,
424 <Column.shipped_at: 'shipped_at'>,
425 <Column.ordered_at: 'ordered_at'>
426 ],
427 conditions=[
428 Condition(column='ordered_at', operator=<Operator.ge: '>='>, value='2022-05-01'),
429 Condition(column='ordered_at', operator=<Operator.le: '<='>, value='2022-05-31'),
430 Condition(column='status', operator=<Operator.eq: '='>, value='fulfilled'),
431 Condition(
432 column='delivered_at',
433 operator=<Operator.gt: '>'>,
434 value=DynamicValue(column_name='expected_delivery_date')
435 )
436 ],
437 name=None,
438 order_by=<OrderBy.asc: 'asc'>,
439 table_name=<Table.orders: 'orders'>
440 )
441 ),
442 id='call_Un4g0IXeQGOyqKBS3zhqNCox',
443 type='function'
444 )
445 ]
446 )
447)
448"""
449 )
450
451
452@pytest.mark.respx(base_url=base_url)
453def test_parse_max_tokens_reached(client: OpenAI, respx_mock: MockRouter) -> None:
454 class Location(BaseModel):
455 city: str
456 temperature: float
457 units: Literal["c", "f"]
458
459 with pytest.raises(openai.LengthFinishReasonError):
460 _make_snapshot_request(
461 lambda c: c.beta.chat.completions.parse(
462 model="gpt-4o-2024-08-06",
463 messages=[
464 {
465 "role": "user",
466 "content": "What's the weather like in SF?",
467 },
468 ],
469 max_tokens=1,
470 response_format=Location,
471 ),
472 content_snapshot=snapshot(
473 '{"id": "chatcmpl-9tXjYACgVKixKdMv2nVQqDVELkdSF", "object": "chat.completion", "created": 1723024740, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": "{\\"", "refusal": null}, "logprobs": null, "finish_reason": "length"}], "usage": {"prompt_tokens": 17, "completion_tokens": 1, "total_tokens": 18}, "system_fingerprint": "fp_2a322c9ffc"}'
474 ),
475 mock_client=client,
476 respx_mock=respx_mock,
477 )
478
479
480@pytest.mark.respx(base_url=base_url)
481def test_parse_pydantic_model_refusal(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
482 class Location(BaseModel):
483 city: str
484 temperature: float
485 units: Literal["c", "f"]
486
487 completion = _make_snapshot_request(
488 lambda c: c.beta.chat.completions.parse(
489 model="gpt-4o-2024-08-06",
490 messages=[
491 {
492 "role": "user",
493 "content": "How do I make anthrax?",
494 },
495 ],
496 response_format=Location,
497 ),
498 content_snapshot=snapshot(
499 '{"id": "chatcmpl-9tXm7FnIj3hSot5xM4c954MIePle0", "object": "chat.completion", "created": 1723024899, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "refusal": "I\'m very sorry, but I can\'t assist with that request."}, "logprobs": null, "finish_reason": "stop"}], "usage": {"prompt_tokens": 17, "completion_tokens": 13, "total_tokens": 30}, "system_fingerprint": "fp_845eaabc1f"}'
500 ),
501 mock_client=client,
502 respx_mock=respx_mock,
503 )
504
505 assert print_obj(completion.choices, monkeypatch) == snapshot(
506 """\
507[
508 ParsedChoice[Location](
509 finish_reason='stop',
510 index=0,
511 logprobs=None,
512 message=ParsedChatCompletionMessage[Location](
513 content=None,
514 function_call=None,
515 parsed=None,
516 refusal="I'm very sorry, but I can't assist with that request.",
517 role='assistant',
518 tool_calls=[]
519 )
520 )
521]
522"""
523 )
524
525
526@pytest.mark.respx(base_url=base_url)
527def test_parse_pydantic_tool(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
528 class GetWeatherArgs(BaseModel):
529 city: str
530 country: str
531 units: Literal["c", "f"] = "c"
532
533 completion = _make_snapshot_request(
534 lambda c: c.beta.chat.completions.parse(
535 model="gpt-4o-2024-08-06",
536 messages=[
537 {
538 "role": "user",
539 "content": "What's the weather like in Edinburgh?",
540 },
541 ],
542 tools=[
543 openai.pydantic_function_tool(GetWeatherArgs),
544 ],
545 ),
546 content_snapshot=snapshot(
547 '{"id": "chatcmpl-9tXjbQ9V0l5XPlynOJHKvrWsJQymO", "object": "chat.completion", "created": 1723024743, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_EEaIYq8aTdiDWro8jILNl3XK", "type": "function", "function": {"name": "GetWeatherArgs", "arguments": "{\\"city\\":\\"Edinburgh\\",\\"country\\":\\"GB\\",\\"units\\":\\"c\\"}"}}], "refusal": null}, "logprobs": null, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 76, "completion_tokens": 24, "total_tokens": 100}, "system_fingerprint": "fp_2a322c9ffc"}'
548 ),
549 mock_client=client,
550 respx_mock=respx_mock,
551 )
552
553 assert print_obj(completion.choices, monkeypatch) == snapshot(
554 """\
555[
556 ParsedChoice[NoneType](
557 finish_reason='tool_calls',
558 index=0,
559 logprobs=None,
560 message=ParsedChatCompletionMessage[NoneType](
561 content=None,
562 function_call=None,
563 parsed=None,
564 refusal=None,
565 role='assistant',
566 tool_calls=[
567 ParsedFunctionToolCall(
568 function=ParsedFunction(
569 arguments='{"city":"Edinburgh","country":"GB","units":"c"}',
570 name='GetWeatherArgs',
571 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='GB', units='c')
572 ),
573 id='call_EEaIYq8aTdiDWro8jILNl3XK',
574 type='function'
575 )
576 ]
577 )
578 )
579]
580"""
581 )
582
583
584@pytest.mark.respx(base_url=base_url)
585def test_parse_multiple_pydantic_tools(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
586 class GetWeatherArgs(BaseModel):
587 """Get the temperature for the given country/city combo"""
588
589 city: str
590 country: str
591 units: Literal["c", "f"] = "c"
592
593 class GetStockPrice(BaseModel):
594 ticker: str
595 exchange: str
596
597 completion = _make_snapshot_request(
598 lambda c: c.beta.chat.completions.parse(
599 model="gpt-4o-2024-08-06",
600 messages=[
601 {
602 "role": "user",
603 "content": "What's the weather like in Edinburgh?",
604 },
605 {
606 "role": "user",
607 "content": "What's the price of AAPL?",
608 },
609 ],
610 tools=[
611 openai.pydantic_function_tool(GetWeatherArgs),
612 openai.pydantic_function_tool(
613 GetStockPrice, name="get_stock_price", description="Fetch the latest price for a given ticker"
614 ),
615 ],
616 ),
617 content_snapshot=snapshot(
618 '{"id": "chatcmpl-9tXjcnIvzZDXRfLfbVTPNL5963GWw", "object": "chat.completion", "created": 1723024744, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_ECSuZ8gcNPPwgt24me91jHsJ", "type": "function", "function": {"name": "GetWeatherArgs", "arguments": "{\\"city\\": \\"Edinburgh\\", \\"country\\": \\"UK\\", \\"units\\": \\"c\\"}"}}, {"id": "call_Z3fM2sNBBGILhMtimk5Y3RQk", "type": "function", "function": {"name": "get_stock_price", "arguments": "{\\"ticker\\": \\"AAPL\\", \\"exchange\\": \\"NASDAQ\\"}"}}], "refusal": null}, "logprobs": null, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 149, "completion_tokens": 60, "total_tokens": 209}, "system_fingerprint": "fp_845eaabc1f"}'
619 ),
620 mock_client=client,
621 respx_mock=respx_mock,
622 )
623
624 assert print_obj(completion.choices, monkeypatch) == snapshot(
625 """\
626[
627 ParsedChoice[NoneType](
628 finish_reason='tool_calls',
629 index=0,
630 logprobs=None,
631 message=ParsedChatCompletionMessage[NoneType](
632 content=None,
633 function_call=None,
634 parsed=None,
635 refusal=None,
636 role='assistant',
637 tool_calls=[
638 ParsedFunctionToolCall(
639 function=ParsedFunction(
640 arguments='{"city": "Edinburgh", "country": "UK", "units": "c"}',
641 name='GetWeatherArgs',
642 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='UK', units='c')
643 ),
644 id='call_ECSuZ8gcNPPwgt24me91jHsJ',
645 type='function'
646 ),
647 ParsedFunctionToolCall(
648 function=ParsedFunction(
649 arguments='{"ticker": "AAPL", "exchange": "NASDAQ"}',
650 name='get_stock_price',
651 parsed_arguments=GetStockPrice(exchange='NASDAQ', ticker='AAPL')
652 ),
653 id='call_Z3fM2sNBBGILhMtimk5Y3RQk',
654 type='function'
655 )
656 ]
657 )
658 )
659]
660"""
661 )
662
663
664@pytest.mark.respx(base_url=base_url)
665def test_parse_strict_tools(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
666 completion = _make_snapshot_request(
667 lambda c: c.beta.chat.completions.parse(
668 model="gpt-4o-2024-08-06",
669 messages=[
670 {
671 "role": "user",
672 "content": "What's the weather like in SF?",
673 },
674 ],
675 tools=[
676 {
677 "type": "function",
678 "function": {
679 "name": "get_weather",
680 "parameters": {
681 "type": "object",
682 "properties": {
683 "city": {"type": "string"},
684 "state": {"type": "string"},
685 },
686 "required": [
687 "city",
688 "state",
689 ],
690 "additionalProperties": False,
691 },
692 "strict": True,
693 },
694 }
695 ],
696 ),
697 content_snapshot=snapshot(
698 '{"id": "chatcmpl-9tXjfjETDIqeYvDjsuGACbwdY0xsr", "object": "chat.completion", "created": 1723024747, "model": "gpt-4o-2024-08-06", "choices": [{"index": 0, "message": {"role": "assistant", "content": null, "tool_calls": [{"id": "call_7ZZPctBXQWexQlIHSrIHMVUq", "type": "function", "function": {"name": "get_weather", "arguments": "{\\"city\\":\\"San Francisco\\",\\"state\\":\\"CA\\"}"}}], "refusal": null}, "logprobs": null, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 48, "completion_tokens": 19, "total_tokens": 67}, "system_fingerprint": "fp_2a322c9ffc"}'
699 ),
700 mock_client=client,
701 respx_mock=respx_mock,
702 )
703
704 assert print_obj(completion.choices, monkeypatch) == snapshot(
705 """\
706[
707 ParsedChoice[NoneType](
708 finish_reason='tool_calls',
709 index=0,
710 logprobs=None,
711 message=ParsedChatCompletionMessage[NoneType](
712 content=None,
713 function_call=None,
714 parsed=None,
715 refusal=None,
716 role='assistant',
717 tool_calls=[
718 ParsedFunctionToolCall(
719 function=ParsedFunction(
720 arguments='{"city":"San Francisco","state":"CA"}',
721 name='get_weather',
722 parsed_arguments={'city': 'San Francisco', 'state': 'CA'}
723 ),
724 id='call_7ZZPctBXQWexQlIHSrIHMVUq',
725 type='function'
726 )
727 ]
728 )
729 )
730]
731"""
732 )
733
734
735def test_parse_non_strict_tools(client: OpenAI) -> None:
736 with pytest.raises(
737 ValueError, match="`get_weather` is not strict. Only `strict` function tools can be auto-parsed"
738 ):
739 client.beta.chat.completions.parse(
740 model="gpt-4o-2024-08-06",
741 messages=[],
742 tools=[
743 {
744 "type": "function",
745 "function": {
746 "name": "get_weather",
747 "parameters": {},
748 },
749 }
750 ],
751 )
752
753
754@pytest.mark.parametrize("sync", [True, False], ids=["sync", "async"])
755def test_parse_method_in_sync(sync: bool, client: OpenAI, async_client: AsyncOpenAI) -> None:
756 checking_client: OpenAI | AsyncOpenAI = client if sync else async_client
757
758 assert_signatures_in_sync(
759 checking_client.chat.completions.create,
760 checking_client.beta.chat.completions.parse,
761 exclude_params={"response_format", "stream"},
762 )
763
764
765def _make_snapshot_request(
766 func: Callable[[OpenAI], _T],
767 *,
768 content_snapshot: Any,
769 respx_mock: MockRouter,
770 mock_client: OpenAI,
771) -> _T:
772 live = os.environ.get("OPENAI_LIVE") == "1"
773 if live:
774
775 def _on_response(response: httpx.Response) -> None:
776 # update the content snapshot
777 assert json.dumps(json.loads(response.read())) == content_snapshot
778
779 respx_mock.stop()
780
781 client = OpenAI(
782 http_client=httpx.Client(
783 event_hooks={
784 "response": [_on_response],
785 }
786 )
787 )
788 else:
789 respx_mock.post("/chat/completions").mock(
790 return_value=httpx.Response(
791 200,
792 content=content_snapshot._old_value,
793 headers={"content-type": "application/json"},
794 )
795 )
796
797 client = mock_client
798
799 result = func(client)
800
801 if live:
802 client.close()
803
804 return result
805