GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

openai/openai-python

Public

mirrored from https://github.com/openai/openai-pythonAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
50de514b910aced32104833acd892bebfb2cf123

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openai-python/tests/lib/chat

tests/lib/chat/test_completions_streaming.py

1058lines · modecode

RawDownload
feat(api): support storing chat completions, enabling evals and model distillation in the dashboard7c8c111
1from __future__ import annotations
2
3import os
4from typing import Any, Generic, Callable, Iterator, cast, overload
5from typing_extensions import Literal, TypeVar
6
7import rich
8import httpx
9import pytest
10from respx import MockRouter
11from pydantic import BaseModel
12from inline_snapshot import external, snapshot, outsource
13
14import openai
15from openai import OpenAI, AsyncOpenAI
16from openai._utils import assert_signatures_in_sync
17from openai._compat import model_copy
18from openai.lib.streaming.chat import (
19 ContentDoneEvent,
20 ChatCompletionStream,
21 ChatCompletionStreamEvent,
22 ChatCompletionStreamManager,
23 ParsedChatCompletionSnapshot,
24)
25from openai.lib._parsing._completions import ResponseFormatT
26
27from ._utils import print_obj
28from ...conftest import base_url
29
30_T = TypeVar("_T")
31
32# all the snapshots in this file are auto-generated from the live API
33#
34# you can update them with
35#
36# `OPENAI_LIVE=1 pytest --inline-snapshot=fix`
37
38
39@pytest.mark.respx(base_url=base_url)
40def test_parse_nothing(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
41 listener = _make_stream_snapshot_request(
42 lambda c: c.beta.chat.completions.stream(
43 model="gpt-4o-2024-08-06",
44 messages=[
45 {
46 "role": "user",
47 "content": "What's the weather like in SF?",
48 },
49 ],
50 ),
51 content_snapshot=snapshot(external("e2aad469b71d*.bin")),
52 mock_client=client,
53 respx_mock=respx_mock,
54 )
55
56 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
57 """\
58[
59 ParsedChoice[NoneType](
60 finish_reason='stop',
61 index=0,
62 logprobs=None,
63 message=ParsedChatCompletionMessage[NoneType](
64 content="I'm unable to provide real-time weather updates. To get the current weather in San Francisco, I
65recommend checking a reliable weather website or a weather app.",
66 function_call=None,
67 parsed=None,
68 refusal=None,
69 role='assistant',
70 tool_calls=[]
71 )
72 )
73]
74"""
75 )
76 assert print_obj(listener.get_event_by_type("content.done"), monkeypatch) == snapshot(
77 """\
78ContentDoneEvent[NoneType](
79 content="I'm unable to provide real-time weather updates. To get the current weather in San Francisco, I recommend
80checking a reliable weather website or a weather app.",
81 parsed=None,
82 type='content.done'
83)
84"""
85 )
86
87
88@pytest.mark.respx(base_url=base_url)
89def test_parse_pydantic_model(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
90 class Location(BaseModel):
91 city: str
92 temperature: float
93 units: Literal["c", "f"]
94
95 done_snapshots: list[ParsedChatCompletionSnapshot] = []
96
97 def on_event(stream: ChatCompletionStream[Location], event: ChatCompletionStreamEvent[Location]) -> None:
98 if event.type == "content.done":
99 done_snapshots.append(model_copy(stream.current_completion_snapshot, deep=True))
100
101 listener = _make_stream_snapshot_request(
102 lambda c: c.beta.chat.completions.stream(
103 model="gpt-4o-2024-08-06",
104 messages=[
105 {
106 "role": "user",
107 "content": "What's the weather like in SF?",
108 },
109 ],
110 response_format=Location,
111 ),
112 content_snapshot=snapshot(external("7e5ea4d12e7c*.bin")),
113 mock_client=client,
114 respx_mock=respx_mock,
115 on_event=on_event,
116 )
117
118 assert len(done_snapshots) == 1
119 assert isinstance(done_snapshots[0].choices[0].message.parsed, Location)
120
121 for event in reversed(listener.events):
122 if event.type == "content.delta":
123 data = cast(Any, event.parsed)
124 assert isinstance(data["city"], str), data
125 assert isinstance(data["temperature"], (int, float)), data
126 assert isinstance(data["units"], str), data
127 break
128 else:
129 rich.print(listener.events)
130 raise AssertionError("Did not find a `content.delta` event")
131
132 assert print_obj(listener.stream.get_final_completion(), monkeypatch) == snapshot(
133 """\
134ParsedChatCompletion[Location](
135 choices=[
136 ParsedChoice[Location](
137 finish_reason='stop',
138 index=0,
139 logprobs=None,
140 message=ParsedChatCompletionMessage[Location](
141 content='{"city":"San Francisco","temperature":61,"units":"f"}',
142 function_call=None,
143 parsed=Location(city='San Francisco', temperature=61.0, units='f'),
144 refusal=None,
145 role='assistant',
146 tool_calls=[]
147 )
148 )
149 ],
150 created=1727346169,
151 id='chatcmpl-ABfw1e5abtU8OwGr15vOreYVb2MiF',
152 model='gpt-4o-2024-08-06',
153 object='chat.completion',
154 service_tier=None,
155 system_fingerprint='fp_5050236cbd',
156 usage=CompletionUsage(
157 completion_tokens=14,
158 completion_tokens_details=CompletionTokensDetails(audio_tokens=None, reasoning_tokens=0),
159 prompt_tokens=79,
160 prompt_tokens_details=None,
161 total_tokens=93
162 )
163)
164"""
165 )
166 assert print_obj(listener.get_event_by_type("content.done"), monkeypatch) == snapshot(
167 """\
168ContentDoneEvent[Location](
169 content='{"city":"San Francisco","temperature":61,"units":"f"}',
170 parsed=Location(city='San Francisco', temperature=61.0, units='f'),
171 type='content.done'
172)
173"""
174 )
175
176
177@pytest.mark.respx(base_url=base_url)
178def test_parse_pydantic_model_multiple_choices(
179 client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch
180) -> None:
181 class Location(BaseModel):
182 city: str
183 temperature: float
184 units: Literal["c", "f"]
185
186 listener = _make_stream_snapshot_request(
187 lambda c: c.beta.chat.completions.stream(
188 model="gpt-4o-2024-08-06",
189 messages=[
190 {
191 "role": "user",
192 "content": "What's the weather like in SF?",
193 },
194 ],
195 n=3,
196 response_format=Location,
197 ),
198 content_snapshot=snapshot(external("a491adda08c3*.bin")),
199 mock_client=client,
200 respx_mock=respx_mock,
201 )
202
203 assert [e.type for e in listener.events] == snapshot(
204 [
205 "chunk",
206 "content.delta",
207 "chunk",
208 "content.delta",
209 "chunk",
210 "content.delta",
211 "chunk",
212 "content.delta",
213 "chunk",
214 "content.delta",
215 "chunk",
216 "content.delta",
217 "chunk",
218 "content.delta",
219 "chunk",
220 "content.delta",
221 "chunk",
222 "content.delta",
223 "chunk",
224 "content.delta",
225 "chunk",
226 "content.delta",
227 "chunk",
228 "content.delta",
229 "chunk",
230 "content.delta",
231 "chunk",
232 "content.delta",
233 "chunk",
234 "content.delta",
235 "chunk",
236 "content.delta",
237 "chunk",
238 "content.delta",
239 "chunk",
240 "content.delta",
241 "chunk",
242 "content.delta",
243 "chunk",
244 "content.delta",
245 "chunk",
246 "content.delta",
247 "chunk",
248 "content.delta",
249 "chunk",
250 "content.delta",
251 "chunk",
252 "content.delta",
253 "chunk",
254 "content.delta",
255 "chunk",
256 "content.delta",
257 "chunk",
258 "content.delta",
259 "chunk",
260 "content.delta",
261 "chunk",
262 "content.delta",
263 "chunk",
264 "content.delta",
265 "chunk",
266 "content.delta",
267 "chunk",
268 "content.delta",
269 "chunk",
270 "content.delta",
271 "chunk",
272 "content.delta",
273 "chunk",
274 "content.delta",
275 "chunk",
276 "content.delta",
277 "chunk",
278 "content.delta",
279 "chunk",
280 "content.delta",
281 "chunk",
282 "content.delta",
283 "chunk",
284 "content.delta",
285 "chunk",
286 "content.delta",
287 "chunk",
288 "content.delta",
289 "chunk",
290 "content.delta",
291 "chunk",
292 "content.delta",
293 "chunk",
294 "content.delta",
295 "chunk",
296 "content.done",
297 "chunk",
298 "content.done",
299 "chunk",
300 "content.done",
301 "chunk",
302 ]
303 )
304 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
305 """\
306[
307 ParsedChoice[Location](
308 finish_reason='stop',
309 index=0,
310 logprobs=None,
311 message=ParsedChatCompletionMessage[Location](
312 content='{"city":"San Francisco","temperature":65,"units":"f"}',
313 function_call=None,
314 parsed=Location(city='San Francisco', temperature=65.0, units='f'),
315 refusal=None,
316 role='assistant',
317 tool_calls=[]
318 )
319 ),
320 ParsedChoice[Location](
321 finish_reason='stop',
322 index=1,
323 logprobs=None,
324 message=ParsedChatCompletionMessage[Location](
325 content='{"city":"San Francisco","temperature":61,"units":"f"}',
326 function_call=None,
327 parsed=Location(city='San Francisco', temperature=61.0, units='f'),
328 refusal=None,
329 role='assistant',
330 tool_calls=[]
331 )
332 ),
333 ParsedChoice[Location](
334 finish_reason='stop',
335 index=2,
336 logprobs=None,
337 message=ParsedChatCompletionMessage[Location](
338 content='{"city":"San Francisco","temperature":59,"units":"f"}',
339 function_call=None,
340 parsed=Location(city='San Francisco', temperature=59.0, units='f'),
341 refusal=None,
342 role='assistant',
343 tool_calls=[]
344 )
345 )
346]
347"""
348 )
349
350
351@pytest.mark.respx(base_url=base_url)
352def test_parse_max_tokens_reached(client: OpenAI, respx_mock: MockRouter) -> None:
353 class Location(BaseModel):
354 city: str
355 temperature: float
356 units: Literal["c", "f"]
357
358 with pytest.raises(openai.LengthFinishReasonError):
359 _make_stream_snapshot_request(
360 lambda c: c.beta.chat.completions.stream(
361 model="gpt-4o-2024-08-06",
362 messages=[
363 {
364 "role": "user",
365 "content": "What's the weather like in SF?",
366 },
367 ],
368 max_tokens=1,
369 response_format=Location,
370 ),
371 content_snapshot=snapshot(external("4cc50a6135d2*.bin")),
372 mock_client=client,
373 respx_mock=respx_mock,
374 )
375
376
377@pytest.mark.respx(base_url=base_url)
378def test_parse_pydantic_model_refusal(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
379 class Location(BaseModel):
380 city: str
381 temperature: float
382 units: Literal["c", "f"]
383
384 listener = _make_stream_snapshot_request(
385 lambda c: c.beta.chat.completions.stream(
386 model="gpt-4o-2024-08-06",
387 messages=[
388 {
389 "role": "user",
390 "content": "How do I make anthrax?",
391 },
392 ],
393 response_format=Location,
394 ),
395 content_snapshot=snapshot(external("173417d55340*.bin")),
396 mock_client=client,
397 respx_mock=respx_mock,
398 )
399
400 assert print_obj(listener.get_event_by_type("refusal.done"), monkeypatch) == snapshot("""\
401RefusalDoneEvent(refusal="I'm sorry, I can't assist with that request.", type='refusal.done')
402""")
403
404 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
405 """\
406[
407 ParsedChoice[Location](
408 finish_reason='stop',
409 index=0,
410 logprobs=None,
411 message=ParsedChatCompletionMessage[Location](
412 content=None,
413 function_call=None,
414 parsed=None,
415 refusal="I'm sorry, I can't assist with that request.",
416 role='assistant',
417 tool_calls=[]
418 )
419 )
420]
421"""
422 )
423
424
425@pytest.mark.respx(base_url=base_url)
426def test_content_logprobs_events(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
427 listener = _make_stream_snapshot_request(
428 lambda c: c.beta.chat.completions.stream(
429 model="gpt-4o-2024-08-06",
430 messages=[
431 {
432 "role": "user",
433 "content": "Say foo",
434 },
435 ],
436 logprobs=True,
437 ),
438 content_snapshot=snapshot(external("83b060bae42e*.bin")),
439 mock_client=client,
440 respx_mock=respx_mock,
441 )
442
443 assert print_obj([e for e in listener.events if e.type.startswith("logprobs")], monkeypatch) == snapshot("""\
444[
445 LogprobsContentDeltaEvent(
446 content=[
447 ChatCompletionTokenLogprob(bytes=[70, 111, 111], logprob=-0.0025094282, token='Foo', top_logprobs=[])
448 ],
449 snapshot=[
450 ChatCompletionTokenLogprob(bytes=[70, 111, 111], logprob=-0.0025094282, token='Foo', top_logprobs=[])
451 ],
452 type='logprobs.content.delta'
453 ),
454 LogprobsContentDeltaEvent(
455 content=[ChatCompletionTokenLogprob(bytes=[33], logprob=-0.26638845, token='!', top_logprobs=[])],
456 snapshot=[
457 ChatCompletionTokenLogprob(bytes=[70, 111, 111], logprob=-0.0025094282, token='Foo', top_logprobs=[]),
458 ChatCompletionTokenLogprob(bytes=[33], logprob=-0.26638845, token='!', top_logprobs=[])
459 ],
460 type='logprobs.content.delta'
461 ),
462 LogprobsContentDoneEvent(
463 content=[
464 ChatCompletionTokenLogprob(bytes=[70, 111, 111], logprob=-0.0025094282, token='Foo', top_logprobs=[]),
465 ChatCompletionTokenLogprob(bytes=[33], logprob=-0.26638845, token='!', top_logprobs=[])
466 ],
467 type='logprobs.content.done'
468 )
469]
470""")
471
472 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot("""\
473[
474 ParsedChoice[NoneType](
475 finish_reason='stop',
476 index=0,
477 logprobs=ChoiceLogprobs(
478 content=[
479 ChatCompletionTokenLogprob(bytes=[70, 111, 111], logprob=-0.0025094282, token='Foo', top_logprobs=[]),
480 ChatCompletionTokenLogprob(bytes=[33], logprob=-0.26638845, token='!', top_logprobs=[])
481 ],
482 refusal=None
483 ),
484 message=ParsedChatCompletionMessage[NoneType](
485 content='Foo!',
486 function_call=None,
487 parsed=None,
488 refusal=None,
489 role='assistant',
490 tool_calls=[]
491 )
492 )
493]
494""")
495
496
497@pytest.mark.respx(base_url=base_url)
498def test_refusal_logprobs_events(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
499 class Location(BaseModel):
500 city: str
501 temperature: float
502 units: Literal["c", "f"]
503
504 listener = _make_stream_snapshot_request(
505 lambda c: c.beta.chat.completions.stream(
506 model="gpt-4o-2024-08-06",
507 messages=[
508 {
509 "role": "user",
510 "content": "How do I make anthrax?",
511 },
512 ],
513 logprobs=True,
514 response_format=Location,
515 ),
516 content_snapshot=snapshot(external("569c877e6942*.bin")),
517 mock_client=client,
518 respx_mock=respx_mock,
519 )
520
521 assert print_obj([e.type for e in listener.events if e.type.startswith("logprobs")], monkeypatch) == snapshot("""\
522[
523 'logprobs.refusal.delta',
524 'logprobs.refusal.delta',
525 'logprobs.refusal.delta',
526 'logprobs.refusal.delta',
527 'logprobs.refusal.delta',
528 'logprobs.refusal.delta',
529 'logprobs.refusal.delta',
530 'logprobs.refusal.delta',
531 'logprobs.refusal.delta',
532 'logprobs.refusal.delta',
533 'logprobs.refusal.delta',
534 'logprobs.refusal.done'
535]
536""")
537
538 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot("""\
539[
540 ParsedChoice[Location](
541 finish_reason='stop',
542 index=0,
543 logprobs=ChoiceLogprobs(
544 content=None,
545 refusal=[
546 ChatCompletionTokenLogprob(bytes=[73, 39, 109], logprob=-0.0012038043, token="I'm", top_logprobs=[]),
547 ChatCompletionTokenLogprob(
548 bytes=[32, 118, 101, 114, 121],
549 logprob=-0.8438816,
550 token=' very',
551 top_logprobs=[]
552 ),
553 ChatCompletionTokenLogprob(
554 bytes=[32, 115, 111, 114, 114, 121],
555 logprob=-3.4121115e-06,
556 token=' sorry',
557 top_logprobs=[]
558 ),
559 ChatCompletionTokenLogprob(bytes=[44], logprob=-3.3809047e-05, token=',', top_logprobs=[]),
560 ChatCompletionTokenLogprob(
561 bytes=[32, 98, 117, 116],
562 logprob=-0.038048144,
563 token=' but',
564 top_logprobs=[]
565 ),
566 ChatCompletionTokenLogprob(bytes=[32, 73], logprob=-0.0016109125, token=' I', top_logprobs=[]),
567 ChatCompletionTokenLogprob(
568 bytes=[32, 99, 97, 110, 39, 116],
569 logprob=-0.0073532974,
570 token=" can't",
571 top_logprobs=[]
572 ),
573 ChatCompletionTokenLogprob(
574 bytes=[32, 97, 115, 115, 105, 115, 116],
575 logprob=-0.0020837625,
576 token=' assist',
577 top_logprobs=[]
578 ),
579 ChatCompletionTokenLogprob(
580 bytes=[32, 119, 105, 116, 104],
581 logprob=-0.00318354,
582 token=' with',
583 top_logprobs=[]
584 ),
585 ChatCompletionTokenLogprob(
586 bytes=[32, 116, 104, 97, 116],
587 logprob=-0.0017186158,
588 token=' that',
589 top_logprobs=[]
590 ),
591 ChatCompletionTokenLogprob(bytes=[46], logprob=-0.57687104, token='.', top_logprobs=[])
592 ]
593 ),
594 message=ParsedChatCompletionMessage[Location](
595 content=None,
596 function_call=None,
597 parsed=None,
598 refusal="I'm very sorry, but I can't assist with that.",
599 role='assistant',
600 tool_calls=[]
601 )
602 )
603]
604""")
605
606
607@pytest.mark.respx(base_url=base_url)
608def test_parse_pydantic_tool(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
609 class GetWeatherArgs(BaseModel):
610 city: str
611 country: str
612 units: Literal["c", "f"] = "c"
613
614 listener = _make_stream_snapshot_request(
615 lambda c: c.beta.chat.completions.stream(
616 model="gpt-4o-2024-08-06",
617 messages=[
618 {
619 "role": "user",
620 "content": "What's the weather like in Edinburgh?",
621 },
622 ],
623 tools=[
624 openai.pydantic_function_tool(GetWeatherArgs),
625 ],
626 ),
627 content_snapshot=snapshot(external("c6aa7e397b71*.bin")),
628 mock_client=client,
629 respx_mock=respx_mock,
630 )
631
632 assert print_obj(listener.stream.current_completion_snapshot.choices, monkeypatch) == snapshot(
633 """\
634[
635 ParsedChoice[object](
636 finish_reason='tool_calls',
637 index=0,
638 logprobs=None,
639 message=ParsedChatCompletionMessage[object](
640 content=None,
641 function_call=None,
642 parsed=None,
643 refusal=None,
644 role='assistant',
645 tool_calls=[
646 ParsedFunctionToolCall(
647 function=ParsedFunction(
648 arguments='{"city":"Edinburgh","country":"UK","units":"c"}',
649 name='GetWeatherArgs',
650 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='UK', units='c')
651 ),
652 id='call_c91SqDXlYFuETYv8mUHzz6pp',
653 index=0,
654 type='function'
655 )
656 ]
657 )
658 )
659]
660"""
661 )
662
663 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
664 """\
665[
666 ParsedChoice[NoneType](
667 finish_reason='tool_calls',
668 index=0,
669 logprobs=None,
670 message=ParsedChatCompletionMessage[NoneType](
671 content=None,
672 function_call=None,
673 parsed=None,
674 refusal=None,
675 role='assistant',
676 tool_calls=[
677 ParsedFunctionToolCall(
678 function=ParsedFunction(
679 arguments='{"city":"Edinburgh","country":"UK","units":"c"}',
680 name='GetWeatherArgs',
681 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='UK', units='c')
682 ),
683 id='call_c91SqDXlYFuETYv8mUHzz6pp',
684 index=0,
685 type='function'
686 )
687 ]
688 )
689 )
690]
691"""
692 )
693
694
695@pytest.mark.respx(base_url=base_url)
696def test_parse_multiple_pydantic_tools(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
697 class GetWeatherArgs(BaseModel):
698 """Get the temperature for the given country/city combo"""
699
700 city: str
701 country: str
702 units: Literal["c", "f"] = "c"
703
704 class GetStockPrice(BaseModel):
705 ticker: str
706 exchange: str
707
708 listener = _make_stream_snapshot_request(
709 lambda c: c.beta.chat.completions.stream(
710 model="gpt-4o-2024-08-06",
711 messages=[
712 {
713 "role": "user",
714 "content": "What's the weather like in Edinburgh?",
715 },
716 {
717 "role": "user",
718 "content": "What's the price of AAPL?",
719 },
720 ],
721 tools=[
722 openai.pydantic_function_tool(GetWeatherArgs),
723 openai.pydantic_function_tool(
724 GetStockPrice, name="get_stock_price", description="Fetch the latest price for a given ticker"
725 ),
726 ],
727 ),
728 content_snapshot=snapshot(external("f82268f2fefd*.bin")),
729 mock_client=client,
730 respx_mock=respx_mock,
731 )
732
733 assert print_obj(listener.stream.current_completion_snapshot.choices, monkeypatch) == snapshot(
734 """\
735[
736 ParsedChoice[object](
737 finish_reason='tool_calls',
738 index=0,
739 logprobs=None,
740 message=ParsedChatCompletionMessage[object](
741 content=None,
742 function_call=None,
743 parsed=None,
744 refusal=None,
745 role='assistant',
746 tool_calls=[
747 ParsedFunctionToolCall(
748 function=ParsedFunction(
749 arguments='{"city": "Edinburgh", "country": "GB", "units": "c"}',
750 name='GetWeatherArgs',
751 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='GB', units='c')
752 ),
753 id='call_JMW1whyEaYG438VE1OIflxA2',
754 index=0,
755 type='function'
756 ),
757 ParsedFunctionToolCall(
758 function=ParsedFunction(
759 arguments='{"ticker": "AAPL", "exchange": "NASDAQ"}',
760 name='get_stock_price',
761 parsed_arguments=GetStockPrice(exchange='NASDAQ', ticker='AAPL')
762 ),
763 id='call_DNYTawLBoN8fj3KN6qU9N1Ou',
764 index=1,
765 type='function'
766 )
767 ]
768 )
769 )
770]
771"""
772 )
773 completion = listener.stream.get_final_completion()
774 assert print_obj(completion.choices[0].message.tool_calls, monkeypatch) == snapshot(
775 """\
776[
777 ParsedFunctionToolCall(
778 function=ParsedFunction(
779 arguments='{"city": "Edinburgh", "country": "GB", "units": "c"}',
780 name='GetWeatherArgs',
781 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='GB', units='c')
782 ),
783 id='call_JMW1whyEaYG438VE1OIflxA2',
784 index=0,
785 type='function'
786 ),
787 ParsedFunctionToolCall(
788 function=ParsedFunction(
789 arguments='{"ticker": "AAPL", "exchange": "NASDAQ"}',
790 name='get_stock_price',
791 parsed_arguments=GetStockPrice(exchange='NASDAQ', ticker='AAPL')
792 ),
793 id='call_DNYTawLBoN8fj3KN6qU9N1Ou',
794 index=1,
795 type='function'
796 )
797]
798"""
799 )
800
801
802@pytest.mark.respx(base_url=base_url)
803def test_parse_strict_tools(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
804 listener = _make_stream_snapshot_request(
805 lambda c: c.beta.chat.completions.stream(
806 model="gpt-4o-2024-08-06",
807 messages=[
808 {
809 "role": "user",
810 "content": "What's the weather like in SF?",
811 },
812 ],
813 tools=[
814 {
815 "type": "function",
816 "function": {
817 "name": "get_weather",
818 "parameters": {
819 "type": "object",
820 "properties": {
821 "city": {"type": "string"},
822 "state": {"type": "string"},
823 },
824 "required": [
825 "city",
826 "state",
827 ],
828 "additionalProperties": False,
829 },
830 "strict": True,
831 },
832 }
833 ],
834 ),
835 content_snapshot=snapshot(external("a247c49c5fcd*.bin")),
836 mock_client=client,
837 respx_mock=respx_mock,
838 )
839
840 assert print_obj(listener.stream.current_completion_snapshot.choices, monkeypatch) == snapshot(
841 """\
842[
843 ParsedChoice[object](
844 finish_reason='tool_calls',
845 index=0,
846 logprobs=None,
847 message=ParsedChatCompletionMessage[object](
848 content=None,
849 function_call=None,
850 parsed=None,
851 refusal=None,
852 role='assistant',
853 tool_calls=[
854 ParsedFunctionToolCall(
855 function=ParsedFunction(
856 arguments='{"city":"San Francisco","state":"CA"}',
857 name='get_weather',
858 parsed_arguments={'city': 'San Francisco', 'state': 'CA'}
859 ),
860 id='call_CTf1nWJLqSeRgDqaCG27xZ74',
861 index=0,
862 type='function'
863 )
864 ]
865 )
866 )
867]
868"""
869 )
870
871
872@pytest.mark.respx(base_url=base_url)
873def test_non_pydantic_response_format(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
874 listener = _make_stream_snapshot_request(
875 lambda c: c.beta.chat.completions.stream(
876 model="gpt-4o-2024-08-06",
877 messages=[
878 {
879 "role": "user",
880 "content": "What's the weather like in SF? Give me any JSON back",
881 },
882 ],
883 response_format={"type": "json_object"},
884 ),
885 content_snapshot=snapshot(external("d61558011839*.bin")),
886 mock_client=client,
887 respx_mock=respx_mock,
888 )
889
890 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
891 """\
892[
893 ParsedChoice[NoneType](
894 finish_reason='stop',
895 index=0,
896 logprobs=None,
897 message=ParsedChatCompletionMessage[NoneType](
898 content='\\n {\\n "location": "San Francisco, CA",\\n "weather": {\\n "temperature": "18°C",\\n
899"condition": "Partly Cloudy",\\n "humidity": "72%",\\n "windSpeed": "15 km/h",\\n "windDirection": "NW"\\n
900},\\n "forecast": [\\n {\\n "day": "Monday",\\n "high": "20°C",\\n "low": "14°C",\\n
901"condition": "Sunny"\\n },\\n {\\n "day": "Tuesday",\\n "high": "19°C",\\n "low": "15°C",\\n
902"condition": "Mostly Cloudy"\\n },\\n {\\n "day": "Wednesday",\\n "high": "18°C",\\n "low":
903"14°C",\\n "condition": "Cloudy"\\n }\\n ]\\n }\\n',
904 function_call=None,
905 parsed=None,
906 refusal=None,
907 role='assistant',
908 tool_calls=[]
909 )
910 )
911]
912"""
913 )
914
915
916@pytest.mark.respx(base_url=base_url)
917def test_allows_non_strict_tools_but_no_parsing(
918 client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch
919) -> None:
920 listener = _make_stream_snapshot_request(
921 lambda c: c.beta.chat.completions.stream(
922 model="gpt-4o-2024-08-06",
923 messages=[{"role": "user", "content": "what's the weather in NYC?"}],
924 tools=[
925 {
926 "type": "function",
927 "function": {
928 "name": "get_weather",
929 "parameters": {"type": "object", "properties": {"city": {"type": "string"}}},
930 },
931 }
932 ],
933 ),
934 content_snapshot=snapshot(external("2018feb66ae1*.bin")),
935 mock_client=client,
936 respx_mock=respx_mock,
937 )
938
939 assert print_obj(listener.get_event_by_type("tool_calls.function.arguments.done"), monkeypatch) == snapshot("""\
940FunctionToolCallArgumentsDoneEvent(
941 arguments='{"city":"New York City"}',
942 index=0,
943 name='get_weather',
944 parsed_arguments=None,
945 type='tool_calls.function.arguments.done'
946)
947""")
948
949 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
950 """\
951[
952 ParsedChoice[NoneType](
953 finish_reason='tool_calls',
954 index=0,
955 logprobs=None,
956 message=ParsedChatCompletionMessage[NoneType](
957 content=None,
958 function_call=None,
959 parsed=None,
960 refusal=None,
961 role='assistant',
962 tool_calls=[
963 ParsedFunctionToolCall(
964 function=ParsedFunction(
965 arguments='{"city":"New York City"}',
966 name='get_weather',
967 parsed_arguments=None
968 ),
969 id='call_4XzlGBLtUe9dy3GVNV4jhq7h',
970 index=0,
971 type='function'
972 )
973 ]
974 )
975 )
976]
977"""
978 )
979
980
981@pytest.mark.parametrize("sync", [True, False], ids=["sync", "async"])
982def test_stream_method_in_sync(sync: bool, client: OpenAI, async_client: AsyncOpenAI) -> None:
983 checking_client: OpenAI | AsyncOpenAI = client if sync else async_client
984
985 assert_signatures_in_sync(
986 checking_client.chat.completions.create,
987 checking_client.beta.chat.completions.stream,
988 exclude_params={"response_format", "stream"},
989 )
990
991
992class StreamListener(Generic[ResponseFormatT]):
993 def __init__(self, stream: ChatCompletionStream[ResponseFormatT]) -> None:
994 self.stream = stream
995 self.events: list[ChatCompletionStreamEvent[ResponseFormatT]] = []
996
997 def __iter__(self) -> Iterator[ChatCompletionStreamEvent[ResponseFormatT]]:
998 for event in self.stream:
999 self.events.append(event)
1000 yield event
1001
1002 @overload
1003 def get_event_by_type(self, event_type: Literal["content.done"]) -> ContentDoneEvent[ResponseFormatT] | None: ...
1004
1005 @overload
1006 def get_event_by_type(self, event_type: str) -> ChatCompletionStreamEvent[ResponseFormatT] | None: ...
1007
1008 def get_event_by_type(self, event_type: str) -> ChatCompletionStreamEvent[ResponseFormatT] | None:
1009 return next((e for e in self.events if e.type == event_type), None)
1010
1011
1012def _make_stream_snapshot_request(
1013 func: Callable[[OpenAI], ChatCompletionStreamManager[ResponseFormatT]],
1014 *,
1015 content_snapshot: Any,
1016 respx_mock: MockRouter,
1017 mock_client: OpenAI,
1018 on_event: Callable[[ChatCompletionStream[ResponseFormatT], ChatCompletionStreamEvent[ResponseFormatT]], Any]
1019 | None = None,
1020) -> StreamListener[ResponseFormatT]:
1021 live = os.environ.get("OPENAI_LIVE") == "1"
1022 if live:
1023
1024 def _on_response(response: httpx.Response) -> None:
1025 # update the content snapshot
1026 assert outsource(response.read()) == content_snapshot
1027
1028 respx_mock.stop()
1029
1030 client = OpenAI(
1031 http_client=httpx.Client(
1032 event_hooks={
1033 "response": [_on_response],
1034 }
1035 )
1036 )
1037 else:
1038 respx_mock.post("/chat/completions").mock(
1039 return_value=httpx.Response(
1040 200,
1041 content=content_snapshot._old_value._load_value(),
1042 headers={"content-type": "text/event-stream"},
1043 )
1044 )
1045
1046 client = mock_client
1047
1048 with func(client) as stream:
1049 listener = StreamListener(stream)
1050
1051 for event in listener:
1052 if on_event:
1053 on_event(stream, event)
1054
1055 if live:
1056 client.close()
1057
1058 return listener