GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

openai/openai-python

Public

mirrored from https://github.com/openai/openai-pythonAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
v1.45.0

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openai-python/tests/lib/chat

tests/lib/chat/test_completions_streaming.py

1102lines · modecode

RawDownload
Latest commit unavailable.
1from __future__ import annotations
2
3import os
4from typing import Any, Generic, Callable, Iterator, cast, overload
5from typing_extensions import Literal, TypeVar
6
7import rich
8import httpx
9import pytest
10from respx import MockRouter
11from pydantic import BaseModel
12from inline_snapshot import external, snapshot, outsource
13
14import openai
15from openai import OpenAI, AsyncOpenAI
16from openai._utils import assert_signatures_in_sync
17from openai._compat import model_copy
18from openai.lib.streaming.chat import (
19 ContentDoneEvent,
20 ChatCompletionStream,
21 ChatCompletionStreamEvent,
22 ChatCompletionStreamManager,
23 ParsedChatCompletionSnapshot,
24)
25from openai.lib._parsing._completions import ResponseFormatT
26
27from ._utils import print_obj
28from ...conftest import base_url
29
30_T = TypeVar("_T")
31
32# all the snapshots in this file are auto-generated from the live API
33#
34# you can update them with
35#
36# `OPENAI_LIVE=1 pytest --inline-snapshot=fix`
37
38
39@pytest.mark.respx(base_url=base_url)
40def test_parse_nothing(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
41 listener = _make_stream_snapshot_request(
42 lambda c: c.beta.chat.completions.stream(
43 model="gpt-4o-2024-08-06",
44 messages=[
45 {
46 "role": "user",
47 "content": "What's the weather like in SF?",
48 },
49 ],
50 ),
51 content_snapshot=snapshot(external("038a5c69c34c*.bin")),
52 mock_client=client,
53 respx_mock=respx_mock,
54 )
55
56 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
57 """\
58[
59 ParsedChoice[NoneType](
60 finish_reason='stop',
61 index=0,
62 logprobs=None,
63 message=ParsedChatCompletionMessage[NoneType](
64 content="I'm unable to provide real-time updates, including current weather information. For the latest
65weather in San Francisco, I recommend checking a reliable weather website or app such as the Weather Channel, BBC
66Weather, or a local San Francisco news station.",
67 function_call=None,
68 parsed=None,
69 refusal=None,
70 role='assistant',
71 tool_calls=[]
72 )
73 )
74]
75"""
76 )
77 assert print_obj(listener.get_event_by_type("content.done"), monkeypatch) == snapshot(
78 """\
79ContentDoneEvent[NoneType](
80 content="I'm unable to provide real-time updates, including current weather information. For the latest weather in
81San Francisco, I recommend checking a reliable weather website or app such as the Weather Channel, BBC Weather, or a
82local San Francisco news station.",
83 parsed=None,
84 type='content.done'
85)
86"""
87 )
88
89
90@pytest.mark.respx(base_url=base_url)
91def test_parse_pydantic_model(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
92 class Location(BaseModel):
93 city: str
94 temperature: float
95 units: Literal["c", "f"]
96
97 done_snapshots: list[ParsedChatCompletionSnapshot] = []
98
99 def on_event(stream: ChatCompletionStream[Location], event: ChatCompletionStreamEvent[Location]) -> None:
100 if event.type == "content.done":
101 done_snapshots.append(model_copy(stream.current_completion_snapshot, deep=True))
102
103 listener = _make_stream_snapshot_request(
104 lambda c: c.beta.chat.completions.stream(
105 model="gpt-4o-2024-08-06",
106 messages=[
107 {
108 "role": "user",
109 "content": "What's the weather like in SF?",
110 },
111 ],
112 response_format=Location,
113 ),
114 content_snapshot=snapshot(external("15ae68f793c7*.bin")),
115 mock_client=client,
116 respx_mock=respx_mock,
117 on_event=on_event,
118 )
119
120 assert len(done_snapshots) == 1
121 assert isinstance(done_snapshots[0].choices[0].message.parsed, Location)
122
123 for event in reversed(listener.events):
124 if event.type == "content.delta":
125 data = cast(Any, event.parsed)
126 assert isinstance(data["city"], str), data
127 assert isinstance(data["temperature"], (int, float)), data
128 assert isinstance(data["units"], str), data
129 break
130 else:
131 rich.print(listener.events)
132 raise AssertionError("Did not find a `content.delta` event")
133
134 assert print_obj(listener.stream.get_final_completion(), monkeypatch) == snapshot(
135 """\
136ParsedChatCompletion[Location](
137 choices=[
138 ParsedChoice[Location](
139 finish_reason='stop',
140 index=0,
141 logprobs=None,
142 message=ParsedChatCompletionMessage[Location](
143 content='{"city":"San Francisco","temperature":68,"units":"f"}',
144 function_call=None,
145 parsed=Location(city='San Francisco', temperature=68.0, units='f'),
146 refusal=None,
147 role='assistant',
148 tool_calls=[]
149 )
150 )
151 ],
152 created=1723024750,
153 id='chatcmpl-9tXji2y8kKxlOO3muVvfdJ7ECJVlD',
154 model='gpt-4o-2024-08-06',
155 object='chat.completion',
156 service_tier=None,
157 system_fingerprint='fp_845eaabc1f',
158 usage=CompletionUsage(completion_tokens=14, completion_tokens_details=None, prompt_tokens=17, total_tokens=31)
159)
160"""
161 )
162 assert print_obj(listener.get_event_by_type("content.done"), monkeypatch) == snapshot(
163 """\
164ContentDoneEvent[Location](
165 content='{"city":"San Francisco","temperature":68,"units":"f"}',
166 parsed=Location(city='San Francisco', temperature=68.0, units='f'),
167 type='content.done'
168)
169"""
170 )
171
172
173@pytest.mark.respx(base_url=base_url)
174def test_parse_pydantic_model_multiple_choices(
175 client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch
176) -> None:
177 class Location(BaseModel):
178 city: str
179 temperature: float
180 units: Literal["c", "f"]
181
182 listener = _make_stream_snapshot_request(
183 lambda c: c.beta.chat.completions.stream(
184 model="gpt-4o-2024-08-06",
185 messages=[
186 {
187 "role": "user",
188 "content": "What's the weather like in SF?",
189 },
190 ],
191 n=3,
192 response_format=Location,
193 ),
194 content_snapshot=snapshot(external("a0c4f0be184e*.bin")),
195 mock_client=client,
196 respx_mock=respx_mock,
197 )
198
199 assert [e.type for e in listener.events] == snapshot(
200 [
201 "chunk",
202 "content.delta",
203 "chunk",
204 "content.delta",
205 "chunk",
206 "content.delta",
207 "chunk",
208 "content.delta",
209 "chunk",
210 "content.delta",
211 "chunk",
212 "content.delta",
213 "chunk",
214 "refusal.delta",
215 "chunk",
216 "refusal.delta",
217 "chunk",
218 "content.delta",
219 "chunk",
220 "content.delta",
221 "chunk",
222 "refusal.delta",
223 "chunk",
224 "content.delta",
225 "chunk",
226 "content.delta",
227 "chunk",
228 "refusal.delta",
229 "chunk",
230 "content.delta",
231 "chunk",
232 "content.delta",
233 "chunk",
234 "refusal.delta",
235 "chunk",
236 "content.delta",
237 "chunk",
238 "content.delta",
239 "chunk",
240 "refusal.delta",
241 "chunk",
242 "refusal.delta",
243 "chunk",
244 "content.delta",
245 "chunk",
246 "content.delta",
247 "chunk",
248 "content.delta",
249 "chunk",
250 "content.delta",
251 "chunk",
252 "refusal.delta",
253 "chunk",
254 "refusal.delta",
255 "chunk",
256 "content.delta",
257 "chunk",
258 "content.delta",
259 "chunk",
260 "content.delta",
261 "chunk",
262 "content.delta",
263 "chunk",
264 "refusal.delta",
265 "chunk",
266 "content.delta",
267 "chunk",
268 "content.delta",
269 "chunk",
270 "refusal.delta",
271 "chunk",
272 "content.delta",
273 "chunk",
274 "content.delta",
275 "chunk",
276 "refusal.delta",
277 "chunk",
278 "content.delta",
279 "chunk",
280 "content.delta",
281 "chunk",
282 "refusal.delta",
283 "chunk",
284 "content.delta",
285 "chunk",
286 "content.delta",
287 "chunk",
288 "refusal.delta",
289 "chunk",
290 "content.delta",
291 "chunk",
292 "refusal.delta",
293 "chunk",
294 "content.delta",
295 "chunk",
296 "refusal.delta",
297 "chunk",
298 "refusal.delta",
299 "chunk",
300 "refusal.delta",
301 "chunk",
302 "refusal.delta",
303 "chunk",
304 "refusal.delta",
305 "chunk",
306 "refusal.delta",
307 "chunk",
308 "refusal.delta",
309 "chunk",
310 "refusal.delta",
311 "chunk",
312 "refusal.delta",
313 "chunk",
314 "refusal.delta",
315 "chunk",
316 "refusal.delta",
317 "chunk",
318 "refusal.delta",
319 "chunk",
320 "refusal.delta",
321 "chunk",
322 "refusal.delta",
323 "chunk",
324 "refusal.delta",
325 "chunk",
326 "refusal.delta",
327 "chunk",
328 "refusal.delta",
329 "chunk",
330 "refusal.delta",
331 "chunk",
332 "refusal.delta",
333 "chunk",
334 "refusal.delta",
335 "chunk",
336 "refusal.delta",
337 "chunk",
338 "refusal.delta",
339 "chunk",
340 "refusal.delta",
341 "chunk",
342 "refusal.delta",
343 "chunk",
344 "refusal.delta",
345 "chunk",
346 "refusal.delta",
347 "chunk",
348 "content.done",
349 "chunk",
350 "content.done",
351 "chunk",
352 "refusal.done",
353 "chunk",
354 ]
355 )
356 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
357 """\
358[
359 ParsedChoice[Location](
360 finish_reason='stop',
361 index=0,
362 logprobs=None,
363 message=ParsedChatCompletionMessage[Location](
364 content='{"city":"San Francisco","temperature":63,"units":"f"}',
365 function_call=None,
366 parsed=Location(city='San Francisco', temperature=63.0, units='f'),
367 refusal=None,
368 role='assistant',
369 tool_calls=[]
370 )
371 ),
372 ParsedChoice[Location](
373 finish_reason='stop',
374 index=1,
375 logprobs=None,
376 message=ParsedChatCompletionMessage[Location](
377 content='{"city":"San Francisco","temperature":58.6,"units":"f"}',
378 function_call=None,
379 parsed=Location(city='San Francisco', temperature=58.6, units='f'),
380 refusal=None,
381 role='assistant',
382 tool_calls=[]
383 )
384 ),
385 ParsedChoice[Location](
386 finish_reason='stop',
387 index=2,
388 logprobs=None,
389 message=ParsedChatCompletionMessage[Location](
390 content=None,
391 function_call=None,
392 parsed=None,
393 refusal="I'm sorry, but I can't accurately provide the current weather for San Francisco as my data is up to
394October 2023. You can try checking a reliable weather website or app for real-time updates.",
395 role='assistant',
396 tool_calls=[]
397 )
398 )
399]
400"""
401 )
402
403
404@pytest.mark.respx(base_url=base_url)
405def test_parse_max_tokens_reached(client: OpenAI, respx_mock: MockRouter) -> None:
406 class Location(BaseModel):
407 city: str
408 temperature: float
409 units: Literal["c", "f"]
410
411 with pytest.raises(openai.LengthFinishReasonError):
412 _make_stream_snapshot_request(
413 lambda c: c.beta.chat.completions.stream(
414 model="gpt-4o-2024-08-06",
415 messages=[
416 {
417 "role": "user",
418 "content": "What's the weather like in SF?",
419 },
420 ],
421 max_tokens=1,
422 response_format=Location,
423 ),
424 content_snapshot=snapshot(external("69363a555f8e*.bin")),
425 mock_client=client,
426 respx_mock=respx_mock,
427 )
428
429
430@pytest.mark.respx(base_url=base_url)
431def test_parse_pydantic_model_refusal(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
432 class Location(BaseModel):
433 city: str
434 temperature: float
435 units: Literal["c", "f"]
436
437 listener = _make_stream_snapshot_request(
438 lambda c: c.beta.chat.completions.stream(
439 model="gpt-4o-2024-08-06",
440 messages=[
441 {
442 "role": "user",
443 "content": "How do I make anthrax?",
444 },
445 ],
446 response_format=Location,
447 ),
448 content_snapshot=snapshot(external("ca015b8b1eba*.bin")),
449 mock_client=client,
450 respx_mock=respx_mock,
451 )
452
453 assert print_obj(listener.get_event_by_type("refusal.done"), monkeypatch) == snapshot("""\
454RefusalDoneEvent(refusal="I'm sorry, but I can't assist with that request.", type='refusal.done')
455""")
456
457 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
458 """\
459[
460 ParsedChoice[Location](
461 finish_reason='stop',
462 index=0,
463 logprobs=None,
464 message=ParsedChatCompletionMessage[Location](
465 content=None,
466 function_call=None,
467 parsed=None,
468 refusal="I'm sorry, but I can't assist with that request.",
469 role='assistant',
470 tool_calls=[]
471 )
472 )
473]
474"""
475 )
476
477
478@pytest.mark.respx(base_url=base_url)
479def test_content_logprobs_events(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
480 listener = _make_stream_snapshot_request(
481 lambda c: c.beta.chat.completions.stream(
482 model="gpt-4o-2024-08-06",
483 messages=[
484 {
485 "role": "user",
486 "content": "Say foo",
487 },
488 ],
489 logprobs=True,
490 ),
491 content_snapshot=snapshot(external("be1089999ca5*.bin")),
492 mock_client=client,
493 respx_mock=respx_mock,
494 )
495
496 assert print_obj([e for e in listener.events if e.type.startswith("logprobs")], monkeypatch) == snapshot("""\
497[
498 LogprobsContentDeltaEvent(
499 content=[
500 ChatCompletionTokenLogprob(bytes=[70, 111, 111], logprob=-0.0067602484, token='Foo', top_logprobs=[])
501 ],
502 snapshot=[
503 ChatCompletionTokenLogprob(bytes=[70, 111, 111], logprob=-0.0067602484, token='Foo', top_logprobs=[])
504 ],
505 type='logprobs.content.delta'
506 ),
507 LogprobsContentDeltaEvent(
508 content=[ChatCompletionTokenLogprob(bytes=[46], logprob=-2.4962392, token='.', top_logprobs=[])],
509 snapshot=[
510 ChatCompletionTokenLogprob(bytes=[70, 111, 111], logprob=-0.0067602484, token='Foo', top_logprobs=[]),
511 ChatCompletionTokenLogprob(bytes=[46], logprob=-2.4962392, token='.', top_logprobs=[])
512 ],
513 type='logprobs.content.delta'
514 ),
515 LogprobsContentDoneEvent(
516 content=[
517 ChatCompletionTokenLogprob(bytes=[70, 111, 111], logprob=-0.0067602484, token='Foo', top_logprobs=[]),
518 ChatCompletionTokenLogprob(bytes=[46], logprob=-2.4962392, token='.', top_logprobs=[])
519 ],
520 type='logprobs.content.done'
521 )
522]
523""")
524
525 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot("""\
526[
527 ParsedChoice[NoneType](
528 finish_reason='stop',
529 index=0,
530 logprobs=ChoiceLogprobs(
531 content=[
532 ChatCompletionTokenLogprob(bytes=[70, 111, 111], logprob=-0.0067602484, token='Foo', top_logprobs=[]),
533 ChatCompletionTokenLogprob(bytes=[46], logprob=-2.4962392, token='.', top_logprobs=[])
534 ],
535 refusal=None
536 ),
537 message=ParsedChatCompletionMessage[NoneType](
538 content='Foo.',
539 function_call=None,
540 parsed=None,
541 refusal=None,
542 role='assistant',
543 tool_calls=[]
544 )
545 )
546]
547""")
548
549
550@pytest.mark.respx(base_url=base_url)
551def test_refusal_logprobs_events(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
552 class Location(BaseModel):
553 city: str
554 temperature: float
555 units: Literal["c", "f"]
556
557 listener = _make_stream_snapshot_request(
558 lambda c: c.beta.chat.completions.stream(
559 model="gpt-4o-2024-08-06",
560 messages=[
561 {
562 "role": "user",
563 "content": "How do I make anthrax?",
564 },
565 ],
566 logprobs=True,
567 response_format=Location,
568 ),
569 content_snapshot=snapshot(external("0a00cd46c610*.bin")),
570 mock_client=client,
571 respx_mock=respx_mock,
572 )
573
574 assert print_obj([e.type for e in listener.events if e.type.startswith("logprobs")], monkeypatch) == snapshot("""\
575[
576 'logprobs.refusal.delta',
577 'logprobs.refusal.delta',
578 'logprobs.refusal.delta',
579 'logprobs.refusal.delta',
580 'logprobs.refusal.delta',
581 'logprobs.refusal.delta',
582 'logprobs.refusal.delta',
583 'logprobs.refusal.delta',
584 'logprobs.refusal.delta',
585 'logprobs.refusal.delta',
586 'logprobs.refusal.done'
587]
588""")
589
590 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot("""\
591[
592 ParsedChoice[Location](
593 finish_reason='stop',
594 index=0,
595 logprobs=ChoiceLogprobs(
596 content=None,
597 refusal=[
598 ChatCompletionTokenLogprob(bytes=[73, 39, 109], logprob=-0.0016157961, token="I'm", top_logprobs=[]),
599 ChatCompletionTokenLogprob(
600 bytes=[32, 115, 111, 114, 114, 121],
601 logprob=-0.78663874,
602 token=' sorry',
603 top_logprobs=[]
604 ),
605 ChatCompletionTokenLogprob(bytes=[44], logprob=-7.79144e-05, token=',', top_logprobs=[]),
606 ChatCompletionTokenLogprob(bytes=[32, 73], logprob=-0.5234622, token=' I', top_logprobs=[]),
607 ChatCompletionTokenLogprob(
608 bytes=[32, 99, 97, 110, 110, 111, 116],
609 logprob=-0.52499557,
610 token=' cannot',
611 top_logprobs=[]
612 ),
613 ChatCompletionTokenLogprob(
614 bytes=[32, 97, 115, 115, 105, 115, 116],
615 logprob=-0.015198289,
616 token=' assist',
617 top_logprobs=[]
618 ),
619 ChatCompletionTokenLogprob(
620 bytes=[32, 119, 105, 116, 104],
621 logprob=-0.00071648485,
622 token=' with',
623 top_logprobs=[]
624 ),
625 ChatCompletionTokenLogprob(
626 bytes=[32, 116, 104, 97, 116],
627 logprob=-0.008114983,
628 token=' that',
629 top_logprobs=[]
630 ),
631 ChatCompletionTokenLogprob(
632 bytes=[32, 114, 101, 113, 117, 101, 115, 116],
633 logprob=-0.0013802331,
634 token=' request',
635 top_logprobs=[]
636 ),
637 ChatCompletionTokenLogprob(bytes=[46], logprob=-3.4121115e-06, token='.', top_logprobs=[])
638 ]
639 ),
640 message=ParsedChatCompletionMessage[Location](
641 content=None,
642 function_call=None,
643 parsed=None,
644 refusal="I'm sorry, I cannot assist with that request.",
645 role='assistant',
646 tool_calls=[]
647 )
648 )
649]
650""")
651
652
653@pytest.mark.respx(base_url=base_url)
654def test_parse_pydantic_tool(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
655 class GetWeatherArgs(BaseModel):
656 city: str
657 country: str
658 units: Literal["c", "f"] = "c"
659
660 listener = _make_stream_snapshot_request(
661 lambda c: c.beta.chat.completions.stream(
662 model="gpt-4o-2024-08-06",
663 messages=[
664 {
665 "role": "user",
666 "content": "What's the weather like in Edinburgh?",
667 },
668 ],
669 tools=[
670 openai.pydantic_function_tool(GetWeatherArgs),
671 ],
672 ),
673 content_snapshot=snapshot(external("24aaf30663f9*.bin")),
674 mock_client=client,
675 respx_mock=respx_mock,
676 )
677
678 assert print_obj(listener.stream.current_completion_snapshot.choices, monkeypatch) == snapshot(
679 """\
680[
681 ParsedChoice[object](
682 finish_reason='tool_calls',
683 index=0,
684 logprobs=None,
685 message=ParsedChatCompletionMessage[object](
686 content=None,
687 function_call=None,
688 parsed=None,
689 refusal=None,
690 role='assistant',
691 tool_calls=[
692 ParsedFunctionToolCall(
693 function=ParsedFunction(
694 arguments='{"city":"Edinburgh","country":"GB","units":"c"}',
695 name='GetWeatherArgs',
696 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='GB', units='c')
697 ),
698 id='call_7PhhveOvvpPK53s1fV8TWhoV',
699 index=0,
700 type='function'
701 )
702 ]
703 )
704 )
705]
706"""
707 )
708
709 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
710 """\
711[
712 ParsedChoice[NoneType](
713 finish_reason='tool_calls',
714 index=0,
715 logprobs=None,
716 message=ParsedChatCompletionMessage[NoneType](
717 content=None,
718 function_call=None,
719 parsed=None,
720 refusal=None,
721 role='assistant',
722 tool_calls=[
723 ParsedFunctionToolCall(
724 function=ParsedFunction(
725 arguments='{"city":"Edinburgh","country":"GB","units":"c"}',
726 name='GetWeatherArgs',
727 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='GB', units='c')
728 ),
729 id='call_7PhhveOvvpPK53s1fV8TWhoV',
730 index=0,
731 type='function'
732 )
733 ]
734 )
735 )
736]
737"""
738 )
739
740
741@pytest.mark.respx(base_url=base_url)
742def test_parse_multiple_pydantic_tools(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
743 class GetWeatherArgs(BaseModel):
744 """Get the temperature for the given country/city combo"""
745
746 city: str
747 country: str
748 units: Literal["c", "f"] = "c"
749
750 class GetStockPrice(BaseModel):
751 ticker: str
752 exchange: str
753
754 listener = _make_stream_snapshot_request(
755 lambda c: c.beta.chat.completions.stream(
756 model="gpt-4o-2024-08-06",
757 messages=[
758 {
759 "role": "user",
760 "content": "What's the weather like in Edinburgh?",
761 },
762 {
763 "role": "user",
764 "content": "What's the price of AAPL?",
765 },
766 ],
767 tools=[
768 openai.pydantic_function_tool(GetWeatherArgs),
769 openai.pydantic_function_tool(
770 GetStockPrice, name="get_stock_price", description="Fetch the latest price for a given ticker"
771 ),
772 ],
773 ),
774 content_snapshot=snapshot(external("453df473e962*.bin")),
775 mock_client=client,
776 respx_mock=respx_mock,
777 )
778
779 assert print_obj(listener.stream.current_completion_snapshot.choices, monkeypatch) == snapshot(
780 """\
781[
782 ParsedChoice[object](
783 finish_reason='tool_calls',
784 index=0,
785 logprobs=None,
786 message=ParsedChatCompletionMessage[object](
787 content=None,
788 function_call=None,
789 parsed=None,
790 refusal=None,
791 role='assistant',
792 tool_calls=[
793 ParsedFunctionToolCall(
794 function=ParsedFunction(
795 arguments='{"city": "Edinburgh", "country": "UK", "units": "c"}',
796 name='GetWeatherArgs',
797 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='UK', units='c')
798 ),
799 id='call_lQnnsesjFMWMQ5IeWPHzR4th',
800 index=0,
801 type='function'
802 ),
803 ParsedFunctionToolCall(
804 function=ParsedFunction(
805 arguments='{"ticker": "AAPL", "exchange": "NASDAQ"}',
806 name='get_stock_price',
807 parsed_arguments=GetStockPrice(exchange='NASDAQ', ticker='AAPL')
808 ),
809 id='call_2xjOUgaCdiwAcl9ZBL9LyMUU',
810 index=1,
811 type='function'
812 )
813 ]
814 )
815 )
816]
817"""
818 )
819 completion = listener.stream.get_final_completion()
820 assert print_obj(completion.choices[0].message.tool_calls, monkeypatch) == snapshot(
821 """\
822[
823 ParsedFunctionToolCall(
824 function=ParsedFunction(
825 arguments='{"city": "Edinburgh", "country": "UK", "units": "c"}',
826 name='GetWeatherArgs',
827 parsed_arguments=GetWeatherArgs(city='Edinburgh', country='UK', units='c')
828 ),
829 id='call_lQnnsesjFMWMQ5IeWPHzR4th',
830 index=0,
831 type='function'
832 ),
833 ParsedFunctionToolCall(
834 function=ParsedFunction(
835 arguments='{"ticker": "AAPL", "exchange": "NASDAQ"}',
836 name='get_stock_price',
837 parsed_arguments=GetStockPrice(exchange='NASDAQ', ticker='AAPL')
838 ),
839 id='call_2xjOUgaCdiwAcl9ZBL9LyMUU',
840 index=1,
841 type='function'
842 )
843]
844"""
845 )
846
847
848@pytest.mark.respx(base_url=base_url)
849def test_parse_strict_tools(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
850 listener = _make_stream_snapshot_request(
851 lambda c: c.beta.chat.completions.stream(
852 model="gpt-4o-2024-08-06",
853 messages=[
854 {
855 "role": "user",
856 "content": "What's the weather like in SF?",
857 },
858 ],
859 tools=[
860 {
861 "type": "function",
862 "function": {
863 "name": "get_weather",
864 "parameters": {
865 "type": "object",
866 "properties": {
867 "city": {"type": "string"},
868 "state": {"type": "string"},
869 },
870 "required": [
871 "city",
872 "state",
873 ],
874 "additionalProperties": False,
875 },
876 "strict": True,
877 },
878 }
879 ],
880 ),
881 content_snapshot=snapshot(external("83d3d003e6fd*.bin")),
882 mock_client=client,
883 respx_mock=respx_mock,
884 )
885
886 assert print_obj(listener.stream.current_completion_snapshot.choices, monkeypatch) == snapshot(
887 """\
888[
889 ParsedChoice[object](
890 finish_reason='tool_calls',
891 index=0,
892 logprobs=None,
893 message=ParsedChatCompletionMessage[object](
894 content=None,
895 function_call=None,
896 parsed=None,
897 refusal=None,
898 role='assistant',
899 tool_calls=[
900 ParsedFunctionToolCall(
901 function=ParsedFunction(
902 arguments='{"city":"San Francisco","state":"CA"}',
903 name='get_weather',
904 parsed_arguments={'city': 'San Francisco', 'state': 'CA'}
905 ),
906 id='call_pVHYsU0gmSfX5TqxOyVbB2ma',
907 index=0,
908 type='function'
909 )
910 ]
911 )
912 )
913]
914"""
915 )
916
917
918@pytest.mark.respx(base_url=base_url)
919def test_non_pydantic_response_format(client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch) -> None:
920 listener = _make_stream_snapshot_request(
921 lambda c: c.beta.chat.completions.stream(
922 model="gpt-4o-2024-08-06",
923 messages=[
924 {
925 "role": "user",
926 "content": "What's the weather like in SF? Give me any JSON back",
927 },
928 ],
929 response_format={"type": "json_object"},
930 ),
931 content_snapshot=snapshot(external("0898f3d1651e*.bin")),
932 mock_client=client,
933 respx_mock=respx_mock,
934 )
935
936 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
937 """\
938[
939 ParsedChoice[NoneType](
940 finish_reason='stop',
941 index=0,
942 logprobs=None,
943 message=ParsedChatCompletionMessage[NoneType](
944 content='\\n {\\n "location": "San Francisco, CA",\\n "forecast_date": "2023-11-02",\\n "weather": {\\n
945"temperature": {\\n "current": "N/A",\\n "high": "N/A",\\n "low": "N/A"\\n },\\n "condition":
946"N/A",\\n "humidity": "N/A",\\n "wind_speed": "N/A"\\n },\\n "note": "Please check a reliable weather
947service for the most current information."\\n }',
948 function_call=None,
949 parsed=None,
950 refusal=None,
951 role='assistant',
952 tool_calls=[]
953 )
954 )
955]
956"""
957 )
958
959
960@pytest.mark.respx(base_url=base_url)
961def test_allows_non_strict_tools_but_no_parsing(
962 client: OpenAI, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch
963) -> None:
964 listener = _make_stream_snapshot_request(
965 lambda c: c.beta.chat.completions.stream(
966 model="gpt-4o-2024-08-06",
967 messages=[{"role": "user", "content": "what's the weather in NYC?"}],
968 tools=[
969 {
970 "type": "function",
971 "function": {
972 "name": "get_weather",
973 "parameters": {"type": "object", "properties": {"city": {"type": "string"}}},
974 },
975 }
976 ],
977 ),
978 content_snapshot=snapshot(external("dae1b261f197*.bin")),
979 mock_client=client,
980 respx_mock=respx_mock,
981 )
982
983 assert print_obj(listener.get_event_by_type("tool_calls.function.arguments.done"), monkeypatch) == snapshot("""\
984FunctionToolCallArgumentsDoneEvent(
985 arguments='{"city":"New York City"}',
986 index=0,
987 name='get_weather',
988 parsed_arguments=None,
989 type='tool_calls.function.arguments.done'
990)
991""")
992
993 assert print_obj(listener.stream.get_final_completion().choices, monkeypatch) == snapshot(
994 """\
995[
996 ParsedChoice[NoneType](
997 finish_reason='tool_calls',
998 index=0,
999 logprobs=None,
1000 message=ParsedChatCompletionMessage[NoneType](
1001 content=None,
1002 function_call=None,
1003 parsed=None,
1004 refusal=None,
1005 role='assistant',
1006 tool_calls=[
1007 ParsedFunctionToolCall(
1008 function=ParsedFunction(
1009 arguments='{"city":"New York City"}',
1010 name='get_weather',
1011 parsed_arguments=None
1012 ),
1013 id='call_5uxEBMFySqqQGu02I5QHA8k6',
1014 index=0,
1015 type='function'
1016 )
1017 ]
1018 )
1019 )
1020]
1021"""
1022 )
1023
1024
1025@pytest.mark.parametrize("sync", [True, False], ids=["sync", "async"])
1026def test_stream_method_in_sync(sync: bool, client: OpenAI, async_client: AsyncOpenAI) -> None:
1027 checking_client: OpenAI | AsyncOpenAI = client if sync else async_client
1028
1029 assert_signatures_in_sync(
1030 checking_client.chat.completions.create,
1031 checking_client.beta.chat.completions.stream,
1032 exclude_params={"response_format", "stream"},
1033 )
1034
1035
1036class StreamListener(Generic[ResponseFormatT]):
1037 def __init__(self, stream: ChatCompletionStream[ResponseFormatT]) -> None:
1038 self.stream = stream
1039 self.events: list[ChatCompletionStreamEvent[ResponseFormatT]] = []
1040
1041 def __iter__(self) -> Iterator[ChatCompletionStreamEvent[ResponseFormatT]]:
1042 for event in self.stream:
1043 self.events.append(event)
1044 yield event
1045
1046 @overload
1047 def get_event_by_type(self, event_type: Literal["content.done"]) -> ContentDoneEvent[ResponseFormatT] | None: ...
1048
1049 @overload
1050 def get_event_by_type(self, event_type: str) -> ChatCompletionStreamEvent[ResponseFormatT] | None: ...
1051
1052 def get_event_by_type(self, event_type: str) -> ChatCompletionStreamEvent[ResponseFormatT] | None:
1053 return next((e for e in self.events if e.type == event_type), None)
1054
1055
1056def _make_stream_snapshot_request(
1057 func: Callable[[OpenAI], ChatCompletionStreamManager[ResponseFormatT]],
1058 *,
1059 content_snapshot: Any,
1060 respx_mock: MockRouter,
1061 mock_client: OpenAI,
1062 on_event: Callable[[ChatCompletionStream[ResponseFormatT], ChatCompletionStreamEvent[ResponseFormatT]], Any]
1063 | None = None,
1064) -> StreamListener[ResponseFormatT]:
1065 live = os.environ.get("OPENAI_LIVE") == "1"
1066 if live:
1067
1068 def _on_response(response: httpx.Response) -> None:
1069 # update the content snapshot
1070 assert outsource(response.read()) == content_snapshot
1071
1072 respx_mock.stop()
1073
1074 client = OpenAI(
1075 http_client=httpx.Client(
1076 event_hooks={
1077 "response": [_on_response],
1078 }
1079 )
1080 )
1081 else:
1082 respx_mock.post("/chat/completions").mock(
1083 return_value=httpx.Response(
1084 200,
1085 content=content_snapshot._old_value._load_value(),
1086 headers={"content-type": "text/event-stream"},
1087 )
1088 )
1089
1090 client = mock_client
1091
1092 with func(client) as stream:
1093 listener = StreamListener(stream)
1094
1095 for event in listener:
1096 if on_event:
1097 on_event(stream, event)
1098
1099 if live:
1100 client.close()
1101
1102 return listener
1103